fix(ziprecruiter): get full description

pull/14/head
Cullen Watson 2023-08-25 21:17:36 -05:00
parent 18dc9a6cf8
commit a64acf9c86
2 changed files with 37 additions and 6 deletions

View File

@ -110,11 +110,15 @@ class IndeedScraper(Scraper):
job_type = IndeedScraper.get_job_type(job) job_type = IndeedScraper.get_job_type(job)
timestamp_seconds = job["pubDate"] / 1000 timestamp_seconds = job["pubDate"] / 1000
date_posted = datetime.fromtimestamp(timestamp_seconds) date_posted = datetime.fromtimestamp(timestamp_seconds)
li_elements = snippet_html.find_all("li")
if li_elements:
description = " ".join(li.text for li in li_elements)
else:
description = None
first_li = snippet_html.find("li") first_li = snippet_html.find("li")
job_post = JobPost( job_post = JobPost(
title=job["normTitle"], title=job["normTitle"],
description=first_li.text if first_li else None, description=description,
company_name=job["company"], company_name=job["company"],
location=Location( location=Location(
city=job.get("jobLocationCity"), city=job.get("jobLocationCity"),

View File

@ -14,6 +14,8 @@ import math
class ZipRecruiterScraper(Scraper): class ZipRecruiterScraper(Scraper):
url = "https://www.ziprecruiter.com"
def __init__(self): def __init__(self):
""" """
Initializes LinkedInScraper with the ZipRecruiter job search url Initializes LinkedInScraper with the ZipRecruiter job search url
@ -21,7 +23,6 @@ class ZipRecruiterScraper(Scraper):
site = Site(Site.ZIP_RECRUITER) site = Site(Site.ZIP_RECRUITER)
super().__init__(site) super().__init__(site)
self.url = "https://www.ziprecruiter.com/jobs-search"
self.jobs_per_page = 20 self.jobs_per_page = 20
self.seen_urls = set() self.seen_urls = set()
@ -61,7 +62,9 @@ class ZipRecruiterScraper(Scraper):
} }
response = session.get( response = session.get(
self.url, headers=ZipRecruiterScraper.headers(), params=params self.url + "/jobs-search",
headers=ZipRecruiterScraper.headers(),
params=params,
) )
if response.status_code != status.HTTP_200_OK: if response.status_code != status.HTTP_200_OK:
@ -69,6 +72,7 @@ class ZipRecruiterScraper(Scraper):
html_string = response.content html_string = response.content
soup = BeautifulSoup(html_string, "html.parser") soup = BeautifulSoup(html_string, "html.parser")
if page == 1: if page == 1:
script_tag = soup.find("script", {"id": "js_variables"}) script_tag = soup.find("script", {"id": "js_variables"})
data = json.loads(script_tag.string) data = json.loads(script_tag.string)
@ -86,9 +90,12 @@ class ZipRecruiterScraper(Scraper):
title = job.find("h2", {"class": "title"}).text title = job.find("h2", {"class": "title"}).text
company = job.find("a", {"class": "company_name"}).text.strip() company = job.find("a", {"class": "company_name"}).text.strip()
description = job.find("p", {"class": "job_snippet"}).text.strip()
job_type_element = job.find("li", {"class": "perk_item perk_type"})
description, job_url = ZipRecruiterScraper.get_description(job_url, session)
if description is None:
description = job.find("p", {"class": "job_snippet"}).text.strip()
job_type_element = job.find("li", {"class": "perk_item perk_type"})
if job_type_element: if job_type_element:
job_type_text = ( job_type_text = (
job_type_element.text.strip() job_type_element.text.strip()
@ -163,6 +170,26 @@ class ZipRecruiterScraper(Scraper):
) )
return job_response return job_response
@classmethod
def get_description(cls, job_page_url: str, session: tls_client.Session) -> str:
"""
Retrieves job description by going to the job page url
:param job_page_url:
:param session:
:return: description or None
"""
response = session.get(
job_page_url, headers=ZipRecruiterScraper.headers(), allow_redirects=True
)
html_string = response.content
soup_job = BeautifulSoup(html_string, "html.parser")
job_description_div = soup_job.find("div", {"class": "job_description"})
if job_description_div:
return job_description_div.text.strip("\n"), response.url
return None, response.url
@staticmethod @staticmethod
def get_interval(interval_str: str): def get_interval(interval_str: str):
""" """