diff --git a/api/core/scrapers/indeed/__init__.py b/api/core/scrapers/indeed/__init__.py index b8c6429..f04432a 100644 --- a/api/core/scrapers/indeed/__init__.py +++ b/api/core/scrapers/indeed/__init__.py @@ -110,11 +110,15 @@ class IndeedScraper(Scraper): job_type = IndeedScraper.get_job_type(job) timestamp_seconds = job["pubDate"] / 1000 date_posted = datetime.fromtimestamp(timestamp_seconds) - + li_elements = snippet_html.find_all("li") + if li_elements: + description = " ".join(li.text for li in li_elements) + else: + description = None first_li = snippet_html.find("li") job_post = JobPost( title=job["normTitle"], - description=first_li.text if first_li else None, + description=description, company_name=job["company"], location=Location( city=job.get("jobLocationCity"), diff --git a/api/core/scrapers/ziprecruiter/__init__.py b/api/core/scrapers/ziprecruiter/__init__.py index 837d237..29b1728 100644 --- a/api/core/scrapers/ziprecruiter/__init__.py +++ b/api/core/scrapers/ziprecruiter/__init__.py @@ -14,6 +14,8 @@ import math class ZipRecruiterScraper(Scraper): + url = "https://www.ziprecruiter.com" + def __init__(self): """ Initializes LinkedInScraper with the ZipRecruiter job search url @@ -21,7 +23,6 @@ class ZipRecruiterScraper(Scraper): site = Site(Site.ZIP_RECRUITER) super().__init__(site) - self.url = "https://www.ziprecruiter.com/jobs-search" self.jobs_per_page = 20 self.seen_urls = set() @@ -61,7 +62,9 @@ class ZipRecruiterScraper(Scraper): } response = session.get( - self.url, headers=ZipRecruiterScraper.headers(), params=params + self.url + "/jobs-search", + headers=ZipRecruiterScraper.headers(), + params=params, ) if response.status_code != status.HTTP_200_OK: @@ -69,6 +72,7 @@ class ZipRecruiterScraper(Scraper): html_string = response.content soup = BeautifulSoup(html_string, "html.parser") + if page == 1: script_tag = soup.find("script", {"id": "js_variables"}) data = json.loads(script_tag.string) @@ -86,9 +90,12 @@ class ZipRecruiterScraper(Scraper): title = job.find("h2", {"class": "title"}).text company = job.find("a", {"class": "company_name"}).text.strip() - description = job.find("p", {"class": "job_snippet"}).text.strip() - job_type_element = job.find("li", {"class": "perk_item perk_type"}) + description, job_url = ZipRecruiterScraper.get_description(job_url, session) + if description is None: + description = job.find("p", {"class": "job_snippet"}).text.strip() + + job_type_element = job.find("li", {"class": "perk_item perk_type"}) if job_type_element: job_type_text = ( job_type_element.text.strip() @@ -163,6 +170,26 @@ class ZipRecruiterScraper(Scraper): ) return job_response + @classmethod + def get_description(cls, job_page_url: str, session: tls_client.Session) -> str: + """ + Retrieves job description by going to the job page url + :param job_page_url: + :param session: + :return: description or None + """ + response = session.get( + job_page_url, headers=ZipRecruiterScraper.headers(), allow_redirects=True + ) + + html_string = response.content + soup_job = BeautifulSoup(html_string, "html.parser") + + job_description_div = soup_job.find("div", {"class": "job_description"}) + if job_description_div: + return job_description_div.text.strip("\n"), response.url + return None, response.url + @staticmethod def get_interval(interval_str: str): """