fix(ziprecruiter): get full description

api/core/scrapers/indeed/__init__.py

@@ -110,11 +110,15 @@ class IndeedScraper(Scraper):
             job_type = IndeedScraper.get_job_type(job)
             timestamp_seconds = job["pubDate"] / 1000
             date_posted = datetime.fromtimestamp(timestamp_seconds)
-
+            li_elements = snippet_html.find_all("li")
+            if li_elements:
+                description = " ".join(li.text for li in li_elements)
+            else:
+                description = None
             first_li = snippet_html.find("li")
             job_post = JobPost(
                 title=job["normTitle"],
-                description=first_li.text if first_li else None,
+                description=description,
                 company_name=job["company"],
                 location=Location(
                     city=job.get("jobLocationCity"),

api/core/scrapers/ziprecruiter/__init__.py

@@ -14,6 +14,8 @@ import math
 
 
 class ZipRecruiterScraper(Scraper):
+    url = "https://www.ziprecruiter.com"
+
     def __init__(self):
         """
         Initializes LinkedInScraper with the ZipRecruiter job search url
@@ -21,7 +23,6 @@ class ZipRecruiterScraper(Scraper):
         site = Site(Site.ZIP_RECRUITER)
         super().__init__(site)
 
-        self.url = "https://www.ziprecruiter.com/jobs-search"
         self.jobs_per_page = 20
         self.seen_urls = set()
 
@@ -61,7 +62,9 @@ class ZipRecruiterScraper(Scraper):
         }
 
         response = session.get(
-            self.url, headers=ZipRecruiterScraper.headers(), params=params
+            self.url + "/jobs-search",
+            headers=ZipRecruiterScraper.headers(),
+            params=params,
         )
 
         if response.status_code != status.HTTP_200_OK:
@@ -69,6 +72,7 @@ class ZipRecruiterScraper(Scraper):
 
         html_string = response.content
         soup = BeautifulSoup(html_string, "html.parser")
+
         if page == 1:
             script_tag = soup.find("script", {"id": "js_variables"})
             data = json.loads(script_tag.string)
@@ -86,9 +90,12 @@ class ZipRecruiterScraper(Scraper):
 
             title = job.find("h2", {"class": "title"}).text
             company = job.find("a", {"class": "company_name"}).text.strip()
-            description = job.find("p", {"class": "job_snippet"}).text.strip()
-            job_type_element = job.find("li", {"class": "perk_item perk_type"})
 
+            description, job_url = ZipRecruiterScraper.get_description(job_url, session)
+            if description is None:
+                description = job.find("p", {"class": "job_snippet"}).text.strip()
+
+            job_type_element = job.find("li", {"class": "perk_item perk_type"})
             if job_type_element:
                 job_type_text = (
                     job_type_element.text.strip()
@@ -163,6 +170,26 @@ class ZipRecruiterScraper(Scraper):
         )
         return job_response
 
+    @classmethod
+    def get_description(cls, job_page_url: str, session: tls_client.Session) -> str:
+        """
+        Retrieves job description by going to the job page url
+        :param job_page_url:
+        :param session:
+        :return:  description or None
+        """
+        response = session.get(
+            job_page_url, headers=ZipRecruiterScraper.headers(), allow_redirects=True
+        )
+
+        html_string = response.content
+        soup_job = BeautifulSoup(html_string, "html.parser")
+
+        job_description_div = soup_job.find("div", {"class": "job_description"})
+        if job_description_div:
+            return job_description_div.text.strip("\n"), response.url
+        return None, response.url
+
     @staticmethod
     def get_interval(interval_str: str):
         """