fix(ziprecruiter): pagination (#97)

* fix(ziprecruiter): pagination

* chore: version

README.md

@@ -80,6 +80,7 @@ Optional
 JobPost
 ├── title (str)
 ├── company (str)
+├── company_url (str)
 ├── job_url (str)
 ├── location (object)
 │   ├── country (str)
@@ -158,16 +159,11 @@ persist, [submit an issue](https://github.com/Bunsly/JobSpy/issues).
 **Q: Received a response code 429?**  
 **A:** This indicates that you have been blocked by the job board site for sending too many requests. All of the job board sites are aggressive with blocking. We recommend:
 
-- Waiting a few seconds between requests.
+- Waiting some time between scrapes (site-dependent).
 - Trying a VPN or proxy to change your IP address.
 
 ---
 
-**Q: Experiencing a "Segmentation fault: 11" on macOS Catalina?**  
-**A:** This is due to `tls_client` dependency not supporting your architecture. Solutions and workarounds include:
-
-- Upgrade to a newer version of MacOS
-- Reach out to the maintainers of [tls_client](https://github.com/bogdanfinn/tls-client) for fixes
 
 
   

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "python-jobspy"
-version = "1.1.38"
+version = "1.1.40"
 description = "Job scraper for LinkedIn, Indeed, Glassdoor & ZipRecruiter"
 authors = ["Zachary Hampton <zachary@bunsly.com>", "Cullen Watson <cullen@bunsly.com>"]
 homepage = "https://github.com/Bunsly/JobSpy"

src/jobspy/scrapers/indeed/__init__.py

@@ -154,8 +154,9 @@ class IndeedScraper(Scraper):
             )
             return job_post
 
+        workers = 10 if scraper_input.full_description else 10  # possibly lessen 10 when fetching desc based on feedback
         jobs = jobs["metaData"]["mosaicProviderJobCardsModel"]["results"]
-        with ThreadPoolExecutor(max_workers=1) as executor:
+        with ThreadPoolExecutor(max_workers=workers) as executor:
             job_results: list[Future] = [
                 executor.submit(process_job, job) for job in jobs
             ]
@@ -206,7 +207,7 @@ class IndeedScraper(Scraper):
         parsed_url = urllib.parse.urlparse(job_page_url)
         params = urllib.parse.parse_qs(parsed_url.query)
         jk_value = params.get("jk", [None])[0]
-        formatted_url = f"{self.url}/viewjob?jk={jk_value}&spa=1"
+        formatted_url = f"{self.url}/m/viewjob?jk={jk_value}&spa=1"
         session = create_session(self.proxy)
 
         try:
@@ -223,10 +224,18 @@ class IndeedScraper(Scraper):
             return None
 
         try:
-            data = json.loads(response.text)
+            soup = BeautifulSoup(response.text, 'html.parser')
-            job_description = data["body"]["jobInfoWrapperModel"]["jobInfoModel"][
+            script_tags = soup.find_all('script')
-                "sanitizedJobDescription"
+
-            ]
+            job_description = ''
+            for tag in script_tags:
+                if 'window._initialData' in tag.text:
+                    json_str = tag.text
+                    json_str = json_str.split('window._initialData=')[1]
+                    json_str = json_str.rsplit(';', 1)[0]
+                    data = json.loads(json_str)
+                    job_description = data["jobInfoWrapperModel"]["jobInfoModel"]["sanitizedJobDescription"]
+                    break
         except (KeyError, TypeError, IndexError):
             return None
 

src/jobspy/scrapers/ziprecruiter/__init__.py

@@ -44,12 +44,12 @@ class ZipRecruiterScraper(Scraper):
         """
         params = self.add_params(scraper_input)
         if continue_token:
-            params["continue"] = continue_token
+            params["continue_from"] = continue_token
         try:
             response = self.session.get(
                 f"https://api.ziprecruiter.com/jobs-app/jobs",
                 headers=self.headers(),
-                params=self.add_params(scraper_input),
+                params=params
             )
             if response.status_code != 200:
                 raise ZipRecruiterException(
@@ -68,7 +68,7 @@ class ZipRecruiterScraper(Scraper):
         with ThreadPoolExecutor(max_workers=self.jobs_per_page) as executor:
             job_results = [executor.submit(self.process_job, job) for job in jobs_list]
 
-        job_list = [result.result() for result in job_results if result.result()]
+        job_list = list(filter(None, (result.result() for result in job_results)))
         return job_list, next_continue_token
 
     def scrape(self, scraper_input: ScraperInput) -> JobResponse:
@@ -95,16 +95,15 @@ class ZipRecruiterScraper(Scraper):
             if not continue_token:
                 break
 
-        if len(job_list) > scraper_input.results_wanted:
+        return JobResponse(jobs=job_list[: scraper_input.results_wanted])
-            job_list = job_list[: scraper_input.results_wanted]
 
-        return JobResponse(jobs=job_list)
+    def process_job(self, job: dict) -> JobPost | None:
-
-    @staticmethod
-    def process_job(job: dict) -> JobPost:
         """Processes an individual job dict from the response"""
         title = job.get("name")
-        job_url = job.get("job_url")
+        job_url = f"https://www.ziprecruiter.com/jobs//j?lvk={job['listing_key']}"
+        if job_url in self.seen_urls:
+            return
+        self.seen_urls.add(job_url)
 
         job_description_html = job.get("job_description", "").strip()
         description_soup = BeautifulSoup(job_description_html, "html.parser")
@@ -173,7 +172,6 @@ class ZipRecruiterScraper(Scraper):
         params = {
             "search": scraper_input.search_term,
             "location": scraper_input.location,
-            "form": "jobs-landing",
         }
         job_type_value = None
         if scraper_input.job_type: