fix(ziprecruiter): init cookies (#82)

examples/JobSpy_AllSites.py

@@ -2,12 +2,11 @@ from jobspy import scrape_jobs
 import pandas as pd
 
 jobs: pd.DataFrame = scrape_jobs(
-    site_name=["indeed", "linkedin", "zip_recruiter"],
+    site_name=["indeed", "linkedin", "zip_recruiter", "glassdoor"],
     search_term="software engineer",
     location="Dallas, TX",
-    results_wanted=50,  # be wary the higher it is, the more likey you'll get blocked (rotating proxy should work tho)
+    results_wanted=25,  # be wary the higher it is, the more likey you'll get blocked (rotating proxy can help tho)
     country_indeed="USA",
-    offset=25  # start jobs from an offset (use if search failed and want to continue)
     # proxy="http://jobspy:5a4vpWtj8EeJ2hoYzk@ca.smartproxy.com:20001",
 )
 

examples/JobSpy_LongScrape.py

@@ -0,0 +1,77 @@
+from jobspy import scrape_jobs
+import pandas as pd
+import os
+import time
+
+# creates csv a new filename if the jobs.csv already exists.
+csv_filename = "jobs.csv"
+counter = 1
+while os.path.exists(csv_filename):
+    csv_filename = f"jobs_{counter}.csv"
+    counter += 1
+
+# results wanted and offset
+results_wanted = 1000
+offset = 0
+
+all_jobs = []
+
+# max retries
+max_retries = 3
+
+# nuumber of results at each iteration
+results_in_each_iteration = 30
+
+while len(all_jobs) < results_wanted:
+    retry_count = 0
+    while retry_count < max_retries:
+        print("Doing from", offset, "to", offset + results_in_each_iteration, "jobs")
+        try:
+            jobs = scrape_jobs(
+                site_name=["indeed"],
+                search_term="software engineer",
+                # New York, NY
+                # Dallas, TX
+
+                # Los Angeles, CA
+                location="Los Angeles, CA",
+                results_wanted=min(results_in_each_iteration, results_wanted - len(all_jobs)),
+                country_indeed="USA",
+                offset=offset,
+                # proxy="http://jobspy:5a4vpWtj8EeJ2hoYzk@ca.smartproxy.com:20001",
+            )
+
+            # Add the scraped jobs to the list
+            all_jobs.extend(jobs.to_dict('records'))
+
+            # Increment the offset for the next page of results
+            offset += results_in_each_iteration
+
+            # Add a delay to avoid rate limiting (you can adjust the delay time as needed)
+            print(f"Scraped {len(all_jobs)} jobs")
+            print("Sleeping secs", 100 * (retry_count + 1))
+            time.sleep(100 * (retry_count + 1))  # Sleep for 2 seconds between requests
+
+            break  # Break out of the retry loop if successful
+        except Exception as e:
+            print(f"Error: {e}")
+            retry_count += 1
+            print("Sleeping secs before retry", 100 * (retry_count + 1))
+            time.sleep(100 * (retry_count + 1))
+            if retry_count >= max_retries:
+                print("Max retries reached. Exiting.")
+                break
+
+# DataFrame from the collected job data
+jobs_df = pd.DataFrame(all_jobs)
+
+# Formatting
+pd.set_option("display.max_columns", None)
+pd.set_option("display.max_rows", None)
+pd.set_option("display.width", None)
+pd.set_option("display.max_colwidth", 50)
+
+print(jobs_df)
+
+jobs_df.to_csv(csv_filename, index=False)
+print(f"Outputted to {csv_filename}")

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "python-jobspy"
-version = "1.1.31"
+version = "1.1.34"
 description = "Job scraper for LinkedIn, Indeed, Glassdoor & ZipRecruiter"
 authors = ["Zachary Hampton <zachary@bunsly.com>", "Cullen Watson <cullen@bunsly.com>"]
 homepage = "https://github.com/Bunsly/JobSpy"

src/jobspy/scrapers/linkedin/__init__.py

@@ -4,6 +4,7 @@ jobspy.scrapers.linkedin
 
 This module contains routines to scrape LinkedIn.
 """
+import random
 from typing import Optional
 from datetime import datetime
 
@@ -16,14 +17,14 @@ from threading import Lock
 from urllib.parse import urlparse, urlunparse
 
 from .. import Scraper, ScraperInput, Site
-from ..utils import count_urgent_words, extract_emails_from_text, get_enum_from_job_type, currency_parser
 from ..exceptions import LinkedInException
+from ..utils import create_session
 from ...jobs import JobPost, Location, JobResponse, JobType, Country, Compensation
+from ..utils import count_urgent_words, extract_emails_from_text, get_enum_from_job_type, currency_parser
 
 
 class LinkedInScraper(Scraper):
-    MAX_RETRIES = 3
-    DELAY = 10
+    DELAY = 3
 
     def __init__(self, proxy: Optional[str] = None):
         """
@@ -57,6 +58,7 @@ class LinkedInScraper(Scraper):
             return mapping.get(job_type_enum, "")
 
         while len(job_list) < scraper_input.results_wanted and page < 1000:
+            session = create_session(is_tls=False, has_retry=True, delay=5)
             params = {
                 "keywords": scraper_input.search_term,
                 "location": scraper_input.location,
@@ -71,44 +73,30 @@ class LinkedInScraper(Scraper):
             }
 
             params = {k: v for k, v in params.items() if v is not None}
-            retries = 0
-            while retries < self.MAX_RETRIES:
-                try:
-                    response = requests.get(
-                        f"{self.url}/jobs-guest/jobs/api/seeMoreJobPostings/search?",
-                        params=params,
-                        allow_redirects=True,
-                        proxies=self.proxy,
-                        timeout=10,
-                    )
-                    response.raise_for_status()
-
-                    break
-                except requests.HTTPError as e:
-                    if hasattr(e, "response") and e.response is not None:
-                        if e.response.status_code in (429, 502):
-                            time.sleep(self.DELAY)
-                            retries += 1
-                            continue
-                        else:
-                            raise LinkedInException(
-                                f"bad response status code: {e.response.status_code}"
-                            )
-                    else:
-                        raise
-                except ProxyError as e:
-                    raise LinkedInException("bad proxy")
-                except Exception as e:
-                    raise LinkedInException(str(e))
-            else:
-                # Raise an exception if the maximum number of retries is reached
-                raise LinkedInException(
-                    "Max retries reached, failed to get a valid response"
+            try:
+                response = session.get(
+                    f"{self.url}/jobs-guest/jobs/api/seeMoreJobPostings/search?",
+                    params=params,
+                    allow_redirects=True,
+                    proxies=self.proxy,
+                    headers=self.headers(),
+                    timeout=10,
                 )
+                response.raise_for_status()
+
+            except requests.HTTPError as e:
+                raise LinkedInException(f"bad response status code: {e.response.status_code}")
+            except ProxyError as e:
+                raise LinkedInException("bad proxy")
+            except Exception as e:
+                raise LinkedInException(str(e))
 
             soup = BeautifulSoup(response.text, "html.parser")
+            job_cards = soup.find_all("div", class_="base-search-card")
+            if len(job_cards) == 0:
+                return JobResponse(jobs=job_list)
 
-            for job_card in soup.find_all("div", class_="base-search-card"):
+            for job_card in job_cards:
                 job_url = None
                 href_tag = job_card.find("a", class_="base-card__full-link")
                 if href_tag and "href" in href_tag.attrs:
@@ -130,6 +118,7 @@ class LinkedInScraper(Scraper):
                     raise LinkedInException("Exception occurred while processing jobs")
 
             page += 25
+            time.sleep(random.uniform(LinkedInScraper.DELAY, LinkedInScraper.DELAY + 2))
 
         job_list = job_list[: scraper_input.results_wanted]
         return JobResponse(jobs=job_list)
@@ -181,22 +170,22 @@ class LinkedInScraper(Scraper):
         benefits_tag = job_card.find("span", class_="result-benefits__text")
         benefits = " ".join(benefits_tag.get_text().split()) if benefits_tag else None
 
-        description, job_type = self.get_job_description(job_url)
-        # description, job_type = None, []
+        # removed to speed up scraping
+        # description, job_type = self.get_job_description(job_url)
 
         return JobPost(
             title=title,
-            description=description,
             company_name=company,
             company_url=company_url,
             location=location,
             date_posted=date_posted,
             job_url=job_url,
-            job_type=job_type,
             compensation=compensation,
             benefits=benefits,
-            emails=extract_emails_from_text(description) if description else None,
-            num_urgent_words=count_urgent_words(description) if description else None,
+            # job_type=job_type,
+            # description=description,
+            # emails=extract_emails_from_text(description) if description else None,
+            # num_urgent_words=count_urgent_words(description) if description else None,
         )
 
     def get_job_description(
@@ -208,12 +197,10 @@ class LinkedInScraper(Scraper):
         :return: description or None
         """
         try:
-            response = requests.get(job_page_url, timeout=5, proxies=self.proxy)
+            session = create_session(is_tls=False, has_retry=True)
+            response = session.get(job_page_url, timeout=5, proxies=self.proxy)
             response.raise_for_status()
         except requests.HTTPError as e:
-            if hasattr(e, "response") and e.response is not None:
-                if e.response.status_code in (429, 502):
-                    time.sleep(self.DELAY)
             return None, None
         except Exception as e:
             return None, None
@@ -287,3 +274,21 @@ class LinkedInScraper(Scraper):
                 )
 
         return location
+
+    @staticmethod
+    def headers() -> dict:
+        return {
+            'authority': 'www.linkedin.com',
+            'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
+            'accept-language': 'en-US,en;q=0.9',
+            'cache-control': 'max-age=0',
+            'sec-ch-ua': '"Not_A Brand";v="8", "Chromium";v="120", "Google Chrome";v="120"',
+            # 'sec-ch-ua-mobile': '?0',
+            # 'sec-ch-ua-platform': '"macOS"',
+            # 'sec-fetch-dest': 'document',
+            # 'sec-fetch-mode': 'navigate',
+            # 'sec-fetch-site': 'none',
+            # 'sec-fetch-user': '?1',
+            'upgrade-insecure-requests': '1',
+            'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'
+        }

src/jobspy/scrapers/utils.py

@@ -29,7 +29,7 @@ def extract_emails_from_text(text: str) -> list[str] | None:
     return email_regex.findall(text)
 
 
-def create_session(proxy: dict | None = None, is_tls: bool = True, has_retry: bool = False):
+def create_session(proxy: dict | None = None, is_tls: bool = True, has_retry: bool = False, delay: int = 1) -> requests.Session:
     """
     Creates a requests session with optional tls, proxy, and retry settings.
 
@@ -51,7 +51,7 @@ def create_session(proxy: dict | None = None, is_tls: bool = True, has_retry: bo
                             connect=3,
                             status=3,
                             status_forcelist=[500, 502, 503, 504, 429],
-                            backoff_factor=1)
+                            backoff_factor=delay)
             adapter = HTTPAdapter(max_retries=retries)
 
             session.mount('http://', adapter)

src/jobspy/scrapers/ziprecruiter/__init__.py

@@ -10,6 +10,7 @@ import re
 from datetime import datetime, date
 from typing import Optional, Tuple, Any
 
+import requests
 from bs4 import BeautifulSoup
 from concurrent.futures import ThreadPoolExecutor
 
@@ -26,6 +27,8 @@ class ZipRecruiterScraper(Scraper):
         """
         site = Site(Site.ZIP_RECRUITER)
         self.url = "https://www.ziprecruiter.com"
+        self.session = create_session(proxy)
+        self.get_cookies()
         super().__init__(site, proxy=proxy)
 
         self.jobs_per_page = 20
@@ -44,12 +47,10 @@ class ZipRecruiterScraper(Scraper):
         if continue_token:
             params["continue"] = continue_token
         try:
-            session = create_session(self.proxy, is_tls=True)
-            response = session.get(
+            response = self.session.get(
                 f"https://api.ziprecruiter.com/jobs-app/jobs",
                 headers=self.headers(),
                 params=self.add_params(scraper_input),
-                timeout_seconds=10,
             )
             if response.status_code != 200:
                 raise ZipRecruiterException(
@@ -156,6 +157,11 @@ class ZipRecruiterScraper(Scraper):
             num_urgent_words=count_urgent_words(description) if description else None,
         )
 
+    def get_cookies(self):
+        url="https://api.ziprecruiter.com/jobs-app/event"
+        data="event_type=session&logged_in=false&number_of_retry=1&property=model%3AiPhone&property=os%3AiOS&property=locale%3Aen_us&property=app_build_number%3A4734&property=app_version%3A91.0&property=manufacturer%3AApple&property=timestamp%3A2024-01-12T12%3A04%3A42-06%3A00&property=screen_height%3A852&property=os_version%3A16.6.1&property=source%3Ainstall&property=screen_width%3A393&property=device_model%3AiPhone%2014%20Pro&property=brand%3AApple"
+        self.session.post(url, data=data, headers=ZipRecruiterScraper.headers())
+
     @staticmethod
     def get_job_type_enum(job_type_str: str) -> list[JobType] | None:
         for job_type in JobType:
@@ -195,12 +201,16 @@ class ZipRecruiterScraper(Scraper):
     @staticmethod
     def headers() -> dict:
         """
-        Returns headers needed for ZipRecruiter API requests
+        Returns headers needed for requests
         :return: dict - Dictionary containing headers
         """
         return {
-            'Host': 'api.ziprecruiter.com',
-            'accept': '*/*',
-            'authorization': 'Basic YTBlZjMyZDYtN2I0Yy00MWVkLWEyODMtYTI1NDAzMzI0YTcyOg==',
-            'Cookie': '__cf_bm=DZ7eJOw6lka.Bwy5jLeDqWanaZ8BJlVAwaXrmcbYnxM-1701505132-0-AfGaVIfTA2kJlmleK14o722vbVwpZ+4UxFznsWv+guvzXSpD9KVEy/+pNzvEZUx88yaEShJwGt3/EVjhHirX/ASustKxg47V/aXRd2XIO2QN; zglobalid=61f94830-1990-4130-b222-d9d0e09c7825.57da9ea9581c.656ae86b; ziprecruiter_browser=018188e0-045b-4ad7-aa50-627a6c3d43aa; ziprecruiter_session=5259b2219bf95b6d2299a1417424bc2edc9f4b38; zva=100000000%3Bvid%3AZWroa0x_F1KEeGeU'
+            "Host": "api.ziprecruiter.com",
+            "accept": "*/*",
+            "x-zr-zva-override": "100000000;vid:ZT1huzm_EQlDTVEc",
+            "x-pushnotificationid": "0ff4983d38d7fc5b3370297f2bcffcf4b3321c418f5c22dd152a0264707602a0",
+            "x-deviceid": "D77B3A92-E589-46A4-8A39-6EF6F1D86006",
+            "user-agent": "Job Search/87.0 (iPhone; CPU iOS 16_6_1 like Mac OS X)",
+            "authorization": "Basic YTBlZjMyZDYtN2I0Yy00MWVkLWEyODMtYTI1NDAzMzI0YTcyOg==",
+            "accept-language": "en-US,en;q=0.9",
         }