issue#270: glassdoor 403 response by rotating user-agent and updating headers

jobspy/__init__.py

@@ -45,6 +45,7 @@ def scrape_jobs(
     hours_old: int = None,
     enforce_annual_salary: bool = False,
     verbose: int = 0,
+    user_agent: str = None,
     **kwargs,
 ) -> pd.DataFrame:
     """
@@ -98,7 +99,7 @@ def scrape_jobs(
 
     def scrape_site(site: Site) -> Tuple[str, JobResponse]:
         scraper_class = SCRAPER_MAPPING[site]
-        scraper = scraper_class(proxies=proxies, ca_cert=ca_cert)
+        scraper = scraper_class(proxies=proxies, ca_cert=ca_cert, user_agent=user_agent)
         scraped_data: JobResponse = scraper.scrape(scraper_input)
         cap_name = site.value.capitalize()
         site_name = "ZipRecruiter" if cap_name == "Zip_recruiter" else cap_name

jobspy/bayt/__init__.py

@@ -25,7 +25,7 @@ class BaytScraper(Scraper):
     band_delay = 3
 
     def __init__(
-        self, proxies: list[str] | str | None = None, ca_cert: str | None = None
+        self, proxies: list[str] | str | None = None, ca_cert: str | None = None, user_agent: str | None = None
     ):
         super().__init__(Site.BAYT, proxies=proxies, ca_cert=ca_cert)
         self.scraper_input = None

jobspy/glassdoor/__init__.py

@@ -34,13 +34,13 @@ log = create_logger("Glassdoor")
 
 class Glassdoor(Scraper):
     def __init__(
-        self, proxies: list[str] | str | None = None, ca_cert: str | None = None
+        self, proxies: list[str] | str | None = None, ca_cert: str | None = None, user_agent: str | None = None
     ):
         """
         Initializes GlassdoorScraper with the Glassdoor job search url
         """
         site = Site(Site.GLASSDOOR)
-        super().__init__(site, proxies=proxies, ca_cert=ca_cert)
+        super().__init__(site, proxies=proxies, ca_cert=ca_cert, user_agent=user_agent)
 
         self.base_url = None
         self.country = None
@@ -65,6 +65,8 @@ class Glassdoor(Scraper):
         )
         token = self._get_csrf_token()
         headers["gd-csrf-token"] = token if token else fallback_token
+        if self.user_agent:
+            headers["user-agent"] = self.user_agent
         self.session.headers.update(headers)
 
         location_id, location_type = self._get_location(

jobspy/google/__init__.py

@@ -22,7 +22,7 @@ from jobspy.google.util import log, find_job_info_initial_page, find_job_info
 
 class Google(Scraper):
     def __init__(
-        self, proxies: list[str] | str | None = None, ca_cert: str | None = None
+        self, proxies: list[str] | str | None = None, ca_cert: str | None = None, user_agent: str | None = None
     ):
         """
         Initializes Google Scraper with the Goodle jobs search url

jobspy/indeed/__init__.py

@@ -28,7 +28,7 @@ log = create_logger("Indeed")
 
 class Indeed(Scraper):
     def __init__(
-        self, proxies: list[str] | str | None = None, ca_cert: str | None = None
+        self, proxies: list[str] | str | None = None, ca_cert: str | None = None, user_agent: str | None = None
     ):
         """
         Initializes IndeedScraper with the Indeed API url

jobspy/linkedin/__init__.py

@@ -50,7 +50,7 @@ class LinkedIn(Scraper):
     jobs_per_page = 25
 
     def __init__(
-        self, proxies: list[str] | str | None = None, ca_cert: str | None = None
+        self, proxies: list[str] | str | None = None, ca_cert: str | None = None, user_agent: str | None = None
     ):
         """
         Initializes LinkedInScraper with the LinkedIn job search url

jobspy/model.py

@@ -320,11 +320,12 @@ class ScraperInput(BaseModel):
 
 class Scraper(ABC):
     def __init__(
-        self, site: Site, proxies: list[str] | None = None, ca_cert: str | None = None
+        self, site: Site, proxies: list[str] | None = None, ca_cert: str | None = None, user_agent: str | None = None
     ):
         self.site = site
         self.proxies = proxies
         self.ca_cert = ca_cert
+        self.user_agent = user_agent
 
     @abstractmethod
     def scrape(self, scraper_input: ScraperInput) -> JobResponse: ...

jobspy/naukri/__init__.py

@@ -44,7 +44,7 @@ class Naukri(Scraper):
     jobs_per_page = 20  
 
     def __init__(
-        self, proxies: list[str] | str | None = None, ca_cert: str | None = None
+        self, proxies: list[str] | str | None = None, ca_cert: str | None = None, user_agent: str | None = None
     ):
         """
         Initializes NaukriScraper with the Naukri API URL

jobspy/ziprecruiter/__init__.py

@@ -38,7 +38,7 @@ class ZipRecruiter(Scraper):
     api_url = "https://api.ziprecruiter.com"
 
     def __init__(
-        self, proxies: list[str] | str | None = None, ca_cert: str | None = None
+        self, proxies: list[str] | str | None = None, ca_cert: str | None = None, user_agent: str | None = None
     ):
         """
         Initializes ZipRecruiterScraper with the ZipRecruiter job search url