turtlebasket/JobSpy
1
0
Fork 0
mirror of https://github.com/Bunsly/JobSpy.git synced 2026-03-10 23:44:30 -07:00
Code Issues Packages Projects Releases Wiki Activity

fix(glassdoor): add retry adapter (#77)

Browse Source
This commit is contained in:
Cullen Watson
2024-01-03 12:04:32 -06:00
committed by GitHub
parent 33d442bf1e
commit a5916edcdd
2 changed files with 20 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
src/jobspy/scrapers/glassdoor/__init__.py
View File

@@ -26,7 +26,7 @@ class GlassdoorScraper(Scraper):
"""
Initializes GlassdoorScraper with the Glassdoor job search url
"""
site = Site(Site.ZIP_RECRUITER)
site = Site(Site.GLASSDOOR)
super().__init__(site, proxy=proxy)
self.url = None
@@ -49,7 +49,7 @@ class GlassdoorScraper(Scraper):
payload = self.add_payload(
scraper_input, location_id, location_type, page_num, cursor
)
session = create_session(self.proxy, is_tls=False)
session = create_session(self.proxy, is_tls=False, has_retry=True)
response = session.post(
f"{self.url}/graph", headers=self.headers(), timeout=10, data=payload
)
@@ -171,7 +171,7 @@ class GlassdoorScraper(Scraper):
if not location or is_remote:
return "11047", "STATE" # remote options
url = f"{self.url}/findPopularLocationAjax.htm?maxLocationsToReturn=10&term={location}"
session = create_session(self.proxy)
session = create_session(self.proxy, has_retry=True)
response = session.get(url)
if response.status_code != 200:
raise GlassdoorException(
@@ -194,7 +194,7 @@ class GlassdoorScraper(Scraper):
location_type: str,
page_num: int,
cursor: str | None = None,
) -> dict[str, str | Any]:
) -> str:
payload = {
"operationName": "JobSearchResultsQuery",
"variables": {