turtlebasket
/
JobSpy
mirror of https://github.com/Bunsly/JobSpy
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

[fix] glassdoor location

pull/67/head v1.1.24
Cullen Watson 2023-10-30 20:19:56 -05:00
parent 0302ab14f5
commit d37f86e1b9
1 changed files with 11 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
src/jobspy/scrapers/glassdoor/__init__.py
View File

@ -140,9 +140,9 @@ class GlassdoorScraper(Scraper):
all_jobs = all_jobs[: scraper_input.results_wanted] all_jobs = all_jobs[: scraper_input.results_wanted]
break break
except Exception as e: except Exception as e:
print(f"Page {page} generated an exception: {e}") raise GlassdoorException(str(e))
except Exception as e: except Exception as e:
print(f"An exception occurred: {e}") raise GlassdoorException(str(e))
return JobResponse(jobs=all_jobs) return JobResponse(jobs=all_jobs)
@ -185,7 +185,7 @@ class GlassdoorScraper(Scraper):
def get_location(self, location: str, is_remote: bool) -> (int, str): def get_location(self, location: str, is_remote: bool) -> (int, str):
if not location or is_remote: if not location or is_remote:
return "11047", "S" # remote options return "11047", "STATE" # remote options
url = f"{self.url}/findPopularLocationAjax.htm?maxLocationsToReturn=10&term={location}" url = f"{self.url}/findPopularLocationAjax.htm?maxLocationsToReturn=10&term={location}"
session = create_session(self.proxy) session = create_session(self.proxy)
response = session.get(url) response = session.get(url)
@ -196,7 +196,12 @@ class GlassdoorScraper(Scraper):
items = response.json() items = response.json()
if not items: if not items:
raise ValueError(f"Location '{location}' not found on Glassdoor") raise ValueError(f"Location '{location}' not found on Glassdoor")
return int(items[0]["locationId"]), items[0]["locationType"] location_type = items[0]["locationType"]
if location_type == "C":
location_type = "CITY"
elif location_type == "S":
location_type = "STATE"
return int(items[0]["locationId"]), location_type
@staticmethod @staticmethod
def add_payload( def add_payload(
@ -213,6 +218,8 @@ class GlassdoorScraper(Scraper):
"filterParams": [], "filterParams": [],
"keyword": scraper_input.search_term, "keyword": scraper_input.search_term,
"numJobsToShow": 30, "numJobsToShow": 30,
"locationType": location_type,
"locationId": int(location_id),
"parameterUrlInput": f"IL.0,12_I{location_type}{location_id}", "parameterUrlInput": f"IL.0,12_I{location_type}{location_id}",
"pageNumber": page_num, "pageNumber": page_num,
"pageCursor": cursor, "pageCursor": cursor,