turtlebasket
/
JobSpy
mirror of https://github.com/Bunsly/JobSpy
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Catch time out exception

pull/120/head
troy-conte 2024-03-01 12:48:52 -05:00 committed by GitHub
parent f8a4eccc6b
commit 87556a3a5d
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 7 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
src/jobspy/scrapers/glassdoor/__init__.py
View File

@ -81,6 +81,10 @@ class GlassdoorScraper(Scraper):
if len(all_jobs) >= scraper_input.results_wanted: if len(all_jobs) >= scraper_input.results_wanted:
all_jobs = all_jobs[: scraper_input.results_wanted] all_jobs = all_jobs[: scraper_input.results_wanted]
break break
except TimeoutError as timeout_exception: # Specific exception for timeouts
print(f"Timeout occurred on page {page}: {str(timeout_exception)}")
# Skip this page and continue to the next
continue
except Exception as e: except Exception as e:
raise GlassdoorException(str(e)) raise GlassdoorException(str(e))
except Exception as e: except Exception as e:
@ -99,7 +103,10 @@ class GlassdoorScraper(Scraper):
Scrapes a page of Glassdoor for jobs with scraper_input criteria Scrapes a page of Glassdoor for jobs with scraper_input criteria
""" """
self.scraper_input = scraper_input self.scraper_input = scraper_input
urlCount = 0
try: try:
logger.error(f'Glassdoor searches: {urlCount}')
urlCount+=1
payload = self._add_payload( payload = self._add_payload(
location_id, location_type, page_num, cursor location_id, location_type, page_num, cursor
) )