Catch time out exception

pull/120/head
troy-conte 2024-03-01 12:48:52 -05:00 committed by GitHub
parent f8a4eccc6b
commit 87556a3a5d
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 7 additions and 0 deletions

View File

@ -81,6 +81,10 @@ class GlassdoorScraper(Scraper):
if len(all_jobs) >= scraper_input.results_wanted:
all_jobs = all_jobs[: scraper_input.results_wanted]
break
except TimeoutError as timeout_exception: # Specific exception for timeouts
print(f"Timeout occurred on page {page}: {str(timeout_exception)}")
# Skip this page and continue to the next
continue
except Exception as e:
raise GlassdoorException(str(e))
except Exception as e:
@ -99,7 +103,10 @@ class GlassdoorScraper(Scraper):
Scrapes a page of Glassdoor for jobs with scraper_input criteria
"""
self.scraper_input = scraper_input
urlCount = 0
try:
logger.error(f'Glassdoor searches: {urlCount}')
urlCount+=1
payload = self._add_payload(
location_id, location_type, page_num, cursor
)