turtlebasket
/
JobSpy
mirror of https://github.com/Bunsly/JobSpy
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

added search updates to console

pull/120/head
troy-conte 2024-03-01 17:11:06 -05:00 committed by GitHub
parent 0feee10b61
commit 2845cc9865
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 4 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
src/jobspy/scrapers/indeed/__init__.py
View File

@ -59,11 +59,13 @@ class IndeedScraper(Scraper):
self.scraper_input = scraper_input self.scraper_input = scraper_input
job_list = self._scrape_page() job_list = self._scrape_page()
pages_processed = 1 pages_processed = 1
urlCount = 0
while len(self.seen_urls) < scraper_input.results_wanted: while len(self.seen_urls) < scraper_input.results_wanted:
pages_to_process = math.ceil((scraper_input.results_wanted - len(self.seen_urls)) / self.jobs_per_page) pages_to_process = math.ceil((scraper_input.results_wanted - len(self.seen_urls)) / self.jobs_per_page)
new_jobs = False new_jobs = False
print(f'Indeed search page: {urlCount}')
urlCount += 1
with ThreadPoolExecutor(max_workers=10) as executor: with ThreadPoolExecutor(max_workers=10) as executor:
futures: list[Future] = [ futures: list[Future] = [
executor.submit(self._scrape_page, page + pages_processed) executor.submit(self._scrape_page, page + pages_processed)