turtlebasket
/
JobSpy
mirror of https://github.com/Bunsly/JobSpy
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

fix(jobs): add 1 to loop counter

pull/12/head
Cullen Watson 2023-07-11 08:51:19 -05:00
parent a5920ddc52
commit 2a96fdc8a5
3 changed files with 14 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
api/core/scrapers/indeed/__init__.py
View File

@ -89,6 +89,7 @@ class IndeedScraper(Scraper):
) )
for job in jobs["metaData"]["mosaicProviderJobCardsModel"]["results"]: for job in jobs["metaData"]["mosaicProviderJobCardsModel"]["results"]:
processed_jobs += 1
job_url = f'{self.job_url}{job["jobkey"]}' job_url = f'{self.job_url}{job["jobkey"]}'
if job_url in seen_urls: if job_url in seen_urls:
continue continue
@ -135,7 +136,10 @@ class IndeedScraper(Scraper):
job_url=job_url, job_url=job_url,
) )
job_list.append(job_post) job_list.append(job_post)
if len(job_list) >= scraper_input.results_wanted: if (
len(job_list) >= scraper_input.results_wanted
or processed_jobs >= total_num_jobs
):
break break
if ( if (

6
api/core/scrapers/linkedin/__init__.py
View File

@ -74,6 +74,7 @@ class LinkedInScraper(Scraper):
"div", "div",
class_="base-card relative w-full hover:no-underline focus:no-underline base-card--link base-search-card base-search-card--link job-search-card", class_="base-card relative w-full hover:no-underline focus:no-underline base-card--link base-search-card base-search-card--link job-search-card",
): ):
processed_jobs += 1
data_entity_urn = job_card.get("data-entity-urn", "") data_entity_urn = job_card.get("data-entity-urn", "")
job_id = ( job_id = (
data_entity_urn.split(":")[-1] if data_entity_urn else "N/A" data_entity_urn.split(":")[-1] if data_entity_urn else "N/A"
@ -113,7 +114,10 @@ class LinkedInScraper(Scraper):
job_url=job_url, job_url=job_url,
) )
job_list.append(job_post) job_list.append(job_post)
if len(job_list) >= scraper_input.results_wanted: if (
len(job_list) >= scraper_input.results_wanted
or processed_jobs >= job_count
):
break break
if ( if (
len(job_list) >= scraper_input.results_wanted len(job_list) >= scraper_input.results_wanted

5
api/core/scrapers/ziprecruiter/__init__.py
View File

@ -114,7 +114,10 @@ class ZipRecruiterScraper(Scraper):
job_url=job_url, job_url=job_url,
) )
job_list.append(job_post) job_list.append(job_post)
if len(job_list) >= scraper_input.results_wanted: if (
len(job_list) >= scraper_input.results_wanted
or processed_jobs >= job_count
):
break break
if ( if (