turtlebasket
/
JobSpy
mirror of https://github.com/Bunsly/JobSpy
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

fix(indeed): offset

pull/214/head 1.1.72
Cullen Watson 2024-10-22 19:25:07 -05:00
parent 6372e41bd9
commit f395597fdd
2 changed files with 8 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
pyproject.toml
View File

@ -1,6 +1,6 @@
[tool.poetry] [tool.poetry]
name = "python-jobspy" name = "python-jobspy"
version = "1.1.71" version = "1.1.72"
description = "Job scraper for LinkedIn, Indeed, Glassdoor & ZipRecruiter" description = "Job scraper for LinkedIn, Indeed, Glassdoor & ZipRecruiter"
authors = ["Zachary Hampton <zachary@bunsly.com>", "Cullen Watson <cullen@bunsly.com>"] authors = ["Zachary Hampton <zachary@bunsly.com>", "Cullen Watson <cullen@bunsly.com>"]
homepage = "https://github.com/Bunsly/JobSpy" homepage = "https://github.com/Bunsly/JobSpy"

16
src/jobspy/scrapers/indeed/__init__.py
View File

@ -69,15 +69,8 @@ class IndeedScraper(Scraper):
page = 1 page = 1
cursor = None cursor = None
offset_pages = math.ceil(self.scraper_input.offset / 100)
for _ in range(offset_pages):
logger.info(f"skipping search page: {page}")
__, cursor = self._scrape_page(cursor)
if not __:
logger.info(f"found no jobs on page: {page}")
break
while len(self.seen_urls) < scraper_input.results_wanted: while len(self.seen_urls) < scraper_input.results_wanted + scraper_input.offset:
logger.info( logger.info(
f"search page: {page} / {math.ceil(scraper_input.results_wanted / 100)}" f"search page: {page} / {math.ceil(scraper_input.results_wanted / 100)}"
) )
@ -87,7 +80,12 @@ class IndeedScraper(Scraper):
break break
job_list += jobs job_list += jobs
page += 1 page += 1
return JobResponse(jobs=job_list[: scraper_input.results_wanted]) return JobResponse(
jobs=job_list[
scraper_input.offset : scraper_input.offset
+ scraper_input.results_wanted
]
)
def _scrape_page(self, cursor: str | None) -> Tuple[list[JobPost], str | None]: def _scrape_page(self, cursor: str | None) -> Tuple[list[JobPost], str | None]:
""" """