turtlebasket
/
JobSpy
mirror of https://github.com/Bunsly/JobSpy
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

fix(indeed): full descr

pull/96/head
Cullen Watson 2024-02-02 18:20:33 -06:00
parent 1ccf2290fe
commit 51fdeea8ba
1 changed files with 15 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
src/jobspy/scrapers/indeed/__init__.py
View File

@ -154,8 +154,9 @@ class IndeedScraper(Scraper):
) )
return job_post return job_post
workers = 10 if scraper_input.full_description else 10 # possibly lessen 10 when fetching desc based on feedback
jobs = jobs["metaData"]["mosaicProviderJobCardsModel"]["results"] jobs = jobs["metaData"]["mosaicProviderJobCardsModel"]["results"]
with ThreadPoolExecutor(max_workers=1) as executor: with ThreadPoolExecutor(max_workers=workers) as executor:
job_results: list[Future] = [ job_results: list[Future] = [
executor.submit(process_job, job) for job in jobs executor.submit(process_job, job) for job in jobs
] ]
@ -206,7 +207,7 @@ class IndeedScraper(Scraper):
parsed_url = urllib.parse.urlparse(job_page_url) parsed_url = urllib.parse.urlparse(job_page_url)
params = urllib.parse.parse_qs(parsed_url.query) params = urllib.parse.parse_qs(parsed_url.query)
jk_value = params.get("jk", [None])[0] jk_value = params.get("jk", [None])[0]
formatted_url = f"{self.url}/viewjob?jk={jk_value}&spa=1" formatted_url = f"{self.url}/m/viewjob?jk={jk_value}&spa=1"
session = create_session(self.proxy) session = create_session(self.proxy)
try: try:
@ -223,10 +224,18 @@ class IndeedScraper(Scraper):
return None return None
try: try:
data = json.loads(response.text) soup = BeautifulSoup(response.text, 'html.parser')
job_description = data["body"]["jobInfoWrapperModel"]["jobInfoModel"][ script_tags = soup.find_all('script')
"sanitizedJobDescription"
] job_description = ''
for tag in script_tags:
if 'window._initialData' in tag.text:
json_str = tag.text
json_str = json_str.split('window._initialData=')[1]
json_str = json_str.rsplit(';', 1)[0]
data = json.loads(json_str)
job_description = data["jobInfoWrapperModel"]["jobInfoModel"]["sanitizedJobDescription"]
break
except (KeyError, TypeError, IndexError): except (KeyError, TypeError, IndexError):
return None return None