turtlebasket
/
JobSpy
mirror of https://github.com/Bunsly/JobSpy
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

fix None issue

pull/18/head
Cullen Watson 2023-08-27 12:40:53 -05:00
parent 122b332aa0
commit 819c22eded
2 changed files with 18 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
.github/workflows/test.yml vendored
View File

@ -61,19 +61,19 @@ jobs:
echo "${response::-3}" > response.json echo "${response::-3}" > response.json
cat response.json cat response.json
# - name: Check error field in response - name: Check error field in response
# run: | run: |
# indeed_error=$(jq '.indeed.error' response.json) indeed_error=$(jq '.indeed.error' response.json)
# linkedin_error=$(jq '.linkedin.error' response.json) linkedin_error=$(jq '.linkedin.error' response.json)
# zip_recruiter_error=$(jq '.zip_recruiter.error' response.json) zip_recruiter_error=$(jq '.zip_recruiter.error' response.json)
#
# if [[ "$indeed_error" != "null" || "$linkedin_error" != "null" || "$zip_recruiter_error" != "null" ]]; then if [[ "$indeed_error" != "null" || "$linkedin_error" != "null" || "$zip_recruiter_error" != "null" ]]; then
# echo "Error found in response:" echo "Error found in response:"
# echo "Indeed Error: $indeed_error" echo "Indeed Error: $indeed_error"
# echo "LinkedIn Error: $linkedin_error" echo "LinkedIn Error: $linkedin_error"
# echo "ZipRecruiter Error: $zip_recruiter_error" echo "ZipRecruiter Error: $zip_recruiter_error"
# exit 1 exit 1
# fi fi
- name: Verify returned_results in response - name: Verify returned_results in response
run: | run: |

8
api/core/scrapers/ziprecruiter/__init__.py
View File

@ -96,7 +96,9 @@ class ZipRecruiterScraper(Scraper):
title = job.find("h2", {"class": "title"}).text title = job.find("h2", {"class": "title"}).text
company = job.find("a", {"class": "company_name"}).text.strip() company = job.find("a", {"class": "company_name"}).text.strip()
description, job_url = ZipRecruiterScraper.get_description(job_url, session) description, updated_job_url = ZipRecruiterScraper.get_description(job_url, session)
if updated_job_url is not None:
job_url = updated_job_url
if description is None: if description is None:
description = job.find("p", {"class": "job_snippet"}).text.strip() description = job.find("p", {"class": "job_snippet"}).text.strip()
@ -185,7 +187,7 @@ class ZipRecruiterScraper(Scraper):
@staticmethod @staticmethod
def get_description( def get_description(
job_page_url: str, session: tls_client.Session job_page_url: str, session: tls_client.Session
) -> Tuple[Optional[str], str]: ) -> Tuple[Optional[str], Optional[str]]:
""" """
Retrieves job description by going to the job page url Retrieves job description by going to the job page url
:param job_page_url: :param job_page_url:
@ -196,7 +198,7 @@ class ZipRecruiterScraper(Scraper):
job_page_url, headers=ZipRecruiterScraper.headers(), allow_redirects=True job_page_url, headers=ZipRecruiterScraper.headers(), allow_redirects=True
) )
if response.status_code not in range(200, 400): if response.status_code not in range(200, 400):
return None return None, None
html_string = response.content html_string = response.content
soup_job = BeautifulSoup(html_string, "html.parser") soup_job = BeautifulSoup(html_string, "html.parser")