Compare commits

..

3 Commits

Author SHA1 Message Date
Cullen
216d3fd39f ziprecruiter: 5s delay 2023-10-28 16:41:32 -05:00
Cullen Watson
d3bfdc0a6e ziprecruiter api (#63) 2023-10-28 16:17:28 -05:00
Cullen Watson
ba5ed803ca use ziprecuriter api (#62) 2023-10-28 15:51:29 -05:00

View File

@@ -5,7 +5,7 @@ jobspy.scrapers.ziprecruiter
This module contains routines to scrape ZipRecruiter. This module contains routines to scrape ZipRecruiter.
""" """
import math import math
import json import time
import re import re
from datetime import datetime, date from datetime import datetime, date
from typing import Optional, Tuple, Any from typing import Optional, Tuple, Any
@@ -68,9 +68,11 @@ class ZipRecruiterScraper(Scraper):
raise ZipRecruiterException("bad proxy") raise ZipRecruiterException("bad proxy")
raise ZipRecruiterException(str(e)) raise ZipRecruiterException(str(e))
time.sleep(5)
response_data = response.json() response_data = response.json()
jobs_list = response_data.get("jobs", []) jobs_list = response_data.get("jobs", [])
next_continue_token = response_data.get('continue', None) next_continue_token = response_data.get('continue', None)
print(len(jobs_list))
with ThreadPoolExecutor(max_workers=10) as executor: with ThreadPoolExecutor(max_workers=10) as executor:
job_results = [ job_results = [
@@ -113,6 +115,7 @@ class ZipRecruiterScraper(Scraper):
title = job.get("name") title = job.get("name")
job_url = job.get("job_url") job_url = job.get("job_url")
description = BeautifulSoup( description = BeautifulSoup(
job.get("job_description", "").strip(), "html.parser" job.get("job_description", "").strip(), "html.parser"
).get_text() ).get_text()
@@ -136,7 +139,6 @@ class ZipRecruiterScraper(Scraper):
else: else:
date_posted = date.today() date_posted = date.today()
return JobPost( return JobPost(
title=title, title=title,
company_name=company, company_name=company,