mirror of
https://github.com/Bunsly/JobSpy.git
synced 2026-03-05 12:04:33 -08:00
Compare commits
3 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
216d3fd39f | ||
|
|
d3bfdc0a6e | ||
|
|
ba5ed803ca |
@@ -5,7 +5,7 @@ jobspy.scrapers.ziprecruiter
|
|||||||
This module contains routines to scrape ZipRecruiter.
|
This module contains routines to scrape ZipRecruiter.
|
||||||
"""
|
"""
|
||||||
import math
|
import math
|
||||||
import json
|
import time
|
||||||
import re
|
import re
|
||||||
from datetime import datetime, date
|
from datetime import datetime, date
|
||||||
from typing import Optional, Tuple, Any
|
from typing import Optional, Tuple, Any
|
||||||
@@ -68,9 +68,11 @@ class ZipRecruiterScraper(Scraper):
|
|||||||
raise ZipRecruiterException("bad proxy")
|
raise ZipRecruiterException("bad proxy")
|
||||||
raise ZipRecruiterException(str(e))
|
raise ZipRecruiterException(str(e))
|
||||||
|
|
||||||
|
time.sleep(5)
|
||||||
response_data = response.json()
|
response_data = response.json()
|
||||||
jobs_list = response_data.get("jobs", [])
|
jobs_list = response_data.get("jobs", [])
|
||||||
next_continue_token = response_data.get('continue', None)
|
next_continue_token = response_data.get('continue', None)
|
||||||
|
print(len(jobs_list))
|
||||||
|
|
||||||
with ThreadPoolExecutor(max_workers=10) as executor:
|
with ThreadPoolExecutor(max_workers=10) as executor:
|
||||||
job_results = [
|
job_results = [
|
||||||
@@ -113,6 +115,7 @@ class ZipRecruiterScraper(Scraper):
|
|||||||
title = job.get("name")
|
title = job.get("name")
|
||||||
job_url = job.get("job_url")
|
job_url = job.get("job_url")
|
||||||
|
|
||||||
|
|
||||||
description = BeautifulSoup(
|
description = BeautifulSoup(
|
||||||
job.get("job_description", "").strip(), "html.parser"
|
job.get("job_description", "").strip(), "html.parser"
|
||||||
).get_text()
|
).get_text()
|
||||||
@@ -136,7 +139,6 @@ class ZipRecruiterScraper(Scraper):
|
|||||||
else:
|
else:
|
||||||
date_posted = date.today()
|
date_posted = date.today()
|
||||||
|
|
||||||
|
|
||||||
return JobPost(
|
return JobPost(
|
||||||
title=title,
|
title=title,
|
||||||
company_name=company,
|
company_name=company,
|
||||||
|
|||||||
Reference in New Issue
Block a user