mirror of https://github.com/Bunsly/JobSpy
Cullen Watson
GitHub
parent
1598d4ff63
commit
1c264b8c58
1469
JobSpy_Demo.ipynb
1469
JobSpy_Demo.ipynb
File diff suppressed because it is too large
Load Diff
91
README.md
91
README.md
|
|
@ -23,7 +23,11 @@ import pandas as pd
|
|||
jobs: pd.DataFrame = scrape_jobs(
|
||||
site_name=["indeed", "linkedin", "zip_recruiter"],
|
||||
search_term="software engineer",
|
||||
results_wanted=10
|
||||
location="Dallas, TX",
|
||||
results_wanted=10,
|
||||
|
||||
# country: only needed for indeed
|
||||
country='USA'
|
||||
)
|
||||
|
||||
if jobs.empty:
|
||||
|
|
@ -65,8 +69,10 @@ Optional
|
|||
├── is_remote (bool)
|
||||
├── results_wanted (int): number of job results to retrieve for each site specified in 'site_type'
|
||||
├── easy_apply (bool): filters for jobs on LinkedIn that have the 'Easy Apply' option
|
||||
├── country (enum): uses the corresponding subdomain on Indeed (e.g. Canada on Indeed is ca.indeed.com
|
||||
```
|
||||
|
||||
|
||||
### JobPost Schema
|
||||
```plaintext
|
||||
JobPost
|
||||
|
|
@ -81,12 +87,91 @@ JobPost
|
|||
├── job_type (enum)
|
||||
├── compensation (object)
|
||||
│ ├── interval (CompensationInterval): yearly, monthly, weekly, daily, hourly
|
||||
│ ├── min_amount (float)
|
||||
│ ├── max_amount (float)
|
||||
│ ├── min_amount (int)
|
||||
│ ├── max_amount (int)
|
||||
│ └── currency (str)
|
||||
└── date_posted (datetime)
|
||||
```
|
||||
|
||||
## Supported Countries for Job Searching
|
||||
|
||||
|
||||
### **LinkedIn**
|
||||
|
||||
LinkedIn searches globally. Use the `location` parameter
|
||||
|
||||
### **ZipRecruiter**
|
||||
|
||||
ZipRecruiter searches for jobs in US/Canada. Use the `location` parameter
|
||||
|
||||
|
||||
### **Indeed**
|
||||
For Indeed, you `location` along with `country` param
|
||||
|
||||
You can specify the following countries when searching on Indeed (use the exact name):
|
||||
|
||||
- Argentina
|
||||
- Australia
|
||||
- Austria
|
||||
- Bahrain
|
||||
- Belgium
|
||||
- Brazil
|
||||
- Canada
|
||||
- Chile
|
||||
- China
|
||||
- Colombia
|
||||
- Costa Rica
|
||||
- Czech Republic
|
||||
- Denmark
|
||||
- Ecuador
|
||||
- Egypt
|
||||
- Finland
|
||||
- France
|
||||
- Germany
|
||||
- Greece
|
||||
- Hong Kong
|
||||
- Hungary
|
||||
- India
|
||||
- Indonesia
|
||||
- Ireland
|
||||
- Israel
|
||||
- Italy
|
||||
- Japan
|
||||
- Kuwait
|
||||
- Luxembourg
|
||||
- Malaysia
|
||||
- Mexico
|
||||
- Morocco
|
||||
- Netherlands
|
||||
- New Zealand
|
||||
- Nigeria
|
||||
- Norway
|
||||
- Oman
|
||||
- Pakistan
|
||||
- Panama
|
||||
- Peru
|
||||
- Philippines
|
||||
- Poland
|
||||
- Portugal
|
||||
- Qatar
|
||||
- Romania
|
||||
- Saudi Arabia
|
||||
- Singapore
|
||||
- South Africa
|
||||
- South Korea
|
||||
- Spain
|
||||
- Sweden
|
||||
- Switzerland
|
||||
- Taiwan
|
||||
- Thailand
|
||||
- Turkey
|
||||
- Ukraine
|
||||
- United Arab Emirates
|
||||
- UK
|
||||
- USA
|
||||
- Uruguay
|
||||
- Venezuela
|
||||
- Vietnam
|
||||
|
||||
## Frequently Asked Questions
|
||||
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
[tool.poetry]
|
||||
name = "python-jobspy"
|
||||
version = "1.0.3"
|
||||
version = "1.1.0"
|
||||
description = "Job scraper for LinkedIn, Indeed & ZipRecruiter"
|
||||
authors = ["Zachary Hampton <zachary@zacharysproducts.com>", "Cullen Watson <cullen@cullen.ai>"]
|
||||
readme = "README.md"
|
||||
|
|
|
|||
|
|
@ -1,15 +1,11 @@
|
|||
import pandas as pd
|
||||
from typing import List, Tuple
|
||||
|
||||
from .jobs import JobType
|
||||
from .jobs import JobType, Location
|
||||
from .scrapers.indeed import IndeedScraper
|
||||
from .scrapers.ziprecruiter import ZipRecruiterScraper
|
||||
from .scrapers.linkedin import LinkedInScraper
|
||||
from .scrapers import (
|
||||
ScraperInput,
|
||||
Site,
|
||||
JobResponse,
|
||||
)
|
||||
from .scrapers import ScraperInput, Site, JobResponse, Country
|
||||
|
||||
|
||||
SCRAPER_MAPPING = {
|
||||
|
|
@ -32,6 +28,7 @@ def scrape_jobs(
|
|||
job_type: JobType = None,
|
||||
easy_apply: bool = False, # linkedin
|
||||
results_wanted: int = 15,
|
||||
country: str = "usa",
|
||||
) -> pd.DataFrame:
|
||||
"""
|
||||
Asynchronously scrapes job data from multiple job sites.
|
||||
|
|
@ -41,9 +38,12 @@ def scrape_jobs(
|
|||
if type(site_name) == str:
|
||||
site_name = _map_str_to_site(site_name)
|
||||
|
||||
country_enum = Country.from_string(country)
|
||||
|
||||
site_type = [site_name] if type(site_name) == Site else site_name
|
||||
scraper_input = ScraperInput(
|
||||
site_type=site_type,
|
||||
country=country_enum,
|
||||
search_term=search_term,
|
||||
location=location,
|
||||
distance=distance,
|
||||
|
|
@ -71,22 +71,15 @@ def scrape_jobs(
|
|||
for job in job_response.jobs:
|
||||
data = job.dict()
|
||||
data["site"] = site
|
||||
|
||||
# Formatting JobType
|
||||
data["job_type"] = data["job_type"].value if data["job_type"] else None
|
||||
|
||||
# Formatting Location
|
||||
location_obj = data.get("location")
|
||||
if location_obj and isinstance(location_obj, dict):
|
||||
data["city"] = location_obj.get("city", "")
|
||||
data["state"] = location_obj.get("state", "")
|
||||
data["country"] = location_obj.get("country", "USA")
|
||||
data["company"] = data["company_name"]
|
||||
if data["job_type"]:
|
||||
# Take the first value from the job type tuple
|
||||
data["job_type"] = data["job_type"].value[0]
|
||||
else:
|
||||
data["city"] = None
|
||||
data["state"] = None
|
||||
data["country"] = None
|
||||
data["job_type"] = None
|
||||
|
||||
data["location"] = Location(**data["location"]).display_location()
|
||||
|
||||
# Formatting Compensation
|
||||
compensation_obj = data.get("compensation")
|
||||
if compensation_obj and isinstance(compensation_obj, dict):
|
||||
data["interval"] = (
|
||||
|
|
@ -111,13 +104,13 @@ def scrape_jobs(
|
|||
desired_order = [
|
||||
"site",
|
||||
"title",
|
||||
"company_name",
|
||||
"city",
|
||||
"state",
|
||||
"company",
|
||||
"location",
|
||||
"job_type",
|
||||
"interval",
|
||||
"min_amount",
|
||||
"max_amount",
|
||||
"currency",
|
||||
"job_url",
|
||||
"description",
|
||||
]
|
||||
|
|
|
|||
|
|
@ -6,24 +6,160 @@ from pydantic import BaseModel, validator
|
|||
|
||||
|
||||
class JobType(Enum):
|
||||
FULL_TIME = "fulltime"
|
||||
PART_TIME = "parttime"
|
||||
CONTRACT = "contract"
|
||||
TEMPORARY = "temporary"
|
||||
INTERNSHIP = "internship"
|
||||
FULL_TIME = (
|
||||
"fulltime",
|
||||
"períodointegral",
|
||||
"estágio/trainee",
|
||||
"cunormăîntreagă",
|
||||
"tiempocompleto",
|
||||
"vollzeit",
|
||||
"voltijds",
|
||||
"tempointegral",
|
||||
"全职",
|
||||
"plnýúvazek",
|
||||
"fuldtid",
|
||||
"دوامكامل",
|
||||
"kokopäivätyö",
|
||||
"tempsplein",
|
||||
"vollzeit",
|
||||
"πλήρηςαπασχόληση",
|
||||
"teljesmunkaidő",
|
||||
"tempopieno",
|
||||
"tempsplein",
|
||||
"heltid",
|
||||
"jornadacompleta",
|
||||
"pełnyetat",
|
||||
"정규직",
|
||||
"100%",
|
||||
"全職",
|
||||
"งานประจำ",
|
||||
"tamzamanlı",
|
||||
"повназайнятість",
|
||||
"toànthờigian",
|
||||
)
|
||||
PART_TIME = ("parttime", "teilzeit")
|
||||
CONTRACT = ("contract", "contractor")
|
||||
TEMPORARY = ("temporary",)
|
||||
INTERNSHIP = ("internship", "prácticas", "ojt(onthejobtraining)", "praktikum")
|
||||
|
||||
PER_DIEM = "perdiem"
|
||||
NIGHTS = "nights"
|
||||
OTHER = "other"
|
||||
SUMMER = "summer"
|
||||
VOLUNTEER = "volunteer"
|
||||
PER_DIEM = ("perdiem",)
|
||||
NIGHTS = ("nights",)
|
||||
OTHER = ("other",)
|
||||
SUMMER = ("summer",)
|
||||
VOLUNTEER = ("volunteer",)
|
||||
|
||||
|
||||
class Country(Enum):
|
||||
ARGENTINA = ("argentina", "ar")
|
||||
AUSTRALIA = ("australia", "au")
|
||||
AUSTRIA = ("austria", "at")
|
||||
BAHRAIN = ("bahrain", "bh")
|
||||
BELGIUM = ("belgium", "be")
|
||||
BRAZIL = ("brazil", "br")
|
||||
CANADA = ("canada", "ca")
|
||||
CHILE = ("chile", "cl")
|
||||
CHINA = ("china", "cn")
|
||||
COLOMBIA = ("colombia", "co")
|
||||
COSTARICA = ("costa rica", "cr")
|
||||
CZECHREPUBLIC = ("czech republic", "cz")
|
||||
DENMARK = ("denmark", "dk")
|
||||
ECUADOR = ("ecuador", "ec")
|
||||
EGYPT = ("egypt", "eg")
|
||||
FINLAND = ("finland", "fi")
|
||||
FRANCE = ("france", "fr")
|
||||
GERMANY = ("germany", "de")
|
||||
GREECE = ("greece", "gr")
|
||||
HONGKONG = ("hong kong", "hk")
|
||||
HUNGARY = ("hungary", "hu")
|
||||
INDIA = ("india", "in")
|
||||
INDONESIA = ("indonesia", "id")
|
||||
IRELAND = ("ireland", "ie")
|
||||
ISRAEL = ("israel", "il")
|
||||
ITALY = ("italy", "it")
|
||||
JAPAN = ("japan", "jp")
|
||||
KUWAIT = ("kuwait", "kw")
|
||||
LUXEMBOURG = ("luxembourg", "lu")
|
||||
MALAYSIA = ("malaysia", "malaysia")
|
||||
MEXICO = ("mexico", "mx")
|
||||
MOROCCO = ("morocco", "ma")
|
||||
NETHERLANDS = ("netherlands", "nl")
|
||||
NEWZEALAND = ("new zealand", "nz")
|
||||
NIGERIA = ("nigeria", "ng")
|
||||
NORWAY = ("norway", "no")
|
||||
OMAN = ("oman", "om")
|
||||
PAKISTAN = ("pakistan", "pk")
|
||||
PANAMA = ("panama", "pa")
|
||||
PERU = ("peru", "pe")
|
||||
PHILIPPINES = ("philippines", "ph")
|
||||
POLAND = ("poland", "pl")
|
||||
PORTUGAL = ("portugal", "pt")
|
||||
QATAR = ("qatar", "qa")
|
||||
ROMANIA = ("romania", "ro")
|
||||
SAUDIARABIA = ("saudi arabia", "sa")
|
||||
SINGAPORE = ("singapore", "sg")
|
||||
SOUTHAFRICA = ("south africa", "za")
|
||||
SOUTHKOREA = ("south korea", "kr")
|
||||
SPAIN = ("spain", "es")
|
||||
SWEDEN = ("sweden", "se")
|
||||
SWITZERLAND = ("switzerland", "ch")
|
||||
TAIWAN = ("taiwan", "tw")
|
||||
THAILAND = ("thailand", "th")
|
||||
TURKEY = ("turkey", "tr")
|
||||
UKRAINE = ("ukraine", "ua")
|
||||
UNITEDARABEMIRATES = ("united arab emirates", "ae")
|
||||
UK = ("uk", "uk")
|
||||
USA = ("usa", "www")
|
||||
URUGUAY = ("uruguay", "uy")
|
||||
VENEZUELA = ("venezuela", "ve")
|
||||
VIETNAM = ("vietnam", "vn")
|
||||
|
||||
# internal for ziprecruiter
|
||||
US_CANADA = ("usa/ca", "www")
|
||||
|
||||
# internal for linkeind
|
||||
WORLDWIDE = ("worldwide", "www")
|
||||
|
||||
def __new__(cls, country, domain):
|
||||
obj = object.__new__(cls)
|
||||
obj._value_ = country
|
||||
obj.domain = domain
|
||||
return obj
|
||||
|
||||
@property
|
||||
def domain_value(self):
|
||||
return self.domain
|
||||
|
||||
@classmethod
|
||||
def from_string(cls, country_str: str):
|
||||
"""Convert a string to the corresponding Country enum."""
|
||||
country_str = country_str.strip().lower()
|
||||
for country in cls:
|
||||
if country.value == country_str:
|
||||
return country
|
||||
valid_countries = [country.value for country in cls]
|
||||
raise ValueError(
|
||||
f"Invalid country string: '{country_str}'. Valid countries (only include this param for Indeed) are: {', '.join(valid_countries)}"
|
||||
)
|
||||
|
||||
|
||||
class Location(BaseModel):
|
||||
country: str = "USA"
|
||||
city: str = None
|
||||
country: Country = None
|
||||
city: Optional[str] = None
|
||||
state: Optional[str] = None
|
||||
|
||||
def display_location(self) -> str:
|
||||
location_parts = []
|
||||
if self.city:
|
||||
location_parts.append(self.city)
|
||||
if self.state:
|
||||
location_parts.append(self.state)
|
||||
if self.country and self.country not in (Country.US_CANADA, Country.WORLDWIDE):
|
||||
if self.country.value in ("usa", "uk"):
|
||||
location_parts.append(self.country.value.upper())
|
||||
else:
|
||||
location_parts.append(self.country.value.title())
|
||||
return ", ".join(location_parts)
|
||||
|
||||
|
||||
class CompensationInterval(Enum):
|
||||
YEARLY = "yearly"
|
||||
|
|
@ -37,7 +173,7 @@ class Compensation(BaseModel):
|
|||
interval: CompensationInterval
|
||||
min_amount: int = None
|
||||
max_amount: int = None
|
||||
currency: str = "USD"
|
||||
currency: Optional[str] = "USD"
|
||||
|
||||
|
||||
class JobPost(BaseModel):
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
from ..jobs import Enum, BaseModel, JobType, JobResponse
|
||||
from ..jobs import Enum, BaseModel, JobType, JobResponse, Country
|
||||
from typing import List, Optional, Any
|
||||
|
||||
|
||||
|
|
@ -18,6 +18,7 @@ class ScraperInput(BaseModel):
|
|||
search_term: str
|
||||
|
||||
location: str = None
|
||||
country: Optional[Country] = Country.USA
|
||||
distance: Optional[int] = None
|
||||
is_remote: bool = False
|
||||
job_type: Optional[JobType] = None
|
||||
|
|
@ -35,9 +36,8 @@ class CommonResponse(BaseModel):
|
|||
|
||||
|
||||
class Scraper:
|
||||
def __init__(self, site: Site, url: str):
|
||||
def __init__(self, site: Site):
|
||||
self.site = site
|
||||
self.url = url
|
||||
|
||||
def scrape(self, scraper_input: ScraperInput) -> JobResponse:
|
||||
...
|
||||
|
|
|
|||
|
|
@ -1,6 +1,8 @@
|
|||
import re
|
||||
import math
|
||||
import io
|
||||
import json
|
||||
import traceback
|
||||
from datetime import datetime
|
||||
from typing import Optional
|
||||
|
||||
|
|
@ -18,7 +20,7 @@ from ...jobs import (
|
|||
JobResponse,
|
||||
JobType,
|
||||
)
|
||||
from .. import Scraper, ScraperInput, Site, StatusException
|
||||
from .. import Scraper, ScraperInput, Site, Country, StatusException
|
||||
|
||||
|
||||
class ParsingException(Exception):
|
||||
|
|
@ -31,8 +33,7 @@ class IndeedScraper(Scraper):
|
|||
Initializes IndeedScraper with the Indeed job search url
|
||||
"""
|
||||
site = Site(Site.INDEED)
|
||||
url = "https://www.indeed.com"
|
||||
super().__init__(site, url)
|
||||
super().__init__(site)
|
||||
|
||||
self.jobs_per_page = 15
|
||||
self.seen_urls = set()
|
||||
|
|
@ -47,16 +48,21 @@ class IndeedScraper(Scraper):
|
|||
:param session:
|
||||
:return: jobs found on page, total number of jobs found for search
|
||||
"""
|
||||
self.country = scraper_input.country
|
||||
domain = self.country.domain_value
|
||||
self.url = f"https://{domain}.indeed.com"
|
||||
|
||||
job_list = []
|
||||
|
||||
params = {
|
||||
"q": scraper_input.search_term,
|
||||
"l": scraper_input.location,
|
||||
"radius": scraper_input.distance,
|
||||
"filter": 0,
|
||||
"start": 0 + page * 10,
|
||||
}
|
||||
if scraper_input.distance:
|
||||
params["radius"] = scraper_input.distance
|
||||
|
||||
sc_values = []
|
||||
if scraper_input.is_remote:
|
||||
sc_values.append("attr(DSQF7)")
|
||||
|
|
@ -65,12 +71,15 @@ class IndeedScraper(Scraper):
|
|||
|
||||
if sc_values:
|
||||
params["sc"] = "0kf:" + "".join(sc_values) + ";"
|
||||
response = session.get(self.url + "/jobs", params=params)
|
||||
response = session.get(self.url + "/jobs", params=params, allow_redirects=True)
|
||||
# print(response.status_code)
|
||||
|
||||
if response.status_code != 200 and response.status_code != 307:
|
||||
if response.status_code not in range(200, 400):
|
||||
raise StatusException(response.status_code)
|
||||
|
||||
soup = BeautifulSoup(response.content, "html.parser")
|
||||
with open("text2.html", "w", encoding="utf-8") as f:
|
||||
f.write(str(soup))
|
||||
if "did not match any jobs" in str(soup):
|
||||
raise ParsingException("Search did not match any jobs")
|
||||
|
||||
|
|
@ -92,8 +101,6 @@ class IndeedScraper(Scraper):
|
|||
if job_url in self.seen_urls:
|
||||
return None
|
||||
|
||||
snippet_html = BeautifulSoup(job["snippet"], "html.parser")
|
||||
|
||||
extracted_salary = job.get("extractedSalary")
|
||||
compensation = None
|
||||
if extracted_salary:
|
||||
|
|
@ -118,11 +125,12 @@ class IndeedScraper(Scraper):
|
|||
date_posted = date_posted.strftime("%Y-%m-%d")
|
||||
|
||||
description = self.get_description(job_url, session)
|
||||
li_elements = snippet_html.find_all("li")
|
||||
with io.StringIO(job["snippet"]) as f:
|
||||
soup = BeautifulSoup(f, "html.parser")
|
||||
li_elements = soup.find_all("li")
|
||||
if description is None and li_elements:
|
||||
description = " ".join(li.text for li in li_elements)
|
||||
|
||||
first_li = snippet_html.find("li")
|
||||
job_post = JobPost(
|
||||
title=job["normTitle"],
|
||||
description=description,
|
||||
|
|
@ -130,6 +138,7 @@ class IndeedScraper(Scraper):
|
|||
location=Location(
|
||||
city=job.get("jobLocationCity"),
|
||||
state=job.get("jobLocationState"),
|
||||
country=self.country,
|
||||
),
|
||||
job_type=job_type,
|
||||
compensation=compensation,
|
||||
|
|
@ -138,7 +147,7 @@ class IndeedScraper(Scraper):
|
|||
)
|
||||
return job_post
|
||||
|
||||
with ThreadPoolExecutor(max_workers=10) as executor:
|
||||
with ThreadPoolExecutor(max_workers=1) as executor:
|
||||
job_results: list[Future] = [
|
||||
executor.submit(process_job, job)
|
||||
for job in jobs["metaData"]["mosaicProviderJobCardsModel"]["results"]
|
||||
|
|
@ -166,7 +175,7 @@ class IndeedScraper(Scraper):
|
|||
#: get first page to initialize session
|
||||
job_list, total_results = self.scrape_page(scraper_input, 0, session)
|
||||
|
||||
with ThreadPoolExecutor(max_workers=10) as executor:
|
||||
with ThreadPoolExecutor(max_workers=1) as executor:
|
||||
futures: list[Future] = [
|
||||
executor.submit(self.scrape_page, scraper_input, page, session)
|
||||
for page in range(1, pages_to_process + 1)
|
||||
|
|
@ -188,6 +197,7 @@ class IndeedScraper(Scraper):
|
|||
error=f"Indeed failed to parse response: {e}",
|
||||
)
|
||||
except Exception as e:
|
||||
print(f"LinkedIn failed to scrape: {e}\n{traceback.format_exc()}")
|
||||
return JobResponse(
|
||||
success=False,
|
||||
error=f"Indeed failed to scrape: {e}",
|
||||
|
|
@ -215,15 +225,23 @@ class IndeedScraper(Scraper):
|
|||
jk_value = params.get("jk", [None])[0]
|
||||
formatted_url = f"{self.url}/viewjob?jk={jk_value}&spa=1"
|
||||
|
||||
response = session.get(formatted_url, allow_redirects=True)
|
||||
try:
|
||||
response = session.get(
|
||||
formatted_url, allow_redirects=True, timeout_seconds=5
|
||||
)
|
||||
except requests.exceptions.Timeout:
|
||||
print("The request timed out.")
|
||||
return None
|
||||
|
||||
if response.status_code not in range(200, 400):
|
||||
print("status code not in range")
|
||||
return None
|
||||
|
||||
raw_description = response.json()["body"]["jobInfoWrapperModel"][
|
||||
"jobInfoModel"
|
||||
]["sanitizedJobDescription"]
|
||||
soup = BeautifulSoup(raw_description, "html.parser")
|
||||
with io.StringIO(raw_description) as f:
|
||||
soup = BeautifulSoup(f, "html.parser")
|
||||
text_content = " ".join(soup.get_text().split()).strip()
|
||||
return text_content
|
||||
|
||||
|
|
@ -237,13 +255,18 @@ class IndeedScraper(Scraper):
|
|||
for taxonomy in job["taxonomyAttributes"]:
|
||||
if taxonomy["label"] == "job-types":
|
||||
if len(taxonomy["attributes"]) > 0:
|
||||
job_type_str = (
|
||||
taxonomy["attributes"][0]["label"]
|
||||
.replace("-", "_")
|
||||
.replace(" ", "_")
|
||||
.upper()
|
||||
)
|
||||
return JobType[job_type_str]
|
||||
label = taxonomy["attributes"][0].get("label")
|
||||
if label:
|
||||
job_type_str = label.replace("-", "").replace(" ", "").lower()
|
||||
# print(f"Debug: job_type_str = {job_type_str}")
|
||||
return IndeedScraper.get_enum_from_value(job_type_str)
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def get_enum_from_value(value_str):
|
||||
for job_type in JobType:
|
||||
if value_str in job_type.value:
|
||||
return job_type
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
|
|
@ -294,7 +317,7 @@ class IndeedScraper(Scraper):
|
|||
:param soup:
|
||||
:return: total_num_jobs
|
||||
"""
|
||||
script = soup.find("script", string=lambda t: "window._initialData" in t)
|
||||
script = soup.find("script", string=lambda t: t and "window._initialData" in t)
|
||||
|
||||
pattern = re.compile(r"window._initialData\s*=\s*({.*})\s*;", re.DOTALL)
|
||||
match = pattern.search(script.string)
|
||||
|
|
|
|||
|
|
@ -22,8 +22,8 @@ class LinkedInScraper(Scraper):
|
|||
Initializes LinkedInScraper with the LinkedIn job search url
|
||||
"""
|
||||
site = Site(Site.LINKEDIN)
|
||||
url = "https://www.linkedin.com"
|
||||
super().__init__(site, url)
|
||||
self.url = "https://www.linkedin.com"
|
||||
super().__init__(site)
|
||||
|
||||
def scrape(self, scraper_input: ScraperInput) -> JobResponse:
|
||||
"""
|
||||
|
|
@ -31,6 +31,7 @@ class LinkedInScraper(Scraper):
|
|||
:param scraper_input:
|
||||
:return: job_response
|
||||
"""
|
||||
self.country = "worldwide"
|
||||
job_list: list[JobPost] = []
|
||||
seen_urls = set()
|
||||
page, processed_jobs, job_count = 0, 0, 0
|
||||
|
|
@ -104,7 +105,7 @@ class LinkedInScraper(Scraper):
|
|||
metadata_card = job_info.find(
|
||||
"div", class_="base-search-card__metadata"
|
||||
)
|
||||
location: Location = LinkedInScraper.get_location(metadata_card)
|
||||
location: Location = self.get_location(metadata_card)
|
||||
|
||||
datetime_tag = metadata_card.find(
|
||||
"time", class_="job-search-card__listdate"
|
||||
|
|
@ -125,7 +126,7 @@ class LinkedInScraper(Scraper):
|
|||
job_url=job_url,
|
||||
job_type=job_type,
|
||||
compensation=Compensation(
|
||||
interval=CompensationInterval.YEARLY, currency="USD"
|
||||
interval=CompensationInterval.YEARLY, currency=None
|
||||
),
|
||||
)
|
||||
job_list.append(job_post)
|
||||
|
|
@ -195,17 +196,24 @@ class LinkedInScraper(Scraper):
|
|||
employment_type = employment_type.lower()
|
||||
employment_type = employment_type.replace("-", "")
|
||||
|
||||
return JobType(employment_type)
|
||||
return LinkedInScraper.get_enum_from_value(employment_type)
|
||||
|
||||
return text_content, get_job_type(soup)
|
||||
|
||||
@staticmethod
|
||||
def get_location(metadata_card: Optional[Tag]) -> Location:
|
||||
def get_enum_from_value(value_str):
|
||||
for job_type in JobType:
|
||||
if value_str in job_type.value:
|
||||
return job_type
|
||||
return None
|
||||
|
||||
def get_location(self, metadata_card: Optional[Tag]) -> Location:
|
||||
"""
|
||||
Extracts the location data from the job metadata card.
|
||||
:param metadata_card
|
||||
:return: location
|
||||
"""
|
||||
location = Location(country=self.country)
|
||||
if metadata_card is not None:
|
||||
location_tag = metadata_card.find(
|
||||
"span", class_="job-search-card__location"
|
||||
|
|
@ -217,6 +225,7 @@ class LinkedInScraper(Scraper):
|
|||
location = Location(
|
||||
city=city,
|
||||
state=state,
|
||||
country=self.country,
|
||||
)
|
||||
|
||||
return location
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
import math
|
||||
import json
|
||||
import re
|
||||
import traceback
|
||||
from datetime import datetime
|
||||
from typing import Optional, Tuple
|
||||
from urllib.parse import urlparse, parse_qs
|
||||
|
|
@ -18,6 +19,7 @@ from ...jobs import (
|
|||
Location,
|
||||
JobResponse,
|
||||
JobType,
|
||||
Country,
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -27,8 +29,8 @@ class ZipRecruiterScraper(Scraper):
|
|||
Initializes LinkedInScraper with the ZipRecruiter job search url
|
||||
"""
|
||||
site = Site(Site.ZIP_RECRUITER)
|
||||
url = "https://www.ziprecruiter.com"
|
||||
super().__init__(site, url)
|
||||
self.url = "https://www.ziprecruiter.com"
|
||||
super().__init__(site)
|
||||
|
||||
self.jobs_per_page = 20
|
||||
self.seen_urls = set()
|
||||
|
|
@ -80,8 +82,10 @@ class ZipRecruiterScraper(Scraper):
|
|||
self.url + "/jobs-search",
|
||||
headers=ZipRecruiterScraper.headers(),
|
||||
params=params,
|
||||
allow_redirects=True,
|
||||
)
|
||||
|
||||
# print(response.status_code)
|
||||
if response.status_code != 200:
|
||||
raise StatusException(response.status_code)
|
||||
|
||||
|
|
@ -144,6 +148,7 @@ class ZipRecruiterScraper(Scraper):
|
|||
error=f"ZipRecruiter returned status code {e.status_code}",
|
||||
)
|
||||
except Exception as e:
|
||||
print(f"ZipRecruiter failed to scrape: {e}\n{traceback.format_exc()}")
|
||||
return JobResponse(
|
||||
success=False,
|
||||
error=f"ZipRecruiter failed to scrape: {e}",
|
||||
|
|
@ -181,15 +186,12 @@ class ZipRecruiterScraper(Scraper):
|
|||
description = job.find("p", {"class": "job_snippet"}).text.strip()
|
||||
|
||||
job_type_element = job.find("li", {"class": "perk_item perk_type"})
|
||||
job_type = None
|
||||
if job_type_element:
|
||||
job_type_text = (
|
||||
job_type_element.text.strip().lower().replace("-", "").replace(" ", "")
|
||||
)
|
||||
if job_type_text == "contractor":
|
||||
job_type_text = "contract"
|
||||
job_type = JobType(job_type_text)
|
||||
else:
|
||||
job_type = None
|
||||
job_type = ZipRecruiterScraper.get_job_type_enum(job_type_text)
|
||||
|
||||
date_posted = ZipRecruiterScraper.get_date_posted(job)
|
||||
|
||||
|
|
@ -206,16 +208,17 @@ class ZipRecruiterScraper(Scraper):
|
|||
return job_post
|
||||
|
||||
def process_job_js(self, job: dict) -> JobPost:
|
||||
# Map the job data to the expected fields by the Pydantic model
|
||||
title = job.get("Title")
|
||||
description = BeautifulSoup(
|
||||
job.get("Snippet", "").strip(), "html.parser"
|
||||
).get_text()
|
||||
|
||||
company = job.get("OrgName")
|
||||
location = Location(city=job.get("City"), state=job.get("State"))
|
||||
location = Location(
|
||||
city=job.get("City"), state=job.get("State"), country=Country.US_CANADA
|
||||
)
|
||||
try:
|
||||
job_type = ZipRecruiterScraper.job_type_from_string(
|
||||
job_type = ZipRecruiterScraper.get_job_type_enum(
|
||||
job.get("EmploymentType", "").replace("-", "_").lower()
|
||||
)
|
||||
except ValueError:
|
||||
|
|
@ -244,6 +247,7 @@ class ZipRecruiterScraper(Scraper):
|
|||
interval=CompensationInterval.YEARLY,
|
||||
min_amount=min_amount,
|
||||
max_amount=max_amount,
|
||||
currency="USD/CAD",
|
||||
)
|
||||
save_job_url = job.get("SaveJobURL", "")
|
||||
posted_time_match = re.search(
|
||||
|
|
@ -270,17 +274,18 @@ class ZipRecruiterScraper(Scraper):
|
|||
return job_post
|
||||
|
||||
@staticmethod
|
||||
def job_type_from_string(value: str) -> Optional[JobType]:
|
||||
if not value:
|
||||
def get_enum_from_value(value_str):
|
||||
for job_type in JobType:
|
||||
if value_str in job_type.value:
|
||||
return job_type
|
||||
return None
|
||||
|
||||
if value.lower() == "contractor":
|
||||
value = "contract"
|
||||
normalized_value = value.replace("_", "")
|
||||
for item in JobType:
|
||||
if item.value == normalized_value:
|
||||
return item
|
||||
raise ValueError(f"Invalid value for JobType: {value}")
|
||||
@staticmethod
|
||||
def get_job_type_enum(job_type_str: str) -> Optional[JobType]:
|
||||
for job_type in JobType:
|
||||
if job_type_str in job_type.value:
|
||||
return job_type
|
||||
return None
|
||||
|
||||
def get_description(self, job_page_url: str) -> Tuple[Optional[str], Optional[str]]:
|
||||
"""
|
||||
|
|
@ -289,11 +294,16 @@ class ZipRecruiterScraper(Scraper):
|
|||
:param session:
|
||||
:return: description or None, response url
|
||||
"""
|
||||
try:
|
||||
response = self.session.get(
|
||||
job_page_url, headers=ZipRecruiterScraper.headers(), allow_redirects=True
|
||||
job_page_url,
|
||||
headers=ZipRecruiterScraper.headers(),
|
||||
allow_redirects=True,
|
||||
timeout_seconds=5,
|
||||
)
|
||||
if response.status_code not in range(200, 400):
|
||||
return None, None
|
||||
except requests.exceptions.Timeout:
|
||||
print("The request timed out.")
|
||||
return None
|
||||
|
||||
html_string = response.content
|
||||
soup_job = BeautifulSoup(html_string, "html.parser")
|
||||
|
|
@ -375,7 +385,10 @@ class ZipRecruiterScraper(Scraper):
|
|||
amounts.append(amount)
|
||||
|
||||
compensation = Compensation(
|
||||
interval=interval, min_amount=min(amounts), max_amount=max(amounts)
|
||||
interval=interval,
|
||||
min_amount=min(amounts),
|
||||
max_amount=max(amounts),
|
||||
currency="USD/CAD",
|
||||
)
|
||||
|
||||
return compensation
|
||||
|
|
@ -399,10 +412,7 @@ class ZipRecruiterScraper(Scraper):
|
|||
city, state = None, None
|
||||
else:
|
||||
city, state = None, None
|
||||
return Location(
|
||||
city=city,
|
||||
state=state,
|
||||
)
|
||||
return Location(city=city, state=state, country=Country.US_CANADA)
|
||||
|
||||
@staticmethod
|
||||
def headers() -> dict:
|
||||
|
|
|
|||
Loading…
Reference in New Issue