2023-09-03 07:29:25 -07:00
|
|
|
import pandas as pd
|
2023-09-03 10:05:50 -07:00
|
|
|
from typing import List, Tuple
|
2023-09-03 07:29:25 -07:00
|
|
|
|
2023-09-05 10:17:22 -07:00
|
|
|
from .jobs import JobType, Location
|
2023-09-03 10:05:50 -07:00
|
|
|
from .scrapers.indeed import IndeedScraper
|
|
|
|
from .scrapers.ziprecruiter import ZipRecruiterScraper
|
|
|
|
from .scrapers.linkedin import LinkedInScraper
|
2023-09-05 10:17:22 -07:00
|
|
|
from .scrapers import ScraperInput, Site, JobResponse, Country
|
2023-09-03 07:29:25 -07:00
|
|
|
|
|
|
|
|
|
|
|
SCRAPER_MAPPING = {
|
|
|
|
Site.LINKEDIN: LinkedInScraper,
|
|
|
|
Site.INDEED: IndeedScraper,
|
|
|
|
Site.ZIP_RECRUITER: ZipRecruiterScraper,
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
def _map_str_to_site(site_name: str) -> Site:
|
|
|
|
return Site[site_name.upper()]
|
|
|
|
|
|
|
|
|
|
|
|
def scrape_jobs(
|
2023-09-03 18:05:31 -07:00
|
|
|
site_name: str | Site | List[Site],
|
|
|
|
search_term: str,
|
|
|
|
location: str = "",
|
|
|
|
distance: int = None,
|
|
|
|
is_remote: bool = False,
|
|
|
|
job_type: JobType = None,
|
|
|
|
easy_apply: bool = False, # linkedin
|
|
|
|
results_wanted: int = 15,
|
2023-09-05 10:17:22 -07:00
|
|
|
country: str = "usa",
|
2023-09-03 07:29:25 -07:00
|
|
|
) -> pd.DataFrame:
|
|
|
|
"""
|
|
|
|
Asynchronously scrapes job data from multiple job sites.
|
|
|
|
:return: results_wanted: pandas dataframe containing job data
|
|
|
|
"""
|
|
|
|
|
|
|
|
if type(site_name) == str:
|
|
|
|
site_name = _map_str_to_site(site_name)
|
|
|
|
|
2023-09-05 10:17:22 -07:00
|
|
|
country_enum = Country.from_string(country)
|
|
|
|
|
2023-09-03 07:29:25 -07:00
|
|
|
site_type = [site_name] if type(site_name) == Site else site_name
|
|
|
|
scraper_input = ScraperInput(
|
|
|
|
site_type=site_type,
|
2023-09-05 10:17:22 -07:00
|
|
|
country=country_enum,
|
2023-09-03 07:29:25 -07:00
|
|
|
search_term=search_term,
|
|
|
|
location=location,
|
|
|
|
distance=distance,
|
|
|
|
is_remote=is_remote,
|
|
|
|
job_type=job_type,
|
|
|
|
easy_apply=easy_apply,
|
|
|
|
results_wanted=results_wanted,
|
|
|
|
)
|
|
|
|
|
|
|
|
def scrape_site(site: Site) -> Tuple[str, JobResponse]:
|
|
|
|
scraper_class = SCRAPER_MAPPING[site]
|
|
|
|
scraper = scraper_class()
|
|
|
|
scraped_data: JobResponse = scraper.scrape(scraper_input)
|
|
|
|
|
|
|
|
return site.value, scraped_data
|
|
|
|
|
|
|
|
results = {}
|
|
|
|
for site in scraper_input.site_type:
|
|
|
|
site_value, scraped_data = scrape_site(site)
|
|
|
|
results[site_value] = scraped_data
|
|
|
|
|
|
|
|
dfs = []
|
|
|
|
|
|
|
|
for site, job_response in results.items():
|
|
|
|
for job in job_response.jobs:
|
|
|
|
data = job.dict()
|
2023-09-03 18:05:31 -07:00
|
|
|
data["site"] = site
|
2023-09-05 10:17:22 -07:00
|
|
|
data["company"] = data["company_name"]
|
|
|
|
if data["job_type"]:
|
|
|
|
# Take the first value from the job type tuple
|
|
|
|
data["job_type"] = data["job_type"].value[0]
|
2023-09-03 07:29:25 -07:00
|
|
|
else:
|
2023-09-05 10:17:22 -07:00
|
|
|
data["job_type"] = None
|
|
|
|
|
|
|
|
data["location"] = Location(**data["location"]).display_location()
|
2023-09-03 07:29:25 -07:00
|
|
|
|
2023-09-03 18:05:31 -07:00
|
|
|
compensation_obj = data.get("compensation")
|
2023-09-03 07:29:25 -07:00
|
|
|
if compensation_obj and isinstance(compensation_obj, dict):
|
2023-09-03 18:05:31 -07:00
|
|
|
data["interval"] = (
|
|
|
|
compensation_obj.get("interval").value
|
|
|
|
if compensation_obj.get("interval")
|
|
|
|
else None
|
|
|
|
)
|
|
|
|
data["min_amount"] = compensation_obj.get("min_amount")
|
|
|
|
data["max_amount"] = compensation_obj.get("max_amount")
|
|
|
|
data["currency"] = compensation_obj.get("currency", "USD")
|
2023-09-03 07:29:25 -07:00
|
|
|
else:
|
2023-09-03 18:05:31 -07:00
|
|
|
data["interval"] = None
|
|
|
|
data["min_amount"] = None
|
|
|
|
data["max_amount"] = None
|
|
|
|
data["currency"] = None
|
2023-09-03 07:29:25 -07:00
|
|
|
|
|
|
|
job_df = pd.DataFrame([data])
|
|
|
|
dfs.append(job_df)
|
|
|
|
|
|
|
|
if dfs:
|
|
|
|
df = pd.concat(dfs, ignore_index=True)
|
2023-09-03 18:05:31 -07:00
|
|
|
desired_order = [
|
|
|
|
"site",
|
|
|
|
"title",
|
2023-09-05 10:17:22 -07:00
|
|
|
"company",
|
|
|
|
"location",
|
2023-09-03 18:05:31 -07:00
|
|
|
"job_type",
|
|
|
|
"interval",
|
|
|
|
"min_amount",
|
|
|
|
"max_amount",
|
2023-09-05 10:17:22 -07:00
|
|
|
"currency",
|
2023-09-03 18:05:31 -07:00
|
|
|
"job_url",
|
|
|
|
"description",
|
|
|
|
]
|
2023-09-03 07:29:25 -07:00
|
|
|
df = df[desired_order]
|
|
|
|
else:
|
|
|
|
df = pd.DataFrame()
|
|
|
|
|
|
|
|
return df
|