mirror of
https://github.com/Bunsly/JobSpy.git
synced 2026-03-05 12:04:33 -08:00
Compare commits
6 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
1089d1f0a5 | ||
|
|
3e93454738 | ||
|
|
0d150d519f | ||
|
|
cc3497f929 | ||
|
|
5986f75346 | ||
|
|
4b7bdb9313 |
26
README.md
26
README.md
@@ -48,7 +48,7 @@ jobs.to_csv("jobs.csv", quoting=csv.QUOTE_NONNUMERIC, escapechar="\\", index=Fal
|
|||||||
### Output
|
### Output
|
||||||
|
|
||||||
```
|
```
|
||||||
SITE TITLE COMPANY_NAME CITY STATE JOB_TYPE INTERVAL MIN_AMOUNT MAX_AMOUNT JOB_URL DESCRIPTION
|
SITE TITLE COMPANY CITY STATE JOB_TYPE INTERVAL MIN_AMOUNT MAX_AMOUNT JOB_URL DESCRIPTION
|
||||||
indeed Software Engineer AMERICAN SYSTEMS Arlington VA None yearly 200000 150000 https://www.indeed.com/viewjob?jk=5e409e577046... THIS POSITION COMES WITH A 10K SIGNING BONUS!...
|
indeed Software Engineer AMERICAN SYSTEMS Arlington VA None yearly 200000 150000 https://www.indeed.com/viewjob?jk=5e409e577046... THIS POSITION COMES WITH A 10K SIGNING BONUS!...
|
||||||
indeed Senior Software Engineer TherapyNotes.com Philadelphia PA fulltime yearly 135000 110000 https://www.indeed.com/viewjob?jk=da39574a40cb... About Us TherapyNotes is the national leader i...
|
indeed Senior Software Engineer TherapyNotes.com Philadelphia PA fulltime yearly 135000 110000 https://www.indeed.com/viewjob?jk=da39574a40cb... About Us TherapyNotes is the national leader i...
|
||||||
linkedin Software Engineer - Early Career Lockheed Martin Sunnyvale CA fulltime yearly None None https://www.linkedin.com/jobs/view/3693012711 Description:By bringing together people that u...
|
linkedin Software Engineer - Early Career Lockheed Martin Sunnyvale CA fulltime yearly None None https://www.linkedin.com/jobs/view/3693012711 Description:By bringing together people that u...
|
||||||
@@ -60,23 +60,23 @@ zip_recruiter Software Developer TEKsystems Phoenix
|
|||||||
### Parameters for `scrape_jobs()`
|
### Parameters for `scrape_jobs()`
|
||||||
|
|
||||||
```plaintext
|
```plaintext
|
||||||
Required
|
|
||||||
├── site_type (List[enum]): linkedin, zip_recruiter, indeed, glassdoor
|
|
||||||
└── search_term (str)
|
|
||||||
Optional
|
Optional
|
||||||
|
├── site_type (list): linkedin, zip_recruiter, indeed, glassdoor (default is all 4)
|
||||||
|
├── search_term (str)
|
||||||
├── location (str)
|
├── location (str)
|
||||||
├── distance (int): in miles, default 50
|
├── distance (int): in miles, default 50
|
||||||
├── job_type (enum): fulltime, parttime, internship, contract
|
├── job_type (str): fulltime, parttime, internship, contract
|
||||||
├── proxy (str): in format 'http://user:pass@host:port'
|
├── proxy (str): in format 'http://user:pass@host:port'
|
||||||
├── is_remote (bool)
|
├── is_remote (bool)
|
||||||
├── linkedin_fetch_description (bool): fetches full description for LinkedIn (slower)
|
|
||||||
├── results_wanted (int): number of job results to retrieve for each site specified in 'site_type'
|
├── results_wanted (int): number of job results to retrieve for each site specified in 'site_type'
|
||||||
├── easy_apply (bool): filters for jobs that are hosted on the job board site (not supported on Indeed)
|
├── easy_apply (bool): filters for jobs that are hosted on the job board site (LinkedIn & Indeed do not allow pairing this with hours_old)
|
||||||
|
├── linkedin_fetch_description (bool): fetches full description for LinkedIn (slower)
|
||||||
├── linkedin_company_ids (list[int): searches for linkedin jobs with specific company ids
|
├── linkedin_company_ids (list[int): searches for linkedin jobs with specific company ids
|
||||||
├── description_format (enum): markdown, html (format type of the job descriptions)
|
├── description_format (str): markdown, html (format type of the job descriptions)
|
||||||
├── country_indeed (enum): filters the country on Indeed (see below for correct spelling)
|
├── country_indeed (str): filters the country on Indeed (see below for correct spelling)
|
||||||
├── offset (num): starts the search from an offset (e.g. 25 will start the search from the 25th result)
|
├── offset (int): starts the search from an offset (e.g. 25 will start the search from the 25th result)
|
||||||
├── hours_old (int): filters jobs by the number of hours since the job was posted (ZipRecruiter and Glassdoor round up to next day. If you use this on Indeed, it will not filter by job_type or is_remote)
|
├── hours_old (int): filters jobs by the number of hours since the job was posted (ZipRecruiter and Glassdoor round up to next day. If you use this on Indeed, it will not filter by job_type/is_remote/easy_apply)
|
||||||
|
├── verbose (int) {0, 1, 2}: Controls the verbosity of the runtime printouts (0 prints only errors, 1 is errors+warnings, 2 is all logs. Default is 2.)
|
||||||
```
|
```
|
||||||
|
|
||||||
### JobPost Schema
|
### JobPost Schema
|
||||||
@@ -119,7 +119,7 @@ Indeed specific
|
|||||||
|
|
||||||
### **LinkedIn**
|
### **LinkedIn**
|
||||||
|
|
||||||
LinkedIn searches globally & uses only the `location` parameter. You can only fetch 1000 jobs max from the LinkedIn endpoint we are using
|
LinkedIn searches globally & uses only the `location` parameter.
|
||||||
|
|
||||||
### **ZipRecruiter**
|
### **ZipRecruiter**
|
||||||
|
|
||||||
@@ -154,7 +154,7 @@ You can specify the following countries when searching on Indeed (use the exact
|
|||||||
|
|
||||||
## Notes
|
## Notes
|
||||||
* Indeed is the best scraper currently with no rate limiting.
|
* Indeed is the best scraper currently with no rate limiting.
|
||||||
* Glassdoor/Ziprecruiter can only fetch 900/1000 jobs from the endpoints we are using on a given search.
|
* All the job board endpoints are capped at around 1000 jobs on a given search.
|
||||||
* LinkedIn is the most restrictive and usually rate limits around the 10th page.
|
* LinkedIn is the most restrictive and usually rate limits around the 10th page.
|
||||||
|
|
||||||
## Frequently Asked Questions
|
## Frequently Asked Questions
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
[tool.poetry]
|
[tool.poetry]
|
||||||
name = "python-jobspy"
|
name = "python-jobspy"
|
||||||
version = "1.1.50"
|
version = "1.1.51"
|
||||||
description = "Job scraper for LinkedIn, Indeed, Glassdoor & ZipRecruiter"
|
description = "Job scraper for LinkedIn, Indeed, Glassdoor & ZipRecruiter"
|
||||||
authors = ["Zachary Hampton <zachary@bunsly.com>", "Cullen Watson <cullen@bunsly.com>"]
|
authors = ["Zachary Hampton <zachary@bunsly.com>", "Cullen Watson <cullen@bunsly.com>"]
|
||||||
homepage = "https://github.com/Bunsly/JobSpy"
|
homepage = "https://github.com/Bunsly/JobSpy"
|
||||||
|
|||||||
@@ -5,7 +5,7 @@ from typing import Tuple
|
|||||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||||
|
|
||||||
from .jobs import JobType, Location
|
from .jobs import JobType, Location
|
||||||
from .scrapers.utils import logger
|
from .scrapers.utils import logger, set_logger_level
|
||||||
from .scrapers.indeed import IndeedScraper
|
from .scrapers.indeed import IndeedScraper
|
||||||
from .scrapers.ziprecruiter import ZipRecruiterScraper
|
from .scrapers.ziprecruiter import ZipRecruiterScraper
|
||||||
from .scrapers.glassdoor import GlassdoorScraper
|
from .scrapers.glassdoor import GlassdoorScraper
|
||||||
@@ -36,11 +36,12 @@ def scrape_jobs(
|
|||||||
linkedin_company_ids: list[int] | None = None,
|
linkedin_company_ids: list[int] | None = None,
|
||||||
offset: int | None = 0,
|
offset: int | None = 0,
|
||||||
hours_old: int = None,
|
hours_old: int = None,
|
||||||
|
verbose: int = 2,
|
||||||
**kwargs,
|
**kwargs,
|
||||||
) -> pd.DataFrame:
|
) -> pd.DataFrame:
|
||||||
"""
|
"""
|
||||||
Simultaneously scrapes job data from multiple job sites.
|
Simultaneously scrapes job data from multiple job sites.
|
||||||
:return: results_wanted: pandas dataframe containing job data
|
:return: pandas dataframe containing job data
|
||||||
"""
|
"""
|
||||||
SCRAPER_MAPPING = {
|
SCRAPER_MAPPING = {
|
||||||
Site.LINKEDIN: LinkedInScraper,
|
Site.LINKEDIN: LinkedInScraper,
|
||||||
@@ -48,6 +49,7 @@ def scrape_jobs(
|
|||||||
Site.ZIP_RECRUITER: ZipRecruiterScraper,
|
Site.ZIP_RECRUITER: ZipRecruiterScraper,
|
||||||
Site.GLASSDOOR: GlassdoorScraper,
|
Site.GLASSDOOR: GlassdoorScraper,
|
||||||
}
|
}
|
||||||
|
set_logger_level(verbose)
|
||||||
|
|
||||||
def map_str_to_site(site_name: str) -> Site:
|
def map_str_to_site(site_name: str) -> Site:
|
||||||
return Site[site_name.upper()]
|
return Site[site_name.upper()]
|
||||||
|
|||||||
@@ -150,6 +150,15 @@ class IndeedScraper(Scraper):
|
|||||||
""".format(
|
""".format(
|
||||||
start=self.scraper_input.hours_old
|
start=self.scraper_input.hours_old
|
||||||
)
|
)
|
||||||
|
elif self.scraper_input.easy_apply:
|
||||||
|
filters_str = """
|
||||||
|
filters: {
|
||||||
|
keyword: {
|
||||||
|
field: "indeedApplyScope",
|
||||||
|
keys: ["DESKTOP"]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
"""
|
||||||
elif self.scraper_input.job_type or self.scraper_input.is_remote:
|
elif self.scraper_input.job_type or self.scraper_input.is_remote:
|
||||||
job_type_key_mapping = {
|
job_type_key_mapping = {
|
||||||
JobType.FULL_TIME: "CF3CP",
|
JobType.FULL_TIME: "CF3CP",
|
||||||
|
|||||||
@@ -21,6 +21,23 @@ if not logger.handlers:
|
|||||||
logger.addHandler(console_handler)
|
logger.addHandler(console_handler)
|
||||||
|
|
||||||
|
|
||||||
|
def set_logger_level(verbose: int = 2):
|
||||||
|
"""
|
||||||
|
Adjusts the logger's level. This function allows the logging level to be changed at runtime.
|
||||||
|
|
||||||
|
Parameters:
|
||||||
|
- verbose: int {0, 1, 2} (default=2, all logs)
|
||||||
|
"""
|
||||||
|
if verbose is None:
|
||||||
|
return
|
||||||
|
level_name = {2: "INFO", 1: "WARNING", 0: "ERROR"}.get(verbose, "INFO")
|
||||||
|
level = getattr(logging, level_name.upper(), None)
|
||||||
|
if level is not None:
|
||||||
|
logger.setLevel(level)
|
||||||
|
else:
|
||||||
|
raise ValueError(f"Invalid log level: {level_name}")
|
||||||
|
|
||||||
|
|
||||||
def markdown_converter(description_html: str):
|
def markdown_converter(description_html: str):
|
||||||
if description_html is None:
|
if description_html is None:
|
||||||
return None
|
return None
|
||||||
|
|||||||
Reference in New Issue
Block a user