diff --git a/pyproject.toml b/pyproject.toml index b419438..a7ff553 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "python-jobspy" -version = "1.1.1" +version = "1.1.2" description = "Job scraper for LinkedIn, Indeed & ZipRecruiter" authors = ["Zachary Hampton ", "Cullen Watson "] readme = "README.md" diff --git a/src/jobspy/__init__.py b/src/jobspy/__init__.py index e2c0fdd..a49d33f 100644 --- a/src/jobspy/__init__.py +++ b/src/jobspy/__init__.py @@ -26,7 +26,7 @@ def _map_str_to_site(site_name: str) -> Site: def scrape_jobs( - site_name: str | Site | List[Site], + site_name: str | List[str] | Site | List[Site], search_term: str, location: str = "", distance: int = None, @@ -43,11 +43,12 @@ def scrape_jobs( """ if type(site_name) == str: - site_name = _map_str_to_site(site_name) + site_type = [_map_str_to_site(site_name)] + else: #: if type(site_name) == list + site_type = [_map_str_to_site(site) if type(site) == str else site_name for site in site_name] country_enum = Country.from_string(country_indeed) - site_type = [site_name] if type(site_name) == Site else site_name scraper_input = ScraperInput( site_type=site_type, country=country_enum, @@ -122,7 +123,6 @@ def scrape_jobs( errors_list = [(key, value) for key, value in errors.items()] errors_df = pd.DataFrame(errors_list, columns=["Site", "Error"]) - if dfs: df = pd.concat(dfs, ignore_index=True) if hyperlinks: diff --git a/src/jobspy/scrapers/indeed/__init__.py b/src/jobspy/scrapers/indeed/__init__.py index 17cbead..0f32841 100644 --- a/src/jobspy/scrapers/indeed/__init__.py +++ b/src/jobspy/scrapers/indeed/__init__.py @@ -78,9 +78,7 @@ class IndeedScraper(Scraper): raise StatusException(response.status_code) soup = BeautifulSoup(response.content, "html.parser") - with open("text2.html", "w", encoding="utf-8") as f: - f.write(str(soup)) - if "did not match any jobs" in str(soup): + if "did not match any jobs" in response.text: raise ParsingException("Search did not match any jobs") jobs = IndeedScraper.parse_jobs( diff --git a/src/tests/test_all.py b/src/tests/test_all.py new file mode 100644 index 0000000..14428f5 --- /dev/null +++ b/src/tests/test_all.py @@ -0,0 +1,10 @@ +from ..jobspy import scrape_jobs + + +def test_all(): + result = scrape_jobs( + site_name=["linkedin", "indeed", "zip_recruiter"], + search_term="software engineer", + results_wanted=5, + ) + assert result is not None and result.errors.empty is True diff --git a/src/tests/test_indeed.py b/src/tests/test_indeed.py index bd9dfd1..c7442b9 100644 --- a/src/tests/test_indeed.py +++ b/src/tests/test_indeed.py @@ -6,4 +6,4 @@ def test_indeed(): site_name="indeed", search_term="software engineer", ) - assert result is not None + assert result is not None and result.errors.empty is True diff --git a/src/tests/test_linkedin.py b/src/tests/test_linkedin.py index af79510..d8f62d1 100644 --- a/src/tests/test_linkedin.py +++ b/src/tests/test_linkedin.py @@ -1,4 +1,4 @@ -from jobspy import scrape_jobs +from ..jobspy import scrape_jobs def test_linkedin(): @@ -6,4 +6,4 @@ def test_linkedin(): site_name="linkedin", search_term="software engineer", ) - assert result is not None + assert result is not None and result.errors.empty is True diff --git a/src/tests/test_ziprecruiter.py b/src/tests/test_ziprecruiter.py index 0fc189a..b648e21 100644 --- a/src/tests/test_ziprecruiter.py +++ b/src/tests/test_ziprecruiter.py @@ -1,4 +1,4 @@ -from jobspy import scrape_jobs +from ..jobspy import scrape_jobs def test_ziprecruiter(): @@ -7,4 +7,4 @@ def test_ziprecruiter(): search_term="software engineer", ) - assert result is not None + assert result is not None and result.errors.empty is True