diff --git a/pyproject.toml b/pyproject.toml index eea69ad..e939358 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "python-jobspy" -version = "1.1.43" +version = "1.1.44" description = "Job scraper for LinkedIn, Indeed, Glassdoor & ZipRecruiter" authors = ["Zachary Hampton ", "Cullen Watson "] homepage = "https://github.com/Bunsly/JobSpy" diff --git a/src/jobspy/__init__.py b/src/jobspy/__init__.py index d7ed0dc..7370c37 100644 --- a/src/jobspy/__init__.py +++ b/src/jobspy/__init__.py @@ -192,4 +192,4 @@ def scrape_jobs( else: jobs_formatted_df = pd.DataFrame() - return jobs_formatted_df.sort_values(by='date_posted', ascending=False) + return jobs_formatted_df.sort_values(by=['site', 'date_posted'], ascending=[True, False]) diff --git a/src/jobspy/scrapers/glassdoor/__init__.py b/src/jobspy/scrapers/glassdoor/__init__.py index 3b3ec7a..3352a2a 100644 --- a/src/jobspy/scrapers/glassdoor/__init__.py +++ b/src/jobspy/scrapers/glassdoor/__init__.py @@ -6,7 +6,6 @@ This module contains routines to scrape Glassdoor. """ import json import requests -from bs4 import BeautifulSoup from typing import Optional from datetime import datetime, timedelta from concurrent.futures import ThreadPoolExecutor, as_completed @@ -14,7 +13,7 @@ from ..utils import count_urgent_words, extract_emails_from_text from .. import Scraper, ScraperInput, Site from ..exceptions import GlassdoorException -from ..utils import create_session, modify_and_get_description +from ..utils import create_session from ...jobs import ( JobPost, Compensation, @@ -35,6 +34,7 @@ class GlassdoorScraper(Scraper): self.url = None self.country = None + self.session = None self.jobs_per_page = 30 self.seen_urls = set() @@ -53,8 +53,7 @@ class GlassdoorScraper(Scraper): payload = self.add_payload( scraper_input, location_id, location_type, page_num, cursor ) - session = create_session(self.proxy, is_tls=False, has_retry=True) - response = session.post( + response = self.session.post( f"{self.url}/graph", headers=self.headers(), timeout=10, data=payload ) if response.status_code != 200: @@ -73,7 +72,6 @@ class GlassdoorScraper(Scraper): with ThreadPoolExecutor(max_workers=self.jobs_per_page) as executor: future_to_job_data = {executor.submit(self.process_job, job): job for job in jobs_data} for future in as_completed(future_to_job_data): - job_data = future_to_job_data[future] try: job_post = future.result() if job_post: @@ -111,7 +109,7 @@ class GlassdoorScraper(Scraper): try: description = self.fetch_job_description(job_id) - except Exception as e : + except: description = None job_post = JobPost( @@ -145,6 +143,8 @@ class GlassdoorScraper(Scraper): all_jobs: list[JobPost] = [] cursor = None max_pages = 30 + self.session = create_session(self.proxy, is_tls=False, has_retry=True) + self.session.get(self.url) try: for page in range( @@ -201,8 +201,7 @@ class GlassdoorScraper(Scraper): return None data = response.json()[0] desc = data['data']['jobview']['job']['description'] - soup = BeautifulSoup(desc, 'html.parser') - return modify_and_get_description(soup) + return desc @staticmethod def parse_compensation(data: dict) -> Optional[Compensation]: @@ -267,7 +266,6 @@ class GlassdoorScraper(Scraper): filter_params.append({"filterKey": "fromAge", "values": str(fromage)}) payload = { "operationName": "JobSearchResultsQuery", - "variables": { "excludeJobListingIds": [], "filterParams": filter_params, @@ -281,21 +279,177 @@ class GlassdoorScraper(Scraper): "fromage": fromage, "sort": "date" }, - "query": "query JobSearchResultsQuery($excludeJobListingIds: [Long!], $keyword: String, $locationId: Int, $locationType: LocationTypeEnum, $numJobsToShow: Int!, $pageCursor: String, $pageNumber: Int, $filterParams: [FilterParams], $originalPageUrl: String, $seoFriendlyUrlInput: String, $parameterUrlInput: String, $seoUrl: Boolean) {\n jobListings(\n contextHolder: {searchParams: {excludeJobListingIds: $excludeJobListingIds, keyword: $keyword, locationId: $locationId, locationType: $locationType, numPerPage: $numJobsToShow, pageCursor: $pageCursor, pageNumber: $pageNumber, filterParams: $filterParams, originalPageUrl: $originalPageUrl, seoFriendlyUrlInput: $seoFriendlyUrlInput, parameterUrlInput: $parameterUrlInput, seoUrl: $seoUrl, searchType: SR}}\n ) {\n companyFilterOptions {\n id\n shortName\n __typename\n }\n filterOptions\n indeedCtk\n jobListings {\n ...JobView\n __typename\n }\n jobListingSeoLinks {\n linkItems {\n position\n url\n __typename\n }\n __typename\n }\n jobSearchTrackingKey\n jobsPageSeoData {\n pageMetaDescription\n pageTitle\n __typename\n }\n paginationCursors {\n cursor\n pageNumber\n __typename\n }\n indexablePageForSeo\n searchResultsMetadata {\n searchCriteria {\n implicitLocation {\n id\n localizedDisplayName\n type\n __typename\n }\n keyword\n location {\n id\n shortName\n localizedShortName\n localizedDisplayName\n type\n __typename\n }\n __typename\n }\n footerVO {\n countryMenu {\n childNavigationLinks {\n id\n link\n textKey\n __typename\n }\n __typename\n }\n __typename\n }\n helpCenterDomain\n helpCenterLocale\n jobAlert {\n jobAlertExists\n __typename\n }\n jobSerpFaq {\n questions {\n answer\n question\n __typename\n }\n __typename\n }\n jobSerpJobOutlook {\n occupation\n paragraph\n __typename\n }\n showMachineReadableJobs\n __typename\n }\n serpSeoLinksVO {\n relatedJobTitlesResults\n searchedJobTitle\n searchedKeyword\n searchedLocationIdAsString\n searchedLocationSeoName\n searchedLocationType\n topCityIdsToNameResults {\n key\n value\n __typename\n }\n topEmployerIdsToNameResults {\n key\n value\n __typename\n }\n topEmployerNameResults\n topOccupationResults\n __typename\n }\n totalJobsCount\n __typename\n }\n}\n\nfragment JobView on JobListingSearchResult {\n jobview {\n header {\n adOrderId\n advertiserType\n adOrderSponsorshipLevel\n ageInDays\n divisionEmployerName\n easyApply\n employer {\n id\n name\n shortName\n __typename\n }\n employerNameFromSearch\n goc\n gocConfidence\n gocId\n jobCountryId\n jobLink\n jobResultTrackingKey\n jobTitleText\n locationName\n locationType\n locId\n needsCommission\n payCurrency\n payPeriod\n payPeriodAdjustedPay {\n p10\n p50\n p90\n __typename\n }\n rating\n salarySource\n savedJobId\n sponsored\n __typename\n }\n job {\n descriptionFragments\n importConfigId\n jobTitleId\n jobTitleText\n listingId\n __typename\n }\n jobListingAdminDetails {\n cpcVal\n importConfigId\n jobListingId\n jobSourceId\n userEligibleForAdminJobDetails\n __typename\n }\n overview {\n shortName\n squareLogoUrl\n __typename\n }\n __typename\n }\n __typename\n}\n", + "query": """ + query JobSearchResultsQuery( + $excludeJobListingIds: [Long!], + $keyword: String, + $locationId: Int, + $locationType: LocationTypeEnum, + $numJobsToShow: Int!, + $pageCursor: String, + $pageNumber: Int, + $filterParams: [FilterParams], + $originalPageUrl: String, + $seoFriendlyUrlInput: String, + $parameterUrlInput: String, + $seoUrl: Boolean + ) { + jobListings( + contextHolder: { + searchParams: { + excludeJobListingIds: $excludeJobListingIds, + keyword: $keyword, + locationId: $locationId, + locationType: $locationType, + numPerPage: $numJobsToShow, + pageCursor: $pageCursor, + pageNumber: $pageNumber, + filterParams: $filterParams, + originalPageUrl: $originalPageUrl, + seoFriendlyUrlInput: $seoFriendlyUrlInput, + parameterUrlInput: $parameterUrlInput, + seoUrl: $seoUrl, + searchType: SR + } + } + ) { + companyFilterOptions { + id + shortName + __typename + } + filterOptions + indeedCtk + jobListings { + ...JobView + __typename + } + jobListingSeoLinks { + linkItems { + position + url + __typename + } + __typename + } + jobSearchTrackingKey + jobsPageSeoData { + pageMetaDescription + pageTitle + __typename + } + paginationCursors { + cursor + pageNumber + __typename + } + indexablePageForSeo + searchResultsMetadata { + searchCriteria { + implicitLocation { + id + localizedDisplayName + type + __typename + } + keyword + location { + id + shortName + localizedShortName + localizedDisplayName + type + __typename + } + __typename + } + helpCenterDomain + helpCenterLocale + jobSerpJobOutlook { + occupation + paragraph + __typename + } + showMachineReadableJobs + __typename + } + totalJobsCount + __typename + } + } + + fragment JobView on JobListingSearchResult { + jobview { + header { + adOrderId + advertiserType + adOrderSponsorshipLevel + ageInDays + divisionEmployerName + easyApply + employer { + id + name + shortName + __typename + } + employerNameFromSearch + goc + gocConfidence + gocId + jobCountryId + jobLink + jobResultTrackingKey + jobTitleText + locationName + locationType + locId + needsCommission + payCurrency + payPeriod + payPeriodAdjustedPay { + p10 + p50 + p90 + __typename + } + rating + salarySource + savedJobId + sponsored + __typename + } + job { + description + importConfigId + jobTitleId + jobTitleText + listingId + __typename + } + jobListingAdminDetails { + cpcVal + importConfigId + jobListingId + jobSourceId + userEligibleForAdminJobDetails + __typename + } + overview { + shortName + squareLogoUrl + __typename + } + __typename + } + __typename + } + """ } - job_type_filters = { - JobType.FULL_TIME: "fulltime", - JobType.PART_TIME: "parttime", - JobType.CONTRACT: "contract", - JobType.INTERNSHIP: "internship", - JobType.TEMPORARY: "temporary", - } - - if scraper_input.job_type in job_type_filters: - filter_value = job_type_filters[scraper_input.job_type] + if scraper_input.job_type: payload["variables"]["filterParams"].append( - {"filterKey": "jobType", "values": filter_value} + {"filterKey": "jobType", "values": scraper_input.job_type.value[0]} ) return json.dumps([payload]) @@ -331,7 +485,6 @@ class GlassdoorScraper(Scraper): "apollographql-client-name": "job-search-next", "apollographql-client-version": "4.65.5", "content-type": "application/json", - "cookie": 'gdId=91e2dfc4-c8b5-4fa7-83d0-11512b80262c; G_ENABLED_IDPS=google; trs=https%3A%2F%2Fwww.redhat.com%2F:referral:referral:2023-07-05+09%3A50%3A14.862:undefined:undefined; g_state={"i_p":1688587331651,"i_l":1}; _cfuvid=.7llazxhYFZWi6EISSPdVjtqF0NMVwzxr_E.cB1jgLs-1697828392979-0-604800000; GSESSIONID=undefined; JSESSIONID=F03DD1B5EE02DB6D842FE42B142F88F3; cass=1; jobsClicked=true; indeedCtk=1hd77b301k79i801; asst=1697829114.2; G_AUTHUSER_H=0; uc=8013A8318C98C517FE6DD0024636DFDEF978FC33266D93A2FAFEF364EACA608949D8B8FA2DC243D62DE271D733EB189D809ABE5B08D7B1AE865D217BD4EEBB97C282F5DA5FEFE79C937E3F6110B2A3A0ADBBA3B4B6DF5A996FEE00516100A65FCB11DA26817BE8D1C1BF6CFE36B5B68A3FDC2CFEC83AB797F7841FBB157C202332FC7E077B56BD39B167BDF3D9866E3B; AWSALB=zxc/Yk1nbWXXT6HjNyn3H4h4950ckVsFV/zOrq5LSoChYLE1qV+hDI8Axi3fUa9rlskndcO0M+Fw+ZnJ+AQ2afBFpyOd1acouLMYgkbEpqpQaWhY6/Gv4QH1zBcJ; AWSALBCORS=zxc/Yk1nbWXXT6HjNyn3H4h4950ckVsFV/zOrq5LSoChYLE1qV+hDI8Axi3fUa9rlskndcO0M+Fw+ZnJ+AQ2afBFpyOd1acouLMYgkbEpqpQaWhY6/Gv4QH1zBcJ; gdsid=1697828393025:1697830776351:668396EDB9E6A832022D34414128093D; at=HkH8Hnqi9uaMC7eu0okqyIwqp07ht9hBvE1_St7E_hRqPvkO9pUeJ1Jcpds4F3g6LL5ADaCNlxrPn0o6DumGMfog8qI1-zxaV_jpiFs3pugntw6WpVyYWdfioIZ1IDKupyteeLQEM1AO4zhGjY_rPZynpsiZBPO_B1au94sKv64rv23yvP56OiWKKfI-8_9hhLACEwWvM-Az7X-4aE2QdFt93VJbXbbGVf07bdDZfimsIkTtgJCLSRhU1V0kEM1Efyu66vo3m77gFFaMW7lxyYnb36I5PdDtEXBm3aL-zR7-qa5ywd94ISEivgqQOA4FPItNhqIlX4XrfD1lxVz6rfPaoTIDi4DI6UMCUjwyPsuv8mn0rYqDfRnmJpZ97fJ5AnhrknAd_6ZWN5v1OrxJczHzcXd8LO820QPoqxzzG13bmSTXLwGSxMUCtSrVsq05hicimQ3jpRt0c1dA4OkTNqF7_770B9JfcHcM8cr8-C4IL56dnOjr9KBGfN1Q2IvZM2cOBRbV7okiNOzKVZ3qJ24AE34WA2F3U6Whiu6H8nIuGG5hSNkVygY6CtglNZfFF9p8pJAZm79PngrrBv-CXFBZmhYLFo46lmFetDkiJ6mirtez4tKpzTIYjIp4_JAkiZFwbLJ2QGH4mK8kyyW0lZiX1DTuQec50N_5wvRo0Gt7nlKxzLsApMnaNhuQeH5ygh_pa381ORo9mQGi0EYF9zk00pa2--z4PtjfQ8KFq36GgpxKy5-o4qgqygZj8F01L8r-FiX2G4C7PREMIpAyHX2A4-_JxA1IS2j12EyqKTLqE9VcP06qm2Z-YuIW3ctmpMxy5G9_KiEiGv17weizhSFnl6SbpAEY-2VSmQ5V6jm3hoMp2jemkuGCRkZeFstLDEPxlzFN7WM; __cf_bm=zGaVjIJw4irf40_7UVw54B6Ohm271RUX4Tc8KVScrbs-1697830777-0-AYv2GnKTnnCU+cY9xHbJunO0DwlLDO6SIBnC/s/qldpKsGK0rRAjD6y8lbyATT/KlS7g29OZaN4fbd0lrJg0KmWbIybZIzfWVLHSYePVuOhu; asst=1697829114.2; at=dFhXf64wsf2TlnWy41xLs7skJkuxgKToEGcjGtDfUvW4oEAJ4tTIR5dKQ8wbwT75aIaGgdCfvcb-da7vwrCGWscCncmfLFQpJ9l-LLwoRfk-pMsxHhd77wvf-W7I0HSm7-Q5lQJqI9WyNGRxOa-RpzBTf4L8_Et4-3FzjPaAoYY5pY1FhuwXbN5asGOAMW-p8cjpbfn3PumlIYuckguWnjrcY2F31YJ_1noeoHM9tCGpymANbqGXRkG6aXY7yCfVXtdgZU1K5SMeaSPZIuF_iLUxjc_corzpNiH6qq7BIAmh-e5Aa-g7cwpZcln1fmwTVw4uTMZf1eLIMTa9WzgqZNkvG-sGaq_XxKA_Wai6xTTkOHfRgm4632Ba2963wdJvkGmUUa3tb_L4_wTgk3eFnHp5JhghLfT2Pe3KidP-yX__vx8JOsqe3fndCkKXgVz7xQKe1Dur-sMNlGwi4LXfguTT2YUI8C5Miq3pj2IHc7dC97eyyAiAM4HvyGWfaXWZcei6oIGrOwMvYgy0AcwFry6SIP2SxLT5TrxinRRuem1r1IcOTJsMJyUPp1QsZ7bOyq9G_0060B4CPyovw5523hEuqLTM-R5e5yavY6C_1DHUyE15C3mrh7kdvmlGZeflnHqkFTEKwwOftm-Mv-CKD5Db9ABFGNxKB2FH7nDH67hfOvm4tGNMzceBPKYJ3wciTt9jK3wy39_7cOYVywfrZ-oLhw_XtsbGSSeGn3HytrfgSADAh2sT0Gg6eCC9Xy1vh-Za337SVLUDXZ73W2xJxxUHBkFzZs8L_Xndo5DsbpWhVs9IYUGyraJdqB3SLgDbAppIBCJl4fx6_DG8-xOQPBvuFMlTROe1JVdHOzXI1GElwFDTuH1pjkg4I2G0NhAbE06Y-1illQE; gdsid=1697828393025:1697831731408:99C30D94108AC3030D61C736DDCDF11C', "gd-csrf-token": "Ft6oHEWlRZrxDww95Cpazw:0pGUrkb2y3TyOpAIqF2vbPmUXoXVkD3oEGDVkvfeCerceQ5-n8mBg3BovySUIjmCPHCaW0H2nQVdqzbtsYqf4Q:wcqRqeegRUa9MVLJGyujVXB7vWFPjdaS1CtrrzJq-ok", "origin": "https://www.glassdoor.com", "referer": "https://www.glassdoor.com/", diff --git a/src/jobspy/scrapers/indeed/__init__.py b/src/jobspy/scrapers/indeed/__init__.py index ce4fbd1..5b05cbd 100644 --- a/src/jobspy/scrapers/indeed/__init__.py +++ b/src/jobspy/scrapers/indeed/__init__.py @@ -11,7 +11,6 @@ import requests from typing import Any from datetime import datetime -import urllib.parse from bs4 import BeautifulSoup from bs4.element import Tag from concurrent.futures import ThreadPoolExecutor, Future @@ -22,7 +21,7 @@ from ..utils import ( extract_emails_from_text, create_session, get_enum_from_job_type, - modify_and_get_description + logger ) from ...jobs import ( JobPost, @@ -50,13 +49,14 @@ class IndeedScraper(Scraper): def scrape_page( self, scraper_input: ScraperInput, page: int - ) -> tuple[list[JobPost], int]: + ) -> list[JobPost]: """ Scrapes a page of Indeed for jobs with scraper_input criteria :param scraper_input: :param page: :return: jobs found on page, total number of jobs found for search """ + job_list = [] self.country = scraper_input.country domain = self.country.indeed_domain_value self.url = f"https://{domain}.indeed.com" @@ -76,14 +76,14 @@ class IndeedScraper(Scraper): ) except Exception as e: if "Proxy responded with" in str(e): - raise IndeedException("bad proxy") - raise IndeedException(str(e)) + logger.error(f'Indeed: Bad proxy') + else: + logger.error(f'Indeed: {str(e)}') + return job_list soup = BeautifulSoup(response.content, "html.parser") - job_list = [] - total_num_jobs = IndeedScraper.total_jobs(soup) if "did not match any jobs" in response.text: - return job_list, total_num_jobs + return job_list jobs = IndeedScraper.parse_jobs( soup @@ -145,7 +145,7 @@ class IndeedScraper(Scraper): job_list = [result.result() for result in job_results if result.result()] - return job_list, total_num_jobs + return job_list def scrape(self, scraper_input: ScraperInput) -> JobResponse: """ @@ -153,7 +153,7 @@ class IndeedScraper(Scraper): :param scraper_input: :return: job_response """ - job_list, total_results = self.scrape_page(scraper_input, 0) + job_list = self.scrape_page(scraper_input, 0) pages_processed = 1 while len(self.seen_urls) < scraper_input.results_wanted: @@ -167,7 +167,7 @@ class IndeedScraper(Scraper): ] for future in futures: - jobs, _ = future.result() + jobs = future.result() if jobs: job_list += jobs new_jobs = True @@ -182,55 +182,7 @@ class IndeedScraper(Scraper): if len(self.seen_urls) > scraper_input.results_wanted: job_list = job_list[:scraper_input.results_wanted] - job_response = JobResponse( - jobs=job_list, - total_results=total_results, - ) - return job_response - - def get_description(self, job_page_url: str) -> str | None: - """ - Retrieves job description by going to the job page url - :param job_page_url: - :return: description - """ - parsed_url = urllib.parse.urlparse(job_page_url) - params = urllib.parse.parse_qs(parsed_url.query) - jk_value = params.get("jk", [None])[0] - formatted_url = f"{self.url}/m/viewjob?jk={jk_value}&spa=1" - session = create_session(self.proxy) - - try: - response = session.get( - formatted_url, - headers=self.get_headers(), - allow_redirects=True, - timeout_seconds=5, - ) - except Exception as e: - return None - - if response.status_code not in range(200, 400): - return None - - try: - soup = BeautifulSoup(response.text, 'html.parser') - script_tags = soup.find_all('script') - - job_description = '' - for tag in script_tags: - if 'window._initialData' in tag.text: - json_str = tag.text - json_str = json_str.split('window._initialData=')[1] - json_str = json_str.rsplit(';', 1)[0] - data = json.loads(json_str) - job_description = data["jobInfoWrapperModel"]["jobInfoModel"]["sanitizedJobDescription"] - break - except (KeyError, TypeError, IndexError): - return None - - soup = BeautifulSoup(job_description, "html.parser") - return modify_and_get_description(soup) + return JobResponse(jobs=job_list) @staticmethod def get_job_type(job: dict) -> list[JobType] | None: @@ -330,24 +282,6 @@ class IndeedScraper(Scraper): "Could not find any results for the search" ) - @staticmethod - def total_jobs(soup: BeautifulSoup) -> int: - """ - Parses the total jobs for that search from soup object - :param soup: - :return: total_num_jobs - """ - script = soup.find("script", string=lambda t: t and "window._initialData" in t) - - pattern = re.compile(r"window._initialData\s*=\s*({.*})\s*;", re.DOTALL) - match = pattern.search(script.string) - total_num_jobs = 0 - if match: - json_str = match.group(1) - data = json.loads(json_str) - total_num_jobs = int(data["searchTitleBarModel"]["totalNumResults"]) - return total_num_jobs - @staticmethod def get_headers(): return { @@ -380,7 +314,7 @@ class IndeedScraper(Scraper): if scraper_input.is_remote: sc_values.append("attr(DSQF7)") if scraper_input.job_type: - sc_values.append("jt({})".format(scraper_input.job_type.value)) + sc_values.append("jt({})".format(scraper_input.job_type.value[0])) if sc_values: params["sc"] = "0kf:" + "".join(sc_values) + ";" @@ -406,7 +340,7 @@ class IndeedScraper(Scraper): taxonomy["label"] == "remote" and len(taxonomy["attributes"]) > 0 for taxonomy in job.get("taxonomyAttributes", []) ) - return is_remote_in_attributes or is_remote_in_description or is_remote_in_location + return is_remote_in_attributes or is_remote_in_description or is_remote_in_location or is_remote_in_taxonomy def get_job_details(self, job_keys: list[str]) -> dict: """ diff --git a/src/jobspy/scrapers/linkedin/__init__.py b/src/jobspy/scrapers/linkedin/__init__.py index af52a06..4833337 100644 --- a/src/jobspy/scrapers/linkedin/__init__.py +++ b/src/jobspy/scrapers/linkedin/__init__.py @@ -31,8 +31,7 @@ from ..utils import ( count_urgent_words, extract_emails_from_text, get_enum_from_job_type, - currency_parser, - modify_and_get_description + currency_parser ) @@ -236,10 +235,15 @@ class LinkedInScraper(Scraper): div_content = soup.find( "div", class_=lambda x: x and "show-more-less-html__markup" in x ) - description = None - if div_content: - description = modify_and_get_description(div_content) + if div_content is not None: + def remove_attributes(tag): + for attr in list(tag.attrs): + del tag[attr] + return tag + + div_content = remove_attributes(div_content) + description = div_content.prettify(formatter="html") def get_job_type( soup_job_type: BeautifulSoup, diff --git a/src/jobspy/scrapers/utils.py b/src/jobspy/scrapers/utils.py index 84e4c0b..5366193 100644 --- a/src/jobspy/scrapers/utils.py +++ b/src/jobspy/scrapers/utils.py @@ -1,4 +1,5 @@ import re +import logging import numpy as np import tls_client @@ -7,14 +8,14 @@ from requests.adapters import HTTPAdapter, Retry from ..jobs import JobType - -def modify_and_get_description(soup): - for li in soup.find_all('li'): - li.string = "- " + li.get_text() - - description = soup.get_text(separator='\n').strip() - description = re.sub(r'\n+', '\n', description) - return description +logger = logging.getLogger("JobSpy") +if not logger.handlers: + logger.setLevel(logging.ERROR) + console_handler = logging.StreamHandler() + console_handler.setLevel(logging.ERROR) + formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') + console_handler.setFormatter(formatter) + logger.addHandler(console_handler) def count_urgent_words(description: str) -> int: @@ -79,6 +80,7 @@ def get_enum_from_job_type(job_type_str: str) -> JobType | None: res = job_type return res + def currency_parser(cur_str): # Remove any non-numerical characters # except for ',' '.' or '-' (e.g. EUR) @@ -94,3 +96,5 @@ def currency_parser(cur_str): num = float(cur_str) return np.round(num, 2) + + diff --git a/src/jobspy/scrapers/ziprecruiter/__init__.py b/src/jobspy/scrapers/ziprecruiter/__init__.py index 1e21dd5..fd7f25a 100644 --- a/src/jobspy/scrapers/ziprecruiter/__init__.py +++ b/src/jobspy/scrapers/ziprecruiter/__init__.py @@ -9,13 +9,12 @@ import time from datetime import datetime, timezone from typing import Optional, Tuple, Any -from bs4 import BeautifulSoup from concurrent.futures import ThreadPoolExecutor from .. import Scraper, ScraperInput, Site from ..exceptions import ZipRecruiterException from ...jobs import JobPost, Compensation, Location, JobResponse, JobType, Country -from ..utils import count_urgent_words, extract_emails_from_text, create_session, modify_and_get_description +from ..utils import count_urgent_words, extract_emails_from_text, create_session class ZipRecruiterScraper(Scraper): @@ -107,9 +106,7 @@ class ZipRecruiterScraper(Scraper): return self.seen_urls.add(job_url) - job_description_html = job.get("job_description", "").strip() - description_soup = BeautifulSoup(job_description_html, "html.parser") - description = modify_and_get_description(description_soup) + description = job.get("job_description", "").strip() company = job.get("hiring_company", {}).get("name") country_value = "usa" if job.get("job_country") == "US" else "canada" @@ -168,25 +165,16 @@ class ZipRecruiterScraper(Scraper): if scraper_input.hours_old: fromage = max(scraper_input.hours_old // 24, 1) if scraper_input.hours_old else None params['days'] = fromage - job_type_value = None + job_type_map = { + JobType.FULL_TIME: 'full_time', + JobType.PART_TIME: 'part_time' + } if scraper_input.job_type: - if scraper_input.job_type.value == "fulltime": - job_type_value = "full_time" - elif scraper_input.job_type.value == "parttime": - job_type_value = "part_time" - else: - job_type_value = scraper_input.job_type.value + params['employment_type'] = job_type_map[scraper_input.job_type] if scraper_input.job_type in job_type_map else scraper_input.job_type.value[0] if scraper_input.easy_apply: params['zipapply'] = 1 - - if job_type_value: - params[ - "refine_by_employment" - ] = f"employment_type:employment_type:{job_type_value}" - if scraper_input.is_remote: - params["refine_by_location_type"] = "only_remote" - + params["remote"] = 1 if scraper_input.distance: params["radius"] = scraper_input.distance