fix job type search (#106)

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "python-jobspy"
-version = "1.1.43"
+version = "1.1.44"
 description = "Job scraper for LinkedIn, Indeed, Glassdoor & ZipRecruiter"
 authors = ["Zachary Hampton <zachary@bunsly.com>", "Cullen Watson <cullen@bunsly.com>"]
 homepage = "https://github.com/Bunsly/JobSpy"

src/jobspy/__init__.py

@@ -192,4 +192,4 @@ def scrape_jobs(
     else:
         jobs_formatted_df = pd.DataFrame()
 
-    return jobs_formatted_df.sort_values(by='date_posted', ascending=False)
+    return jobs_formatted_df.sort_values(by=['site', 'date_posted'], ascending=[True, False])

src/jobspy/scrapers/glassdoor/__init__.py

@@ -6,7 +6,6 @@ This module contains routines to scrape Glassdoor.
 """
 import json
 import requests
-from bs4 import BeautifulSoup
 from typing import Optional
 from datetime import datetime, timedelta
 from concurrent.futures import ThreadPoolExecutor, as_completed
@@ -14,7 +13,7 @@ from ..utils import count_urgent_words, extract_emails_from_text
 
 from .. import Scraper, ScraperInput, Site
 from ..exceptions import GlassdoorException
-from ..utils import create_session, modify_and_get_description
+from ..utils import create_session
 from ...jobs import (
     JobPost,
     Compensation,
@@ -35,6 +34,7 @@ class GlassdoorScraper(Scraper):
 
         self.url = None
         self.country = None
+        self.session = None
         self.jobs_per_page = 30
         self.seen_urls = set()
 
@@ -53,8 +53,7 @@ class GlassdoorScraper(Scraper):
             payload = self.add_payload(
                 scraper_input, location_id, location_type, page_num, cursor
             )
-            session = create_session(self.proxy, is_tls=False, has_retry=True)
+            response = self.session.post(
-            response = session.post(
                 f"{self.url}/graph", headers=self.headers(), timeout=10, data=payload
             )
             if response.status_code != 200:
@@ -73,7 +72,6 @@ class GlassdoorScraper(Scraper):
         with ThreadPoolExecutor(max_workers=self.jobs_per_page) as executor:
             future_to_job_data = {executor.submit(self.process_job, job): job for job in jobs_data}
             for future in as_completed(future_to_job_data):
-                job_data = future_to_job_data[future]
                 try:
                     job_post = future.result()
                     if job_post:
@@ -111,7 +109,7 @@ class GlassdoorScraper(Scraper):
 
         try:
             description = self.fetch_job_description(job_id)
-        except Exception as e :
+        except:
             description = None
 
         job_post = JobPost(
@@ -145,6 +143,8 @@ class GlassdoorScraper(Scraper):
         all_jobs: list[JobPost] = []
         cursor = None
         max_pages = 30
+        self.session = create_session(self.proxy, is_tls=False, has_retry=True)
+        self.session.get(self.url)
 
         try:
             for page in range(
@@ -201,8 +201,7 @@ class GlassdoorScraper(Scraper):
             return None
         data = response.json()[0]
         desc = data['data']['jobview']['job']['description']
-        soup = BeautifulSoup(desc, 'html.parser')
+        return desc
-        return modify_and_get_description(soup)
 
     @staticmethod
     def parse_compensation(data: dict) -> Optional[Compensation]:
@@ -267,7 +266,6 @@ class GlassdoorScraper(Scraper):
             filter_params.append({"filterKey": "fromAge", "values": str(fromage)})
         payload = {
             "operationName": "JobSearchResultsQuery",
-
             "variables": {
                 "excludeJobListingIds": [],
                 "filterParams": filter_params,
@@ -281,21 +279,177 @@ class GlassdoorScraper(Scraper):
                 "fromage": fromage,
                 "sort": "date"
             },
-            "query": "query JobSearchResultsQuery($excludeJobListingIds: [Long!], $keyword: String, $locationId: Int, $locationType: LocationTypeEnum, $numJobsToShow: Int!, $pageCursor: String, $pageNumber: Int, $filterParams: [FilterParams], $originalPageUrl: String, $seoFriendlyUrlInput: String, $parameterUrlInput: String, $seoUrl: Boolean) {\n  jobListings(\n    contextHolder: {searchParams: {excludeJobListingIds: $excludeJobListingIds, keyword: $keyword, locationId: $locationId, locationType: $locationType, numPerPage: $numJobsToShow, pageCursor: $pageCursor, pageNumber: $pageNumber, filterParams: $filterParams, originalPageUrl: $originalPageUrl, seoFriendlyUrlInput: $seoFriendlyUrlInput, parameterUrlInput: $parameterUrlInput, seoUrl: $seoUrl, searchType: SR}}\n  ) {\n    companyFilterOptions {\n      id\n      shortName\n      __typename\n    }\n    filterOptions\n    indeedCtk\n    jobListings {\n      ...JobView\n      __typename\n    }\n    jobListingSeoLinks {\n      linkItems {\n        position\n        url\n        __typename\n      }\n      __typename\n    }\n    jobSearchTrackingKey\n    jobsPageSeoData {\n      pageMetaDescription\n      pageTitle\n      __typename\n    }\n    paginationCursors {\n      cursor\n      pageNumber\n      __typename\n    }\n    indexablePageForSeo\n    searchResultsMetadata {\n      searchCriteria {\n        implicitLocation {\n          id\n          localizedDisplayName\n          type\n          __typename\n        }\n        keyword\n        location {\n          id\n          shortName\n          localizedShortName\n          localizedDisplayName\n          type\n          __typename\n        }\n        __typename\n      }\n      footerVO {\n        countryMenu {\n          childNavigationLinks {\n            id\n            link\n            textKey\n            __typename\n          }\n          __typename\n        }\n        __typename\n      }\n      helpCenterDomain\n      helpCenterLocale\n      jobAlert {\n        jobAlertExists\n        __typename\n      }\n      jobSerpFaq {\n        questions {\n          answer\n          question\n          __typename\n        }\n        __typename\n      }\n      jobSerpJobOutlook {\n        occupation\n        paragraph\n        __typename\n      }\n      showMachineReadableJobs\n      __typename\n    }\n    serpSeoLinksVO {\n      relatedJobTitlesResults\n      searchedJobTitle\n      searchedKeyword\n      searchedLocationIdAsString\n      searchedLocationSeoName\n      searchedLocationType\n      topCityIdsToNameResults {\n        key\n        value\n        __typename\n      }\n      topEmployerIdsToNameResults {\n        key\n        value\n        __typename\n      }\n      topEmployerNameResults\n      topOccupationResults\n      __typename\n    }\n    totalJobsCount\n    __typename\n  }\n}\n\nfragment JobView on JobListingSearchResult {\n  jobview {\n    header {\n      adOrderId\n      advertiserType\n      adOrderSponsorshipLevel\n      ageInDays\n      divisionEmployerName\n      easyApply\n      employer {\n        id\n        name\n        shortName\n        __typename\n      }\n      employerNameFromSearch\n      goc\n      gocConfidence\n      gocId\n      jobCountryId\n      jobLink\n      jobResultTrackingKey\n      jobTitleText\n      locationName\n      locationType\n      locId\n      needsCommission\n      payCurrency\n      payPeriod\n      payPeriodAdjustedPay {\n        p10\n        p50\n        p90\n        __typename\n      }\n      rating\n      salarySource\n      savedJobId\n      sponsored\n      __typename\n    }\n    job {\n      descriptionFragments\n      importConfigId\n      jobTitleId\n      jobTitleText\n      listingId\n      __typename\n    }\n    jobListingAdminDetails {\n      cpcVal\n      importConfigId\n      jobListingId\n      jobSourceId\n      userEligibleForAdminJobDetails\n      __typename\n    }\n    overview {\n      shortName\n      squareLogoUrl\n      __typename\n    }\n    __typename\n  }\n  __typename\n}\n",
+            "query": """
+            query JobSearchResultsQuery(
+                $excludeJobListingIds: [Long!], 
+                $keyword: String, 
+                $locationId: Int, 
+                $locationType: LocationTypeEnum, 
+                $numJobsToShow: Int!, 
+                $pageCursor: String, 
+                $pageNumber: Int, 
+                $filterParams: [FilterParams], 
+                $originalPageUrl: String, 
+                $seoFriendlyUrlInput: String, 
+                $parameterUrlInput: String, 
+                $seoUrl: Boolean
+            ) {
+                jobListings(
+                    contextHolder: {
+                        searchParams: {
+                            excludeJobListingIds: $excludeJobListingIds, 
+                            keyword: $keyword, 
+                            locationId: $locationId, 
+                            locationType: $locationType, 
+                            numPerPage: $numJobsToShow, 
+                            pageCursor: $pageCursor, 
+                            pageNumber: $pageNumber, 
+                            filterParams: $filterParams, 
+                            originalPageUrl: $originalPageUrl, 
+                            seoFriendlyUrlInput: $seoFriendlyUrlInput, 
+                            parameterUrlInput: $parameterUrlInput, 
+                            seoUrl: $seoUrl, 
+                            searchType: SR
+                        }
+                    }
+                ) {
+                    companyFilterOptions {
+                        id
+                        shortName
+                        __typename
+                    }
+                    filterOptions
+                    indeedCtk
+                    jobListings {
+                        ...JobView
+                        __typename
+                    }
+                    jobListingSeoLinks {
+                        linkItems {
+                            position
+                            url
+                            __typename
+                        }
+                        __typename
+                    }
+                    jobSearchTrackingKey
+                    jobsPageSeoData {
+                        pageMetaDescription
+                        pageTitle
+                        __typename
+                    }
+                    paginationCursors {
+                        cursor
+                        pageNumber
+                        __typename
+                    }
+                    indexablePageForSeo
+                    searchResultsMetadata {
+                        searchCriteria {
+                            implicitLocation {
+                                id
+                                localizedDisplayName
+                                type
+                                __typename
+                            }
+                            keyword
+                            location {
+                                id
+                                shortName
+                                localizedShortName
+                                localizedDisplayName
+                                type
+                                __typename
+                            }
+                            __typename
+                        }
+                        helpCenterDomain
+                        helpCenterLocale
+                        jobSerpJobOutlook {
+                            occupation
+                            paragraph
+                            __typename
+                        }
+                        showMachineReadableJobs
+                        __typename
+                    }
+                    totalJobsCount
+                    __typename
+                }
             }
 
-        job_type_filters = {
+            fragment JobView on JobListingSearchResult {
-            JobType.FULL_TIME: "fulltime",
+                jobview {
-            JobType.PART_TIME: "parttime",
+                    header {
-            JobType.CONTRACT: "contract",
+                        adOrderId
-            JobType.INTERNSHIP: "internship",
+                        advertiserType
-            JobType.TEMPORARY: "temporary",
+                        adOrderSponsorshipLevel
+                        ageInDays
+                        divisionEmployerName
+                        easyApply
+                        employer {
+                            id
+                            name
+                            shortName
+                            __typename
+                        }
+                        employerNameFromSearch
+                        goc
+                        gocConfidence
+                        gocId
+                        jobCountryId
+                        jobLink
+                        jobResultTrackingKey
+                        jobTitleText
+                        locationName
+                        locationType
+                        locId
+                        needsCommission
+                        payCurrency
+                        payPeriod
+                        payPeriodAdjustedPay {
+                            p10
+                            p50
+                            p90
+                            __typename
+                        }
+                        rating
+                        salarySource
+                        savedJobId
+                        sponsored
+                        __typename
+                    }
+                    job {
+                        description
+                        importConfigId
+                        jobTitleId
+                        jobTitleText
+                        listingId
+                        __typename
+                    }
+                    jobListingAdminDetails {
+                        cpcVal
+                        importConfigId
+                        jobListingId
+                        jobSourceId
+                        userEligibleForAdminJobDetails
+                        __typename
+                    }
+                    overview {
+                        shortName
+                        squareLogoUrl
+                        __typename
+                    }
+                    __typename
+                }
+                __typename
+            }
+            """
         }
 
-        if scraper_input.job_type in job_type_filters:
+        if scraper_input.job_type:
-            filter_value = job_type_filters[scraper_input.job_type]
             payload["variables"]["filterParams"].append(
-                {"filterKey": "jobType", "values": filter_value}
+                {"filterKey": "jobType", "values": scraper_input.job_type.value[0]}
             )
         return json.dumps([payload])
 
@@ -331,7 +485,6 @@ class GlassdoorScraper(Scraper):
             "apollographql-client-name": "job-search-next",
             "apollographql-client-version": "4.65.5",
             "content-type": "application/json",
-            "cookie": 'gdId=91e2dfc4-c8b5-4fa7-83d0-11512b80262c; G_ENABLED_IDPS=google; trs=https%3A%2F%2Fwww.redhat.com%2F:referral:referral:2023-07-05+09%3A50%3A14.862:undefined:undefined; g_state={"i_p":1688587331651,"i_l":1}; _cfuvid=.7llazxhYFZWi6EISSPdVjtqF0NMVwzxr_E.cB1jgLs-1697828392979-0-604800000; GSESSIONID=undefined; JSESSIONID=F03DD1B5EE02DB6D842FE42B142F88F3; cass=1; jobsClicked=true; indeedCtk=1hd77b301k79i801; asst=1697829114.2; G_AUTHUSER_H=0; uc=8013A8318C98C517FE6DD0024636DFDEF978FC33266D93A2FAFEF364EACA608949D8B8FA2DC243D62DE271D733EB189D809ABE5B08D7B1AE865D217BD4EEBB97C282F5DA5FEFE79C937E3F6110B2A3A0ADBBA3B4B6DF5A996FEE00516100A65FCB11DA26817BE8D1C1BF6CFE36B5B68A3FDC2CFEC83AB797F7841FBB157C202332FC7E077B56BD39B167BDF3D9866E3B; AWSALB=zxc/Yk1nbWXXT6HjNyn3H4h4950ckVsFV/zOrq5LSoChYLE1qV+hDI8Axi3fUa9rlskndcO0M+Fw+ZnJ+AQ2afBFpyOd1acouLMYgkbEpqpQaWhY6/Gv4QH1zBcJ; AWSALBCORS=zxc/Yk1nbWXXT6HjNyn3H4h4950ckVsFV/zOrq5LSoChYLE1qV+hDI8Axi3fUa9rlskndcO0M+Fw+ZnJ+AQ2afBFpyOd1acouLMYgkbEpqpQaWhY6/Gv4QH1zBcJ; gdsid=1697828393025:1697830776351:668396EDB9E6A832022D34414128093D; at=HkH8Hnqi9uaMC7eu0okqyIwqp07ht9hBvE1_St7E_hRqPvkO9pUeJ1Jcpds4F3g6LL5ADaCNlxrPn0o6DumGMfog8qI1-zxaV_jpiFs3pugntw6WpVyYWdfioIZ1IDKupyteeLQEM1AO4zhGjY_rPZynpsiZBPO_B1au94sKv64rv23yvP56OiWKKfI-8_9hhLACEwWvM-Az7X-4aE2QdFt93VJbXbbGVf07bdDZfimsIkTtgJCLSRhU1V0kEM1Efyu66vo3m77gFFaMW7lxyYnb36I5PdDtEXBm3aL-zR7-qa5ywd94ISEivgqQOA4FPItNhqIlX4XrfD1lxVz6rfPaoTIDi4DI6UMCUjwyPsuv8mn0rYqDfRnmJpZ97fJ5AnhrknAd_6ZWN5v1OrxJczHzcXd8LO820QPoqxzzG13bmSTXLwGSxMUCtSrVsq05hicimQ3jpRt0c1dA4OkTNqF7_770B9JfcHcM8cr8-C4IL56dnOjr9KBGfN1Q2IvZM2cOBRbV7okiNOzKVZ3qJ24AE34WA2F3U6Whiu6H8nIuGG5hSNkVygY6CtglNZfFF9p8pJAZm79PngrrBv-CXFBZmhYLFo46lmFetDkiJ6mirtez4tKpzTIYjIp4_JAkiZFwbLJ2QGH4mK8kyyW0lZiX1DTuQec50N_5wvRo0Gt7nlKxzLsApMnaNhuQeH5ygh_pa381ORo9mQGi0EYF9zk00pa2--z4PtjfQ8KFq36GgpxKy5-o4qgqygZj8F01L8r-FiX2G4C7PREMIpAyHX2A4-_JxA1IS2j12EyqKTLqE9VcP06qm2Z-YuIW3ctmpMxy5G9_KiEiGv17weizhSFnl6SbpAEY-2VSmQ5V6jm3hoMp2jemkuGCRkZeFstLDEPxlzFN7WM; __cf_bm=zGaVjIJw4irf40_7UVw54B6Ohm271RUX4Tc8KVScrbs-1697830777-0-AYv2GnKTnnCU+cY9xHbJunO0DwlLDO6SIBnC/s/qldpKsGK0rRAjD6y8lbyATT/KlS7g29OZaN4fbd0lrJg0KmWbIybZIzfWVLHSYePVuOhu; asst=1697829114.2; at=dFhXf64wsf2TlnWy41xLs7skJkuxgKToEGcjGtDfUvW4oEAJ4tTIR5dKQ8wbwT75aIaGgdCfvcb-da7vwrCGWscCncmfLFQpJ9l-LLwoRfk-pMsxHhd77wvf-W7I0HSm7-Q5lQJqI9WyNGRxOa-RpzBTf4L8_Et4-3FzjPaAoYY5pY1FhuwXbN5asGOAMW-p8cjpbfn3PumlIYuckguWnjrcY2F31YJ_1noeoHM9tCGpymANbqGXRkG6aXY7yCfVXtdgZU1K5SMeaSPZIuF_iLUxjc_corzpNiH6qq7BIAmh-e5Aa-g7cwpZcln1fmwTVw4uTMZf1eLIMTa9WzgqZNkvG-sGaq_XxKA_Wai6xTTkOHfRgm4632Ba2963wdJvkGmUUa3tb_L4_wTgk3eFnHp5JhghLfT2Pe3KidP-yX__vx8JOsqe3fndCkKXgVz7xQKe1Dur-sMNlGwi4LXfguTT2YUI8C5Miq3pj2IHc7dC97eyyAiAM4HvyGWfaXWZcei6oIGrOwMvYgy0AcwFry6SIP2SxLT5TrxinRRuem1r1IcOTJsMJyUPp1QsZ7bOyq9G_0060B4CPyovw5523hEuqLTM-R5e5yavY6C_1DHUyE15C3mrh7kdvmlGZeflnHqkFTEKwwOftm-Mv-CKD5Db9ABFGNxKB2FH7nDH67hfOvm4tGNMzceBPKYJ3wciTt9jK3wy39_7cOYVywfrZ-oLhw_XtsbGSSeGn3HytrfgSADAh2sT0Gg6eCC9Xy1vh-Za337SVLUDXZ73W2xJxxUHBkFzZs8L_Xndo5DsbpWhVs9IYUGyraJdqB3SLgDbAppIBCJl4fx6_DG8-xOQPBvuFMlTROe1JVdHOzXI1GElwFDTuH1pjkg4I2G0NhAbE06Y-1illQE; gdsid=1697828393025:1697831731408:99C30D94108AC3030D61C736DDCDF11C',
             "gd-csrf-token": "Ft6oHEWlRZrxDww95Cpazw:0pGUrkb2y3TyOpAIqF2vbPmUXoXVkD3oEGDVkvfeCerceQ5-n8mBg3BovySUIjmCPHCaW0H2nQVdqzbtsYqf4Q:wcqRqeegRUa9MVLJGyujVXB7vWFPjdaS1CtrrzJq-ok",
             "origin": "https://www.glassdoor.com",
             "referer": "https://www.glassdoor.com/",

src/jobspy/scrapers/indeed/__init__.py

@@ -11,7 +11,6 @@ import requests
 from typing import Any
 from datetime import datetime
 
-import urllib.parse
 from bs4 import BeautifulSoup
 from bs4.element import Tag
 from concurrent.futures import ThreadPoolExecutor, Future
@@ -22,7 +21,7 @@ from ..utils import (
     extract_emails_from_text,
     create_session,
     get_enum_from_job_type,
-    modify_and_get_description
+    logger
 )
 from ...jobs import (
     JobPost,
@@ -50,13 +49,14 @@ class IndeedScraper(Scraper):
 
     def scrape_page(
         self, scraper_input: ScraperInput, page: int
-    ) -> tuple[list[JobPost], int]:
+    ) -> list[JobPost]:
         """
         Scrapes a page of Indeed for jobs with scraper_input criteria
         :param scraper_input:
         :param page:
         :return: jobs found on page, total number of jobs found for search
         """
+        job_list = []
         self.country = scraper_input.country
         domain = self.country.indeed_domain_value
         self.url = f"https://{domain}.indeed.com"
@@ -76,14 +76,14 @@ class IndeedScraper(Scraper):
                 )
         except Exception as e:
             if "Proxy responded with" in str(e):
-                raise IndeedException("bad proxy")
+                logger.error(f'Indeed: Bad proxy')
-            raise IndeedException(str(e))
+            else:
+                logger.error(f'Indeed: {str(e)}')
+            return job_list
 
         soup = BeautifulSoup(response.content, "html.parser")
-        job_list = []
-        total_num_jobs = IndeedScraper.total_jobs(soup)
         if "did not match any jobs" in response.text:
-            return job_list, total_num_jobs
+            return job_list
 
         jobs = IndeedScraper.parse_jobs(
             soup
@@ -145,7 +145,7 @@ class IndeedScraper(Scraper):
 
         job_list = [result.result() for result in job_results if result.result()]
 
-        return job_list, total_num_jobs
+        return job_list
 
     def scrape(self, scraper_input: ScraperInput) -> JobResponse:
         """
@@ -153,7 +153,7 @@ class IndeedScraper(Scraper):
         :param scraper_input:
         :return: job_response
         """
-        job_list, total_results = self.scrape_page(scraper_input, 0)
+        job_list = self.scrape_page(scraper_input, 0)
         pages_processed = 1
 
         while len(self.seen_urls) < scraper_input.results_wanted:
@@ -167,7 +167,7 @@ class IndeedScraper(Scraper):
                 ]
 
                 for future in futures:
-                    jobs, _ = future.result()
+                    jobs = future.result()
                     if jobs:
                         job_list += jobs
                         new_jobs = True
@@ -182,55 +182,7 @@ class IndeedScraper(Scraper):
         if len(self.seen_urls) > scraper_input.results_wanted:
             job_list = job_list[:scraper_input.results_wanted]
 
-        job_response = JobResponse(
+        return JobResponse(jobs=job_list)
-            jobs=job_list,
-            total_results=total_results,
-        )
-        return job_response
-
-    def get_description(self, job_page_url: str) -> str | None:
-        """
-        Retrieves job description by going to the job page url
-        :param job_page_url:
-        :return: description
-        """
-        parsed_url = urllib.parse.urlparse(job_page_url)
-        params = urllib.parse.parse_qs(parsed_url.query)
-        jk_value = params.get("jk", [None])[0]
-        formatted_url = f"{self.url}/m/viewjob?jk={jk_value}&spa=1"
-        session = create_session(self.proxy)
-
-        try:
-            response = session.get(
-                formatted_url,
-                headers=self.get_headers(),
-                allow_redirects=True,
-                timeout_seconds=5,
-            )
-        except Exception as e:
-            return None
-
-        if response.status_code not in range(200, 400):
-            return None
-
-        try:
-            soup = BeautifulSoup(response.text, 'html.parser')
-            script_tags = soup.find_all('script')
-
-            job_description = ''
-            for tag in script_tags:
-                if 'window._initialData' in tag.text:
-                    json_str = tag.text
-                    json_str = json_str.split('window._initialData=')[1]
-                    json_str = json_str.rsplit(';', 1)[0]
-                    data = json.loads(json_str)
-                    job_description = data["jobInfoWrapperModel"]["jobInfoModel"]["sanitizedJobDescription"]
-                    break
-        except (KeyError, TypeError, IndexError):
-            return None
-
-        soup = BeautifulSoup(job_description, "html.parser")
-        return modify_and_get_description(soup)
 
     @staticmethod
     def get_job_type(job: dict) -> list[JobType] | None:
@@ -330,24 +282,6 @@ class IndeedScraper(Scraper):
                 "Could not find any results for the search"
             )
 
-    @staticmethod
-    def total_jobs(soup: BeautifulSoup) -> int:
-        """
-        Parses the total jobs for that search from soup object
-        :param soup:
-        :return: total_num_jobs
-        """
-        script = soup.find("script", string=lambda t: t and "window._initialData" in t)
-
-        pattern = re.compile(r"window._initialData\s*=\s*({.*})\s*;", re.DOTALL)
-        match = pattern.search(script.string)
-        total_num_jobs = 0
-        if match:
-            json_str = match.group(1)
-            data = json.loads(json_str)
-            total_num_jobs = int(data["searchTitleBarModel"]["totalNumResults"])
-        return total_num_jobs
-
     @staticmethod
     def get_headers():
         return {
@@ -380,7 +314,7 @@ class IndeedScraper(Scraper):
         if scraper_input.is_remote:
             sc_values.append("attr(DSQF7)")
         if scraper_input.job_type:
-            sc_values.append("jt({})".format(scraper_input.job_type.value))
+            sc_values.append("jt({})".format(scraper_input.job_type.value[0]))
 
         if sc_values:
             params["sc"] = "0kf:" + "".join(sc_values) + ";"
@@ -406,7 +340,7 @@ class IndeedScraper(Scraper):
             taxonomy["label"] == "remote" and len(taxonomy["attributes"]) > 0
             for taxonomy in job.get("taxonomyAttributes", [])
         )
-        return is_remote_in_attributes or is_remote_in_description or is_remote_in_location
+        return is_remote_in_attributes or is_remote_in_description or is_remote_in_location or is_remote_in_taxonomy
 
     def get_job_details(self, job_keys: list[str]) -> dict:
         """

src/jobspy/scrapers/linkedin/__init__.py

@@ -31,8 +31,7 @@ from ..utils import (
     count_urgent_words,
     extract_emails_from_text,
     get_enum_from_job_type,
-    currency_parser,
+    currency_parser
-    modify_and_get_description
 )
 
 
@@ -236,10 +235,15 @@ class LinkedInScraper(Scraper):
         div_content = soup.find(
             "div", class_=lambda x: x and "show-more-less-html__markup" in x
         )
-
         description = None
-        if div_content:
+        if div_content is not None:
-            description = modify_and_get_description(div_content)
+            def remove_attributes(tag):
+                for attr in list(tag.attrs):
+                    del tag[attr]
+                return tag
+
+            div_content = remove_attributes(div_content)
+            description = div_content.prettify(formatter="html")
 
         def get_job_type(
             soup_job_type: BeautifulSoup,

src/jobspy/scrapers/utils.py

@@ -1,4 +1,5 @@
 import re
+import logging
 import numpy as np
 
 import tls_client
@@ -7,14 +8,14 @@ from requests.adapters import HTTPAdapter, Retry
 
 from ..jobs import JobType
 
-
+logger = logging.getLogger("JobSpy")
-def modify_and_get_description(soup):
+if not logger.handlers:
-    for li in soup.find_all('li'):
+    logger.setLevel(logging.ERROR)
-        li.string = "- " + li.get_text()
+    console_handler = logging.StreamHandler()
-
+    console_handler.setLevel(logging.ERROR)
-    description = soup.get_text(separator='\n').strip()
+    formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
-    description = re.sub(r'\n+', '\n', description)
+    console_handler.setFormatter(formatter)
-    return description
+    logger.addHandler(console_handler)
 
 
 def count_urgent_words(description: str) -> int:
@@ -79,6 +80,7 @@ def get_enum_from_job_type(job_type_str: str) -> JobType | None:
             res = job_type
     return res
 
+
 def currency_parser(cur_str):
     # Remove any non-numerical characters
     # except for ',' '.' or '-' (e.g. EUR)
@@ -94,3 +96,5 @@ def currency_parser(cur_str):
         num = float(cur_str)
 
     return np.round(num, 2)
+
+

src/jobspy/scrapers/ziprecruiter/__init__.py

@@ -9,13 +9,12 @@ import time
 from datetime import datetime, timezone
 from typing import Optional, Tuple, Any
 
-from bs4 import BeautifulSoup
 from concurrent.futures import ThreadPoolExecutor
 
 from .. import Scraper, ScraperInput, Site
 from ..exceptions import ZipRecruiterException
 from ...jobs import JobPost, Compensation, Location, JobResponse, JobType, Country
-from ..utils import count_urgent_words, extract_emails_from_text, create_session, modify_and_get_description
+from ..utils import count_urgent_words, extract_emails_from_text, create_session
 
 
 class ZipRecruiterScraper(Scraper):
@@ -107,9 +106,7 @@ class ZipRecruiterScraper(Scraper):
             return
         self.seen_urls.add(job_url)
 
-        job_description_html = job.get("job_description", "").strip()
+        description = job.get("job_description", "").strip()
-        description_soup = BeautifulSoup(job_description_html, "html.parser")
-        description = modify_and_get_description(description_soup)
 
         company = job.get("hiring_company", {}).get("name")
         country_value = "usa" if job.get("job_country") == "US" else "canada"
@@ -168,25 +165,16 @@ class ZipRecruiterScraper(Scraper):
         if scraper_input.hours_old:
             fromage = max(scraper_input.hours_old // 24, 1) if scraper_input.hours_old else None
             params['days'] = fromage
-        job_type_value = None
+        job_type_map = {
+            JobType.FULL_TIME: 'full_time',
+            JobType.PART_TIME: 'part_time'
+        }
         if scraper_input.job_type:
-            if scraper_input.job_type.value == "fulltime":
+            params['employment_type'] = job_type_map[scraper_input.job_type] if scraper_input.job_type in job_type_map else scraper_input.job_type.value[0]
-                job_type_value = "full_time"
-            elif scraper_input.job_type.value == "parttime":
-                job_type_value = "part_time"
-            else:
-                job_type_value = scraper_input.job_type.value
         if scraper_input.easy_apply:
             params['zipapply'] = 1
-
-        if job_type_value:
-            params[
-                "refine_by_employment"
-            ] = f"employment_type:employment_type:{job_type_value}"
-
         if scraper_input.is_remote:
-            params["refine_by_location_type"] = "only_remote"
+            params["remote"] = 1
-
         if scraper_input.distance:
             params["radius"] = scraper_input.distance