docs: readme

* fix(ziprecruiter): pagination

* chore: version

README.md

@@ -69,7 +69,8 @@ Optional
 ├── is_remote (bool)
 ├── full_description (bool): fetches full description for Indeed / LinkedIn (much slower)
 ├── results_wanted (int): number of job results to retrieve for each site specified in 'site_type'
-├── easy_apply (bool): filters for jobs that are hosted on LinkedIn
+├── easy_apply (bool): filters for jobs that are hosted on the job board site
+├── linkedin_company_ids (list[int): searches for linkedin jobs with specific company ids
 ├── country_indeed (enum): filters the country on Indeed (see below for correct spelling)
 ├── offset (num): starts the search from an offset (e.g. 25 will start the search from the 25th result)
 ```
@@ -80,6 +81,7 @@ Optional
 JobPost
 ├── title (str)
 ├── company (str)
+├── company_url (str)
 ├── job_url (str)
 ├── location (object)
 │   ├── country (str)
@@ -158,16 +160,11 @@ persist, [submit an issue](https://github.com/Bunsly/JobSpy/issues).
 **Q: Received a response code 429?**  
 **A:** This indicates that you have been blocked by the job board site for sending too many requests. All of the job board sites are aggressive with blocking. We recommend:
 
-- Waiting a few seconds between requests.
+- Waiting some time between scrapes (site-dependent).
 - Trying a VPN or proxy to change your IP address.
 
 ---
 
-**Q: Experiencing a "Segmentation fault: 11" on macOS Catalina?**  
-**A:** This is due to `tls_client` dependency not supporting your architecture. Solutions and workarounds include:
-
-- Upgrade to a newer version of MacOS
-- Reach out to the maintainers of [tls_client](https://github.com/bogdanfinn/tls-client) for fixes
 
 
   

poetry.lock

@@ -1053,16 +1053,6 @@ files = [
     {file = "MarkupSafe-2.1.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:5bbe06f8eeafd38e5d0a4894ffec89378b6c6a625ff57e3028921f8ff59318ac"},
     {file = "MarkupSafe-2.1.3-cp311-cp311-win32.whl", hash = "sha256:dd15ff04ffd7e05ffcb7fe79f1b98041b8ea30ae9234aed2a9168b5797c3effb"},
     {file = "MarkupSafe-2.1.3-cp311-cp311-win_amd64.whl", hash = "sha256:134da1eca9ec0ae528110ccc9e48041e0828d79f24121a1a146161103c76e686"},
-    {file = "MarkupSafe-2.1.3-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:f698de3fd0c4e6972b92290a45bd9b1536bffe8c6759c62471efaa8acb4c37bc"},
-    {file = "MarkupSafe-2.1.3-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:aa57bd9cf8ae831a362185ee444e15a93ecb2e344c8e52e4d721ea3ab6ef1823"},
-    {file = "MarkupSafe-2.1.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ffcc3f7c66b5f5b7931a5aa68fc9cecc51e685ef90282f4a82f0f5e9b704ad11"},
-    {file = "MarkupSafe-2.1.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:47d4f1c5f80fc62fdd7777d0d40a2e9dda0a05883ab11374334f6c4de38adffd"},
-    {file = "MarkupSafe-2.1.3-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1f67c7038d560d92149c060157d623c542173016c4babc0c1913cca0564b9939"},
-    {file = "MarkupSafe-2.1.3-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:9aad3c1755095ce347e26488214ef77e0485a3c34a50c5a5e2471dff60b9dd9c"},
-    {file = "MarkupSafe-2.1.3-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:14ff806850827afd6b07a5f32bd917fb7f45b046ba40c57abdb636674a8b559c"},
-    {file = "MarkupSafe-2.1.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8f9293864fe09b8149f0cc42ce56e3f0e54de883a9de90cd427f191c346eb2e1"},
-    {file = "MarkupSafe-2.1.3-cp312-cp312-win32.whl", hash = "sha256:715d3562f79d540f251b99ebd6d8baa547118974341db04f5ad06d5ea3eb8007"},
-    {file = "MarkupSafe-2.1.3-cp312-cp312-win_amd64.whl", hash = "sha256:1b8dd8c3fd14349433c79fa8abeb573a55fc0fdd769133baac1f5e07abf54aeb"},
     {file = "MarkupSafe-2.1.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:8e254ae696c88d98da6555f5ace2279cf7cd5b3f52be2b5cf97feafe883b58d2"},
     {file = "MarkupSafe-2.1.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cb0932dc158471523c9637e807d9bfb93e06a95cbf010f1a38b98623b929ef2b"},
     {file = "MarkupSafe-2.1.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9402b03f1a1b4dc4c19845e5c749e3ab82d5078d16a2a4c2cd2df62d57bb0707"},
@@ -2270,13 +2260,13 @@ test = ["flake8", "isort", "pytest"]
 
 [[package]]
 name = "tls-client"
-version = "0.2.1"
+version = "1.0"
 description = "Advanced Python HTTP Client."
 optional = false
 python-versions = "*"
 files = [
-    {file = "tls_client-0.2.1-py3-none-any.whl", hash = "sha256:124a710952b979d5e20b4e2b7879b7958d6e48a259d0f5b83101055eb173f0bd"},
-    {file = "tls_client-0.2.1.tar.gz", hash = "sha256:473fb4c671d9d4ca6b818548ab6e955640dd589767bfce520830c5618c2f2e2b"},
+    {file = "tls_client-1.0-py3-none-any.whl", hash = "sha256:f1183f5e18cb31914bd62d11b350a33ea0293ea80fb91d69a3072821dece3e66"},
+    {file = "tls_client-1.0.tar.gz", hash = "sha256:7f6de48ad4a0ef69b72682c76ce604155971e07b4bfb2148a36276194ae3e7a0"},
 ]
 
 [[package]]
@@ -2445,4 +2435,4 @@ files = [
 [metadata]
 lock-version = "2.0"
 python-versions = "^3.10"
-content-hash = "f966f3979873eec2c3b13460067f5aa414c69aa8ab5cd3239c1cfa564fcb5deb"
+content-hash = "404a77d78066cbb2ef71015562baf44aa11d12aac29a191c1ccc7758bfda598a"

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "python-jobspy"
-version = "1.1.35"
+version = "1.1.42"
 description = "Job scraper for LinkedIn, Indeed, Glassdoor & ZipRecruiter"
 authors = ["Zachary Hampton <zachary@bunsly.com>", "Cullen Watson <cullen@bunsly.com>"]
 homepage = "https://github.com/Bunsly/JobSpy"
@@ -13,7 +13,7 @@ packages = [
 [tool.poetry.dependencies]
 python = "^3.10"
 requests = "^2.31.0"
-tls-client = "^0.2.1"
+tls-client = "*"
 beautifulsoup4 = "^4.12.2"
 pandas = "^2.1.0"
 NUMPY = "1.24.2"

src/jobspy/__init__.py

@@ -1,7 +1,7 @@
 import pandas as pd
+from typing import Tuple
 import concurrent.futures
 from concurrent.futures import ThreadPoolExecutor
-from typing import Tuple, Optional
 
 from .jobs import JobType, Location
 from .scrapers.indeed import IndeedScraper
@@ -29,19 +29,20 @@ def _map_str_to_site(site_name: str) -> Site:
 
 
 def scrape_jobs(
-    site_name: str | list[str] | Site | list[Site],
-    search_term: str,
-    location: str = "",
-    distance: int = None,
+    site_name: str | list[str] | Site | list[Site] | None = None,
+    search_term: str | None = None,
+    location: str | None = None,
+    distance: int | None = None,
     is_remote: bool = False,
-    job_type: str = None,
-    easy_apply: bool = False,  # linkedin
+    job_type: str | None = None,
+    easy_apply: bool | None = None,
     results_wanted: int = 15,
     country_indeed: str = "usa",
     hyperlinks: bool = False,
-    proxy: Optional[str] = None,
-    full_description: Optional[bool] = False,
-    offset: Optional[int] = 0,
+    proxy: str | None = None,
+    full_description: bool | None = False,
+    linkedin_company_ids: list[int] | None = None,
+    offset: int | None = 0,
 ) -> pd.DataFrame:
     """
     Simultaneously scrapes job data from multiple job sites.
@@ -56,18 +57,23 @@ def scrape_jobs(
 
     job_type = get_enum_from_value(job_type) if job_type else None
 
-    if type(site_name) == str:
-        site_type = [_map_str_to_site(site_name)]
-    else:  #: if type(site_name) == list
-        site_type = [
-            _map_str_to_site(site) if type(site) == str else site_name
-            for site in site_name
-        ]
+    def get_site_type():
+        site_types = list(Site)
+        if isinstance(site_name, str):
+            site_types = [_map_str_to_site(site_name)]
+        elif isinstance(site_name, Site):
+            site_types = [site_name]
+        elif isinstance(site_name, list):
+            site_types = [
+                _map_str_to_site(site) if isinstance(site, str) else site
+                for site in site_name
+            ]
+        return site_types
 
     country_enum = Country.from_string(country_indeed)
 
     scraper_input = ScraperInput(
-        site_type=site_type,
+        site_type=get_site_type(),
         country=country_enum,
         search_term=search_term,
         location=location,
@@ -77,6 +83,7 @@ def scrape_jobs(
         easy_apply=easy_apply,
         full_description=full_description,
         results_wanted=results_wanted,
+        linkedin_company_ids=linkedin_company_ids,
         offset=offset,
     )
 

src/jobspy/jobs/__init__.py

@@ -1,7 +1,7 @@
-from typing import Union, Optional
+from typing import Optional
 from datetime import date
 from enum import Enum
-from pydantic import BaseModel, validator
+from pydantic import BaseModel
 
 
 class JobType(Enum):

src/jobspy/scrapers/__init__.py

@@ -1,5 +1,4 @@
 from ..jobs import Enum, BaseModel, JobType, JobResponse, Country
-from typing import List, Optional, Any
 
 
 class Site(Enum):
@@ -10,23 +9,24 @@ class Site(Enum):
 
 
 class ScraperInput(BaseModel):
-    site_type: List[Site]
-    search_term: str
+    site_type: list[Site]
+    search_term: str | None = None
 
-    location: str = None
-    country: Optional[Country] = Country.USA
-    distance: Optional[int] = None
+    location: str | None = None
+    country: Country | None = Country.USA
+    distance: int | None = None
     is_remote: bool = False
-    job_type: Optional[JobType] = None
-    easy_apply: bool = None  # linkedin
+    job_type: JobType | None = None
+    easy_apply: bool | None = None
     full_description: bool = False
     offset: int = 0
+    linkedin_company_ids: list[int] | None = None
 
     results_wanted: int = 15
 
 
 class Scraper:
-    def __init__(self, site: Site, proxy: Optional[List[str]] = None):
+    def __init__(self, site: Site, proxy: list[str] | None = None):
         self.site = site
         self.proxy = (lambda p: {"http": p, "https": p} if p else None)(proxy)
 

src/jobspy/scrapers/glassdoor/__init__.py

@@ -14,7 +14,7 @@ from ..utils import count_urgent_words, extract_emails_from_text
 
 from .. import Scraper, ScraperInput, Site
 from ..exceptions import GlassdoorException
-from ..utils import create_session
+from ..utils import create_session, modify_and_get_description
 from ...jobs import (
     JobPost,
     Compensation,
@@ -88,13 +88,14 @@ class GlassdoorScraper(Scraper):
     def process_job(self, job_data):
         """Processes a single job and fetches its description."""
         job_id = job_data["jobview"]["job"]["listingId"]
-        job_url = f'{self.url}/job-listing/?jl={job_id}'
+        job_url = f'{self.url}job-listing/j?jl={job_id}'
         if job_url in self.seen_urls:
             return None
         self.seen_urls.add(job_url)
         job = job_data["jobview"]
         title = job["job"]["jobTitleText"]
         company_name = job["header"]["employerNameFromSearch"]
+        company_id = job_data['jobview']['header']['employer']['id']
         location_name = job["header"].get("locationName", "")
         location_type = job["header"].get("locationType", "")
         age_in_days = job["header"].get("ageInDays")
@@ -115,6 +116,7 @@ class GlassdoorScraper(Scraper):
 
         job_post = JobPost(
             title=title,
+            company_url=f"{self.url}Overview/W-EI_IE{company_id}.htm" if company_id else None,
             company_name=company_name,
             date_posted=date_posted,
             job_url=job_url,
@@ -200,9 +202,7 @@ class GlassdoorScraper(Scraper):
         data = response.json()[0]
         desc = data['data']['jobview']['job']['description']
         soup = BeautifulSoup(desc, 'html.parser')
-        description = soup.get_text(separator='\n')
-
-        return description
+        return modify_and_get_description(soup)
 
     @staticmethod
     def parse_compensation(data: dict) -> Optional[Compensation]:
@@ -246,6 +246,8 @@ class GlassdoorScraper(Scraper):
             location_type = "CITY"
         elif location_type == "S":
             location_type = "STATE"
+        elif location_type == 'N':
+            location_type = "COUNTRY"
         return int(items[0]["locationId"]), location_type
 
     @staticmethod
@@ -260,7 +262,7 @@ class GlassdoorScraper(Scraper):
             "operationName": "JobSearchResultsQuery",
             "variables": {
                 "excludeJobListingIds": [],
-                "filterParams": [],
+                "filterParams": [{"filterKey": "applicationType", "values": "1"}] if scraper_input.easy_apply else [],
                 "keyword": scraper_input.search_term,
                 "numJobsToShow": 30,
                 "locationType": location_type,
@@ -292,12 +294,11 @@ class GlassdoorScraper(Scraper):
         for job_type in JobType:
             if job_type_str in job_type.value:
                 return [job_type]
-        return None
 
     @staticmethod
-    def parse_location(location_name: str) -> Location:
+    def parse_location(location_name: str) -> Location | None:
         if not location_name or location_name == "Remote":
-            return None
+            return
         city, _, state = location_name.partition(", ")
         return Location(city=city, state=state)
 
@@ -306,7 +307,6 @@ class GlassdoorScraper(Scraper):
         for cursor_data in pagination_cursors:
             if cursor_data["pageNumber"] == page_num:
                 return cursor_data["cursor"]
-        return None
 
     @staticmethod
     def headers() -> dict:

src/jobspy/scrapers/indeed/__init__.py

@@ -8,6 +8,7 @@ import re
 import math
 import io
 import json
+from typing import Any
 from datetime import datetime
 
 import urllib.parse
@@ -21,6 +22,7 @@ from ..utils import (
     extract_emails_from_text,
     create_session,
     get_enum_from_job_type,
+    modify_and_get_description
 )
 from ...jobs import (
     JobPost,
@@ -43,7 +45,7 @@ class IndeedScraper(Scraper):
         site = Site(Site.INDEED)
         super().__init__(site, proxy=proxy)
 
-        self.jobs_per_page = 15
+        self.jobs_per_page = 25
         self.seen_urls = set()
 
     def scrape_page(
@@ -59,30 +61,12 @@ class IndeedScraper(Scraper):
         domain = self.country.indeed_domain_value
         self.url = f"https://{domain}.indeed.com"
 
-        params = {
-            "q": scraper_input.search_term,
-            "l": scraper_input.location,
-            "filter": 0,
-            "start": scraper_input.offset + page * 10,
-            "sort": "date"
-        }
-        if scraper_input.distance:
-            params["radius"] = scraper_input.distance
-
-        sc_values = []
-        if scraper_input.is_remote:
-            sc_values.append("attr(DSQF7)")
-        if scraper_input.job_type:
-            sc_values.append("jt({})".format(scraper_input.job_type.value))
-
-        if sc_values:
-            params["sc"] = "0kf:" + "".join(sc_values) + ";"
         try:
             session = create_session(self.proxy)
             response = session.get(
-                f"{self.url}/jobs",
+                f"{self.url}/m/jobs",
                 headers=self.get_headers(),
-                params=params,
+                params=self.add_params(scraper_input, page),
                 allow_redirects=True,
                 timeout_seconds=10,
             )
@@ -111,8 +95,8 @@ class IndeedScraper(Scraper):
         ):
             raise IndeedException("No jobs found.")
 
-        def process_job(job) -> JobPost | None:
-            job_url = f'{self.url}/jobs/viewjob?jk={job["jobkey"]}'
+        def process_job(job: dict) -> JobPost | None:
+            job_url = f'{self.url}/m/jobs/viewjob?jk={job["jobkey"]}'
             job_url_client = f'{self.url}/viewjob?jk={job["jobkey"]}'
             if job_url in self.seen_urls:
                 return None
@@ -170,8 +154,9 @@ class IndeedScraper(Scraper):
             )
             return job_post
 
+        workers = 10 if scraper_input.full_description else 10  # possibly lessen 10 when fetching desc based on feedback
         jobs = jobs["metaData"]["mosaicProviderJobCardsModel"]["results"]
-        with ThreadPoolExecutor(max_workers=1) as executor:
+        with ThreadPoolExecutor(max_workers=workers) as executor:
             job_results: list[Future] = [
                 executor.submit(process_job, job) for job in jobs
             ]
@@ -193,7 +178,7 @@ class IndeedScraper(Scraper):
         #: get first page to initialize session
         job_list, total_results = self.scrape_page(scraper_input, 0)
 
-        with ThreadPoolExecutor(max_workers=1) as executor:
+        with ThreadPoolExecutor(max_workers=10) as executor:
             futures: list[Future] = [
                 executor.submit(self.scrape_page, scraper_input, page)
                 for page in range(1, pages_to_process + 1)
@@ -222,7 +207,7 @@ class IndeedScraper(Scraper):
         parsed_url = urllib.parse.urlparse(job_page_url)
         params = urllib.parse.parse_qs(parsed_url.query)
         jk_value = params.get("jk", [None])[0]
-        formatted_url = f"{self.url}/viewjob?jk={jk_value}&spa=1"
+        formatted_url = f"{self.url}/m/viewjob?jk={jk_value}&spa=1"
         session = create_session(self.proxy)
 
         try:
@@ -239,17 +224,23 @@ class IndeedScraper(Scraper):
             return None
 
         try:
-            data = json.loads(response.text)
-            job_description = data["body"]["jobInfoWrapperModel"]["jobInfoModel"][
-                "sanitizedJobDescription"
-            ]
+            soup = BeautifulSoup(response.text, 'html.parser')
+            script_tags = soup.find_all('script')
+
+            job_description = ''
+            for tag in script_tags:
+                if 'window._initialData' in tag.text:
+                    json_str = tag.text
+                    json_str = json_str.split('window._initialData=')[1]
+                    json_str = json_str.rsplit(';', 1)[0]
+                    data = json.loads(json_str)
+                    job_description = data["jobInfoWrapperModel"]["jobInfoModel"]["sanitizedJobDescription"]
+                    break
         except (KeyError, TypeError, IndexError):
             return None
 
         soup = BeautifulSoup(job_description, "html.parser")
-        text_content = "\n".join(soup.stripped_strings)
-
-        return text_content
+        return modify_and_get_description(soup)
 
     @staticmethod
     def get_job_type(job: dict) -> list[JobType] | None:
@@ -332,17 +323,14 @@ class IndeedScraper(Scraper):
     @staticmethod
     def get_headers():
         return {
-            "authority": "www.indeed.com",
-            "accept": "*/*",
-            "accept-language": "en-US,en;q=0.9",
-            "referer": "https://www.indeed.com/viewjob?jk=fe6182337d72c7b1&tk=1hcbfcmd0k62t802&from=serp&vjs=3&advn=8132938064490989&adid=408692607&ad=-6NYlbfkN0A3Osc99MJFDKjquSk4WOGT28ALb_ad4QMtrHreCb9ICg6MiSVy9oDAp3evvOrI7Q-O9qOtQTg1EPbthP9xWtBN2cOuVeHQijxHjHpJC65TjDtftH3AXeINjBvAyDrE8DrRaAXl8LD3Fs1e_xuDHQIssdZ2Mlzcav8m5jHrA0fA64ZaqJV77myldaNlM7-qyQpy4AsJQfvg9iR2MY7qeC5_FnjIgjKIy_lNi9OPMOjGRWXA94CuvC7zC6WeiJmBQCHISl8IOBxf7EdJZlYdtzgae3593TFxbkd6LUwbijAfjax39aAuuCXy3s9C4YgcEP3TwEFGQoTpYu9Pmle-Ae1tHGPgsjxwXkgMm7Cz5mBBdJioglRCj9pssn-1u1blHZM4uL1nK9p1Y6HoFgPUU9xvKQTHjKGdH8d4y4ETyCMoNF4hAIyUaysCKdJKitC8PXoYaWhDqFtSMR4Jys8UPqUV&xkcb=SoDD-_M3JLQfWnQTDh0LbzkdCdPP&xpse=SoBa6_I3JLW9FlWZlB0PbzkdCdPP&sjdu=i6xVERweJM_pVUvgf-MzuaunBTY7G71J5eEX6t4DrDs5EMPQdODrX7Nn-WIPMezoqr5wA_l7Of-3CtoiUawcHw",
-            "sec-ch-ua": '"Google Chrome";v="119", "Chromium";v="119", "Not?A_Brand";v="24"',
-            "sec-ch-ua-mobile": "?0",
-            "sec-ch-ua-platform": '"Windows"',
-            "sec-fetch-dest": "empty",
-            "sec-fetch-mode": "cors",
-            "sec-fetch-site": "same-origin",
-            "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36",
+          'Host': 'www.indeed.com',
+          'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
+          'sec-fetch-site': 'same-origin',
+          'sec-fetch-dest': 'document',
+          'accept-language': 'en-US,en;q=0.9',
+          'sec-fetch-mode': 'navigate',
+          'user-agent': 'Mozilla/5.0 (iPhone; CPU iPhone OS 16_6_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148 Indeed App 192.0',
+          'referer': 'https://www.indeed.com/m/jobs?q=software%20intern&l=Dallas%2C%20TX&from=serpso&rq=1&rsIdx=3',
         }
 
     @staticmethod
@@ -355,3 +343,29 @@ class IndeedScraper(Scraper):
             if taxonomy["label"] == "remote" and len(taxonomy["attributes"]) > 0:
                 return True
         return False
+
+    @staticmethod
+    def add_params(scraper_input: ScraperInput, page: int) -> dict[str, str | Any]:
+        params = {
+            "q": scraper_input.search_term,
+            "l": scraper_input.location if scraper_input.location else scraper_input.country.value[0].split(',')[-1],
+            "filter": 0,
+            "start": scraper_input.offset + page * 10,
+            "sort": "date"
+        }
+        if scraper_input.distance:
+            params["radius"] = scraper_input.distance
+
+        sc_values = []
+        if scraper_input.is_remote:
+            sc_values.append("attr(DSQF7)")
+        if scraper_input.job_type:
+            sc_values.append("jt({})".format(scraper_input.job_type.value))
+
+        if sc_values:
+            params["sc"] = "0kf:" + "".join(sc_values) + ";"
+
+        if scraper_input.easy_apply:
+            params['iafilter'] = 1
+
+        return params

src/jobspy/scrapers/linkedin/__init__.py

@@ -4,23 +4,36 @@ jobspy.scrapers.linkedin
 
 This module contains routines to scrape LinkedIn.
 """
+import time
 import random
 from typing import Optional
 from datetime import datetime
 
 import requests
-import time
 from requests.exceptions import ProxyError
-from bs4 import BeautifulSoup
-from bs4.element import Tag
 from threading import Lock
+from bs4.element import Tag
+from bs4 import BeautifulSoup
 from urllib.parse import urlparse, urlunparse
 
 from .. import Scraper, ScraperInput, Site
 from ..exceptions import LinkedInException
 from ..utils import create_session
-from ...jobs import JobPost, Location, JobResponse, JobType, Country, Compensation
-from ..utils import count_urgent_words, extract_emails_from_text, get_enum_from_job_type, currency_parser
+from ...jobs import (
+    JobPost,
+    Location,
+    JobResponse,
+    JobType,
+    Country,
+    Compensation
+)
+from ..utils import (
+    count_urgent_words,
+    extract_emails_from_text,
+    get_enum_from_job_type,
+    currency_parser,
+    modify_and_get_description
+)
 
 
 class LinkedInScraper(Scraper):
@@ -57,7 +70,9 @@ class LinkedInScraper(Scraper):
 
             return mapping.get(job_type_enum, "")
 
-        while len(job_list) < scraper_input.results_wanted and page < 1000:
+        continue_search = lambda: len(job_list) < scraper_input.results_wanted and page < 1000
+
+        while continue_search():
             session = create_session(is_tls=False, has_retry=True, delay=5)
             params = {
                 "keywords": scraper_input.search_term,
@@ -70,6 +85,7 @@ class LinkedInScraper(Scraper):
                 "pageNum": 0,
                 "start": page + scraper_input.offset,
                 "f_AL": "true" if scraper_input.easy_apply else None,
+                "f_C": ','.join(map(str, scraper_input.linkedin_company_ids)) if scraper_input.linkedin_company_ids else None
             }
 
             params = {k: v for k, v in params.items() if v is not None}
@@ -117,8 +133,9 @@ class LinkedInScraper(Scraper):
                 except Exception as e:
                     raise LinkedInException("Exception occurred while processing jobs")
 
-            page += 25
-            time.sleep(random.uniform(LinkedInScraper.DELAY, LinkedInScraper.DELAY + 2))
+            if continue_search():
+                time.sleep(random.uniform(LinkedInScraper.DELAY, LinkedInScraper.DELAY + 2))
+                page += 25
 
         job_list = job_list[: scraper_input.results_wanted]
         return JobResponse(jobs=job_list)
@@ -213,7 +230,7 @@ class LinkedInScraper(Scraper):
 
         description = None
         if div_content:
-            description = "\n".join(line.strip() for line in div_content.get_text(separator="\n").splitlines() if line.strip())
+            description = modify_and_get_description(div_content)
 
         def get_job_type(
             soup_job_type: BeautifulSoup,

src/jobspy/scrapers/utils.py

@@ -8,6 +8,15 @@ from requests.adapters import HTTPAdapter, Retry
 from ..jobs import JobType
 
 
+def modify_and_get_description(soup):
+    for li in soup.find_all('li'):
+        li.string = "- " + li.get_text()
+
+    description = soup.get_text(separator='\n').strip()
+    description = re.sub(r'\n+', '\n', description)
+    return description
+
+
 def count_urgent_words(description: str) -> int:
     """
     Count the number of urgent words or phrases in a job description.

src/jobspy/scrapers/ziprecruiter/__init__.py

@@ -6,18 +6,16 @@ This module contains routines to scrape ZipRecruiter.
 """
 import math
 import time
-import re
-from datetime import datetime, date
+from datetime import datetime, timezone
 from typing import Optional, Tuple, Any
 
-import requests
 from bs4 import BeautifulSoup
 from concurrent.futures import ThreadPoolExecutor
 
 from .. import Scraper, ScraperInput, Site
 from ..exceptions import ZipRecruiterException
-from ..utils import count_urgent_words, extract_emails_from_text, create_session
 from ...jobs import JobPost, Compensation, Location, JobResponse, JobType, Country
+from ..utils import count_urgent_words, extract_emails_from_text, create_session, modify_and_get_description
 
 
 class ZipRecruiterScraper(Scraper):
@@ -33,6 +31,7 @@ class ZipRecruiterScraper(Scraper):
 
         self.jobs_per_page = 20
         self.seen_urls = set()
+        self.delay = 5
 
     def find_jobs_in_page(
         self, scraper_input: ScraperInput, continue_token: str | None = None
@@ -45,12 +44,12 @@ class ZipRecruiterScraper(Scraper):
         """
         params = self.add_params(scraper_input)
         if continue_token:
-            params["continue"] = continue_token
+            params["continue_from"] = continue_token
         try:
             response = self.session.get(
                 f"https://api.ziprecruiter.com/jobs-app/jobs",
                 headers=self.headers(),
-                params=self.add_params(scraper_input),
+                params=params
             )
             if response.status_code != 200:
                 raise ZipRecruiterException(
@@ -61,7 +60,6 @@ class ZipRecruiterScraper(Scraper):
                 raise ZipRecruiterException("bad proxy")
             raise ZipRecruiterException(str(e))
 
-        time.sleep(5)
         response_data = response.json()
         jobs_list = response_data.get("jobs", [])
         next_continue_token = response_data.get("continue", None)
@@ -69,7 +67,7 @@ class ZipRecruiterScraper(Scraper):
         with ThreadPoolExecutor(max_workers=self.jobs_per_page) as executor:
             job_results = [executor.submit(self.process_job, job) for job in jobs_list]
 
-        job_list = [result.result() for result in job_results if result.result()]
+        job_list = list(filter(None, (result.result() for result in job_results)))
         return job_list, next_continue_token
 
     def scrape(self, scraper_input: ScraperInput) -> JobResponse:
@@ -87,6 +85,9 @@ class ZipRecruiterScraper(Scraper):
             if len(job_list) >= scraper_input.results_wanted:
                 break
 
+            if page > 1:
+                time.sleep(self.delay)
+
             jobs_on_page, continue_token = self.find_jobs_in_page(
                 scraper_input, continue_token
             )
@@ -96,22 +97,21 @@ class ZipRecruiterScraper(Scraper):
             if not continue_token:
                 break
 
-        if len(job_list) > scraper_input.results_wanted:
-            job_list = job_list[: scraper_input.results_wanted]
+        return JobResponse(jobs=job_list[: scraper_input.results_wanted])
 
-        return JobResponse(jobs=job_list)
-
-    @staticmethod
-    def process_job(job: dict) -> JobPost:
+    def process_job(self, job: dict) -> JobPost | None:
         """Processes an individual job dict from the response"""
         title = job.get("name")
-        job_url = job.get("job_url")
+        job_url = f"https://www.ziprecruiter.com/jobs//j?lvk={job['listing_key']}"
+        if job_url in self.seen_urls:
+            return
+        self.seen_urls.add(job_url)
 
-        description = BeautifulSoup(
-            job.get("job_description", "").strip(), "html.parser"
-        ).get_text(separator="\n")
+        job_description_html = job.get("job_description", "").strip()
+        description_soup = BeautifulSoup(job_description_html, "html.parser")
+        description = modify_and_get_description(description_soup)
 
-        company = job["hiring_company"].get("name") if "hiring_company" in job else None
+        company = job.get("hiring_company", {}).get("name")
         country_value = "usa" if job.get("job_country") == "US" else "canada"
         country_enum = Country.from_string(country_value)
 
@@ -121,17 +121,7 @@ class ZipRecruiterScraper(Scraper):
         job_type = ZipRecruiterScraper.get_job_type_enum(
             job.get("employment_type", "").replace("_", "").lower()
         )
-
-        save_job_url = job.get("SaveJobURL", "")
-        posted_time_match = re.search(
-            r"posted_time=(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z)", save_job_url
-        )
-        if posted_time_match:
-            date_time_str = posted_time_match.group(1)
-            date_posted_obj = datetime.strptime(date_time_str, "%Y-%m-%dT%H:%M:%SZ")
-            date_posted = date_posted_obj.date()
-        else:
-            date_posted = date.today()
+        date_posted = datetime.fromisoformat(job['posted_time'].rstrip("Z")).date()
 
         return JobPost(
             title=title,
@@ -174,7 +164,6 @@ class ZipRecruiterScraper(Scraper):
         params = {
             "search": scraper_input.search_term,
             "location": scraper_input.location,
-            "form": "jobs-landing",
         }
         job_type_value = None
         if scraper_input.job_type:
@@ -184,6 +173,8 @@ class ZipRecruiterScraper(Scraper):
                 job_type_value = "part_time"
             else:
                 job_type_value = scraper_input.job_type.value
+        if scraper_input.easy_apply:
+            params['zipapply'] = 1
 
         if job_type_value:
             params[
@@ -196,6 +187,8 @@ class ZipRecruiterScraper(Scraper):
         if scraper_input.distance:
             params["radius"] = scraper_input.distance
 
+        params = {k: v for k, v in params.items() if v is not None}
+
         return params
 
     @staticmethod