update for artifact with run ID

git commit -m "Added requirements.txt"
git push origin main

.github/workflows/job_scraper_dynamic.yml

@@ -0,0 +1,49 @@
+name: JobSpy Scraper Dynamic Workflow
+
+on:
+  workflow_dispatch:
+    inputs:
+      user_email:
+        description: 'Email of user'
+        required: true
+        default: 'Branden@autoemployme.onmicrosoft.com'
+
+permissions:
+  contents: read
+  id-token: write
+
+jobs:
+  scrape_jobs:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout Repo
+        uses: actions/checkout@v3
+
+      - name: Set Up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.10'
+
+      - name: Install Dependencies
+        run: |
+          pip install --upgrade pip
+          pip install -r requirements.txt
+
+      - name: Sanitize Email + Create Run ID
+        id: vars
+        run: |
+          safe_email=$(echo "${{ github.event.inputs.user_email }}" | sed 's/@/_at_/g; s/\./_/g')
+          run_id=$(date +%s)
+          echo "safe_email=$safe_email" >> $GITHUB_OUTPUT
+          echo "run_id=$run_id" >> $GITHUB_OUTPUT
+
+      - name: Run Job Scraper
+        run: |
+          python job_scraper_dynamic.py "${{ github.event.inputs.user_email }}" "${{ steps.vars.outputs.run_id }}"
+
+      - name: Upload Output Artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: jobspy_output_${{ steps.vars.outputs.safe_email }}_${{ steps.vars.outputs.run_id }}
+          path: outputs/jobspy_output_${{ steps.vars.outputs.safe_email }}_${{ steps.vars.outputs.run_id }}.csv

.github/workflows/jobspy_scraper.yml

@@ -0,0 +1,48 @@
+name: JobSpy Scraper Workflow
+
+on:
+  workflow_dispatch:  # Allows manual trigger from GitHub or Power Automate
+  # Remove or comment out the schedule to prevent auto-runs
+  # schedule:
+  #   - cron: '0 */6 * * *'  # Runs every 6 hours (DISABLED)
+
+permissions:
+  actions: read
+  contents: read
+  id-token: write
+
+jobs:
+  scrape_jobs:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.10'
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r requirements.txt
+
+      - name: Run JobSpy Scraper
+        run: python job_scraper_exact_match.py
+
+      - name: Debug - Check if jobspy_output.csv exists
+        run: |
+          if [ ! -f jobspy_output.csv ]; then
+            echo "❌ ERROR: jobspy_output.csv not found!"
+            exit 1
+          else
+            echo "✅ jobspy_output.csv found, proceeding to upload..."
+          fi
+
+      - name: Upload JobSpy Output as Artifact
+        uses: actions/upload-artifact@v4  # Explicitly using latest version
+        with:
+          name: jobspy-results
+          path: jobspy_output.csv

.github/workflows/publish-to-pypi.yml

@@ -1,9 +1,13 @@
-name: Publish Python 🐍 distributions 📦 to PyPI
-on: push
+name: Publish JobSpy to PyPi
+on:
+  push:
+    branches:
+      - main
+  workflow_dispatch:
 
 jobs:
   build-n-publish:
-    name: Build and publish Python 🐍 distributions 📦 to PyPI
+    name: Build and publish JobSpy to PyPi
     runs-on: ubuntu-latest
 
     steps:
@@ -27,7 +31,7 @@ jobs:
           build
 
       - name: Publish distribution 📦 to PyPI
-        if: startsWith(github.ref, 'refs/tags')
+        if: startsWith(github.ref, 'refs/tags') || github.event_name == 'workflow_dispatch'
         uses: pypa/gh-action-pypi-publish@release/v1
         with:
           password: ${{ secrets.PYPI_API_TOKEN }}

.github/workflows/python-test.yml

@@ -1,22 +0,0 @@
-name: Python Tests
-
-on:
-  pull_request:
-    branches:
-      - main
-
-jobs:
-  test:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - name: Set up Python
-        uses: actions/setup-python@v2
-        with:
-          python-version: '3.8'
-      - name: Install dependencies
-        run: |
-          pip install poetry
-          poetry install
-      - name: Run tests
-        run: poetry run pytest src/tests/test_all.py

README.md

@@ -1,17 +1,12 @@
 <img src="https://github.com/cullenwatson/JobSpy/assets/78247585/ae185b7e-e444-4712-8bb9-fa97f53e896b" width="400">
 
-**JobSpy** is a simple, yet comprehensive, job scraping library.
-
-**Not technical?** Try out the web scraping tool on our site at [usejobspy.com](https://usejobspy.com).
-
-*Looking to build a data-focused software product?* **[Book a call](https://bunsly.com/)** *to
-work with us.*
+**JobSpy** is a job scraping library with the goal of aggregating all the jobs from popular job boards with one tool.
 
 ## Features
 
-- Scrapes job postings from **LinkedIn**, **Indeed**, **Glassdoor**, & **ZipRecruiter** simultaneously
-- Aggregates the job postings in a Pandas DataFrame
-- Proxies support
+- Scrapes job postings from **LinkedIn**, **Indeed**, **Glassdoor**, **Google**, **ZipRecruiter**, & **Bayt** concurrently
+- Aggregates the job postings in a dataframe
+- Proxies support to bypass blocking
 
 ![jobspy](https://github.com/cullenwatson/JobSpy/assets/78247585/ec7ef355-05f6-4fd3-8161-a817e31c5c57)
 
@@ -30,16 +25,16 @@ import csv
 from jobspy import scrape_jobs
 
 jobs = scrape_jobs(
-    site_name=["indeed", "linkedin", "zip_recruiter", "glassdoor"],
+    site_name=["indeed", "linkedin", "zip_recruiter", "glassdoor", "google", "bayt"],
     search_term="software engineer",
-    location="Dallas, TX",
+    google_search_term="software engineer jobs near San Francisco, CA since yesterday",
+    location="San Francisco, CA",
     results_wanted=20,
-    hours_old=72, # (only Linkedin/Indeed is hour specific, others round up to days old)
-    country_indeed='USA',  # only needed for indeed / glassdoor
+    hours_old=72,
+    country_indeed='USA',
     
-    # linkedin_fetch_description=True # get full description , direct job url , company industry and job level (seniority level) for linkedin (slower)
+    # linkedin_fetch_description=True # gets more info such as description, direct job url (slower)
     # proxies=["208.195.175.46:65095", "208.195.175.45:65095", "localhost"],
-    
 )
 print(f"Found {len(jobs)} jobs")
 print(jobs.head())
@@ -63,10 +58,13 @@ zip_recruiter Software Developer                 TEKsystems        Phoenix
 ```plaintext
 Optional
 ├── site_name (list|str): 
-|    linkedin, zip_recruiter, indeed, glassdoor 
-|    (default is all four)
+|    linkedin, zip_recruiter, indeed, glassdoor, google, bayt
+|    (default is all)
 │
 ├── search_term (str)
+|
+├── google_search_term (str)
+|     search term for google jobs. This is the only param for filtering google jobs.
 │
 ├── location (str)
 │
@@ -79,14 +77,14 @@ Optional
 ├── proxies (list): 
 |    in format ['user:pass@host:port', 'localhost']
 |    each job board scraper will round robin through the proxies
-│
+|
 ├── is_remote (bool)
 │
 ├── results_wanted (int): 
 |    number of job results to retrieve for each site specified in 'site_name'
 │
 ├── easy_apply (bool): 
-|    filters for jobs that are hosted on the job board site
+|    filters for jobs that are hosted on the job board site (LinkedIn easy apply filter no longer works)
 │
 ├── description_format (str): 
 |    markdown, html (Format type of the job descriptions. Default is markdown.)
@@ -113,6 +111,9 @@ Optional
 |
 ├── enforce_annual_salary (bool): 
 |    converts wages to annual salary
+|
+├── ca_cert (str)
+|    path to CA Certificate file for proxies
 ```
 
 ```
@@ -128,49 +129,6 @@ Optional
 |    - easy_apply
 ```
 
-
-### JobPost Schema
-
-```plaintext
-JobPost
-├── title
-├── company
-├── company_url
-├── job_url
-├── location
-│   ├── country
-│   ├── city
-│   ├── state
-├── description
-├── job_type: fulltime, parttime, internship, contract
-├── job_function
-│   ├── interval: yearly, monthly, weekly, daily, hourly
-│   ├── min_amount
-│   ├── max_amount
-│   ├── currency
-│   └── salary_source: direct_data, description (parsed from posting)
-├── date_posted
-├── emails
-└── is_remote
-
-Linkedin specific
-└── job_level
-
-Linkedin & Indeed specific
-└── company_industry
-
-Indeed specific
-├── company_country
-├── company_addresses
-├── company_employees_label
-├── company_revenue_label
-├── company_description
-├── ceo_name
-├── ceo_photo_url
-├── logo_photo_url
-└── banner_photo_url
-```
-
 ## Supported Countries for Job Searching
 
 ### **LinkedIn**
@@ -207,6 +165,11 @@ You can specify the following countries when searching on Indeed (use the exact
 | United Arab Emirates | UK*          | USA*       | Uruguay        |
 | Venezuela            | Vietnam*     |            |                |
 
+### **Bayt**
+
+Bayt only uses the search_term parameter currently and searches internationally
+
+
 
 ## Notes
 * Indeed is the best scraper currently with no rate limiting.  
@@ -217,7 +180,23 @@ You can specify the following countries when searching on Indeed (use the exact
 
 ---
 **Q: Why is Indeed giving unrelated roles?**  
-**A:** Indeed is searching each one of your terms e.g. software intern, it searches software OR intern. Try search_term='"software intern"' in quotes for stricter searching
+**A:** Indeed searches the description too.
+
+- use - to remove words
+- "" for exact match
+
+Example of a good Indeed query
+
+```py
+search_term='"engineering intern" software summer (java OR python OR c++) 2025 -tax -marketing'
+```
+
+This searches the description/title and must include software, summer, 2025, one of the languages, engineering intern exactly, no tax, no marketing.
+
+---
+
+**Q: No results when using "google"?**  
+**A:** You have to use super specific syntax. Search for google jobs on your browser and then whatever pops up in the google jobs search box after applying some filters is what you need to copy & paste into the google_search_term. 
 
 ---
 
@@ -229,8 +208,41 @@ You can specify the following countries when searching on Indeed (use the exact
 
 ---
 
-**Q: Encountering issues with your queries?**  
-**A:** Try reducing the number of `results_wanted` and/or broadening the filters. If problems
-persist, [submit an issue](https://github.com/Bunsly/JobSpy/issues).
+### JobPost Schema
 
----
+```plaintext
+JobPost
+├── title
+├── company
+├── company_url
+├── job_url
+├── location
+│   ├── country
+│   ├── city
+│   ├── state
+├── description
+├── job_type: fulltime, parttime, internship, contract
+├── job_function
+│   ├── interval: yearly, monthly, weekly, daily, hourly
+│   ├── min_amount
+│   ├── max_amount
+│   ├── currency
+│   └── salary_source: direct_data, description (parsed from posting)
+├── date_posted
+├── emails
+└── is_remote
+
+Linkedin specific
+└── job_level
+
+Linkedin & Indeed specific
+└── company_industry
+
+Indeed specific
+├── company_country
+├── company_addresses
+├── company_employees_label
+├── company_revenue_label
+├── company_description
+└── company_logo
+```

configs/config.json

@@ -0,0 +1,8 @@
+{
+    "search_terms": ["IT Support", "Help Desk"],
+    "results_wanted": 50,
+    "max_days_old": 7,
+    "target_state": "NY",
+    "user_email": "Branden@autoemployme.onmicrosoft.com"
+  }
+  

configs/config_Branden_at_autoemployme_onmicrosoft_com.json

@@ -0,0 +1,8 @@
+{
+    "search_terms": ["Testing", "Help Desk", "Support"],
+    "results_wanted": 50,
+    "max_days_old": 7,
+    "target_state": "NY",
+    "user_email": "Branden@autoemployme.onmicrosoft.com"
+  }
+  

job_scraper.py

@@ -0,0 +1,116 @@
+import csv
+import datetime
+from jobspy.google import Google
+from jobspy.linkedin import LinkedIn
+from jobspy.indeed import Indeed
+from jobspy.ziprecruiter import ZipRecruiter
+from jobspy.model import ScraperInput
+
+# Define job sources
+sources = {
+    "google": Google,
+    "linkedin": LinkedIn,
+    "indeed": Indeed,
+    "zip_recruiter": ZipRecruiter,
+}
+
+# Define search preferences
+search_terms = ["Automation Engineer", "CRM Manager", "Implementation Specialist"]
+results_wanted = 200  # Fetch more jobs
+max_days_old = 2  # Fetch jobs posted in last 48 hours
+target_state = "NY"  # Only keep jobs from New York
+
+
+def scrape_jobs(search_terms, results_wanted, max_days_old, target_state):
+    """Scrape jobs from multiple sources and filter by state."""
+    all_jobs = []
+    today = datetime.date.today()
+    print("\n🔎 DEBUG: Fetching jobs for search terms:", search_terms)
+
+    for search_term in search_terms:
+        for source_name, source_class in sources.items():
+            print(f"\n🚀 Scraping {search_term} from {source_name}...")
+
+            scraper = source_class()
+            search_criteria = ScraperInput(
+                site_type=[source_name],
+                search_term=search_term,
+                results_wanted=results_wanted,
+            )
+
+            job_response = scraper.scrape(search_criteria)
+
+            for job in job_response.jobs:
+                # Normalize location fields
+                location_city = job.location.city.strip() if job.location.city else "Unknown"
+                location_state = job.location.state.strip().upper() if job.location.state else "Unknown"
+                location_country = str(job.location.country) if job.location.country else "Unknown"
+
+                # Debug: Show all jobs being fetched
+                print(f"📍 Fetched Job: {job.title} - {location_city}, {location_state}, {location_country}")
+
+                # Ensure the job is recent
+                if job.date_posted and (today - job.date_posted).days <= max_days_old:
+                    if location_state == target_state or job.is_remote:
+                        print(f"✅ MATCH (In NY or Remote): {job.title} - {location_city}, {location_state} (Posted {job.date_posted})")
+
+                        all_jobs.append({
+                            "Job ID": job.id,
+                            "Job Title (Primary)": job.title,
+                            "Company Name": job.company_name if job.company_name else "Unknown",
+                            "Industry": job.company_industry if job.company_industry else "Not Provided",
+                            "Experience Level": job.job_level if job.job_level else "Not Provided",
+                            "Job Type": job.job_type[0].name if job.job_type else "Not Provided",
+                            "Is Remote": job.is_remote,
+                            "Currency": job.compensation.currency if job.compensation else "",
+                            "Salary Min": job.compensation.min_amount if job.compensation else "",
+                            "Salary Max": job.compensation.max_amount if job.compensation else "",
+                            "Date Posted": job.date_posted.strftime("%Y-%m-%d") if job.date_posted else "Not Provided",
+                            "Location City": location_city,
+                            "Location State": location_state,
+                            "Location Country": location_country,
+                            "Job URL": job.job_url,
+                            "Job Description": job.description[:500] if job.description else "No description available",
+                            "Job Source": source_name
+                        })
+                    else:
+                        print(f"❌ Ignored (Wrong State): {job.title} - {location_city}, {location_state} (Posted {job.date_posted})")
+                else:
+                    print(f"⏳ Ignored (Too Old): {job.title} - {location_city}, {location_state} (Posted {job.date_posted})")
+
+    print(f"\n✅ {len(all_jobs)} jobs retrieved in NY")
+    return all_jobs
+
+
+def save_jobs_to_csv(jobs, filename="jobspy_output.csv"):
+    """Save job data to a CSV file."""
+    if not jobs:
+        print("⚠️ No jobs found matching criteria.")
+        return
+
+    fieldnames = [
+        "Job ID", "Job Title (Primary)", "Company Name", "Industry",
+        "Experience Level", "Job Type", "Is Remote", "Currency",
+        "Salary Min", "Salary Max", "Date Posted", "Location City",
+        "Location State", "Location Country", "Job URL", "Job Description",
+        "Job Source"
+    ]
+
+    with open(filename, mode="w", newline="", encoding="utf-8") as file:
+        writer = csv.DictWriter(file, fieldnames=fieldnames)
+        writer.writeheader()
+        writer.writerows(jobs)
+
+    print(f"✅ Jobs saved to {filename} ({len(jobs)} entries)")
+
+
+# Run the scraper with multiple job searches
+job_data = scrape_jobs(
+    search_terms=search_terms,
+    results_wanted=results_wanted,
+    max_days_old=max_days_old,
+    target_state=target_state
+)
+
+# Save results to CSV
+save_jobs_to_csv(job_data)

job_scraper_dynamic.py

@@ -0,0 +1,94 @@
+import csv, datetime, os, sys, json
+from jobspy.google import Google
+from jobspy.linkedin import LinkedIn
+from jobspy.indeed import Indeed
+from jobspy.model import ScraperInput
+
+sources = {
+    "google": Google,
+    "linkedin": LinkedIn,
+    "indeed": Indeed,
+}
+
+def sanitize_email(email):
+    return email.replace("@", "_at_").replace(".", "_")
+
+def load_config(email):
+    safe_email = sanitize_email(email)
+    config_path = os.path.join("configs", f"config_{safe_email}.json")
+    if not os.path.exists(config_path):
+        raise FileNotFoundError(f"❌ Config for {email} not found at {config_path}")
+    with open(config_path, "r", encoding="utf-8") as f:
+        return json.load(f), safe_email
+
+def scrape_jobs(search_terms, results_wanted, max_days_old, target_state):
+    today = datetime.date.today()
+    all_jobs = []
+
+    for term in search_terms:
+        for source, Scraper in sources.items():
+            print(f"🔍 Scraping {term} from {source}")
+            scraper = Scraper()
+            try:
+                jobs = scraper.scrape(ScraperInput(
+                    site_type=[source],
+                    search_term=term,
+                    results_wanted=results_wanted
+                )).jobs
+            except Exception as e:
+                print(f"⚠️ {source} error: {e}")
+                continue
+
+            for job in jobs:
+                if job.date_posted and (today - job.date_posted).days <= max_days_old:
+                    if target_state == (job.location.state or "").upper() or job.is_remote:
+                        if any(term.lower() in job.title.lower() for term in search_terms):
+                            all_jobs.append({
+                                "Job ID": job.id,
+                                "Job Title (Primary)": job.title,
+                                "Company Name": job.company_name or "Unknown",
+                                "Industry": job.company_industry or "Not Provided",
+                                "Experience Level": job.job_level or "Not Provided",
+                                "Job Type": job.job_type[0].name if job.job_type else "Not Provided",
+                                "Is Remote": job.is_remote,
+                                "Currency": job.compensation.currency if job.compensation else "",
+                                "Salary Min": job.compensation.min_amount if job.compensation else "",
+                                "Salary Max": job.compensation.max_amount if job.compensation else "",
+                                "Date Posted": job.date_posted.strftime("%Y-%m-%d"),
+                                "Location City": job.location.city or "Unknown",
+                                "Location State": (job.location.state or "Unknown").upper(),
+                                "Location Country": job.location.country or "Unknown",
+                                "Job URL": job.job_url,
+                                "Job Description": job.description.replace(",", "") if job.description else "No description",
+                                "Job Source": source
+                            })
+    print(f"✅ Found {len(all_jobs)} jobs")
+    return all_jobs
+
+def save_to_csv(jobs, path):
+    os.makedirs(os.path.dirname(path), exist_ok=True)
+    fieldnames = [
+        "Job ID", "Job Title (Primary)", "Company Name", "Industry",
+        "Experience Level", "Job Type", "Is Remote", "Currency",
+        "Salary Min", "Salary Max", "Date Posted", "Location City",
+        "Location State", "Location Country", "Job URL", "Job Description", "Job Source"
+    ]
+    header = "|~|".join(fieldnames)
+    rows = [header] + ["|~|".join(str(job.get(col, "Not Provided")).replace(",", "").strip() for col in fieldnames) for job in jobs]
+    with open(path, "w", encoding="utf-8") as f:
+        f.write(",".join(rows))
+    print(f"💾 Saved output to: {path}")
+
+if __name__ == "__main__":
+    try:
+        if len(sys.argv) != 3:
+            raise ValueError("❌ Usage: python job_scraper_dynamic.py <user_email> <run_id>")
+
+        user_email, run_id = sys.argv[1], sys.argv[2]
+        config, safe_email = load_config(user_email)
+        jobs = scrape_jobs(config["search_terms"], config["results_wanted"], config["max_days_old"], config["target_state"])
+        save_to_csv(jobs, f"outputs/jobspy_output_{safe_email}_{run_id}.csv")
+
+    except Exception as e:
+        print(f"❌ Fatal error: {e}")
+        sys.exit(1)

job_scraper_exact_match.py

@@ -0,0 +1,146 @@
+import csv
+import datetime
+import os
+from jobspy.google import Google
+from jobspy.linkedin import LinkedIn
+from jobspy.indeed import Indeed
+from jobspy.model import ScraperInput
+
+# Define job sources
+sources = {
+    "google": Google,
+    "linkedin": LinkedIn,
+    "indeed": Indeed,
+}
+
+# Define search preferences
+search_terms = ["Automation Engineer", "CRM Manager", "Implementation Specialist", "CRM", "Project Manager", "POS", "Microsoft Power", "IT Support"]
+results_wanted = 100  # Fetch more jobs
+max_days_old = 2  # Fetch jobs posted in last 48 hours
+target_state = "NY"  # Only keep jobs from New York
+
+def scrape_jobs(search_terms, results_wanted, max_days_old, target_state):
+    """Scrape jobs from multiple sources and filter by state."""
+    all_jobs = []
+    today = datetime.date.today()
+    
+    print("\n🔎 DEBUG: Fetching jobs for search terms:", search_terms)
+
+    for search_term in search_terms:
+        for source_name, source_class in sources.items():
+            print(f"\n🚀 Scraping {search_term} from {source_name}...")
+
+            scraper = source_class()
+            search_criteria = ScraperInput(
+                site_type=[source_name],
+                search_term=search_term,
+                results_wanted=results_wanted,
+            )
+
+            job_response = scraper.scrape(search_criteria)
+
+            for job in job_response.jobs:
+                # Normalize location fields
+                location_city = job.location.city.strip() if job.location.city else "Unknown"
+                location_state = job.location.state.strip().upper() if job.location.state else "Unknown"
+                location_country = str(job.location.country) if job.location.country else "Unknown"
+
+                # Debug: Show all jobs being fetched
+                print(f"📍 Fetched Job: {job.title} - {location_city}, {location_state}, {location_country}")
+
+                # Exclude jobs that don’t explicitly match the search terms
+                if not any(term.lower() in job.title.lower() for term in search_terms):
+                    print(f"🚫 Excluding: {job.title} (Doesn't match {search_terms})")
+                    continue  # Skip this job
+
+                # Ensure the job is recent
+                if job.date_posted and (today - job.date_posted).days <= max_days_old:
+                    # Only accept jobs if they're in NY or Remote
+                    if location_state == target_state or job.is_remote:
+                        print(f"✅ MATCH: {job.title} - {location_city}, {location_state} (Posted {job.date_posted})")
+                        all_jobs.append({
+                            "Job ID": job.id,
+                            "Job Title (Primary)": job.title,
+                            "Company Name": job.company_name if job.company_name else "Unknown",
+                            "Industry": job.company_industry if job.company_industry else "Not Provided",
+                            "Experience Level": job.job_level if job.job_level else "Not Provided",
+                            "Job Type": job.job_type[0].name if job.job_type else "Not Provided",
+                            "Is Remote": job.is_remote,
+                            "Currency": job.compensation.currency if job.compensation else "",
+                            "Salary Min": job.compensation.min_amount if job.compensation else "",
+                            "Salary Max": job.compensation.max_amount if job.compensation else "",
+                            "Date Posted": job.date_posted.strftime("%Y-%m-%d") if job.date_posted else "Not Provided",
+                            "Location City": location_city,
+                            "Location State": location_state,
+                            "Location Country": location_country,
+                            "Job URL": job.job_url,
+                            "Job Description": job.description.replace(",", "") if job.description else "No description available",
+                            "Job Source": source_name
+                        })
+                    else:
+                        print(f"❌ Ignored (Wrong State): {job.title} - {location_city}, {location_state} (Posted {job.date_posted})")
+                else:
+                    print(f"⏳ Ignored (Too Old): {job.title} - {location_city}, {location_state} (Posted {job.date_posted})")
+
+    print(f"\n✅ {len(all_jobs)} jobs retrieved in NY")
+    return all_jobs
+
+
+def save_jobs_to_csv(jobs, filename="jobspy_output.csv"):
+    """Save job data to a CSV file with custom formatting:
+       - Fields within a record are separated by the custom delimiter |~|
+       - Records are separated by a comma
+       - All commas in field values are removed
+       - Blank fields are replaced with 'Not Provided'
+    """
+    if not jobs:
+        print("⚠️ No jobs found matching criteria.")
+        return
+
+    # Remove old CSV file before writing
+    if os.path.exists(filename):
+        os.remove(filename)
+
+    fieldnames = [
+        "Job ID", "Job Title (Primary)", "Company Name", "Industry",
+        "Experience Level", "Job Type", "Is Remote", "Currency",
+        "Salary Min", "Salary Max", "Date Posted", "Location City",
+        "Location State", "Location Country", "Job URL", "Job Description",
+        "Job Source"
+    ]
+
+    # Build header record using custom field delimiter
+    header_record = "|~|".join(fieldnames)
+    records = [header_record]
+
+    for job in jobs:
+        row = []
+        for field in fieldnames:
+            value = str(job.get(field, "")).strip()
+            if not value:
+                value = "Not Provided"
+            # Remove all commas from the value
+            value = value.replace(",", "")
+            row.append(value)
+        # Join fields with the custom delimiter
+        record = "|~|".join(row)
+        records.append(record)
+    
+    # Join records with a comma as the record separator
+    output = ",".join(records)
+    with open(filename, "w", encoding="utf-8") as file:
+        file.write(output)
+
+    print(f"✅ Jobs saved to {filename} ({len(jobs)} entries)")
+
+
+# Run the scraper with multiple job searches
+job_data = scrape_jobs(
+    search_terms=search_terms,
+    results_wanted=results_wanted,
+    max_days_old=max_days_old,
+    target_state=target_state
+)
+
+# Save results to CSV with custom formatting
+save_jobs_to_csv(job_data)

jobspy/__init__.py

@@ -1,27 +1,33 @@
 from __future__ import annotations
 
-import pandas as pd
-from typing import Tuple
 from concurrent.futures import ThreadPoolExecutor, as_completed
+from typing import Tuple
 
-from .jobs import JobType, Location
-from .scrapers.utils import logger, set_logger_level, extract_salary
-from .scrapers.indeed import IndeedScraper
-from .scrapers.ziprecruiter import ZipRecruiterScraper
-from .scrapers.glassdoor import GlassdoorScraper
-from .scrapers.linkedin import LinkedInScraper
-from .scrapers import SalarySource, ScraperInput, Site, JobResponse, Country
-from .scrapers.exceptions import (
-    LinkedInException,
-    IndeedException,
-    ZipRecruiterException,
-    GlassdoorException,
+import pandas as pd
+
+from jobspy.bayt import BaytScraper
+from jobspy.glassdoor import Glassdoor
+from jobspy.google import Google
+from jobspy.indeed import Indeed
+from jobspy.linkedin import LinkedIn
+from jobspy.model import JobType, Location, JobResponse, Country
+from jobspy.model import SalarySource, ScraperInput, Site
+from jobspy.util import (
+    set_logger_level,
+    extract_salary,
+    create_logger,
+    get_enum_from_value,
+    map_str_to_site,
+    convert_to_annual,
+    desired_order,
 )
+from jobspy.ziprecruiter import ZipRecruiter
 
 
 def scrape_jobs(
     site_name: str | list[str] | Site | list[Site] | None = None,
     search_term: str | None = None,
+    google_search_term: str | None = None,
     location: str | None = None,
     distance: int | None = 50,
     is_remote: bool = False,
@@ -29,38 +35,30 @@ def scrape_jobs(
     easy_apply: bool | None = None,
     results_wanted: int = 15,
     country_indeed: str = "usa",
-    hyperlinks: bool = False,
     proxies: list[str] | str | None = None,
+    ca_cert: str | None = None,
     description_format: str = "markdown",
     linkedin_fetch_description: bool | None = False,
     linkedin_company_ids: list[int] | None = None,
     offset: int | None = 0,
     hours_old: int = None,
     enforce_annual_salary: bool = False,
-    verbose: int = 2,
+    verbose: int = 0,
     **kwargs,
 ) -> pd.DataFrame:
     """
-    Simultaneously scrapes job data from multiple job sites.
-    :return: pandas dataframe containing job data
+    Scrapes job data from job boards concurrently
+    :return: Pandas DataFrame containing job data
     """
     SCRAPER_MAPPING = {
-        Site.LINKEDIN: LinkedInScraper,
-        Site.INDEED: IndeedScraper,
-        Site.ZIP_RECRUITER: ZipRecruiterScraper,
-        Site.GLASSDOOR: GlassdoorScraper,
+        Site.LINKEDIN: LinkedIn,
+        Site.INDEED: Indeed,
+        Site.ZIP_RECRUITER: ZipRecruiter,
+        Site.GLASSDOOR: Glassdoor,
+        Site.GOOGLE: Google,
+        Site.BAYT: BaytScraper,
     }
     set_logger_level(verbose)
-
-    def map_str_to_site(site_name: str) -> Site:
-        return Site[site_name.upper()]
-
-    def get_enum_from_value(value_str):
-        for job_type in JobType:
-            if value_str in job_type.value:
-                return job_type
-        raise Exception(f"Invalid job type: {value_str}")
-
     job_type = get_enum_from_value(job_type) if job_type else None
 
     def get_site_type():
@@ -82,6 +80,7 @@ def scrape_jobs(
         site_type=get_site_type(),
         country=country_enum,
         search_term=search_term,
+        google_search_term=google_search_term,
         location=location,
         distance=distance,
         is_remote=is_remote,
@@ -97,11 +96,11 @@ def scrape_jobs(
 
     def scrape_site(site: Site) -> Tuple[str, JobResponse]:
         scraper_class = SCRAPER_MAPPING[site]
-        scraper = scraper_class(proxies=proxies)
+        scraper = scraper_class(proxies=proxies, ca_cert=ca_cert)
         scraped_data: JobResponse = scraper.scrape(scraper_input)
         cap_name = site.value.capitalize()
         site_name = "ZipRecruiter" if cap_name == "Zip_recruiter" else cap_name
-        logger.info(f"{site_name} finished scraping")
+        create_logger(site_name).info(f"finished scraping")
         return site.value, scraped_data
 
     site_to_jobs_dict = {}
@@ -119,28 +118,12 @@ def scrape_jobs(
             site_value, scraped_data = future.result()
             site_to_jobs_dict[site_value] = scraped_data
 
-    def convert_to_annual(job_data: dict):
-        if job_data["interval"] == "hourly":
-            job_data["min_amount"] *= 2080
-            job_data["max_amount"] *= 2080
-        if job_data["interval"] == "monthly":
-            job_data["min_amount"] *= 12
-            job_data["max_amount"] *= 12
-        if job_data["interval"] == "weekly":
-            job_data["min_amount"] *= 52
-            job_data["max_amount"] *= 52
-        if job_data["interval"] == "daily":
-            job_data["min_amount"] *= 260
-            job_data["max_amount"] *= 260
-        job_data["interval"] = "yearly"
-
     jobs_dfs: list[pd.DataFrame] = []
 
     for site, job_response in site_to_jobs_dict.items():
         for job in job_response.jobs:
             job_data = job.dict()
             job_url = job_data["job_url"]
-            job_data["job_url_hyper"] = f'<a href="{job_url}">{job_url}</a>'
             job_data["site"] = site
             job_data["company"] = job_data["company_name"]
             job_data["job_type"] = (
@@ -203,41 +186,6 @@ def scrape_jobs(
         # Step 2: Concatenate the filtered DataFrames
         jobs_df = pd.concat(filtered_dfs, ignore_index=True)
 
-        # Desired column order
-        desired_order = [
-            "id",
-            "site",
-            "job_url_hyper" if hyperlinks else "job_url",
-            "job_url_direct",
-            "title",
-            "company",
-            "location",
-            "job_type",
-            "date_posted",
-            "salary_source",
-            "interval",
-            "min_amount",
-            "max_amount",
-            "currency",
-            "is_remote",
-            "job_level",
-            "job_function",
-            "company_industry",
-            "listing_type",
-            "emails",
-            "description",
-            "company_url",
-            "company_url_direct",
-            "company_addresses",
-            "company_num_employees",
-            "company_revenue",
-            "company_description",
-            "logo_photo_url",
-            "banner_photo_url",
-            "ceo_name",
-            "ceo_photo_url",
-        ]
-
         # Step 3: Ensure all desired columns are present, adding missing ones as empty
         for column in desired_order:
             if column not in jobs_df.columns:
@@ -247,6 +195,8 @@ def scrape_jobs(
         jobs_df = jobs_df[desired_order]
 
         # Step 4: Sort the DataFrame as required
-        return jobs_df.sort_values(by=["site", "date_posted"], ascending=[True, False])
+        return jobs_df.sort_values(
+            by=["site", "date_posted"], ascending=[True, False]
+        ).reset_index(drop=True)
     else:
         return pd.DataFrame()

jobspy/bayt/__init__.py

@@ -0,0 +1,145 @@
+from __future__ import annotations
+
+import random
+import time
+
+from bs4 import BeautifulSoup
+
+from jobspy.model import (
+    Scraper,
+    ScraperInput,
+    Site,
+    JobPost,
+    JobResponse,
+    Location,
+    Country,
+)
+from jobspy.util import create_logger, create_session
+
+log = create_logger("Bayt")
+
+
+class BaytScraper(Scraper):
+    base_url = "https://www.bayt.com"
+    delay = 2
+    band_delay = 3
+
+    def __init__(
+        self, proxies: list[str] | str | None = None, ca_cert: str | None = None
+    ):
+        super().__init__(Site.BAYT, proxies=proxies, ca_cert=ca_cert)
+        self.scraper_input = None
+        self.session = None
+        self.country = "worldwide"
+
+    def scrape(self, scraper_input: ScraperInput) -> JobResponse:
+        self.scraper_input = scraper_input
+        self.session = create_session(
+            proxies=self.proxies, ca_cert=self.ca_cert, is_tls=False, has_retry=True
+        )
+        job_list: list[JobPost] = []
+        page = 1
+        results_wanted = (
+            scraper_input.results_wanted if scraper_input.results_wanted else 10
+        )
+
+        while len(job_list) < results_wanted:
+            log.info(f"Fetching Bayt jobs page {page}")
+            job_elements = self._fetch_jobs(self.scraper_input.search_term, page)
+            if not job_elements:
+                break
+
+            if job_elements:
+                log.debug(
+                    "First job element snippet:\n" + job_elements[0].prettify()[:500]
+                )
+
+            initial_count = len(job_list)
+            for job in job_elements:
+                try:
+                    job_post = self._extract_job_info(job)
+                    if job_post:
+                        job_list.append(job_post)
+                        if len(job_list) >= results_wanted:
+                            break
+                    else:
+                        log.debug(
+                            "Extraction returned None. Job snippet:\n"
+                            + job.prettify()[:500]
+                        )
+                except Exception as e:
+                    log.error(f"Bayt: Error extracting job info: {str(e)}")
+                    continue
+
+            if len(job_list) == initial_count:
+                log.info(f"No new jobs found on page {page}. Ending pagination.")
+                break
+
+            page += 1
+            time.sleep(random.uniform(self.delay, self.delay + self.band_delay))
+
+        job_list = job_list[: scraper_input.results_wanted]
+        return JobResponse(jobs=job_list)
+
+    def _fetch_jobs(self, query: str, page: int) -> list | None:
+        """
+        Grabs the job results for the given query and page number.
+        """
+        try:
+            url = f"{self.base_url}/en/international/jobs/{query}-jobs/?page={page}"
+            response = self.session.get(url)
+            response.raise_for_status()
+            soup = BeautifulSoup(response.text, "html.parser")
+            job_listings = soup.find_all("li", attrs={"data-js-job": ""})
+            log.debug(f"Found {len(job_listings)} job listing elements")
+            return job_listings
+        except Exception as e:
+            log.error(f"Bayt: Error fetching jobs - {str(e)}")
+            return None
+
+    def _extract_job_info(self, job: BeautifulSoup) -> JobPost | None:
+        """
+        Extracts the job information from a single job listing.
+        """
+        # Find the h2 element holding the title and link (no class filtering)
+        job_general_information = job.find("h2")
+        if not job_general_information:
+            return
+
+        job_title = job_general_information.get_text(strip=True)
+        job_url = self._extract_job_url(job_general_information)
+        if not job_url:
+            return
+
+        # Extract company name using the original approach:
+        company_tag = job.find("div", class_="t-nowrap p10l")
+        company_name = (
+            company_tag.find("span").get_text(strip=True)
+            if company_tag and company_tag.find("span")
+            else None
+        )
+
+        # Extract location using the original approach:
+        location_tag = job.find("div", class_="t-mute t-small")
+        location = location_tag.get_text(strip=True) if location_tag else None
+
+        job_id = f"bayt-{abs(hash(job_url))}"
+        location_obj = Location(
+            city=location,
+            country=Country.from_string(self.country),
+        )
+        return JobPost(
+            id=job_id,
+            title=job_title,
+            company_name=company_name,
+            location=location_obj,
+            job_url=job_url,
+        )
+
+    def _extract_job_url(self, job_general_information: BeautifulSoup) -> str | None:
+        """
+        Pulls the job URL from the 'a' within the h2 element.
+        """
+        a_tag = job_general_information.find("a")
+        if a_tag and a_tag.has_attr("href"):
+            return self.base_url + a_tag["href"].strip()

jobspy/exception.py

@@ -1,5 +1,5 @@
 """
-jobspy.scrapers.exceptions
+jobspy.jobboard.exceptions
 ~~~~~~~~~~~~~~~~~~~
 
 This module contains the set of Scrapers' exceptions.
@@ -24,3 +24,13 @@ class ZipRecruiterException(Exception):
 class GlassdoorException(Exception):
     def __init__(self, message=None):
         super().__init__(message or "An error occurred with Glassdoor")
+
+
+class GoogleJobsException(Exception):
+    def __init__(self, message=None):
+        super().__init__(message or "An error occurred with Google Jobs")
+
+
+class BaytException(Exception):
+    def __init__(self, message=None):
+        super().__init__(message or "An error occurred with Bayt")

jobspy/glassdoor/__init__.py

@@ -0,0 +1,320 @@
+from __future__ import annotations
+
+import re
+import json
+import requests
+from typing import Tuple
+from datetime import datetime, timedelta
+from concurrent.futures import ThreadPoolExecutor, as_completed
+
+from jobspy.glassdoor.constant import fallback_token, query_template, headers
+from jobspy.glassdoor.util import (
+    get_cursor_for_page,
+    parse_compensation,
+    parse_location,
+)
+from jobspy.util import (
+    extract_emails_from_text,
+    create_logger,
+    create_session,
+    markdown_converter,
+)
+from jobspy.exception import GlassdoorException
+from jobspy.model import (
+    JobPost,
+    JobResponse,
+    DescriptionFormat,
+    Scraper,
+    ScraperInput,
+    Site,
+)
+
+log = create_logger("Glassdoor")
+
+
+class Glassdoor(Scraper):
+    def __init__(
+        self, proxies: list[str] | str | None = None, ca_cert: str | None = None
+    ):
+        """
+        Initializes GlassdoorScraper with the Glassdoor job search url
+        """
+        site = Site(Site.GLASSDOOR)
+        super().__init__(site, proxies=proxies, ca_cert=ca_cert)
+
+        self.base_url = None
+        self.country = None
+        self.session = None
+        self.scraper_input = None
+        self.jobs_per_page = 30
+        self.max_pages = 30
+        self.seen_urls = set()
+
+    def scrape(self, scraper_input: ScraperInput) -> JobResponse:
+        """
+        Scrapes Glassdoor for jobs with scraper_input criteria.
+        :param scraper_input: Information about job search criteria.
+        :return: JobResponse containing a list of jobs.
+        """
+        self.scraper_input = scraper_input
+        self.scraper_input.results_wanted = min(900, scraper_input.results_wanted)
+        self.base_url = self.scraper_input.country.get_glassdoor_url()
+
+        self.session = create_session(
+            proxies=self.proxies, ca_cert=self.ca_cert, has_retry=True
+        )
+        token = self._get_csrf_token()
+        headers["gd-csrf-token"] = token if token else fallback_token
+        self.session.headers.update(headers)
+
+        location_id, location_type = self._get_location(
+            scraper_input.location, scraper_input.is_remote
+        )
+        if location_type is None:
+            log.error("Glassdoor: location not parsed")
+            return JobResponse(jobs=[])
+        job_list: list[JobPost] = []
+        cursor = None
+
+        range_start = 1 + (scraper_input.offset // self.jobs_per_page)
+        tot_pages = (scraper_input.results_wanted // self.jobs_per_page) + 2
+        range_end = min(tot_pages, self.max_pages + 1)
+        for page in range(range_start, range_end):
+            log.info(f"search page: {page} / {range_end - 1}")
+            try:
+                jobs, cursor = self._fetch_jobs_page(
+                    scraper_input, location_id, location_type, page, cursor
+                )
+                job_list.extend(jobs)
+                if not jobs or len(job_list) >= scraper_input.results_wanted:
+                    job_list = job_list[: scraper_input.results_wanted]
+                    break
+            except Exception as e:
+                log.error(f"Glassdoor: {str(e)}")
+                break
+        return JobResponse(jobs=job_list)
+
+    def _fetch_jobs_page(
+        self,
+        scraper_input: ScraperInput,
+        location_id: int,
+        location_type: str,
+        page_num: int,
+        cursor: str | None,
+    ) -> Tuple[list[JobPost], str | None]:
+        """
+        Scrapes a page of Glassdoor for jobs with scraper_input criteria
+        """
+        jobs = []
+        self.scraper_input = scraper_input
+        try:
+            payload = self._add_payload(location_id, location_type, page_num, cursor)
+            response = self.session.post(
+                f"{self.base_url}/graph",
+                timeout_seconds=15,
+                data=payload,
+            )
+            if response.status_code != 200:
+                exc_msg = f"bad response status code: {response.status_code}"
+                raise GlassdoorException(exc_msg)
+            res_json = response.json()[0]
+            if "errors" in res_json:
+                raise ValueError("Error encountered in API response")
+        except (
+            requests.exceptions.ReadTimeout,
+            GlassdoorException,
+            ValueError,
+            Exception,
+        ) as e:
+            log.error(f"Glassdoor: {str(e)}")
+            return jobs, None
+
+        jobs_data = res_json["data"]["jobListings"]["jobListings"]
+
+        with ThreadPoolExecutor(max_workers=self.jobs_per_page) as executor:
+            future_to_job_data = {
+                executor.submit(self._process_job, job): job for job in jobs_data
+            }
+            for future in as_completed(future_to_job_data):
+                try:
+                    job_post = future.result()
+                    if job_post:
+                        jobs.append(job_post)
+                except Exception as exc:
+                    raise GlassdoorException(f"Glassdoor generated an exception: {exc}")
+
+        return jobs, get_cursor_for_page(
+            res_json["data"]["jobListings"]["paginationCursors"], page_num + 1
+        )
+
+    def _get_csrf_token(self):
+        """
+        Fetches csrf token needed for API by visiting a generic page
+        """
+        res = self.session.get(f"{self.base_url}/Job/computer-science-jobs.htm")
+        pattern = r'"token":\s*"([^"]+)"'
+        matches = re.findall(pattern, res.text)
+        token = None
+        if matches:
+            token = matches[0]
+        return token
+
+    def _process_job(self, job_data):
+        """
+        Processes a single job and fetches its description.
+        """
+        job_id = job_data["jobview"]["job"]["listingId"]
+        job_url = f"{self.base_url}job-listing/j?jl={job_id}"
+        if job_url in self.seen_urls:
+            return None
+        self.seen_urls.add(job_url)
+        job = job_data["jobview"]
+        title = job["job"]["jobTitleText"]
+        company_name = job["header"]["employerNameFromSearch"]
+        company_id = job_data["jobview"]["header"]["employer"]["id"]
+        location_name = job["header"].get("locationName", "")
+        location_type = job["header"].get("locationType", "")
+        age_in_days = job["header"].get("ageInDays")
+        is_remote, location = False, None
+        date_diff = (datetime.now() - timedelta(days=age_in_days)).date()
+        date_posted = date_diff if age_in_days is not None else None
+
+        if location_type == "S":
+            is_remote = True
+        else:
+            location = parse_location(location_name)
+
+        compensation = parse_compensation(job["header"])
+        try:
+            description = self._fetch_job_description(job_id)
+        except:
+            description = None
+        company_url = f"{self.base_url}Overview/W-EI_IE{company_id}.htm"
+        company_logo = (
+            job_data["jobview"].get("overview", {}).get("squareLogoUrl", None)
+        )
+        listing_type = (
+            job_data["jobview"]
+            .get("header", {})
+            .get("adOrderSponsorshipLevel", "")
+            .lower()
+        )
+        return JobPost(
+            id=f"gd-{job_id}",
+            title=title,
+            company_url=company_url if company_id else None,
+            company_name=company_name,
+            date_posted=date_posted,
+            job_url=job_url,
+            location=location,
+            compensation=compensation,
+            is_remote=is_remote,
+            description=description,
+            emails=extract_emails_from_text(description) if description else None,
+            company_logo=company_logo,
+            listing_type=listing_type,
+        )
+
+    def _fetch_job_description(self, job_id):
+        """
+        Fetches the job description for a single job ID.
+        """
+        url = f"{self.base_url}/graph"
+        body = [
+            {
+                "operationName": "JobDetailQuery",
+                "variables": {
+                    "jl": job_id,
+                    "queryString": "q",
+                    "pageTypeEnum": "SERP",
+                },
+                "query": """
+                query JobDetailQuery($jl: Long!, $queryString: String, $pageTypeEnum: PageTypeEnum) {
+                    jobview: jobView(
+                        listingId: $jl
+                        contextHolder: {queryString: $queryString, pageTypeEnum: $pageTypeEnum}
+                    ) {
+                        job {
+                            description
+                            __typename
+                        }
+                        __typename
+                    }
+                }
+                """,
+            }
+        ]
+        res = requests.post(url, json=body, headers=headers)
+        if res.status_code != 200:
+            return None
+        data = res.json()[0]
+        desc = data["data"]["jobview"]["job"]["description"]
+        if self.scraper_input.description_format == DescriptionFormat.MARKDOWN:
+            desc = markdown_converter(desc)
+        return desc
+
+    def _get_location(self, location: str, is_remote: bool) -> (int, str):
+        if not location or is_remote:
+            return "11047", "STATE"  # remote options
+        url = f"{self.base_url}/findPopularLocationAjax.htm?maxLocationsToReturn=10&term={location}"
+        res = self.session.get(url)
+        if res.status_code != 200:
+            if res.status_code == 429:
+                err = f"429 Response - Blocked by Glassdoor for too many requests"
+                log.error(err)
+                return None, None
+            else:
+                err = f"Glassdoor response status code {res.status_code}"
+                err += f" - {res.text}"
+                log.error(f"Glassdoor response status code {res.status_code}")
+                return None, None
+        items = res.json()
+
+        if not items:
+            raise ValueError(f"Location '{location}' not found on Glassdoor")
+        location_type = items[0]["locationType"]
+        if location_type == "C":
+            location_type = "CITY"
+        elif location_type == "S":
+            location_type = "STATE"
+        elif location_type == "N":
+            location_type = "COUNTRY"
+        return int(items[0]["locationId"]), location_type
+
+    def _add_payload(
+        self,
+        location_id: int,
+        location_type: str,
+        page_num: int,
+        cursor: str | None = None,
+    ) -> str:
+        fromage = None
+        if self.scraper_input.hours_old:
+            fromage = max(self.scraper_input.hours_old // 24, 1)
+        filter_params = []
+        if self.scraper_input.easy_apply:
+            filter_params.append({"filterKey": "applicationType", "values": "1"})
+        if fromage:
+            filter_params.append({"filterKey": "fromAge", "values": str(fromage)})
+        payload = {
+            "operationName": "JobSearchResultsQuery",
+            "variables": {
+                "excludeJobListingIds": [],
+                "filterParams": filter_params,
+                "keyword": self.scraper_input.search_term,
+                "numJobsToShow": 30,
+                "locationType": location_type,
+                "locationId": int(location_id),
+                "parameterUrlInput": f"IL.0,12_I{location_type}{location_id}",
+                "pageNumber": page_num,
+                "pageCursor": cursor,
+                "fromage": fromage,
+                "sort": "date",
+            },
+            "query": query_template,
+        }
+        if self.scraper_input.job_type:
+            payload["variables"]["filterParams"].append(
+                {"filterKey": "jobType", "values": self.scraper_input.job_type.value[0]}
+            )
+        return json.dumps([payload])

jobspy/glassdoor/constant.py

@@ -0,0 +1,184 @@
+headers = {
+    "authority": "www.glassdoor.com",
+    "accept": "*/*",
+    "accept-language": "en-US,en;q=0.9",
+    "apollographql-client-name": "job-search-next",
+    "apollographql-client-version": "4.65.5",
+    "content-type": "application/json",
+    "origin": "https://www.glassdoor.com",
+    "referer": "https://www.glassdoor.com/",
+    "sec-ch-ua": '"Chromium";v="118", "Google Chrome";v="118", "Not=A?Brand";v="99"',
+    "sec-ch-ua-mobile": "?0",
+    "sec-ch-ua-platform": '"macOS"',
+    "sec-fetch-dest": "empty",
+    "sec-fetch-mode": "cors",
+    "sec-fetch-site": "same-origin",
+    "user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36",
+}
+query_template = """
+        query JobSearchResultsQuery(
+            $excludeJobListingIds: [Long!], 
+            $keyword: String, 
+            $locationId: Int, 
+            $locationType: LocationTypeEnum, 
+            $numJobsToShow: Int!, 
+            $pageCursor: String, 
+            $pageNumber: Int, 
+            $filterParams: [FilterParams], 
+            $originalPageUrl: String, 
+            $seoFriendlyUrlInput: String, 
+            $parameterUrlInput: String, 
+            $seoUrl: Boolean
+        ) {
+            jobListings(
+                contextHolder: {
+                    searchParams: {
+                        excludeJobListingIds: $excludeJobListingIds, 
+                        keyword: $keyword, 
+                        locationId: $locationId, 
+                        locationType: $locationType, 
+                        numPerPage: $numJobsToShow, 
+                        pageCursor: $pageCursor, 
+                        pageNumber: $pageNumber, 
+                        filterParams: $filterParams, 
+                        originalPageUrl: $originalPageUrl, 
+                        seoFriendlyUrlInput: $seoFriendlyUrlInput, 
+                        parameterUrlInput: $parameterUrlInput, 
+                        seoUrl: $seoUrl, 
+                        searchType: SR
+                    }
+                }
+            ) {
+                companyFilterOptions {
+                    id
+                    shortName
+                    __typename
+                }
+                filterOptions
+                indeedCtk
+                jobListings {
+                    ...JobView
+                    __typename
+                }
+                jobListingSeoLinks {
+                    linkItems {
+                        position
+                        url
+                        __typename
+                    }
+                    __typename
+                }
+                jobSearchTrackingKey
+                jobsPageSeoData {
+                    pageMetaDescription
+                    pageTitle
+                    __typename
+                }
+                paginationCursors {
+                    cursor
+                    pageNumber
+                    __typename
+                }
+                indexablePageForSeo
+                searchResultsMetadata {
+                    searchCriteria {
+                        implicitLocation {
+                            id
+                            localizedDisplayName
+                            type
+                            __typename
+                        }
+                        keyword
+                        location {
+                            id
+                            shortName
+                            localizedShortName
+                            localizedDisplayName
+                            type
+                            __typename
+                        }
+                        __typename
+                    }
+                    helpCenterDomain
+                    helpCenterLocale
+                    jobSerpJobOutlook {
+                        occupation
+                        paragraph
+                        __typename
+                    }
+                    showMachineReadableJobs
+                    __typename
+                }
+                totalJobsCount
+                __typename
+            }
+        }
+
+        fragment JobView on JobListingSearchResult {
+            jobview {
+                header {
+                    adOrderId
+                    advertiserType
+                    adOrderSponsorshipLevel
+                    ageInDays
+                    divisionEmployerName
+                    easyApply
+                    employer {
+                        id
+                        name
+                        shortName
+                        __typename
+                    }
+                    employerNameFromSearch
+                    goc
+                    gocConfidence
+                    gocId
+                    jobCountryId
+                    jobLink
+                    jobResultTrackingKey
+                    jobTitleText
+                    locationName
+                    locationType
+                    locId
+                    needsCommission
+                    payCurrency
+                    payPeriod
+                    payPeriodAdjustedPay {
+                        p10
+                        p50
+                        p90
+                        __typename
+                    }
+                    rating
+                    salarySource
+                    savedJobId
+                    sponsored
+                    __typename
+                }
+                job {
+                    description
+                    importConfigId
+                    jobTitleId
+                    jobTitleText
+                    listingId
+                    __typename
+                }
+                jobListingAdminDetails {
+                    cpcVal
+                    importConfigId
+                    jobListingId
+                    jobSourceId
+                    userEligibleForAdminJobDetails
+                    __typename
+                }
+                overview {
+                    shortName
+                    squareLogoUrl
+                    __typename
+                }
+                __typename
+            }
+            __typename
+        }
+"""
+fallback_token = "Ft6oHEWlRZrxDww95Cpazw:0pGUrkb2y3TyOpAIqF2vbPmUXoXVkD3oEGDVkvfeCerceQ5-n8mBg3BovySUIjmCPHCaW0H2nQVdqzbtsYqf4Q:wcqRqeegRUa9MVLJGyujVXB7vWFPjdaS1CtrrzJq-ok"

jobspy/glassdoor/util.py

@@ -0,0 +1,42 @@
+from jobspy.model import Compensation, CompensationInterval, Location, JobType
+
+
+def parse_compensation(data: dict) -> Compensation | None:
+    pay_period = data.get("payPeriod")
+    adjusted_pay = data.get("payPeriodAdjustedPay")
+    currency = data.get("payCurrency", "USD")
+    if not pay_period or not adjusted_pay:
+        return None
+
+    interval = None
+    if pay_period == "ANNUAL":
+        interval = CompensationInterval.YEARLY
+    elif pay_period:
+        interval = CompensationInterval.get_interval(pay_period)
+    min_amount = int(adjusted_pay.get("p10") // 1)
+    max_amount = int(adjusted_pay.get("p90") // 1)
+    return Compensation(
+        interval=interval,
+        min_amount=min_amount,
+        max_amount=max_amount,
+        currency=currency,
+    )
+
+
+def get_job_type_enum(job_type_str: str) -> list[JobType] | None:
+    for job_type in JobType:
+        if job_type_str in job_type.value:
+            return [job_type]
+
+
+def parse_location(location_name: str) -> Location | None:
+    if not location_name or location_name == "Remote":
+        return
+    city, _, state = location_name.partition(", ")
+    return Location(city=city, state=state)
+
+
+def get_cursor_for_page(pagination_cursors, page_num):
+    for cursor_data in pagination_cursors:
+        if cursor_data["pageNumber"] == page_num:
+            return cursor_data["cursor"]

jobspy/google/__init__.py

@@ -0,0 +1,202 @@
+from __future__ import annotations
+
+import math
+import re
+import json
+from typing import Tuple
+from datetime import datetime, timedelta
+
+from jobspy.google.constant import headers_jobs, headers_initial, async_param
+from jobspy.model import (
+    Scraper,
+    ScraperInput,
+    Site,
+    JobPost,
+    JobResponse,
+    Location,
+    JobType,
+)
+from jobspy.util import extract_emails_from_text, extract_job_type, create_session
+from jobspy.google.util import log, find_job_info_initial_page, find_job_info
+
+
+class Google(Scraper):
+    def __init__(
+        self, proxies: list[str] | str | None = None, ca_cert: str | None = None
+    ):
+        """
+        Initializes Google Scraper with the Goodle jobs search url
+        """
+        site = Site(Site.GOOGLE)
+        super().__init__(site, proxies=proxies, ca_cert=ca_cert)
+
+        self.country = None
+        self.session = None
+        self.scraper_input = None
+        self.jobs_per_page = 10
+        self.seen_urls = set()
+        self.url = "https://www.google.com/search"
+        self.jobs_url = "https://www.google.com/async/callback:550"
+
+    def scrape(self, scraper_input: ScraperInput) -> JobResponse:
+        """
+        Scrapes Google for jobs with scraper_input criteria.
+        :param scraper_input: Information about job search criteria.
+        :return: JobResponse containing a list of jobs.
+        """
+        self.scraper_input = scraper_input
+        self.scraper_input.results_wanted = min(900, scraper_input.results_wanted)
+
+        self.session = create_session(
+            proxies=self.proxies, ca_cert=self.ca_cert, is_tls=False, has_retry=True
+        )
+        forward_cursor, job_list = self._get_initial_cursor_and_jobs()
+        if forward_cursor is None:
+            log.warning(
+                "initial cursor not found, try changing your query or there was at most 10 results"
+            )
+            return JobResponse(jobs=job_list)
+
+        page = 1
+
+        while (
+            len(self.seen_urls) < scraper_input.results_wanted + scraper_input.offset
+            and forward_cursor
+        ):
+            log.info(
+                f"search page: {page} / {math.ceil(scraper_input.results_wanted / self.jobs_per_page)}"
+            )
+            try:
+                jobs, forward_cursor = self._get_jobs_next_page(forward_cursor)
+            except Exception as e:
+                log.error(f"failed to get jobs on page: {page}, {e}")
+                break
+            if not jobs:
+                log.info(f"found no jobs on page: {page}")
+                break
+            job_list += jobs
+            page += 1
+        return JobResponse(
+            jobs=job_list[
+                scraper_input.offset : scraper_input.offset
+                + scraper_input.results_wanted
+            ]
+        )
+
+    def _get_initial_cursor_and_jobs(self) -> Tuple[str, list[JobPost]]:
+        """Gets initial cursor and jobs to paginate through job listings"""
+        query = f"{self.scraper_input.search_term} jobs"
+
+        def get_time_range(hours_old):
+            if hours_old <= 24:
+                return "since yesterday"
+            elif hours_old <= 72:
+                return "in the last 3 days"
+            elif hours_old <= 168:
+                return "in the last week"
+            else:
+                return "in the last month"
+
+        job_type_mapping = {
+            JobType.FULL_TIME: "Full time",
+            JobType.PART_TIME: "Part time",
+            JobType.INTERNSHIP: "Internship",
+            JobType.CONTRACT: "Contract",
+        }
+
+        if self.scraper_input.job_type in job_type_mapping:
+            query += f" {job_type_mapping[self.scraper_input.job_type]}"
+
+        if self.scraper_input.location:
+            query += f" near {self.scraper_input.location}"
+
+        if self.scraper_input.hours_old:
+            time_filter = get_time_range(self.scraper_input.hours_old)
+            query += f" {time_filter}"
+
+        if self.scraper_input.is_remote:
+            query += " remote"
+
+        if self.scraper_input.google_search_term:
+            query = self.scraper_input.google_search_term
+
+        params = {"q": query, "udm": "8"}
+        response = self.session.get(self.url, headers=headers_initial, params=params)
+
+        pattern_fc = r'<div jsname="Yust4d"[^>]+data-async-fc="([^"]+)"'
+        match_fc = re.search(pattern_fc, response.text)
+        data_async_fc = match_fc.group(1) if match_fc else None
+        jobs_raw = find_job_info_initial_page(response.text)
+        jobs = []
+        for job_raw in jobs_raw:
+            job_post = self._parse_job(job_raw)
+            if job_post:
+                jobs.append(job_post)
+        return data_async_fc, jobs
+
+    def _get_jobs_next_page(self, forward_cursor: str) -> Tuple[list[JobPost], str]:
+        params = {"fc": [forward_cursor], "fcv": ["3"], "async": [async_param]}
+        response = self.session.get(self.jobs_url, headers=headers_jobs, params=params)
+        return self._parse_jobs(response.text)
+
+    def _parse_jobs(self, job_data: str) -> Tuple[list[JobPost], str]:
+        """
+        Parses jobs on a page with next page cursor
+        """
+        start_idx = job_data.find("[[[")
+        end_idx = job_data.rindex("]]]") + 3
+        s = job_data[start_idx:end_idx]
+        parsed = json.loads(s)[0]
+
+        pattern_fc = r'data-async-fc="([^"]+)"'
+        match_fc = re.search(pattern_fc, job_data)
+        data_async_fc = match_fc.group(1) if match_fc else None
+        jobs_on_page = []
+        for array in parsed:
+            _, job_data = array
+            if not job_data.startswith("[[["):
+                continue
+            job_d = json.loads(job_data)
+
+            job_info = find_job_info(job_d)
+            job_post = self._parse_job(job_info)
+            if job_post:
+                jobs_on_page.append(job_post)
+        return jobs_on_page, data_async_fc
+
+    def _parse_job(self, job_info: list):
+        job_url = job_info[3][0][0] if job_info[3] and job_info[3][0] else None
+        if job_url in self.seen_urls:
+            return
+        self.seen_urls.add(job_url)
+
+        title = job_info[0]
+        company_name = job_info[1]
+        location = city = job_info[2]
+        state = country = date_posted = None
+        if location and "," in location:
+            city, state, *country = [*map(lambda x: x.strip(), location.split(","))]
+
+        days_ago_str = job_info[12]
+        if type(days_ago_str) == str:
+            match = re.search(r"\d+", days_ago_str)
+            days_ago = int(match.group()) if match else None
+            date_posted = (datetime.now() - timedelta(days=days_ago)).date()
+
+        description = job_info[19]
+
+        job_post = JobPost(
+            id=f"go-{job_info[28]}",
+            title=title,
+            company_name=company_name,
+            location=Location(
+                city=city, state=state, country=country[0] if country else None
+            ),
+            job_url=job_url,
+            date_posted=date_posted,
+            is_remote="remote" in description.lower() or "wfh" in description.lower(),
+            description=description,
+            emails=extract_emails_from_text(description),
+            job_type=extract_job_type(description),
+        )
+        return job_post

jobspy/google/constant.py

@@ -0,0 +1,52 @@
+headers_initial = {
+    "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
+    "accept-language": "en-US,en;q=0.9",
+    "priority": "u=0, i",
+    "referer": "https://www.google.com/",
+    "sec-ch-prefers-color-scheme": "dark",
+    "sec-ch-ua": '"Chromium";v="130", "Google Chrome";v="130", "Not?A_Brand";v="99"',
+    "sec-ch-ua-arch": '"arm"',
+    "sec-ch-ua-bitness": '"64"',
+    "sec-ch-ua-form-factors": '"Desktop"',
+    "sec-ch-ua-full-version": '"130.0.6723.58"',
+    "sec-ch-ua-full-version-list": '"Chromium";v="130.0.6723.58", "Google Chrome";v="130.0.6723.58", "Not?A_Brand";v="99.0.0.0"',
+    "sec-ch-ua-mobile": "?0",
+    "sec-ch-ua-model": '""',
+    "sec-ch-ua-platform": '"macOS"',
+    "sec-ch-ua-platform-version": '"15.0.1"',
+    "sec-ch-ua-wow64": "?0",
+    "sec-fetch-dest": "document",
+    "sec-fetch-mode": "navigate",
+    "sec-fetch-site": "same-origin",
+    "sec-fetch-user": "?1",
+    "upgrade-insecure-requests": "1",
+    "user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36",
+    "x-browser-channel": "stable",
+    "x-browser-copyright": "Copyright 2024 Google LLC. All rights reserved.",
+    "x-browser-year": "2024",
+}
+
+headers_jobs = {
+    "accept": "*/*",
+    "accept-language": "en-US,en;q=0.9",
+    "priority": "u=1, i",
+    "referer": "https://www.google.com/",
+    "sec-ch-prefers-color-scheme": "dark",
+    "sec-ch-ua": '"Chromium";v="130", "Google Chrome";v="130", "Not?A_Brand";v="99"',
+    "sec-ch-ua-arch": '"arm"',
+    "sec-ch-ua-bitness": '"64"',
+    "sec-ch-ua-form-factors": '"Desktop"',
+    "sec-ch-ua-full-version": '"130.0.6723.58"',
+    "sec-ch-ua-full-version-list": '"Chromium";v="130.0.6723.58", "Google Chrome";v="130.0.6723.58", "Not?A_Brand";v="99.0.0.0"',
+    "sec-ch-ua-mobile": "?0",
+    "sec-ch-ua-model": '""',
+    "sec-ch-ua-platform": '"macOS"',
+    "sec-ch-ua-platform-version": '"15.0.1"',
+    "sec-ch-ua-wow64": "?0",
+    "sec-fetch-dest": "empty",
+    "sec-fetch-mode": "cors",
+    "sec-fetch-site": "same-origin",
+    "user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36",
+}
+
+async_param = "_basejs:/xjs/_/js/k=xjs.s.en_US.JwveA-JiKmg.2018.O/am=AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIAAAAAAAAACAAAoICAAAAAAAKMAfAAAAIAQAAAAAAAAAAAAACCAAAEJDAAACAAAAAGABAIAAARBAAABAAAAAgAgQAABAASKAfv8JAAABAAAAAAwAQAQACQAAAAAAcAEAQABoCAAAABAAAIABAACAAAAEAAAAFAAAAAAAAAAAAAAAAAAAAAAAAACAQADoBwAAAAAAAAAAAAAQBAAAAATQAAoACOAHAAAAAAAAAQAAAIIAAAA_ZAACAAAAAAAAcB8APB4wHFJ4AAAAAAAAAAAAAAAACECCYA5If0EACAAAAAAAAAAAAAAAAAAAUgRNXG4AMAE/dg=0/br=1/rs=ACT90oGxMeaFMCopIHq5tuQM-6_3M_VMjQ,_basecss:/xjs/_/ss/k=xjs.s.IwsGu62EDtU.L.B1.O/am=QOoQIAQAAAQAREADEBAAAAAAAAAAAAAAAAAAAAAgAQAAIAAAgAQAAAIAIAIAoEwCAADIC8AfsgEAawwAPkAAjgoAGAAAAAAAAEADAAAAAAIgAECHAAAAAAAAAAABAQAggAARQAAAQCEAAAAAIAAAABgAAAAAIAQIACCAAfB-AAFIQABoCEA_CgEAAIABAACEgHAEwwAEFQAM4CgAAAAAAAAAAAAACABCAAAAQEAAABAgAMCPAAA4AoE2BAEAggSAAIoAQAAAAAgAAAAACCAQAAAxEwA_ZAACAAAAAAAAAAkAAAAAAAAgAAAAAAAAAAAAAAAAAAAAAAAAQAEAAAAAAAAAAAAAAAAAAAAAQA/br=1/rs=ACT90oGZc36t3uUQkj0srnIvvbHjO2hgyg,_basecomb:/xjs/_/js/k=xjs.s.en_US.JwveA-JiKmg.2018.O/ck=xjs.s.IwsGu62EDtU.L.B1.O/am=QOoQIAQAAAQAREADEBAAAAAAAAAAAAAAAAAAAAAgAQAAIAAAgAQAAAKAIAoIqEwCAADIK8AfsgEAawwAPkAAjgoAGAAACCAAAEJDAAACAAIgAGCHAIAAARBAAABBAQAggAgRQABAQSOAfv8JIAABABgAAAwAYAQICSCAAfB-cAFIQABoCEA_ChEAAIABAACEgHAEwwAEFQAM4CgAAAAAAAAAAAAACABCAACAQEDoBxAgAMCPAAA4AoE2BAEAggTQAIoASOAHAAgAAAAACSAQAIIxEwA_ZAACAAAAAAAAcB8APB4wHFJ4AAAAAAAAAAAAAAAACECCYA5If0EACAAAAAAAAAAAAAAAAAAAUgRNXG4AMAE/d=1/ed=1/dg=0/br=1/ujg=1/rs=ACT90oFNLTjPzD_OAqhhtXwe2pg1T3WpBg,_fmt:prog,_id:fc_5FwaZ86OKsfdwN4P4La3yA4_2"

jobspy/google/util.py

@@ -0,0 +1,41 @@
+import re
+
+from jobspy.util import create_logger
+
+log = create_logger("Google")
+
+
+def find_job_info(jobs_data: list | dict) -> list | None:
+    """Iterates through the JSON data to find the job listings"""
+    if isinstance(jobs_data, dict):
+        for key, value in jobs_data.items():
+            if key == "520084652" and isinstance(value, list):
+                return value
+            else:
+                result = find_job_info(value)
+                if result:
+                    return result
+    elif isinstance(jobs_data, list):
+        for item in jobs_data:
+            result = find_job_info(item)
+            if result:
+                return result
+    return None
+
+
+def find_job_info_initial_page(html_text: str):
+    pattern = f'520084652":(' + r"\[.*?\]\s*])\s*}\s*]\s*]\s*]\s*]\s*]"
+    results = []
+    matches = re.finditer(pattern, html_text)
+
+    import json
+
+    for match in matches:
+        try:
+            parsed_data = json.loads(match.group(1))
+            results.append(parsed_data)
+
+        except json.JSONDecodeError as e:
+            log.error(f"Failed to parse match: {str(e)}")
+            results.append({"raw_match": match.group(0), "error": str(e)})
+    return results

jobspy/indeed/__init__.py

@@ -0,0 +1,262 @@
+from __future__ import annotations
+
+import math
+from datetime import datetime
+from typing import Tuple
+
+from jobspy.indeed.constant import job_search_query, api_headers
+from jobspy.indeed.util import is_job_remote, get_compensation, get_job_type
+from jobspy.model import (
+    Scraper,
+    ScraperInput,
+    Site,
+    JobPost,
+    Location,
+    JobResponse,
+    JobType,
+    DescriptionFormat,
+)
+from jobspy.util import (
+    extract_emails_from_text,
+    markdown_converter,
+    create_session,
+    create_logger,
+)
+
+log = create_logger("Indeed")
+
+
+class Indeed(Scraper):
+    def __init__(
+        self, proxies: list[str] | str | None = None, ca_cert: str | None = None
+    ):
+        """
+        Initializes IndeedScraper with the Indeed API url
+        """
+        super().__init__(Site.INDEED, proxies=proxies)
+
+        self.session = create_session(
+            proxies=self.proxies, ca_cert=ca_cert, is_tls=False
+        )
+        self.scraper_input = None
+        self.jobs_per_page = 100
+        self.num_workers = 10
+        self.seen_urls = set()
+        self.headers = None
+        self.api_country_code = None
+        self.base_url = None
+        self.api_url = "https://apis.indeed.com/graphql"
+
+    def scrape(self, scraper_input: ScraperInput) -> JobResponse:
+        """
+        Scrapes Indeed for jobs with scraper_input criteria
+        :param scraper_input:
+        :return: job_response
+        """
+        self.scraper_input = scraper_input
+        domain, self.api_country_code = self.scraper_input.country.indeed_domain_value
+        self.base_url = f"https://{domain}.indeed.com"
+        self.headers = api_headers.copy()
+        self.headers["indeed-co"] = self.scraper_input.country.indeed_domain_value
+        job_list = []
+        page = 1
+
+        cursor = None
+
+        while len(self.seen_urls) < scraper_input.results_wanted + scraper_input.offset:
+            log.info(
+                f"search page: {page} / {math.ceil(scraper_input.results_wanted / self.jobs_per_page)}"
+            )
+            jobs, cursor = self._scrape_page(cursor)
+            if not jobs:
+                log.info(f"found no jobs on page: {page}")
+                break
+            job_list += jobs
+            page += 1
+        return JobResponse(
+            jobs=job_list[
+                scraper_input.offset : scraper_input.offset
+                + scraper_input.results_wanted
+            ]
+        )
+
+    def _scrape_page(self, cursor: str | None) -> Tuple[list[JobPost], str | None]:
+        """
+        Scrapes a page of Indeed for jobs with scraper_input criteria
+        :param cursor:
+        :return: jobs found on page, next page cursor
+        """
+        jobs = []
+        new_cursor = None
+        filters = self._build_filters()
+        search_term = (
+            self.scraper_input.search_term.replace('"', '\\"')
+            if self.scraper_input.search_term
+            else ""
+        )
+        query = job_search_query.format(
+            what=(f'what: "{search_term}"' if search_term else ""),
+            location=(
+                f'location: {{where: "{self.scraper_input.location}", radius: {self.scraper_input.distance}, radiusUnit: MILES}}'
+                if self.scraper_input.location
+                else ""
+            ),
+            dateOnIndeed=self.scraper_input.hours_old,
+            cursor=f'cursor: "{cursor}"' if cursor else "",
+            filters=filters,
+        )
+        payload = {
+            "query": query,
+        }
+        api_headers_temp = api_headers.copy()
+        api_headers_temp["indeed-co"] = self.api_country_code
+        response = self.session.post(
+            self.api_url,
+            headers=api_headers_temp,
+            json=payload,
+            timeout=10,
+            verify=False,
+        )
+        if not response.ok:
+            log.info(
+                f"responded with status code: {response.status_code} (submit GitHub issue if this appears to be a bug)"
+            )
+            return jobs, new_cursor
+        data = response.json()
+        jobs = data["data"]["jobSearch"]["results"]
+        new_cursor = data["data"]["jobSearch"]["pageInfo"]["nextCursor"]
+
+        job_list = []
+        for job in jobs:
+            processed_job = self._process_job(job["job"])
+            if processed_job:
+                job_list.append(processed_job)
+
+        return job_list, new_cursor
+
+    def _build_filters(self):
+        """
+        Builds the filters dict for job type/is_remote. If hours_old is provided, composite filter for job_type/is_remote is not possible.
+        IndeedApply: filters: { keyword: { field: "indeedApplyScope", keys: ["DESKTOP"] } }
+        """
+        filters_str = ""
+        if self.scraper_input.hours_old:
+            filters_str = """
+            filters: {{
+                date: {{
+                  field: "dateOnIndeed",
+                  start: "{start}h"
+                }}
+            }}
+            """.format(
+                start=self.scraper_input.hours_old
+            )
+        elif self.scraper_input.easy_apply:
+            filters_str = """
+            filters: {
+                keyword: {
+                  field: "indeedApplyScope",
+                  keys: ["DESKTOP"]
+                }
+            }
+            """
+        elif self.scraper_input.job_type or self.scraper_input.is_remote:
+            job_type_key_mapping = {
+                JobType.FULL_TIME: "CF3CP",
+                JobType.PART_TIME: "75GKK",
+                JobType.CONTRACT: "NJXCK",
+                JobType.INTERNSHIP: "VDTG7",
+            }
+
+            keys = []
+            if self.scraper_input.job_type:
+                key = job_type_key_mapping[self.scraper_input.job_type]
+                keys.append(key)
+
+            if self.scraper_input.is_remote:
+                keys.append("DSQF7")
+
+            if keys:
+                keys_str = '", "'.join(keys)
+                filters_str = f"""
+                filters: {{
+                  composite: {{
+                    filters: [{{
+                      keyword: {{
+                        field: "attributes",
+                        keys: ["{keys_str}"]
+                      }}
+                    }}]
+                  }}
+                }}
+                """
+        return filters_str
+
+    def _process_job(self, job: dict) -> JobPost | None:
+        """
+        Parses the job dict into JobPost model
+        :param job: dict to parse
+        :return: JobPost if it's a new job
+        """
+        job_url = f'{self.base_url}/viewjob?jk={job["key"]}'
+        if job_url in self.seen_urls:
+            return
+        self.seen_urls.add(job_url)
+        description = job["description"]["html"]
+        if self.scraper_input.description_format == DescriptionFormat.MARKDOWN:
+            description = markdown_converter(description)
+            description = description.replace(",", "")
+
+
+        job_type = get_job_type(job["attributes"])
+        timestamp_seconds = job["datePublished"] / 1000
+        date_posted = datetime.fromtimestamp(timestamp_seconds).strftime("%Y-%m-%d")
+        employer = job["employer"].get("dossier") if job["employer"] else None
+        employer_details = employer.get("employerDetails", {}) if employer else {}
+        rel_url = job["employer"]["relativeCompanyPageUrl"] if job["employer"] else None
+        return JobPost(
+            id=f'in-{job["key"]}',
+            title=job["title"],
+            description=description,
+            company_name=job["employer"].get("name") if job.get("employer") else None,
+            company_url=(f"{self.base_url}{rel_url}" if job["employer"] else None),
+            company_url_direct=(
+                employer["links"]["corporateWebsite"] if employer else None
+            ),
+            location=Location(
+                city=job.get("location", {}).get("city"),
+                state=job.get("location", {}).get("admin1Code"),
+                country=job.get("location", {}).get("countryCode"),
+            ),
+            job_type=job_type,
+            compensation=get_compensation(job["compensation"]),
+            date_posted=date_posted,
+            job_url=job_url,
+            job_url_direct=(
+                job["recruit"].get("viewJobUrl") if job.get("recruit") else None
+            ),
+            emails=extract_emails_from_text(description) if description else None,
+            is_remote=is_job_remote(job, description),
+            company_addresses=(
+                employer_details["addresses"][0]
+                if employer_details.get("addresses")
+                else None
+            ),
+            company_industry=(
+                employer_details["industry"]
+                .replace("Iv1", "")
+                .replace("_", " ")
+                .title()
+                .strip()
+                if employer_details.get("industry")
+                else None
+            ),
+            company_num_employees=employer_details.get("employeesLocalizedLabel"),
+            company_revenue=employer_details.get("revenueLocalizedLabel"),
+            company_description=employer_details.get("briefDescription"),
+            company_logo=(
+                employer["images"].get("squareLogoUrl")
+                if employer and employer.get("images")
+                else None
+            ),
+        )

jobspy/indeed/constant.py

@@ -0,0 +1,109 @@
+job_search_query = """
+    query GetJobData {{
+        jobSearch(
+        {what}
+        {location}
+        limit: 100
+        {cursor}
+        sort: RELEVANCE
+        {filters}
+        ) {{
+        pageInfo {{
+            nextCursor
+        }}
+        results {{
+            trackingKey
+            job {{
+            source {{
+                name
+            }}
+            key
+            title
+            datePublished
+            dateOnIndeed
+            description {{
+                html
+            }}
+            location {{
+                countryName
+                countryCode
+                admin1Code
+                city
+                postalCode
+                streetAddress
+                formatted {{
+                short
+                long
+                }}
+            }}
+            compensation {{
+                estimated {{
+                currencyCode
+                baseSalary {{
+                    unitOfWork
+                    range {{
+                    ... on Range {{
+                        min
+                        max
+                    }}
+                    }}
+                }}
+                }}
+                baseSalary {{
+                unitOfWork
+                range {{
+                    ... on Range {{
+                    min
+                    max
+                    }}
+                }}
+                }}
+                currencyCode
+            }}
+            attributes {{
+                key
+                label
+            }}
+            employer {{
+                relativeCompanyPageUrl
+                name
+                dossier {{
+                    employerDetails {{
+                    addresses
+                    industry
+                    employeesLocalizedLabel
+                    revenueLocalizedLabel
+                    briefDescription
+                    ceoName
+                    ceoPhotoUrl
+                    }}
+                    images {{
+                        headerImageUrl
+                        squareLogoUrl
+                    }}
+                    links {{
+                    corporateWebsite
+                }}
+                }}
+            }}
+            recruit {{
+                viewJobUrl
+                detailedSalary
+                workSchedule
+            }}
+            }}
+        }}
+        }}
+    }}
+    """
+
+api_headers = {
+    "Host": "apis.indeed.com",
+    "content-type": "application/json",
+    "indeed-api-key": "161092c2017b5bbab13edb12461a62d5a833871e7cad6d9d475304573de67ac8",
+    "accept": "application/json",
+    "indeed-locale": "en-US",
+    "accept-language": "en-US,en;q=0.9",
+    "user-agent": "Mozilla/5.0 (iPhone; CPU iPhone OS 16_6_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148 Indeed App 193.1",
+    "indeed-app-info": "appv=193.1; appid=com.indeed.jobsearch; osv=16.6.1; os=ios; dtype=phone",
+}

jobspy/indeed/util.py

@@ -0,0 +1,80 @@
+from jobspy.model import CompensationInterval, JobType, Compensation
+from jobspy.util import get_enum_from_job_type
+
+
+def get_job_type(attributes: list) -> list[JobType]:
+    """
+    Parses the attributes to get list of job types
+    :param attributes:
+    :return: list of JobType
+    """
+    job_types: list[JobType] = []
+    for attribute in attributes:
+        job_type_str = attribute["label"].replace("-", "").replace(" ", "").lower()
+        job_type = get_enum_from_job_type(job_type_str)
+        if job_type:
+            job_types.append(job_type)
+    return job_types
+
+
+def get_compensation(compensation: dict) -> Compensation | None:
+    """
+    Parses the job to get compensation
+    :param sssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrompensation:
+    :return: compensation object
+    """
+    if not compensation["baseSalary"] and not compensation["estimated"]:
+        return None
+    comp = (
+        compensation["baseSalary"]
+        if compensation["baseSalary"]
+        else compensation["estimated"]["baseSalary"]
+    )
+    if not comp:
+        return None
+    interval = get_compensation_interval(comp["unitOfWork"])
+    if not interval:
+        return None
+    min_range = comp["range"].get("min")
+    max_range = comp["range"].get("max")
+    return Compensation(
+        interval=interval,
+        min_amount=int(min_range) if min_range is not None else None,
+        max_amount=int(max_range) if max_range is not None else None,
+        currency=(
+            compensation["estimated"]["currencyCode"]
+            if compensation["estimated"]
+            else compensation["currencyCode"]
+        ),
+    )
+
+
+def is_job_remote(job: dict, description: str) -> bool:
+    """
+    Searches the description, location, and attributes to check if job is remote
+    """
+    remote_keywords = ["remote", "work from home", "wfh"]
+    is_remote_in_attributes = any(
+        any(keyword in attr["label"].lower() for keyword in remote_keywords)
+        for attr in job["attributes"]
+    )
+    is_remote_in_location = any(
+        keyword in job["location"]["formatted"]["long"].lower()
+        for keyword in remote_keywords
+    )
+    return is_remote_in_attributes or is_remote_in_location
+
+
+def get_compensation_interval(interval: str) -> CompensationInterval:
+    interval_mapping = {
+        "DAY": "DAILY",
+        "YEAR": "YEARLY",
+        "HOUR": "HOURLY",
+        "WEEK": "WEEKLY",
+        "MONTH": "MONTHLY",
+    }
+    mapped_interval = interval_mapping.get(interval.upper(), None)
+    if mapped_interval and mapped_interval in CompensationInterval.__members__:
+        return CompensationInterval[mapped_interval]
+    else:
+        raise ValueError(f"Unsupported interval: {interval}")

jobspy/linkedin/__init__.py

@@ -1,62 +1,69 @@
-"""
-jobspy.scrapers.linkedin
-~~~~~~~~~~~~~~~~~~~
-
-This module contains routines to scrape LinkedIn.
-"""
-
 from __future__ import annotations
 
-import time
+import math
 import random
-import regex as re
-from typing import Optional
+import time
 from datetime import datetime
-
-from bs4.element import Tag
-from bs4 import BeautifulSoup
+from typing import Optional
 from urllib.parse import urlparse, urlunparse, unquote
 
-from .. import Scraper, ScraperInput, Site
-from ..exceptions import LinkedInException
-from ..utils import create_session, remove_attributes
-from ...jobs import (
+import regex as re
+from bs4 import BeautifulSoup
+from bs4.element import Tag
+
+from jobspy.exception import LinkedInException
+from jobspy.linkedin.constant import headers
+from jobspy.linkedin.util import (
+    job_type_code,
+    parse_job_type,
+    parse_job_level,
+    parse_company_industry,
+)
+from jobspy.model import (
     JobPost,
     Location,
     JobResponse,
-    JobType,
     Country,
     Compensation,
     DescriptionFormat,
+    Scraper,
+    ScraperInput,
+    Site,
 )
-from ..utils import (
-    logger,
+from jobspy.util import (
     extract_emails_from_text,
-    get_enum_from_job_type,
     currency_parser,
     markdown_converter,
+    create_session,
+    remove_attributes,
+    create_logger,
 )
 
+log = create_logger("LinkedIn")
 
-class LinkedInScraper(Scraper):
+
+class LinkedIn(Scraper):
     base_url = "https://www.linkedin.com"
     delay = 3
     band_delay = 4
     jobs_per_page = 25
 
-    def __init__(self, proxies: list[str] | str | None = None):
+    def __init__(
+        self, proxies: list[str] | str | None = None, ca_cert: str | None = None
+    ):
         """
         Initializes LinkedInScraper with the LinkedIn job search url
         """
-        super().__init__(Site.LINKEDIN, proxies=proxies)
+        super().__init__(Site.LINKEDIN, proxies=proxies, ca_cert=ca_cert)
         self.session = create_session(
             proxies=self.proxies,
+            ca_cert=ca_cert,
             is_tls=False,
             has_retry=True,
             delay=5,
             clear_cookies=True,
         )
-        self.session.headers.update(self.headers)
+        self.session.headers.update(headers)
         self.scraper_input = None
         self.country = "worldwide"
         self.job_url_direct_regex = re.compile(r'(?<=\?url=)[^"]+')
@@ -70,29 +77,31 @@ class LinkedInScraper(Scraper):
         self.scraper_input = scraper_input
         job_list: list[JobPost] = []
         seen_ids = set()
-        page = scraper_input.offset // 10 * 10 if scraper_input.offset else 0
+        start = scraper_input.offset // 10 * 10 if scraper_input.offset else 0
         request_count = 0
         seconds_old = (
             scraper_input.hours_old * 3600 if scraper_input.hours_old else None
         )
         continue_search = (
-            lambda: len(job_list) < scraper_input.results_wanted and page < 1000
+            lambda: len(job_list) < scraper_input.results_wanted and start < 1000
         )
         while continue_search():
             request_count += 1
-            logger.info(f"LinkedIn search page: {request_count}")
+            log.info(
+                f"search page: {request_count} / {math.ceil(scraper_input.results_wanted / 10)}"
+            )
             params = {
                 "keywords": scraper_input.search_term,
                 "location": scraper_input.location,
                 "distance": scraper_input.distance,
                 "f_WT": 2 if scraper_input.is_remote else None,
                 "f_JT": (
-                    self.job_type_code(scraper_input.job_type)
+                    job_type_code(scraper_input.job_type)
                     if scraper_input.job_type
                     else None
                 ),
                 "pageNum": 0,
-                "start": page,
+                "start": start,
                 "f_AL": "true" if scraper_input.easy_apply else None,
                 "f_C": (
                     ",".join(map(str, scraper_input.linkedin_company_ids))
@@ -118,13 +127,13 @@ class LinkedInScraper(Scraper):
                     else:
                         err = f"LinkedIn response status code {response.status_code}"
                         err += f" - {response.text}"
-                    logger.error(err)
+                    log.error(err)
                     return JobResponse(jobs=job_list)
             except Exception as e:
                 if "Proxy responded with" in str(e):
-                    logger.error(f"LinkedIn: Bad proxy")
+                    log.error(f"LinkedIn: Bad proxy")
                 else:
-                    logger.error(f"LinkedIn: {str(e)}")
+                    log.error(f"LinkedIn: {str(e)}")
                 return JobResponse(jobs=job_list)
 
             soup = BeautifulSoup(response.text, "html.parser")
@@ -154,7 +163,7 @@ class LinkedInScraper(Scraper):
 
             if continue_search():
                 time.sleep(random.uniform(self.delay, self.delay + self.band_delay))
-                page += len(job_list)
+                start += len(job_list)
 
         job_list = job_list[: scraper_input.results_wanted]
         return JobResponse(jobs=job_list)
@@ -208,9 +217,11 @@ class LinkedInScraper(Scraper):
         job_details = {}
         if full_descr:
             job_details = self._get_job_details(job_id)
+            description = description.replace(",", "")
+
 
         return JobPost(
-            id=job_id,
+            id=f"li-{job_id}",
             title=title,
             company_name=company,
             company_url=company_url,
@@ -224,7 +235,7 @@ class LinkedInScraper(Scraper):
             description=job_details.get("description"),
             job_url_direct=job_details.get("job_url_direct"),
             emails=extract_emails_from_text(job_details.get("description")),
-            logo_photo_url=job_details.get("logo_photo_url"),
+            company_logo=job_details.get("company_logo"),
             job_function=job_details.get("job_function"),
         )
 
@@ -266,15 +277,19 @@ class LinkedInScraper(Scraper):
             )
             if job_function_span:
                 job_function = job_function_span.text.strip()
+
+        company_logo = (
+            logo_image.get("data-delayed-url")
+            if (logo_image := soup.find("img", {"class": "artdeco-entity-image"}))
+            else None
+        )
         return {
             "description": description,
-            "job_level": self._parse_job_level(soup),
-            "company_industry": self._parse_company_industry(soup),
-            "job_type": self._parse_job_type(soup),
+            "job_level": parse_job_level(soup),
+            "company_industry": parse_company_industry(soup),
+            "job_type": parse_job_type(soup),
             "job_url_direct": self._parse_job_url_direct(soup),
-            "logo_photo_url": soup.find("img", {"class": "artdeco-entity-image"}).get(
-                "data-delayed-url"
-            ),
+            "company_logo": company_logo,
             "job_function": job_function,
         }
 
@@ -304,77 +319,6 @@ class LinkedInScraper(Scraper):
                 location = Location(city=city, state=state, country=country)
         return location
 
-    @staticmethod
-    def _parse_job_type(soup_job_type: BeautifulSoup) -> list[JobType] | None:
-        """
-        Gets the job type from job page
-        :param soup_job_type:
-        :return: JobType
-        """
-        h3_tag = soup_job_type.find(
-            "h3",
-            class_="description__job-criteria-subheader",
-            string=lambda text: "Employment type" in text,
-        )
-        employment_type = None
-        if h3_tag:
-            employment_type_span = h3_tag.find_next_sibling(
-                "span",
-                class_="description__job-criteria-text description__job-criteria-text--criteria",
-            )
-            if employment_type_span:
-                employment_type = employment_type_span.get_text(strip=True)
-                employment_type = employment_type.lower()
-                employment_type = employment_type.replace("-", "")
-
-        return [get_enum_from_job_type(employment_type)] if employment_type else []
-
-    @staticmethod
-    def _parse_job_level(soup_job_level: BeautifulSoup) -> str | None:
-        """
-        Gets the job level from job page
-        :param soup_job_level:
-        :return: str
-        """
-        h3_tag = soup_job_level.find(
-            "h3",
-            class_="description__job-criteria-subheader",
-            string=lambda text: "Seniority level" in text,
-        )
-        job_level = None
-        if h3_tag:
-            job_level_span = h3_tag.find_next_sibling(
-                "span",
-                class_="description__job-criteria-text description__job-criteria-text--criteria",
-            )
-            if job_level_span:
-                job_level = job_level_span.get_text(strip=True)
-
-        return job_level
-
-    @staticmethod
-    def _parse_company_industry(soup_industry: BeautifulSoup) -> str | None:
-        """
-        Gets the company industry from job page
-        :param soup_industry:
-        :return: str
-        """
-        h3_tag = soup_industry.find(
-            "h3",
-            class_="description__job-criteria-subheader",
-            string=lambda text: "Industries" in text,
-        )
-        industry = None
-        if h3_tag:
-            industry_span = h3_tag.find_next_sibling(
-                "span",
-                class_="description__job-criteria-text description__job-criteria-text--criteria",
-            )
-            if industry_span:
-                industry = industry_span.get_text(strip=True)
-
-        return industry
-
     def _parse_job_url_direct(self, soup: BeautifulSoup) -> str | None:
         """
         Gets the job url direct from job page
@@ -391,22 +335,3 @@ class LinkedInScraper(Scraper):
                 job_url_direct = unquote(job_url_direct_match.group())
 
         return job_url_direct
-
-    @staticmethod
-    def job_type_code(job_type_enum: JobType) -> str:
-        return {
-            JobType.FULL_TIME: "F",
-            JobType.PART_TIME: "P",
-            JobType.INTERNSHIP: "I",
-            JobType.CONTRACT: "C",
-            JobType.TEMPORARY: "T",
-        }.get(job_type_enum, "")
-
-    headers = {
-        "authority": "www.linkedin.com",
-        "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
-        "accept-language": "en-US,en;q=0.9",
-        "cache-control": "max-age=0",
-        "upgrade-insecure-requests": "1",
-        "user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
-    }

jobspy/linkedin/constant.py

@@ -0,0 +1,8 @@
+headers = {
+    "authority": "www.linkedin.com",
+    "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
+    "accept-language": "en-US,en;q=0.9",
+    "cache-control": "max-age=0",
+    "upgrade-insecure-requests": "1",
+    "user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
+}

jobspy/linkedin/util.py

@@ -0,0 +1,85 @@
+from bs4 import BeautifulSoup
+
+from jobspy.model import JobType
+from jobspy.util import get_enum_from_job_type
+
+
+def job_type_code(job_type_enum: JobType) -> str:
+    return {
+        JobType.FULL_TIME: "F",
+        JobType.PART_TIME: "P",
+        JobType.INTERNSHIP: "I",
+        JobType.CONTRACT: "C",
+        JobType.TEMPORARY: "T",
+    }.get(job_type_enum, "")
+
+
+def parse_job_type(soup_job_type: BeautifulSoup) -> list[JobType] | None:
+    """
+    Gets the job type from job page
+    :param soup_job_type:
+    :return: JobType
+    """
+    h3_tag = soup_job_type.find(
+        "h3",
+        class_="description__job-criteria-subheader",
+        string=lambda text: "Employment type" in text,
+    )
+    employment_type = None
+    if h3_tag:
+        employment_type_span = h3_tag.find_next_sibling(
+            "span",
+            class_="description__job-criteria-text description__job-criteria-text--criteria",
+        )
+        if employment_type_span:
+            employment_type = employment_type_span.get_text(strip=True)
+            employment_type = employment_type.lower()
+            employment_type = employment_type.replace("-", "")
+
+    return [get_enum_from_job_type(employment_type)] if employment_type else []
+
+
+def parse_job_level(soup_job_level: BeautifulSoup) -> str | None:
+    """
+    Gets the job level from job page
+    :param soup_job_level:
+    :return: str
+    """
+    h3_tag = soup_job_level.find(
+        "h3",
+        class_="description__job-criteria-subheader",
+        string=lambda text: "Seniority level" in text,
+    )
+    job_level = None
+    if h3_tag:
+        job_level_span = h3_tag.find_next_sibling(
+            "span",
+            class_="description__job-criteria-text description__job-criteria-text--criteria",
+        )
+        if job_level_span:
+            job_level = job_level_span.get_text(strip=True)
+
+    return job_level
+
+
+def parse_company_industry(soup_industry: BeautifulSoup) -> str | None:
+    """
+    Gets the company industry from job page
+    :param soup_industry:
+    :return: str
+    """
+    h3_tag = soup_industry.find(
+        "h3",
+        class_="description__job-criteria-subheader",
+        string=lambda text: "Industries" in text,
+    )
+    industry = None
+    if h3_tag:
+        industry_span = h3_tag.find_next_sibling(
+            "span",
+            class_="description__job-criteria-text description__job-criteria-text--criteria",
+        )
+        if industry_span:
+            industry = industry_span.get_text(strip=True)
+
+    return industry

jobspy/model.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+from abc import ABC, abstractmethod
 from typing import Optional
 from datetime import date
 from enum import Enum
@@ -68,16 +69,20 @@ class Country(Enum):
     AUSTRIA = ("austria", "at", "at")
     BAHRAIN = ("bahrain", "bh")
     BELGIUM = ("belgium", "be", "fr:be")
+    BULGARIA = ("bulgaria", "bg")
     BRAZIL = ("brazil", "br", "com.br")
     CANADA = ("canada", "ca", "ca")
     CHILE = ("chile", "cl")
     CHINA = ("china", "cn")
     COLOMBIA = ("colombia", "co")
     COSTARICA = ("costa rica", "cr")
+    CROATIA = ("croatia", "hr")
+    CYPRUS = ("cyprus", "cy")
     CZECHREPUBLIC = ("czech republic,czechia", "cz")
     DENMARK = ("denmark", "dk")
     ECUADOR = ("ecuador", "ec")
     EGYPT = ("egypt", "eg")
+    ESTONIA = ("estonia", "ee")
     FINLAND = ("finland", "fi")
     FRANCE = ("france", "fr", "fr")
     GERMANY = ("germany", "de", "de")
@@ -91,8 +96,11 @@ class Country(Enum):
     ITALY = ("italy", "it", "it")
     JAPAN = ("japan", "jp")
     KUWAIT = ("kuwait", "kw")
+    LATVIA = ("latvia", "lv")
+    LITHUANIA = ("lithuania", "lt")
     LUXEMBOURG = ("luxembourg", "lu")
     MALAYSIA = ("malaysia", "malaysia:my", "com")
+    MALTA = ("malta", "malta:mt", "mt")
     MEXICO = ("mexico", "mx", "com.mx")
     MOROCCO = ("morocco", "ma")
     NETHERLANDS = ("netherlands", "nl", "nl")
@@ -110,6 +118,8 @@ class Country(Enum):
     ROMANIA = ("romania", "ro")
     SAUDIARABIA = ("saudi arabia", "sa")
     SINGAPORE = ("singapore", "sg", "sg")
+    SLOVAKIA = ("slovakia", "sk")
+    SLOVENIA = ("slovenia", "sl")
     SOUTHAFRICA = ("south africa", "za")
     SOUTHKOREA = ("south korea", "kr")
     SPAIN = ("spain", "es", "es")
@@ -117,7 +127,7 @@ class Country(Enum):
     SWITZERLAND = ("switzerland", "ch", "de:ch")
     TAIWAN = ("taiwan", "tw")
     THAILAND = ("thailand", "th")
-    TURKEY = ("turkey", "tr")
+    TURKEY = ("türkiye,turkey", "tr")
     UKRAINE = ("ukraine", "ua")
     UNITEDARABEMIRATES = ("united arab emirates", "ae")
     UK = ("uk,united kingdom", "uk:gb", "co.uk")
@@ -255,9 +265,7 @@ class JobPost(BaseModel):
     company_num_employees: str | None = None
     company_revenue: str | None = None
     company_description: str | None = None
-    ceo_name: str | None = None
-    ceo_photo_url: str | None = None
-    logo_photo_url: str | None = None
+    company_logo: str | None = None
     banner_photo_url: str | None = None
 
     # linkedin only atm
@@ -266,3 +274,49 @@ class JobPost(BaseModel):
 
 class JobResponse(BaseModel):
     jobs: list[JobPost] = []
+
+
+class Site(Enum):
+    LINKEDIN = "linkedin"
+    INDEED = "indeed"
+    ZIP_RECRUITER = "zip_recruiter"
+    GLASSDOOR = "glassdoor"
+    GOOGLE = "google"
+    BAYT = "bayt"
+
+
+class SalarySource(Enum):
+    DIRECT_DATA = "direct_data"
+    DESCRIPTION = "description"
+
+
+class ScraperInput(BaseModel):
+    site_type: list[Site]
+    search_term: str | None = None
+    google_search_term: str | None = None
+
+    location: str | None = None
+    country: Country | None = Country.USA
+    distance: int | None = None
+    is_remote: bool = False
+    job_type: JobType | None = None
+    easy_apply: bool | None = None
+    offset: int = 0
+    linkedin_fetch_description: bool = False
+    linkedin_company_ids: list[int] | None = None
+    description_format: DescriptionFormat | None = DescriptionFormat.MARKDOWN
+
+    results_wanted: int = 15
+    hours_old: int | None = None
+
+
+class Scraper(ABC):
+    def __init__(
+        self, site: Site, proxies: list[str] | None = None, ca_cert: str | None = None
+    ):
+        self.site = site
+        self.proxies = proxies
+        self.ca_cert = ca_cert
+
+    @abstractmethod
+    def scrape(self, scraper_input: ScraperInput) -> JobResponse: ...

jobspy/util.py

@@ -1,26 +1,32 @@
 from __future__ import annotations
 
-import re
 import logging
+import re
 from itertools import cycle
 
+import numpy as np
 import requests
 import tls_client
-import numpy as np
+import urllib3
 from markdownify import markdownify as md
 from requests.adapters import HTTPAdapter, Retry
 
-from ..jobs import CompensationInterval, JobType
+from jobspy.model import CompensationInterval, JobType, Site
 
-logger = logging.getLogger("JobSpy")
-logger.propagate = False
-if not logger.handlers:
-    logger.setLevel(logging.INFO)
-    console_handler = logging.StreamHandler()
-    format = "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
-    formatter = logging.Formatter(format)
-    console_handler.setFormatter(formatter)
-    logger.addHandler(console_handler)
+urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
+
+
+def create_logger(name: str):
+    logger = logging.getLogger(f"JobSpy:{name}")
+    logger.propagate = False
+    if not logger.handlers:
+        logger.setLevel(logging.INFO)
+        console_handler = logging.StreamHandler()
+        format = "%(asctime)s - %(levelname)s - %(name)s - %(message)s"
+        formatter = logging.Formatter(format)
+        console_handler.setFormatter(formatter)
+        logger.addHandler(console_handler)
+    return logger
 
 
 class RotatingProxySession:
@@ -100,6 +106,7 @@ class TLSRotating(RotatingProxySession, tls_client.Session):
 def create_session(
     *,
     proxies: dict | str | None = None,
+    ca_cert: str | None = None,
     is_tls: bool = True,
     has_retry: bool = False,
     delay: int = 1,
@@ -119,10 +126,13 @@ def create_session(
             clear_cookies=clear_cookies,
         )
 
+    if ca_cert:
+        session.verify = ca_cert
+
     return session
 
 
-def set_logger_level(verbose: int = 2):
+def set_logger_level(verbose: int):
     """
     Adjusts the logger's level. This function allows the logging level to be changed at runtime.
 
@@ -134,7 +144,9 @@ def set_logger_level(verbose: int = 2):
     level_name = {2: "INFO", 1: "WARNING", 0: "ERROR"}.get(verbose, "INFO")
     level = getattr(logging, level_name.upper(), None)
     if level is not None:
-        logger.setLevel(level)
+        for logger_name in logging.root.manager.loggerDict:
+            if logger_name.startswith("JobSpy:"):
+                logging.getLogger(logger_name).setLevel(level)
     else:
         raise ValueError(f"Invalid log level: {level_name}")
 
@@ -195,6 +207,10 @@ def extract_salary(
     monthly_threshold=30000,
     enforce_annual_salary=False,
 ):
+    """
+    Extracts salary information from a string and returns the salary interval, min and max salary values, and currency.
+    (TODO: Needs test cases as the regex is complicated and may not cover all edge cases)
+    """
     if not salary_str:
         return None, None, None, None
 
@@ -251,3 +267,81 @@ def extract_salary(
             else:
                 return interval, min_salary, max_salary, "USD"
     return None, None, None, None
+
+
+def extract_job_type(description: str):
+    if not description:
+        return []
+
+    keywords = {
+        JobType.FULL_TIME: r"full\s?time",
+        JobType.PART_TIME: r"part\s?time",
+        JobType.INTERNSHIP: r"internship",
+        JobType.CONTRACT: r"contract",
+    }
+
+    listing_types = []
+    for key, pattern in keywords.items():
+        if re.search(pattern, description, re.IGNORECASE):
+            listing_types.append(key)
+
+    return listing_types if listing_types else None
+
+
+def map_str_to_site(site_name: str) -> Site:
+    return Site[site_name.upper()]
+
+
+def get_enum_from_value(value_str):
+    for job_type in JobType:
+        if value_str in job_type.value:
+            return job_type
+    raise Exception(f"Invalid job type: {value_str}")
+
+
+def convert_to_annual(job_data: dict):
+    if job_data["interval"] == "hourly":
+        job_data["min_amount"] *= 2080
+        job_data["max_amount"] *= 2080
+    if job_data["interval"] == "monthly":
+        job_data["min_amount"] *= 12
+        job_data["max_amount"] *= 12
+    if job_data["interval"] == "weekly":
+        job_data["min_amount"] *= 52
+        job_data["max_amount"] *= 52
+    if job_data["interval"] == "daily":
+        job_data["min_amount"] *= 260
+        job_data["max_amount"] *= 260
+    job_data["interval"] = "yearly"
+
+
+desired_order = [
+    "id",
+    "site",
+    "job_url",
+    "job_url_direct",
+    "title",
+    "company",
+    "location",
+    "date_posted",
+    "job_type",
+    "salary_source",
+    "interval",
+    "min_amount",
+    "max_amount",
+    "currency",
+    "is_remote",
+    "job_level",
+    "job_function",
+    "listing_type",
+    "emails",
+    "description",
+    "company_industry",
+    "company_url",
+    "company_logo",
+    "company_url_direct",
+    "company_addresses",
+    "company_num_employees",
+    "company_revenue",
+    "company_description",
+]

jobspy/ziprecruiter/__init__.py

@@ -1,54 +1,53 @@
-"""
-jobspy.scrapers.ziprecruiter
-~~~~~~~~~~~~~~~~~~~
-
-This module contains routines to scrape ZipRecruiter.
-"""
-
 from __future__ import annotations
 
 import json
 import math
 import re
 import time
-from datetime import datetime
-from typing import Optional, Tuple, Any
-
 from concurrent.futures import ThreadPoolExecutor
+from datetime import datetime
 
 from bs4 import BeautifulSoup
 
-from .. import Scraper, ScraperInput, Site
-from ..utils import (
-    logger,
+from jobspy.ziprecruiter.constant import headers, get_cookie_data
+from jobspy.util import (
     extract_emails_from_text,
     create_session,
     markdown_converter,
     remove_attributes,
+    create_logger,
 )
-from ...jobs import (
+from jobspy.model import (
     JobPost,
     Compensation,
     Location,
     JobResponse,
-    JobType,
     Country,
     DescriptionFormat,
+    Scraper,
+    ScraperInput,
+    Site,
 )
+from jobspy.ziprecruiter.util import get_job_type_enum, add_params
+
+log = create_logger("ZipRecruiter")
 
 
-class ZipRecruiterScraper(Scraper):
+class ZipRecruiter(Scraper):
     base_url = "https://www.ziprecruiter.com"
     api_url = "https://api.ziprecruiter.com"
 
-    def __init__(self, proxies: list[str] | str | None = None):
+    def __init__(
+        self, proxies: list[str] | str | None = None, ca_cert: str | None = None
+    ):
         """
         Initializes ZipRecruiterScraper with the ZipRecruiter job search url
         """
         super().__init__(Site.ZIP_RECRUITER, proxies=proxies)
 
         self.scraper_input = None
-        self.session = create_session(proxies=proxies)
+        self.session = create_session(proxies=proxies, ca_cert=ca_cert)
+        self.session.headers.update(headers)
         self._get_cookies()
 
         self.delay = 5
@@ -71,7 +70,7 @@ class ZipRecruiterScraper(Scraper):
                 break
             if page > 1:
                 time.sleep(self.delay)
-            logger.info(f"ZipRecruiter search page: {page}")
+            log.info(f"search page: {page} / {max_pages}")
             jobs_on_page, continue_token = self._find_jobs_in_page(
                 scraper_input, continue_token
             )
@@ -85,7 +84,7 @@ class ZipRecruiterScraper(Scraper):
 
     def _find_jobs_in_page(
         self, scraper_input: ScraperInput, continue_token: str | None = None
-    ) -> Tuple[list[JobPost], Optional[str]]:
+    ) -> tuple[list[JobPost], str | None]:
         """
         Scrapes a page of ZipRecruiter for jobs with scraper_input criteria
         :param scraper_input:
@@ -93,26 +92,24 @@ class ZipRecruiterScraper(Scraper):
         :return: jobs found on page
         """
         jobs_list = []
-        params = self._add_params(scraper_input)
+        params = add_params(scraper_input)
         if continue_token:
             params["continue_from"] = continue_token
         try:
-            res = self.session.get(
-                f"{self.api_url}/jobs-app/jobs", headers=self.headers, params=params
-            )
+            res = self.session.get(f"{self.api_url}/jobs-app/jobs", params=params)
             if res.status_code not in range(200, 400):
                 if res.status_code == 429:
                     err = "429 Response - Blocked by ZipRecruiter for too many requests"
                 else:
                     err = f"ZipRecruiter response status code {res.status_code}"
                     err += f" with response: {res.text}"  # ZipRecruiter likely not available in EU
-                logger.error(err)
+                log.error(err)
                 return jobs_list, ""
         except Exception as e:
             if "Proxy responded with" in str(e):
-                logger.error(f"Indeed: Bad proxy")
+                log.error(f"Indeed: Bad proxy")
             else:
-                logger.error(f"Indeed: {str(e)}")
+                log.error(f"Indeed: {str(e)}")
             return jobs_list, ""
 
         res_data = res.json()
@@ -148,7 +145,7 @@ class ZipRecruiterScraper(Scraper):
         location = Location(
             city=job.get("job_city"), state=job.get("job_state"), country=country_enum
         )
-        job_type = self._get_job_type_enum(
+        job_type = get_job_type_enum(
             job.get("employment_type", "").replace("_", "").lower()
         )
         date_posted = datetime.fromisoformat(job["posted_time"].rstrip("Z")).date()
@@ -160,7 +157,7 @@ class ZipRecruiterScraper(Scraper):
         description_full, job_url_direct = self._get_descr(job_url)
 
         return JobPost(
-            id=str(job["listing_key"]),
+            id=f'zr-{job["listing_key"]}',
             title=title,
             company_name=company,
             location=location,
@@ -180,7 +177,7 @@ class ZipRecruiterScraper(Scraper):
         )
 
     def _get_descr(self, job_url):
-        res = self.session.get(job_url, headers=self.headers, allow_redirects=True)
+        res = self.session.get(job_url, allow_redirects=True)
         description_full = job_url_direct = None
         if res.ok:
             soup = BeautifulSoup(res.text, "html.parser")
@@ -197,13 +194,17 @@ class ZipRecruiterScraper(Scraper):
                 else ""
             )
             description_full = job_description_clean + company_description_clean
-            script_tag = soup.find("script", type="application/json")
-            if script_tag:
-                job_json = json.loads(script_tag.string)
-                job_url_val = job_json["model"].get("saveJobURL", "")
-                m = re.search(r"job_url=(.+)", job_url_val)
-                if m:
-                    job_url_direct = m.group(1)
+
+            try:
+                script_tag = soup.find("script", type="application/json")
+                if script_tag:
+                    job_json = json.loads(script_tag.string)
+                    job_url_val = job_json["model"].get("saveJobURL", "")
+                    m = re.search(r"job_url=(.+)", job_url_val)
+                    if m:
+                        job_url_direct = m.group(1)
+            except:
+                job_url_direct = None
 
             if self.scraper_input.description_format == DescriptionFormat.MARKDOWN:
                 description_full = markdown_converter(description_full)
@@ -211,44 +212,8 @@ class ZipRecruiterScraper(Scraper):
         return description_full, job_url_direct
 
     def _get_cookies(self):
-        data = "event_type=session&logged_in=false&number_of_retry=1&property=model%3AiPhone&property=os%3AiOS&property=locale%3Aen_us&property=app_build_number%3A4734&property=app_version%3A91.0&property=manufacturer%3AApple&property=timestamp%3A2024-01-12T12%3A04%3A42-06%3A00&property=screen_height%3A852&property=os_version%3A16.6.1&property=source%3Ainstall&property=screen_width%3A393&property=device_model%3AiPhone%2014%20Pro&property=brand%3AApple"
+        """
+        Sends a session event to the API with device properties.
+        """
         url = f"{self.api_url}/jobs-app/event"
-        self.session.post(url, data=data, headers=self.headers)
-
-    @staticmethod
-    def _get_job_type_enum(job_type_str: str) -> list[JobType] | None:
-        for job_type in JobType:
-            if job_type_str in job_type.value:
-                return [job_type]
-        return None
-
-    @staticmethod
-    def _add_params(scraper_input) -> dict[str, str | Any]:
-        params = {
-            "search": scraper_input.search_term,
-            "location": scraper_input.location,
-        }
-        if scraper_input.hours_old:
-            params["days"] = max(scraper_input.hours_old // 24, 1)
-        job_type_map = {JobType.FULL_TIME: "full_time", JobType.PART_TIME: "part_time"}
-        if scraper_input.job_type:
-            job_type = scraper_input.job_type
-            params["employment_type"] = job_type_map.get(job_type, job_type.value[0])
-        if scraper_input.easy_apply:
-            params["zipapply"] = 1
-        if scraper_input.is_remote:
-            params["remote"] = 1
-        if scraper_input.distance:
-            params["radius"] = scraper_input.distance
-        return {k: v for k, v in params.items() if v is not None}
-
-    headers = {
-        "Host": "api.ziprecruiter.com",
-        "accept": "*/*",
-        "x-zr-zva-override": "100000000;vid:ZT1huzm_EQlDTVEc",
-        "x-pushnotificationid": "0ff4983d38d7fc5b3370297f2bcffcf4b3321c418f5c22dd152a0264707602a0",
-        "x-deviceid": "D77B3A92-E589-46A4-8A39-6EF6F1D86006",
-        "user-agent": "Job Search/87.0 (iPhone; CPU iOS 16_6_1 like Mac OS X)",
-        "authorization": "Basic YTBlZjMyZDYtN2I0Yy00MWVkLWEyODMtYTI1NDAzMzI0YTcyOg==",
-        "accept-language": "en-US,en;q=0.9",
-    }
+        self.session.post(url, data=get_cookie_data)

jobspy/ziprecruiter/constant.py

@@ -0,0 +1,29 @@
+headers = {
+    "Host": "api.ziprecruiter.com",
+    "accept": "*/*",
+    "x-zr-zva-override": "100000000;vid:ZT1huzm_EQlDTVEc",
+    "x-pushnotificationid": "0ff4983d38d7fc5b3370297f2bcffcf4b3321c418f5c22dd152a0264707602a0",
+    "x-deviceid": "D77B3A92-E589-46A4-8A39-6EF6F1D86006",
+    "user-agent": "Job Search/87.0 (iPhone; CPU iOS 16_6_1 like Mac OS X)",
+    "authorization": "Basic YTBlZjMyZDYtN2I0Yy00MWVkLWEyODMtYTI1NDAzMzI0YTcyOg==",
+    "accept-language": "en-US,en;q=0.9",
+}
+
+get_cookie_data = [
+    ("event_type", "session"),
+    ("logged_in", "false"),
+    ("number_of_retry", "1"),
+    ("property", "model:iPhone"),
+    ("property", "os:iOS"),
+    ("property", "locale:en_us"),
+    ("property", "app_build_number:4734"),
+    ("property", "app_version:91.0"),
+    ("property", "manufacturer:Apple"),
+    ("property", "timestamp:2025-01-12T12:04:42-06:00"),
+    ("property", "screen_height:852"),
+    ("property", "os_version:16.6.1"),
+    ("property", "source:install"),
+    ("property", "screen_width:393"),
+    ("property", "device_model:iPhone 14 Pro"),
+    ("property", "brand:Apple"),
+]

jobspy/ziprecruiter/util.py

@@ -0,0 +1,31 @@
+from jobspy.model import JobType
+
+
+def add_params(scraper_input) -> dict[str, str | int]:
+    params: dict[str, str | int] = {
+        "search": scraper_input.search_term,
+        "location": scraper_input.location,
+    }
+    if scraper_input.hours_old:
+        params["days"] = max(scraper_input.hours_old // 24, 1)
+
+    job_type_map = {JobType.FULL_TIME: "full_time", JobType.PART_TIME: "part_time"}
+    if scraper_input.job_type:
+        job_type = scraper_input.job_type
+        params["employment_type"] = job_type_map.get(job_type, job_type.value[0])
+
+    if scraper_input.easy_apply:
+        params["zipapply"] = 1
+    if scraper_input.is_remote:
+        params["remote"] = 1
+    if scraper_input.distance:
+        params["radius"] = scraper_input.distance
+
+    return {k: v for k, v in params.items() if v is not None}
+
+
+def get_job_type_enum(job_type_str: str) -> list[JobType] | None:
+    for job_type in JobType:
+        if job_type_str in job_type.value:
+            return [job_type]
+    return None

poetry.toml

@@ -1,2 +0,0 @@
-[virtualenvs]
-in-project = true

pyproject.toml

@@ -1,37 +1,33 @@
+[build-system]
+requires = [ "poetry-core",]
+build-backend = "poetry.core.masonry.api"
+
 [tool.poetry]
 name = "python-jobspy"
-version = "1.1.66"
-description = "Job scraper for LinkedIn, Indeed, Glassdoor & ZipRecruiter"
-authors = ["Zachary Hampton <zachary@bunsly.com>", "Cullen Watson <cullen@bunsly.com>"]
-homepage = "https://github.com/Bunsly/JobSpy"
+version = "1.1.78"
+description = "Job scraper for LinkedIn, Indeed, Glassdoor, ZipRecruiter & Bayt"
+authors = ["Cullen Watson <cullen@cullenwatson.com>", "Zachary Hampton <zachary@zacharysproducts.com>"]
+homepage = "https://github.com/cullenwatson/JobSpy"
 readme = "README.md"
-keywords = ['jobs-scraper', 'linkedin', 'indeed', 'glassdoor', 'ziprecruiter']
+keywords = [ "jobs-scraper", "linkedin", "indeed", "glassdoor", "ziprecruiter", "bayt"]
+[[tool.poetry.packages]]
+include = "jobspy"
 
-packages = [
-    { include = "jobspy", from = "src" }
-]
+[tool.black]
+line-length = 88
 
 [tool.poetry.dependencies]
-python = "^3.10"
+python = "^3.10 || ^3.12"
 requests = "^2.31.0"
 beautifulsoup4 = "^4.12.2"
 pandas = "^2.1.0"
 NUMPY = "1.26.3"
 pydantic = "^2.3.0"
 tls-client = "^1.0.1"
-markdownify = "^0.11.6"
+markdownify = "^0.13.1"
 regex = "^2024.4.28"
 
-
 [tool.poetry.group.dev.dependencies]
-pytest = "^7.4.1"
 jupyter = "^1.0.0"
 black = "*"
 pre-commit = "*"
-
-[build-system]
-requires = ["poetry-core"]
-build-backend = "poetry.core.masonry.api"
-
-[tool.black]
-line-length = 88

requirements.txt

@@ -0,0 +1,118 @@
+annotated-types==0.7.0
+anyio==4.6.2.post1
+argon2-cffi==23.1.0
+argon2-cffi-bindings==21.2.0
+arrow==1.3.0
+asttokens==2.4.1
+async-lru==2.0.4
+attrs==24.2.0
+babel==2.16.0
+beautifulsoup4==4.12.3
+black==24.10.0
+bleach==6.1.0
+certifi==2024.8.30
+cffi==1.17.1
+cfgv==3.4.0
+charset-normalizer==3.4.0
+click==8.1.7
+comm==0.2.2
+debugpy==1.8.7
+decorator==5.1.1
+defusedxml==0.7.1
+distlib==0.3.9
+executing==2.1.0
+fastjsonschema==2.20.0
+filelock==3.16.1
+fqdn==1.5.1
+h11==0.14.0
+httpcore==1.0.6
+httpx==0.27.2
+identify==2.6.1
+idna==3.10
+ipykernel==6.29.5
+ipython==8.28.0
+ipywidgets==8.1.5
+isoduration==20.11.0
+jedi==0.19.1
+Jinja2==3.1.4
+json5==0.9.25
+jsonpointer==3.0.0
+jsonschema==4.23.0
+jsonschema-specifications==2024.10.1
+jupyter==1.1.1
+jupyter-console==6.6.3
+jupyter-events==0.10.0
+jupyter-lsp==2.2.5
+jupyter_client==8.6.3
+jupyter_core==5.7.2
+jupyter_server==2.14.2
+jupyter_server_terminals==0.5.3
+jupyterlab==4.2.5
+jupyterlab_pygments==0.3.0
+jupyterlab_server==2.27.3
+jupyterlab_widgets==3.0.13
+markdownify==0.13.1
+MarkupSafe==3.0.2
+matplotlib-inline==0.1.7
+mistune==3.0.2
+mypy-extensions==1.0.0
+nbclient==0.10.0
+nbconvert==7.16.4
+nbformat==5.10.4
+nest-asyncio==1.6.0
+nodeenv==1.9.1
+notebook==7.2.2
+notebook_shim==0.2.4
+numpy==1.26.3
+overrides==7.7.0
+packaging==24.1
+pandas==2.2.3
+pandocfilters==1.5.1
+parso==0.8.4
+pathspec==0.12.1
+pexpect==4.9.0
+platformdirs==4.3.6
+pre_commit==4.0.1
+prometheus_client==0.21.0
+prompt_toolkit==3.0.48
+psutil==6.1.0
+ptyprocess==0.7.0
+pure_eval==0.2.3
+pycparser==2.22
+pydantic==2.9.2
+pydantic_core==2.23.4
+Pygments==2.18.0
+python-dateutil==2.9.0.post0
+-e git+https://github.com/fakebranden/JobSpy@60819a8fcabbd3eaba7741b673023612dc3d3692#egg=python_jobspy
+python-json-logger==2.0.7
+pytz==2024.2
+PyYAML==6.0.2
+pyzmq==26.2.0
+referencing==0.35.1
+regex==2024.9.11
+requests==2.32.3
+rfc3339-validator==0.1.4
+rfc3986-validator==0.1.1
+rpds-py==0.20.0
+Send2Trash==1.8.3
+setuptools==75.2.0
+six==1.16.0
+sniffio==1.3.1
+soupsieve==2.6
+stack-data==0.6.3
+terminado==0.18.1
+tinycss2==1.3.0
+tls-client==1.0.1
+tornado==6.4.1
+traitlets==5.14.3
+types-python-dateutil==2.9.0.20241003
+typing_extensions==4.12.2
+tzdata==2024.2
+uri-template==1.3.0
+urllib3==2.2.3
+virtualenv==20.27.0
+wcwidth==0.2.13
+webcolors==24.8.0
+webencodings==0.5.1
+websocket-client==1.8.0
+widgetsnbextension==4.0.13

src/jobspy/scrapers/__init__.py

@@ -1,50 +0,0 @@
-from __future__ import annotations
-
-from abc import ABC, abstractmethod
-
-from ..jobs import (
-    Enum,
-    BaseModel,
-    JobType,
-    JobResponse,
-    Country,
-    DescriptionFormat,
-)
-
-
-class Site(Enum):
-    LINKEDIN = "linkedin"
-    INDEED = "indeed"
-    ZIP_RECRUITER = "zip_recruiter"
-    GLASSDOOR = "glassdoor"
-
-class SalarySource(Enum):
-    DIRECT_DATA = "direct_data"
-    DESCRIPTION = "description"
-
-class ScraperInput(BaseModel):
-    site_type: list[Site]
-    search_term: str | None = None
-
-    location: str | None = None
-    country: Country | None = Country.USA
-    distance: int | None = None
-    is_remote: bool = False
-    job_type: JobType | None = None
-    easy_apply: bool | None = None
-    offset: int = 0
-    linkedin_fetch_description: bool = False
-    linkedin_company_ids: list[int] | None = None
-    description_format: DescriptionFormat | None = DescriptionFormat.MARKDOWN
-
-    results_wanted: int = 15
-    hours_old: int | None = None
-
-
-class Scraper(ABC):
-    def __init__(self, site: Site, proxies: list[str] | None = None):
-        self.proxies = proxies
-        self.site = site
-
-    @abstractmethod
-    def scrape(self, scraper_input: ScraperInput) -> JobResponse: ...

src/jobspy/scrapers/glassdoor/__init__.py

@@ -1,545 +0,0 @@
-"""
-jobspy.scrapers.glassdoor
-~~~~~~~~~~~~~~~~~~~
-
-This module contains routines to scrape Glassdoor.
-"""
-
-from __future__ import annotations
-
-import re
-import json
-import requests
-from typing import Optional, Tuple
-from datetime import datetime, timedelta
-from concurrent.futures import ThreadPoolExecutor, as_completed
-
-from .. import Scraper, ScraperInput, Site
-from ..utils import extract_emails_from_text
-from ..exceptions import GlassdoorException
-from ..utils import (
-    create_session,
-    markdown_converter,
-    logger,
-)
-from ...jobs import (
-    JobPost,
-    Compensation,
-    CompensationInterval,
-    Location,
-    JobResponse,
-    JobType,
-    DescriptionFormat,
-)
-
-
-class GlassdoorScraper(Scraper):
-    def __init__(self, proxies: list[str] | str | None = None):
-        """
-        Initializes GlassdoorScraper with the Glassdoor job search url
-        """
-        site = Site(Site.GLASSDOOR)
-        super().__init__(site, proxies=proxies)
-
-        self.base_url = None
-        self.country = None
-        self.session = None
-        self.scraper_input = None
-        self.jobs_per_page = 30
-        self.max_pages = 30
-        self.seen_urls = set()
-
-    def scrape(self, scraper_input: ScraperInput) -> JobResponse:
-        """
-        Scrapes Glassdoor for jobs with scraper_input criteria.
-        :param scraper_input: Information about job search criteria.
-        :return: JobResponse containing a list of jobs.
-        """
-        self.scraper_input = scraper_input
-        self.scraper_input.results_wanted = min(900, scraper_input.results_wanted)
-        self.base_url = self.scraper_input.country.get_glassdoor_url()
-
-        self.session = create_session(proxies=self.proxies, is_tls=True, has_retry=True)
-        token = self._get_csrf_token()
-        self.headers["gd-csrf-token"] = token if token else self.fallback_token
-
-        location_id, location_type = self._get_location(
-            scraper_input.location, scraper_input.is_remote
-        )
-        if location_type is None:
-            logger.error("Glassdoor: location not parsed")
-            return JobResponse(jobs=[])
-        job_list: list[JobPost] = []
-        cursor = None
-
-        range_start = 1 + (scraper_input.offset // self.jobs_per_page)
-        tot_pages = (scraper_input.results_wanted // self.jobs_per_page) + 2
-        range_end = min(tot_pages, self.max_pages + 1)
-        for page in range(range_start, range_end):
-            logger.info(f"Glassdoor search page: {page}")
-            try:
-                jobs, cursor = self._fetch_jobs_page(
-                    scraper_input, location_id, location_type, page, cursor
-                )
-                job_list.extend(jobs)
-                if not jobs or len(job_list) >= scraper_input.results_wanted:
-                    job_list = job_list[: scraper_input.results_wanted]
-                    break
-            except Exception as e:
-                logger.error(f"Glassdoor: {str(e)}")
-                break
-        return JobResponse(jobs=job_list)
-
-    def _fetch_jobs_page(
-        self,
-        scraper_input: ScraperInput,
-        location_id: int,
-        location_type: str,
-        page_num: int,
-        cursor: str | None,
-    ) -> Tuple[list[JobPost], str | None]:
-        """
-        Scrapes a page of Glassdoor for jobs with scraper_input criteria
-        """
-        jobs = []
-        self.scraper_input = scraper_input
-        try:
-            payload = self._add_payload(location_id, location_type, page_num, cursor)
-            response = self.session.post(
-                f"{self.base_url}/graph",
-                headers=self.headers,
-                timeout_seconds=15,
-                data=payload,
-            )
-            if response.status_code != 200:
-                exc_msg = f"bad response status code: {response.status_code}"
-                raise GlassdoorException(exc_msg)
-            res_json = response.json()[0]
-            if "errors" in res_json:
-                raise ValueError("Error encountered in API response")
-        except (
-            requests.exceptions.ReadTimeout,
-            GlassdoorException,
-            ValueError,
-            Exception,
-        ) as e:
-            logger.error(f"Glassdoor: {str(e)}")
-            return jobs, None
-
-        jobs_data = res_json["data"]["jobListings"]["jobListings"]
-
-        with ThreadPoolExecutor(max_workers=self.jobs_per_page) as executor:
-            future_to_job_data = {
-                executor.submit(self._process_job, job): job for job in jobs_data
-            }
-            for future in as_completed(future_to_job_data):
-                try:
-                    job_post = future.result()
-                    if job_post:
-                        jobs.append(job_post)
-                except Exception as exc:
-                    raise GlassdoorException(f"Glassdoor generated an exception: {exc}")
-
-        return jobs, self.get_cursor_for_page(
-            res_json["data"]["jobListings"]["paginationCursors"], page_num + 1
-        )
-
-    def _get_csrf_token(self):
-        """
-        Fetches csrf token needed for API by visiting a generic page
-        """
-        res = self.session.get(
-            f"{self.base_url}/Job/computer-science-jobs.htm", headers=self.headers
-        )
-        pattern = r'"token":\s*"([^"]+)"'
-        matches = re.findall(pattern, res.text)
-        token = None
-        if matches:
-            token = matches[0]
-        return token
-
-    def _process_job(self, job_data):
-        """
-        Processes a single job and fetches its description.
-        """
-        job_id = job_data["jobview"]["job"]["listingId"]
-        job_url = f"{self.base_url}job-listing/j?jl={job_id}"
-        if job_url in self.seen_urls:
-            return None
-        self.seen_urls.add(job_url)
-        job = job_data["jobview"]
-        title = job["job"]["jobTitleText"]
-        company_name = job["header"]["employerNameFromSearch"]
-        company_id = job_data["jobview"]["header"]["employer"]["id"]
-        location_name = job["header"].get("locationName", "")
-        location_type = job["header"].get("locationType", "")
-        age_in_days = job["header"].get("ageInDays")
-        is_remote, location = False, None
-        date_diff = (datetime.now() - timedelta(days=age_in_days)).date()
-        date_posted = date_diff if age_in_days is not None else None
-
-        if location_type == "S":
-            is_remote = True
-        else:
-            location = self.parse_location(location_name)
-
-        compensation = self.parse_compensation(job["header"])
-        try:
-            description = self._fetch_job_description(job_id)
-        except:
-            description = None
-        company_url = f"{self.base_url}Overview/W-EI_IE{company_id}.htm"
-        company_logo = (
-            job_data["jobview"].get("overview", {}).get("squareLogoUrl", None)
-        )
-        listing_type = (
-            job_data["jobview"]
-            .get("header", {})
-            .get("adOrderSponsorshipLevel", "")
-            .lower()
-        )
-        return JobPost(
-            id=str(job_id),
-            title=title,
-            company_url=company_url if company_id else None,
-            company_name=company_name,
-            date_posted=date_posted,
-            job_url=job_url,
-            location=location,
-            compensation=compensation,
-            is_remote=is_remote,
-            description=description,
-            emails=extract_emails_from_text(description) if description else None,
-            logo_photo_url=company_logo,
-            listing_type=listing_type,
-        )
-
-    def _fetch_job_description(self, job_id):
-        """
-        Fetches the job description for a single job ID.
-        """
-        url = f"{self.base_url}/graph"
-        body = [
-            {
-                "operationName": "JobDetailQuery",
-                "variables": {
-                    "jl": job_id,
-                    "queryString": "q",
-                    "pageTypeEnum": "SERP",
-                },
-                "query": """
-                query JobDetailQuery($jl: Long!, $queryString: String, $pageTypeEnum: PageTypeEnum) {
-                    jobview: jobView(
-                        listingId: $jl
-                        contextHolder: {queryString: $queryString, pageTypeEnum: $pageTypeEnum}
-                    ) {
-                        job {
-                            description
-                            __typename
-                        }
-                        __typename
-                    }
-                }
-                """,
-            }
-        ]
-        res = requests.post(url, json=body, headers=self.headers)
-        if res.status_code != 200:
-            return None
-        data = res.json()[0]
-        desc = data["data"]["jobview"]["job"]["description"]
-        if self.scraper_input.description_format == DescriptionFormat.MARKDOWN:
-            desc = markdown_converter(desc)
-        return desc
-
-    def _get_location(self, location: str, is_remote: bool) -> (int, str):
-        if not location or is_remote:
-            return "11047", "STATE"  # remote options
-        url = f"{self.base_url}/findPopularLocationAjax.htm?maxLocationsToReturn=10&term={location}"
-        res = self.session.get(url, headers=self.headers)
-        if res.status_code != 200:
-            if res.status_code == 429:
-                err = f"429 Response - Blocked by Glassdoor for too many requests"
-                logger.error(err)
-                return None, None
-            else:
-                err = f"Glassdoor response status code {res.status_code}"
-                err += f" - {res.text}"
-                logger.error(f"Glassdoor response status code {res.status_code}")
-                return None, None
-        items = res.json()
-
-        if not items:
-            raise ValueError(f"Location '{location}' not found on Glassdoor")
-        location_type = items[0]["locationType"]
-        if location_type == "C":
-            location_type = "CITY"
-        elif location_type == "S":
-            location_type = "STATE"
-        elif location_type == "N":
-            location_type = "COUNTRY"
-        return int(items[0]["locationId"]), location_type
-
-    def _add_payload(
-        self,
-        location_id: int,
-        location_type: str,
-        page_num: int,
-        cursor: str | None = None,
-    ) -> str:
-        fromage = None
-        if self.scraper_input.hours_old:
-            fromage = max(self.scraper_input.hours_old // 24, 1)
-        filter_params = []
-        if self.scraper_input.easy_apply:
-            filter_params.append({"filterKey": "applicationType", "values": "1"})
-        if fromage:
-            filter_params.append({"filterKey": "fromAge", "values": str(fromage)})
-        payload = {
-            "operationName": "JobSearchResultsQuery",
-            "variables": {
-                "excludeJobListingIds": [],
-                "filterParams": filter_params,
-                "keyword": self.scraper_input.search_term,
-                "numJobsToShow": 30,
-                "locationType": location_type,
-                "locationId": int(location_id),
-                "parameterUrlInput": f"IL.0,12_I{location_type}{location_id}",
-                "pageNumber": page_num,
-                "pageCursor": cursor,
-                "fromage": fromage,
-                "sort": "date",
-            },
-            "query": self.query_template,
-        }
-        if self.scraper_input.job_type:
-            payload["variables"]["filterParams"].append(
-                {"filterKey": "jobType", "values": self.scraper_input.job_type.value[0]}
-            )
-        return json.dumps([payload])
-
-    @staticmethod
-    def parse_compensation(data: dict) -> Optional[Compensation]:
-        pay_period = data.get("payPeriod")
-        adjusted_pay = data.get("payPeriodAdjustedPay")
-        currency = data.get("payCurrency", "USD")
-        if not pay_period or not adjusted_pay:
-            return None
-
-        interval = None
-        if pay_period == "ANNUAL":
-            interval = CompensationInterval.YEARLY
-        elif pay_period:
-            interval = CompensationInterval.get_interval(pay_period)
-        min_amount = int(adjusted_pay.get("p10") // 1)
-        max_amount = int(adjusted_pay.get("p90") // 1)
-        return Compensation(
-            interval=interval,
-            min_amount=min_amount,
-            max_amount=max_amount,
-            currency=currency,
-        )
-
-    @staticmethod
-    def get_job_type_enum(job_type_str: str) -> list[JobType] | None:
-        for job_type in JobType:
-            if job_type_str in job_type.value:
-                return [job_type]
-
-    @staticmethod
-    def parse_location(location_name: str) -> Location | None:
-        if not location_name or location_name == "Remote":
-            return
-        city, _, state = location_name.partition(", ")
-        return Location(city=city, state=state)
-
-    @staticmethod
-    def get_cursor_for_page(pagination_cursors, page_num):
-        for cursor_data in pagination_cursors:
-            if cursor_data["pageNumber"] == page_num:
-                return cursor_data["cursor"]
-
-    fallback_token = "Ft6oHEWlRZrxDww95Cpazw:0pGUrkb2y3TyOpAIqF2vbPmUXoXVkD3oEGDVkvfeCerceQ5-n8mBg3BovySUIjmCPHCaW0H2nQVdqzbtsYqf4Q:wcqRqeegRUa9MVLJGyujVXB7vWFPjdaS1CtrrzJq-ok"
-    headers = {
-        "authority": "www.glassdoor.com",
-        "accept": "*/*",
-        "accept-language": "en-US,en;q=0.9",
-        "apollographql-client-name": "job-search-next",
-        "apollographql-client-version": "4.65.5",
-        "content-type": "application/json",
-        "origin": "https://www.glassdoor.com",
-        "referer": "https://www.glassdoor.com/",
-        "sec-ch-ua": '"Chromium";v="118", "Google Chrome";v="118", "Not=A?Brand";v="99"',
-        "sec-ch-ua-mobile": "?0",
-        "sec-ch-ua-platform": '"macOS"',
-        "sec-fetch-dest": "empty",
-        "sec-fetch-mode": "cors",
-        "sec-fetch-site": "same-origin",
-        "user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36",
-    }
-    query_template = """
-            query JobSearchResultsQuery(
-                $excludeJobListingIds: [Long!], 
-                $keyword: String, 
-                $locationId: Int, 
-                $locationType: LocationTypeEnum, 
-                $numJobsToShow: Int!, 
-                $pageCursor: String, 
-                $pageNumber: Int, 
-                $filterParams: [FilterParams], 
-                $originalPageUrl: String, 
-                $seoFriendlyUrlInput: String, 
-                $parameterUrlInput: String, 
-                $seoUrl: Boolean
-            ) {
-                jobListings(
-                    contextHolder: {
-                        searchParams: {
-                            excludeJobListingIds: $excludeJobListingIds, 
-                            keyword: $keyword, 
-                            locationId: $locationId, 
-                            locationType: $locationType, 
-                            numPerPage: $numJobsToShow, 
-                            pageCursor: $pageCursor, 
-                            pageNumber: $pageNumber, 
-                            filterParams: $filterParams, 
-                            originalPageUrl: $originalPageUrl, 
-                            seoFriendlyUrlInput: $seoFriendlyUrlInput, 
-                            parameterUrlInput: $parameterUrlInput, 
-                            seoUrl: $seoUrl, 
-                            searchType: SR
-                        }
-                    }
-                ) {
-                    companyFilterOptions {
-                        id
-                        shortName
-                        __typename
-                    }
-                    filterOptions
-                    indeedCtk
-                    jobListings {
-                        ...JobView
-                        __typename
-                    }
-                    jobListingSeoLinks {
-                        linkItems {
-                            position
-                            url
-                            __typename
-                        }
-                        __typename
-                    }
-                    jobSearchTrackingKey
-                    jobsPageSeoData {
-                        pageMetaDescription
-                        pageTitle
-                        __typename
-                    }
-                    paginationCursors {
-                        cursor
-                        pageNumber
-                        __typename
-                    }
-                    indexablePageForSeo
-                    searchResultsMetadata {
-                        searchCriteria {
-                            implicitLocation {
-                                id
-                                localizedDisplayName
-                                type
-                                __typename
-                            }
-                            keyword
-                            location {
-                                id
-                                shortName
-                                localizedShortName
-                                localizedDisplayName
-                                type
-                                __typename
-                            }
-                            __typename
-                        }
-                        helpCenterDomain
-                        helpCenterLocale
-                        jobSerpJobOutlook {
-                            occupation
-                            paragraph
-                            __typename
-                        }
-                        showMachineReadableJobs
-                        __typename
-                    }
-                    totalJobsCount
-                    __typename
-                }
-            }
-
-            fragment JobView on JobListingSearchResult {
-                jobview {
-                    header {
-                        adOrderId
-                        advertiserType
-                        adOrderSponsorshipLevel
-                        ageInDays
-                        divisionEmployerName
-                        easyApply
-                        employer {
-                            id
-                            name
-                            shortName
-                            __typename
-                        }
-                        employerNameFromSearch
-                        goc
-                        gocConfidence
-                        gocId
-                        jobCountryId
-                        jobLink
-                        jobResultTrackingKey
-                        jobTitleText
-                        locationName
-                        locationType
-                        locId
-                        needsCommission
-                        payCurrency
-                        payPeriod
-                        payPeriodAdjustedPay {
-                            p10
-                            p50
-                            p90
-                            __typename
-                        }
-                        rating
-                        salarySource
-                        savedJobId
-                        sponsored
-                        __typename
-                    }
-                    job {
-                        description
-                        importConfigId
-                        jobTitleId
-                        jobTitleText
-                        listingId
-                        __typename
-                    }
-                    jobListingAdminDetails {
-                        cpcVal
-                        importConfigId
-                        jobListingId
-                        jobSourceId
-                        userEligibleForAdminJobDetails
-                        __typename
-                    }
-                    overview {
-                        shortName
-                        squareLogoUrl
-                        __typename
-                    }
-                    __typename
-                }
-                __typename
-            }
-    """

src/jobspy/scrapers/indeed/__init__.py

@@ -1,458 +0,0 @@
-"""
-jobspy.scrapers.indeed
-~~~~~~~~~~~~~~~~~~~
-
-This module contains routines to scrape Indeed.
-"""
-
-from __future__ import annotations
-
-import math
-from typing import Tuple
-from datetime import datetime
-from concurrent.futures import ThreadPoolExecutor, Future
-
-from .. import Scraper, ScraperInput, Site
-from ..utils import (
-    extract_emails_from_text,
-    get_enum_from_job_type,
-    markdown_converter,
-    logger,
-    create_session,
-)
-from ...jobs import (
-    JobPost,
-    Compensation,
-    CompensationInterval,
-    Location,
-    JobResponse,
-    JobType,
-    DescriptionFormat,
-)
-
-
-class IndeedScraper(Scraper):
-    def __init__(self, proxies: list[str] | str | None = None):
-        """
-        Initializes IndeedScraper with the Indeed API url
-        """
-        super().__init__(Site.INDEED, proxies=proxies)
-
-        self.session = create_session(proxies=self.proxies, is_tls=False)
-        self.scraper_input = None
-        self.jobs_per_page = 100
-        self.num_workers = 10
-        self.seen_urls = set()
-        self.headers = None
-        self.api_country_code = None
-        self.base_url = None
-        self.api_url = "https://apis.indeed.com/graphql"
-
-    def scrape(self, scraper_input: ScraperInput) -> JobResponse:
-        """
-        Scrapes Indeed for jobs with scraper_input criteria
-        :param scraper_input:
-        :return: job_response
-        """
-        self.scraper_input = scraper_input
-        domain, self.api_country_code = self.scraper_input.country.indeed_domain_value
-        self.base_url = f"https://{domain}.indeed.com"
-        self.headers = self.api_headers.copy()
-        self.headers["indeed-co"] = self.scraper_input.country.indeed_domain_value
-        job_list = []
-        page = 1
-
-        cursor = None
-        offset_pages = math.ceil(self.scraper_input.offset / 100)
-        for _ in range(offset_pages):
-            logger.info(f"Indeed skipping search page: {page}")
-            __, cursor = self._scrape_page(cursor)
-            if not __:
-                logger.info(f"Indeed found no jobs on page: {page}")
-                break
-
-        while len(self.seen_urls) < scraper_input.results_wanted:
-            logger.info(f"Indeed search page: {page}")
-            jobs, cursor = self._scrape_page(cursor)
-            if not jobs:
-                logger.info(f"Indeed found no jobs on page: {page}")
-                break
-            job_list += jobs
-            page += 1
-        return JobResponse(jobs=job_list[: scraper_input.results_wanted])
-
-    def _scrape_page(self, cursor: str | None) -> Tuple[list[JobPost], str | None]:
-        """
-        Scrapes a page of Indeed for jobs with scraper_input criteria
-        :param cursor:
-        :return: jobs found on page, next page cursor
-        """
-        jobs = []
-        new_cursor = None
-        filters = self._build_filters()
-        search_term = (
-            self.scraper_input.search_term.replace('"', '\\"')
-            if self.scraper_input.search_term
-            else ""
-        )
-        query = self.job_search_query.format(
-            what=(f'what: "{search_term}"' if search_term else ""),
-            location=(
-                f'location: {{where: "{self.scraper_input.location}", radius: {self.scraper_input.distance}, radiusUnit: MILES}}'
-                if self.scraper_input.location
-                else ""
-            ),
-            dateOnIndeed=self.scraper_input.hours_old,
-            cursor=f'cursor: "{cursor}"' if cursor else "",
-            filters=filters,
-        )
-        payload = {
-            "query": query,
-        }
-        api_headers = self.api_headers.copy()
-        api_headers["indeed-co"] = self.api_country_code
-        response = self.session.post(
-            self.api_url,
-            headers=api_headers,
-            json=payload,
-            timeout=10,
-        )
-        if response.status_code != 200:
-            logger.info(
-                f"Indeed responded with status code: {response.status_code} (submit GitHub issue if this appears to be a bug)"
-            )
-            return jobs, new_cursor
-        data = response.json()
-        jobs = data["data"]["jobSearch"]["results"]
-        new_cursor = data["data"]["jobSearch"]["pageInfo"]["nextCursor"]
-
-        with ThreadPoolExecutor(max_workers=self.num_workers) as executor:
-            job_results: list[Future] = [
-                executor.submit(self._process_job, job["job"]) for job in jobs
-            ]
-        job_list = [result.result() for result in job_results if result.result()]
-        return job_list, new_cursor
-
-    def _build_filters(self):
-        """
-        Builds the filters dict for job type/is_remote. If hours_old is provided, composite filter for job_type/is_remote is not possible.
-        IndeedApply: filters: { keyword: { field: "indeedApplyScope", keys: ["DESKTOP"] } }
-        """
-        filters_str = ""
-        if self.scraper_input.hours_old:
-            filters_str = """
-            filters: {{
-                date: {{
-                  field: "dateOnIndeed",
-                  start: "{start}h"
-                }}
-            }}
-            """.format(
-                start=self.scraper_input.hours_old
-            )
-        elif self.scraper_input.easy_apply:
-            filters_str = """
-            filters: {
-                keyword: {
-                  field: "indeedApplyScope",
-                  keys: ["DESKTOP"]
-                }
-            }
-            """
-        elif self.scraper_input.job_type or self.scraper_input.is_remote:
-            job_type_key_mapping = {
-                JobType.FULL_TIME: "CF3CP",
-                JobType.PART_TIME: "75GKK",
-                JobType.CONTRACT: "NJXCK",
-                JobType.INTERNSHIP: "VDTG7",
-            }
-
-            keys = []
-            if self.scraper_input.job_type:
-                key = job_type_key_mapping[self.scraper_input.job_type]
-                keys.append(key)
-
-            if self.scraper_input.is_remote:
-                keys.append("DSQF7")
-
-            if keys:
-                keys_str = '", "'.join(keys)
-                filters_str = f"""
-                filters: {{
-                  composite: {{
-                    filters: [{{
-                      keyword: {{
-                        field: "attributes",
-                        keys: ["{keys_str}"]
-                      }}
-                    }}]
-                  }}
-                }}
-                """
-        return filters_str
-
-    def _process_job(self, job: dict) -> JobPost | None:
-        """
-        Parses the job dict into JobPost model
-        :param job: dict to parse
-        :return: JobPost if it's a new job
-        """
-        job_url = f'{self.base_url}/viewjob?jk={job["key"]}'
-        if job_url in self.seen_urls:
-            return
-        self.seen_urls.add(job_url)
-        description = job["description"]["html"]
-        if self.scraper_input.description_format == DescriptionFormat.MARKDOWN:
-            description = markdown_converter(description)
-
-        job_type = self._get_job_type(job["attributes"])
-        timestamp_seconds = job["datePublished"] / 1000
-        date_posted = datetime.fromtimestamp(timestamp_seconds).strftime("%Y-%m-%d")
-        employer = job["employer"].get("dossier") if job["employer"] else None
-        employer_details = employer.get("employerDetails", {}) if employer else {}
-        rel_url = job["employer"]["relativeCompanyPageUrl"] if job["employer"] else None
-        return JobPost(
-            id=str(job["key"]),
-            title=job["title"],
-            description=description,
-            company_name=job["employer"].get("name") if job.get("employer") else None,
-            company_url=(f"{self.base_url}{rel_url}" if job["employer"] else None),
-            company_url_direct=(
-                employer["links"]["corporateWebsite"] if employer else None
-            ),
-            location=Location(
-                city=job.get("location", {}).get("city"),
-                state=job.get("location", {}).get("admin1Code"),
-                country=job.get("location", {}).get("countryCode"),
-            ),
-            job_type=job_type,
-            compensation=self._get_compensation(job["compensation"]),
-            date_posted=date_posted,
-            job_url=job_url,
-            job_url_direct=(
-                job["recruit"].get("viewJobUrl") if job.get("recruit") else None
-            ),
-            emails=extract_emails_from_text(description) if description else None,
-            is_remote=self._is_job_remote(job, description),
-            company_addresses=(
-                employer_details["addresses"][0]
-                if employer_details.get("addresses")
-                else None
-            ),
-            company_industry=(
-                employer_details["industry"]
-                .replace("Iv1", "")
-                .replace("_", " ")
-                .title()
-                .strip()
-                if employer_details.get("industry")
-                else None
-            ),
-            company_num_employees=employer_details.get("employeesLocalizedLabel"),
-            company_revenue=employer_details.get("revenueLocalizedLabel"),
-            company_description=employer_details.get("briefDescription"),
-            ceo_name=employer_details.get("ceoName"),
-            ceo_photo_url=employer_details.get("ceoPhotoUrl"),
-            logo_photo_url=(
-                employer["images"].get("squareLogoUrl")
-                if employer and employer.get("images")
-                else None
-            ),
-            banner_photo_url=(
-                employer["images"].get("headerImageUrl")
-                if employer and employer.get("images")
-                else None
-            ),
-        )
-
-    @staticmethod
-    def _get_job_type(attributes: list) -> list[JobType]:
-        """
-        Parses the attributes to get list of job types
-        :param attributes:
-        :return: list of JobType
-        """
-        job_types: list[JobType] = []
-        for attribute in attributes:
-            job_type_str = attribute["label"].replace("-", "").replace(" ", "").lower()
-            job_type = get_enum_from_job_type(job_type_str)
-            if job_type:
-                job_types.append(job_type)
-        return job_types
-
-    @staticmethod
-    def _get_compensation(compensation: dict) -> Compensation | None:
-        """
-        Parses the job to get compensation
-        :param job:
-        :return: compensation object
-        """
-        if not compensation["baseSalary"] and not compensation["estimated"]:
-            return None
-        comp = (
-            compensation["baseSalary"]
-            if compensation["baseSalary"]
-            else compensation["estimated"]["baseSalary"]
-        )
-        if not comp:
-            return None
-        interval = IndeedScraper._get_compensation_interval(comp["unitOfWork"])
-        if not interval:
-            return None
-        min_range = comp["range"].get("min")
-        max_range = comp["range"].get("max")
-        return Compensation(
-            interval=interval,
-            min_amount=int(min_range) if min_range is not None else None,
-            max_amount=int(max_range) if max_range is not None else None,
-            currency=(
-                compensation["estimated"]["currencyCode"]
-                if compensation["estimated"]
-                else compensation["currencyCode"]
-            ),
-        )
-
-    @staticmethod
-    def _is_job_remote(job: dict, description: str) -> bool:
-        """
-        Searches the description, location, and attributes to check if job is remote
-        """
-        remote_keywords = ["remote", "work from home", "wfh"]
-        is_remote_in_attributes = any(
-            any(keyword in attr["label"].lower() for keyword in remote_keywords)
-            for attr in job["attributes"]
-        )
-        is_remote_in_description = any(
-            keyword in description.lower() for keyword in remote_keywords
-        )
-        is_remote_in_location = any(
-            keyword in job["location"]["formatted"]["long"].lower()
-            for keyword in remote_keywords
-        )
-        return (
-            is_remote_in_attributes or is_remote_in_description or is_remote_in_location
-        )
-
-    @staticmethod
-    def _get_compensation_interval(interval: str) -> CompensationInterval:
-        interval_mapping = {
-            "DAY": "DAILY",
-            "YEAR": "YEARLY",
-            "HOUR": "HOURLY",
-            "WEEK": "WEEKLY",
-            "MONTH": "MONTHLY",
-        }
-        mapped_interval = interval_mapping.get(interval.upper(), None)
-        if mapped_interval and mapped_interval in CompensationInterval.__members__:
-            return CompensationInterval[mapped_interval]
-        else:
-            raise ValueError(f"Unsupported interval: {interval}")
-
-    api_headers = {
-        "Host": "apis.indeed.com",
-        "content-type": "application/json",
-        "indeed-api-key": "161092c2017b5bbab13edb12461a62d5a833871e7cad6d9d475304573de67ac8",
-        "accept": "application/json",
-        "indeed-locale": "en-US",
-        "accept-language": "en-US,en;q=0.9",
-        "user-agent": "Mozilla/5.0 (iPhone; CPU iPhone OS 16_6_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148 Indeed App 193.1",
-        "indeed-app-info": "appv=193.1; appid=com.indeed.jobsearch; osv=16.6.1; os=ios; dtype=phone",
-    }
-    job_search_query = """
-        query GetJobData {{
-          jobSearch(
-            {what}
-            {location}
-            limit: 100
-            {cursor}
-            sort: RELEVANCE
-            {filters}
-          ) {{
-            pageInfo {{
-              nextCursor
-            }}
-            results {{
-              trackingKey
-              job {{
-                source {{
-                  name
-                }}
-                key
-                title
-                datePublished
-                dateOnIndeed
-                description {{
-                  html
-                }}
-                location {{
-                  countryName
-                  countryCode
-                  admin1Code
-                  city
-                  postalCode
-                  streetAddress
-                  formatted {{
-                    short
-                    long
-                  }}
-                }}
-                compensation {{
-                  estimated {{
-                    currencyCode
-                    baseSalary {{
-                      unitOfWork
-                      range {{
-                        ... on Range {{
-                          min
-                          max
-                        }}
-                      }}
-                    }}
-                  }}
-                  baseSalary {{
-                    unitOfWork
-                    range {{
-                      ... on Range {{
-                        min
-                        max
-                      }}
-                    }}
-                  }}
-                  currencyCode
-                }}
-                attributes {{
-                  key
-                  label
-                }}
-                employer {{
-                  relativeCompanyPageUrl
-                  name
-                  dossier {{
-                      employerDetails {{
-                        addresses
-                        industry
-                        employeesLocalizedLabel
-                        revenueLocalizedLabel
-                        briefDescription
-                        ceoName
-                        ceoPhotoUrl
-                      }}
-                      images {{
-                            headerImageUrl
-                            squareLogoUrl
-                      }}
-                      links {{
-                        corporateWebsite
-                    }}
-                  }}
-                }}
-                recruit {{
-                  viewJobUrl
-                  detailedSalary
-                  workSchedule
-                }}
-              }}
-            }}
-          }}
-        }}
-        """

src/tests/test_all.py

@@ -1,18 +0,0 @@
-from ..jobspy import scrape_jobs
-import pandas as pd
-
-
-def test_all():
-    result = scrape_jobs(
-        site_name=[
-            "linkedin",
-            "indeed",
-            "glassdoor",
-        ],  # ziprecruiter needs good ip, and temp fix to pass test on ci
-        search_term="engineer",
-        results_wanted=5,
-    )
-
-    assert (
-        isinstance(result, pd.DataFrame) and len(result) == 15
-    ), "Result should be a non-empty DataFrame"

src/tests/test_glassdoor.py

@@ -1,13 +0,0 @@
-from ..jobspy import scrape_jobs
-import pandas as pd
-
-
-def test_glassdoor():
-    result = scrape_jobs(
-        site_name="glassdoor",
-        search_term="engineer",
-        results_wanted=5,
-    )
-    assert (
-        isinstance(result, pd.DataFrame) and len(result) == 5
-    ), "Result should be a non-empty DataFrame"

src/tests/test_indeed.py

@@ -1,13 +0,0 @@
-from ..jobspy import scrape_jobs
-import pandas as pd
-
-
-def test_indeed():
-    result = scrape_jobs(
-        site_name="indeed",
-        search_term="engineer",
-        results_wanted=5,
-    )
-    assert (
-        isinstance(result, pd.DataFrame) and len(result) == 5
-    ), "Result should be a non-empty DataFrame"

src/tests/test_linkedin.py

@@ -1,9 +0,0 @@
-from ..jobspy import scrape_jobs
-import pandas as pd
-
-
-def test_linkedin():
-    result = scrape_jobs(site_name="linkedin", search_term="engineer", results_wanted=5)
-    assert (
-        isinstance(result, pd.DataFrame) and len(result) == 5
-    ), "Result should be a non-empty DataFrame"

src/tests/test_ziprecruiter.py

@@ -1,12 +0,0 @@
-from ..jobspy import scrape_jobs
-import pandas as pd
-
-
-def test_ziprecruiter():
-    result = scrape_jobs(
-        site_name="zip_recruiter", search_term="software engineer", results_wanted=5
-    )
-
-    assert (
-        isinstance(result, pd.DataFrame) and len(result) == 5
-    ), "Result should be a non-empty DataFrame"