fix:key error (#176)

Changed Numpy to the most recent version so the package can properly install

.github/workflows/python-test.yml

@@ -0,0 +1,22 @@
+name: Python Tests
+
+on:
+  pull_request:
+    branches:
+      - main
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: '3.8'
+      - name: Install dependencies
+        run: |
+          pip install poetry
+          poetry install
+      - name: Run tests
+        run: poetry run pytest src/tests/

README.md

@@ -37,7 +37,7 @@ jobs = scrape_jobs(
     hours_old=72, # (only Linkedin/Indeed is hour specific, others round up to days old)
     country_indeed='USA',  # only needed for indeed / glassdoor
     
-    # linkedin_fetch_description=True # get full description and direct job url for linkedin (slower)
+    # linkedin_fetch_description=True # get full description , direct job url , company industry and job level (seniority level) for linkedin (slower)
     # proxies=["208.195.175.46:65095", "208.195.175.45:65095", "localhost"],
     
 )
@@ -78,7 +78,7 @@ Optional
 │
 ├── proxies (list): 
 |    in format ['user:pass@host:port', 'localhost']
-|    each job board will round robin through the proxies
+|    each job board scraper will round robin through the proxies
 │
 ├── is_remote (bool)
 │
@@ -110,6 +110,9 @@ Optional
 |
 ├── country_indeed (str): 
 |    filters the country on Indeed & Glassdoor (see below for correct spelling)
+|
+├── enforce_annual_salary (bool): 
+|    converts wages to annual salary
 ```
 
 ```
@@ -130,37 +133,42 @@ Optional
 
 ```plaintext
 JobPost
-├── title (str)
-├── company (str)
-├── company_url (str)
-├── job_url (str)
-├── location (object)
-│   ├── country (str)
-│   ├── city (str)
-│   ├── state (str)
-├── description (str)
-├── job_type (str): fulltime, parttime, internship, contract
-├── job_function (str)
-├── compensation (object)
-│   ├── interval (str): yearly, monthly, weekly, daily, hourly
-│   ├── min_amount (int)
-│   ├── max_amount (int)
-│   └── currency (enum)
-├── date_posted (date)
-├── emails (str)
-└── is_remote (bool)
+├── title
+├── company
+├── company_url
+├── job_url
+├── location
+│   ├── country
+│   ├── city
+│   ├── state
+├── description
+├── job_type: fulltime, parttime, internship, contract
+├── job_function
+│   ├── interval: yearly, monthly, weekly, daily, hourly
+│   ├── min_amount
+│   ├── max_amount
+│   ├── currency
+│   └── salary_source: direct_data, description (parsed from posting)
+├── date_posted
+├── emails
+└── is_remote
+
+Linkedin specific
+└── job_level
+
+Linkedin & Indeed specific
+└── company_industry
 
 Indeed specific
-├── company_country (str)
-└── company_addresses (str)
-└── company_industry (str)
-└── company_employees_label (str)
-└── company_revenue_label (str)
-└── company_description (str)
-└── ceo_name (str)
-└── ceo_photo_url (str)
-└── logo_photo_url (str)
-└── banner_photo_url (str)
+├── company_country
+├── company_addresses
+├── company_employees_label
+├── company_revenue_label
+├── company_description
+├── ceo_name
+├── ceo_photo_url
+├── logo_photo_url
+└── banner_photo_url
 ```
 
 ## Supported Countries for Job Searching

poetry.toml

@@ -0,0 +1,2 @@
+[virtualenvs]
+in-project = true

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "python-jobspy"
-version = "1.1.57"
+version = "1.1.61"
 description = "Job scraper for LinkedIn, Indeed, Glassdoor & ZipRecruiter"
 authors = ["Zachary Hampton <zachary@bunsly.com>", "Cullen Watson <cullen@bunsly.com>"]
 homepage = "https://github.com/Bunsly/JobSpy"
@@ -15,7 +15,7 @@ python = "^3.10"
 requests = "^2.31.0"
 beautifulsoup4 = "^4.12.2"
 pandas = "^2.1.0"
-NUMPY = "1.24.2"
+NUMPY = "1.26.3"
 pydantic = "^2.3.0"
 tls-client = "^1.0.1"
 markdownify = "^0.11.6"

src/jobspy/__init__.py

@@ -10,7 +10,7 @@ from .scrapers.indeed import IndeedScraper
 from .scrapers.ziprecruiter import ZipRecruiterScraper
 from .scrapers.glassdoor import GlassdoorScraper
 from .scrapers.linkedin import LinkedInScraper
-from .scrapers import ScraperInput, Site, JobResponse, Country
+from .scrapers import SalarySource, ScraperInput, Site, JobResponse, Country
 from .scrapers.exceptions import (
     LinkedInException,
     IndeedException,
@@ -36,6 +36,7 @@ def scrape_jobs(
     linkedin_company_ids: list[int] | None = None,
     offset: int | None = 0,
     hours_old: int = None,
+    enforce_annual_salary: bool = False,
     verbose: int = 2,
     **kwargs,
 ) -> pd.DataFrame:
@@ -165,7 +166,8 @@ def scrape_jobs(
                 job_data["min_amount"] = compensation_obj.get("min_amount")
                 job_data["max_amount"] = compensation_obj.get("max_amount")
                 job_data["currency"] = compensation_obj.get("currency", "USD")
-                if (
+                job_data["salary_source"] = SalarySource.DIRECT_DATA.value
+                if enforce_annual_salary and (
                     job_data["interval"]
                     and job_data["interval"] != "yearly"
                     and job_data["min_amount"]
@@ -180,8 +182,17 @@ def scrape_jobs(
                         job_data["min_amount"],
                         job_data["max_amount"],
                         job_data["currency"],
-                    ) = extract_salary(job_data["description"])
+                    ) = extract_salary(
+                        job_data["description"],
+                        enforce_annual_salary=enforce_annual_salary,
+                    )
+                    job_data["salary_source"] = SalarySource.DESCRIPTION.value
 
+            job_data["salary_source"] = (
+                job_data["salary_source"]
+                if "min_amount" in job_data and job_data["min_amount"]
+                else None
+            )
             job_df = pd.DataFrame([job_data])
             jobs_dfs.append(job_df)
 
@@ -203,18 +214,21 @@ def scrape_jobs(
             "location",
             "job_type",
             "date_posted",
+            "salary_source",
             "interval",
             "min_amount",
             "max_amount",
             "currency",
             "is_remote",
+            "job_level",
             "job_function",
+            "company_industry",
+            "listing_type",
             "emails",
             "description",
             "company_url",
             "company_url_direct",
             "company_addresses",
-            "company_industry",
             "company_num_employees",
             "company_revenue",
             "company_description",

src/jobspy/jobs/__init__.py

@@ -242,10 +242,16 @@ class JobPost(BaseModel):
     date_posted: date | None = None
     emails: list[str] | None = None
     is_remote: bool | None = None
+    listing_type: str | None = None
+
+    # linkedin specific
+    job_level: str | None = None
+
+    # linkedin and indeed specific
+    company_industry: str | None = None
 
     # indeed specific
     company_addresses: str | None = None
-    company_industry: str | None = None
     company_num_employees: str | None = None
     company_revenue: str | None = None
     company_description: str | None = None

src/jobspy/scrapers/__init__.py

@@ -18,6 +18,9 @@ class Site(Enum):
     ZIP_RECRUITER = "zip_recruiter"
     GLASSDOOR = "glassdoor"
 
+class SalarySource(Enum):
+    DIRECT_DATA = "direct_data"
+    DESCRIPTION = "description"
 
 class ScraperInput(BaseModel):
     site_type: list[Site]

src/jobspy/scrapers/glassdoor/__init__.py

@@ -189,6 +189,15 @@ class GlassdoorScraper(Scraper):
         except:
             description = None
         company_url = f"{self.base_url}Overview/W-EI_IE{company_id}.htm"
+        company_logo = (
+            job_data["jobview"].get("overview", {}).get("squareLogoUrl", None)
+        )
+        listing_type = (
+            job_data["jobview"]
+            .get("header", {})
+            .get("adOrderSponsorshipLevel", "")
+            .lower()
+        )
         return JobPost(
             id=str(job_id),
             title=title,
@@ -201,6 +210,8 @@ class GlassdoorScraper(Scraper):
             is_remote=is_remote,
             description=description,
             emails=extract_emails_from_text(description) if description else None,
+            logo_photo_url=company_logo,
+            listing_type=listing_type,
         )
 
     def _fetch_job_description(self, job_id):

src/jobspy/scrapers/indeed/__init__.py

@@ -176,7 +176,7 @@ class IndeedScraper(Scraper):
                 keys.append("DSQF7")
 
             if keys:
-                keys_str = '", "'.join(keys)  # Prepare your keys string
+                keys_str = '", "'.join(keys)
                 filters_str = f"""
                 filters: {{
                   composite: {{
@@ -226,7 +226,7 @@ class IndeedScraper(Scraper):
                 country=job.get("location", {}).get("countryCode"),
             ),
             job_type=job_type,
-            compensation=self._get_compensation(job),
+            compensation=self._get_compensation(job["compensation"]),
             date_posted=date_posted,
             job_url=job_url,
             job_url_direct=(
@@ -244,6 +244,7 @@ class IndeedScraper(Scraper):
                 .replace("Iv1", "")
                 .replace("_", " ")
                 .title()
+                .strip()
                 if employer_details.get("industry")
                 else None
             ),
@@ -280,14 +281,19 @@ class IndeedScraper(Scraper):
         return job_types
 
     @staticmethod
-    def _get_compensation(job: dict) -> Compensation | None:
+    def _get_compensation(compensation: dict) -> Compensation | None:
         """
         Parses the job to get compensation
         :param job:
-        :param job:
         :return: compensation object
         """
-        comp = job["compensation"]["baseSalary"]
+        if not compensation["baseSalary"] and not compensation["estimated"]:
+            return None
+        comp = (
+            compensation["baseSalary"]
+            if compensation["baseSalary"]
+            else compensation["estimated"]["baseSalary"]
+        )
         if not comp:
             return None
         interval = IndeedScraper._get_compensation_interval(comp["unitOfWork"])
@@ -299,7 +305,11 @@ class IndeedScraper(Scraper):
             interval=interval,
             min_amount=int(min_range) if min_range is not None else None,
             max_amount=int(max_range) if max_range is not None else None,
-            currency=job["compensation"]["currencyCode"],
+            currency=(
+                compensation["estimated"]["currencyCode"]
+                if compensation["estimated"]
+                else compensation["currencyCode"]
+            ),
         )
 
     @staticmethod
@@ -353,7 +363,6 @@ class IndeedScraper(Scraper):
           jobSearch(
             {what}
             {location}
-            includeSponsoredResults: NONE
             limit: 100
             sort: DATE
             {cursor}
@@ -365,6 +374,9 @@ class IndeedScraper(Scraper):
             results {{
               trackingKey
               job {{
+                source {{
+                  name
+                }}
                 key
                 title
                 datePublished
@@ -385,6 +397,18 @@ class IndeedScraper(Scraper):
                   }}
                 }}
                 compensation {{
+                  estimated {{
+                    currencyCode
+                    baseSalary {{
+                      unitOfWork
+                      range {{
+                        ... on Range {{
+                          min
+                          max
+                        }}
+                      }}
+                    }}
+                  }}
                   baseSalary {{
                     unitOfWork
                     range {{

src/jobspy/scrapers/linkedin/__init__.py

@@ -219,6 +219,8 @@ class LinkedInScraper(Scraper):
             job_url=f"{self.base_url}/jobs/view/{job_id}",
             compensation=compensation,
             job_type=job_details.get("job_type"),
+            job_level=job_details.get("job_level", "").lower(),
+            company_industry=job_details.get("company_industry"),
             description=job_details.get("description"),
             job_url_direct=job_details.get("job_url_direct"),
             emails=extract_emails_from_text(job_details.get("description")),
@@ -266,6 +268,8 @@ class LinkedInScraper(Scraper):
                 job_function = job_function_span.text.strip()
         return {
             "description": description,
+            "job_level": self._parse_job_level(soup),
+            "company_industry": self._parse_company_industry(soup),
             "job_type": self._parse_job_type(soup),
             "job_url_direct": self._parse_job_url_direct(soup),
             "logo_photo_url": soup.find("img", {"class": "artdeco-entity-image"}).get(
@@ -325,6 +329,52 @@ class LinkedInScraper(Scraper):
 
         return [get_enum_from_job_type(employment_type)] if employment_type else []
 
+    @staticmethod
+    def _parse_job_level(soup_job_level: BeautifulSoup) -> str | None:
+        """
+        Gets the job level from job page
+        :param soup_job_level:
+        :return: str
+        """
+        h3_tag = soup_job_level.find(
+            "h3",
+            class_="description__job-criteria-subheader",
+            string=lambda text: "Seniority level" in text,
+        )
+        job_level = None
+        if h3_tag:
+            job_level_span = h3_tag.find_next_sibling(
+                "span",
+                class_="description__job-criteria-text description__job-criteria-text--criteria",
+            )
+            if job_level_span:
+                job_level = job_level_span.get_text(strip=True)
+
+        return job_level
+
+    @staticmethod
+    def _parse_company_industry(soup_industry: BeautifulSoup) -> str | None:
+        """
+        Gets the company industry from job page
+        :param soup_industry:
+        :return: str
+        """
+        h3_tag = soup_industry.find(
+            "h3",
+            class_="description__job-criteria-subheader",
+            string=lambda text: "Industries" in text,
+        )
+        industry = None
+        if h3_tag:
+            industry_span = h3_tag.find_next_sibling(
+                "span",
+                class_="description__job-criteria-text description__job-criteria-text--criteria",
+            )
+            if industry_span:
+                industry = industry_span.get_text(strip=True)
+
+        return industry
+
     def _parse_job_url_direct(self, soup: BeautifulSoup) -> str | None:
         """
         Gets the job url direct from job page

src/jobspy/scrapers/utils.py

@@ -10,7 +10,7 @@ import numpy as np
 from markdownify import markdownify as md
 from requests.adapters import HTTPAdapter, Retry
 
-from ..jobs import JobType
+from ..jobs import CompensationInterval, JobType
 
 logger = logging.getLogger("JobSpy")
 logger.propagate = False
@@ -193,6 +193,7 @@ def extract_salary(
     upper_limit=700000,
     hourly_threshold=350,
     monthly_threshold=30000,
+    enforce_annual_salary=False,
 ):
     if not salary_str:
         return None, None, None, None
@@ -220,20 +221,30 @@ def extract_salary(
 
         # Convert to annual if less than the hourly threshold
         if min_salary < hourly_threshold:
-            min_salary = convert_hourly_to_annual(min_salary)
+            interval = CompensationInterval.HOURLY.value
+            annual_min_salary = convert_hourly_to_annual(min_salary)
             if max_salary < hourly_threshold:
-                max_salary = convert_hourly_to_annual(max_salary)
+                annual_max_salary = convert_hourly_to_annual(max_salary)
 
         elif min_salary < monthly_threshold:
-            min_salary = convert_monthly_to_annual(min_salary)
+            interval = CompensationInterval.MONTHLY.value
+            annual_min_salary = convert_monthly_to_annual(min_salary)
             if max_salary < monthly_threshold:
-                max_salary = convert_monthly_to_annual(max_salary)
+                annual_max_salary = convert_monthly_to_annual(max_salary)
+
+        else:
+            interval = CompensationInterval.YEARLY.value
+            annual_min_salary = min_salary
+            annual_max_salary = max_salary
 
         # Ensure salary range is within specified limits
         if (
-            lower_limit <= min_salary <= upper_limit
-            and lower_limit <= max_salary <= upper_limit
-            and min_salary < max_salary
+            lower_limit <= annual_min_salary <= upper_limit
+            and lower_limit <= annual_max_salary <= upper_limit
+            and annual_min_salary < annual_max_salary
         ):
-            return "yearly", min_salary, max_salary, "USD"
+            if enforce_annual_salary:
+                return interval, annual_min_salary, annual_max_salary, "USD"
+            else:
+                return interval, min_salary, max_salary, "USD"
     return None, None, None, None

src/jobspy/scrapers/ziprecruiter/__init__.py

@@ -135,6 +135,7 @@ class ZipRecruiterScraper(Scraper):
         self.seen_urls.add(job_url)
 
         description = job.get("job_description", "").strip()
+        listing_type = job.get("buyer_type", "")
         description = (
             markdown_converter(description)
             if self.scraper_input.description_format == DescriptionFormat.MARKDOWN
@@ -175,6 +176,7 @@ class ZipRecruiterScraper(Scraper):
             description=description_full if description_full else description,
             emails=extract_emails_from_text(description) if description else None,
             job_url_direct=job_url_direct,
+            listing_type=listing_type,
         )
 
     def _get_descr(self, job_url):

src/tests/test_all.py

@@ -5,10 +5,10 @@ import pandas as pd
 def test_all():
     result = scrape_jobs(
         site_name=["linkedin", "indeed", "zip_recruiter", "glassdoor"],
-        search_term="software engineer",
+        search_term="engineer",
         results_wanted=5,
     )
 
     assert (
-        isinstance(result, pd.DataFrame) and not result.empty
+        isinstance(result, pd.DataFrame) and len(result) == 20
     ), "Result should be a non-empty DataFrame"

src/tests/test_glassdoor.py

@@ -2,10 +2,12 @@ from ..jobspy import scrape_jobs
 import pandas as pd
 
 
-def test_indeed():
+def test_glassdoor():
     result = scrape_jobs(
-        site_name="glassdoor", search_term="software engineer", country_indeed="USA"
+        site_name="glassdoor",
+        search_term="engineer",
+        results_wanted=5,
     )
     assert (
-        isinstance(result, pd.DataFrame) and not result.empty
+        isinstance(result, pd.DataFrame) and len(result) == 5
     ), "Result should be a non-empty DataFrame"

src/tests/test_indeed.py

@@ -4,8 +4,10 @@ import pandas as pd
 
 def test_indeed():
     result = scrape_jobs(
-        site_name="indeed", search_term="software engineer", country_indeed="usa"
+        site_name="indeed",
+        search_term="engineer",
+        results_wanted=5,
     )
     assert (
-        isinstance(result, pd.DataFrame) and not result.empty
+        isinstance(result, pd.DataFrame) and len(result) == 5
     ), "Result should be a non-empty DataFrame"

src/tests/test_linkedin.py

@@ -3,10 +3,7 @@ import pandas as pd
 
 
 def test_linkedin():
-    result = scrape_jobs(
-        site_name="linkedin",
-        search_term="software engineer",
-    )
+    result = scrape_jobs(site_name="linkedin", search_term="engineer", results_wanted=5)
     assert (
-        isinstance(result, pd.DataFrame) and not result.empty
+        isinstance(result, pd.DataFrame) and len(result) == 5
     ), "Result should be a non-empty DataFrame"

src/tests/test_ziprecruiter.py

@@ -4,10 +4,9 @@ import pandas as pd
 
 def test_ziprecruiter():
     result = scrape_jobs(
-        site_name="zip_recruiter",
-        search_term="software engineer",
+        site_name="zip_recruiter", search_term="software engineer", results_wanted=5
     )
 
     assert (
-        isinstance(result, pd.DataFrame) and not result.empty
+        isinstance(result, pd.DataFrame) and len(result) == 5
     ), "Result should be a non-empty DataFrame"