chore:readme

.github/CODEOWNERS

@@ -1 +0,0 @@
-* @cullenwatson

README.md

@@ -89,6 +89,7 @@ Optional
 |
 ├── user_agent (str): 
 |    override the default user agent which may be outdated
+|
 │
 ├── description_format (str): 
 |    markdown, html (Format type of the job descriptions. Default is markdown.)

jobspy/__init__.py

@@ -107,7 +107,6 @@ def scrape_jobs(
         scraped_data: JobResponse = scraper.scrape(scraper_input)
         cap_name = site.value.capitalize()
         site_name = "ZipRecruiter" if cap_name == "Zip_recruiter" else cap_name
-        site_name = "LinkedIn" if cap_name == "Linkedin" else cap_name
         create_logger(site_name).info(f"finished scraping")
         return site.value, scraped_data
 

jobspy/bdjobs/__init__.py

@@ -46,7 +46,7 @@ class BDJobs(Scraper):
     band_delay = 3
 
     def __init__(
-        self, proxies: list[str] | str | None = None, ca_cert: str | None = None, user_agent: str | None = None
+        self, proxies: list[str] | str | None = None, ca_cert: str | None = None
     ):
         """
         Initializes BDJobsScraper with the BDJobs job search url

jobspy/linkedin/__init__.py

@@ -35,7 +35,6 @@ from jobspy.util import (
     extract_emails_from_text,
     currency_parser,
     markdown_converter,
-    plain_converter,
     create_session,
     remove_attributes,
     create_logger,
@@ -165,7 +164,7 @@ class LinkedIn(Scraper):
 
             if continue_search():
                 time.sleep(random.uniform(self.delay, self.delay + self.band_delay))
-                start += len(job_cards)
+                start += len(job_list)
 
         job_list = job_list[: scraper_input.results_wanted]
         return JobResponse(jobs=job_list)
@@ -209,10 +208,6 @@ class LinkedIn(Scraper):
             if metadata_card
             else None
         )
-        if not datetime_tag and metadata_card:
-            datetime_tag = metadata_card.find(
-                "time", class_="job-search-card__listdate--new"
-            )
         date_posted = None
         if datetime_tag and "datetime" in datetime_tag.attrs:
             datetime_str = datetime_tag["datetime"]
@@ -272,8 +267,7 @@ class LinkedIn(Scraper):
             description = div_content.prettify(formatter="html")
             if self.scraper_input.description_format == DescriptionFormat.MARKDOWN:
                 description = markdown_converter(description)
-            elif self.scraper_input.description_format == DescriptionFormat.PLAIN:
-                description = plain_converter(description)
+
         h3_tag = soup.find(
             "h3", text=lambda text: text and "Job function" in text.strip()
         )

jobspy/model.py

@@ -234,7 +234,7 @@ class Compensation(BaseModel):
 class DescriptionFormat(Enum):
     MARKDOWN = "markdown"
     HTML = "html"
-    PLAIN = "plain"
+
 
 class JobPost(BaseModel):
     id: str | None = None

jobspy/naukri/__init__.py

@@ -164,15 +164,12 @@ class Naukri(Scraper):
         date_posted = self._parse_date(job.get("footerPlaceholderLabel"), job.get("createdDate"))
 
         job_url = f"https://www.naukri.com{job.get('jdURL', f'/job/{job_id}')}"
-        raw_description = job.get("jobDescription") if full_descr else None
-
-        job_type = parse_job_type(raw_description) if raw_description else None
-        company_industry = parse_company_industry(raw_description) if raw_description else None
-
-        description = raw_description
+        description = job.get("jobDescription") if full_descr else None
         if description and self.scraper_input.description_format == DescriptionFormat.MARKDOWN:
             description = markdown_converter(description)
 
+        job_type = parse_job_type(description) if description else None
+        company_industry = parse_company_industry(description) if description else None
         is_remote = is_job_remote(title, description or "", location)
         company_logo = job.get("logoPathV3") or job.get("logoPath")
 

jobspy/naukri/util.py

@@ -5,12 +5,10 @@ from jobspy.model import JobType, Location
 from jobspy.util import get_enum_from_job_type
 
 
-def parse_job_type(soup: BeautifulSoup |str) -> list[JobType] | None:
+def parse_job_type(soup: BeautifulSoup) -> list[JobType] | None:
     """
     Gets the job type from the job page
     """
-    if isinstance(soup, str):
-        soup = BeautifulSoup(soup, "html.parser")
     job_type_tag = soup.find("span", class_="job-type")
     if job_type_tag:
         job_type_str = job_type_tag.get_text(strip=True).lower().replace("-", "")
@@ -18,12 +16,10 @@ def parse_job_type(soup: BeautifulSoup |str) -> list[JobType] | None:
     return None
 
 
-def parse_company_industry(soup: BeautifulSoup | str) -> str | None:
+def parse_company_industry(soup: BeautifulSoup) -> str | None:
     """
     Gets the company industry from the job page
     """
-    if isinstance(soup, str):
-        soup = BeautifulSoup(soup, "html.parser")
     industry_tag = soup.find("span", class_="industry")
     return industry_tag.get_text(strip=True) if industry_tag else None
 

jobspy/util.py

@@ -157,15 +157,6 @@ def markdown_converter(description_html: str):
     markdown = md(description_html)
     return markdown.strip()
 
-def plain_converter(decription_html:str):
-    from bs4 import BeautifulSoup
-    if decription_html is None:
-        return None
-    soup = BeautifulSoup(decription_html, "html.parser")
-    text = soup.get_text(separator=" ")
-    text = re.sub(r'\s+',' ',text)
-    return text.strip()
-
 
 def extract_emails_from_text(text: str) -> list[str] | None:
     if not text:

poetry.lock

@@ -749,6 +749,17 @@ files = [
 [package.extras]
 all = ["flake8 (>=7.1.1)", "mypy (>=1.11.2)", "pytest (>=8.3.2)", "ruff (>=0.6.2)"]
 
+[[package]]
+name = "iniconfig"
+version = "2.0.0"
+description = "brain-dead simple config-ini parsing"
+optional = false
+python-versions = ">=3.7"
+files = [
+    {file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"},
+    {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"},
+]
+
 [[package]]
 name = "ipykernel"
 version = "6.29.5"
@@ -1218,13 +1229,13 @@ files = [
 
 [[package]]
 name = "markdownify"
-version = "1.1.0"
+version = "0.13.1"
 description = "Convert HTML to markdown."
 optional = false
 python-versions = "*"
 files = [
-    {file = "markdownify-1.1.0-py3-none-any.whl", hash = "sha256:32a5a08e9af02c8a6528942224c91b933b4bd2c7d078f9012943776fc313eeef"},
-    {file = "markdownify-1.1.0.tar.gz", hash = "sha256:449c0bbbf1401c5112379619524f33b63490a8fa479456d41de9dc9e37560ebd"},
+    {file = "markdownify-0.13.1-py3-none-any.whl", hash = "sha256:1d181d43d20902bcc69d7be85b5316ed174d0dda72ff56e14ae4c95a4a407d22"},
+    {file = "markdownify-0.13.1.tar.gz", hash = "sha256:ab257f9e6bd4075118828a28c9d02f8a4bfeb7421f558834aa79b2dfeb32a098"},
 ]
 
 [package.dependencies]
@@ -1699,6 +1710,21 @@ docs = ["furo (>=2024.8.6)", "proselint (>=0.14)", "sphinx (>=8.0.2)", "sphinx-a
 test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=8.3.2)", "pytest-cov (>=5)", "pytest-mock (>=3.14)"]
 type = ["mypy (>=1.11.2)"]
 
+[[package]]
+name = "pluggy"
+version = "1.5.0"
+description = "plugin and hook calling mechanisms for python"
+optional = false
+python-versions = ">=3.8"
+files = [
+    {file = "pluggy-1.5.0-py3-none-any.whl", hash = "sha256:44e1ad92c8ca002de6377e165f3e0f1be63266ab4d554740532335b9d75ea669"},
+    {file = "pluggy-1.5.0.tar.gz", hash = "sha256:2cffa88e94fdc978c4c574f15f9e59b7f4201d439195c3715ca9e2486f1d0cf1"},
+]
+
+[package.extras]
+dev = ["pre-commit", "tox"]
+testing = ["pytest", "pytest-benchmark"]
+
 [[package]]
 name = "pre-commit"
 version = "4.0.1"
@@ -1949,6 +1975,28 @@ files = [
 [package.extras]
 windows-terminal = ["colorama (>=0.4.6)"]
 
+[[package]]
+name = "pytest"
+version = "7.4.4"
+description = "pytest: simple powerful testing with Python"
+optional = false
+python-versions = ">=3.7"
+files = [
+    {file = "pytest-7.4.4-py3-none-any.whl", hash = "sha256:b090cdf5ed60bf4c45261be03239c2c1c22df034fbffe691abe93cd80cea01d8"},
+    {file = "pytest-7.4.4.tar.gz", hash = "sha256:2cf0005922c6ace4a3e2ec8b4080eb0d9753fdc93107415332f50ce9e7994280"},
+]
+
+[package.dependencies]
+colorama = {version = "*", markers = "sys_platform == \"win32\""}
+exceptiongroup = {version = ">=1.0.0rc8", markers = "python_version < \"3.11\""}
+iniconfig = "*"
+packaging = "*"
+pluggy = ">=0.12,<2.0"
+tomli = {version = ">=1.0.0", markers = "python_version < \"3.11\""}
+
+[package.extras]
+testing = ["argcomplete", "attrs (>=19.2.0)", "hypothesis (>=3.56)", "mock", "nose", "pygments (>=2.7.2)", "requests", "setuptools", "xmlschema"]
+
 [[package]]
 name = "python-dateutil"
 version = "2.9.0.post0"
@@ -2821,4 +2869,4 @@ files = [
 [metadata]
 lock-version = "2.0"
 python-versions = "^3.10"
-content-hash = "6260adc8f96f6cf1ba4e2c23f05504c19e67140b9d346aed3d12eea6957b2104"
+content-hash = "57169347d2ce0ff19c4d3024ce000651bb3a816e36f454618f480741094fb4a7"

pyproject.toml

@@ -21,10 +21,10 @@ python = "^3.10"
 requests = "^2.31.0"
 beautifulsoup4 = "^4.12.2"
 pandas = "^2.1.0"
-numpy = ">=1.26.0"
+NUMPY = "1.26.3"
 pydantic = "^2.3.0"
 tls-client = "^1.0.1"
-markdownify = "^1.1.0"
+markdownify = "^0.13.1"
 regex = "^2024.4.28"
 
 [tool.poetry.group.dev.dependencies]