Indeed country support (#38)

src/jobspy/scrapers/indeed/__init__.py

@@ -1,6 +1,8 @@
 import re
 import math
+import io
 import json
+import traceback
 from datetime import datetime
 from typing import Optional
 
@@ -18,7 +20,7 @@ from ...jobs import (
     JobResponse,
     JobType,
 )
-from .. import Scraper, ScraperInput, Site, StatusException
+from .. import Scraper, ScraperInput, Site, Country, StatusException
 
 
 class ParsingException(Exception):
@@ -31,8 +33,7 @@ class IndeedScraper(Scraper):
         Initializes IndeedScraper with the Indeed job search url
         """
         site = Site(Site.INDEED)
-        url = "https://www.indeed.com"
-        super().__init__(site, url)
+        super().__init__(site)
 
         self.jobs_per_page = 15
         self.seen_urls = set()
@@ -47,16 +48,21 @@ class IndeedScraper(Scraper):
         :param session:
         :return: jobs found on page, total number of jobs found for search
         """
+        self.country = scraper_input.country
+        domain = self.country.domain_value
+        self.url = f"https://{domain}.indeed.com"
 
         job_list = []
 
         params = {
             "q": scraper_input.search_term,
             "l": scraper_input.location,
-            "radius": scraper_input.distance,
             "filter": 0,
             "start": 0 + page * 10,
         }
+        if scraper_input.distance:
+            params["radius"] = scraper_input.distance
+
         sc_values = []
         if scraper_input.is_remote:
             sc_values.append("attr(DSQF7)")
@@ -65,12 +71,15 @@ class IndeedScraper(Scraper):
 
         if sc_values:
             params["sc"] = "0kf:" + "".join(sc_values) + ";"
-        response = session.get(self.url + "/jobs", params=params)
+        response = session.get(self.url + "/jobs", params=params, allow_redirects=True)
+        # print(response.status_code)
 
-        if response.status_code != 200 and response.status_code != 307:
+        if response.status_code not in range(200, 400):
             raise StatusException(response.status_code)
 
         soup = BeautifulSoup(response.content, "html.parser")
+        with open("text2.html", "w", encoding="utf-8") as f:
+            f.write(str(soup))
         if "did not match any jobs" in str(soup):
             raise ParsingException("Search did not match any jobs")
 
@@ -92,8 +101,6 @@ class IndeedScraper(Scraper):
             if job_url in self.seen_urls:
                 return None
 
-            snippet_html = BeautifulSoup(job["snippet"], "html.parser")
-
             extracted_salary = job.get("extractedSalary")
             compensation = None
             if extracted_salary:
@@ -118,11 +125,12 @@ class IndeedScraper(Scraper):
             date_posted = date_posted.strftime("%Y-%m-%d")
 
             description = self.get_description(job_url, session)
-            li_elements = snippet_html.find_all("li")
-            if description is None and li_elements:
-                description = " ".join(li.text for li in li_elements)
+            with io.StringIO(job["snippet"]) as f:
+                soup = BeautifulSoup(f, "html.parser")
+                li_elements = soup.find_all("li")
+                if description is None and li_elements:
+                    description = " ".join(li.text for li in li_elements)
 
-            first_li = snippet_html.find("li")
             job_post = JobPost(
                 title=job["normTitle"],
                 description=description,
@@ -130,6 +138,7 @@ class IndeedScraper(Scraper):
                 location=Location(
                     city=job.get("jobLocationCity"),
                     state=job.get("jobLocationState"),
+                    country=self.country,
                 ),
                 job_type=job_type,
                 compensation=compensation,
@@ -138,7 +147,7 @@ class IndeedScraper(Scraper):
             )
             return job_post
 
-        with ThreadPoolExecutor(max_workers=10) as executor:
+        with ThreadPoolExecutor(max_workers=1) as executor:
             job_results: list[Future] = [
                 executor.submit(process_job, job)
                 for job in jobs["metaData"]["mosaicProviderJobCardsModel"]["results"]
@@ -166,7 +175,7 @@ class IndeedScraper(Scraper):
             #: get first page to initialize session
             job_list, total_results = self.scrape_page(scraper_input, 0, session)
 
-            with ThreadPoolExecutor(max_workers=10) as executor:
+            with ThreadPoolExecutor(max_workers=1) as executor:
                 futures: list[Future] = [
                     executor.submit(self.scrape_page, scraper_input, page, session)
                     for page in range(1, pages_to_process + 1)
@@ -188,6 +197,7 @@ class IndeedScraper(Scraper):
                 error=f"Indeed failed to parse response: {e}",
             )
         except Exception as e:
+            print(f"LinkedIn failed to scrape: {e}\n{traceback.format_exc()}")
             return JobResponse(
                 success=False,
                 error=f"Indeed failed to scrape: {e}",
@@ -215,17 +225,25 @@ class IndeedScraper(Scraper):
         jk_value = params.get("jk", [None])[0]
         formatted_url = f"{self.url}/viewjob?jk={jk_value}&spa=1"
 
-        response = session.get(formatted_url, allow_redirects=True)
+        try:
+            response = session.get(
+                formatted_url, allow_redirects=True, timeout_seconds=5
+            )
+        except requests.exceptions.Timeout:
+            print("The request timed out.")
+            return None
 
         if response.status_code not in range(200, 400):
+            print("status code not in range")
             return None
 
         raw_description = response.json()["body"]["jobInfoWrapperModel"][
             "jobInfoModel"
         ]["sanitizedJobDescription"]
-        soup = BeautifulSoup(raw_description, "html.parser")
-        text_content = " ".join(soup.get_text().split()).strip()
-        return text_content
+        with io.StringIO(raw_description) as f:
+            soup = BeautifulSoup(f, "html.parser")
+            text_content = " ".join(soup.get_text().split()).strip()
+            return text_content
 
     @staticmethod
     def get_job_type(job: dict) -> Optional[JobType]:
@@ -237,13 +255,18 @@ class IndeedScraper(Scraper):
         for taxonomy in job["taxonomyAttributes"]:
             if taxonomy["label"] == "job-types":
                 if len(taxonomy["attributes"]) > 0:
-                    job_type_str = (
-                        taxonomy["attributes"][0]["label"]
-                        .replace("-", "_")
-                        .replace(" ", "_")
-                        .upper()
-                    )
-                    return JobType[job_type_str]
+                    label = taxonomy["attributes"][0].get("label")
+                    if label:
+                        job_type_str = label.replace("-", "").replace(" ", "").lower()
+                        # print(f"Debug: job_type_str = {job_type_str}")
+                        return IndeedScraper.get_enum_from_value(job_type_str)
+        return None
+
+    @staticmethod
+    def get_enum_from_value(value_str):
+        for job_type in JobType:
+            if value_str in job_type.value:
+                return job_type
         return None
 
     @staticmethod
@@ -294,7 +317,7 @@ class IndeedScraper(Scraper):
         :param soup:
         :return: total_num_jobs
         """
-        script = soup.find("script", string=lambda t: "window._initialData" in t)
+        script = soup.find("script", string=lambda t: t and "window._initialData" in t)
 
         pattern = re.compile(r"window._initialData\s*=\s*({.*})\s*;", re.DOTALL)
         match = pattern.search(script.string)