Validation error (#35)

This commit is contained in:
Cullen Watson
2023-09-03 20:05:31 -05:00
committed by GitHub
parent 69b47a2053
commit 7ae7ecdee8
13 changed files with 519 additions and 486 deletions

View File

@@ -1,9 +1,8 @@
import re
import sys
import math
import json
from datetime import datetime
from typing import Optional, Tuple, List
from typing import Optional
import tls_client
import urllib.parse
@@ -11,7 +10,14 @@ from bs4 import BeautifulSoup
from bs4.element import Tag
from concurrent.futures import ThreadPoolExecutor, Future
from ...jobs import JobPost, Compensation, CompensationInterval, Location, JobResponse, JobType
from ...jobs import (
JobPost,
Compensation,
CompensationInterval,
Location,
JobResponse,
JobType,
)
from .. import Scraper, ScraperInput, Site, StatusException
@@ -61,10 +67,7 @@ class IndeedScraper(Scraper):
params["sc"] = "0kf:" + "".join(sc_values) + ";"
response = session.get(self.url + "/jobs", params=params)
if (
response.status_code != 200
and response.status_code != 307
):
if response.status_code != 200 and response.status_code != 307:
raise StatusException(response.status_code)
soup = BeautifulSoup(response.content, "html.parser")
@@ -136,8 +139,10 @@ class IndeedScraper(Scraper):
return job_post
with ThreadPoolExecutor(max_workers=10) as executor:
job_results: list[Future] = [executor.submit(process_job, job) for job in
jobs["metaData"]["mosaicProviderJobCardsModel"]["results"]]
job_results: list[Future] = [
executor.submit(process_job, job)
for job in jobs["metaData"]["mosaicProviderJobCardsModel"]["results"]
]
job_list = [result.result() for result in job_results if result.result()]