turtlebasket/JobSpy
1
0
Fork 0
mirror of https://github.com/Bunsly/JobSpy.git synced 2026-03-06 04:24:30 -08:00
Code Issues Packages Projects Releases Wiki Activity

refactor(jobs): switch bad response handling to use fastapi.HTTPException

Browse Source
This commit is contained in:
Cullen Watson
2023-07-09 07:37:18 -05:00
parent 9ef41979fd
commit dd0047a5bb
5 changed files with 36 additions and 51 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
api/core/scrapers/indeed/__init__.py
View File

@@ -1,13 +1,13 @@
import json
import re
import json
from math import ceil
import tls_client
from bs4 import BeautifulSoup
from fastapi import HTTPException, status
from api.core.scrapers import Scraper, ScraperInput, Site
from api.core.jobs import *
from api.core.utils import handle_response
from api.core.scrapers import Scraper, ScraperInput, Site
class IndeedScraper(Scraper):
@@ -29,9 +29,11 @@ class IndeedScraper(Scraper):
}
response = session.get(self.url, params=params)
success, result = handle_response(response)
if not success:
return result
if response.status_code != status.HTTP_200_OK:
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail=f"Response returned {response.status_code} {response.reason}",
)
soup = BeautifulSoup(response.content, "html.parser")