Proxy support (#44)

* add proxy support

* return as data frame
This commit is contained in:
Cullen Watson
2023-09-07 11:28:17 -05:00
committed by GitHub
parent a37e7f235e
commit 59f739018a
10 changed files with 366 additions and 319 deletions

View File

@@ -2,11 +2,6 @@ from ..jobs import Enum, BaseModel, JobType, JobResponse, Country
from typing import List, Optional, Any
class StatusException(Exception):
def __init__(self, status_code: int):
self.status_code = status_code
class Site(Enum):
LINKEDIN = "linkedin"
INDEED = "indeed"
@@ -28,8 +23,9 @@ class ScraperInput(BaseModel):
class Scraper:
def __init__(self, site: Site):
def __init__(self, site: Site, proxy: Optional[List[str]] = None):
self.site = site
self.proxy = (lambda p: {"http": p, "https": p} if p else None)(proxy)
def scrape(self, scraper_input: ScraperInput) -> JobResponse:
...