diff --git a/homeharvest/core/scrapers/__init__.py b/homeharvest/core/scrapers/__init__.py index 5e2da68..ad58f78 100644 --- a/homeharvest/core/scrapers/__init__.py +++ b/homeharvest/core/scrapers/__init__.py @@ -16,7 +16,7 @@ class Scraper: self.location = scraper_input.location self.listing_type = scraper_input.listing_type - self.session = requests.Session() + self.session = requests.Session(proxies=scraper_input.proxy) self.listing_type = scraper_input.listing_type self.site_name = scraper_input.site_name diff --git a/homeharvest/core/scrapers/realtor/__init__.py b/homeharvest/core/scrapers/realtor/__init__.py index f8a63bf..ccddf71 100644 --- a/homeharvest/core/scrapers/realtor/__init__.py +++ b/homeharvest/core/scrapers/realtor/__init__.py @@ -39,7 +39,6 @@ class RealtorScraper(Scraper): "https://parser-external.geo.moveaws.com/suggest", params=params, headers=headers, - proxies=self.proxy, ) response_json = response.json() @@ -105,7 +104,7 @@ class RealtorScraper(Scraper): "variables": variables, } - response = self.session.post(self.search_url, json=payload, proxies=self.proxy) + response = self.session.post(self.search_url, json=payload) response_json = response.json() property_info = response_json["data"]["property"] @@ -218,7 +217,7 @@ class RealtorScraper(Scraper): "variables": variables, } - response = self.session.post(self.search_url, json=payload, proxies=self.proxy) + response = self.session.post(self.search_url, json=payload) response.raise_for_status() response_json = response.json() diff --git a/homeharvest/core/scrapers/redfin/__init__.py b/homeharvest/core/scrapers/redfin/__init__.py index 57ec57d..e2fb0d8 100644 --- a/homeharvest/core/scrapers/redfin/__init__.py +++ b/homeharvest/core/scrapers/redfin/__init__.py @@ -16,7 +16,7 @@ class RedfinScraper(Scraper): self.location ) - response = self.session.get(url, proxies=self.proxy) + response = self.session.get(url) response_json = json.loads(response.text.replace("{}&&", "")) def get_region_type(match_type: str): @@ -111,7 +111,7 @@ class RedfinScraper(Scraper): def _handle_rentals(self, region_id, region_type): url = f"https://www.redfin.com/stingray/api/v1/search/rentals?al=1&isRentals=true®ion_id={region_id}®ion_type={region_type}&num_homes=100000" - response = self.session.get(url, proxies=self.proxy) + response = self.session.get(url) response.raise_for_status() homes = response.json() @@ -211,7 +211,7 @@ class RedfinScraper(Scraper): home_id ) - response = self.session.get(url, proxies=self.proxy) + response = self.session.get(url) response_json = json.loads(response.text.replace("{}&&", "")) parsed_home = self._parse_home( @@ -233,7 +233,7 @@ class RedfinScraper(Scraper): url = f"https://www.redfin.com/stingray/api/gis?al=1®ion_id={region_id}®ion_type={region_type}&num_homes=100000" else: url = f"https://www.redfin.com/stingray/api/gis?al=1®ion_id={region_id}®ion_type={region_type}&sold_within_days=30&num_homes=100000" - response = self.session.get(url, proxies=self.proxy) + response = self.session.get(url) response_json = json.loads(response.text.replace("{}&&", "")) homes = [ self._parse_home(home) for home in response_json["payload"]["homes"] diff --git a/homeharvest/core/scrapers/zillow/__init__.py b/homeharvest/core/scrapers/zillow/__init__.py index 8a9e135..0b7b70d 100644 --- a/homeharvest/core/scrapers/zillow/__init__.py +++ b/homeharvest/core/scrapers/zillow/__init__.py @@ -26,13 +26,13 @@ class ZillowScraper(Scraper): "}&abKey=6666272a-4b99-474c-b857-110ec438732b&clientId=homepage-render" ).format(location) - response = self.session.get(url, proxies=self.proxy) + response = self.session.get(url) return response.json()["results"] != [] def search(self): resp = self.session.get( - self.url, headers=self._get_headers(), proxies=self.proxy + self.url, headers=self._get_headers() ) resp.raise_for_status() content = resp.text @@ -131,7 +131,7 @@ class ZillowScraper(Scraper): "isDebugRequest": False, } resp = self.session.put( - url, headers=self._get_headers(), json=payload, proxies=self.proxy + url, headers=self._get_headers(), json=payload ) resp.raise_for_status() a = resp.json()