turtlebasket
/
HomeHarvest
mirror of https://github.com/Bunsly/HomeHarvest.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

refactor: move proxy to session

pull/11/head
Cullen Watson 2023-09-19 15:48:46 -05:00
parent 5f31beda46
commit 1f0feb836d
4 changed files with 10 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
homeharvest/core/scrapers/__init__.py
View File

@ -16,7 +16,7 @@ class Scraper:
self.location = scraper_input.location self.location = scraper_input.location
self.listing_type = scraper_input.listing_type self.listing_type = scraper_input.listing_type
self.session = requests.Session() self.session = requests.Session(proxies=scraper_input.proxy)
self.listing_type = scraper_input.listing_type self.listing_type = scraper_input.listing_type
self.site_name = scraper_input.site_name self.site_name = scraper_input.site_name

5
homeharvest/core/scrapers/realtor/__init__.py
View File

@ -39,7 +39,6 @@ class RealtorScraper(Scraper):
"https://parser-external.geo.moveaws.com/suggest", "https://parser-external.geo.moveaws.com/suggest",
params=params, params=params,
headers=headers, headers=headers,
proxies=self.proxy,
) )
response_json = response.json() response_json = response.json()
@ -105,7 +104,7 @@ class RealtorScraper(Scraper):
"variables": variables, "variables": variables,
} }
response = self.session.post(self.search_url, json=payload, proxies=self.proxy) response = self.session.post(self.search_url, json=payload)
response_json = response.json() response_json = response.json()
property_info = response_json["data"]["property"] property_info = response_json["data"]["property"]
@ -218,7 +217,7 @@ class RealtorScraper(Scraper):
"variables": variables, "variables": variables,
} }
response = self.session.post(self.search_url, json=payload, proxies=self.proxy) response = self.session.post(self.search_url, json=payload)
response.raise_for_status() response.raise_for_status()
response_json = response.json() response_json = response.json()

8
homeharvest/core/scrapers/redfin/__init__.py
View File

@ -16,7 +16,7 @@ class RedfinScraper(Scraper):
self.location self.location
) )
response = self.session.get(url, proxies=self.proxy) response = self.session.get(url)
response_json = json.loads(response.text.replace("{}&&", "")) response_json = json.loads(response.text.replace("{}&&", ""))
def get_region_type(match_type: str): def get_region_type(match_type: str):
@ -111,7 +111,7 @@ class RedfinScraper(Scraper):
def _handle_rentals(self, region_id, region_type): def _handle_rentals(self, region_id, region_type):
url = f"https://www.redfin.com/stingray/api/v1/search/rentals?al=1&isRentals=true&region_id={region_id}&region_type={region_type}&num_homes=100000" url = f"https://www.redfin.com/stingray/api/v1/search/rentals?al=1&isRentals=true&region_id={region_id}&region_type={region_type}&num_homes=100000"
response = self.session.get(url, proxies=self.proxy) response = self.session.get(url)
response.raise_for_status() response.raise_for_status()
homes = response.json() homes = response.json()
@ -211,7 +211,7 @@ class RedfinScraper(Scraper):
home_id home_id
) )
response = self.session.get(url, proxies=self.proxy) response = self.session.get(url)
response_json = json.loads(response.text.replace("{}&&", "")) response_json = json.loads(response.text.replace("{}&&", ""))
parsed_home = self._parse_home( parsed_home = self._parse_home(
@ -233,7 +233,7 @@ class RedfinScraper(Scraper):
url = f"https://www.redfin.com/stingray/api/gis?al=1&region_id={region_id}&region_type={region_type}&num_homes=100000" url = f"https://www.redfin.com/stingray/api/gis?al=1&region_id={region_id}&region_type={region_type}&num_homes=100000"
else: else:
url = f"https://www.redfin.com/stingray/api/gis?al=1&region_id={region_id}&region_type={region_type}&sold_within_days=30&num_homes=100000" url = f"https://www.redfin.com/stingray/api/gis?al=1&region_id={region_id}&region_type={region_type}&sold_within_days=30&num_homes=100000"
response = self.session.get(url, proxies=self.proxy) response = self.session.get(url)
response_json = json.loads(response.text.replace("{}&&", "")) response_json = json.loads(response.text.replace("{}&&", ""))
homes = [ homes = [
self._parse_home(home) for home in response_json["payload"]["homes"] self._parse_home(home) for home in response_json["payload"]["homes"]

6
homeharvest/core/scrapers/zillow/__init__.py
View File

@ -26,13 +26,13 @@ class ZillowScraper(Scraper):
"}&abKey=6666272a-4b99-474c-b857-110ec438732b&clientId=homepage-render" "}&abKey=6666272a-4b99-474c-b857-110ec438732b&clientId=homepage-render"
).format(location) ).format(location)
response = self.session.get(url, proxies=self.proxy) response = self.session.get(url)
return response.json()["results"] != [] return response.json()["results"] != []
def search(self): def search(self):
resp = self.session.get( resp = self.session.get(
self.url, headers=self._get_headers(), proxies=self.proxy self.url, headers=self._get_headers()
) )
resp.raise_for_status() resp.raise_for_status()
content = resp.text content = resp.text
@ -131,7 +131,7 @@ class ZillowScraper(Scraper):
"isDebugRequest": False, "isDebugRequest": False,
} }
resp = self.session.put( resp = self.session.put(
url, headers=self._get_headers(), json=payload, proxies=self.proxy url, headers=self._get_headers(), json=payload
) )
resp.raise_for_status() resp.raise_for_status()
a = resp.json() a = resp.json()