- single address support

pull/1/head
Zachary Hampton 2023-09-16 14:34:10 -07:00
parent 5ea0fa0bdb
commit 2f3b012747
2 changed files with 51 additions and 16 deletions

View File

@ -16,9 +16,11 @@ class RedfinScraper(Scraper):
def get_region_type(match_type: str): def get_region_type(match_type: str):
if match_type == "4": if match_type == "4":
return "2" return "2" #: zip
elif match_type == "2": elif match_type == "2":
return "6" return "6" #: city
elif match_type == "1":
return "address" #: address, needs to be handled differently
if response_json['payload']['exactMatch'] is not None: if response_json['payload']['exactMatch'] is not None:
target = response_json['payload']['exactMatch'] target = response_json['payload']['exactMatch']
@ -28,20 +30,30 @@ class RedfinScraper(Scraper):
return target['id'].split('_')[1], get_region_type(target['type']) return target['id'].split('_')[1], get_region_type(target['type'])
@staticmethod @staticmethod
def parse_home(home: dict) -> Property: def parse_home(home: dict, single_search: bool = False) -> Property:
address = Address(
address_one=home['streetLine']['value'],
city=home['city'],
state=home['state'],
zip_code=home['zip']
)
url = 'https://www.redfin.com{}'.format(home['url'])
def get_value(key: str) -> Any | None: def get_value(key: str) -> Any | None:
if key in home and 'value' in home[key]: if key in home and 'value' in home[key]:
return home[key]['value'] return home[key]['value']
if not single_search:
address = Address(
address_one=get_value('streetLine'),
city=home['city'],
state=home['state'],
zip_code=home['zip']
)
else:
address_info = home['streetAddress']
address = Address(
address_one=address_info['assembledAddress'],
city=home['city'],
state=home['state'],
zip_code=home['zip']
)
url = 'https://www.redfin.com{}'.format(home['url'])
return Property( return Property(
address=address, address=address,
url=url, url=url,
@ -50,22 +62,41 @@ class RedfinScraper(Scraper):
stories=home['stories'] if 'stories' in home else None, stories=home['stories'] if 'stories' in home else None,
agent_name=get_value('listingAgent'), agent_name=get_value('listingAgent'),
description=home['listingRemarks'] if 'listingRemarks' in home else None, description=home['listingRemarks'] if 'listingRemarks' in home else None,
year_built=get_value('yearBuilt'), year_built=get_value('yearBuilt') if not single_search else home['yearBuilt'],
square_feet=get_value('sqFt'), square_feet=get_value('sqFt'),
price_per_square_foot=get_value('pricePerSqFt'), price_per_square_foot=get_value('pricePerSqFt'),
price=get_value('price'), price=get_value('price'),
mls_id=get_value('mlsId') mls_id=get_value('mlsId')
) )
def handle_address(self, home_id: str):
"""
EPs:
https://www.redfin.com/stingray/api/home/details/initialInfo?al=1&path=/TX/Austin/70-Rainey-St-78701/unit-1608/home/147337694
https://www.redfin.com/stingray/api/home/details/mainHouseInfoPanelInfo?propertyId=147337694&accessLevel=3
https://www.redfin.com/stingray/api/home/details/aboveTheFold?propertyId=147337694&accessLevel=3
https://www.redfin.com/stingray/api/home/details/belowTheFold?propertyId=147337694&accessLevel=3
"""
url = "https://www.redfin.com/stingray/api/home/details/aboveTheFold?propertyId={}&accessLevel=3".format(home_id)
response = self.session.get(url)
response_json = json.loads(response.text.replace('{}&&', ''))
parsed_home = self.parse_home(response_json['payload']['addressSectionInfo'], single_search=True)
return [parsed_home]
def search(self): def search(self):
region_id, region_type = self.handle_location() region_id, region_type = self.handle_location()
if region_type == "address":
home_id = region_id
return self.handle_address(home_id)
url = 'https://www.redfin.com/stingray/api/gis?al=1&region_id={}&region_type={}'.format(region_id, region_type) url = 'https://www.redfin.com/stingray/api/gis?al=1&region_id={}&region_type={}'.format(region_id, region_type)
response = self.session.get(url) response = self.session.get(url)
response_json = json.loads(response.text.replace('{}&&', '')) response_json = json.loads(response.text.replace('{}&&', ''))
homes = [self.parse_home(home) for home in response_json['payload']['homes']] homes = [self.parse_home(home) for home in response_json['payload']['homes']] #: support buildings
return homes return homes

View File

@ -3,6 +3,10 @@ from homeharvest import scrape_property
def test_redfin(): def test_redfin():
results = [ results = [
scrape_property(
location="2530 Al Lipscomb Way",
site_name="redfin"
),
scrape_property( scrape_property(
location="Phoenix, AZ, USA", location="Phoenix, AZ, USA",
site_name="redfin" site_name="redfin"