- redfin city support

- test case updates
- types addition
- docs grammar
pull/1/head
Zachary Hampton 2023-09-16 13:39:03 -07:00
parent 2d6e746ae9
commit 5ea0fa0bdb
6 changed files with 43 additions and 15 deletions

View File

@ -2,7 +2,7 @@
## RoadMap ## RoadMap
- Currently supports scraping from **RedFin** - Currently, supports scraping from **RedFin**
- Coming soon: Support for **Zillow** and other real estate platforms - Coming soon: Support for **Zillow** and other real estate platforms
- Under consideration: Excel plugin to attract a wider audience - Under consideration: Excel plugin to attract a wider audience

View File

@ -36,7 +36,13 @@ class RealtorScraper(Scraper):
return response_json['autocomplete'][0] return response_json['autocomplete'][0]
def search(self): def search(self):
location_info = self.handle_location() location_info = self.handle_location()
location_type = location_info['area_type'] location_type = location_info['area_type']
"""
property types:
apartment + building + commercial + condo_townhome + condo_townhome_rowhome_coop + condos + coop + duplex_triplex + farm + investment + land + mobile + multi_family + rental + single_family + townhomes
"""
print('a') print('a')

View File

@ -14,10 +14,18 @@ class RedfinScraper(Scraper):
response = self.session.get(url) response = self.session.get(url)
response_json = json.loads(response.text.replace('{}&&', '')) response_json = json.loads(response.text.replace('{}&&', ''))
def get_region_type(match_type: str):
if match_type == "4":
return "2"
elif match_type == "2":
return "6"
if response_json['payload']['exactMatch'] is not None: if response_json['payload']['exactMatch'] is not None:
return response_json['payload']['exactMatch']['id'].split('_')[1] target = response_json['payload']['exactMatch']
else: else:
return response_json['payload']['sections'][0]['rows'][0].split('_')[1] target = response_json['payload']['sections'][0]['rows'][0]
return target['id'].split('_')[1], get_region_type(target['type'])
@staticmethod @staticmethod
def parse_home(home: dict) -> Property: def parse_home(home: dict) -> Property:
@ -50,9 +58,9 @@ class RedfinScraper(Scraper):
) )
def search(self): def search(self):
region_id = self.handle_location() region_id, region_type = self.handle_location()
url = 'https://www.redfin.com/stingray/api/gis?al=1&region_id={}&region_type=2'.format(region_id) url = 'https://www.redfin.com/stingray/api/gis?al=1&region_id={}&region_type={}'.format(region_id, region_type)
response = self.session.get(url) response = self.session.get(url)
response_json = json.loads(response.text.replace('{}&&', '')) response_json = json.loads(response.text.replace('{}&&', ''))

View File

@ -33,3 +33,5 @@ class Property:
price_per_square_foot: int | None = None price_per_square_foot: int | None = None
price: int | None = None price: int | None = None
mls_id: str | None = None mls_id: str | None = None
property_type: str | None = None

View File

@ -2,9 +2,11 @@ from homeharvest import scrape_property
def test_realtor(): def test_realtor():
result = scrape_property( results = [
location="85281", scrape_property(
site_name="realtor.com" location="85281",
) site_name="realtor.com"
),
]
assert result is not None assert all([result is not None for result in results])

View File

@ -2,9 +2,19 @@ from homeharvest import scrape_property
def test_redfin(): def test_redfin():
result = scrape_property( results = [
site_name="redfin", scrape_property(
location="85281" location="Phoenix, AZ, USA",
) site_name="redfin"
),
scrape_property(
location="Dallas, TX, USA",
site_name="redfin"
),
scrape_property(
location="85281",
site_name="redfin"
),
]
assert result is not None assert all([result is not None for result in results])