From 1f1ca8068ff563d36a4844f73cc63e506279c78c Mon Sep 17 00:00:00 2001 From: Zachary Hampton <69336300+ZacharyHampton@users.noreply.github.com> Date: Mon, 2 Oct 2023 10:28:13 -0700 Subject: [PATCH 01/19] - realtor.com default --- README.md | 30 +++++++++++++++--------------- homeharvest/__init__.py | 8 +++++--- 2 files changed, 20 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 74fade3..87d9819 100644 --- a/README.md +++ b/README.md @@ -29,6 +29,21 @@ pip install homeharvest ## Usage +### Python + +```py +from homeharvest import scrape_property +import pandas as pd + +properties: pd.DataFrame = scrape_property( + location="85281", + listing_type="for_rent" # for_sale / sold +) + +#: Note, to export to CSV or Excel, use properties.to_csv() or properties.to_excel(). +print(properties) +``` + ### CLI ```bash @@ -44,21 +59,6 @@ By default: - If `-f` or `--filename` is left blank, the default is `HomeHarvest_`. - If `-p` or `--proxy` is not provided, the scraper uses the local IP. - Use `-k` or `--keep_duplicates` to keep duplicate properties based on address. If not provided, duplicates will be removed. -### Python - -```py -from homeharvest import scrape_property -import pandas as pd - -properties: pd.DataFrame = scrape_property( - site_name=["zillow", "realtor.com", "redfin"], - location="85281", - listing_type="for_rent" # for_sale / sold -) - -#: Note, to export to CSV or Excel, use properties.to_csv() or properties.to_excel(). -print(properties) -``` ## Output ```py diff --git a/homeharvest/__init__.py b/homeharvest/__init__.py index 2b60e3b..8fe7d0d 100644 --- a/homeharvest/__init__.py +++ b/homeharvest/__init__.py @@ -132,7 +132,7 @@ def _scrape_single_site(location: str, site_name: str, listing_type: str, proxy: def scrape_property( location: str, - site_name: Union[str, list[str]] = None, + site_name: Union[str, list[str]] = "realtor.com", listing_type: str = "for_sale", proxy: str = None, keep_duplicates: bool = False @@ -140,12 +140,14 @@ def scrape_property( """ Scrape property from various sites from a given location and listing type. - :returns: pd.DataFrame + :param keep_duplicates: + :param proxy: :param location: US Location (e.g. 'San Francisco, CA', 'Cook County, IL', '85281', '2530 Al Lipscomb Way') :param site_name: Site name or list of site names (e.g. ['realtor.com', 'zillow'], 'redfin') :param listing_type: Listing type (e.g. 'for_sale', 'for_rent', 'sold') - :return: pd.DataFrame containing properties + :returns: pd.DataFrame containing properties """ + if site_name is None: site_name = list(_scrapers.keys()) From 40bbf76db103bd6895f3140d306b65bb1f644cde Mon Sep 17 00:00:00 2001 From: Zachary Hampton <69336300+ZacharyHampton@users.noreply.github.com> Date: Mon, 2 Oct 2023 13:58:47 -0700 Subject: [PATCH 02/19] - realtor radius --- homeharvest/__init__.py | 9 +- homeharvest/core/scrapers/__init__.py | 2 + homeharvest/core/scrapers/realtor/__init__.py | 184 ++++++++++-------- tests/test_realtor.py | 10 + 4 files changed, 123 insertions(+), 82 deletions(-) diff --git a/homeharvest/__init__.py b/homeharvest/__init__.py index 8fe7d0d..f489674 100644 --- a/homeharvest/__init__.py +++ b/homeharvest/__init__.py @@ -106,7 +106,7 @@ def _process_result(result: Property) -> pd.DataFrame: return properties_df -def _scrape_single_site(location: str, site_name: str, listing_type: str, proxy: str = None) -> pd.DataFrame: +def _scrape_single_site(location: str, site_name: str, listing_type: str, radius: float, proxy: str = None) -> pd.DataFrame: """ Helper function to scrape a single site. """ @@ -117,6 +117,7 @@ def _scrape_single_site(location: str, site_name: str, listing_type: str, proxy: listing_type=ListingType[listing_type.upper()], site_name=SiteName.get_by_value(site_name.lower()), proxy=proxy, + radius=radius, ) site = _scrapers[site_name.lower()](scraper_input) @@ -134,12 +135,14 @@ def scrape_property( location: str, site_name: Union[str, list[str]] = "realtor.com", listing_type: str = "for_sale", + radius: float = None, proxy: str = None, keep_duplicates: bool = False ) -> pd.DataFrame: """ Scrape property from various sites from a given location and listing type. + :param radius: Radius in miles to find comparable properties on individual addresses :param keep_duplicates: :param proxy: :param location: US Location (e.g. 'San Francisco, CA', 'Cook County, IL', '85281', '2530 Al Lipscomb Way') @@ -157,12 +160,12 @@ def scrape_property( results = [] if len(site_name) == 1: - final_df = _scrape_single_site(location, site_name[0], listing_type, proxy) + final_df = _scrape_single_site(location, site_name[0], listing_type, radius, proxy) results.append(final_df) else: with ThreadPoolExecutor() as executor: futures = { - executor.submit(_scrape_single_site, location, s_name, listing_type, proxy): s_name + executor.submit(_scrape_single_site, location, s_name, listing_type, radius, proxy): s_name for s_name in site_name } diff --git a/homeharvest/core/scrapers/__init__.py b/homeharvest/core/scrapers/__init__.py index e900dbe..0ab548b 100644 --- a/homeharvest/core/scrapers/__init__.py +++ b/homeharvest/core/scrapers/__init__.py @@ -9,6 +9,7 @@ class ScraperInput: location: str listing_type: ListingType site_name: SiteName + radius: float | None = None proxy: str | None = None @@ -29,6 +30,7 @@ class Scraper: self.listing_type = scraper_input.listing_type self.site_name = scraper_input.site_name + self.radius = scraper_input.radius def search(self) -> list[Property]: ... diff --git a/homeharvest/core/scrapers/realtor/__init__.py b/homeharvest/core/scrapers/realtor/__init__.py index 78ecc84..e1cb8e7 100644 --- a/homeharvest/core/scrapers/realtor/__init__.py +++ b/homeharvest/core/scrapers/realtor/__init__.py @@ -153,76 +153,90 @@ class RealtorScraper(Scraper): ) ] - def handle_area(self, variables: dict, return_total: bool = False) -> list[Property] | int: + def handle_area(self, variables: dict, is_for_comps: bool = False, return_total: bool = False) -> list[Property] | int: """ Handles a location area & returns a list of properties """ - query = ( - """query Home_search( - $city: String, - $county: [String], - $state_code: String, - $postal_code: String - $offset: Int, - ) { - home_search( - query: { - city: $city - county: $county - postal_code: $postal_code - state_code: $state_code - status: %s - } - limit: 200 - offset: $offset + + results_query = """{ + count + total + results { + property_id + description { + baths + beds + lot_sqft + sqft + text + sold_price + stories + year_built + garage + unit_number + floor_number + } + location { + address { + city + country + line + postal_code + state_code + state + street_direction + street_name + street_number + street_post_direction + street_suffix + unit + coordinate { + lon + lat + } + } + } + list_price + price_per_sqft + source { + id + } + } + }}""" + + if not is_for_comps: + query = ( + """query Home_search( + $city: String, + $county: [String], + $state_code: String, + $postal_code: String + $offset: Int, ) { - count - total - results { - property_id - description { - baths - beds - lot_sqft - sqft - text - sold_price - stories - year_built - garage - unit_number - floor_number + home_search( + query: { + city: $city + county: $county + postal_code: $postal_code + state_code: $state_code + status: %s } - location { - address { - city - country - line - postal_code - state_code - state - street_direction - street_name - street_number - street_post_direction - street_suffix - unit - coordinate { - lon - lat - } - } - } - list_price - price_per_sqft - source { - id - } - } - } - }""" - % self.listing_type.value.lower() - ) + limit: 200 + offset: $offset + ) %s""" + % (self.listing_type.value.lower(), results_query)) + else: + query = ( + """query Property_search( + $coordinates: [Float]! + $radius: String! + $offset: Int!, + ) { + property_search( + query: { nearby: { coordinates: $coordinates, radius: $radius } } + limit: 200 + offset: $offset + ) %s""" % results_query) payload = { "query": query, @@ -232,9 +246,10 @@ class RealtorScraper(Scraper): response = self.session.post(self.search_url, json=payload) response.raise_for_status() response_json = response.json() + search_key = "home_search" if not is_for_comps else "property_search" if return_total: - return response_json["data"]["home_search"]["total"] + return response_json["data"][search_key]["total"] properties: list[Property] = [] @@ -242,13 +257,13 @@ class RealtorScraper(Scraper): response_json is None or "data" not in response_json or response_json["data"] is None - or "home_search" not in response_json["data"] - or response_json["data"]["home_search"] is None - or "results" not in response_json["data"]["home_search"] + or search_key not in response_json["data"] + or response_json["data"][search_key] is None + or "results" not in response_json["data"][search_key] ): return [] - for result in response_json["data"]["home_search"]["results"]: + for result in response_json["data"][search_key]["results"]: self.counter += 1 address_one, _ = parse_address_one(result["location"]["address"]["line"]) realty_property = Property( @@ -297,21 +312,31 @@ class RealtorScraper(Scraper): def search(self): location_info = self.handle_location() location_type = location_info["area_type"] + is_for_comps = self.radius is not None and location_type == "address" - if location_type == "address": + if location_type == "address" and not is_for_comps: property_id = location_info["mpr_id"] return self.handle_address(property_id) offset = 0 - search_variables = { - "city": location_info.get("city"), - "county": location_info.get("county"), - "state_code": location_info.get("state_code"), - "postal_code": location_info.get("postal_code"), - "offset": offset, - } - total = self.handle_area(search_variables, return_total=True) + if not is_for_comps: + search_variables = { + "city": location_info.get("city"), + "county": location_info.get("county"), + "state_code": location_info.get("state_code"), + "postal_code": location_info.get("postal_code"), + "offset": offset, + } + else: + coordinates = list(location_info["centroid"].values()) + search_variables = { + "coordinates": coordinates, + "radius": "{}mi".format(self.radius), + "offset": offset, + } + + total = self.handle_area(search_variables, return_total=True, is_for_comps=is_for_comps) homes = [] with ThreadPoolExecutor(max_workers=10) as executor: @@ -320,6 +345,7 @@ class RealtorScraper(Scraper): self.handle_area, variables=search_variables | {"offset": i}, return_total=False, + is_for_comps=is_for_comps, ) for i in range(0, total, 200) ] diff --git a/tests/test_realtor.py b/tests/test_realtor.py index 3b23529..db8cb51 100644 --- a/tests/test_realtor.py +++ b/tests/test_realtor.py @@ -7,6 +7,16 @@ from homeharvest.exceptions import ( ) +def test_realtor_comps(): + result = scrape_property( + location="2530 Al Lipscomb Way", + site_name="realtor.com", + radius=0.5, + ) + + print(result) + + def test_realtor(): results = [ scrape_property( From 088088ae515a70c7a9eac6379ca29dbd86298abd Mon Sep 17 00:00:00 2001 From: Zachary Hampton <69336300+ZacharyHampton@users.noreply.github.com> Date: Tue, 3 Oct 2023 15:05:17 -0700 Subject: [PATCH 03/19] - last x days param --- homeharvest/__init__.py | 9 ++++--- homeharvest/core/scrapers/__init__.py | 2 ++ homeharvest/core/scrapers/realtor/__init__.py | 24 +++++++++++++++---- tests/test_realtor.py | 14 ++++++++++- 4 files changed, 41 insertions(+), 8 deletions(-) diff --git a/homeharvest/__init__.py b/homeharvest/__init__.py index f489674..332ff7e 100644 --- a/homeharvest/__init__.py +++ b/homeharvest/__init__.py @@ -106,7 +106,7 @@ def _process_result(result: Property) -> pd.DataFrame: return properties_df -def _scrape_single_site(location: str, site_name: str, listing_type: str, radius: float, proxy: str = None) -> pd.DataFrame: +def _scrape_single_site(location: str, site_name: str, listing_type: str, radius: float, proxy: str = None, sold_last_x_days: int = None) -> pd.DataFrame: """ Helper function to scrape a single site. """ @@ -118,6 +118,7 @@ def _scrape_single_site(location: str, site_name: str, listing_type: str, radius site_name=SiteName.get_by_value(site_name.lower()), proxy=proxy, radius=radius, + sold_last_x_days=sold_last_x_days ) site = _scrapers[site_name.lower()](scraper_input) @@ -136,12 +137,14 @@ def scrape_property( site_name: Union[str, list[str]] = "realtor.com", listing_type: str = "for_sale", radius: float = None, + sold_last_x_days: int = None, proxy: str = None, keep_duplicates: bool = False ) -> pd.DataFrame: """ Scrape property from various sites from a given location and listing type. + :param sold_last_x_days: Sold in last x days :param radius: Radius in miles to find comparable properties on individual addresses :param keep_duplicates: :param proxy: @@ -160,12 +163,12 @@ def scrape_property( results = [] if len(site_name) == 1: - final_df = _scrape_single_site(location, site_name[0], listing_type, radius, proxy) + final_df = _scrape_single_site(location, site_name[0], listing_type, radius, proxy, sold_last_x_days) results.append(final_df) else: with ThreadPoolExecutor() as executor: futures = { - executor.submit(_scrape_single_site, location, s_name, listing_type, radius, proxy): s_name + executor.submit(_scrape_single_site, location, s_name, listing_type, radius, proxy, sold_last_x_days): s_name for s_name in site_name } diff --git a/homeharvest/core/scrapers/__init__.py b/homeharvest/core/scrapers/__init__.py index 0ab548b..bc418e3 100644 --- a/homeharvest/core/scrapers/__init__.py +++ b/homeharvest/core/scrapers/__init__.py @@ -11,6 +11,7 @@ class ScraperInput: site_name: SiteName radius: float | None = None proxy: str | None = None + sold_last_x_days: int | None = None class Scraper: @@ -31,6 +32,7 @@ class Scraper: self.listing_type = scraper_input.listing_type self.site_name = scraper_input.site_name self.radius = scraper_input.radius + self.sold_last_x_days = scraper_input.sold_last_x_days def search(self) -> list[Property]: ... diff --git a/homeharvest/core/scrapers/realtor/__init__.py b/homeharvest/core/scrapers/realtor/__init__.py index e1cb8e7..6449dfd 100644 --- a/homeharvest/core/scrapers/realtor/__init__.py +++ b/homeharvest/core/scrapers/realtor/__init__.py @@ -4,7 +4,7 @@ homeharvest.realtor.__init__ This module implements the scraper for relator.com """ -from ..models import Property, Address +from ..models import Property, Address, ListingType from .. import Scraper from ....exceptions import NoResultsFound from ....utils import parse_address_one, parse_address_two @@ -204,6 +204,10 @@ class RealtorScraper(Scraper): } }}""" + sold_date_param = ('sold_date: { min: "$today-%sD" }' % self.sold_last_x_days + if self.listing_type == ListingType.SOLD and self.sold_last_x_days is not None + else "") + if not is_for_comps: query = ( """query Home_search( @@ -220,11 +224,17 @@ class RealtorScraper(Scraper): postal_code: $postal_code state_code: $state_code status: %s + %s } limit: 200 offset: $offset ) %s""" - % (self.listing_type.value.lower(), results_query)) + % ( + self.listing_type.value.lower(), + sold_date_param, + results_query + ) + ) else: query = ( """query Property_search( @@ -233,10 +243,16 @@ class RealtorScraper(Scraper): $offset: Int!, ) { property_search( - query: { nearby: { coordinates: $coordinates, radius: $radius } } + query: { + nearby: { + coordinates: $coordinates + radius: $radius + } + %s + } limit: 200 offset: $offset - ) %s""" % results_query) + ) %s""" % (sold_date_param, results_query)) payload = { "query": query, diff --git a/tests/test_realtor.py b/tests/test_realtor.py index db8cb51..56d3ef3 100644 --- a/tests/test_realtor.py +++ b/tests/test_realtor.py @@ -14,7 +14,19 @@ def test_realtor_comps(): radius=0.5, ) - print(result) + assert result is not None and len(result) > 0 + + +def test_realtor_last_x_days_sold(): + days_result_30 = scrape_property( + location="Dallas, TX", site_name="realtor.com", listing_type="sold", sold_last_x_days=30 + ) + + days_result_10 = scrape_property( + location="Dallas, TX", site_name="realtor.com", listing_type="sold", sold_last_x_days=10 + ) + + assert all([result is not None for result in [days_result_30, days_result_10]]) and len(days_result_30) != len(days_result_10) def test_realtor(): From 29664e4eee3db0ef978b68d28ffe45155142d7dc Mon Sep 17 00:00:00 2001 From: Zachary Hampton <69336300+ZacharyHampton@users.noreply.github.com> Date: Tue, 3 Oct 2023 22:21:16 -0700 Subject: [PATCH 04/19] - cullen merge --- homeharvest/__init__.py | 98 +---- homeharvest/core/scrapers/models.py | 100 ++---- homeharvest/core/scrapers/realtor/__init__.py | 271 ++++++++------ homeharvest/core/scrapers/redfin/__init__.py | 246 ------------- homeharvest/core/scrapers/zillow/__init__.py | 335 ------------------ homeharvest/utils.py | 98 +++-- tests/test_realtor.py | 13 +- tests/test_redfin.py | 35 -- tests/test_utils.py | 24 -- tests/test_zillow.py | 34 -- 10 files changed, 258 insertions(+), 996 deletions(-) delete mode 100644 homeharvest/core/scrapers/redfin/__init__.py delete mode 100644 homeharvest/core/scrapers/zillow/__init__.py delete mode 100644 tests/test_redfin.py delete mode 100644 tests/test_utils.py delete mode 100644 tests/test_zillow.py diff --git a/homeharvest/__init__.py b/homeharvest/__init__.py index 332ff7e..6eb5157 100644 --- a/homeharvest/__init__.py +++ b/homeharvest/__init__.py @@ -4,17 +4,14 @@ import concurrent.futures from concurrent.futures import ThreadPoolExecutor from .core.scrapers import ScraperInput -from .core.scrapers.redfin import RedfinScraper +from .utils import process_result, ordered_properties from .core.scrapers.realtor import RealtorScraper -from .core.scrapers.zillow import ZillowScraper from .core.scrapers.models import ListingType, Property, SiteName from .exceptions import InvalidSite, InvalidListingType _scrapers = { - "redfin": RedfinScraper, "realtor.com": RealtorScraper, - "zillow": ZillowScraper, } @@ -26,86 +23,6 @@ def _validate_input(site_name: str, listing_type: str) -> None: raise InvalidListingType(f"Provided listing type, '{listing_type}', does not exist.") -def _get_ordered_properties(result: Property) -> list[str]: - return [ - "property_url", - "site_name", - "listing_type", - "property_type", - "status_text", - "baths_min", - "baths_max", - "beds_min", - "beds_max", - "sqft_min", - "sqft_max", - "price_min", - "price_max", - "unit_count", - "tax_assessed_value", - "price_per_sqft", - "lot_area_value", - "lot_area_unit", - "address_one", - "address_two", - "city", - "state", - "zip_code", - "posted_time", - "area_min", - "bldg_name", - "stories", - "year_built", - "agent_name", - "agent_phone", - "agent_email", - "days_on_market", - "sold_date", - "mls_id", - "img_src", - "latitude", - "longitude", - "description", - ] - - -def _process_result(result: Property) -> pd.DataFrame: - prop_data = result.__dict__ - - prop_data["site_name"] = prop_data["site_name"].value - prop_data["listing_type"] = prop_data["listing_type"].value.lower() - if "property_type" in prop_data and prop_data["property_type"] is not None: - prop_data["property_type"] = prop_data["property_type"].value.lower() - else: - prop_data["property_type"] = None - if "address" in prop_data: - address_data = prop_data["address"] - prop_data["address_one"] = address_data.address_one - prop_data["address_two"] = address_data.address_two - prop_data["city"] = address_data.city - prop_data["state"] = address_data.state - prop_data["zip_code"] = address_data.zip_code - - del prop_data["address"] - - if "agent" in prop_data and prop_data["agent"] is not None: - agent_data = prop_data["agent"] - prop_data["agent_name"] = agent_data.name - prop_data["agent_phone"] = agent_data.phone - prop_data["agent_email"] = agent_data.email - - del prop_data["agent"] - else: - prop_data["agent_name"] = None - prop_data["agent_phone"] = None - prop_data["agent_email"] = None - - properties_df = pd.DataFrame([prop_data]) - properties_df = properties_df[_get_ordered_properties(result)] - - return properties_df - - def _scrape_single_site(location: str, site_name: str, listing_type: str, radius: float, proxy: str = None, sold_last_x_days: int = None) -> pd.DataFrame: """ Helper function to scrape a single site. @@ -124,22 +41,20 @@ def _scrape_single_site(location: str, site_name: str, listing_type: str, radius site = _scrapers[site_name.lower()](scraper_input) results = site.search() - properties_dfs = [_process_result(result) for result in results] - properties_dfs = [df.dropna(axis=1, how="all") for df in properties_dfs if not df.empty] + properties_dfs = [process_result(result) for result in results] if not properties_dfs: return pd.DataFrame() - return pd.concat(properties_dfs, ignore_index=True) + return pd.concat(properties_dfs, ignore_index=True, axis=0)[ordered_properties] def scrape_property( location: str, - site_name: Union[str, list[str]] = "realtor.com", + #: site_name: Union[str, list[str]] = "realtor.com", listing_type: str = "for_sale", radius: float = None, sold_last_x_days: int = None, proxy: str = None, - keep_duplicates: bool = False ) -> pd.DataFrame: """ Scrape property from various sites from a given location and listing type. @@ -153,6 +68,7 @@ def scrape_property( :param listing_type: Listing type (e.g. 'for_sale', 'for_rent', 'sold') :returns: pd.DataFrame containing properties """ + site_name = "realtor.com" if site_name is None: site_name = list(_scrapers.keys()) @@ -183,13 +99,11 @@ def scrape_property( final_df = pd.concat(results, ignore_index=True) - columns_to_track = ["address_one", "address_two", "city"] + columns_to_track = ["Street", "Unit", "Zip"] #: validate they exist, otherwise create them for col in columns_to_track: if col not in final_df.columns: final_df[col] = None - if not keep_duplicates: - final_df = final_df.drop_duplicates(subset=columns_to_track, keep="first") return final_df diff --git a/homeharvest/core/scrapers/models.py b/homeharvest/core/scrapers/models.py index ed75999..00d2a3b 100644 --- a/homeharvest/core/scrapers/models.py +++ b/homeharvest/core/scrapers/models.py @@ -1,7 +1,6 @@ from dataclasses import dataclass from enum import Enum -from typing import Tuple -from datetime import datetime +from typing import Optional class SiteName(Enum): @@ -23,46 +22,13 @@ class ListingType(Enum): SOLD = "SOLD" -class PropertyType(Enum): - HOUSE = "HOUSE" - BUILDING = "BUILDING" - CONDO = "CONDO" - TOWNHOUSE = "TOWNHOUSE" - SINGLE_FAMILY = "SINGLE_FAMILY" - MULTI_FAMILY = "MULTI_FAMILY" - MANUFACTURED = "MANUFACTURED" - NEW_CONSTRUCTION = "NEW_CONSTRUCTION" - APARTMENT = "APARTMENT" - APARTMENTS = "APARTMENTS" - LAND = "LAND" - LOT = "LOT" - OTHER = "OTHER" - - BLANK = "BLANK" - - @classmethod - def from_int_code(cls, code): - mapping = { - 1: cls.HOUSE, - 2: cls.CONDO, - 3: cls.TOWNHOUSE, - 4: cls.MULTI_FAMILY, - 5: cls.LAND, - 6: cls.OTHER, - 8: cls.SINGLE_FAMILY, - 13: cls.SINGLE_FAMILY, - } - - return mapping.get(code, cls.BLANK) - - @dataclass class Address: - address_one: str | None = None - address_two: str | None = "#" + street: str | None = None + unit: str | None = None city: str | None = None state: str | None = None - zip_code: str | None = None + zip: str | None = None @dataclass @@ -74,47 +40,31 @@ class Agent: @dataclass class Property: - property_url: str - site_name: SiteName - listing_type: ListingType - address: Address - property_type: PropertyType | None = None - - # house for sale - tax_assessed_value: int | None = None - lot_area_value: float | None = None - lot_area_unit: str | None = None - stories: int | None = None - year_built: int | None = None - price_per_sqft: int | None = None + property_url: str | None = None + mls: str | None = None mls_id: str | None = None + status: str | None = None + style: str | None = None - agent: Agent | None = None - img_src: str | None = None - description: str | None = None - status_text: str | None = None - posted_time: datetime | None = None + beds: int | None = None + baths_full: int | None = None + baths_half: int | None = None + list_price: int | None = None + list_date: str | None = None + sold_price: int | None = None + last_sold_date: str | None = None + prc_sqft: float | None = None + est_sf: int | None = None + lot_sf: int | None = None + hoa_fee: int | None = None - # building for sale - bldg_name: str | None = None - area_min: int | None = None - - beds_min: int | None = None - beds_max: int | None = None - - baths_min: float | None = None - baths_max: float | None = None - - sqft_min: int | None = None - sqft_max: int | None = None - - price_min: int | None = None - price_max: int | None = None - - unit_count: int | None = None + address: Address | None = None + yr_blt: int | None = None latitude: float | None = None longitude: float | None = None - sold_date: datetime | None = None - days_on_market: int | None = None + stories: int | None = None + prkg_gar: float | None = None + + neighborhoods: Optional[str] = None diff --git a/homeharvest/core/scrapers/realtor/__init__.py b/homeharvest/core/scrapers/realtor/__init__.py index 6449dfd..af0fe8f 100644 --- a/homeharvest/core/scrapers/realtor/__init__.py +++ b/homeharvest/core/scrapers/realtor/__init__.py @@ -7,7 +7,6 @@ This module implements the scraper for relator.com from ..models import Property, Address, ListingType from .. import Scraper from ....exceptions import NoResultsFound -from ....utils import parse_address_one, parse_address_two from concurrent.futures import ThreadPoolExecutor, as_completed @@ -118,91 +117,105 @@ class RealtorScraper(Scraper): response_json = response.json() property_info = response_json["data"]["property"] - address_one, address_two = parse_address_one(property_info["address"]["line"]) return [ Property( - site_name=self.site_name, - address=Address( - address_one=address_one, - address_two=address_two, - city=property_info["address"]["city"], - state=property_info["address"]["state_code"], - zip_code=property_info["address"]["postal_code"], - ), property_url="https://www.realtor.com/realestateandhomes-detail/" - + property_info["details"]["permalink"], + + property_info["details"]["permalink"], stories=property_info["details"]["stories"], - year_built=property_info["details"]["year_built"], - price_per_sqft=property_info["basic"]["price"] // property_info["basic"]["sqft"] - if property_info["basic"]["sqft"] is not None and property_info["basic"]["price"] is not None - else None, mls_id=property_id, - listing_type=self.listing_type, - lot_area_value=property_info["public_record"]["lot_size"] - if property_info["public_record"] is not None - else None, - beds_min=property_info["basic"]["beds"], - beds_max=property_info["basic"]["beds"], - baths_min=property_info["basic"]["baths"], - baths_max=property_info["basic"]["baths"], - sqft_min=property_info["basic"]["sqft"], - sqft_max=property_info["basic"]["sqft"], - price_min=property_info["basic"]["price"], - price_max=property_info["basic"]["price"], ) ] - def handle_area(self, variables: dict, is_for_comps: bool = False, return_total: bool = False) -> list[Property] | int: + def handle_area(self, variables: dict, is_for_comps: bool = False, return_total: bool = False) -> list[ + Property] | int: """ Handles a location area & returns a list of properties """ results_query = """{ - count - total - results { - property_id - description { - baths - beds - lot_sqft - sqft - text - sold_price - stories - year_built - garage - unit_number - floor_number - } - location { - address { - city - country - line - postal_code - state_code - state - street_direction - street_name - street_number - street_post_direction - street_suffix - unit - coordinate { - lon - lat + count + total + results { + property_id + list_date + status + last_sold_price + last_sold_date + hoa { + fee + } + description { + baths_full + baths_half + beds + lot_sqft + sqft + sold_price + year_built + garage + sold_price + type + sub_type + name + stories + } + source { + raw { + area + status + style + } + last_update_date + contract_date + id + listing_id + name + type + listing_href + community_id + management_id + corporation_id + subdivision_status + spec_id + plan_id + tier_rank + feed_type + } + location { + address { + city + country + line + postal_code + state_code + state + coordinate { + lon + lat + } + street_direction + street_name + street_number + street_post_direction + street_suffix + unit + } + neighborhoods { + name + } + } + list_price + price_per_sqft + style_category_tags { + exterior + } + source { + id + } + } } - } - } - list_price - price_per_sqft - source { - id - } - } - }}""" + }""" sold_date_param = ('sold_date: { min: "$today-%sD" }' % self.sold_last_x_days if self.listing_type == ListingType.SOLD and self.sold_last_x_days is not None @@ -210,7 +223,7 @@ class RealtorScraper(Scraper): if not is_for_comps: query = ( - """query Home_search( + """query Home_search( $city: String, $county: [String], $state_code: String, @@ -229,15 +242,15 @@ class RealtorScraper(Scraper): limit: 200 offset: $offset ) %s""" - % ( - self.listing_type.value.lower(), - sold_date_param, - results_query - ) + % ( + self.listing_type.value.lower(), + sold_date_param, + results_query + ) ) else: query = ( - """query Property_search( + """query Property_search( $coordinates: [Float]! $radius: String! $offset: Int!, @@ -270,56 +283,80 @@ class RealtorScraper(Scraper): properties: list[Property] = [] if ( - response_json is None - or "data" not in response_json - or response_json["data"] is None - or search_key not in response_json["data"] - or response_json["data"][search_key] is None - or "results" not in response_json["data"][search_key] + response_json is None + or "data" not in response_json + or response_json["data"] is None + or search_key not in response_json["data"] + or response_json["data"][search_key] is None + or "results" not in response_json["data"][search_key] ): return [] for result in response_json["data"][search_key]["results"]: self.counter += 1 - address_one, _ = parse_address_one(result["location"]["address"]["line"]) + mls = ( + result["source"].get("id") + if "source" in result and isinstance(result["source"], dict) + else None + ) + mls_id = ( + result["source"].get("listing_id") + if "source" in result and isinstance(result["source"], dict) + else None + ) + + if not mls_id: + continue + # not type + + neighborhoods_list = [] + neighborhoods = result["location"].get("neighborhoods", []) + + if neighborhoods: + for neighborhood in neighborhoods: + name = neighborhood.get("name") + if name: + neighborhoods_list.append(name) + + neighborhoods_str = ( + ", ".join(neighborhoods_list) if neighborhoods_list else None + ) + + able_to_get_lat_long = result and result.get("location") and result["location"].get("address") and result["location"]["address"].get("coordinate") + realty_property = Property( + property_url="https://www.realtor.com/realestateandhomes-detail/" + + result["property_id"], + mls=mls, + mls_id=mls_id, + status=result["status"].upper(), + style=result["description"]["type"].upper(), + beds=result["description"]["beds"], + baths_full=result["description"]["baths_full"], + baths_half=result["description"]["baths_half"], + est_sf=result["description"]["sqft"], + lot_sf=result["description"]["lot_sqft"], + list_price=result["list_price"], + list_date=result["list_date"].split("T")[0] + if result["list_date"] + else None, + sold_price=result["description"]["sold_price"], + prc_sqft=result["price_per_sqft"], + last_sold_date=result["last_sold_date"], + hoa_fee=result["hoa"]["fee"] if result.get("hoa") and isinstance(result["hoa"], dict) else None, address=Address( - address_one=address_one, + street=f"{result['location']['address']['street_number']} {result['location']['address']['street_name']} {result['location']['address']['street_suffix']}", + unit=result["location"]["address"]["unit"], city=result["location"]["address"]["city"], state=result["location"]["address"]["state_code"], - zip_code=result["location"]["address"]["postal_code"], - address_two=parse_address_two(result["location"]["address"]["unit"]), + zip=result["location"]["address"]["postal_code"], ), - latitude=result["location"]["address"]["coordinate"]["lat"] - if result - and result.get("location") - and result["location"].get("address") - and result["location"]["address"].get("coordinate") - and "lat" in result["location"]["address"]["coordinate"] - else None, - longitude=result["location"]["address"]["coordinate"]["lon"] - if result - and result.get("location") - and result["location"].get("address") - and result["location"]["address"].get("coordinate") - and "lon" in result["location"]["address"]["coordinate"] - else None, - site_name=self.site_name, - property_url="https://www.realtor.com/realestateandhomes-detail/" + result["property_id"], + yr_blt=result["description"]["year_built"], + latitude=result["location"]["address"]["coordinate"].get("lat") if able_to_get_lat_long else None, + longitude=result["location"]["address"]["coordinate"].get("lon") if able_to_get_lat_long else None, + prkg_gar=result["description"]["garage"], stories=result["description"]["stories"], - year_built=result["description"]["year_built"], - price_per_sqft=result["price_per_sqft"], - mls_id=result["property_id"], - listing_type=self.listing_type, - lot_area_value=result["description"]["lot_sqft"], - beds_min=result["description"]["beds"], - beds_max=result["description"]["beds"], - baths_min=result["description"]["baths"], - baths_max=result["description"]["baths"], - sqft_min=result["description"]["sqft"], - sqft_max=result["description"]["sqft"], - price_min=result["list_price"], - price_max=result["list_price"], + neighborhoods=neighborhoods_str, ) properties.append(realty_property) diff --git a/homeharvest/core/scrapers/redfin/__init__.py b/homeharvest/core/scrapers/redfin/__init__.py deleted file mode 100644 index 80b91f8..0000000 --- a/homeharvest/core/scrapers/redfin/__init__.py +++ /dev/null @@ -1,246 +0,0 @@ -""" -homeharvest.redfin.__init__ -~~~~~~~~~~~~ - -This module implements the scraper for redfin.com -""" -import json -from typing import Any -from .. import Scraper -from ....utils import parse_address_two, parse_address_one -from ..models import Property, Address, PropertyType, ListingType, SiteName, Agent -from ....exceptions import NoResultsFound, SearchTooBroad -from datetime import datetime - - -class RedfinScraper(Scraper): - def __init__(self, scraper_input): - super().__init__(scraper_input) - self.listing_type = scraper_input.listing_type - - def _handle_location(self): - url = "https://www.redfin.com/stingray/do/location-autocomplete?v=2&al=1&location={}".format(self.location) - - response = self.session.get(url) - response_json = json.loads(response.text.replace("{}&&", "")) - - def get_region_type(match_type: str): - if match_type == "4": - return "2" #: zip - elif match_type == "2": - return "6" #: city - elif match_type == "1": - return "address" #: address, needs to be handled differently - elif match_type == "11": - return "state" - - if "exactMatch" not in response_json["payload"]: - raise NoResultsFound("No results found for location: {}".format(self.location)) - - if response_json["payload"]["exactMatch"] is not None: - target = response_json["payload"]["exactMatch"] - else: - target = response_json["payload"]["sections"][0]["rows"][0] - - return target["id"].split("_")[1], get_region_type(target["type"]) - - def _parse_home(self, home: dict, single_search: bool = False) -> Property: - def get_value(key: str) -> Any | None: - if key in home and "value" in home[key]: - return home[key]["value"] - - if not single_search: - address = Address( - address_one=parse_address_one(get_value("streetLine"))[0], - address_two=parse_address_one(get_value("streetLine"))[1], - city=home.get("city"), - state=home.get("state"), - zip_code=home.get("zip"), - ) - else: - address_info = home.get("streetAddress") - address_one, address_two = parse_address_one(address_info.get("assembledAddress")) - - address = Address( - address_one=address_one, - address_two=address_two, - city=home.get("city"), - state=home.get("state"), - zip_code=home.get("zip"), - ) - - url = "https://www.redfin.com{}".format(home["url"]) - lot_size_data = home.get("lotSize") - - if not isinstance(lot_size_data, int): - lot_size = lot_size_data.get("value", None) if isinstance(lot_size_data, dict) else None - else: - lot_size = lot_size_data - - lat_long = get_value("latLong") - - return Property( - site_name=self.site_name, - listing_type=self.listing_type, - address=address, - property_url=url, - beds_min=home["beds"] if "beds" in home else None, - beds_max=home["beds"] if "beds" in home else None, - baths_min=home["baths"] if "baths" in home else None, - baths_max=home["baths"] if "baths" in home else None, - price_min=get_value("price"), - price_max=get_value("price"), - sqft_min=get_value("sqFt"), - sqft_max=get_value("sqFt"), - stories=home["stories"] if "stories" in home else None, - agent=Agent( #: listingAgent, some have sellingAgent as well - name=home['listingAgent'].get('name') if 'listingAgent' in home else None, - phone=home['listingAgent'].get('phone') if 'listingAgent' in home else None, - ), - description=home["listingRemarks"] if "listingRemarks" in home else None, - year_built=get_value("yearBuilt") if not single_search else home.get("yearBuilt"), - lot_area_value=lot_size, - property_type=PropertyType.from_int_code(home.get("propertyType")), - price_per_sqft=get_value("pricePerSqFt") if type(home.get("pricePerSqFt")) != int else home.get("pricePerSqFt"), - mls_id=get_value("mlsId"), - latitude=lat_long.get('latitude') if lat_long else None, - longitude=lat_long.get('longitude') if lat_long else None, - sold_date=datetime.fromtimestamp(home['soldDate'] / 1000) if 'soldDate' in home else None, - days_on_market=get_value("dom") - ) - - def _handle_rentals(self, region_id, region_type): - url = f"https://www.redfin.com/stingray/api/v1/search/rentals?al=1&isRentals=true®ion_id={region_id}®ion_type={region_type}&num_homes=100000" - - response = self.session.get(url) - response.raise_for_status() - homes = response.json() - - properties_list = [] - - for home in homes["homes"]: - home_data = home["homeData"] - rental_data = home["rentalExtension"] - - property_url = f"https://www.redfin.com{home_data.get('url', '')}" - address_info = home_data.get("addressInfo", {}) - centroid = address_info.get("centroid", {}).get("centroid", {}) - address = Address( - address_one=parse_address_one(address_info.get("formattedStreetLine"))[0], - city=address_info.get("city"), - state=address_info.get("state"), - zip_code=address_info.get("zip"), - ) - - price_range = rental_data.get("rentPriceRange", {"min": None, "max": None}) - bed_range = rental_data.get("bedRange", {"min": None, "max": None}) - bath_range = rental_data.get("bathRange", {"min": None, "max": None}) - sqft_range = rental_data.get("sqftRange", {"min": None, "max": None}) - - property_ = Property( - property_url=property_url, - site_name=SiteName.REDFIN, - listing_type=ListingType.FOR_RENT, - address=address, - description=rental_data.get("description"), - latitude=centroid.get("latitude"), - longitude=centroid.get("longitude"), - baths_min=bath_range.get("min"), - baths_max=bath_range.get("max"), - beds_min=bed_range.get("min"), - beds_max=bed_range.get("max"), - price_min=price_range.get("min"), - price_max=price_range.get("max"), - sqft_min=sqft_range.get("min"), - sqft_max=sqft_range.get("max"), - img_src=home_data.get("staticMapUrl"), - posted_time=rental_data.get("lastUpdated"), - bldg_name=rental_data.get("propertyName"), - ) - - properties_list.append(property_) - - if not properties_list: - raise NoResultsFound("No rentals found for the given location.") - - return properties_list - - def _parse_building(self, building: dict) -> Property: - street_address = " ".join( - [ - building["address"]["streetNumber"], - building["address"]["directionalPrefix"], - building["address"]["streetName"], - building["address"]["streetType"], - ] - ) - return Property( - site_name=self.site_name, - property_type=PropertyType("BUILDING"), - address=Address( - address_one=parse_address_one(street_address)[0], - city=building["address"]["city"], - state=building["address"]["stateOrProvinceCode"], - zip_code=building["address"]["postalCode"], - address_two=parse_address_two( - " ".join( - [ - building["address"]["unitType"], - building["address"]["unitValue"], - ] - ) - ), - ), - property_url="https://www.redfin.com{}".format(building["url"]), - listing_type=self.listing_type, - unit_count=building.get("numUnitsForSale"), - ) - - def handle_address(self, home_id: str): - """ - EPs: - https://www.redfin.com/stingray/api/home/details/initialInfo?al=1&path=/TX/Austin/70-Rainey-St-78701/unit-1608/home/147337694 - https://www.redfin.com/stingray/api/home/details/mainHouseInfoPanelInfo?propertyId=147337694&accessLevel=3 - https://www.redfin.com/stingray/api/home/details/aboveTheFold?propertyId=147337694&accessLevel=3 - https://www.redfin.com/stingray/api/home/details/belowTheFold?propertyId=147337694&accessLevel=3 - """ - url = "https://www.redfin.com/stingray/api/home/details/aboveTheFold?propertyId={}&accessLevel=3".format( - home_id - ) - - response = self.session.get(url) - response_json = json.loads(response.text.replace("{}&&", "")) - - parsed_home = self._parse_home(response_json["payload"]["addressSectionInfo"], single_search=True) - return [parsed_home] - - def search(self): - region_id, region_type = self._handle_location() - - if region_type == "state": - raise SearchTooBroad("State searches are not supported, please use a more specific location.") - - if region_type == "address": - home_id = region_id - return self.handle_address(home_id) - - if self.listing_type == ListingType.FOR_RENT: - return self._handle_rentals(region_id, region_type) - else: - if self.listing_type == ListingType.FOR_SALE: - url = f"https://www.redfin.com/stingray/api/gis?al=1®ion_id={region_id}®ion_type={region_type}&num_homes=100000" - else: - url = f"https://www.redfin.com/stingray/api/gis?al=1®ion_id={region_id}®ion_type={region_type}&sold_within_days=30&num_homes=100000" - response = self.session.get(url) - response_json = json.loads(response.text.replace("{}&&", "")) - - if "payload" in response_json: - homes_list = response_json["payload"].get("homes", []) - buildings_list = response_json["payload"].get("buildings", {}).values() - - homes = [self._parse_home(home) for home in homes_list] + [ - self._parse_building(building) for building in buildings_list - ] - return homes - else: - return [] diff --git a/homeharvest/core/scrapers/zillow/__init__.py b/homeharvest/core/scrapers/zillow/__init__.py deleted file mode 100644 index ba55a01..0000000 --- a/homeharvest/core/scrapers/zillow/__init__.py +++ /dev/null @@ -1,335 +0,0 @@ -""" -homeharvest.zillow.__init__ -~~~~~~~~~~~~ - -This module implements the scraper for zillow.com -""" -import re -import json - -import tls_client - -from .. import Scraper -from requests.exceptions import HTTPError -from ....utils import parse_address_one, parse_address_two -from ....exceptions import GeoCoordsNotFound, NoResultsFound -from ..models import Property, Address, ListingType, PropertyType, Agent -import urllib.parse -from datetime import datetime, timedelta - - -class ZillowScraper(Scraper): - def __init__(self, scraper_input): - session = tls_client.Session( - client_identifier="chrome112", random_tls_extension_order=True - ) - - super().__init__(scraper_input, session) - - self.session.headers.update({ - 'authority': 'www.zillow.com', - 'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9', - 'accept-language': 'en-US,en;q=0.9', - 'cache-control': 'max-age=0', - 'sec-fetch-dest': 'document', - 'sec-fetch-mode': 'navigate', - 'sec-fetch-site': 'same-origin', - 'sec-fetch-user': '?1', - 'upgrade-insecure-requests': '1', - 'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.0.0 Safari/537.36', - }) - - if not self.is_plausible_location(self.location): - raise NoResultsFound("Invalid location input: {}".format(self.location)) - - listing_type_to_url_path = { - ListingType.FOR_SALE: "for_sale", - ListingType.FOR_RENT: "for_rent", - ListingType.SOLD: "recently_sold", - } - - self.url = f"https://www.zillow.com/homes/{listing_type_to_url_path[self.listing_type]}/{self.location}_rb/" - - def is_plausible_location(self, location: str) -> bool: - url = ( - "https://www.zillowstatic.com/autocomplete/v3/suggestions?q={" - "}&abKey=6666272a-4b99-474c-b857-110ec438732b&clientId=homepage-render" - ).format(urllib.parse.quote(location)) - - resp = self.session.get(url) - - return resp.json()["results"] != [] - - def search(self): - resp = self.session.get(self.url) - if resp.status_code != 200: - raise HTTPError( - f"bad response status code: {resp.status_code}" - ) - content = resp.text - - match = re.search( - r'', - content, - re.DOTALL, - ) - if not match: - raise NoResultsFound("No results were found for Zillow with the given Location.") - - json_str = match.group(1) - data = json.loads(json_str) - - if "searchPageState" in data["props"]["pageProps"]: - pattern = r'window\.mapBounds = \{\s*"west":\s*(-?\d+\.\d+),\s*"east":\s*(-?\d+\.\d+),\s*"south":\s*(-?\d+\.\d+),\s*"north":\s*(-?\d+\.\d+)\s*\};' - - match = re.search(pattern, content) - - if match: - coords = [float(coord) for coord in match.groups()] - return self._fetch_properties_backend(coords) - - else: - raise GeoCoordsNotFound("Box bounds could not be located.") - - elif "gdpClientCache" in data["props"]["pageProps"]: - gdp_client_cache = json.loads(data["props"]["pageProps"]["gdpClientCache"]) - main_key = list(gdp_client_cache.keys())[0] - - property_data = gdp_client_cache[main_key]["property"] - property = self._get_single_property_page(property_data) - - return [property] - raise NoResultsFound("Specific property data not found in the response.") - - def _fetch_properties_backend(self, coords): - url = "https://www.zillow.com/async-create-search-page-state" - - filter_state_for_sale = { - "sortSelection": { - # "value": "globalrelevanceex" - "value": "days" - }, - "isAllHomes": {"value": True}, - } - - filter_state_for_rent = { - "isForRent": {"value": True}, - "isForSaleByAgent": {"value": False}, - "isForSaleByOwner": {"value": False}, - "isNewConstruction": {"value": False}, - "isComingSoon": {"value": False}, - "isAuction": {"value": False}, - "isForSaleForeclosure": {"value": False}, - "isAllHomes": {"value": True}, - } - - filter_state_sold = { - "isRecentlySold": {"value": True}, - "isForSaleByAgent": {"value": False}, - "isForSaleByOwner": {"value": False}, - "isNewConstruction": {"value": False}, - "isComingSoon": {"value": False}, - "isAuction": {"value": False}, - "isForSaleForeclosure": {"value": False}, - "isAllHomes": {"value": True}, - } - - selected_filter = ( - filter_state_for_rent - if self.listing_type == ListingType.FOR_RENT - else filter_state_for_sale - if self.listing_type == ListingType.FOR_SALE - else filter_state_sold - ) - - payload = { - "searchQueryState": { - "pagination": {}, - "isMapVisible": True, - "mapBounds": { - "west": coords[0], - "east": coords[1], - "south": coords[2], - "north": coords[3], - }, - "filterState": selected_filter, - "isListVisible": True, - "mapZoom": 11, - }, - "wants": {"cat1": ["mapResults"]}, - "isDebugRequest": False, - } - resp = self.session.put(url, json=payload) - if resp.status_code != 200: - raise HTTPError( - f"bad response status code: {resp.status_code}" - ) - return self._parse_properties(resp.json()) - - @staticmethod - def parse_posted_time(time: str) -> datetime: - int_time = int(time.split(" ")[0]) - - if "hour" in time: - return datetime.now() - timedelta(hours=int_time) - - if "day" in time: - return datetime.now() - timedelta(days=int_time) - - def _parse_properties(self, property_data: dict): - mapresults = property_data["cat1"]["searchResults"]["mapResults"] - - properties_list = [] - - for result in mapresults: - if "hdpData" in result: - home_info = result["hdpData"]["homeInfo"] - address_data = { - "address_one": parse_address_one(home_info.get("streetAddress"))[0], - "address_two": parse_address_two(home_info["unit"]) if "unit" in home_info else "#", - "city": home_info.get("city"), - "state": home_info.get("state"), - "zip_code": home_info.get("zipcode"), - } - property_obj = Property( - site_name=self.site_name, - address=Address(**address_data), - property_url=f"https://www.zillow.com{result['detailUrl']}", - tax_assessed_value=int(home_info["taxAssessedValue"]) if "taxAssessedValue" in home_info else None, - property_type=PropertyType(home_info.get("homeType")), - listing_type=ListingType( - home_info["statusType"] if "statusType" in home_info else self.listing_type - ), - status_text=result.get("statusText"), - posted_time=self.parse_posted_time(result["variableData"]["text"]) - if "variableData" in result - and "text" in result["variableData"] - and result["variableData"]["type"] == "TIME_ON_INFO" - else None, - price_min=home_info.get("price"), - price_max=home_info.get("price"), - beds_min=int(home_info["bedrooms"]) if "bedrooms" in home_info else None, - beds_max=int(home_info["bedrooms"]) if "bedrooms" in home_info else None, - baths_min=home_info.get("bathrooms"), - baths_max=home_info.get("bathrooms"), - sqft_min=int(home_info["livingArea"]) if "livingArea" in home_info else None, - sqft_max=int(home_info["livingArea"]) if "livingArea" in home_info else None, - price_per_sqft=int(home_info["price"] // home_info["livingArea"]) - if "livingArea" in home_info and home_info["livingArea"] != 0 and "price" in home_info - else None, - latitude=result["latLong"]["latitude"], - longitude=result["latLong"]["longitude"], - lot_area_value=round(home_info["lotAreaValue"], 2) if "lotAreaValue" in home_info else None, - lot_area_unit=home_info.get("lotAreaUnit"), - img_src=result.get("imgSrc"), - ) - - properties_list.append(property_obj) - - elif "isBuilding" in result: - price_string = result["price"].replace("$", "").replace(",", "").replace("+/mo", "") - - match = re.search(r"(\d+)", price_string) - price_value = int(match.group(1)) if match else None - building_obj = Property( - property_url=f"https://www.zillow.com{result['detailUrl']}", - site_name=self.site_name, - property_type=PropertyType("BUILDING"), - listing_type=ListingType(result["statusType"]), - img_src=result.get("imgSrc"), - address=self._extract_address(result["address"]), - baths_min=result.get("minBaths"), - area_min=result.get("minArea"), - bldg_name=result.get("communityName"), - status_text=result.get("statusText"), - price_min=price_value if "+/mo" in result.get("price") else None, - price_max=price_value if "+/mo" in result.get("price") else None, - latitude=result.get("latLong", {}).get("latitude"), - longitude=result.get("latLong", {}).get("longitude"), - unit_count=result.get("unitCount"), - ) - - properties_list.append(building_obj) - - return properties_list - - def _get_single_property_page(self, property_data: dict): - """ - This method is used when a user enters the exact location & zillow returns just one property - """ - url = ( - f"https://www.zillow.com{property_data['hdpUrl']}" - if "zillow.com" not in property_data["hdpUrl"] - else property_data["hdpUrl"] - ) - address_data = property_data["address"] - address_one, address_two = parse_address_one(address_data["streetAddress"]) - address = Address( - address_one=address_one, - address_two=address_two if address_two else "#", - city=address_data["city"], - state=address_data["state"], - zip_code=address_data["zipcode"], - ) - property_type = property_data.get("homeType", None) - return Property( - site_name=self.site_name, - property_url=url, - property_type=PropertyType(property_type) if property_type in PropertyType.__members__ else None, - listing_type=self.listing_type, - address=address, - year_built=property_data.get("yearBuilt"), - tax_assessed_value=property_data.get("taxAssessedValue"), - lot_area_value=property_data.get("lotAreaValue"), - lot_area_unit=property_data["lotAreaUnits"].lower() if "lotAreaUnits" in property_data else None, - agent=Agent( - name=property_data.get("attributionInfo", {}).get("agentName") - ), - stories=property_data.get("resoFacts", {}).get("stories"), - mls_id=property_data.get("attributionInfo", {}).get("mlsId"), - beds_min=property_data.get("bedrooms"), - beds_max=property_data.get("bedrooms"), - baths_min=property_data.get("bathrooms"), - baths_max=property_data.get("bathrooms"), - price_min=property_data.get("price"), - price_max=property_data.get("price"), - sqft_min=property_data.get("livingArea"), - sqft_max=property_data.get("livingArea"), - price_per_sqft=property_data.get("resoFacts", {}).get("pricePerSquareFoot"), - latitude=property_data.get("latitude"), - longitude=property_data.get("longitude"), - img_src=property_data.get("streetViewTileImageUrlMediumAddress"), - description=property_data.get("description"), - ) - - def _extract_address(self, address_str): - """ - Extract address components from a string formatted like '555 Wedglea Dr, Dallas, TX', - and return an Address object. - """ - parts = address_str.split(", ") - - if len(parts) != 3: - raise ValueError(f"Unexpected address format: {address_str}") - - address_one = parts[0].strip() - city = parts[1].strip() - state_zip = parts[2].split(" ") - - if len(state_zip) == 1: - state = state_zip[0].strip() - zip_code = None - elif len(state_zip) == 2: - state = state_zip[0].strip() - zip_code = state_zip[1].strip() - else: - raise ValueError(f"Unexpected state/zip format in address: {address_str}") - - address_one, address_two = parse_address_one(address_one) - return Address( - address_one=address_one, - address_two=address_two if address_two else "#", - city=city, - state=state, - zip_code=zip_code, - ) diff --git a/homeharvest/utils.py b/homeharvest/utils.py index 2aeedee..f522cd4 100644 --- a/homeharvest/utils.py +++ b/homeharvest/utils.py @@ -1,38 +1,76 @@ -import re +from .core.scrapers.models import Property +import pandas as pd + +ordered_properties = [ + "PropertyURL", + "MLS", + "MLS #", + "Status", + "Style", + "Street", + "Unit", + "City", + "State", + "Zip", + "Beds", + "FB", + "NumHB", + "EstSF", + "YrBlt", + "ListPrice", + "Lst Date", + "Sold Price", + "COEDate", + "LotSFApx", + "PrcSqft", + "LATITUDE", + "LONGITUDE", + "Stories", + "HOAFee", + "PrkgGar", + "Community", +] -def parse_address_one(street_address: str) -> tuple: - if not street_address: - return street_address, "#" +def process_result(result: Property) -> pd.DataFrame: + prop_data = {prop: None for prop in ordered_properties} + prop_data.update(result.__dict__) + prop_data["PropertyURL"] = prop_data["property_url"] + prop_data["MLS"] = prop_data["mls"] + prop_data["MLS #"] = prop_data["mls_id"] + prop_data["Status"] = prop_data["status"] + prop_data["Style"] = prop_data["style"] - apt_match = re.search( - r"(APT\s*[\dA-Z]+|#[\dA-Z]+|UNIT\s*[\dA-Z]+|LOT\s*[\dA-Z]+|SUITE\s*[\dA-Z]+)$", - street_address, - re.I, - ) + if "address" in prop_data: + address_data = prop_data["address"] + prop_data["Street"] = address_data.street + prop_data["Unit"] = address_data.unit + prop_data["City"] = address_data.city + prop_data["State"] = address_data.state + prop_data["Zip"] = address_data.zip - if apt_match: - apt_str = apt_match.group().strip() - cleaned_apt_str = re.sub(r"(APT\s*|UNIT\s*|LOT\s*|SUITE\s*)", "#", apt_str, flags=re.I) + prop_data["Community"] = prop_data["neighborhoods"] + prop_data["Beds"] = prop_data["beds"] + prop_data["FB"] = prop_data["baths_full"] + prop_data["NumHB"] = prop_data["baths_half"] + prop_data["EstSF"] = prop_data["est_sf"] + prop_data["ListPrice"] = prop_data["list_price"] + prop_data["Lst Date"] = prop_data["list_date"] + prop_data["Sold Price"] = prop_data["sold_price"] + prop_data["COEDate"] = prop_data["last_sold_date"] + prop_data["LotSFApx"] = prop_data["lot_sf"] + prop_data["HOAFee"] = prop_data["hoa_fee"] - main_address = street_address.replace(apt_str, "").strip() - return main_address, cleaned_apt_str - else: - return street_address, "#" + if prop_data.get("prc_sqft") is not None: + prop_data["PrcSqft"] = round(prop_data["prc_sqft"], 2) + prop_data["YrBlt"] = prop_data["yr_blt"] + prop_data["LATITUDE"] = prop_data["latitude"] + prop_data["LONGITUDE"] = prop_data["longitude"] + prop_data["Stories"] = prop_data["stories"] + prop_data["PrkgGar"] = prop_data["prkg_gar"] -def parse_address_two(street_address: str): - if not street_address: - return "#" - apt_match = re.search( - r"(APT\s*[\dA-Z]+|#[\dA-Z]+|UNIT\s*[\dA-Z]+|LOT\s*[\dA-Z]+|SUITE\s*[\dA-Z]+)$", - street_address, - re.I, - ) + properties_df = pd.DataFrame([prop_data]) + properties_df = properties_df.reindex(columns=ordered_properties) - if apt_match: - apt_str = apt_match.group().strip() - apt_str = re.sub(r"(APT\s*|UNIT\s*|LOT\s*|SUITE\s*)", "#", apt_str, flags=re.I) - return apt_str - else: - return "#" + return properties_df[ordered_properties] \ No newline at end of file diff --git a/tests/test_realtor.py b/tests/test_realtor.py index 56d3ef3..a1768b4 100644 --- a/tests/test_realtor.py +++ b/tests/test_realtor.py @@ -10,7 +10,6 @@ from homeharvest.exceptions import ( def test_realtor_comps(): result = scrape_property( location="2530 Al Lipscomb Way", - site_name="realtor.com", radius=0.5, ) @@ -19,11 +18,11 @@ def test_realtor_comps(): def test_realtor_last_x_days_sold(): days_result_30 = scrape_property( - location="Dallas, TX", site_name="realtor.com", listing_type="sold", sold_last_x_days=30 + location="Dallas, TX", listing_type="sold", sold_last_x_days=30 ) days_result_10 = scrape_property( - location="Dallas, TX", site_name="realtor.com", listing_type="sold", sold_last_x_days=10 + location="Dallas, TX", listing_type="sold", sold_last_x_days=10 ) assert all([result is not None for result in [days_result_30, days_result_10]]) and len(days_result_30) != len(days_result_10) @@ -33,16 +32,15 @@ def test_realtor(): results = [ scrape_property( location="2530 Al Lipscomb Way", - site_name="realtor.com", listing_type="for_sale", ), scrape_property( - location="Phoenix, AZ", site_name=["realtor.com"], listing_type="for_rent" + location="Phoenix, AZ", listing_type="for_rent" ), #: does not support "city, state, USA" format scrape_property( - location="Dallas, TX", site_name="realtor.com", listing_type="sold" + location="Dallas, TX", listing_type="sold" ), #: does not support "city, state, USA" format - scrape_property(location="85281", site_name="realtor.com"), + scrape_property(location="85281"), ] assert all([result is not None for result in results]) @@ -52,7 +50,6 @@ def test_realtor(): bad_results += [ scrape_property( location="abceefg ju098ot498hh9", - site_name="realtor.com", listing_type="for_sale", ) ] diff --git a/tests/test_redfin.py b/tests/test_redfin.py deleted file mode 100644 index 6904499..0000000 --- a/tests/test_redfin.py +++ /dev/null @@ -1,35 +0,0 @@ -from homeharvest import scrape_property -from homeharvest.exceptions import ( - InvalidSite, - InvalidListingType, - NoResultsFound, - GeoCoordsNotFound, - SearchTooBroad, -) - - -def test_redfin(): - results = [ - scrape_property(location="San Diego", site_name="redfin", listing_type="for_sale"), - scrape_property(location="2530 Al Lipscomb Way", site_name="redfin", listing_type="for_sale"), - scrape_property(location="Phoenix, AZ, USA", site_name=["redfin"], listing_type="for_rent"), - scrape_property(location="Dallas, TX, USA", site_name="redfin", listing_type="sold"), - scrape_property(location="85281", site_name="redfin"), - ] - - assert all([result is not None for result in results]) - - bad_results = [] - try: - bad_results += [ - scrape_property( - location="abceefg ju098ot498hh9", - site_name="redfin", - listing_type="for_sale", - ), - scrape_property(location="Florida", site_name="redfin", listing_type="for_rent"), - ] - except (InvalidSite, InvalidListingType, NoResultsFound, GeoCoordsNotFound, SearchTooBroad): - assert True - - assert all([result is None for result in bad_results]) diff --git a/tests/test_utils.py b/tests/test_utils.py deleted file mode 100644 index d21ee77..0000000 --- a/tests/test_utils.py +++ /dev/null @@ -1,24 +0,0 @@ -from homeharvest.utils import parse_address_one, parse_address_two - - -def test_parse_address_one(): - test_data = [ - ("4303 E Cactus Rd Apt 126", ("4303 E Cactus Rd", "#126")), - ("1234 Elm Street apt 2B", ("1234 Elm Street", "#2B")), - ("1234 Elm Street UNIT 3A", ("1234 Elm Street", "#3A")), - ("1234 Elm Street unit 3A", ("1234 Elm Street", "#3A")), - ("1234 Elm Street SuIte 3A", ("1234 Elm Street", "#3A")), - ] - - for input_data, (exp_addr_one, exp_addr_two) in test_data: - address_one, address_two = parse_address_one(input_data) - assert address_one == exp_addr_one - assert address_two == exp_addr_two - - -def test_parse_address_two(): - test_data = [("Apt 126", "#126"), ("apt 2B", "#2B"), ("UNIT 3A", "#3A"), ("unit 3A", "#3A"), ("SuIte 3A", "#3A")] - - for input_data, expected in test_data: - output = parse_address_two(input_data) - assert output == expected diff --git a/tests/test_zillow.py b/tests/test_zillow.py deleted file mode 100644 index dfcc55d..0000000 --- a/tests/test_zillow.py +++ /dev/null @@ -1,34 +0,0 @@ -from homeharvest import scrape_property -from homeharvest.exceptions import ( - InvalidSite, - InvalidListingType, - NoResultsFound, - GeoCoordsNotFound, -) - - -def test_zillow(): - results = [ - scrape_property(location="2530 Al Lipscomb Way", site_name="zillow", listing_type="for_sale"), - scrape_property(location="Phoenix, AZ, USA", site_name=["zillow"], listing_type="for_rent"), - scrape_property(location="Surprise, AZ", site_name=["zillow"], listing_type="for_sale"), - scrape_property(location="Dallas, TX, USA", site_name="zillow", listing_type="sold"), - scrape_property(location="85281", site_name="zillow"), - scrape_property(location="3268 88th st s, Lakewood", site_name="zillow", listing_type="for_rent"), - ] - - assert all([result is not None for result in results]) - - bad_results = [] - try: - bad_results += [ - scrape_property( - location="abceefg ju098ot498hh9", - site_name="zillow", - listing_type="for_sale", - ) - ] - except (InvalidSite, InvalidListingType, NoResultsFound, GeoCoordsNotFound): - assert True - - assert all([result is None for result in bad_results]) From bf81ef413f4b82efafd3cdcca3e8e6e0be82d9fd Mon Sep 17 00:00:00 2001 From: Zachary Hampton <69336300+ZacharyHampton@users.noreply.github.com> Date: Tue, 3 Oct 2023 22:22:09 -0700 Subject: [PATCH 05/19] - version bump --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 5724747..c79f04e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "homeharvest" -version = "0.2.19" +version = "0.3.0" description = "Real estate scraping library supporting Zillow, Realtor.com & Redfin." authors = ["Zachary Hampton ", "Cullen Watson "] homepage = "https://github.com/ZacharyHampton/HomeHarvest" From d2879734e656edecc979fb084c5779168bd86a79 Mon Sep 17 00:00:00 2001 From: Zachary Hampton <69336300+ZacharyHampton@users.noreply.github.com> Date: Tue, 3 Oct 2023 22:25:29 -0700 Subject: [PATCH 06/19] - cli update --- homeharvest/cli.py | 27 ++++++++++----------------- 1 file changed, 10 insertions(+), 17 deletions(-) diff --git a/homeharvest/cli.py b/homeharvest/cli.py index c9deae8..6d4d72d 100644 --- a/homeharvest/cli.py +++ b/homeharvest/cli.py @@ -7,15 +7,6 @@ def main(): parser = argparse.ArgumentParser(description="Home Harvest Property Scraper") parser.add_argument("location", type=str, help="Location to scrape (e.g., San Francisco, CA)") - parser.add_argument( - "-s", - "--site_name", - type=str, - nargs="*", - default=None, - help="Site name(s) to scrape from (e.g., realtor, zillow)", - ) - parser.add_argument( "-l", "--listing_type", @@ -42,18 +33,20 @@ def main(): help="Name of the output file (without extension)", ) - parser.add_argument( - "-k", - "--keep_duplicates", - action="store_true", - help="Keep duplicate properties based on address" - ) - parser.add_argument("-p", "--proxy", type=str, default=None, help="Proxy to use for scraping") + parser.add_argument("-d", "--days", type=int, default=None, help="Sold in last _ days filter.") + + parser.add_argument( + "-r", + "--radius", + type=float, + default=None, + help="Get comparable properties within _ (eg. 0.0) miles. Only applicable for individual addresses." + ) args = parser.parse_args() - result = scrape_property(args.location, args.site_name, args.listing_type, proxy=args.proxy, keep_duplicates=args.keep_duplicates) + result = scrape_property(args.location, args.listing_type, proxy=args.proxy) if not args.filename: timestamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S") From f06a01678c033b8fe8f8ac7f16a6f1556ff03ed2 Mon Sep 17 00:00:00 2001 From: Zachary Hampton <69336300+ZacharyHampton@users.noreply.github.com> Date: Tue, 3 Oct 2023 22:31:23 -0700 Subject: [PATCH 07/19] - cli readme update --- README.md | 36 ++++++++++++++++++++++++------------ 1 file changed, 24 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 87d9819..5610f47 100644 --- a/README.md +++ b/README.md @@ -46,19 +46,31 @@ print(properties) ### CLI -```bash -homeharvest "San Francisco, CA" -s zillow realtor.com redfin -l for_rent -o excel -f HomeHarvest ``` - -This will scrape properties from the specified sites for the given location and listing type, and save the results to an Excel file named `HomeHarvest.xlsx`. - -By default: -- If `-s` or `--site_name` is not provided, it will scrape from all available sites. -- If `-l` or `--listing_type` is left blank, the default is `for_sale`. Other options are `for_rent` or `sold`. -- The `-o` or `--output` default format is `excel`. Options are `csv` or `excel`. -- If `-f` or `--filename` is left blank, the default is `HomeHarvest_`. -- If `-p` or `--proxy` is not provided, the scraper uses the local IP. -- Use `-k` or `--keep_duplicates` to keep duplicate properties based on address. If not provided, duplicates will be removed. +usage: homeharvest [-h] [-l {for_sale,for_rent,sold}] [-o {excel,csv}] [-f FILENAME] [-p PROXY] [-d DAYS] [-r RADIUS] location + +Home Harvest Property Scraper + +positional arguments: + location Location to scrape (e.g., San Francisco, CA) + +options: + -h, --help show this help message and exit + -l {for_sale,for_rent,sold}, --listing_type {for_sale,for_rent,sold} + Listing type to scrape + -o {excel,csv}, --output {excel,csv} + Output format + -f FILENAME, --filename FILENAME + Name of the output file (without extension) + -p PROXY, --proxy PROXY + Proxy to use for scraping + -d DAYS, --days DAYS Sold in last _ days filter. + -r RADIUS, --radius RADIUS + Get comparable properties within _ (eg. 0.0) miles. Only applicable for individual addresses. +``` +```bash +> homeharvest "San Francisco, CA" -l for_rent -o excel -f HomeHarvest +``` ## Output ```py From f8c0dd766da55a124673ea64629b9c64a3268ef7 Mon Sep 17 00:00:00 2001 From: Zachary Hampton <69336300+ZacharyHampton@users.noreply.github.com> Date: Tue, 3 Oct 2023 23:33:53 -0700 Subject: [PATCH 08/19] - realtor support --- homeharvest/core/scrapers/realtor/__init__.py | 60 +++++++++++++------ tests/test_realtor.py | 17 ++++++ 2 files changed, 59 insertions(+), 18 deletions(-) diff --git a/homeharvest/core/scrapers/realtor/__init__.py b/homeharvest/core/scrapers/realtor/__init__.py index af0fe8f..fb79a37 100644 --- a/homeharvest/core/scrapers/realtor/__init__.py +++ b/homeharvest/core/scrapers/realtor/__init__.py @@ -127,8 +127,7 @@ class RealtorScraper(Scraper): ) ] - def handle_area(self, variables: dict, is_for_comps: bool = False, return_total: bool = False) -> list[ - Property] | int: + def general_search(self, variables: dict, search_type: str, return_total: bool = False) -> list[Property] | int: """ Handles a location area & returns a list of properties """ @@ -221,7 +220,7 @@ class RealtorScraper(Scraper): if self.listing_type == ListingType.SOLD and self.sold_last_x_days is not None else "") - if not is_for_comps: + if search_type == "area": query = ( """query Home_search( $city: String, @@ -248,7 +247,7 @@ class RealtorScraper(Scraper): results_query ) ) - else: + elif search_type == "comp_address": query = ( """query Property_search( $coordinates: [Float]! @@ -266,6 +265,20 @@ class RealtorScraper(Scraper): limit: 200 offset: $offset ) %s""" % (sold_date_param, results_query)) + else: + query = ( + """query Property_search( + $property_id: [ID]! + $offset: Int!, + ) { + property_search( + query: { + property_id: $property_id + %s + } + limit: 200 + offset: $offset + ) %s""" % (sold_date_param, results_query)) payload = { "query": query, @@ -275,7 +288,7 @@ class RealtorScraper(Scraper): response = self.session.post(self.search_url, json=payload) response.raise_for_status() response_json = response.json() - search_key = "home_search" if not is_for_comps else "property_search" + search_key = "home_search" if search_type == "area" else "property_search" if return_total: return response_json["data"][search_key]["total"] @@ -367,38 +380,49 @@ class RealtorScraper(Scraper): location_type = location_info["area_type"] is_for_comps = self.radius is not None and location_type == "address" - if location_type == "address" and not is_for_comps: - property_id = location_info["mpr_id"] - return self.handle_address(property_id) - offset = 0 + search_variables = { + "offset": offset, + } - if not is_for_comps: - search_variables = { + search_type = "comp_address" if is_for_comps \ + else "address" if location_type == "address" and not is_for_comps \ + else "area" + + if location_type == "address" and not is_for_comps: #: single address search, non comps + property_id = location_info["mpr_id"] + search_variables = search_variables | {"property_id": property_id} + + general_search = self.general_search(search_variables, search_type) + if general_search: + return general_search + else: + return self.handle_address(property_id) #: TODO: support single address search for query by property address (can go from property -> listing to get better data) + + elif not is_for_comps: #: area search + search_variables = search_variables | { "city": location_info.get("city"), "county": location_info.get("county"), "state_code": location_info.get("state_code"), "postal_code": location_info.get("postal_code"), - "offset": offset, } - else: + else: #: comps search coordinates = list(location_info["centroid"].values()) - search_variables = { + search_variables = search_variables | { "coordinates": coordinates, "radius": "{}mi".format(self.radius), - "offset": offset, } - total = self.handle_area(search_variables, return_total=True, is_for_comps=is_for_comps) + total = self.general_search(search_variables, return_total=True, search_type=search_type) homes = [] with ThreadPoolExecutor(max_workers=10) as executor: futures = [ executor.submit( - self.handle_area, + self.general_search, variables=search_variables | {"offset": i}, return_total=False, - is_for_comps=is_for_comps, + search_type=search_type, ) for i in range(0, total, 200) ] diff --git a/tests/test_realtor.py b/tests/test_realtor.py index a1768b4..1c06848 100644 --- a/tests/test_realtor.py +++ b/tests/test_realtor.py @@ -11,6 +11,8 @@ def test_realtor_comps(): result = scrape_property( location="2530 Al Lipscomb Way", radius=0.5, + sold_last_x_days=180, + listing_type="sold", ) assert result is not None and len(result) > 0 @@ -28,6 +30,21 @@ def test_realtor_last_x_days_sold(): assert all([result is not None for result in [days_result_30, days_result_10]]) and len(days_result_30) != len(days_result_10) +def test_realtor_single_property(): + results = [ + scrape_property( + location="15509 N 172nd Dr, Surprise, AZ 85388", + listing_type="for_sale", + ), + scrape_property( + location="2530 Al Lipscomb Way", + listing_type="for_sale", + ) + ] + + assert all([result is not None for result in results]) + + def test_realtor(): results = [ scrape_property( From 51bde20c3c9fcf6bd8bcab23be430c14481ed5fa Mon Sep 17 00:00:00 2001 From: Cullen Watson Date: Wed, 4 Oct 2023 08:58:55 -0500 Subject: [PATCH 09/19] [chore]: clean up --- README.md | 155 ++++---- .../HomeHarvest_Demo.ipynb | 0 homeharvest/__init__.py | 24 +- homeharvest/cli.py | 5 +- homeharvest/core/scrapers/models.py | 43 +-- homeharvest/core/scrapers/realtor/__init__.py | 360 ++++++++---------- homeharvest/exceptions.py | 12 - homeharvest/utils.py | 26 +- 8 files changed, 277 insertions(+), 348 deletions(-) rename HomeHarvest_Demo.ipynb => examples/HomeHarvest_Demo.ipynb (100%) diff --git a/README.md b/README.md index 5610f47..f2ccc80 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ -**HomeHarvest** is a simple, yet comprehensive, real estate scraping library. +**HomeHarvest** is a simple, yet comprehensive, real estate scraping library that extracts and formats data in the style of MLS listings. [![Try with Replit](https://replit.com/badge?caption=Try%20with%20Replit)](https://replit.com/@ZacharyHampton/HomeHarvestDemo) @@ -11,10 +11,14 @@ Check out another project we wrote: ***[JobSpy](https://github.com/cullenwatson/JobSpy)** – a Python package for job scraping* -## Features +## HomeHarvest Features -- Scrapes properties from **Zillow**, **Realtor.com** & **Redfin** simultaneously -- Aggregates the properties in a Pandas DataFrame +- **Source**: Fetches properties directly from **Realtor.com**. +- **Data Format**: Structures data to resemble MLS listings. +- **Export Flexibility**: Options to save as either CSV or Excel. +- **Usage Modes**: + - **CLI**: For users who prefer command-line operations. + - **Python**: For those who'd like to integrate scraping into their Python scripts. [Video Guide for HomeHarvest](https://youtu.be/JnV7eR2Ve2o) - _updated for release v0.2.7_ @@ -29,21 +33,6 @@ pip install homeharvest ## Usage -### Python - -```py -from homeharvest import scrape_property -import pandas as pd - -properties: pd.DataFrame = scrape_property( - location="85281", - listing_type="for_rent" # for_sale / sold -) - -#: Note, to export to CSV or Excel, use properties.to_csv() or properties.to_excel(). -print(properties) -``` - ### CLI ``` @@ -55,7 +44,6 @@ positional arguments: location Location to scrape (e.g., San Francisco, CA) options: - -h, --help show this help message and exit -l {for_sale,for_rent,sold}, --listing_type {for_sale,for_rent,sold} Listing type to scrape -o {excel,csv}, --output {excel,csv} @@ -72,104 +60,107 @@ options: > homeharvest "San Francisco, CA" -l for_rent -o excel -f HomeHarvest ``` -## Output +### Python + ```py ->>> properties.head() - property_url site_name listing_type apt_min_price apt_max_price ... -0 https://www.redfin.com/AZ/Tempe/1003-W-Washing... redfin for_rent 1666.0 2750.0 ... -1 https://www.redfin.com/AZ/Tempe/VELA-at-Town-L... redfin for_rent 1665.0 3763.0 ... -2 https://www.redfin.com/AZ/Tempe/Camden-Tempe/a... redfin for_rent 1939.0 3109.0 ... -3 https://www.redfin.com/AZ/Tempe/Emerson-Park/a... redfin for_rent 1185.0 1817.0 ... -4 https://www.redfin.com/AZ/Tempe/Rio-Paradiso-A... redfin for_rent 1470.0 2235.0 ... -[5 rows x 41 columns] +from homeharvest import scrape_property +from datetime import datetime + +# Generate filename based on current timestamp +current_timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") +filename = f"output/{current_timestamp}.csv" + +properties = scrape_property( + location="San Diego, CA", + listing_type="sold", # for_sale, for_rent +) +print(f"Number of properties: {len(properties)}") +properties.to_csv(filename, index=False) ``` -### Parameters for `scrape_properties()` + +## Output ```plaintext +>>> properties.head() + MLS MLS # Status Style ... COEDate LotSFApx PrcSqft Stories +0 SDCA 230018348 SOLD CONDOS ... 2023-10-03 290110 803 2 +1 SDCA 230016614 SOLD TOWNHOMES ... 2023-10-03 None 838 3 +2 SDCA 230016367 SOLD CONDOS ... 2023-10-03 30056 649 1 +3 MRCA NDP2306335 SOLD SINGLE_FAMILY ... 2023-10-03 7519 661 2 +4 SDCA 230014532 SOLD CONDOS ... 2023-10-03 None 752 1 +[5 rows x 22 columns] +``` + +### Parameters for `scrape_property()` +``` Required ├── location (str): address in various formats e.g. just zip, full address, city/state, etc. └── listing_type (enum): for_rent, for_sale, sold Optional -├── site_name (list[enum], default=all three sites): zillow, realtor.com, redfin -├── proxy (str): in format 'http://user:pass@host:port' or [https, socks] -└── keep_duplicates (bool, default=False): whether to keep or remove duplicate properties based on address +├── radius_for_comps (float): Radius in miles to find comparable properties based on individual addresses. +├── sold_last_x_days (int): Number of past days to filter sold properties. +├── proxy (str): in format 'http://user:pass@host:port' ``` - ### Property Schema ```plaintext Property ├── Basic Information: -│ ├── property_url (str) -│ ├── site_name (enum): zillow, redfin, realtor.com -│ ├── listing_type (enum): for_sale, for_rent, sold -│ └── property_type (enum): house, apartment, condo, townhouse, single_family, multi_family, building +│ ├── property_url (str) +│ ├── mls (str) +│ ├── mls_id (str) +│ └── status (str) ├── Address Details: -│ ├── street_address (str) -│ ├── city (str) -│ ├── state (str) -│ ├── zip_code (str) -│ ├── unit (str) -│ └── country (str) +│ ├── street (str) +│ ├── unit (str) +│ ├── city (str) +│ ├── state (str) +│ └── zip (str) -├── House for Sale Features: -│ ├── tax_assessed_value (int) -│ ├── lot_area_value (float) -│ ├── lot_area_unit (str) -│ ├── stories (int) -│ ├── year_built (int) -│ └── price_per_sqft (int) +├── Property Description: +│ ├── style (str) +│ ├── beds (int) +│ ├── baths_full (int) +│ ├── baths_half (int) +│ ├── sqft (int) +│ ├── lot_sqft (int) +│ ├── sold_price (int) +│ ├── year_built (int) +│ ├── garage (float) +│ └── stories (int) -├── Building for Sale and Apartment Details: -│ ├── bldg_name (str) -│ ├── beds_min (int) -│ ├── beds_max (int) -│ ├── baths_min (float) -│ ├── baths_max (float) -│ ├── sqft_min (int) -│ ├── sqft_max (int) -│ ├── price_min (int) -│ ├── price_max (int) -│ ├── area_min (int) -│ └── unit_count (int) +├── Property Listing Details: +│ ├── list_price (int) +│ ├── list_date (str) +│ ├── last_sold_date (str) +│ ├── prc_sqft (int) +│ └── hoa_fee (int) -├── Miscellaneous Details: -│ ├── mls_id (str) -│ ├── agent_name (str) -│ ├── img_src (str) -│ ├── description (str) -│ ├── status_text (str) -│ └── posted_time (str) - -└── Location Details: - ├── latitude (float) - └── longitude (float) +├── Location Details: +│ ├── latitude (float) +│ ├── longitude (float) +│ └── neighborhoods (str) ``` ## Supported Countries for Property Scraping -* **Zillow**: contains listings in the **US** & **Canada** * **Realtor.com**: mainly from the **US** but also has international listings -* **Redfin**: listings mainly in the **US**, **Canada**, & has expanded to some areas in **Mexico** ### Exceptions The following exceptions may be raised when using HomeHarvest: -- `InvalidSite` - valid options: `zillow`, `redfin`, `realtor.com` - `InvalidListingType` - valid options: `for_sale`, `for_rent`, `sold` - `NoResultsFound` - no properties found from your input -- `GeoCoordsNotFound` - if Zillow scraper is not able to derive geo-coordinates from the location you input ## Frequently Asked Questions - --- -**Q: Encountering issues with your queries?** -**A:** Try a single site and/or broaden the location. If problems persist, [submit an issue](https://github.com/ZacharyHampton/HomeHarvest/issues). +**Q: Encountering issues with your searches?** +**A:** Try to broaden the location. If problems persist, [submit an issue](https://github.com/ZacharyHampton/HomeHarvest/issues). --- **Q: Received a Forbidden 403 response code?** -**A:** This indicates that you have been blocked by the real estate site for sending too many requests. Currently, **Zillow** is particularly aggressive with blocking. We recommend: +**A:** This indicates that you have been blocked by Realtor.com for sending too many requests. We recommend: - Waiting a few seconds between requests. - Trying a VPN to change your IP address. diff --git a/HomeHarvest_Demo.ipynb b/examples/HomeHarvest_Demo.ipynb similarity index 100% rename from HomeHarvest_Demo.ipynb rename to examples/HomeHarvest_Demo.ipynb diff --git a/homeharvest/__init__.py b/homeharvest/__init__.py index 6eb5157..0e6ce6d 100644 --- a/homeharvest/__init__.py +++ b/homeharvest/__init__.py @@ -1,5 +1,4 @@ import pandas as pd -from typing import Union import concurrent.futures from concurrent.futures import ThreadPoolExecutor @@ -7,7 +6,7 @@ from .core.scrapers import ScraperInput from .utils import process_result, ordered_properties from .core.scrapers.realtor import RealtorScraper from .core.scrapers.models import ListingType, Property, SiteName -from .exceptions import InvalidSite, InvalidListingType +from .exceptions import InvalidListingType _scrapers = { @@ -15,10 +14,7 @@ _scrapers = { } -def _validate_input(site_name: str, listing_type: str) -> None: - if site_name.lower() not in _scrapers: - raise InvalidSite(f"Provided site, '{site_name}', does not exist.") - +def _validate_input(listing_type: str) -> None: if listing_type.upper() not in ListingType.__members__: raise InvalidListingType(f"Provided listing type, '{listing_type}', does not exist.") @@ -27,7 +23,7 @@ def _scrape_single_site(location: str, site_name: str, listing_type: str, radius """ Helper function to scrape a single site. """ - _validate_input(site_name, listing_type) + _validate_input(listing_type) scraper_input = ScraperInput( location=location, @@ -40,6 +36,7 @@ def _scrape_single_site(location: str, site_name: str, listing_type: str, radius site = _scrapers[site_name.lower()](scraper_input) results = site.search() + print(f"found {len(results)}") properties_dfs = [process_result(result) for result in results] if not properties_dfs: @@ -50,22 +47,19 @@ def _scrape_single_site(location: str, site_name: str, listing_type: str, radius def scrape_property( location: str, - #: site_name: Union[str, list[str]] = "realtor.com", listing_type: str = "for_sale", radius: float = None, sold_last_x_days: int = None, proxy: str = None, ) -> pd.DataFrame: """ - Scrape property from various sites from a given location and listing type. + Scrape properties from Realtor.com based on a given location and listing type. - :param sold_last_x_days: Sold in last x days - :param radius: Radius in miles to find comparable properties on individual addresses - :param keep_duplicates: - :param proxy: :param location: US Location (e.g. 'San Francisco, CA', 'Cook County, IL', '85281', '2530 Al Lipscomb Way') - :param site_name: Site name or list of site names (e.g. ['realtor.com', 'zillow'], 'redfin') - :param listing_type: Listing type (e.g. 'for_sale', 'for_rent', 'sold') + :param listing_type: Listing type (e.g. 'for_sale', 'for_rent', 'sold'). Default is 'for_sale'. + :param radius: Radius in miles to find comparable properties on individual addresses. Optional. + :param sold_last_x_days: Number of past days to filter sold properties. Optional. + :param proxy: Proxy IP address to be used for scraping. Optional. :returns: pd.DataFrame containing properties """ site_name = "realtor.com" diff --git a/homeharvest/cli.py b/homeharvest/cli.py index 6d4d72d..65850b4 100644 --- a/homeharvest/cli.py +++ b/homeharvest/cli.py @@ -38,7 +38,8 @@ def main(): parser.add_argument( "-r", - "--radius", + "--sold-properties-radius", + dest="sold_properties_radius", # This makes sure the parsed argument is stored as radius_for_comps in args type=float, default=None, help="Get comparable properties within _ (eg. 0.0) miles. Only applicable for individual addresses." @@ -46,7 +47,7 @@ def main(): args = parser.parse_args() - result = scrape_property(args.location, args.listing_type, proxy=args.proxy) + result = scrape_property(args.location, args.listing_type, radius_for_comps=args.radius_for_comps, proxy=args.proxy) if not args.filename: timestamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S") diff --git a/homeharvest/core/scrapers/models.py b/homeharvest/core/scrapers/models.py index 00d2a3b..a8ae258 100644 --- a/homeharvest/core/scrapers/models.py +++ b/homeharvest/core/scrapers/models.py @@ -32,39 +32,34 @@ class Address: @dataclass -class Agent: - name: str - phone: str | None = None - email: str | None = None +class Description: + style: str | None = None + beds: int | None = None + baths_full: int | None = None + baths_half: int | None = None + sqft: int | None = None + lot_sqft: int | None = None + sold_price: int | None = None + year_built: int | None = None + garage: float | None = None + stories: int | None = None @dataclass class Property: - property_url: str | None = None + property_url: str mls: str | None = None mls_id: str | None = None status: str | None = None - style: str | None = None - - beds: int | None = None - baths_full: int | None = None - baths_half: int | None = None - list_price: int | None = None - list_date: str | None = None - sold_price: int | None = None - last_sold_date: str | None = None - prc_sqft: float | None = None - est_sf: int | None = None - lot_sf: int | None = None - hoa_fee: int | None = None - address: Address | None = None - yr_blt: int | None = None + list_price: int | None = None + list_date: str | None = None + last_sold_date: str | None = None + prc_sqft: int | None = None + hoa_fee: int | None = None + description: Description | None = None + latitude: float | None = None longitude: float | None = None - - stories: int | None = None - prkg_gar: float | None = None - neighborhoods: Optional[str] = None diff --git a/homeharvest/core/scrapers/realtor/__init__.py b/homeharvest/core/scrapers/realtor/__init__.py index fb79a37..629a653 100644 --- a/homeharvest/core/scrapers/realtor/__init__.py +++ b/homeharvest/core/scrapers/realtor/__init__.py @@ -2,38 +2,26 @@ homeharvest.realtor.__init__ ~~~~~~~~~~~~ -This module implements the scraper for relator.com +This module implements the scraper for realtor.com """ -from ..models import Property, Address, ListingType +from typing import Dict, Union, Optional +from concurrent.futures import ThreadPoolExecutor, as_completed + from .. import Scraper from ....exceptions import NoResultsFound -from concurrent.futures import ThreadPoolExecutor, as_completed +from ..models import Property, Address, ListingType, Description class RealtorScraper(Scraper): + SEARCH_URL = "https://www.realtor.com/api/v1/rdc_search_srp?client_id=rdc-search-new-communities&schema=vesta" + PROPERTY_URL = "https://www.realtor.com/realestateandhomes-detail/" + ADDRESS_AUTOCOMPLETE_URL = "https://parser-external.geo.moveaws.com/suggest" + def __init__(self, scraper_input): self.counter = 1 super().__init__(scraper_input) - self.search_url = ( - "https://www.realtor.com/api/v1/rdc_search_srp?client_id=rdc-search-new-communities&schema=vesta" - ) def handle_location(self): - headers = { - "authority": "parser-external.geo.moveaws.com", - "accept": "*/*", - "accept-language": "en-US,en;q=0.9", - "origin": "https://www.realtor.com", - "referer": "https://www.realtor.com/", - "sec-ch-ua": '"Chromium";v="116", "Not)A;Brand";v="24", "Google Chrome";v="116"', - "sec-ch-ua-mobile": "?0", - "sec-ch-ua-platform": '"Windows"', - "sec-fetch-dest": "empty", - "sec-fetch-mode": "cors", - "sec-fetch-site": "cross-site", - "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36", - } - params = { "input": self.location, "client_id": self.listing_type.value.lower().replace("_", "-"), @@ -42,9 +30,8 @@ class RealtorScraper(Scraper): } response = self.session.get( - "https://parser-external.geo.moveaws.com/suggest", + self.ADDRESS_AUTOCOMPLETE_URL, params=params, - headers=headers, ) response_json = response.json() @@ -70,22 +57,19 @@ class RealtorScraper(Scraper): stories } address { - address_validation_code - city - country - county - line - postal_code - state_code - street_direction - street_name street_number + street_name street_suffix - street_post_direction - unit_value unit - unit_descriptor - zip + city + state_code + postal_code + location { + coordinate { + lat + lon + } + } } basic { baths @@ -113,25 +97,24 @@ class RealtorScraper(Scraper): "variables": variables, } - response = self.session.post(self.search_url, json=payload) + response = self.session.post(self.SEARCH_URL, json=payload) response_json = response.json() property_info = response_json["data"]["property"] return [ Property( - property_url="https://www.realtor.com/realestateandhomes-detail/" - + property_info["details"]["permalink"], - stories=property_info["details"]["stories"], mls_id=property_id, + property_url=f"{self.PROPERTY_URL}{property_info['details']['permalink']}", + address=self._parse_address(property_info, search_type="handle_address"), + description=self._parse_description(property_info) ) ] - def general_search(self, variables: dict, search_type: str, return_total: bool = False) -> list[Property] | int: + def general_search(self, variables: dict, search_type: str) -> Dict[str, Union[int, list[Property]]]: """ Handles a location area & returns a list of properties """ - results_query = """{ count total @@ -141,86 +124,87 @@ class RealtorScraper(Scraper): status last_sold_price last_sold_date - hoa { - fee - } + list_price + price_per_sqft description { + sqft + beds baths_full baths_half - beds lot_sqft - sqft sold_price year_built garage sold_price type - sub_type name stories } source { - raw { - area - status - style - } - last_update_date - contract_date id listing_id - name - type - listing_href - community_id - management_id - corporation_id - subdivision_status - spec_id - plan_id - tier_rank - feed_type + } + hoa { + fee } location { address { + street_number + street_name + street_suffix + unit city - country - line - postal_code state_code - state + postal_code coordinate { lon lat } - street_direction - street_name - street_number - street_post_direction - street_suffix - unit } neighborhoods { - name + name } } - list_price - price_per_sqft - style_category_tags { - exterior - } - source { - id - } } } }""" sold_date_param = ('sold_date: { min: "$today-%sD" }' % self.sold_last_x_days - if self.listing_type == ListingType.SOLD and self.sold_last_x_days is not None + if self.listing_type == ListingType.SOLD and self.sold_last_x_days else "") + sort_param = ('sort: [{ field: sold_date, direction: desc }]' + if self.listing_type == ListingType.SOLD + else 'sort: [{ field: list_date, direction: desc }]') - if search_type == "area": + if search_type == "comps": + print('general - comps') + query = ( + """query Property_search( + $coordinates: [Float]! + $radius: String! + $offset: Int!, + ) { + property_search( + query: { + nearby: { + coordinates: $coordinates + radius: $radius + } + status: %s + %s + } + %s + limit: 200 + offset: $offset + ) %s""" % ( + self.listing_type.value.lower(), + sold_date_param, + sort_param, + results_query + ) + ) + else: + print('general - not comps') query = ( """query Home_search( $city: String, @@ -238,60 +222,27 @@ class RealtorScraper(Scraper): status: %s %s } + %s limit: 200 offset: $offset ) %s""" % ( self.listing_type.value.lower(), sold_date_param, + sort_param, results_query ) ) - elif search_type == "comp_address": - query = ( - """query Property_search( - $coordinates: [Float]! - $radius: String! - $offset: Int!, - ) { - property_search( - query: { - nearby: { - coordinates: $coordinates - radius: $radius - } - %s - } - limit: 200 - offset: $offset - ) %s""" % (sold_date_param, results_query)) - else: - query = ( - """query Property_search( - $property_id: [ID]! - $offset: Int!, - ) { - property_search( - query: { - property_id: $property_id - %s - } - limit: 200 - offset: $offset - ) %s""" % (sold_date_param, results_query)) payload = { "query": query, "variables": variables, } - response = self.session.post(self.search_url, json=payload) + response = self.session.post(self.SEARCH_URL, json=payload) response.raise_for_status() response_json = response.json() - search_key = "home_search" if search_type == "area" else "property_search" - - if return_total: - return response_json["data"][search_key]["total"] + search_key = "property_search" if search_type == "comps" else "home_search" properties: list[Property] = [] @@ -303,7 +254,7 @@ class RealtorScraper(Scraper): or response_json["data"][search_key] is None or "results" not in response_json["data"][search_key] ): - return [] + return {"total": 0, "properties": []} for result in response_json["data"][search_key]["results"]: self.counter += 1 @@ -312,122 +263,131 @@ class RealtorScraper(Scraper): if "source" in result and isinstance(result["source"], dict) else None ) - mls_id = ( - result["source"].get("listing_id") - if "source" in result and isinstance(result["source"], dict) - else None - ) - if not mls_id: + if not mls: continue - # not type - - neighborhoods_list = [] - neighborhoods = result["location"].get("neighborhoods", []) - - if neighborhoods: - for neighborhood in neighborhoods: - name = neighborhood.get("name") - if name: - neighborhoods_list.append(name) - - neighborhoods_str = ( - ", ".join(neighborhoods_list) if neighborhoods_list else None - ) able_to_get_lat_long = result and result.get("location") and result["location"].get("address") and result["location"]["address"].get("coordinate") realty_property = Property( - property_url="https://www.realtor.com/realestateandhomes-detail/" - + result["property_id"], mls=mls, - mls_id=mls_id, + mls_id=result["source"].get("listing_id") if "source" in result and isinstance(result["source"], dict) else None, + property_url=f"{self.PROPERTY_URL}{result['property_id']}", status=result["status"].upper(), - style=result["description"]["type"].upper(), - beds=result["description"]["beds"], - baths_full=result["description"]["baths_full"], - baths_half=result["description"]["baths_half"], - est_sf=result["description"]["sqft"], - lot_sf=result["description"]["lot_sqft"], list_price=result["list_price"], - list_date=result["list_date"].split("T")[0] - if result["list_date"] - else None, - sold_price=result["description"]["sold_price"], - prc_sqft=result["price_per_sqft"], - last_sold_date=result["last_sold_date"], + list_date=result["list_date"].split("T")[0] if result.get("list_date") else None, + prc_sqft=result.get("price_per_sqft"), + last_sold_date=result.get("last_sold_date"), hoa_fee=result["hoa"]["fee"] if result.get("hoa") and isinstance(result["hoa"], dict) else None, - address=Address( - street=f"{result['location']['address']['street_number']} {result['location']['address']['street_name']} {result['location']['address']['street_suffix']}", - unit=result["location"]["address"]["unit"], - city=result["location"]["address"]["city"], - state=result["location"]["address"]["state_code"], - zip=result["location"]["address"]["postal_code"], - ), - yr_blt=result["description"]["year_built"], latitude=result["location"]["address"]["coordinate"].get("lat") if able_to_get_lat_long else None, longitude=result["location"]["address"]["coordinate"].get("lon") if able_to_get_lat_long else None, - prkg_gar=result["description"]["garage"], - stories=result["description"]["stories"], - neighborhoods=neighborhoods_str, + address=self._parse_address(result, search_type="general_search"), + neighborhoods=self._parse_neighborhoods(result), + description=self._parse_description(result) ) properties.append(realty_property) - return properties + + # print(response_json["data"]["property_search"], variables["offset"]) + # print(response_json["data"]["home_search"]["total"], variables["offset"]) + return { + "total": response_json["data"][search_key]["total"], + "properties": properties, + } def search(self): location_info = self.handle_location() location_type = location_info["area_type"] - is_for_comps = self.radius is not None and location_type == "address" - offset = 0 search_variables = { - "offset": offset, + "offset": 0, } - search_type = "comp_address" if is_for_comps \ - else "address" if location_type == "address" and not is_for_comps \ - else "area" + search_type = "comps" if self.radius and location_type == "address" else "area" + print(search_type) + if location_type == "address": + if not self.radius: #: single address search, non comps + property_id = location_info["mpr_id"] + search_variables |= {"property_id": property_id} + return self.handle_address(property_id) - if location_type == "address" and not is_for_comps: #: single address search, non comps - property_id = location_info["mpr_id"] - search_variables = search_variables | {"property_id": property_id} + else: #: general search, comps (radius) + coordinates = list(location_info["centroid"].values()) + search_variables |= { + "coordinates": coordinates, + "radius": "{}mi".format(self.radius), + } - general_search = self.general_search(search_variables, search_type) - if general_search: - return general_search - else: - return self.handle_address(property_id) #: TODO: support single address search for query by property address (can go from property -> listing to get better data) - - elif not is_for_comps: #: area search - search_variables = search_variables | { + else: #: general search, location + search_variables |= { "city": location_info.get("city"), "county": location_info.get("county"), "state_code": location_info.get("state_code"), "postal_code": location_info.get("postal_code"), } - else: #: comps search - coordinates = list(location_info["centroid"].values()) - search_variables = search_variables | { - "coordinates": coordinates, - "radius": "{}mi".format(self.radius), - } - total = self.general_search(search_variables, return_total=True, search_type=search_type) + result = self.general_search(search_variables, search_type=search_type) + total = result["total"] + homes = result["properties"] - homes = [] with ThreadPoolExecutor(max_workers=10) as executor: futures = [ executor.submit( self.general_search, variables=search_variables | {"offset": i}, - return_total=False, search_type=search_type, ) - for i in range(0, total, 200) + for i in range(200, min(total, 10000), 200) ] for future in as_completed(futures): - homes.extend(future.result()) + homes.extend(future.result()["properties"]) return homes + + @staticmethod + def _parse_neighborhoods(result: dict) -> Optional[str]: + neighborhoods_list = [] + neighborhoods = result["location"].get("neighborhoods", []) + + if neighborhoods: + for neighborhood in neighborhoods: + name = neighborhood.get("name") + if name: + neighborhoods_list.append(name) + + return ", ".join(neighborhoods_list) if neighborhoods_list else None + + @staticmethod + def _parse_address(result: dict, search_type): + if search_type == "general_search": + return Address( + street=f"{result['location']['address']['street_number']} {result['location']['address']['street_name']} {result['location']['address']['street_suffix']}", + unit=result["location"]["address"]["unit"], + city=result["location"]["address"]["city"], + state=result["location"]["address"]["state_code"], + zip=result["location"]["address"]["postal_code"], + ) + return Address( + street=f"{result['address']['street_number']} {result['address']['street_name']} {result['address']['street_suffix']}", + unit=result['address']['unit'], + city=result['address']['city'], + state=result['address']['state_code'], + zip=result['address']['postal_code'], + ) + + @staticmethod + def _parse_description(result: dict) -> Description: + description_data = result.get("description", {}) + return Description( + style=description_data.get("type", "").upper(), + beds=description_data.get("beds"), + baths_full=description_data.get("baths_full"), + baths_half=description_data.get("baths_half"), + sqft=description_data.get("sqft"), + lot_sqft=description_data.get("lot_sqft"), + sold_price=description_data.get("sold_price"), + year_built=description_data.get("year_built"), + garage=description_data.get("garage"), + stories=description_data.get("stories"), + ) \ No newline at end of file diff --git a/homeharvest/exceptions.py b/homeharvest/exceptions.py index 95eedbc..f018c97 100644 --- a/homeharvest/exceptions.py +++ b/homeharvest/exceptions.py @@ -1,18 +1,6 @@ -class InvalidSite(Exception): - """Raised when a provided site is does not exist.""" - - class InvalidListingType(Exception): """Raised when a provided listing type is does not exist.""" class NoResultsFound(Exception): """Raised when no results are found for the given location""" - - -class GeoCoordsNotFound(Exception): - """Raised when no property is found for the given address""" - - -class SearchTooBroad(Exception): - """Raised when the search is too broad""" diff --git a/homeharvest/utils.py b/homeharvest/utils.py index f522cd4..58fbce0 100644 --- a/homeharvest/utils.py +++ b/homeharvest/utils.py @@ -39,7 +39,6 @@ def process_result(result: Property) -> pd.DataFrame: prop_data["MLS"] = prop_data["mls"] prop_data["MLS #"] = prop_data["mls_id"] prop_data["Status"] = prop_data["status"] - prop_data["Style"] = prop_data["style"] if "address" in prop_data: address_data = prop_data["address"] @@ -49,26 +48,27 @@ def process_result(result: Property) -> pd.DataFrame: prop_data["State"] = address_data.state prop_data["Zip"] = address_data.zip - prop_data["Community"] = prop_data["neighborhoods"] - prop_data["Beds"] = prop_data["beds"] - prop_data["FB"] = prop_data["baths_full"] - prop_data["NumHB"] = prop_data["baths_half"] - prop_data["EstSF"] = prop_data["est_sf"] prop_data["ListPrice"] = prop_data["list_price"] prop_data["Lst Date"] = prop_data["list_date"] - prop_data["Sold Price"] = prop_data["sold_price"] prop_data["COEDate"] = prop_data["last_sold_date"] - prop_data["LotSFApx"] = prop_data["lot_sf"] + prop_data["PrcSqft"] = prop_data["prc_sqft"] prop_data["HOAFee"] = prop_data["hoa_fee"] - if prop_data.get("prc_sqft") is not None: - prop_data["PrcSqft"] = round(prop_data["prc_sqft"], 2) + description = result.description + prop_data["Style"] = description.style + prop_data["Beds"] = description.beds + prop_data["FB"] = description.baths_full + prop_data["NumHB"] = description.baths_half + prop_data["EstSF"] = description.sqft + prop_data["LotSFApx"] = description.lot_sqft + prop_data["Sold Price"] = description.sold_price + prop_data["YrBlt"] = description.year_built + prop_data["PrkgGar"] = description.garage + prop_data["Stories"] = description.stories - prop_data["YrBlt"] = prop_data["yr_blt"] prop_data["LATITUDE"] = prop_data["latitude"] prop_data["LONGITUDE"] = prop_data["longitude"] - prop_data["Stories"] = prop_data["stories"] - prop_data["PrkgGar"] = prop_data["prkg_gar"] + prop_data["Community"] = prop_data["neighborhoods"] properties_df = pd.DataFrame([prop_data]) properties_df = properties_df.reindex(columns=ordered_properties) From c4870677c27c754f77229ef52c920ecb962cdc98 Mon Sep 17 00:00:00 2001 From: Cullen Watson Date: Wed, 4 Oct 2023 10:11:53 -0500 Subject: [PATCH 10/19] [enh]: make last_x_days generic add mls_only make radius generic --- README.md | 95 ++++++++------ examples/HomeHarvest_Demo.ipynb | 5 +- examples/HomeHarvest_Demo.py | 18 +++ homeharvest/__init__.py | 108 ++++------------ homeharvest/cli.py | 36 +++++- homeharvest/core/scrapers/__init__.py | 14 +- homeharvest/core/scrapers/realtor/__init__.py | 120 ++++++++++-------- homeharvest/utils.py | 11 +- tests/test_realtor.py | 14 +- 9 files changed, 220 insertions(+), 201 deletions(-) create mode 100644 examples/HomeHarvest_Demo.py diff --git a/README.md b/README.md index f2ccc80..f8b2525 100644 --- a/README.md +++ b/README.md @@ -36,13 +36,13 @@ pip install homeharvest ### CLI ``` -usage: homeharvest [-h] [-l {for_sale,for_rent,sold}] [-o {excel,csv}] [-f FILENAME] [-p PROXY] [-d DAYS] [-r RADIUS] location - +usage: homeharvest [-l {for_sale,for_rent,sold}] [-o {excel,csv}] [-f FILENAME] [-p PROXY] [-d DAYS] [-r RADIUS] [-m] location + Home Harvest Property Scraper - + positional arguments: location Location to scrape (e.g., San Francisco, CA) - + options: -l {for_sale,for_rent,sold}, --listing_type {for_sale,for_rent,sold} Listing type to scrape @@ -54,7 +54,8 @@ options: Proxy to use for scraping -d DAYS, --days DAYS Sold in last _ days filter. -r RADIUS, --radius RADIUS - Get comparable properties within _ (eg. 0.0) miles. Only applicable for individual addresses. + Get comparable properties within _ (eg. 0.0) miles. Only applicable for individual addresses. + -m, --mls_only If set, fetches only MLS listings. ``` ```bash > homeharvest "San Francisco, CA" -l for_rent -o excel -f HomeHarvest @@ -73,9 +74,14 @@ filename = f"output/{current_timestamp}.csv" properties = scrape_property( location="San Diego, CA", listing_type="sold", # for_sale, for_rent + last_x_days=30, # sold/listed in last 30 days + mls_only=True, # only fetch MLS listings ) print(f"Number of properties: {len(properties)}") + +# Export to csv properties.to_csv(filename, index=False) +print(properties.head()) ``` @@ -94,12 +100,23 @@ properties.to_csv(filename, index=False) ### Parameters for `scrape_property()` ``` Required -├── location (str): address in various formats e.g. just zip, full address, city/state, etc. -└── listing_type (enum): for_rent, for_sale, sold +├── location (str): The address in various formats - this could be just a zip code, a full address, or city/state, etc. +└── listing_type (option): Choose the type of listing. + - 'for_rent' + - 'for_sale' + - 'sold' + Optional -├── radius_for_comps (float): Radius in miles to find comparable properties based on individual addresses. -├── sold_last_x_days (int): Number of past days to filter sold properties. -├── proxy (str): in format 'http://user:pass@host:port' +├── radius (decimal): Radius in miles to find comparable properties based on individual addresses. +│ Example: 5.5 (fetches properties within a 5.5-mile radius if location is set to a specific address; otherwise, ignored) +│ +├── last_x_days (integer): Number of past days to filter properties. Utilizes 'COEDate' for 'sold' listing types, and 'Lst Date' for others (for_rent, for_sale). +│ Example: 30 (fetches properties listed/sold in the last 30 days) +│ +├── mls_only (True/False): If set, fetches only MLS listings (mainly applicable to 'sold' listings) +│ +└── proxy (string): In format 'http://user:pass@host:port' + ``` ### Property Schema ```plaintext @@ -111,51 +128,49 @@ Property │ └── status (str) ├── Address Details: -│ ├── street (str) -│ ├── unit (str) -│ ├── city (str) -│ ├── state (str) -│ └── zip (str) +│ ├── street +│ ├── unit +│ ├── city +│ ├── state +│ └── zip ├── Property Description: -│ ├── style (str) -│ ├── beds (int) -│ ├── baths_full (int) -│ ├── baths_half (int) -│ ├── sqft (int) -│ ├── lot_sqft (int) -│ ├── sold_price (int) -│ ├── year_built (int) -│ ├── garage (float) -│ └── stories (int) +│ ├── style +│ ├── beds +│ ├── baths_full +│ ├── baths_half +│ ├── sqft +│ ├── lot_sqft +│ ├── sold_price +│ ├── year_built +│ ├── garage +│ └── stories ├── Property Listing Details: -│ ├── list_price (int) -│ ├── list_date (str) -│ ├── last_sold_date (str) -│ ├── prc_sqft (int) -│ └── hoa_fee (int) +│ ├── list_price +│ ├── list_date +│ ├── last_sold_date +│ ├── prc_sqft +│ └── hoa_fee ├── Location Details: -│ ├── latitude (float) -│ ├── longitude (float) -│ └── neighborhoods (str) +│ ├── latitude +│ ├── longitude +│ └── neighborhoods ``` -## Supported Countries for Property Scraping - -* **Realtor.com**: mainly from the **US** but also has international listings ### Exceptions The following exceptions may be raised when using HomeHarvest: - `InvalidListingType` - valid options: `for_sale`, `for_rent`, `sold` -- `NoResultsFound` - no properties found from your input - +- `NoResultsFound` - no properties found from your search + + ## Frequently Asked Questions --- **Q: Encountering issues with your searches?** -**A:** Try to broaden the location. If problems persist, [submit an issue](https://github.com/ZacharyHampton/HomeHarvest/issues). +**A:** Try to broaden the parameters you're using. If problems persist, [submit an issue](https://github.com/ZacharyHampton/HomeHarvest/issues). --- @@ -163,7 +178,7 @@ The following exceptions may be raised when using HomeHarvest: **A:** This indicates that you have been blocked by Realtor.com for sending too many requests. We recommend: - Waiting a few seconds between requests. -- Trying a VPN to change your IP address. +- Trying a VPN or useing a proxy as a parameter to scrape_property() to change your IP address. --- diff --git a/examples/HomeHarvest_Demo.ipynb b/examples/HomeHarvest_Demo.ipynb index fb9106b..43a28be 100644 --- a/examples/HomeHarvest_Demo.ipynb +++ b/examples/HomeHarvest_Demo.ipynb @@ -31,7 +31,7 @@ "metadata": {}, "outputs": [], "source": [ - "# scrapes all 3 sites by default\n", + "# check for sale properties\n", "scrape_property(\n", " location=\"dallas\",\n", " listing_type=\"for_sale\"\n", @@ -53,7 +53,6 @@ "# search a specific address\n", "scrape_property(\n", " location=\"2530 Al Lipscomb Way\",\n", - " site_name=\"zillow\",\n", " listing_type=\"for_sale\"\n", ")" ] @@ -68,7 +67,6 @@ "# check rentals\n", "scrape_property(\n", " location=\"chicago, illinois\",\n", - " site_name=[\"redfin\", \"zillow\"],\n", " listing_type=\"for_rent\"\n", ")" ] @@ -88,7 +86,6 @@ "# check sold properties\n", "scrape_property(\n", " location=\"90210\",\n", - " site_name=[\"redfin\"],\n", " listing_type=\"sold\"\n", ")" ] diff --git a/examples/HomeHarvest_Demo.py b/examples/HomeHarvest_Demo.py new file mode 100644 index 0000000..9e8e053 --- /dev/null +++ b/examples/HomeHarvest_Demo.py @@ -0,0 +1,18 @@ +from homeharvest import scrape_property +from datetime import datetime + +# Generate filename based on current timestamp +current_timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") +filename = f"output/{current_timestamp}.csv" + +properties = scrape_property( + location="San Diego, CA", + listing_type="sold", # for_sale, for_rent + last_x_days=30, # sold/listed in last 30 days + mls_only=True, # only fetch MLS listings +) +print(f"Number of properties: {len(properties)}") + +# Export to csv +properties.to_csv(filename, index=False) +print(properties.head()) \ No newline at end of file diff --git a/homeharvest/__init__.py b/homeharvest/__init__.py index 0e6ce6d..5d68d1d 100644 --- a/homeharvest/__init__.py +++ b/homeharvest/__init__.py @@ -1,103 +1,41 @@ +import warnings import pandas as pd -import concurrent.futures -from concurrent.futures import ThreadPoolExecutor - from .core.scrapers import ScraperInput -from .utils import process_result, ordered_properties +from .utils import process_result, ordered_properties, validate_input from .core.scrapers.realtor import RealtorScraper -from .core.scrapers.models import ListingType, Property, SiteName -from .exceptions import InvalidListingType - - -_scrapers = { - "realtor.com": RealtorScraper, -} - - -def _validate_input(listing_type: str) -> None: - if listing_type.upper() not in ListingType.__members__: - raise InvalidListingType(f"Provided listing type, '{listing_type}', does not exist.") - - -def _scrape_single_site(location: str, site_name: str, listing_type: str, radius: float, proxy: str = None, sold_last_x_days: int = None) -> pd.DataFrame: - """ - Helper function to scrape a single site. - """ - _validate_input(listing_type) - - scraper_input = ScraperInput( - location=location, - listing_type=ListingType[listing_type.upper()], - site_name=SiteName.get_by_value(site_name.lower()), - proxy=proxy, - radius=radius, - sold_last_x_days=sold_last_x_days - ) - - site = _scrapers[site_name.lower()](scraper_input) - results = site.search() - print(f"found {len(results)}") - - properties_dfs = [process_result(result) for result in results] - if not properties_dfs: - return pd.DataFrame() - - return pd.concat(properties_dfs, ignore_index=True, axis=0)[ordered_properties] +from .core.scrapers.models import ListingType +from .exceptions import InvalidListingType, NoResultsFound def scrape_property( location: str, listing_type: str = "for_sale", radius: float = None, - sold_last_x_days: int = None, + mls_only: bool = False, + last_x_days: int = None, proxy: str = None, ) -> pd.DataFrame: """ Scrape properties from Realtor.com based on a given location and listing type. - - :param location: US Location (e.g. 'San Francisco, CA', 'Cook County, IL', '85281', '2530 Al Lipscomb Way') - :param listing_type: Listing type (e.g. 'for_sale', 'for_rent', 'sold'). Default is 'for_sale'. - :param radius: Radius in miles to find comparable properties on individual addresses. Optional. - :param sold_last_x_days: Number of past days to filter sold properties. Optional. - :param proxy: Proxy IP address to be used for scraping. Optional. - :returns: pd.DataFrame containing properties """ - site_name = "realtor.com" + validate_input(listing_type) - if site_name is None: - site_name = list(_scrapers.keys()) + scraper_input = ScraperInput( + location=location, + listing_type=ListingType[listing_type.upper()], + proxy=proxy, + radius=radius, + mls_only=mls_only, + last_x_days=last_x_days, + ) - if not isinstance(site_name, list): - site_name = [site_name] + site = RealtorScraper(scraper_input) + results = site.search() - results = [] + properties_dfs = [process_result(result) for result in results] + if not properties_dfs: + raise NoResultsFound("no results found for the query") - if len(site_name) == 1: - final_df = _scrape_single_site(location, site_name[0], listing_type, radius, proxy, sold_last_x_days) - results.append(final_df) - else: - with ThreadPoolExecutor() as executor: - futures = { - executor.submit(_scrape_single_site, location, s_name, listing_type, radius, proxy, sold_last_x_days): s_name - for s_name in site_name - } - - for future in concurrent.futures.as_completed(futures): - result = future.result() - results.append(result) - - results = [df for df in results if not df.empty and not df.isna().all().all()] - - if not results: - return pd.DataFrame() - - final_df = pd.concat(results, ignore_index=True) - - columns_to_track = ["Street", "Unit", "Zip"] - - #: validate they exist, otherwise create them - for col in columns_to_track: - if col not in final_df.columns: - final_df[col] = None - - return final_df + with warnings.catch_warnings(): + warnings.simplefilter("ignore", category=FutureWarning) + return pd.concat(properties_dfs, ignore_index=True, axis=0)[ordered_properties] diff --git a/homeharvest/cli.py b/homeharvest/cli.py index 65850b4..b732486 100644 --- a/homeharvest/cli.py +++ b/homeharvest/cli.py @@ -5,7 +5,9 @@ from homeharvest import scrape_property def main(): parser = argparse.ArgumentParser(description="Home Harvest Property Scraper") - parser.add_argument("location", type=str, help="Location to scrape (e.g., San Francisco, CA)") + parser.add_argument( + "location", type=str, help="Location to scrape (e.g., San Francisco, CA)" + ) parser.add_argument( "-l", @@ -33,21 +35,41 @@ def main(): help="Name of the output file (without extension)", ) - parser.add_argument("-p", "--proxy", type=str, default=None, help="Proxy to use for scraping") - parser.add_argument("-d", "--days", type=int, default=None, help="Sold in last _ days filter.") + parser.add_argument( + "-p", "--proxy", type=str, default=None, help="Proxy to use for scraping" + ) + parser.add_argument( + "-d", + "--days", + type=int, + default=None, + help="Sold/listed in last _ days filter.", + ) parser.add_argument( "-r", - "--sold-properties-radius", - dest="sold_properties_radius", # This makes sure the parsed argument is stored as radius_for_comps in args + "--radius", type=float, default=None, - help="Get comparable properties within _ (eg. 0.0) miles. Only applicable for individual addresses." + help="Get comparable properties within _ (eg. 0.0) miles. Only applicable for individual addresses.", + ) + parser.add_argument( + "-m", + "--mls_only", + action="store_true", + help="If set, fetches only MLS listings.", ) args = parser.parse_args() - result = scrape_property(args.location, args.listing_type, radius_for_comps=args.radius_for_comps, proxy=args.proxy) + result = scrape_property( + args.location, + args.listing_type, + radius=args.radius, + proxy=args.proxy, + mls_only=args.mls_only, + last_x_days=args.days, + ) if not args.filename: timestamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S") diff --git a/homeharvest/core/scrapers/__init__.py b/homeharvest/core/scrapers/__init__.py index bc418e3..1ce4431 100644 --- a/homeharvest/core/scrapers/__init__.py +++ b/homeharvest/core/scrapers/__init__.py @@ -8,14 +8,18 @@ from .models import Property, ListingType, SiteName class ScraperInput: location: str listing_type: ListingType - site_name: SiteName radius: float | None = None + mls_only: bool | None = None proxy: str | None = None - sold_last_x_days: int | None = None + last_x_days: int | None = None class Scraper: - def __init__(self, scraper_input: ScraperInput, session: requests.Session | tls_client.Session = None): + def __init__( + self, + scraper_input: ScraperInput, + session: requests.Session | tls_client.Session = None, + ): self.location = scraper_input.location self.listing_type = scraper_input.listing_type @@ -30,9 +34,9 @@ class Scraper: self.session.proxies.update(proxies) self.listing_type = scraper_input.listing_type - self.site_name = scraper_input.site_name self.radius = scraper_input.radius - self.sold_last_x_days = scraper_input.sold_last_x_days + self.last_x_days = scraper_input.last_x_days + self.mls_only = scraper_input.mls_only def search(self) -> list[Property]: ... diff --git a/homeharvest/core/scrapers/realtor/__init__.py b/homeharvest/core/scrapers/realtor/__init__.py index 629a653..de1dcc4 100644 --- a/homeharvest/core/scrapers/realtor/__init__.py +++ b/homeharvest/core/scrapers/realtor/__init__.py @@ -106,12 +106,16 @@ class RealtorScraper(Scraper): Property( mls_id=property_id, property_url=f"{self.PROPERTY_URL}{property_info['details']['permalink']}", - address=self._parse_address(property_info, search_type="handle_address"), - description=self._parse_description(property_info) + address=self._parse_address( + property_info, search_type="handle_address" + ), + description=self._parse_description(property_info), ) ] - def general_search(self, variables: dict, search_type: str) -> Dict[str, Union[int, list[Property]]]: + def general_search( + self, variables: dict, search_type: str + ) -> Dict[str, Union[int, list[Property]]]: """ Handles a location area & returns a list of properties """ @@ -169,17 +173,23 @@ class RealtorScraper(Scraper): } }""" - sold_date_param = ('sold_date: { min: "$today-%sD" }' % self.sold_last_x_days - if self.listing_type == ListingType.SOLD and self.sold_last_x_days - else "") - sort_param = ('sort: [{ field: sold_date, direction: desc }]' - if self.listing_type == ListingType.SOLD - else 'sort: [{ field: list_date, direction: desc }]') + date_param = ( + 'sold_date: { min: "$today-%sD" }' % self.last_x_days + if self.listing_type == ListingType.SOLD and self.last_x_days + else ( + 'list_date: { min: "$today-%sD" }' % self.last_x_days + if self.last_x_days + else "" + ) + ) + sort_param = ( + "sort: [{ field: sold_date, direction: desc }]" + if self.listing_type == ListingType.SOLD + else "sort: [{ field: list_date, direction: desc }]" + ) if search_type == "comps": - print('general - comps') - query = ( - """query Property_search( + query = """query Property_search( $coordinates: [Float]! $radius: String! $offset: Int!, @@ -197,16 +207,13 @@ class RealtorScraper(Scraper): limit: 200 offset: $offset ) %s""" % ( - self.listing_type.value.lower(), - sold_date_param, - sort_param, - results_query - ) + self.listing_type.value.lower(), + date_param, + sort_param, + results_query, ) else: - print('general - not comps') - query = ( - """query Home_search( + query = """query Home_search( $city: String, $county: [String], $state_code: String, @@ -225,13 +232,11 @@ class RealtorScraper(Scraper): %s limit: 200 offset: $offset - ) %s""" - % ( - self.listing_type.value.lower(), - sold_date_param, - sort_param, - results_query - ) + ) %s""" % ( + self.listing_type.value.lower(), + date_param, + sort_param, + results_query, ) payload = { @@ -247,12 +252,12 @@ class RealtorScraper(Scraper): properties: list[Property] = [] if ( - response_json is None - or "data" not in response_json - or response_json["data"] is None - or search_key not in response_json["data"] - or response_json["data"][search_key] is None - or "results" not in response_json["data"][search_key] + response_json is None + or "data" not in response_json + or response_json["data"] is None + or search_key not in response_json["data"] + or response_json["data"][search_key] is None + or "results" not in response_json["data"][search_key] ): return {"total": 0, "properties": []} @@ -264,32 +269,44 @@ class RealtorScraper(Scraper): else None ) - if not mls: + if not mls and self.mls_only: continue - able_to_get_lat_long = result and result.get("location") and result["location"].get("address") and result["location"]["address"].get("coordinate") + able_to_get_lat_long = ( + result + and result.get("location") + and result["location"].get("address") + and result["location"]["address"].get("coordinate") + ) realty_property = Property( mls=mls, - mls_id=result["source"].get("listing_id") if "source" in result and isinstance(result["source"], dict) else None, + mls_id=result["source"].get("listing_id") + if "source" in result and isinstance(result["source"], dict) + else None, property_url=f"{self.PROPERTY_URL}{result['property_id']}", status=result["status"].upper(), list_price=result["list_price"], - list_date=result["list_date"].split("T")[0] if result.get("list_date") else None, + list_date=result["list_date"].split("T")[0] + if result.get("list_date") + else None, prc_sqft=result.get("price_per_sqft"), last_sold_date=result.get("last_sold_date"), - hoa_fee=result["hoa"]["fee"] if result.get("hoa") and isinstance(result["hoa"], dict) else None, - latitude=result["location"]["address"]["coordinate"].get("lat") if able_to_get_lat_long else None, - longitude=result["location"]["address"]["coordinate"].get("lon") if able_to_get_lat_long else None, + hoa_fee=result["hoa"]["fee"] + if result.get("hoa") and isinstance(result["hoa"], dict) + else None, + latitude=result["location"]["address"]["coordinate"].get("lat") + if able_to_get_lat_long + else None, + longitude=result["location"]["address"]["coordinate"].get("lon") + if able_to_get_lat_long + else None, address=self._parse_address(result, search_type="general_search"), neighborhoods=self._parse_neighborhoods(result), - description=self._parse_description(result) + description=self._parse_description(result), ) properties.append(realty_property) - - # print(response_json["data"]["property_search"], variables["offset"]) - # print(response_json["data"]["home_search"]["total"], variables["offset"]) return { "total": response_json["data"][search_key]["total"], "properties": properties, @@ -304,14 +321,13 @@ class RealtorScraper(Scraper): } search_type = "comps" if self.radius and location_type == "address" else "area" - print(search_type) if location_type == "address": - if not self.radius: #: single address search, non comps + if not self.radius: #: single address search, non comps property_id = location_info["mpr_id"] search_variables |= {"property_id": property_id} return self.handle_address(property_id) - else: #: general search, comps (radius) + else: #: general search, comps (radius) coordinates = list(location_info["centroid"].values()) search_variables |= { "coordinates": coordinates, @@ -370,10 +386,10 @@ class RealtorScraper(Scraper): ) return Address( street=f"{result['address']['street_number']} {result['address']['street_name']} {result['address']['street_suffix']}", - unit=result['address']['unit'], - city=result['address']['city'], - state=result['address']['state_code'], - zip=result['address']['postal_code'], + unit=result["address"]["unit"], + city=result["address"]["city"], + state=result["address"]["state_code"], + zip=result["address"]["postal_code"], ) @staticmethod @@ -390,4 +406,4 @@ class RealtorScraper(Scraper): year_built=description_data.get("year_built"), garage=description_data.get("garage"), stories=description_data.get("stories"), - ) \ No newline at end of file + ) diff --git a/homeharvest/utils.py b/homeharvest/utils.py index 58fbce0..1f7f717 100644 --- a/homeharvest/utils.py +++ b/homeharvest/utils.py @@ -1,4 +1,4 @@ -from .core.scrapers.models import Property +from .core.scrapers.models import Property, ListingType import pandas as pd ordered_properties = [ @@ -73,4 +73,11 @@ def process_result(result: Property) -> pd.DataFrame: properties_df = pd.DataFrame([prop_data]) properties_df = properties_df.reindex(columns=ordered_properties) - return properties_df[ordered_properties] \ No newline at end of file + return properties_df[ordered_properties] + + +def validate_input(listing_type: str) -> None: + if listing_type.upper() not in ListingType.__members__: + raise InvalidListingType( + f"Provided listing type, '{listing_type}', does not exist." + ) diff --git a/tests/test_realtor.py b/tests/test_realtor.py index 1c06848..15b7e09 100644 --- a/tests/test_realtor.py +++ b/tests/test_realtor.py @@ -9,10 +9,10 @@ from homeharvest.exceptions import ( def test_realtor_comps(): result = scrape_property( - location="2530 Al Lipscomb Way", - radius=0.5, - sold_last_x_days=180, - listing_type="sold", + location="2530 Al Lipscomb Way", + radius=0.5, + sold_last_x_days=180, + listing_type="sold", ) assert result is not None and len(result) > 0 @@ -27,7 +27,9 @@ def test_realtor_last_x_days_sold(): location="Dallas, TX", listing_type="sold", sold_last_x_days=10 ) - assert all([result is not None for result in [days_result_30, days_result_10]]) and len(days_result_30) != len(days_result_10) + assert all( + [result is not None for result in [days_result_30, days_result_10]] + ) and len(days_result_30) != len(days_result_10) def test_realtor_single_property(): @@ -39,7 +41,7 @@ def test_realtor_single_property(): scrape_property( location="2530 Al Lipscomb Way", listing_type="for_sale", - ) + ), ] assert all([result is not None for result in results]) From 68e15ce69686bc205b4d3234a09c012af888ee5a Mon Sep 17 00:00:00 2001 From: Cullen Watson Date: Wed, 4 Oct 2023 10:14:11 -0500 Subject: [PATCH 11/19] [docs] clarify example --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index f8b2525..0ae0978 100644 --- a/README.md +++ b/README.md @@ -52,7 +52,7 @@ options: Name of the output file (without extension) -p PROXY, --proxy PROXY Proxy to use for scraping - -d DAYS, --days DAYS Sold in last _ days filter. + -d DAYS, --days DAYS Sold/listed in last _ days filter. -r RADIUS, --radius RADIUS Get comparable properties within _ (eg. 0.0) miles. Only applicable for individual addresses. -m, --mls_only If set, fetches only MLS listings. @@ -73,8 +73,8 @@ filename = f"output/{current_timestamp}.csv" properties = scrape_property( location="San Diego, CA", - listing_type="sold", # for_sale, for_rent - last_x_days=30, # sold/listed in last 30 days + listing_type="sold", # or (for_sale, for_rent) + last_x_days=30, # sold in last 30 days - listed in last x days if (for_sale, for_rent) mls_only=True, # only fetch MLS listings ) print(f"Number of properties: {len(properties)}") From 446d5488b863660ce667043777a4f7fe869d65ae Mon Sep 17 00:00:00 2001 From: Zachary Hampton <69336300+ZacharyHampton@users.noreply.github.com> Date: Wed, 4 Oct 2023 10:07:32 -0700 Subject: [PATCH 12/19] - single address support again --- homeharvest/core/scrapers/realtor/__init__.py | 176 +++++++++++++++++- tests/test_realtor.py | 8 +- 2 files changed, 172 insertions(+), 12 deletions(-) diff --git a/homeharvest/core/scrapers/realtor/__init__.py b/homeharvest/core/scrapers/realtor/__init__.py index de1dcc4..cc2382a 100644 --- a/homeharvest/core/scrapers/realtor/__init__.py +++ b/homeharvest/core/scrapers/realtor/__init__.py @@ -13,7 +13,7 @@ from ..models import Property, Address, ListingType, Description class RealtorScraper(Scraper): - SEARCH_URL = "https://www.realtor.com/api/v1/rdc_search_srp?client_id=rdc-search-new-communities&schema=vesta" + SEARCH_GQL_URL = "https://www.realtor.com/api/v1/rdc_search_srp?client_id=rdc-search-new-communities&schema=vesta" PROPERTY_URL = "https://www.realtor.com/realestateandhomes-detail/" ADDRESS_AUTOCOMPLETE_URL = "https://parser-external.geo.moveaws.com/suggest" @@ -42,6 +42,145 @@ class RealtorScraper(Scraper): return result[0] + def handle_listing(self, listing_id: str) -> list[Property]: + query = """query Listing($listing_id: ID!) { + listing(id: $listing_id) { + source { + id + listing_id + } + address { + street_number + street_name + street_suffix + unit + city + state_code + postal_code + location { + coordinate { + lat + lon + } + } + } + basic { + sqft + beds + baths_full + baths_half + lot_sqft + sold_price + sold_price + type + price + status + sold_date + list_date + } + details { + year_built + stories + garage + permalink + } + } + }""" + + variables = {"listing_id": listing_id} + payload = { + "query": query, + "variables": variables, + } + + response = self.session.post(self.SEARCH_GQL_URL, json=payload) + response_json = response.json() + + property_info = response_json["data"]["listing"] + + mls = ( + property_info["source"].get("id") + if "source" in property_info and isinstance(property_info["source"], dict) + else None + ) + + able_to_get_lat_long = ( + property_info + and property_info.get("address") + and property_info["address"].get("location") + and property_info["address"]["location"].get("coordinate") + ) + + listing = Property( + mls=mls, + mls_id=property_info["source"].get("listing_id") + if "source" in property_info and isinstance(property_info["source"], dict) + else None, + property_url=f"{self.PROPERTY_URL}{property_info['details']['permalink']}", + status=property_info["basic"]["status"].upper(), + list_price=property_info["basic"]["price"], + list_date=property_info["basic"]["list_date"].split("T")[0] + if property_info["basic"].get("list_date") + else None, + prc_sqft=property_info["basic"].get("price") / property_info["basic"].get("sqft") + if property_info["basic"].get("price") and property_info["basic"].get("sqft") + else None, + last_sold_date=property_info["basic"]["sold_date"].split("T")[0] + if property_info["basic"].get("sold_date") + else None, + latitude=property_info["address"]["location"]["coordinate"].get("lat") + if able_to_get_lat_long + else None, + longitude=property_info["address"]["location"]["coordinate"].get("lon") + if able_to_get_lat_long + else None, + address=self._parse_address(property_info, search_type="handle_listing"), + description=Description( + style=property_info["basic"].get("type", "").upper(), + beds=property_info["basic"].get("beds"), + baths_full=property_info["basic"].get("baths_full"), + baths_half=property_info["basic"].get("baths_half"), + sqft=property_info["basic"].get("sqft"), + lot_sqft=property_info["basic"].get("lot_sqft"), + sold_price=property_info["basic"].get("sold_price"), + year_built=property_info["details"].get("year_built"), + garage=property_info["details"].get("garage"), + stories=property_info["details"].get("stories"), + ) + ) + + return [listing] + + def get_latest_listing_id(self, property_id: str) -> str | None: + query = """query Property($property_id: ID!) { + property(id: $property_id) { + listings { + listing_id + primary + } + } + } + """ + + variables = {"property_id": property_id} + payload = { + "query": query, + "variables": variables, + } + + response = self.session.post(self.SEARCH_GQL_URL, json=payload) + response_json = response.json() + + property_info = response_json["data"]["property"] + if property_info["listings"] is None: + return None + + primary_listing = next((listing for listing in property_info["listings"] if listing["primary"]), None) + if primary_listing: + return primary_listing["listing_id"] + else: + return property_info["listings"][0]["listing_id"] + def handle_address(self, property_id: str) -> list[Property]: """ Handles a specific address & returns one property @@ -97,7 +236,7 @@ class RealtorScraper(Scraper): "variables": variables, } - response = self.session.post(self.SEARCH_URL, json=payload) + response = self.session.post(self.SEARCH_GQL_URL, json=payload) response_json = response.json() property_info = response_json["data"]["property"] @@ -182,6 +321,7 @@ class RealtorScraper(Scraper): else "" ) ) + sort_param = ( "sort: [{ field: sold_date, direction: desc }]" if self.listing_type == ListingType.SOLD @@ -212,7 +352,7 @@ class RealtorScraper(Scraper): sort_param, results_query, ) - else: + elif search_type == "area": query = """query Home_search( $city: String, $county: [String], @@ -238,16 +378,29 @@ class RealtorScraper(Scraper): sort_param, results_query, ) + else: + query = ( + """query Property_search( + $property_id: [ID]! + $offset: Int!, + ) { + property_search( + query: { + property_id: $property_id + } + limit: 1 + offset: $offset + ) %s""" % results_query) payload = { "query": query, "variables": variables, } - response = self.session.post(self.SEARCH_URL, json=payload) + response = self.session.post(self.SEARCH_GQL_URL, json=payload) response.raise_for_status() response_json = response.json() - search_key = "property_search" if search_type == "comps" else "home_search" + search_key = "home_search" if search_type == "area" else "property_search" properties: list[Property] = [] @@ -320,12 +473,21 @@ class RealtorScraper(Scraper): "offset": 0, } - search_type = "comps" if self.radius and location_type == "address" else "area" + search_type = "comps" if self.radius and location_type == "address" else "address" if location_type == "address" and not self.radius else "area" if location_type == "address": if not self.radius: #: single address search, non comps property_id = location_info["mpr_id"] search_variables |= {"property_id": property_id} - return self.handle_address(property_id) + + gql_results = self.general_search(search_variables, search_type=search_type) + if gql_results["total"] == 0: + listing_id = self.get_latest_listing_id(property_id) + if listing_id is None: + return self.handle_address(property_id) + else: + return self.handle_listing(listing_id) + else: + return gql_results["properties"] else: #: general search, comps (radius) coordinates = list(location_info["centroid"].values()) diff --git a/tests/test_realtor.py b/tests/test_realtor.py index 15b7e09..12b6d8d 100644 --- a/tests/test_realtor.py +++ b/tests/test_realtor.py @@ -1,9 +1,7 @@ from homeharvest import scrape_property from homeharvest.exceptions import ( - InvalidSite, InvalidListingType, NoResultsFound, - GeoCoordsNotFound, ) @@ -11,7 +9,7 @@ def test_realtor_comps(): result = scrape_property( location="2530 Al Lipscomb Way", radius=0.5, - sold_last_x_days=180, + last_x_days=180, listing_type="sold", ) @@ -20,11 +18,11 @@ def test_realtor_comps(): def test_realtor_last_x_days_sold(): days_result_30 = scrape_property( - location="Dallas, TX", listing_type="sold", sold_last_x_days=30 + location="Dallas, TX", listing_type="sold", last_x_days=30 ) days_result_10 = scrape_property( - location="Dallas, TX", listing_type="sold", sold_last_x_days=10 + location="Dallas, TX", listing_type="sold", last_x_days=10 ) assert all( From 6bb68766fc2afab8d81a1656c93ffd372338c951 Mon Sep 17 00:00:00 2001 From: Zachary Hampton <69336300+ZacharyHampton@users.noreply.github.com> Date: Wed, 4 Oct 2023 12:04:05 -0700 Subject: [PATCH 13/19] - realtor tests --- tests/test_realtor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_realtor.py b/tests/test_realtor.py index 12b6d8d..7ace9af 100644 --- a/tests/test_realtor.py +++ b/tests/test_realtor.py @@ -70,7 +70,7 @@ def test_realtor(): listing_type="for_sale", ) ] - except (InvalidSite, InvalidListingType, NoResultsFound, GeoCoordsNotFound): + except (InvalidListingType, NoResultsFound): assert True assert all([result is None for result in bad_results]) From de692faae215fe3b7a2f2ff10089dde75f1f8b28 Mon Sep 17 00:00:00 2001 From: Zachary Hampton <69336300+ZacharyHampton@users.noreply.github.com> Date: Wed, 4 Oct 2023 18:06:06 -0700 Subject: [PATCH 14/19] - rename last_x_days - docstrings for scrape_property --- README.md | 11 +++++------ examples/HomeHarvest_Demo.py | 2 +- homeharvest/__init__.py | 10 ++++++++-- homeharvest/cli.py | 2 +- tests/test_realtor.py | 6 +++--- 5 files changed, 18 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 0ae0978..81fa8f1 100644 --- a/README.md +++ b/README.md @@ -61,7 +61,7 @@ options: > homeharvest "San Francisco, CA" -l for_rent -o excel -f HomeHarvest ``` -### Python +### Python ```py from homeharvest import scrape_property @@ -72,10 +72,10 @@ current_timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") filename = f"output/{current_timestamp}.csv" properties = scrape_property( - location="San Diego, CA", - listing_type="sold", # or (for_sale, for_rent) - last_x_days=30, # sold in last 30 days - listed in last x days if (for_sale, for_rent) - mls_only=True, # only fetch MLS listings + location="San Diego, CA", + listing_type="sold", # or (for_sale, for_rent) + property_younger_than=30, # sold in last 30 days - listed in last x days if (for_sale, for_rent) + mls_only=True, # only fetch MLS listings ) print(f"Number of properties: {len(properties)}") @@ -84,7 +84,6 @@ properties.to_csv(filename, index=False) print(properties.head()) ``` - ## Output ```plaintext >>> properties.head() diff --git a/examples/HomeHarvest_Demo.py b/examples/HomeHarvest_Demo.py index 9e8e053..b7e999f 100644 --- a/examples/HomeHarvest_Demo.py +++ b/examples/HomeHarvest_Demo.py @@ -8,7 +8,7 @@ filename = f"output/{current_timestamp}.csv" properties = scrape_property( location="San Diego, CA", listing_type="sold", # for_sale, for_rent - last_x_days=30, # sold/listed in last 30 days + property_younger_than=30, # sold/listed in last 30 days mls_only=True, # only fetch MLS listings ) print(f"Number of properties: {len(properties)}") diff --git a/homeharvest/__init__.py b/homeharvest/__init__.py index 5d68d1d..cec1687 100644 --- a/homeharvest/__init__.py +++ b/homeharvest/__init__.py @@ -12,11 +12,17 @@ def scrape_property( listing_type: str = "for_sale", radius: float = None, mls_only: bool = False, - last_x_days: int = None, + property_younger_than: int = None, proxy: str = None, ) -> pd.DataFrame: """ Scrape properties from Realtor.com based on a given location and listing type. + :param location: Location to search (e.g. "Dallas, TX", "85281", "2530 Al Lipscomb Way") + :param listing_type: Listing Type (for_sale, for_rent, sold) + :param radius: Get properties within _ (e.g. 1.0) miles. Only applicable for individual addresses. + :param mls_only: If set, fetches only listings with MLS IDs. + :param property_younger_than: Get properties sold/listed in last _ days. + :param proxy: Proxy to use for scraping """ validate_input(listing_type) @@ -26,7 +32,7 @@ def scrape_property( proxy=proxy, radius=radius, mls_only=mls_only, - last_x_days=last_x_days, + last_x_days=property_younger_than, ) site = RealtorScraper(scraper_input) diff --git a/homeharvest/cli.py b/homeharvest/cli.py index b732486..198de12 100644 --- a/homeharvest/cli.py +++ b/homeharvest/cli.py @@ -68,7 +68,7 @@ def main(): radius=args.radius, proxy=args.proxy, mls_only=args.mls_only, - last_x_days=args.days, + property_younger_than=args.days, ) if not args.filename: diff --git a/tests/test_realtor.py b/tests/test_realtor.py index 7ace9af..43af136 100644 --- a/tests/test_realtor.py +++ b/tests/test_realtor.py @@ -9,7 +9,7 @@ def test_realtor_comps(): result = scrape_property( location="2530 Al Lipscomb Way", radius=0.5, - last_x_days=180, + property_younger_than=180, listing_type="sold", ) @@ -18,11 +18,11 @@ def test_realtor_comps(): def test_realtor_last_x_days_sold(): days_result_30 = scrape_property( - location="Dallas, TX", listing_type="sold", last_x_days=30 + location="Dallas, TX", listing_type="sold", property_younger_than=30 ) days_result_10 = scrape_property( - location="Dallas, TX", listing_type="sold", last_x_days=10 + location="Dallas, TX", listing_type="sold", property_younger_than=10 ) assert all( From 8a5f0dc2c95c2f102508053a2d7bdaa45709d9d8 Mon Sep 17 00:00:00 2001 From: Zachary Hampton <69336300+ZacharyHampton@users.noreply.github.com> Date: Wed, 4 Oct 2023 18:25:01 -0700 Subject: [PATCH 15/19] - pending or contingent support --- homeharvest/__init__.py | 3 +++ homeharvest/core/scrapers/__init__.py | 2 ++ homeharvest/core/scrapers/realtor/__init__.py | 10 +++++++--- tests/test_realtor.py | 15 +++++++++++++++ 4 files changed, 27 insertions(+), 3 deletions(-) diff --git a/homeharvest/__init__.py b/homeharvest/__init__.py index cec1687..63aa13f 100644 --- a/homeharvest/__init__.py +++ b/homeharvest/__init__.py @@ -13,6 +13,7 @@ def scrape_property( radius: float = None, mls_only: bool = False, property_younger_than: int = None, + pending_or_contingent: bool = False, proxy: str = None, ) -> pd.DataFrame: """ @@ -22,6 +23,7 @@ def scrape_property( :param radius: Get properties within _ (e.g. 1.0) miles. Only applicable for individual addresses. :param mls_only: If set, fetches only listings with MLS IDs. :param property_younger_than: Get properties sold/listed in last _ days. + :param pending_or_contingent: If set, fetches only pending or contingent listings. Only applicable for for_sale listings from general area searches. :param proxy: Proxy to use for scraping """ validate_input(listing_type) @@ -33,6 +35,7 @@ def scrape_property( radius=radius, mls_only=mls_only, last_x_days=property_younger_than, + pending_or_contingent=pending_or_contingent, ) site = RealtorScraper(scraper_input) diff --git a/homeharvest/core/scrapers/__init__.py b/homeharvest/core/scrapers/__init__.py index 1ce4431..f82b321 100644 --- a/homeharvest/core/scrapers/__init__.py +++ b/homeharvest/core/scrapers/__init__.py @@ -12,6 +12,7 @@ class ScraperInput: mls_only: bool | None = None proxy: str | None = None last_x_days: int | None = None + pending_or_contingent: bool | None = None class Scraper: @@ -37,6 +38,7 @@ class Scraper: self.radius = scraper_input.radius self.last_x_days = scraper_input.last_x_days self.mls_only = scraper_input.mls_only + self.pending_or_contingent = scraper_input.pending_or_contingent def search(self) -> list[Property]: ... diff --git a/homeharvest/core/scrapers/realtor/__init__.py b/homeharvest/core/scrapers/realtor/__init__.py index cc2382a..f0e47c2 100644 --- a/homeharvest/core/scrapers/realtor/__init__.py +++ b/homeharvest/core/scrapers/realtor/__init__.py @@ -328,7 +328,9 @@ class RealtorScraper(Scraper): else "sort: [{ field: list_date, direction: desc }]" ) - if search_type == "comps": + pending_or_contingent_param = "or_filters: { contingent: true, pending: true }" if self.pending_or_contingent else "" + + if search_type == "comps": #: comps search, came from an address query = """query Property_search( $coordinates: [Float]! $radius: String! @@ -352,7 +354,7 @@ class RealtorScraper(Scraper): sort_param, results_query, ) - elif search_type == "area": + elif search_type == "area": #: general search, came from a general location query = """query Home_search( $city: String, $county: [String], @@ -368,6 +370,7 @@ class RealtorScraper(Scraper): state_code: $state_code status: %s %s + %s } %s limit: 200 @@ -375,10 +378,11 @@ class RealtorScraper(Scraper): ) %s""" % ( self.listing_type.value.lower(), date_param, + pending_or_contingent_param, sort_param, results_query, ) - else: + else: #: general search, came from an address query = ( """query Property_search( $property_id: [ID]! diff --git a/tests/test_realtor.py b/tests/test_realtor.py index 43af136..a498112 100644 --- a/tests/test_realtor.py +++ b/tests/test_realtor.py @@ -5,6 +5,21 @@ from homeharvest.exceptions import ( ) +def test_realtor_pending_or_contingent(): + pending_or_contingent_result = scrape_property( + location="Surprise, AZ", + pending_or_contingent=True, + ) + + regular_result = scrape_property( + location="Surprise, AZ", + pending_or_contingent=False, + ) + + assert all([result is not None for result in [pending_or_contingent_result, regular_result]]) + assert len(pending_or_contingent_result) != len(regular_result) + + def test_realtor_comps(): result = scrape_property( location="2530 Al Lipscomb Way", From 37e20f44691c2c2d23199464a581211d852960aa Mon Sep 17 00:00:00 2001 From: Zachary Hampton <69336300+ZacharyHampton@users.noreply.github.com> Date: Wed, 4 Oct 2023 18:44:47 -0700 Subject: [PATCH 16/19] - remove neighborhoods - rename data --- homeharvest/core/scrapers/realtor/__init__.py | 2 +- homeharvest/utils.py | 98 ++++++++----------- 2 files changed, 44 insertions(+), 56 deletions(-) diff --git a/homeharvest/core/scrapers/realtor/__init__.py b/homeharvest/core/scrapers/realtor/__init__.py index f0e47c2..fcd96b2 100644 --- a/homeharvest/core/scrapers/realtor/__init__.py +++ b/homeharvest/core/scrapers/realtor/__init__.py @@ -459,7 +459,7 @@ class RealtorScraper(Scraper): if able_to_get_lat_long else None, address=self._parse_address(result, search_type="general_search"), - neighborhoods=self._parse_neighborhoods(result), + #: neighborhoods=self._parse_neighborhoods(result), description=self._parse_description(result), ) properties.append(realty_property) diff --git a/homeharvest/utils.py b/homeharvest/utils.py index 1f7f717..5d125e1 100644 --- a/homeharvest/utils.py +++ b/homeharvest/utils.py @@ -1,74 +1,62 @@ from .core.scrapers.models import Property, ListingType import pandas as pd +from .exceptions import InvalidListingType ordered_properties = [ - "PropertyURL", - "MLS", - "MLS #", - "Status", - "Style", - "Street", - "Unit", - "City", - "State", - "Zip", - "Beds", - "FB", - "NumHB", - "EstSF", - "YrBlt", - "ListPrice", - "Lst Date", - "Sold Price", - "COEDate", - "LotSFApx", - "PrcSqft", - "LATITUDE", - "LONGITUDE", - "Stories", - "HOAFee", - "PrkgGar", - "Community", + "property_url", + "mls", + "mls_id", + "status", + "style", + "street", + "unit", + "city", + "state", + "zip_code", + "beds", + "full_baths", + "half_baths", + "sqft", + "year_built", + "list_price", + "list_date", + "sold_price", + "last_sold_date", + "lot_sqft", + "price_per_sqft", + "latitude", + "longitude", + "stories", + "hoa_fee", + "parking_garage", ] def process_result(result: Property) -> pd.DataFrame: prop_data = {prop: None for prop in ordered_properties} prop_data.update(result.__dict__) - prop_data["PropertyURL"] = prop_data["property_url"] - prop_data["MLS"] = prop_data["mls"] - prop_data["MLS #"] = prop_data["mls_id"] - prop_data["Status"] = prop_data["status"] if "address" in prop_data: address_data = prop_data["address"] - prop_data["Street"] = address_data.street - prop_data["Unit"] = address_data.unit - prop_data["City"] = address_data.city - prop_data["State"] = address_data.state - prop_data["Zip"] = address_data.zip + prop_data["street"] = address_data.street + prop_data["unit"] = address_data.unit + prop_data["city"] = address_data.city + prop_data["state"] = address_data.state + prop_data["zip_code"] = address_data.zip - prop_data["ListPrice"] = prop_data["list_price"] - prop_data["Lst Date"] = prop_data["list_date"] - prop_data["COEDate"] = prop_data["last_sold_date"] - prop_data["PrcSqft"] = prop_data["prc_sqft"] - prop_data["HOAFee"] = prop_data["hoa_fee"] + prop_data["price_per_sqft"] = prop_data["prc_sqft"] description = result.description - prop_data["Style"] = description.style - prop_data["Beds"] = description.beds - prop_data["FB"] = description.baths_full - prop_data["NumHB"] = description.baths_half - prop_data["EstSF"] = description.sqft - prop_data["LotSFApx"] = description.lot_sqft - prop_data["Sold Price"] = description.sold_price - prop_data["YrBlt"] = description.year_built - prop_data["PrkgGar"] = description.garage - prop_data["Stories"] = description.stories - - prop_data["LATITUDE"] = prop_data["latitude"] - prop_data["LONGITUDE"] = prop_data["longitude"] - prop_data["Community"] = prop_data["neighborhoods"] + prop_data["style"] = description.style + prop_data["beds"] = description.beds + prop_data["full_baths"] = description.baths_full + prop_data["half_baths"] = description.baths_half + prop_data["sqft"] = description.sqft + prop_data["lot_sqft"] = description.lot_sqft + prop_data["sold_price"] = description.sold_price + prop_data["year_built"] = description.year_built + prop_data["parking_garage"] = description.garage + prop_data["stories"] = description.stories properties_df = pd.DataFrame([prop_data]) properties_df = properties_df.reindex(columns=ordered_properties) From 4e7824803283b67909e4c93711c22fac9abaa461 Mon Sep 17 00:00:00 2001 From: Cullen Watson Date: Wed, 4 Oct 2023 21:17:49 -0500 Subject: [PATCH 17/19] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 81fa8f1..76313d7 100644 --- a/README.md +++ b/README.md @@ -109,7 +109,7 @@ Optional ├── radius (decimal): Radius in miles to find comparable properties based on individual addresses. │ Example: 5.5 (fetches properties within a 5.5-mile radius if location is set to a specific address; otherwise, ignored) │ -├── last_x_days (integer): Number of past days to filter properties. Utilizes 'COEDate' for 'sold' listing types, and 'Lst Date' for others (for_rent, for_sale). +├── property_younger_than (integer): Number of past days to filter properties. Utilizes 'COEDate' for 'sold' listing types, and 'Lst Date' for others (for_rent, for_sale). │ Example: 30 (fetches properties listed/sold in the last 30 days) │ ├── mls_only (True/False): If set, fetches only MLS listings (mainly applicable to 'sold' listings) From 4dbb064fe947da38515132e502914bb8d6e62c43 Mon Sep 17 00:00:00 2001 From: Cullen Watson Date: Wed, 4 Oct 2023 21:21:45 -0500 Subject: [PATCH 18/19] [docs]: Update README.md --- README.md | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 76313d7..87b9aa7 100644 --- a/README.md +++ b/README.md @@ -121,41 +121,42 @@ Optional ```plaintext Property ├── Basic Information: -│ ├── property_url (str) -│ ├── mls (str) -│ ├── mls_id (str) -│ └── status (str) +│ ├── property_url +│ ├── mls +│ ├── mls_id +│ └── status ├── Address Details: │ ├── street │ ├── unit │ ├── city │ ├── state -│ └── zip +│ └── zip_code ├── Property Description: │ ├── style │ ├── beds -│ ├── baths_full -│ ├── baths_half +│ ├── full_baths +│ ├── half_baths │ ├── sqft -│ ├── lot_sqft -│ ├── sold_price │ ├── year_built -│ ├── garage -│ └── stories +│ ├── stories +│ └── lot_sqft ├── Property Listing Details: │ ├── list_price │ ├── list_date +│ ├── sold_price │ ├── last_sold_date -│ ├── prc_sqft +│ ├── price_per_sqft │ └── hoa_fee ├── Location Details: │ ├── latitude │ ├── longitude -│ └── neighborhoods + +└── Parking Details: + └── parking_garage ``` ### Exceptions From 2d092c595f1d49a92db5fde6011b53994c1057c3 Mon Sep 17 00:00:00 2001 From: Cullen Watson Date: Wed, 4 Oct 2023 21:24:24 -0500 Subject: [PATCH 19/19] [docs]: Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 87b9aa7..e7c72b1 100644 --- a/README.md +++ b/README.md @@ -109,7 +109,7 @@ Optional ├── radius (decimal): Radius in miles to find comparable properties based on individual addresses. │ Example: 5.5 (fetches properties within a 5.5-mile radius if location is set to a specific address; otherwise, ignored) │ -├── property_younger_than (integer): Number of past days to filter properties. Utilizes 'COEDate' for 'sold' listing types, and 'Lst Date' for others (for_rent, for_sale). +├── property_younger_than (integer): Number of past days to filter properties. Utilizes 'last_sold_date' for 'sold' listing types, and 'list_date' for others (for_rent, for_sale). │ Example: 30 (fetches properties listed/sold in the last 30 days) │ ├── mls_only (True/False): If set, fetches only MLS listings (mainly applicable to 'sold' listings)