diff --git a/homeharvest/__init__.py b/homeharvest/__init__.py index 2b60e3b..d2f34e0 100644 --- a/homeharvest/__init__.py +++ b/homeharvest/__init__.py @@ -3,139 +3,60 @@ from typing import Union import concurrent.futures from concurrent.futures import ThreadPoolExecutor +from .utils import process_result, ordered_properties from .core.scrapers import ScraperInput -from .core.scrapers.redfin import RedfinScraper from .core.scrapers.realtor import RealtorScraper -from .core.scrapers.zillow import ZillowScraper -from .core.scrapers.models import ListingType, Property, SiteName +from .core.scrapers.models import Status, Property, SiteName from .exceptions import InvalidSite, InvalidListingType _scrapers = { - "redfin": RedfinScraper, "realtor.com": RealtorScraper, - "zillow": ZillowScraper, } -def _validate_input(site_name: str, listing_type: str) -> None: +def _validate_input(site_name: str, status: str) -> None: if site_name.lower() not in _scrapers: raise InvalidSite(f"Provided site, '{site_name}', does not exist.") - if listing_type.upper() not in ListingType.__members__: - raise InvalidListingType(f"Provided listing type, '{listing_type}', does not exist.") + if status.upper() not in Status.__members__: + raise InvalidListingType(f"Provided listing type, '{status}', does not exist.") -def _get_ordered_properties(result: Property) -> list[str]: - return [ - "property_url", - "site_name", - "listing_type", - "property_type", - "status_text", - "baths_min", - "baths_max", - "beds_min", - "beds_max", - "sqft_min", - "sqft_max", - "price_min", - "price_max", - "unit_count", - "tax_assessed_value", - "price_per_sqft", - "lot_area_value", - "lot_area_unit", - "address_one", - "address_two", - "city", - "state", - "zip_code", - "posted_time", - "area_min", - "bldg_name", - "stories", - "year_built", - "agent_name", - "agent_phone", - "agent_email", - "days_on_market", - "sold_date", - "mls_id", - "img_src", - "latitude", - "longitude", - "description", - ] - - -def _process_result(result: Property) -> pd.DataFrame: - prop_data = result.__dict__ - - prop_data["site_name"] = prop_data["site_name"].value - prop_data["listing_type"] = prop_data["listing_type"].value.lower() - if "property_type" in prop_data and prop_data["property_type"] is not None: - prop_data["property_type"] = prop_data["property_type"].value.lower() - else: - prop_data["property_type"] = None - if "address" in prop_data: - address_data = prop_data["address"] - prop_data["address_one"] = address_data.address_one - prop_data["address_two"] = address_data.address_two - prop_data["city"] = address_data.city - prop_data["state"] = address_data.state - prop_data["zip_code"] = address_data.zip_code - - del prop_data["address"] - - if "agent" in prop_data and prop_data["agent"] is not None: - agent_data = prop_data["agent"] - prop_data["agent_name"] = agent_data.name - prop_data["agent_phone"] = agent_data.phone - prop_data["agent_email"] = agent_data.email - - del prop_data["agent"] - else: - prop_data["agent_name"] = None - prop_data["agent_phone"] = None - prop_data["agent_email"] = None - - properties_df = pd.DataFrame([prop_data]) - properties_df = properties_df[_get_ordered_properties(result)] - - return properties_df - - -def _scrape_single_site(location: str, site_name: str, listing_type: str, proxy: str = None) -> pd.DataFrame: +def _scrape_single_site( + location: str, site_name: str, status: str, proxy: str = None, timeframe: str = None +) -> pd.DataFrame: """ Helper function to scrape a single site. """ - _validate_input(site_name, listing_type) + print(status) + _validate_input(site_name, status) scraper_input = ScraperInput( location=location, - listing_type=ListingType[listing_type.upper()], + status=status, site_name=SiteName.get_by_value(site_name.lower()), proxy=proxy, + timeframe=timeframe, ) site = _scrapers[site_name.lower()](scraper_input) results = site.search() + print(f"Found {len(results)} results for {site_name}") - properties_dfs = [_process_result(result) for result in results] - properties_dfs = [df.dropna(axis=1, how="all") for df in properties_dfs if not df.empty] + properties_dfs = [process_result(result) for result in results] if not properties_dfs: return pd.DataFrame() - return pd.concat(properties_dfs, ignore_index=True) + return pd.concat(properties_dfs, ignore_index=True, axis=0)[ordered_properties] def scrape_property( location: str, + timeframe: str, site_name: Union[str, list[str]] = None, - listing_type: str = "for_sale", + status: str = "sale", proxy: str = None, - keep_duplicates: bool = False ) -> pd.DataFrame: """ Scrape property from various sites from a given location and listing type. @@ -155,12 +76,14 @@ def scrape_property( results = [] if len(site_name) == 1: - final_df = _scrape_single_site(location, site_name[0], listing_type, proxy) + final_df = _scrape_single_site(location, site_name[0], status, proxy, timeframe) results.append(final_df) else: with ThreadPoolExecutor() as executor: futures = { - executor.submit(_scrape_single_site, location, s_name, listing_type, proxy): s_name + executor.submit( + _scrape_single_site, location, s_name, status, proxy, timeframe + ): s_name for s_name in site_name } @@ -175,13 +98,11 @@ def scrape_property( final_df = pd.concat(results, ignore_index=True) - columns_to_track = ["address_one", "address_two", "city"] + columns_to_track = ["Street", "Unit", "Zip"] #: validate they exist, otherwise create them for col in columns_to_track: if col not in final_df.columns: final_df[col] = None - if not keep_duplicates: - final_df = final_df.drop_duplicates(subset=columns_to_track, keep="first") return final_df diff --git a/homeharvest/cli.py b/homeharvest/cli.py index c9deae8..d239310 100644 --- a/homeharvest/cli.py +++ b/homeharvest/cli.py @@ -5,7 +5,9 @@ from homeharvest import scrape_property def main(): parser = argparse.ArgumentParser(description="Home Harvest Property Scraper") - parser.add_argument("location", type=str, help="Location to scrape (e.g., San Francisco, CA)") + parser.add_argument( + "location", type=str, help="Location to scrape (e.g., San Francisco, CA)" + ) parser.add_argument( "-s", @@ -46,14 +48,22 @@ def main(): "-k", "--keep_duplicates", action="store_true", - help="Keep duplicate properties based on address" + help="Keep duplicate properties based on address", ) - parser.add_argument("-p", "--proxy", type=str, default=None, help="Proxy to use for scraping") + parser.add_argument( + "-p", "--proxy", type=str, default=None, help="Proxy to use for scraping" + ) args = parser.parse_args() - result = scrape_property(args.location, args.site_name, args.listing_type, proxy=args.proxy, keep_duplicates=args.keep_duplicates) + result = scrape_property( + args.location, + args.site_name, + args.listing_type, + proxy=args.proxy, + keep_duplicates=args.keep_duplicates, + ) if not args.filename: timestamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S") diff --git a/homeharvest/core/scrapers/__init__.py b/homeharvest/core/scrapers/__init__.py index e900dbe..79f4e93 100644 --- a/homeharvest/core/scrapers/__init__.py +++ b/homeharvest/core/scrapers/__init__.py @@ -1,21 +1,38 @@ from dataclasses import dataclass import requests import tls_client -from .models import Property, ListingType, SiteName +from typing import Optional +from .models import Property, SiteName, Status +from ...exceptions import InvalidTimeFrame + +VALID_TIMEFRAMES = ["1W", "1M", "3M", "6M", "1Y"] +VALID_STATUSES = ["sold", "for_sale", "for_rent"] @dataclass class ScraperInput: location: str - listing_type: ListingType - site_name: SiteName - proxy: str | None = None + status: str + site_name: str + proxy: Optional[str] = None + timeframe: Optional[str] = None + + def __post_init__(self): + if self.timeframe and self.timeframe not in VALID_TIMEFRAMES: + raise InvalidTimeFrame(f"Invalid timeframe provided: {self.timeframe}") + if self.status and self.status not in VALID_STATUSES: + raise InvalidTimeFrame(f"Invalid status provided: {self.status}") class Scraper: - def __init__(self, scraper_input: ScraperInput, session: requests.Session | tls_client.Session = None): + def __init__( + self, + scraper_input: ScraperInput, + session: requests.Session | tls_client.Session = None, + ): self.location = scraper_input.location - self.listing_type = scraper_input.listing_type + self.status = scraper_input.status + self.timeframe = scraper_input.timeframe if not session: self.session = requests.Session() @@ -27,7 +44,7 @@ class Scraper: proxies = {"http": proxy_url, "https": proxy_url} self.session.proxies.update(proxies) - self.listing_type = scraper_input.listing_type + self.listing_type = scraper_input.status self.site_name = scraper_input.site_name def search(self) -> list[Property]: diff --git a/homeharvest/core/scrapers/models.py b/homeharvest/core/scrapers/models.py index ed75999..4f20094 100644 --- a/homeharvest/core/scrapers/models.py +++ b/homeharvest/core/scrapers/models.py @@ -1,7 +1,6 @@ from dataclasses import dataclass from enum import Enum -from typing import Tuple -from datetime import datetime +from typing import Optional class SiteName(Enum): @@ -17,52 +16,19 @@ class SiteName(Enum): raise ValueError(f"{value} not found in {cls}") -class ListingType(Enum): +class Status(Enum): FOR_SALE = "FOR_SALE" FOR_RENT = "FOR_RENT" SOLD = "SOLD" -class PropertyType(Enum): - HOUSE = "HOUSE" - BUILDING = "BUILDING" - CONDO = "CONDO" - TOWNHOUSE = "TOWNHOUSE" - SINGLE_FAMILY = "SINGLE_FAMILY" - MULTI_FAMILY = "MULTI_FAMILY" - MANUFACTURED = "MANUFACTURED" - NEW_CONSTRUCTION = "NEW_CONSTRUCTION" - APARTMENT = "APARTMENT" - APARTMENTS = "APARTMENTS" - LAND = "LAND" - LOT = "LOT" - OTHER = "OTHER" - - BLANK = "BLANK" - - @classmethod - def from_int_code(cls, code): - mapping = { - 1: cls.HOUSE, - 2: cls.CONDO, - 3: cls.TOWNHOUSE, - 4: cls.MULTI_FAMILY, - 5: cls.LAND, - 6: cls.OTHER, - 8: cls.SINGLE_FAMILY, - 13: cls.SINGLE_FAMILY, - } - - return mapping.get(code, cls.BLANK) - - @dataclass class Address: - address_one: str | None = None - address_two: str | None = "#" + street: str | None = None + unit: str | None = None city: str | None = None state: str | None = None - zip_code: str | None = None + zip: str | None = None @dataclass @@ -74,47 +40,31 @@ class Agent: @dataclass class Property: - property_url: str - site_name: SiteName - listing_type: ListingType - address: Address - property_type: PropertyType | None = None - - # house for sale - tax_assessed_value: int | None = None - lot_area_value: float | None = None - lot_area_unit: str | None = None - stories: int | None = None - year_built: int | None = None - price_per_sqft: int | None = None + property_url: str | None = None + mls: str | None = None mls_id: str | None = None + status: str | None = None + style: str | None = None - agent: Agent | None = None - img_src: str | None = None - description: str | None = None - status_text: str | None = None - posted_time: datetime | None = None + beds: int | None = None + baths_full: int | None = None + baths_half: int | None = None + list_price: int | None = None + list_date: str | None = None + sold_price: int | None = None + last_sold_date: str | None = None + prc_sqft: float | None = None + est_sf: int | None = None + lot_sf: int | None = None + hoa_fee: int | None = None - # building for sale - bldg_name: str | None = None - area_min: int | None = None - - beds_min: int | None = None - beds_max: int | None = None - - baths_min: float | None = None - baths_max: float | None = None - - sqft_min: int | None = None - sqft_max: int | None = None - - price_min: int | None = None - price_max: int | None = None - - unit_count: int | None = None + address: Address | None = None + yr_blt: int | None = None latitude: float | None = None longitude: float | None = None - sold_date: datetime | None = None - days_on_market: int | None = None + stories: int | None = None + prkg_gar: float | None = None + + neighborhoods: Optional[str] = None diff --git a/homeharvest/core/scrapers/realtor/__init__.py b/homeharvest/core/scrapers/realtor/__init__.py index 78ecc84..d0e3352 100644 --- a/homeharvest/core/scrapers/realtor/__init__.py +++ b/homeharvest/core/scrapers/realtor/__init__.py @@ -4,10 +4,10 @@ homeharvest.realtor.__init__ This module implements the scraper for relator.com """ +from typing import Dict, Union from ..models import Property, Address from .. import Scraper from ....exceptions import NoResultsFound -from ....utils import parse_address_one, parse_address_two from concurrent.futures import ThreadPoolExecutor, as_completed @@ -15,9 +15,7 @@ class RealtorScraper(Scraper): def __init__(self, scraper_input): self.counter = 1 super().__init__(scraper_input) - self.search_url = ( - "https://www.realtor.com/api/v1/rdc_search_srp?client_id=rdc-search-new-communities&schema=vesta" - ) + self.endpoint = "https://www.realtor.com/api/v1/rdc_search_srp?client_id=rdc-search-new-communities&schema=vesta" def handle_location(self): headers = { @@ -37,7 +35,7 @@ class RealtorScraper(Scraper): params = { "input": self.location, - "client_id": self.listing_type.value.lower().replace("_", "-"), + "client_id": self.listing_type.lower().replace("_", "-"), "limit": "1", "area_types": "city,state,county,postal_code,address,street,neighborhood,school,school_district,university,park", } @@ -68,7 +66,6 @@ class RealtorScraper(Scraper): garage permalink year_built - stories } address { address_validation_code @@ -100,7 +97,6 @@ class RealtorScraper(Scraper): public_record { lot_size sqft - stories units year_built } @@ -114,56 +110,48 @@ class RealtorScraper(Scraper): "variables": variables, } - response = self.session.post(self.search_url, json=payload) + response = self.session.post(self.endpoint, json=payload) response_json = response.json() property_info = response_json["data"]["property"] - address_one, address_two = parse_address_one(property_info["address"]["line"]) return [ Property( - site_name=self.site_name, - address=Address( - address_one=address_one, - address_two=address_two, - city=property_info["address"]["city"], - state=property_info["address"]["state_code"], - zip_code=property_info["address"]["postal_code"], - ), property_url="https://www.realtor.com/realestateandhomes-detail/" + property_info["details"]["permalink"], - stories=property_info["details"]["stories"], - year_built=property_info["details"]["year_built"], - price_per_sqft=property_info["basic"]["price"] // property_info["basic"]["sqft"] - if property_info["basic"]["sqft"] is not None and property_info["basic"]["price"] is not None + address=Address( + street=f"{property_info['address']['street_number']} {property_info['address']['street_name']} {property_info['address']['street_suffix']}", + unit=property_info["address"]["unit_value"], + city=property_info["address"]["city"], + state=property_info["address"]["state_code"], + zip=property_info["address"]["postal_code"], + ), + yr_blt=property_info["details"]["year_built"], + prc_sqft=property_info["basic"]["price"] + // property_info["basic"]["sqft"] + if property_info["basic"]["sqft"] is not None + and property_info["basic"]["price"] is not None else None, - mls_id=property_id, - listing_type=self.listing_type, - lot_area_value=property_info["public_record"]["lot_size"] - if property_info["public_record"] is not None - else None, - beds_min=property_info["basic"]["beds"], - beds_max=property_info["basic"]["beds"], - baths_min=property_info["basic"]["baths"], - baths_max=property_info["basic"]["baths"], - sqft_min=property_info["basic"]["sqft"], - sqft_max=property_info["basic"]["sqft"], - price_min=property_info["basic"]["price"], - price_max=property_info["basic"]["price"], + status=self.status.upper(), + beds=property_info["basic"]["beds"], + baths_full=property_info["basic"]["baths"], + lot_sf=property_info["basic"]["lot_sqft"], + est_sf=property_info["basic"]["sqft"], + list_price=property_info["basic"]["price"], + sold_price=property_info["basic"]["sold_price"], ) ] - def handle_area(self, variables: dict, return_total: bool = False) -> list[Property] | int: + def handle_area(self, variables: dict) -> Dict[str, Union[int, list[Property]]]: """ Handles a location area & returns a list of properties """ - query = ( - """query Home_search( + query = """query Home_search( $city: String, $county: [String], $state_code: String, - $postal_code: String - $offset: Int, + $postal_code: String, + $offset: Int ) { home_search( query: { @@ -172,26 +160,66 @@ class RealtorScraper(Scraper): postal_code: $postal_code state_code: $state_code status: %s + sold_date: { + min: %s + } } limit: 200 offset: $offset + sort: [ + { + field: sold_date, + direction: desc + } + ] ) { count total results { property_id + list_date + status + last_sold_price + last_sold_date + hoa { + fee + } description { - baths + baths_full + baths_half beds lot_sqft sqft - text sold_price - stories year_built garage - unit_number - floor_number + sold_price + type + sub_type + name + stories + } + source { + raw { + area + status + style + } + last_update_date + contract_date + id + listing_id + name + type + listing_href + community_id + management_id + corporation_id + subdivision_status + spec_id + plan_id + tier_rank + feed_type } location { address { @@ -201,41 +229,44 @@ class RealtorScraper(Scraper): postal_code state_code state + coordinate { + lon + lat + } street_direction street_name street_number street_post_direction street_suffix unit - coordinate { - lon - lat - } + } + neighborhoods { + name } } list_price price_per_sqft + style_category_tags { + exterior} + source { id } } } - }""" - % self.listing_type.value.lower() + }""" % ( + self.status, + f'"$nowUTC-{self.timeframe}"', ) payload = { "query": query, "variables": variables, } - - response = self.session.post(self.search_url, json=payload) + response = self.session.post(self.endpoint, json=payload) response.raise_for_status() response_json = response.json() - if return_total: - return response_json["data"]["home_search"]["total"] - properties: list[Property] = [] if ( @@ -246,19 +277,67 @@ class RealtorScraper(Scraper): or response_json["data"]["home_search"] is None or "results" not in response_json["data"]["home_search"] ): - return [] + return {"total": 0, "properties": []} for result in response_json["data"]["home_search"]["results"]: self.counter += 1 - address_one, _ = parse_address_one(result["location"]["address"]["line"]) + mls = ( + result["source"].get("id") + if "source" in result and isinstance(result["source"], dict) + else None + ) + mls_id = ( + result["source"].get("listing_id") + if "source" in result and isinstance(result["source"], dict) + else None + ) + + if not mls_id: + continue + # not type + + neighborhoods_list = [] + neighborhoods = result["location"].get("neighborhoods", []) + + if neighborhoods: + for neighborhood in neighborhoods: + name = neighborhood.get("name") + if name: + neighborhoods_list.append(name) + + neighborhoods_str = ( + ", ".join(neighborhoods_list) if neighborhoods_list else None + ) + realty_property = Property( + property_url="https://www.realtor.com/realestateandhomes-detail/" + + result["property_id"], + mls=mls, + mls_id=mls_id, + # status=(result["source"]["raw"].get("status").upper() if 'source' in result and isinstance(result["source"], dict) and "raw" in result["source"] and isinstance(result["source"]["raw"], dict) else None), + status=result["status"].upper(), + style=result["description"]["type"].upper(), + beds=result["description"]["beds"], + baths_full=result["description"]["baths_full"], + baths_half=result["description"]["baths_half"], + est_sf=result["description"]["sqft"], + lot_sf=result["description"]["lot_sqft"], + list_price=result["list_price"], + list_date=result["list_date"].split("T")[0], + sold_price=result["description"]["sold_price"], + prc_sqft=result["price_per_sqft"], + last_sold_date=result["last_sold_date"], + hoa_fee=result["hoa"]["fee"] + if result.get("hoa") and isinstance(result["hoa"], dict) + else None, address=Address( - address_one=address_one, + street=f"{result['location']['address']['street_number']} {result['location']['address']['street_name']} {result['location']['address']['street_suffix']}", + unit=result["location"]["address"]["unit"], city=result["location"]["address"]["city"], state=result["location"]["address"]["state_code"], - zip_code=result["location"]["address"]["postal_code"], - address_two=parse_address_two(result["location"]["address"]["unit"]), + zip=result["location"]["address"]["postal_code"], ), + yr_blt=result["description"]["year_built"], latitude=result["location"]["address"]["coordinate"]["lat"] if result and result.get("location") @@ -273,26 +352,16 @@ class RealtorScraper(Scraper): and result["location"]["address"].get("coordinate") and "lon" in result["location"]["address"]["coordinate"] else None, - site_name=self.site_name, - property_url="https://www.realtor.com/realestateandhomes-detail/" + result["property_id"], + prkg_gar=result["description"]["garage"], stories=result["description"]["stories"], - year_built=result["description"]["year_built"], - price_per_sqft=result["price_per_sqft"], - mls_id=result["property_id"], - listing_type=self.listing_type, - lot_area_value=result["description"]["lot_sqft"], - beds_min=result["description"]["beds"], - beds_max=result["description"]["beds"], - baths_min=result["description"]["baths"], - baths_max=result["description"]["baths"], - sqft_min=result["description"]["sqft"], - sqft_max=result["description"]["sqft"], - price_min=result["list_price"], - price_max=result["list_price"], + neighborhoods=neighborhoods_str, ) properties.append(realty_property) - return properties + return { + "total": response_json["data"]["home_search"]["total"], + "properties": properties, + } def search(self): location_info = self.handle_location() @@ -311,20 +380,20 @@ class RealtorScraper(Scraper): "offset": offset, } - total = self.handle_area(search_variables, return_total=True) + result = self.handle_area(search_variables) + total = result["total"] + homes = result["properties"] - homes = [] with ThreadPoolExecutor(max_workers=10) as executor: futures = [ executor.submit( self.handle_area, variables=search_variables | {"offset": i}, - return_total=False, ) - for i in range(0, total, 200) + for i in range(200, min(total, 10000), 200) ] for future in as_completed(futures): - homes.extend(future.result()) + homes.extend(future.result()["properties"]) return homes diff --git a/homeharvest/core/scrapers/redfin/__init__.py b/homeharvest/core/scrapers/redfin/__init__.py deleted file mode 100644 index 80b91f8..0000000 --- a/homeharvest/core/scrapers/redfin/__init__.py +++ /dev/null @@ -1,246 +0,0 @@ -""" -homeharvest.redfin.__init__ -~~~~~~~~~~~~ - -This module implements the scraper for redfin.com -""" -import json -from typing import Any -from .. import Scraper -from ....utils import parse_address_two, parse_address_one -from ..models import Property, Address, PropertyType, ListingType, SiteName, Agent -from ....exceptions import NoResultsFound, SearchTooBroad -from datetime import datetime - - -class RedfinScraper(Scraper): - def __init__(self, scraper_input): - super().__init__(scraper_input) - self.listing_type = scraper_input.listing_type - - def _handle_location(self): - url = "https://www.redfin.com/stingray/do/location-autocomplete?v=2&al=1&location={}".format(self.location) - - response = self.session.get(url) - response_json = json.loads(response.text.replace("{}&&", "")) - - def get_region_type(match_type: str): - if match_type == "4": - return "2" #: zip - elif match_type == "2": - return "6" #: city - elif match_type == "1": - return "address" #: address, needs to be handled differently - elif match_type == "11": - return "state" - - if "exactMatch" not in response_json["payload"]: - raise NoResultsFound("No results found for location: {}".format(self.location)) - - if response_json["payload"]["exactMatch"] is not None: - target = response_json["payload"]["exactMatch"] - else: - target = response_json["payload"]["sections"][0]["rows"][0] - - return target["id"].split("_")[1], get_region_type(target["type"]) - - def _parse_home(self, home: dict, single_search: bool = False) -> Property: - def get_value(key: str) -> Any | None: - if key in home and "value" in home[key]: - return home[key]["value"] - - if not single_search: - address = Address( - address_one=parse_address_one(get_value("streetLine"))[0], - address_two=parse_address_one(get_value("streetLine"))[1], - city=home.get("city"), - state=home.get("state"), - zip_code=home.get("zip"), - ) - else: - address_info = home.get("streetAddress") - address_one, address_two = parse_address_one(address_info.get("assembledAddress")) - - address = Address( - address_one=address_one, - address_two=address_two, - city=home.get("city"), - state=home.get("state"), - zip_code=home.get("zip"), - ) - - url = "https://www.redfin.com{}".format(home["url"]) - lot_size_data = home.get("lotSize") - - if not isinstance(lot_size_data, int): - lot_size = lot_size_data.get("value", None) if isinstance(lot_size_data, dict) else None - else: - lot_size = lot_size_data - - lat_long = get_value("latLong") - - return Property( - site_name=self.site_name, - listing_type=self.listing_type, - address=address, - property_url=url, - beds_min=home["beds"] if "beds" in home else None, - beds_max=home["beds"] if "beds" in home else None, - baths_min=home["baths"] if "baths" in home else None, - baths_max=home["baths"] if "baths" in home else None, - price_min=get_value("price"), - price_max=get_value("price"), - sqft_min=get_value("sqFt"), - sqft_max=get_value("sqFt"), - stories=home["stories"] if "stories" in home else None, - agent=Agent( #: listingAgent, some have sellingAgent as well - name=home['listingAgent'].get('name') if 'listingAgent' in home else None, - phone=home['listingAgent'].get('phone') if 'listingAgent' in home else None, - ), - description=home["listingRemarks"] if "listingRemarks" in home else None, - year_built=get_value("yearBuilt") if not single_search else home.get("yearBuilt"), - lot_area_value=lot_size, - property_type=PropertyType.from_int_code(home.get("propertyType")), - price_per_sqft=get_value("pricePerSqFt") if type(home.get("pricePerSqFt")) != int else home.get("pricePerSqFt"), - mls_id=get_value("mlsId"), - latitude=lat_long.get('latitude') if lat_long else None, - longitude=lat_long.get('longitude') if lat_long else None, - sold_date=datetime.fromtimestamp(home['soldDate'] / 1000) if 'soldDate' in home else None, - days_on_market=get_value("dom") - ) - - def _handle_rentals(self, region_id, region_type): - url = f"https://www.redfin.com/stingray/api/v1/search/rentals?al=1&isRentals=true®ion_id={region_id}®ion_type={region_type}&num_homes=100000" - - response = self.session.get(url) - response.raise_for_status() - homes = response.json() - - properties_list = [] - - for home in homes["homes"]: - home_data = home["homeData"] - rental_data = home["rentalExtension"] - - property_url = f"https://www.redfin.com{home_data.get('url', '')}" - address_info = home_data.get("addressInfo", {}) - centroid = address_info.get("centroid", {}).get("centroid", {}) - address = Address( - address_one=parse_address_one(address_info.get("formattedStreetLine"))[0], - city=address_info.get("city"), - state=address_info.get("state"), - zip_code=address_info.get("zip"), - ) - - price_range = rental_data.get("rentPriceRange", {"min": None, "max": None}) - bed_range = rental_data.get("bedRange", {"min": None, "max": None}) - bath_range = rental_data.get("bathRange", {"min": None, "max": None}) - sqft_range = rental_data.get("sqftRange", {"min": None, "max": None}) - - property_ = Property( - property_url=property_url, - site_name=SiteName.REDFIN, - listing_type=ListingType.FOR_RENT, - address=address, - description=rental_data.get("description"), - latitude=centroid.get("latitude"), - longitude=centroid.get("longitude"), - baths_min=bath_range.get("min"), - baths_max=bath_range.get("max"), - beds_min=bed_range.get("min"), - beds_max=bed_range.get("max"), - price_min=price_range.get("min"), - price_max=price_range.get("max"), - sqft_min=sqft_range.get("min"), - sqft_max=sqft_range.get("max"), - img_src=home_data.get("staticMapUrl"), - posted_time=rental_data.get("lastUpdated"), - bldg_name=rental_data.get("propertyName"), - ) - - properties_list.append(property_) - - if not properties_list: - raise NoResultsFound("No rentals found for the given location.") - - return properties_list - - def _parse_building(self, building: dict) -> Property: - street_address = " ".join( - [ - building["address"]["streetNumber"], - building["address"]["directionalPrefix"], - building["address"]["streetName"], - building["address"]["streetType"], - ] - ) - return Property( - site_name=self.site_name, - property_type=PropertyType("BUILDING"), - address=Address( - address_one=parse_address_one(street_address)[0], - city=building["address"]["city"], - state=building["address"]["stateOrProvinceCode"], - zip_code=building["address"]["postalCode"], - address_two=parse_address_two( - " ".join( - [ - building["address"]["unitType"], - building["address"]["unitValue"], - ] - ) - ), - ), - property_url="https://www.redfin.com{}".format(building["url"]), - listing_type=self.listing_type, - unit_count=building.get("numUnitsForSale"), - ) - - def handle_address(self, home_id: str): - """ - EPs: - https://www.redfin.com/stingray/api/home/details/initialInfo?al=1&path=/TX/Austin/70-Rainey-St-78701/unit-1608/home/147337694 - https://www.redfin.com/stingray/api/home/details/mainHouseInfoPanelInfo?propertyId=147337694&accessLevel=3 - https://www.redfin.com/stingray/api/home/details/aboveTheFold?propertyId=147337694&accessLevel=3 - https://www.redfin.com/stingray/api/home/details/belowTheFold?propertyId=147337694&accessLevel=3 - """ - url = "https://www.redfin.com/stingray/api/home/details/aboveTheFold?propertyId={}&accessLevel=3".format( - home_id - ) - - response = self.session.get(url) - response_json = json.loads(response.text.replace("{}&&", "")) - - parsed_home = self._parse_home(response_json["payload"]["addressSectionInfo"], single_search=True) - return [parsed_home] - - def search(self): - region_id, region_type = self._handle_location() - - if region_type == "state": - raise SearchTooBroad("State searches are not supported, please use a more specific location.") - - if region_type == "address": - home_id = region_id - return self.handle_address(home_id) - - if self.listing_type == ListingType.FOR_RENT: - return self._handle_rentals(region_id, region_type) - else: - if self.listing_type == ListingType.FOR_SALE: - url = f"https://www.redfin.com/stingray/api/gis?al=1®ion_id={region_id}®ion_type={region_type}&num_homes=100000" - else: - url = f"https://www.redfin.com/stingray/api/gis?al=1®ion_id={region_id}®ion_type={region_type}&sold_within_days=30&num_homes=100000" - response = self.session.get(url) - response_json = json.loads(response.text.replace("{}&&", "")) - - if "payload" in response_json: - homes_list = response_json["payload"].get("homes", []) - buildings_list = response_json["payload"].get("buildings", {}).values() - - homes = [self._parse_home(home) for home in homes_list] + [ - self._parse_building(building) for building in buildings_list - ] - return homes - else: - return [] diff --git a/homeharvest/core/scrapers/zillow/__init__.py b/homeharvest/core/scrapers/zillow/__init__.py deleted file mode 100644 index ba55a01..0000000 --- a/homeharvest/core/scrapers/zillow/__init__.py +++ /dev/null @@ -1,335 +0,0 @@ -""" -homeharvest.zillow.__init__ -~~~~~~~~~~~~ - -This module implements the scraper for zillow.com -""" -import re -import json - -import tls_client - -from .. import Scraper -from requests.exceptions import HTTPError -from ....utils import parse_address_one, parse_address_two -from ....exceptions import GeoCoordsNotFound, NoResultsFound -from ..models import Property, Address, ListingType, PropertyType, Agent -import urllib.parse -from datetime import datetime, timedelta - - -class ZillowScraper(Scraper): - def __init__(self, scraper_input): - session = tls_client.Session( - client_identifier="chrome112", random_tls_extension_order=True - ) - - super().__init__(scraper_input, session) - - self.session.headers.update({ - 'authority': 'www.zillow.com', - 'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9', - 'accept-language': 'en-US,en;q=0.9', - 'cache-control': 'max-age=0', - 'sec-fetch-dest': 'document', - 'sec-fetch-mode': 'navigate', - 'sec-fetch-site': 'same-origin', - 'sec-fetch-user': '?1', - 'upgrade-insecure-requests': '1', - 'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.0.0 Safari/537.36', - }) - - if not self.is_plausible_location(self.location): - raise NoResultsFound("Invalid location input: {}".format(self.location)) - - listing_type_to_url_path = { - ListingType.FOR_SALE: "for_sale", - ListingType.FOR_RENT: "for_rent", - ListingType.SOLD: "recently_sold", - } - - self.url = f"https://www.zillow.com/homes/{listing_type_to_url_path[self.listing_type]}/{self.location}_rb/" - - def is_plausible_location(self, location: str) -> bool: - url = ( - "https://www.zillowstatic.com/autocomplete/v3/suggestions?q={" - "}&abKey=6666272a-4b99-474c-b857-110ec438732b&clientId=homepage-render" - ).format(urllib.parse.quote(location)) - - resp = self.session.get(url) - - return resp.json()["results"] != [] - - def search(self): - resp = self.session.get(self.url) - if resp.status_code != 200: - raise HTTPError( - f"bad response status code: {resp.status_code}" - ) - content = resp.text - - match = re.search( - r'', - content, - re.DOTALL, - ) - if not match: - raise NoResultsFound("No results were found for Zillow with the given Location.") - - json_str = match.group(1) - data = json.loads(json_str) - - if "searchPageState" in data["props"]["pageProps"]: - pattern = r'window\.mapBounds = \{\s*"west":\s*(-?\d+\.\d+),\s*"east":\s*(-?\d+\.\d+),\s*"south":\s*(-?\d+\.\d+),\s*"north":\s*(-?\d+\.\d+)\s*\};' - - match = re.search(pattern, content) - - if match: - coords = [float(coord) for coord in match.groups()] - return self._fetch_properties_backend(coords) - - else: - raise GeoCoordsNotFound("Box bounds could not be located.") - - elif "gdpClientCache" in data["props"]["pageProps"]: - gdp_client_cache = json.loads(data["props"]["pageProps"]["gdpClientCache"]) - main_key = list(gdp_client_cache.keys())[0] - - property_data = gdp_client_cache[main_key]["property"] - property = self._get_single_property_page(property_data) - - return [property] - raise NoResultsFound("Specific property data not found in the response.") - - def _fetch_properties_backend(self, coords): - url = "https://www.zillow.com/async-create-search-page-state" - - filter_state_for_sale = { - "sortSelection": { - # "value": "globalrelevanceex" - "value": "days" - }, - "isAllHomes": {"value": True}, - } - - filter_state_for_rent = { - "isForRent": {"value": True}, - "isForSaleByAgent": {"value": False}, - "isForSaleByOwner": {"value": False}, - "isNewConstruction": {"value": False}, - "isComingSoon": {"value": False}, - "isAuction": {"value": False}, - "isForSaleForeclosure": {"value": False}, - "isAllHomes": {"value": True}, - } - - filter_state_sold = { - "isRecentlySold": {"value": True}, - "isForSaleByAgent": {"value": False}, - "isForSaleByOwner": {"value": False}, - "isNewConstruction": {"value": False}, - "isComingSoon": {"value": False}, - "isAuction": {"value": False}, - "isForSaleForeclosure": {"value": False}, - "isAllHomes": {"value": True}, - } - - selected_filter = ( - filter_state_for_rent - if self.listing_type == ListingType.FOR_RENT - else filter_state_for_sale - if self.listing_type == ListingType.FOR_SALE - else filter_state_sold - ) - - payload = { - "searchQueryState": { - "pagination": {}, - "isMapVisible": True, - "mapBounds": { - "west": coords[0], - "east": coords[1], - "south": coords[2], - "north": coords[3], - }, - "filterState": selected_filter, - "isListVisible": True, - "mapZoom": 11, - }, - "wants": {"cat1": ["mapResults"]}, - "isDebugRequest": False, - } - resp = self.session.put(url, json=payload) - if resp.status_code != 200: - raise HTTPError( - f"bad response status code: {resp.status_code}" - ) - return self._parse_properties(resp.json()) - - @staticmethod - def parse_posted_time(time: str) -> datetime: - int_time = int(time.split(" ")[0]) - - if "hour" in time: - return datetime.now() - timedelta(hours=int_time) - - if "day" in time: - return datetime.now() - timedelta(days=int_time) - - def _parse_properties(self, property_data: dict): - mapresults = property_data["cat1"]["searchResults"]["mapResults"] - - properties_list = [] - - for result in mapresults: - if "hdpData" in result: - home_info = result["hdpData"]["homeInfo"] - address_data = { - "address_one": parse_address_one(home_info.get("streetAddress"))[0], - "address_two": parse_address_two(home_info["unit"]) if "unit" in home_info else "#", - "city": home_info.get("city"), - "state": home_info.get("state"), - "zip_code": home_info.get("zipcode"), - } - property_obj = Property( - site_name=self.site_name, - address=Address(**address_data), - property_url=f"https://www.zillow.com{result['detailUrl']}", - tax_assessed_value=int(home_info["taxAssessedValue"]) if "taxAssessedValue" in home_info else None, - property_type=PropertyType(home_info.get("homeType")), - listing_type=ListingType( - home_info["statusType"] if "statusType" in home_info else self.listing_type - ), - status_text=result.get("statusText"), - posted_time=self.parse_posted_time(result["variableData"]["text"]) - if "variableData" in result - and "text" in result["variableData"] - and result["variableData"]["type"] == "TIME_ON_INFO" - else None, - price_min=home_info.get("price"), - price_max=home_info.get("price"), - beds_min=int(home_info["bedrooms"]) if "bedrooms" in home_info else None, - beds_max=int(home_info["bedrooms"]) if "bedrooms" in home_info else None, - baths_min=home_info.get("bathrooms"), - baths_max=home_info.get("bathrooms"), - sqft_min=int(home_info["livingArea"]) if "livingArea" in home_info else None, - sqft_max=int(home_info["livingArea"]) if "livingArea" in home_info else None, - price_per_sqft=int(home_info["price"] // home_info["livingArea"]) - if "livingArea" in home_info and home_info["livingArea"] != 0 and "price" in home_info - else None, - latitude=result["latLong"]["latitude"], - longitude=result["latLong"]["longitude"], - lot_area_value=round(home_info["lotAreaValue"], 2) if "lotAreaValue" in home_info else None, - lot_area_unit=home_info.get("lotAreaUnit"), - img_src=result.get("imgSrc"), - ) - - properties_list.append(property_obj) - - elif "isBuilding" in result: - price_string = result["price"].replace("$", "").replace(",", "").replace("+/mo", "") - - match = re.search(r"(\d+)", price_string) - price_value = int(match.group(1)) if match else None - building_obj = Property( - property_url=f"https://www.zillow.com{result['detailUrl']}", - site_name=self.site_name, - property_type=PropertyType("BUILDING"), - listing_type=ListingType(result["statusType"]), - img_src=result.get("imgSrc"), - address=self._extract_address(result["address"]), - baths_min=result.get("minBaths"), - area_min=result.get("minArea"), - bldg_name=result.get("communityName"), - status_text=result.get("statusText"), - price_min=price_value if "+/mo" in result.get("price") else None, - price_max=price_value if "+/mo" in result.get("price") else None, - latitude=result.get("latLong", {}).get("latitude"), - longitude=result.get("latLong", {}).get("longitude"), - unit_count=result.get("unitCount"), - ) - - properties_list.append(building_obj) - - return properties_list - - def _get_single_property_page(self, property_data: dict): - """ - This method is used when a user enters the exact location & zillow returns just one property - """ - url = ( - f"https://www.zillow.com{property_data['hdpUrl']}" - if "zillow.com" not in property_data["hdpUrl"] - else property_data["hdpUrl"] - ) - address_data = property_data["address"] - address_one, address_two = parse_address_one(address_data["streetAddress"]) - address = Address( - address_one=address_one, - address_two=address_two if address_two else "#", - city=address_data["city"], - state=address_data["state"], - zip_code=address_data["zipcode"], - ) - property_type = property_data.get("homeType", None) - return Property( - site_name=self.site_name, - property_url=url, - property_type=PropertyType(property_type) if property_type in PropertyType.__members__ else None, - listing_type=self.listing_type, - address=address, - year_built=property_data.get("yearBuilt"), - tax_assessed_value=property_data.get("taxAssessedValue"), - lot_area_value=property_data.get("lotAreaValue"), - lot_area_unit=property_data["lotAreaUnits"].lower() if "lotAreaUnits" in property_data else None, - agent=Agent( - name=property_data.get("attributionInfo", {}).get("agentName") - ), - stories=property_data.get("resoFacts", {}).get("stories"), - mls_id=property_data.get("attributionInfo", {}).get("mlsId"), - beds_min=property_data.get("bedrooms"), - beds_max=property_data.get("bedrooms"), - baths_min=property_data.get("bathrooms"), - baths_max=property_data.get("bathrooms"), - price_min=property_data.get("price"), - price_max=property_data.get("price"), - sqft_min=property_data.get("livingArea"), - sqft_max=property_data.get("livingArea"), - price_per_sqft=property_data.get("resoFacts", {}).get("pricePerSquareFoot"), - latitude=property_data.get("latitude"), - longitude=property_data.get("longitude"), - img_src=property_data.get("streetViewTileImageUrlMediumAddress"), - description=property_data.get("description"), - ) - - def _extract_address(self, address_str): - """ - Extract address components from a string formatted like '555 Wedglea Dr, Dallas, TX', - and return an Address object. - """ - parts = address_str.split(", ") - - if len(parts) != 3: - raise ValueError(f"Unexpected address format: {address_str}") - - address_one = parts[0].strip() - city = parts[1].strip() - state_zip = parts[2].split(" ") - - if len(state_zip) == 1: - state = state_zip[0].strip() - zip_code = None - elif len(state_zip) == 2: - state = state_zip[0].strip() - zip_code = state_zip[1].strip() - else: - raise ValueError(f"Unexpected state/zip format in address: {address_str}") - - address_one, address_two = parse_address_one(address_one) - return Address( - address_one=address_one, - address_two=address_two if address_two else "#", - city=city, - state=state, - zip_code=zip_code, - ) diff --git a/homeharvest/exceptions.py b/homeharvest/exceptions.py index 95eedbc..55a5dca 100644 --- a/homeharvest/exceptions.py +++ b/homeharvest/exceptions.py @@ -16,3 +16,7 @@ class GeoCoordsNotFound(Exception): class SearchTooBroad(Exception): """Raised when the search is too broad""" + + +class InvalidTimeFrame(Exception): + """Raised when the time frame is invalid""" diff --git a/homeharvest/utils.py b/homeharvest/utils.py index 2aeedee..603f9b4 100644 --- a/homeharvest/utils.py +++ b/homeharvest/utils.py @@ -1,38 +1,76 @@ -import re +from .core.scrapers.models import Property +import pandas as pd + +ordered_properties = [ + "PropertyURL", + "MLS", + "MLS #", + "Status", + "Style", + "Street", + "Unit", + "City", + "State", + "Zip", + "Beds", + "FB", + "NumHB", + "EstSF", + "YrBlt", + "ListPrice", + "Lst Date", + "Sold Price", + "COEDate", + "LotSFApx", + "PrcSqft", + "LATITUDE", + "LONGITUDE", + "Stories", + "HOAFee", + "PrkgGar", + "Community", +] -def parse_address_one(street_address: str) -> tuple: - if not street_address: - return street_address, "#" +def process_result(result: Property) -> pd.DataFrame: + prop_data = {prop: None for prop in ordered_properties} + prop_data.update(result.__dict__) + prop_data["PropertyURL"] = prop_data["property_url"] + prop_data["MLS"] = prop_data["mls"] + prop_data["MLS #"] = prop_data["mls_id"] + prop_data["Status"] = prop_data["status"] + prop_data["Style"] = prop_data["style"] - apt_match = re.search( - r"(APT\s*[\dA-Z]+|#[\dA-Z]+|UNIT\s*[\dA-Z]+|LOT\s*[\dA-Z]+|SUITE\s*[\dA-Z]+)$", - street_address, - re.I, - ) + if "address" in prop_data: + address_data = prop_data["address"] + prop_data["Street"] = address_data.street + prop_data["Unit"] = address_data.unit + prop_data["City"] = address_data.city + prop_data["State"] = address_data.state + prop_data["Zip"] = address_data.zip - if apt_match: - apt_str = apt_match.group().strip() - cleaned_apt_str = re.sub(r"(APT\s*|UNIT\s*|LOT\s*|SUITE\s*)", "#", apt_str, flags=re.I) + prop_data["Community"] = prop_data["neighborhoods"] + prop_data["Beds"] = prop_data["beds"] + prop_data["FB"] = prop_data["baths_full"] + prop_data["NumHB"] = prop_data["baths_half"] + prop_data["EstSF"] = prop_data["est_sf"] + prop_data["ListPrice"] = prop_data["list_price"] + prop_data["Lst Date"] = prop_data["list_date"] + prop_data["Sold Price"] = prop_data["sold_price"] + prop_data["COEDate"] = prop_data["last_sold_date"] + prop_data["LotSFApx"] = prop_data["lot_sf"] + prop_data["HOAFee"] = prop_data["hoa_fee"] - main_address = street_address.replace(apt_str, "").strip() - return main_address, cleaned_apt_str - else: - return street_address, "#" + if prop_data.get("prc_sqft") is not None: + prop_data["PrcSqft"] = round(prop_data["prc_sqft"], 2) + prop_data["YrBlt"] = prop_data["yr_blt"] + prop_data["LATITUDE"] = prop_data["latitude"] + prop_data["LONGITUDE"] = prop_data["longitude"] + prop_data["Stories"] = prop_data["stories"] + prop_data["PrkgGar"] = prop_data["prkg_gar"] -def parse_address_two(street_address: str): - if not street_address: - return "#" - apt_match = re.search( - r"(APT\s*[\dA-Z]+|#[\dA-Z]+|UNIT\s*[\dA-Z]+|LOT\s*[\dA-Z]+|SUITE\s*[\dA-Z]+)$", - street_address, - re.I, - ) + properties_df = pd.DataFrame([prop_data]) + properties_df = properties_df.reindex(columns=ordered_properties) - if apt_match: - apt_str = apt_match.group().strip() - apt_str = re.sub(r"(APT\s*|UNIT\s*|LOT\s*|SUITE\s*)", "#", apt_str, flags=re.I) - return apt_str - else: - return "#" + return properties_df[ordered_properties] diff --git a/tests/test_realtor.py b/tests/test_realtor.py index 3b23529..557ca64 100644 --- a/tests/test_realtor.py +++ b/tests/test_realtor.py @@ -12,13 +12,13 @@ def test_realtor(): scrape_property( location="2530 Al Lipscomb Way", site_name="realtor.com", - listing_type="for_sale", + status="for_sale", ), scrape_property( - location="Phoenix, AZ", site_name=["realtor.com"], listing_type="for_rent" + location="Phoenix, AZ", site_name=["realtor.com"], status="for_rent" ), #: does not support "city, state, USA" format scrape_property( - location="Dallas, TX", site_name="realtor.com", listing_type="sold" + location="Dallas, TX", site_name="realtor.com", status="sold" ), #: does not support "city, state, USA" format scrape_property(location="85281", site_name="realtor.com"), ] @@ -31,7 +31,7 @@ def test_realtor(): scrape_property( location="abceefg ju098ot498hh9", site_name="realtor.com", - listing_type="for_sale", + status="sale", ) ] except (InvalidSite, InvalidListingType, NoResultsFound, GeoCoordsNotFound): diff --git a/tests/test_redfin.py b/tests/test_redfin.py index 6904499..421a7f0 100644 --- a/tests/test_redfin.py +++ b/tests/test_redfin.py @@ -10,10 +10,14 @@ from homeharvest.exceptions import ( def test_redfin(): results = [ - scrape_property(location="San Diego", site_name="redfin", listing_type="for_sale"), - scrape_property(location="2530 Al Lipscomb Way", site_name="redfin", listing_type="for_sale"), - scrape_property(location="Phoenix, AZ, USA", site_name=["redfin"], listing_type="for_rent"), - scrape_property(location="Dallas, TX, USA", site_name="redfin", listing_type="sold"), + scrape_property(location="San Diego", site_name="redfin", status="sale"), + scrape_property( + location="2530 Al Lipscomb Way", site_name="redfin", status="sale" + ), + scrape_property( + location="Phoenix, AZ, USA", site_name=["redfin"], status="rent" + ), + scrape_property(location="Dallas, TX, USA", site_name="redfin", status="sold"), scrape_property(location="85281", site_name="redfin"), ] @@ -25,11 +29,17 @@ def test_redfin(): scrape_property( location="abceefg ju098ot498hh9", site_name="redfin", - listing_type="for_sale", + status="sale", ), - scrape_property(location="Florida", site_name="redfin", listing_type="for_rent"), + scrape_property(location="Florida", site_name="redfin", status="for_rent"), ] - except (InvalidSite, InvalidListingType, NoResultsFound, GeoCoordsNotFound, SearchTooBroad): + except ( + InvalidSite, + InvalidListingType, + NoResultsFound, + GeoCoordsNotFound, + SearchTooBroad, + ): assert True assert all([result is None for result in bad_results]) diff --git a/tests/test_utils.py b/tests/test_utils.py index d21ee77..ed9f0ef 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -17,7 +17,13 @@ def test_parse_address_one(): def test_parse_address_two(): - test_data = [("Apt 126", "#126"), ("apt 2B", "#2B"), ("UNIT 3A", "#3A"), ("unit 3A", "#3A"), ("SuIte 3A", "#3A")] + test_data = [ + ("Apt 126", "#126"), + ("apt 2B", "#2B"), + ("UNIT 3A", "#3A"), + ("unit 3A", "#3A"), + ("SuIte 3A", "#3A"), + ] for input_data, expected in test_data: output = parse_address_two(input_data) diff --git a/tests/test_zillow.py b/tests/test_zillow.py index dfcc55d..6f8e27d 100644 --- a/tests/test_zillow.py +++ b/tests/test_zillow.py @@ -9,12 +9,24 @@ from homeharvest.exceptions import ( def test_zillow(): results = [ - scrape_property(location="2530 Al Lipscomb Way", site_name="zillow", listing_type="for_sale"), - scrape_property(location="Phoenix, AZ, USA", site_name=["zillow"], listing_type="for_rent"), - scrape_property(location="Surprise, AZ", site_name=["zillow"], listing_type="for_sale"), - scrape_property(location="Dallas, TX, USA", site_name="zillow", listing_type="sold"), + scrape_property( + location="2530 Al Lipscomb Way", site_name="zillow", listing_type="for_sale" + ), + scrape_property( + location="Phoenix, AZ, USA", site_name=["zillow"], listing_type="for_rent" + ), + scrape_property( + location="Surprise, AZ", site_name=["zillow"], listing_type="for_sale" + ), + scrape_property( + location="Dallas, TX, USA", site_name="zillow", listing_type="sold" + ), scrape_property(location="85281", site_name="zillow"), - scrape_property(location="3268 88th st s, Lakewood", site_name="zillow", listing_type="for_rent"), + scrape_property( + location="3268 88th st s, Lakewood", + site_name="zillow", + listing_type="for_rent", + ), ] assert all([result is not None for result in results])