.github/FUNDING.yml

@@ -1 +0,0 @@
-github: Bunsly

README.md

@@ -2,6 +2,10 @@
 
 **HomeHarvest** is a real estate scraping library that extracts and formats data in the style of MLS listings.
 
+**Not technical?** Try out the web scraping tool on our site at [tryhomeharvest.com](https://tryhomeharvest.com).
+
+*Looking to build a data-focused software product?* **[Book a call](https://bunsly.com)** *to work with us.*
+
 ## HomeHarvest Features
 
 - **Source**: Fetches properties directly from **Realtor.com**.
@@ -36,7 +40,6 @@ properties = scrape_property(
   listing_type="sold",  # or (for_sale, for_rent, pending)
   past_days=30,  # sold in last 30 days - listed in last 30 days if (for_sale, for_rent)
 
-  # property_type=['single_family','multi_family'],
   # date_from="2023-05-01", # alternative to past_days
   # date_to="2023-05-28",
   # foreclosure=True
@@ -65,30 +68,13 @@ print(properties.head())
 ```
 Required
 ├── location (str): The address in various formats - this could be just a zip code, a full address, or city/state, etc.
-├── listing_type (option): Choose the type of listing.
+└── listing_type (option): Choose the type of listing.
     - 'for_rent'
     - 'for_sale'
     - 'sold'
-    - 'pending' (for pending/contingent sales)
+    - 'pending'
 
 Optional
-├── property_type (list): Choose the type of properties.
-    - 'single_family'
-    - 'multi_family'
-    - 'condos'
-    - 'condo_townhome_rowhome_coop'
-    - 'condo_townhome'
-    - 'townhomes'
-    - 'duplex_triplex'
-    - 'farm'
-    - 'land'
-    - 'mobile'
-│
-├── return_type (option): Choose the return type.
-│    - 'pandas' (default)
-│    - 'pydantic'
-│    - 'raw' (json)
-│
 ├── radius (decimal): Radius in miles to find comparable properties based on individual addresses.
 │    Example: 5.5 (fetches properties within a 5.5-mile radius if location is set to a specific address; otherwise, ignored)
 │
@@ -108,7 +94,7 @@ Optional
 │
 ├── extra_property_data (True/False): Increases requests by O(n). If set, this fetches additional property data for general searches (e.g. schools, tax appraisals etc.)
 │
-├── exclude_pending (True/False): If set, excludes 'pending' properties from the 'for_sale' results unless listing_type is 'pending'
+├── exclude_pending (True/False): If set, excludes pending properties from the results unless listing_type is 'pending'
 │
 └── limit (integer): Limit the number of properties to fetch. Max & default is 10000.
 ```
@@ -155,14 +141,6 @@ Property
 │ ├── new_construction
 │ └── hoa_fee
 
-├── Tax Information:
-│  ├── year
-│  ├── tax
-│  ├── assessment
-│  │   ├── building
-│  │   ├── land
-│  │   └── total
-
 ├── Location Details:
 │ ├── latitude
 │ ├── longitude

examples/HomeHarvest_Demo.ipynb

@@ -0,0 +1,141 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "cb48903e-5021-49fe-9688-45cd0bc05d0f",
+   "metadata": {
+    "is_executing": true
+   },
+   "outputs": [],
+   "source": [
+    "from homeharvest import scrape_property\n",
+    "import pandas as pd"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "156488ce-0d5f-43c5-87f4-c33e9c427860",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "pd.set_option('display.max_columns', None)  # Show all columns\n",
+    "pd.set_option('display.max_rows', None)     # Show all rows\n",
+    "pd.set_option('display.width', None)        # Auto-adjust display width to fit console\n",
+    "pd.set_option('display.max_colwidth', 50)   # Limit max column width to 50 characters"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "1c8b9744-8606-4e9b-8add-b90371a249a7",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# check for sale properties\n",
+    "scrape_property(\n",
+    "    location=\"dallas\",\n",
+    "    listing_type=\"for_sale\"\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "aaf86093",
+   "metadata": {
+    "collapsed": false,
+    "jupyter": {
+     "outputs_hidden": false
+    }
+   },
+   "outputs": [],
+   "source": [
+    "# search a specific address\n",
+    "scrape_property(\n",
+    "    location=\"2530 Al Lipscomb Way\",\n",
+    "    listing_type=\"for_sale\"\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "ab7b4c21-da1d-4713-9df4-d7425d8ce21e",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# check rentals\n",
+    "scrape_property(\n",
+    "    location=\"chicago, illinois\",\n",
+    "    listing_type=\"for_rent\"\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "af280cd3",
+   "metadata": {
+    "collapsed": false,
+    "jupyter": {
+     "outputs_hidden": false
+    }
+   },
+   "outputs": [],
+   "source": [
+    "# check sold properties\n",
+    "properties = scrape_property(\n",
+    "    location=\"90210\",\n",
+    "    listing_type=\"sold\",\n",
+    "    past_days=10\n",
+    ")\n",
+    "display(properties)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "628c1ce2",
+   "metadata": {
+    "collapsed": false,
+    "is_executing": true,
+    "jupyter": {
+     "outputs_hidden": false
+    }
+   },
+   "outputs": [],
+   "source": [
+    "# display clickable URLs\n",
+    "from IPython.display import display, HTML\n",
+    "properties['property_url'] = '<a href=\"' + properties['property_url'] + '\" target=\"_blank\">' + properties['property_url'] + '</a>'\n",
+    "\n",
+    "html = properties.to_html(escape=False)\n",
+    "truncate_width = f'<style>.dataframe td {{ max-width: 200px; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; }}</style>{html}'\n",
+    "display(HTML(truncate_width))"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.11"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

examples/HomeHarvest_Demo.py

@@ -0,0 +1,20 @@
+from homeharvest import scrape_property
+from datetime import datetime
+
+# Generate filename based on current timestamp
+current_timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+filename = f"HomeHarvest_{current_timestamp}.csv"
+
+properties = scrape_property(
+    location="San Diego, CA",
+    listing_type="sold",  # or (for_sale, for_rent)
+    past_days=30,  # sold in last 30 days - listed in last x days if (for_sale, for_rent)
+    # pending_or_contingent=True # use on for_sale listings to find pending / contingent listings
+    # mls_only=True,  # only fetch MLS listings
+    # proxy="http://user:pass@host:port"  # use a proxy to change your IP address
+)
+print(f"Number of properties: {len(properties)}")
+
+# Export to csv
+properties.to_csv(filename, index=False)
+print(properties.head())

examples/price_of_land.py

@@ -1,104 +0,0 @@
-"""
-This script scrapes sold and pending sold land listings in past year for a list of zip codes and saves the data to individual Excel files.
-It adds two columns to the data: 'lot_acres' and 'ppa' (price per acre) for user to analyze average price of land in a zip code.
-"""
-
-import os
-import pandas as pd
-from homeharvest import scrape_property
-
-
-def get_property_details(zip: str, listing_type):
-    properties = scrape_property(location=zip, listing_type=listing_type, property_type=["land"], past_days=365)
-    if not properties.empty:
-        properties["lot_acres"] = properties["lot_sqft"].apply(lambda x: x / 43560 if pd.notnull(x) else None)
-
-        properties = properties[properties["sqft"].isnull()]
-        properties["ppa"] = properties.apply(
-            lambda row: (
-                int(
-                    (
-                        row["sold_price"]
-                        if (pd.notnull(row["sold_price"]) and row["status"] == "SOLD")
-                        else row["list_price"]
-                    )
-                    / row["lot_acres"]
-                )
-                if pd.notnull(row["lot_acres"])
-                and row["lot_acres"] > 0
-                and (pd.notnull(row["sold_price"]) or pd.notnull(row["list_price"]))
-                else None
-            ),
-            axis=1,
-        )
-        properties["ppa"] = properties["ppa"].astype("Int64")
-        selected_columns = [
-            "property_url",
-            "property_id",
-            "style",
-            "status",
-            "street",
-            "city",
-            "state",
-            "zip_code",
-            "county",
-            "list_date",
-            "last_sold_date",
-            "list_price",
-            "sold_price",
-            "lot_sqft",
-            "lot_acres",
-            "ppa",
-        ]
-        properties = properties[selected_columns]
-    return properties
-
-
-def output_to_excel(zip_code, sold_df, pending_df):
-    root_folder = os.getcwd()
-    zip_folder = os.path.join(root_folder, "zips", zip_code)
-
-    # Create zip code folder if it doesn't exist
-    os.makedirs(zip_folder, exist_ok=True)
-
-    # Define file paths
-    sold_file = os.path.join(zip_folder, f"{zip_code}_sold.xlsx")
-    pending_file = os.path.join(zip_folder, f"{zip_code}_pending.xlsx")
-
-    # Save individual sold and pending files
-    sold_df.to_excel(sold_file, index=False)
-    pending_df.to_excel(pending_file, index=False)
-
-
-zip_codes = map(
-    str,
-    [
-        22920,
-        77024,
-        78028,
-        24553,
-        22967,
-        22971,
-        22922,
-        22958,
-        22969,
-        22949,
-        22938,
-        24599,
-        24562,
-        22976,
-        24464,
-        22964,
-        24581,
-    ],
-)
-
-combined_df = pd.DataFrame()
-for zip in zip_codes:
-    sold_df = get_property_details(zip, "sold")
-    pending_df = get_property_details(zip, "pending")
-    combined_df = pd.concat([combined_df, sold_df, pending_df], ignore_index=True)
-    output_to_excel(zip, sold_df, pending_df)
-
-combined_file = os.path.join(os.getcwd(), "zips", "combined.xlsx")
-combined_df.to_excel(combined_file, index=False)

homeharvest/__init__.py

@@ -3,14 +3,12 @@ import pandas as pd
 from .core.scrapers import ScraperInput
 from .utils import process_result, ordered_properties, validate_input, validate_dates, validate_limit
 from .core.scrapers.realtor import RealtorScraper
-from .core.scrapers.models import ListingType, SearchPropertyType, ReturnType, Property
+from .core.scrapers.models import ListingType
 
 
 def scrape_property(
     location: str,
     listing_type: str = "for_sale",
-    return_type: str = "pandas",
-    property_type: list[str] | None = None,
     radius: float = None,
     mls_only: bool = False,
     past_days: int = None,
@@ -20,14 +18,12 @@ def scrape_property(
     foreclosure: bool = None,
     extra_property_data: bool = True,
     exclude_pending: bool = False,
-    limit: int = 10000
+    limit: int = 10000,
-) -> pd.DataFrame | list[dict] | list[Property]:
+) -> pd.DataFrame:
     """
     Scrape properties from Realtor.com based on a given location and listing type.
     :param location: Location to search (e.g. "Dallas, TX", "85281", "2530 Al Lipscomb Way")
     :param listing_type: Listing Type (for_sale, for_rent, sold, pending)
-    :param return_type: Return type (pandas, pydantic, raw)
-    :param property_type: Property Type (single_family, multi_family, condos, condo_townhome_rowhome_coop, condo_townhome, townhomes, duplex_triplex, farm, land, mobile)
     :param radius: Get properties within _ (e.g. 1.0) miles. Only applicable for individual addresses.
     :param mls_only: If set, fetches only listings with MLS IDs.
     :param proxy: Proxy to use for scraping
@@ -44,9 +40,7 @@ def scrape_property(
 
     scraper_input = ScraperInput(
         location=location,
-        listing_type=ListingType(listing_type.upper()),
+        listing_type=ListingType[listing_type.upper()],
-        return_type=ReturnType(return_type.lower()),
-        property_type=[SearchPropertyType[prop.upper()] for prop in property_type] if property_type else None,
         proxy=proxy,
         radius=radius,
         mls_only=mls_only,
@@ -62,9 +56,6 @@ def scrape_property(
     site = RealtorScraper(scraper_input)
     results = site.search()
 
-    if scraper_input.return_type != ReturnType.pandas:
-        return results
-
     properties_dfs = [df for result in results if not (df := process_result(result)).empty]
     if not properties_dfs:
         return pd.DataFrame()
@@ -72,6 +63,4 @@ def scrape_property(
     with warnings.catch_warnings():
         warnings.simplefilter("ignore", category=FutureWarning)
 
-        return pd.concat(properties_dfs, ignore_index=True, axis=0)[ordered_properties].replace(
+        return pd.concat(properties_dfs, ignore_index=True, axis=0)[ordered_properties].replace({"None": pd.NA, None: pd.NA, "": pd.NA})
-            {"None": pd.NA, None: pd.NA, "": pd.NA}
-        )

homeharvest/core/scrapers/__init__.py

@@ -1,13 +1,11 @@
 from __future__ import annotations
 from dataclasses import dataclass
-from typing import Union
-
 import requests
 from requests.adapters import HTTPAdapter
 from urllib3.util.retry import Retry
 import uuid
 from ...exceptions import AuthenticationError
-from .models import Property, ListingType, SiteName, SearchPropertyType, ReturnType
+from .models import Property, ListingType, SiteName
 import json
 
 
@@ -15,7 +13,6 @@ import json
 class ScraperInput:
     location: str
     listing_type: ListingType
-    property_type: list[SearchPropertyType] | None = None
     radius: float | None = None
     mls_only: bool | None = False
     proxy: str | None = None
@@ -26,7 +23,6 @@ class ScraperInput:
     extra_property_data: bool | None = True
     exclude_pending: bool | None = False
     limit: int = 10000
-    return_type: ReturnType = ReturnType.pandas
 
 
 class Scraper:
@@ -38,12 +34,11 @@ class Scraper:
     ):
         self.location = scraper_input.location
         self.listing_type = scraper_input.listing_type
-        self.property_type = scraper_input.property_type
 
         if not self.session:
             Scraper.session = requests.Session()
             retries = Retry(
-                total=3, backoff_factor=4, status_forcelist=[429, 403], allowed_methods=frozenset(["GET", "POST"])
+                total=3, backoff_factor=3, status_forcelist=[429, 403], allowed_methods=frozenset(["GET", "POST"])
             )
 
             adapter = HTTPAdapter(max_retries=retries)
@@ -51,21 +46,21 @@ class Scraper:
             Scraper.session.mount("https://", adapter)
             Scraper.session.headers.update(
                 {
-                    "accept": "application/json, text/javascript",
+                    'accept': 'application/json, text/javascript',
-                    "accept-language": "en-US,en;q=0.9",
+                    'accept-language': 'en-US,en;q=0.9',
-                    "cache-control": "no-cache",
+                    'cache-control': 'no-cache',
-                    "content-type": "application/json",
+                    'content-type': 'application/json',
-                    "origin": "https://www.realtor.com",
+                    'origin': 'https://www.realtor.com',
-                    "pragma": "no-cache",
+                    'pragma': 'no-cache',
-                    "priority": "u=1, i",
+                    'priority': 'u=1, i',
-                    "rdc-ab-tests": "commute_travel_time_variation:v1",
+                    'rdc-ab-tests': 'commute_travel_time_variation:v1',
-                    "sec-ch-ua": '"Not)A;Brand";v="99", "Google Chrome";v="127", "Chromium";v="127"',
+                    'sec-ch-ua': '"Not)A;Brand";v="99", "Google Chrome";v="127", "Chromium";v="127"',
-                    "sec-ch-ua-mobile": "?0",
+                    'sec-ch-ua-mobile': '?0',
-                    "sec-ch-ua-platform": '"Windows"',
+                    'sec-ch-ua-platform': '"Windows"',
-                    "sec-fetch-dest": "empty",
+                    'sec-fetch-dest': 'empty',
-                    "sec-fetch-mode": "cors",
+                    'sec-fetch-mode': 'cors',
-                    "sec-fetch-site": "same-origin",
+                    'sec-fetch-site': 'same-origin',
-                    "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36",
+                    'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36',
                 }
             )
 
@@ -84,9 +79,8 @@ class Scraper:
         self.extra_property_data = scraper_input.extra_property_data
         self.exclude_pending = scraper_input.exclude_pending
         self.limit = scraper_input.limit
-        self.return_type = scraper_input.return_type
 
-    def search(self) -> list[Union[Property | dict]]: ...
+    def search(self) -> list[Property]: ...
 
     @staticmethod
     def _parse_home(home) -> Property: ...
@@ -100,29 +94,27 @@ class Scraper:
         response = requests.post(
             "https://graph.realtor.com/auth/token",
             headers={
-                "Host": "graph.realtor.com",
+                'Host': 'graph.realtor.com',
-                "Accept": "*/*",
+                'Accept': '*/*',
-                "Content-Type": "Application/json",
+                'Content-Type': 'Application/json',
-                "X-Client-ID": "rdc_mobile_native,iphone",
+                'X-Client-ID': 'rdc_mobile_native,iphone',
-                "X-Visitor-ID": device_id,
+                'X-Visitor-ID': device_id,
-                "X-Client-Version": "24.21.23.679885",
+                'X-Client-Version': '24.21.23.679885',
-                "Accept-Language": "en-US,en;q=0.9",
+                'Accept-Language': 'en-US,en;q=0.9',
-                "User-Agent": "Realtor.com/24.21.23.679885 CFNetwork/1494.0.7 Darwin/23.4.0",
+                'User-Agent': 'Realtor.com/24.21.23.679885 CFNetwork/1494.0.7 Darwin/23.4.0',
             },
-            data=json.dumps(
+            data=json.dumps({
-                {
+                "grant_type": "device_mobile",
-                    "grant_type": "device_mobile",
+                "device_id": device_id,
-                    "device_id": device_id,
+                "client_app_id": "rdc_mobile_native,24.21.23.679885,iphone"
-                    "client_app_id": "rdc_mobile_native,24.21.23.679885,iphone",
+            }))
-                }
-            ),
-        )
 
         data = response.json()
 
         if not (access_token := data.get("access_token")):
             raise AuthenticationError(
-                "Failed to get access token, use a proxy/vpn or wait a moment and try again.", response=response
+                "Failed to get access token, use a proxy/vpn or wait a moment and try again.",
+                response=response
             )
 
         return access_token

homeharvest/core/scrapers/models.py

@@ -4,12 +4,6 @@ from enum import Enum
 from typing import Optional
 
 
-class ReturnType(Enum):
-    pydantic = "pydantic"
-    pandas = "pandas"
-    raw = "raw"
-
-
 class SiteName(Enum):
     ZILLOW = "zillow"
     REDFIN = "redfin"
@@ -23,20 +17,6 @@ class SiteName(Enum):
         raise ValueError(f"{value} not found in {cls}")
 
 
-class SearchPropertyType(Enum):
-    SINGLE_FAMILY = "single_family"
-    APARTMENT = "apartment"
-    CONDOS = "condos"
-    CONDO_TOWNHOME_ROWHOME_COOP = "condo_townhome_rowhome_coop"
-    CONDO_TOWNHOME = "condo_townhome"
-    TOWNHOMES = "townhomes"
-    DUPLEX_TRIPLEX = "duplex_triplex"
-    FARM = "farm"
-    LAND = "land"
-    MULTI_FAMILY = "multi_family"
-    MOBILE = "mobile"
-
-
 class ListingType(Enum):
     FOR_SALE = "FOR_SALE"
     FOR_RENT = "FOR_RENT"
@@ -155,9 +135,6 @@ class Property:
     property_url: str
 
     property_id: str
-    #: allows_cats: bool
-    #: allows_dogs: bool
-
     listing_id: str | None = None
 
     mls: str | None = None
@@ -177,8 +154,6 @@ class Property:
     hoa_fee: int | None = None
     days_on_mls: int | None = None
     description: Description | None = None
-    tags: list[str] | None = None
-    details: list[dict] | None = None
 
     latitude: float | None = None
     longitude: float | None = None
@@ -188,7 +163,5 @@ class Property:
     nearby_schools: list[str] = None
     assessed_value: int | None = None
     estimated_value: int | None = None
-    tax: int | None = None
-    tax_history: list[dict] | None = None
 
     advertisers: Advertisers | None = None

homeharvest/core/scrapers/realtor/__init__.py

@@ -6,35 +6,13 @@ This module implements the scraper for realtor.com
 """
 
 from __future__ import annotations
-
-import json
 from concurrent.futures import ThreadPoolExecutor, as_completed
 from datetime import datetime
-from json import JSONDecodeError
 from typing import Dict, Union, Optional
 
-from tenacity import (
-    retry,
-    retry_if_exception_type,
-    wait_exponential,
-    stop_after_attempt,
-)
-
 from .. import Scraper
-from ..models import (
+from ..models import Property, Address, ListingType, Description, PropertyType, Agent, Broker, Builder, Advertisers, Office
-    Property,
+from .queries import GENERAL_RESULTS_QUERY, SEARCH_HOMES_DATA, HOMES_DATA
-    Address,
-    ListingType,
-    Description,
-    PropertyType,
-    Agent,
-    Broker,
-    Builder,
-    Advertisers,
-    Office,
-    ReturnType
-)
-from .queries import GENERAL_RESULTS_QUERY, SEARCH_HOMES_DATA, HOMES_DATA, HOME_FRAGMENT
 
 
 class RealtorScraper(Scraper):
@@ -103,12 +81,9 @@ class RealtorScraper(Scraper):
             return property_info["listings"][0]["listing_id"]
 
     def handle_home(self, property_id: str) -> list[Property]:
-        query = (
+        query = """query Home($property_id: ID!) {
-            """query Home($property_id: ID!) {
                     home(property_id: $property_id) %s
-                }"""
+                }""" % HOMES_DATA
-            % HOMES_DATA
-        )
 
         variables = {"property_id": property_id}
         payload = {
@@ -121,7 +96,9 @@ class RealtorScraper(Scraper):
 
         property_info = response_json["data"]["home"]
 
-        return [self.process_property(property_info)]
+        return [
+            self.process_property(property_info, "home")
+        ]
 
     @staticmethod
     def process_advertisers(advertisers: list[dict] | None) -> Advertisers | None:
@@ -145,7 +122,7 @@ class RealtorScraper(Scraper):
                     phones=advertiser.get("phones"),
                 )
 
-                if advertiser.get("broker") and advertiser["broker"].get("name"):  #: has a broker
+                if advertiser.get('broker') and advertiser["broker"].get('name'):  #: has a broker
                     processed_advertisers.broker = Broker(
                         uuid=_parse_fulfillment_id(advertiser["broker"].get("fulfillment_id")),
                         name=advertiser["broker"].get("name"),
@@ -169,27 +146,28 @@ class RealtorScraper(Scraper):
 
         return processed_advertisers
 
-    def process_property(self, result: dict) -> Property | None:
+    def process_property(self, result: dict, query_name: str) -> Property | None:
         mls = result["source"].get("id") if "source" in result and isinstance(result["source"], dict) else None
 
         if not mls and self.mls_only:
             return
 
         able_to_get_lat_long = (
-            result
+                result
-            and result.get("location")
+                and result.get("location")
-            and result["location"].get("address")
+                and result["location"].get("address")
-            and result["location"]["address"].get("coordinate")
+                and result["location"]["address"].get("coordinate")
         )
 
-        is_pending = result["flags"].get("is_pending")
+        is_pending = result["flags"].get("is_pending") or result["flags"].get("is_contingent")
-        is_contingent = result["flags"].get("is_contingent")
 
-        if (is_pending or is_contingent) and (self.exclude_pending and self.listing_type != ListingType.PENDING):
+        if is_pending and (self.exclude_pending and self.listing_type != ListingType.PENDING):
             return
 
         property_id = result["property_id"]
-        prop_details = self.process_extra_property_details(result) if self.extra_property_data else {}
+        prop_details = self.get_prop_details(property_id) if self.extra_property_data and query_name != "home" else {}
+        if not prop_details:
+            prop_details = self.process_extra_property_details(result)
 
         property_estimates_root = result.get("current_estimates") or result.get("estimates", {}).get("currentValues")
         estimated_value = self.get_key(property_estimates_root, [0, "estimate"])
@@ -206,33 +184,31 @@ class RealtorScraper(Scraper):
             property_url=result["href"],
             property_id=property_id,
             listing_id=result.get("listing_id"),
-            status=("PENDING" if is_pending else "CONTINGENT" if is_contingent else result["status"].upper()),
+            status="PENDING" if is_pending else result["status"].upper(),
             list_price=result["list_price"],
             list_price_min=result["list_price_min"],
             list_price_max=result["list_price_max"],
-            list_date=(result["list_date"].split("T")[0] if result.get("list_date") else None),
+            list_date=result["list_date"].split("T")[0] if result.get("list_date") else None,
             prc_sqft=result.get("price_per_sqft"),
             last_sold_date=result.get("last_sold_date"),
             new_construction=result["flags"].get("is_new_construction") is True,
-            hoa_fee=(result["hoa"]["fee"] if result.get("hoa") and isinstance(result["hoa"], dict) else None),
+            hoa_fee=result["hoa"]["fee"] if result.get("hoa") and isinstance(result["hoa"], dict) else None,
-            latitude=(result["location"]["address"]["coordinate"].get("lat") if able_to_get_lat_long else None),
+            latitude=result["location"]["address"]["coordinate"].get("lat") if able_to_get_lat_long else None,
-            longitude=(result["location"]["address"]["coordinate"].get("lon") if able_to_get_lat_long else None),
+            longitude=result["location"]["address"]["coordinate"].get("lon") if able_to_get_lat_long else None,
             address=self._parse_address(result, search_type="general_search"),
             description=self._parse_description(result),
             neighborhoods=self._parse_neighborhoods(result),
-            county=(result["location"]["county"].get("name") if result["location"]["county"] else None),
+            county=result["location"]["county"].get("name") if result["location"]["county"] else None,
-            fips_code=(result["location"]["county"].get("fips_code") if result["location"]["county"] else None),
+            fips_code=result["location"]["county"].get("fips_code") if result["location"]["county"] else None,
             days_on_mls=self.calculate_days_on_mls(result),
             nearby_schools=prop_details.get("schools"),
             assessed_value=prop_details.get("assessed_value"),
             estimated_value=estimated_value if estimated_value else None,
             advertisers=advertisers,
-            tax=prop_details.get("tax"),
-            tax_history=prop_details.get("tax_history"),
         )
         return realty_property
 
-    def general_search(self, variables: dict, search_type: str) -> Dict[str, Union[int, Union[list[Property], list[dict]]]]:
+    def general_search(self, variables: dict, search_type: str) -> Dict[str, Union[int, list[Property]]]:
         """
         Handles a location area & returns a list of properties
         """
@@ -249,15 +225,10 @@ class RealtorScraper(Scraper):
             elif self.last_x_days:
                 date_param = f'list_date: {{ min: "$today-{self.last_x_days}D" }}'
 
-        property_type_param = ""
-        if self.property_type:
-            property_types = [pt.value for pt in self.property_type]
-            property_type_param = f"type: {json.dumps(property_types)}"
-
         sort_param = (
             "sort: [{ field: sold_date, direction: desc }]"
             if self.listing_type == ListingType.SOLD
-            else ""  #: "sort: [{ field: list_date, direction: desc }]"  #: prioritize normal fractal sort from realtor
+            else "sort: [{ field: list_date, direction: desc }]"
         )
 
         pending_or_contingent_param = (
@@ -288,7 +259,6 @@ class RealtorScraper(Scraper):
                                 status: %s
                                 %s
                                 %s
-                                %s
                             }
                             %s
                             limit: 200
@@ -298,27 +268,29 @@ class RealtorScraper(Scraper):
                 is_foreclosure,
                 listing_type.value.lower(),
                 date_param,
-                property_type_param,
                 pending_or_contingent_param,
                 sort_param,
                 GENERAL_RESULTS_QUERY,
             )
         elif search_type == "area":  #: general search, came from a general location
             query = """query Home_search(
-                                $location: String!,
+                                $city: String,
+                                $county: [String],
+                                $state_code: String,
+                                $postal_code: String
                                 $offset: Int,
                             ) {
                                 home_search(
                                     query: {
                                         %s
-                                        search_location: {location: $location}
+                                        city: $city
+                                        county: $county
+                                        postal_code: $postal_code
+                                        state_code: $state_code
                                         status: %s
-                                        unique: true
-                                        %s
                                         %s
                                         %s
                                     }
-                                    bucket: { sort: "fractal_v1.1.3_fr" }
                                     %s
                                     limit: 200
                                     offset: $offset
@@ -327,14 +299,13 @@ class RealtorScraper(Scraper):
                 is_foreclosure,
                 listing_type.value.lower(),
                 date_param,
-                property_type_param,
                 pending_or_contingent_param,
                 sort_param,
                 GENERAL_RESULTS_QUERY,
             )
         else:  #: general search, came from an address
             query = (
-                """query Property_search(
+                    """query Property_search(
                         $property_id: [ID]!
                         $offset: Int!,
                     ) {
@@ -346,7 +317,7 @@ class RealtorScraper(Scraper):
                             offset: $offset
                         ) %s 
                     }"""
-                % GENERAL_RESULTS_QUERY
+                    % GENERAL_RESULTS_QUERY
             )
 
         payload = {
@@ -358,15 +329,15 @@ class RealtorScraper(Scraper):
         response_json = response.json()
         search_key = "home_search" if "home_search" in query else "property_search"
 
-        properties: list[Union[Property, dict]] = []
+        properties: list[Property] = []
 
         if (
-            response_json is None
+                response_json is None
-            or "data" not in response_json
+                or "data" not in response_json
-            or response_json["data"] is None
+                or response_json["data"] is None
-            or search_key not in response_json["data"]
+                or search_key not in response_json["data"]
-            or response_json["data"][search_key] is None
+                or response_json["data"][search_key] is None
-            or "results" not in response_json["data"][search_key]
+                or "results" not in response_json["data"][search_key]
         ):
             return {"total": 0, "properties": []}
 
@@ -376,25 +347,17 @@ class RealtorScraper(Scraper):
 
         #: limit the number of properties to be processed
         #: example, if your offset is 200, and your limit is 250, return 50
-        properties_list: list[dict] = properties_list[: self.limit - offset]
+        properties_list = properties_list[:self.limit - offset]
 
-        if self.extra_property_data:
+        with ThreadPoolExecutor(max_workers=self.NUM_PROPERTY_WORKERS) as executor:
-            property_ids = [data["property_id"] for data in properties_list]
+            futures = [
-            extra_property_details = self.get_bulk_prop_details(property_ids) or {}
+                executor.submit(self.process_property, result, search_key) for result in properties_list
+            ]
 
-            for result in properties_list:
+            for future in as_completed(futures):
-                result.update(extra_property_details.get(result["property_id"], {}))
+                result = future.result()
-
+                if result:
-        if self.return_type != ReturnType.raw:
+                    properties.append(result)
-            with ThreadPoolExecutor(max_workers=self.NUM_PROPERTY_WORKERS) as executor:
-                futures = [executor.submit(self.process_property, result) for result in properties_list]
-
-                for future in as_completed(futures):
-                    result = future.result()
-                    if result:
-                        properties.append(result)
-        else:
-            properties = properties_list
 
         return {
             "total": total_properties,
@@ -439,7 +402,10 @@ class RealtorScraper(Scraper):
 
         else:  #: general search, location
             search_variables |= {
-                "location": self.location,
+                "city": location_info.get("city"),
+                "county": location_info.get("county"),
+                "state_code": location_info.get("state_code"),
+                "postal_code": location_info.get("postal_code"),
             }
 
         if self.foreclosure:
@@ -456,11 +422,7 @@ class RealtorScraper(Scraper):
                     variables=search_variables | {"offset": i},
                     search_type=search_type,
                 )
-                for i in range(
+                for i in range(self.DEFAULT_PAGE_SIZE, min(total, self.limit), self.DEFAULT_PAGE_SIZE)
-                    self.DEFAULT_PAGE_SIZE,
-                    min(total, self.limit),
-                    self.DEFAULT_PAGE_SIZE,
-                )
             ]
 
             for future in as_completed(futures):
@@ -482,75 +444,35 @@ class RealtorScraper(Scraper):
     def process_extra_property_details(self, result: dict) -> dict:
         schools = self.get_key(result, ["nearbySchools", "schools"])
         assessed_value = self.get_key(result, ["taxHistory", 0, "assessment", "total"])
-        tax_history = self.get_key(result, ["taxHistory"])
 
         schools = [school["district"]["name"] for school in schools if school["district"].get("name")]
-
-        # Process tax history
-        latest_tax = None
-        processed_tax_history = None
-        if tax_history and isinstance(tax_history, list):
-            tax_history = sorted(tax_history, key=lambda x: x.get("year", 0), reverse=True)
-
-            if tax_history and "tax" in tax_history[0]:
-                latest_tax = tax_history[0]["tax"]
-
-            processed_tax_history = []
-            for entry in tax_history:
-                if "year" in entry and "tax" in entry:
-                    processed_entry = {
-                        "year": entry["year"],
-                        "tax": entry["tax"],
-                    }
-                    if "assessment" in entry and isinstance(entry["assessment"], dict):
-                        processed_entry["assessment"] = {
-                            "building": entry["assessment"].get("building"),
-                            "land": entry["assessment"].get("land"),
-                            "total": entry["assessment"].get("total"),
-                        }
-                    processed_tax_history.append(processed_entry)
-
         return {
             "schools": schools if schools else None,
             "assessed_value": assessed_value if assessed_value else None,
-            "tax": latest_tax,
-            "tax_history": processed_tax_history,
         }
 
-    @retry(
+    def get_prop_details(self, property_id: str) -> dict:
-        retry=retry_if_exception_type(JSONDecodeError),
+        if not self.extra_property_data:
-        wait=wait_exponential(min=4, max=10),
-        stop=stop_after_attempt(3),
-    )
-    def get_bulk_prop_details(self, property_ids: list[str]) -> dict:
-        """
-        Fetch extra property details for multiple properties in a single GraphQL query.
-        Returns a map of property_id to its details.
-        """
-        if not self.extra_property_data or not property_ids:
             return {}
 
-        property_ids = list(set(property_ids))
+        query = """query GetHome($property_id: ID!) {
+                    home(property_id: $property_id) {
+                        __typename
 
-        # Construct the bulk query
+                        nearbySchools: nearby_schools(radius: 5.0, limit_per_level: 3) {
-        fragments = "\n".join(
+                            __typename schools { district { __typename id name } }
-            f'home_{property_id}: home(property_id: {property_id}) {{ ...HomeData }}'
+                        }
-            for property_id in property_ids
+                        taxHistory: tax_history { __typename tax year assessment { __typename building land total } }
-        )
+                    }
-        query = f"""{HOME_FRAGMENT}
+                }"""
 
-        query GetHomes {{
+        variables = {"property_id": property_id}
-            {fragments}
+        response = self.session.post(self.SEARCH_GQL_URL, json={"query": query, "variables": variables})
-        }}"""
 
-        response = self.session.post(self.SEARCH_GQL_URL, json={"query": query})
         data = response.json()
+        property_details = data["data"]["home"]
 
-        if "data" not in data:
+        return self.process_extra_property_details(property_details)
-            return {}
-
-        properties = data["data"]
-        return {data.replace('home_', ''): properties[data] for data in properties if properties[data]}
 
     @staticmethod
     def _parse_neighborhoods(result: dict) -> Optional[str]:
@@ -612,22 +534,20 @@ class RealtorScraper(Scraper):
             style = style.upper()
 
         primary_photo = ""
-        if (primary_photo_info := result.get("primary_photo")) and (
+        if (primary_photo_info := result.get('primary_photo')) and (primary_photo_href := primary_photo_info.get("href")):
-            primary_photo_href := primary_photo_info.get("href")
-        ):
             primary_photo = primary_photo_href.replace("s.jpg", "od-w480_h360_x2.webp?w=1080&q=75")
 
         return Description(
             primary_photo=primary_photo,
             alt_photos=RealtorScraper.process_alt_photos(result.get("photos", [])),
-            style=(PropertyType.__getitem__(style) if style and style in PropertyType.__members__ else None),
+            style=PropertyType.__getitem__(style) if style and style in PropertyType.__members__ else None,
             beds=description_data.get("beds"),
             baths_full=description_data.get("baths_full"),
             baths_half=description_data.get("baths_half"),
             sqft=description_data.get("sqft"),
             lot_sqft=description_data.get("lot_sqft"),
             sold_price=(
-                result.get("last_sold_price") or description_data.get("sold_price")
+                result.get('last_sold_price') or description_data.get("sold_price")
                 if result.get("last_sold_date") or result["list_price"] != description_data.get("sold_price")
                 else None
             ),  #: has a sold date or list and sold price are different
@@ -661,8 +581,4 @@ class RealtorScraper(Scraper):
         if not photos_info:
             return None
 
-        return [
+        return [photo_info["href"].replace("s.jpg", "od-w480_h360_x2.webp?w=1080&q=75") for photo_info in photos_info if photo_info.get("href")]
-            photo_info["href"].replace("s.jpg", "od-w480_h360_x2.webp?w=1080&q=75")
-            for photo_info in photos_info
-            if photo_info.get("href")
-        ]

homeharvest/core/scrapers/realtor/queries.py

@@ -11,34 +11,6 @@ _SEARCH_HOMES_DATA_BASE = """{
     list_price_max
     list_price_min
     price_per_sqft
-    tags
-    details {
-        category
-        text
-        parent_category
-    }
-    pet_policy {
-        cats
-        dogs
-        dogs_small
-        dogs_large
-        __typename
-    }
-    units {
-        availability {
-          date
-          __typename
-        }
-        description {
-          baths_consolidated
-          baths
-          beds
-          sqft
-          __typename
-        }
-        list_price
-        __typename
-    }
     flags {
         is_contingent
         is_pending
@@ -92,14 +64,11 @@ _SEARCH_HOMES_DATA_BASE = """{
     tax_record {
         public_record_id
     }
-    primary_photo(https: true) {
+    primary_photo {
         href
     }
-    photos(https: true) {
+    photos {
         href
-        tags {
-            label
-        }
     }
     advertisers {
         email
@@ -147,63 +116,15 @@ _SEARCH_HOMES_DATA_BASE = """{
         }
         rental_management {
             name
-            href
             fulfillment_id
         }
     }
     """
 
-
-HOME_FRAGMENT = """
-fragment HomeData on Home {
-    property_id
-    nearbySchools: nearby_schools(radius: 5.0, limit_per_level: 3) {
-        __typename schools { district { __typename id name } }
-    }
-    taxHistory: tax_history { __typename tax year assessment { __typename building land total } }
-    monthly_fees {
-        description
-        display_amount
-    }
-    one_time_fees {
-        description
-        display_amount
-    }
-    parking {
-        unassigned_space_rent
-        assigned_spaces_available
-        description
-        assigned_space_rent
-    }
-    terms {
-        text
-        category
-    }
-}
-"""
-
 HOMES_DATA = """%s
                 nearbySchools: nearby_schools(radius: 5.0, limit_per_level: 3) {
                             __typename schools { district { __typename id name } }
                         }
-                monthly_fees {
-                    description
-                    display_amount
-                }
-                one_time_fees {
-                    description
-                    display_amount
-                }
-                parking {
-                    unassigned_space_rent
-                    assigned_spaces_available
-                    description
-                    assigned_space_rent
-                }
-                terms {
-                    text
-                    category
-                }
                 taxHistory: tax_history { __typename tax year assessment { __typename building land total } }
                 estimates {
                     __typename
@@ -220,19 +141,19 @@ HOMES_DATA = """%s
 }""" % _SEARCH_HOMES_DATA_BASE
 
 SEARCH_HOMES_DATA = """%s
-current_estimates {
+                    current_estimates {
-    __typename
+                            __typename
-    source {
+                            source {
-        __typename
+                                __typename
-        type
+                                type
-        name
+                                name
-    }
+                            }
-    estimate
+                            estimate
-    estimateHigh: estimate_high
+                            estimateHigh: estimate_high
-    estimateLow: estimate_low
+                            estimateLow: estimate_low
-    date
+                            date
-    isBestHomeValue: isbest_homevalue
+                            isBestHomeValue: isbest_homevalue
-}
+                        }
 }""" % _SEARCH_HOMES_DATA_BASE
 
 GENERAL_RESULTS_QUERY = """{

homeharvest/utils.py

@@ -33,8 +33,6 @@ ordered_properties = [
     "last_sold_date",
     "assessed_value",
     "estimated_value",
-    "tax",
-    "tax_history",
     "new_construction",
     "lot_sqft",
     "price_per_sqft",
@@ -117,11 +115,8 @@ def process_result(result: Property) -> pd.DataFrame:
     if description:
         prop_data["primary_photo"] = description.primary_photo
         prop_data["alt_photos"] = ", ".join(description.alt_photos) if description.alt_photos else None
-        prop_data["style"] = (
+        prop_data["style"] = description.style if isinstance(description.style,
-            description.style
+                                                             str) else description.style.value if description.style else None
-            if isinstance(description.style, str)
-            else description.style.value if description.style else None
-        )
         prop_data["beds"] = description.beds
         prop_data["full_baths"] = description.baths_full
         prop_data["half_baths"] = description.baths_half

poetry.lock

@@ -1,4 +1,4 @@
-# This file is automatically @generated by Poetry 1.8.4 and should not be changed by hand.
+# This file is automatically @generated by Poetry 1.8.2 and should not be changed by hand.
 
 [[package]]
 name = "annotated-types"
@@ -667,21 +667,6 @@ files = [
     {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"},
 ]
 
-[[package]]
-name = "tenacity"
-version = "9.0.0"
-description = "Retry code until it succeeds"
-optional = false
-python-versions = ">=3.8"
-files = [
-    {file = "tenacity-9.0.0-py3-none-any.whl", hash = "sha256:93de0c98785b27fcf659856aa9f54bfbd399e29969b0621bc7f762bd441b4539"},
-    {file = "tenacity-9.0.0.tar.gz", hash = "sha256:807f37ca97d62aa361264d497b0e31e92b8027044942bfa756160d908320d73b"},
-]
-
-[package.extras]
-doc = ["reno", "sphinx"]
-test = ["pytest", "tornado (>=4.5)", "typeguard"]
-
 [[package]]
 name = "tomli"
 version = "2.0.1"
@@ -755,4 +740,4 @@ test = ["covdefaults (>=2.3)", "coverage (>=7.2.7)", "coverage-enable-subprocess
 [metadata]
 lock-version = "2.0"
 python-versions = ">=3.9,<3.13"
-content-hash = "cefc11b1bf5ad99d628f6d08f6f03003522cc1b6e48b519230d99d716a5c165c"
+content-hash = "21ef9cfb35c446a375a2b74c37691d7031afb1e4f66a8b63cb7c1669470689d2"

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "homeharvest"
-version = "0.4.7"
+version = "0.4.3"
 description = "Real estate scraping library"
 authors = ["Zachary Hampton <zachary@bunsly.com>", "Cullen Watson <cullen@bunsly.com>"]
 homepage = "https://github.com/Bunsly/HomeHarvest"
@@ -14,7 +14,6 @@ python = ">=3.9,<3.13"
 requests = "^2.31.0"
 pandas = "^2.1.1"
 pydantic = "^2.7.4"
-tenacity = "^9.0.0"
 
 
 [tool.poetry.group.dev.dependencies]

tests/test_realtor.py

@@ -1,5 +1,4 @@
-from homeharvest import scrape_property, Property
+from homeharvest import scrape_property
-import pandas as pd
 
 
 def test_realtor_pending_or_contingent():
@@ -106,12 +105,8 @@ def test_realtor():
             location="2530 Al Lipscomb Way",
             listing_type="for_sale",
         ),
-        scrape_property(
+        scrape_property(location="Phoenix, AZ", listing_type="for_rent", limit=1000),  #: does not support "city, state, USA" format
-            location="Phoenix, AZ", listing_type="for_rent", limit=1000
+        scrape_property(location="Dallas, TX", listing_type="sold", limit=1000),  #: does not support "city, state, USA" format
-        ),  #: does not support "city, state, USA" format
-        scrape_property(
-            location="Dallas, TX", listing_type="sold", limit=1000
-        ),  #: does not support "city, state, USA" format
         scrape_property(location="85281"),
     ]
 
@@ -119,13 +114,11 @@ def test_realtor():
 
 
 def test_realtor_city():
-    results = scrape_property(location="Atlanta, GA", listing_type="for_sale", limit=1000)
+    results = scrape_property(
-
+        location="Atlanta, GA",
-    assert results is not None and len(results) > 0
+        listing_type="for_sale",
-
+        limit=1000
-
+    )
-def test_realtor_land():
-    results = scrape_property(location="Atlanta, GA", listing_type="for_sale", property_type=["land"], limit=1000)
 
     assert results is not None and len(results) > 0
 
@@ -248,10 +241,9 @@ def test_apartment_list_price():
     results = results[results["style"] == "APARTMENT"]
 
     #: get percentage of results with atleast 1 of any column not none, list_price, list_price_min, list_price_max
-    assert (
+    assert len(results[results[["list_price", "list_price_min", "list_price_max"]].notnull().any(axis=1)]) / len(
-        len(results[results[["list_price", "list_price_min", "list_price_max"]].notnull().any(axis=1)]) / len(results)
+        results
-        > 0.5
+    ) > 0.5
-    )
 
 
 def test_builder_exists():
@@ -288,15 +280,3 @@ def test_phone_number_matching():
 
     #: assert phone numbers are the same
     assert row["agent_phones"].values[0] == matching_row["agent_phones"].values[0]
-
-
-def test_return_type():
-    results = {
-        "pandas": scrape_property(location="Surprise, AZ", listing_type="for_rent", limit=100),
-        "pydantic": scrape_property(location="Surprise, AZ", listing_type="for_rent", limit=100, return_type="pydantic"),
-        "raw": scrape_property(location="Surprise, AZ", listing_type="for_rent", limit=100, return_type="raw"),
-    }
-
-    assert isinstance(results["pandas"], pd.DataFrame)
-    assert isinstance(results["pydantic"][0], Property)
-    assert isinstance(results["raw"][0], dict)