- bug fixes

- optimized get extra fields with query clustering

.github/FUNDING.yml

@@ -0,0 +1 @@
+github: Bunsly

README.md

@@ -2,10 +2,6 @@
 
 **HomeHarvest** is a real estate scraping library that extracts and formats data in the style of MLS listings.
 
-**Not technical?** Try out the web scraping tool on our site at [tryhomeharvest.com](https://tryhomeharvest.com).
-
-*Looking to build a data-focused software product?* **[Book a call](https://bunsly.com)** *to work with us.*
-
 ## HomeHarvest Features
 
 - **Source**: Fetches properties directly from **Realtor.com**.
@@ -40,6 +36,7 @@ properties = scrape_property(
   listing_type="sold",  # or (for_sale, for_rent, pending)
   past_days=30,  # sold in last 30 days - listed in last 30 days if (for_sale, for_rent)
 
+  # property_type=['single_family','multi_family'],
   # date_from="2023-05-01", # alternative to past_days
   # date_to="2023-05-28",
   # foreclosure=True
@@ -68,13 +65,30 @@ print(properties.head())
 ```
 Required
 ├── location (str): The address in various formats - this could be just a zip code, a full address, or city/state, etc.
-└── listing_type (option): Choose the type of listing.
+├── listing_type (option): Choose the type of listing.
     - 'for_rent'
     - 'for_sale'
     - 'sold'
-    - 'pending'
+    - 'pending' (for pending/contingent sales)
 
 Optional
+├── property_type (list): Choose the type of properties.
+    - 'single_family'
+    - 'multi_family'
+    - 'condos'
+    - 'condo_townhome_rowhome_coop'
+    - 'condo_townhome'
+    - 'townhomes'
+    - 'duplex_triplex'
+    - 'farm'
+    - 'land'
+    - 'mobile'
+│
+├── return_type (option): Choose the return type.
+│    - 'pandas' (default)
+│    - 'pydantic'
+│    - 'raw' (json)
+│
 ├── radius (decimal): Radius in miles to find comparable properties based on individual addresses.
 │    Example: 5.5 (fetches properties within a 5.5-mile radius if location is set to a specific address; otherwise, ignored)
 │
@@ -94,7 +108,7 @@ Optional
 │
 ├── extra_property_data (True/False): Increases requests by O(n). If set, this fetches additional property data for general searches (e.g. schools, tax appraisals etc.)
 │
-├── exclude_pending (True/False): If set, excludes pending properties from the results unless listing_type is 'pending'
+├── exclude_pending (True/False): If set, excludes 'pending' properties from the 'for_sale' results unless listing_type is 'pending'
 │
 └── limit (integer): Limit the number of properties to fetch. Max & default is 10000.
 ```
@@ -104,6 +118,8 @@ Optional
 Property
 ├── Basic Information:
 │ ├── property_url
+│ ├── property_id
+│ ├── listing_id
 │ ├── mls
 │ ├── mls_id
 │ └── status
@@ -139,6 +155,14 @@ Property
 │ ├── new_construction
 │ └── hoa_fee
 
+├── Tax Information:
+│  ├── year
+│  ├── tax
+│  ├── assessment
+│  │   ├── building
+│  │   ├── land
+│  │   └── total
+
 ├── Location Details:
 │ ├── latitude
 │ ├── longitude

examples/HomeHarvest_Demo.ipynb

@@ -1,141 +0,0 @@
-{
- "cells": [
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "cb48903e-5021-49fe-9688-45cd0bc05d0f",
-   "metadata": {
-    "is_executing": true
-   },
-   "outputs": [],
-   "source": [
-    "from homeharvest import scrape_property\n",
-    "import pandas as pd"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "156488ce-0d5f-43c5-87f4-c33e9c427860",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "pd.set_option('display.max_columns', None)  # Show all columns\n",
-    "pd.set_option('display.max_rows', None)     # Show all rows\n",
-    "pd.set_option('display.width', None)        # Auto-adjust display width to fit console\n",
-    "pd.set_option('display.max_colwidth', 50)   # Limit max column width to 50 characters"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "1c8b9744-8606-4e9b-8add-b90371a249a7",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# check for sale properties\n",
-    "scrape_property(\n",
-    "    location=\"dallas\",\n",
-    "    listing_type=\"for_sale\"\n",
-    ")"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "aaf86093",
-   "metadata": {
-    "collapsed": false,
-    "jupyter": {
-     "outputs_hidden": false
-    }
-   },
-   "outputs": [],
-   "source": [
-    "# search a specific address\n",
-    "scrape_property(\n",
-    "    location=\"2530 Al Lipscomb Way\",\n",
-    "    listing_type=\"for_sale\"\n",
-    ")"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "ab7b4c21-da1d-4713-9df4-d7425d8ce21e",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# check rentals\n",
-    "scrape_property(\n",
-    "    location=\"chicago, illinois\",\n",
-    "    listing_type=\"for_rent\"\n",
-    ")"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "af280cd3",
-   "metadata": {
-    "collapsed": false,
-    "jupyter": {
-     "outputs_hidden": false
-    }
-   },
-   "outputs": [],
-   "source": [
-    "# check sold properties\n",
-    "properties = scrape_property(\n",
-    "    location=\"90210\",\n",
-    "    listing_type=\"sold\",\n",
-    "    past_days=10\n",
-    ")\n",
-    "display(properties)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "628c1ce2",
-   "metadata": {
-    "collapsed": false,
-    "is_executing": true,
-    "jupyter": {
-     "outputs_hidden": false
-    }
-   },
-   "outputs": [],
-   "source": [
-    "# display clickable URLs\n",
-    "from IPython.display import display, HTML\n",
-    "properties['property_url'] = '<a href=\"' + properties['property_url'] + '\" target=\"_blank\">' + properties['property_url'] + '</a>'\n",
-    "\n",
-    "html = properties.to_html(escape=False)\n",
-    "truncate_width = f'<style>.dataframe td {{ max-width: 200px; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; }}</style>{html}'\n",
-    "display(HTML(truncate_width))"
-   ]
-  }
- ],
- "metadata": {
-  "kernelspec": {
-   "display_name": "Python 3 (ipykernel)",
-   "language": "python",
-   "name": "python3"
-  },
-  "language_info": {
-   "codemirror_mode": {
-    "name": "ipython",
-    "version": 3
-   },
-   "file_extension": ".py",
-   "mimetype": "text/x-python",
-   "name": "python",
-   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.10.11"
-  }
- },
- "nbformat": 4,
- "nbformat_minor": 5
-}

examples/HomeHarvest_Demo.py

@@ -1,20 +0,0 @@
-from homeharvest import scrape_property
-from datetime import datetime
-
-# Generate filename based on current timestamp
-current_timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
-filename = f"HomeHarvest_{current_timestamp}.csv"
-
-properties = scrape_property(
-    location="San Diego, CA",
-    listing_type="sold",  # or (for_sale, for_rent)
-    past_days=30,  # sold in last 30 days - listed in last x days if (for_sale, for_rent)
-    # pending_or_contingent=True # use on for_sale listings to find pending / contingent listings
-    # mls_only=True,  # only fetch MLS listings
-    # proxy="http://user:pass@host:port"  # use a proxy to change your IP address
-)
-print(f"Number of properties: {len(properties)}")
-
-# Export to csv
-properties.to_csv(filename, index=False)
-print(properties.head())

examples/price_of_land.py

@@ -0,0 +1,104 @@
+"""
+This script scrapes sold and pending sold land listings in past year for a list of zip codes and saves the data to individual Excel files.
+It adds two columns to the data: 'lot_acres' and 'ppa' (price per acre) for user to analyze average price of land in a zip code.
+"""
+
+import os
+import pandas as pd
+from homeharvest import scrape_property
+
+
+def get_property_details(zip: str, listing_type):
+    properties = scrape_property(location=zip, listing_type=listing_type, property_type=["land"], past_days=365)
+    if not properties.empty:
+        properties["lot_acres"] = properties["lot_sqft"].apply(lambda x: x / 43560 if pd.notnull(x) else None)
+
+        properties = properties[properties["sqft"].isnull()]
+        properties["ppa"] = properties.apply(
+            lambda row: (
+                int(
+                    (
+                        row["sold_price"]
+                        if (pd.notnull(row["sold_price"]) and row["status"] == "SOLD")
+                        else row["list_price"]
+                    )
+                    / row["lot_acres"]
+                )
+                if pd.notnull(row["lot_acres"])
+                and row["lot_acres"] > 0
+                and (pd.notnull(row["sold_price"]) or pd.notnull(row["list_price"]))
+                else None
+            ),
+            axis=1,
+        )
+        properties["ppa"] = properties["ppa"].astype("Int64")
+        selected_columns = [
+            "property_url",
+            "property_id",
+            "style",
+            "status",
+            "street",
+            "city",
+            "state",
+            "zip_code",
+            "county",
+            "list_date",
+            "last_sold_date",
+            "list_price",
+            "sold_price",
+            "lot_sqft",
+            "lot_acres",
+            "ppa",
+        ]
+        properties = properties[selected_columns]
+    return properties
+
+
+def output_to_excel(zip_code, sold_df, pending_df):
+    root_folder = os.getcwd()
+    zip_folder = os.path.join(root_folder, "zips", zip_code)
+
+    # Create zip code folder if it doesn't exist
+    os.makedirs(zip_folder, exist_ok=True)
+
+    # Define file paths
+    sold_file = os.path.join(zip_folder, f"{zip_code}_sold.xlsx")
+    pending_file = os.path.join(zip_folder, f"{zip_code}_pending.xlsx")
+
+    # Save individual sold and pending files
+    sold_df.to_excel(sold_file, index=False)
+    pending_df.to_excel(pending_file, index=False)
+
+
+zip_codes = map(
+    str,
+    [
+        22920,
+        77024,
+        78028,
+        24553,
+        22967,
+        22971,
+        22922,
+        22958,
+        22969,
+        22949,
+        22938,
+        24599,
+        24562,
+        22976,
+        24464,
+        22964,
+        24581,
+    ],
+)
+
+combined_df = pd.DataFrame()
+for zip in zip_codes:
+    sold_df = get_property_details(zip, "sold")
+    pending_df = get_property_details(zip, "pending")
+    combined_df = pd.concat([combined_df, sold_df, pending_df], ignore_index=True)
+    output_to_excel(zip, sold_df, pending_df)
+
+combined_file = os.path.join(os.getcwd(), "zips", "combined.xlsx")
+combined_df.to_excel(combined_file, index=False)

homeharvest/__init__.py

@@ -3,12 +3,14 @@ import pandas as pd
 from .core.scrapers import ScraperInput
 from .utils import process_result, ordered_properties, validate_input, validate_dates, validate_limit
 from .core.scrapers.realtor import RealtorScraper
-from .core.scrapers.models import ListingType
+from .core.scrapers.models import ListingType, SearchPropertyType, ReturnType, Property
 
 
 def scrape_property(
     location: str,
     listing_type: str = "for_sale",
+    return_type: str = "pandas",
+    property_type: list[str] | None = None,
     radius: float = None,
     mls_only: bool = False,
     past_days: int = None,
@@ -18,12 +20,14 @@ def scrape_property(
     foreclosure: bool = None,
     extra_property_data: bool = True,
     exclude_pending: bool = False,
-    limit: int = 10000,
+    limit: int = 10000
-) -> pd.DataFrame:
+) -> pd.DataFrame | list[dict] | list[Property]:
     """
     Scrape properties from Realtor.com based on a given location and listing type.
     :param location: Location to search (e.g. "Dallas, TX", "85281", "2530 Al Lipscomb Way")
     :param listing_type: Listing Type (for_sale, for_rent, sold, pending)
+    :param return_type: Return type (pandas, pydantic, raw)
+    :param property_type: Property Type (single_family, multi_family, condos, condo_townhome_rowhome_coop, condo_townhome, townhomes, duplex_triplex, farm, land, mobile)
     :param radius: Get properties within _ (e.g. 1.0) miles. Only applicable for individual addresses.
     :param mls_only: If set, fetches only listings with MLS IDs.
     :param proxy: Proxy to use for scraping
@@ -40,7 +44,9 @@ def scrape_property(
 
     scraper_input = ScraperInput(
         location=location,
-        listing_type=ListingType[listing_type.upper()],
+        listing_type=ListingType(listing_type.upper()),
+        return_type=ReturnType(return_type.lower()),
+        property_type=[SearchPropertyType[prop.upper()] for prop in property_type] if property_type else None,
         proxy=proxy,
         radius=radius,
         mls_only=mls_only,
@@ -56,6 +62,9 @@ def scrape_property(
     site = RealtorScraper(scraper_input)
     results = site.search()
 
+    if scraper_input.return_type != ReturnType.pandas:
+        return results
+
     properties_dfs = [df for result in results if not (df := process_result(result)).empty]
     if not properties_dfs:
         return pd.DataFrame()
@@ -63,4 +72,6 @@ def scrape_property(
     with warnings.catch_warnings():
         warnings.simplefilter("ignore", category=FutureWarning)
 
-        return pd.concat(properties_dfs, ignore_index=True, axis=0)[ordered_properties].replace({"None": pd.NA, None: pd.NA, "": pd.NA})
+        return pd.concat(properties_dfs, ignore_index=True, axis=0)[ordered_properties].replace(
+            {"None": pd.NA, None: pd.NA, "": pd.NA}
+        )

homeharvest/core/scrapers/__init__.py

@@ -1,11 +1,13 @@
 from __future__ import annotations
 from dataclasses import dataclass
+from typing import Union
+
 import requests
 from requests.adapters import HTTPAdapter
 from urllib3.util.retry import Retry
 import uuid
 from ...exceptions import AuthenticationError
-from .models import Property, ListingType, SiteName
+from .models import Property, ListingType, SiteName, SearchPropertyType, ReturnType
 import json
 
 
@@ -13,6 +15,7 @@ import json
 class ScraperInput:
     location: str
     listing_type: ListingType
+    property_type: list[SearchPropertyType] | None = None
     radius: float | None = None
     mls_only: bool | None = False
     proxy: str | None = None
@@ -23,6 +26,7 @@ class ScraperInput:
     extra_property_data: bool | None = True
     exclude_pending: bool | None = False
     limit: int = 10000
+    return_type: ReturnType = ReturnType.pandas
 
 
 class Scraper:
@@ -34,11 +38,12 @@ class Scraper:
     ):
         self.location = scraper_input.location
         self.listing_type = scraper_input.listing_type
+        self.property_type = scraper_input.property_type
 
         if not self.session:
             Scraper.session = requests.Session()
             retries = Retry(
-                total=3, backoff_factor=3, status_forcelist=[429, 403], allowed_methods=frozenset(["GET", "POST"])
+                total=3, backoff_factor=4, status_forcelist=[429, 403], allowed_methods=frozenset(["GET", "POST"])
             )
 
             adapter = HTTPAdapter(max_retries=retries)
@@ -46,8 +51,21 @@ class Scraper:
             Scraper.session.mount("https://", adapter)
             Scraper.session.headers.update(
                 {
-                    "auth": f"Bearer {self.get_access_token()}",
+                    "accept": "application/json, text/javascript",
-                    "apollographql-client-name": "com.move.Realtor-apollo-ios",
+                    "accept-language": "en-US,en;q=0.9",
+                    "cache-control": "no-cache",
+                    "content-type": "application/json",
+                    "origin": "https://www.realtor.com",
+                    "pragma": "no-cache",
+                    "priority": "u=1, i",
+                    "rdc-ab-tests": "commute_travel_time_variation:v1",
+                    "sec-ch-ua": '"Not)A;Brand";v="99", "Google Chrome";v="127", "Chromium";v="127"',
+                    "sec-ch-ua-mobile": "?0",
+                    "sec-ch-ua-platform": '"Windows"',
+                    "sec-fetch-dest": "empty",
+                    "sec-fetch-mode": "cors",
+                    "sec-fetch-site": "same-origin",
+                    "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36",
                 }
             )
 
@@ -66,8 +84,9 @@ class Scraper:
         self.extra_property_data = scraper_input.extra_property_data
         self.exclude_pending = scraper_input.exclude_pending
         self.limit = scraper_input.limit
+        self.return_type = scraper_input.return_type
 
-    def search(self) -> list[Property]: ...
+    def search(self) -> list[Union[Property | dict]]: ...
 
     @staticmethod
     def _parse_home(home) -> Property: ...
@@ -81,27 +100,29 @@ class Scraper:
         response = requests.post(
             "https://graph.realtor.com/auth/token",
             headers={
-                'Host': 'graph.realtor.com',
+                "Host": "graph.realtor.com",
-                'Accept': '*/*',
+                "Accept": "*/*",
-                'Content-Type': 'Application/json',
+                "Content-Type": "Application/json",
-                'X-Client-ID': 'rdc_mobile_native,iphone',
+                "X-Client-ID": "rdc_mobile_native,iphone",
-                'X-Visitor-ID': device_id,
+                "X-Visitor-ID": device_id,
-                'X-Client-Version': '24.21.23.679885',
+                "X-Client-Version": "24.21.23.679885",
-                'Accept-Language': 'en-US,en;q=0.9',
+                "Accept-Language": "en-US,en;q=0.9",
-                'User-Agent': 'Realtor.com/24.21.23.679885 CFNetwork/1494.0.7 Darwin/23.4.0',
+                "User-Agent": "Realtor.com/24.21.23.679885 CFNetwork/1494.0.7 Darwin/23.4.0",
             },
-            data=json.dumps({
+            data=json.dumps(
+                {
                     "grant_type": "device_mobile",
                     "device_id": device_id,
-                "client_app_id": "rdc_mobile_native,24.21.23.679885,iphone"
+                    "client_app_id": "rdc_mobile_native,24.21.23.679885,iphone",
-            }))
+                }
+            ),
+        )
 
         data = response.json()
 
         if not (access_token := data.get("access_token")):
             raise AuthenticationError(
-                "Failed to get access token, use a proxy/vpn or wait a moment and try again.",
+                "Failed to get access token, use a proxy/vpn or wait a moment and try again.", response=response
-                response=response
             )
 
         return access_token

homeharvest/core/scrapers/models.py

@@ -4,6 +4,12 @@ from enum import Enum
 from typing import Optional
 
 
+class ReturnType(Enum):
+    pydantic = "pydantic"
+    pandas = "pandas"
+    raw = "raw"
+
+
 class SiteName(Enum):
     ZILLOW = "zillow"
     REDFIN = "redfin"
@@ -17,6 +23,20 @@ class SiteName(Enum):
         raise ValueError(f"{value} not found in {cls}")
 
 
+class SearchPropertyType(Enum):
+    SINGLE_FAMILY = "single_family"
+    APARTMENT = "apartment"
+    CONDOS = "condos"
+    CONDO_TOWNHOME_ROWHOME_COOP = "condo_townhome_rowhome_coop"
+    CONDO_TOWNHOME = "condo_townhome"
+    TOWNHOMES = "townhomes"
+    DUPLEX_TRIPLEX = "duplex_triplex"
+    FARM = "farm"
+    LAND = "land"
+    MULTI_FAMILY = "multi_family"
+    MOBILE = "mobile"
+
+
 class ListingType(Enum):
     FOR_SALE = "FOR_SALE"
     FOR_RENT = "FOR_RENT"
@@ -106,6 +126,7 @@ class Agent(Entity):
 
 @dataclass
 class Office(Entity):
+    mls_set: str | None = None
     email: str | None = None
     href: str | None = None
     phones: list[dict] | AgentPhone | None = None
@@ -132,6 +153,13 @@ class Advertisers:
 @dataclass
 class Property:
     property_url: str
+
+    property_id: str
+    #: allows_cats: bool
+    #: allows_dogs: bool
+
+    listing_id: str | None = None
+
     mls: str | None = None
     mls_id: str | None = None
     status: str | None = None
@@ -149,6 +177,8 @@ class Property:
     hoa_fee: int | None = None
     days_on_mls: int | None = None
     description: Description | None = None
+    tags: list[str] | None = None
+    details: list[dict] | None = None
 
     latitude: float | None = None
     longitude: float | None = None
@@ -158,5 +188,7 @@ class Property:
     nearby_schools: list[str] = None
     assessed_value: int | None = None
     estimated_value: int | None = None
+    tax: int | None = None
+    tax_history: list[dict] | None = None
 
     advertisers: Advertisers | None = None

homeharvest/core/scrapers/realtor/__init__.py

@@ -6,13 +6,35 @@ This module implements the scraper for realtor.com
 """
 
 from __future__ import annotations
+
+import json
 from concurrent.futures import ThreadPoolExecutor, as_completed
 from datetime import datetime
+from json import JSONDecodeError
 from typing import Dict, Union, Optional
 
+from tenacity import (
+    retry,
+    retry_if_exception_type,
+    wait_exponential,
+    stop_after_attempt,
+)
+
 from .. import Scraper
-from ..models import Property, Address, ListingType, Description, PropertyType, Agent, Broker, Builder, Advertisers, Office
+from ..models import (
-from .queries import GENERAL_RESULTS_QUERY, SEARCH_HOMES_DATA, HOMES_DATA
+    Property,
+    Address,
+    ListingType,
+    Description,
+    PropertyType,
+    Agent,
+    Broker,
+    Builder,
+    Advertisers,
+    Office,
+    ReturnType
+)
+from .queries import GENERAL_RESULTS_QUERY, SEARCH_HOMES_DATA, HOMES_DATA, HOME_FRAGMENT
 
 
 class RealtorScraper(Scraper):
@@ -81,9 +103,12 @@ class RealtorScraper(Scraper):
             return property_info["listings"][0]["listing_id"]
 
     def handle_home(self, property_id: str) -> list[Property]:
-        query = """query Home($property_id: ID!) {
+        query = (
+            """query Home($property_id: ID!) {
                     home(property_id: $property_id) %s
-                }""" % HOMES_DATA
+                }"""
+            % HOMES_DATA
+        )
 
         variables = {"property_id": property_id}
         payload = {
@@ -96,9 +121,7 @@ class RealtorScraper(Scraper):
 
         property_info = response_json["data"]["home"]
 
-        return [
+        return [self.process_property(property_info)]
-            self.process_property(property_info, "home")
-        ]
 
     @staticmethod
     def process_advertisers(advertisers: list[dict] | None) -> Advertisers | None:
@@ -122,7 +145,7 @@ class RealtorScraper(Scraper):
                     phones=advertiser.get("phones"),
                 )
 
-                if advertiser.get('broker') and advertiser["broker"].get('name'):  #: has a broker
+                if advertiser.get("broker") and advertiser["broker"].get("name"):  #: has a broker
                     processed_advertisers.broker = Broker(
                         uuid=_parse_fulfillment_id(advertiser["broker"].get("fulfillment_id")),
                         name=advertiser["broker"].get("name"),
@@ -130,7 +153,8 @@ class RealtorScraper(Scraper):
 
                 if advertiser.get("office"):  #: has an office
                     processed_advertisers.office = Office(
-                        uuid=_parse_fulfillment_id(advertiser["office"].get("fulfillment_id")) or advertiser["office"].get("mls_set"),
+                        uuid=_parse_fulfillment_id(advertiser["office"].get("fulfillment_id")),
+                        mls_set=advertiser["office"].get("mls_set"),
                         name=advertiser["office"].get("name"),
                         email=advertiser["office"].get("email"),
                         phones=advertiser["office"].get("phones"),
@@ -145,7 +169,7 @@ class RealtorScraper(Scraper):
 
         return processed_advertisers
 
-    def process_property(self, result: dict, query_name: str) -> Property | None:
+    def process_property(self, result: dict) -> Property | None:
         mls = result["source"].get("id") if "source" in result and isinstance(result["source"], dict) else None
 
         if not mls and self.mls_only:
@@ -158,15 +182,14 @@ class RealtorScraper(Scraper):
             and result["location"]["address"].get("coordinate")
         )
 
-        is_pending = result["flags"].get("is_pending") or result["flags"].get("is_contingent")
+        is_pending = result["flags"].get("is_pending")
+        is_contingent = result["flags"].get("is_contingent")
 
-        if is_pending and (self.exclude_pending and self.listing_type != ListingType.PENDING):
+        if (is_pending or is_contingent) and (self.exclude_pending and self.listing_type != ListingType.PENDING):
             return
 
         property_id = result["property_id"]
-        prop_details = self.get_prop_details(property_id) if self.extra_property_data and query_name != "home" else {}
+        prop_details = self.process_extra_property_details(result) if self.extra_property_data else {}
-        if not prop_details:
-            prop_details = self.process_extra_property_details(result)
 
         property_estimates_root = result.get("current_estimates") or result.get("estimates", {}).get("currentValues")
         estimated_value = self.get_key(property_estimates_root, [0, "estimate"])
@@ -180,36 +203,36 @@ class RealtorScraper(Scraper):
                 if "source" in result and isinstance(result["source"], dict)
                 else None
             ),
-            property_url=(
+            property_url=result["href"],
-                f"{self.PROPERTY_URL}{property_id}"
+            property_id=property_id,
-                if self.listing_type != ListingType.FOR_RENT
+            listing_id=result.get("listing_id"),
-                else f"{self.PROPERTY_URL}M{property_id}?listing_status=rental"
+            status=("PENDING" if is_pending else "CONTINGENT" if is_contingent else result["status"].upper()),
-            ),
-            status="PENDING" if is_pending else result["status"].upper(),
             list_price=result["list_price"],
             list_price_min=result["list_price_min"],
             list_price_max=result["list_price_max"],
-            list_date=result["list_date"].split("T")[0] if result.get("list_date") else None,
+            list_date=(result["list_date"].split("T")[0] if result.get("list_date") else None),
             prc_sqft=result.get("price_per_sqft"),
             last_sold_date=result.get("last_sold_date"),
             new_construction=result["flags"].get("is_new_construction") is True,
-            hoa_fee=result["hoa"]["fee"] if result.get("hoa") and isinstance(result["hoa"], dict) else None,
+            hoa_fee=(result["hoa"]["fee"] if result.get("hoa") and isinstance(result["hoa"], dict) else None),
-            latitude=result["location"]["address"]["coordinate"].get("lat") if able_to_get_lat_long else None,
+            latitude=(result["location"]["address"]["coordinate"].get("lat") if able_to_get_lat_long else None),
-            longitude=result["location"]["address"]["coordinate"].get("lon") if able_to_get_lat_long else None,
+            longitude=(result["location"]["address"]["coordinate"].get("lon") if able_to_get_lat_long else None),
             address=self._parse_address(result, search_type="general_search"),
             description=self._parse_description(result),
             neighborhoods=self._parse_neighborhoods(result),
-            county=result["location"]["county"].get("name") if result["location"]["county"] else None,
+            county=(result["location"]["county"].get("name") if result["location"]["county"] else None),
-            fips_code=result["location"]["county"].get("fips_code") if result["location"]["county"] else None,
+            fips_code=(result["location"]["county"].get("fips_code") if result["location"]["county"] else None),
             days_on_mls=self.calculate_days_on_mls(result),
             nearby_schools=prop_details.get("schools"),
             assessed_value=prop_details.get("assessed_value"),
             estimated_value=estimated_value if estimated_value else None,
             advertisers=advertisers,
+            tax=prop_details.get("tax"),
+            tax_history=prop_details.get("tax_history"),
         )
         return realty_property
 
-    def general_search(self, variables: dict, search_type: str) -> Dict[str, Union[int, list[Property]]]:
+    def general_search(self, variables: dict, search_type: str) -> Dict[str, Union[int, Union[list[Property], list[dict]]]]:
         """
         Handles a location area & returns a list of properties
         """
@@ -226,10 +249,15 @@ class RealtorScraper(Scraper):
             elif self.last_x_days:
                 date_param = f'list_date: {{ min: "$today-{self.last_x_days}D" }}'
 
+        property_type_param = ""
+        if self.property_type:
+            property_types = [pt.value for pt in self.property_type]
+            property_type_param = f"type: {json.dumps(property_types)}"
+
         sort_param = (
             "sort: [{ field: sold_date, direction: desc }]"
             if self.listing_type == ListingType.SOLD
-            else "sort: [{ field: list_date, direction: desc }]"
+            else ""  #: "sort: [{ field: list_date, direction: desc }]"  #: prioritize normal fractal sort from realtor
         )
 
         pending_or_contingent_param = (
@@ -260,6 +288,7 @@ class RealtorScraper(Scraper):
                                 status: %s
                                 %s
                                 %s
+                                %s
                             }
                             %s
                             limit: 200
@@ -269,29 +298,27 @@ class RealtorScraper(Scraper):
                 is_foreclosure,
                 listing_type.value.lower(),
                 date_param,
+                property_type_param,
                 pending_or_contingent_param,
                 sort_param,
                 GENERAL_RESULTS_QUERY,
             )
         elif search_type == "area":  #: general search, came from a general location
             query = """query Home_search(
-                                $city: String,
+                                $location: String!,
-                                $county: [String],
-                                $state_code: String,
-                                $postal_code: String
                                 $offset: Int,
                             ) {
                                 home_search(
                                     query: {
                                         %s
-                                        city: $city
+                                        search_location: {location: $location}
-                                        county: $county
-                                        postal_code: $postal_code
-                                        state_code: $state_code
                                         status: %s
+                                        unique: true
+                                        %s
                                         %s
                                         %s
                                     }
+                                    bucket: { sort: "fractal_v1.1.3_fr" }
                                     %s
                                     limit: 200
                                     offset: $offset
@@ -300,6 +327,7 @@ class RealtorScraper(Scraper):
                 is_foreclosure,
                 listing_type.value.lower(),
                 date_param,
+                property_type_param,
                 pending_or_contingent_param,
                 sort_param,
                 GENERAL_RESULTS_QUERY,
@@ -330,7 +358,7 @@ class RealtorScraper(Scraper):
         response_json = response.json()
         search_key = "home_search" if "home_search" in query else "property_search"
 
-        properties: list[Property] = []
+        properties: list[Union[Property, dict]] = []
 
         if (
             response_json is None
@@ -348,17 +376,25 @@ class RealtorScraper(Scraper):
 
         #: limit the number of properties to be processed
         #: example, if your offset is 200, and your limit is 250, return 50
-        properties_list = properties_list[:self.limit - offset]
+        properties_list: list[dict] = properties_list[: self.limit - offset]
 
+        if self.extra_property_data:
+            property_ids = [data["property_id"] for data in properties_list]
+            extra_property_details = self.get_bulk_prop_details(property_ids) or {}
+
+            for result in properties_list:
+                result.update(extra_property_details.get(result["property_id"], {}))
+
+        if self.return_type != ReturnType.raw:
             with ThreadPoolExecutor(max_workers=self.NUM_PROPERTY_WORKERS) as executor:
-            futures = [
+                futures = [executor.submit(self.process_property, result) for result in properties_list]
-                executor.submit(self.process_property, result, search_key) for result in properties_list
-            ]
 
                 for future in as_completed(futures):
                     result = future.result()
                     if result:
                         properties.append(result)
+        else:
+            properties = properties_list
 
         return {
             "total": total_properties,
@@ -403,10 +439,7 @@ class RealtorScraper(Scraper):
 
         else:  #: general search, location
             search_variables |= {
-                "city": location_info.get("city"),
+                "location": self.location,
-                "county": location_info.get("county"),
-                "state_code": location_info.get("state_code"),
-                "postal_code": location_info.get("postal_code"),
             }
 
         if self.foreclosure:
@@ -423,7 +456,11 @@ class RealtorScraper(Scraper):
                     variables=search_variables | {"offset": i},
                     search_type=search_type,
                 )
-                for i in range(self.DEFAULT_PAGE_SIZE, min(total, self.limit), self.DEFAULT_PAGE_SIZE)
+                for i in range(
+                    self.DEFAULT_PAGE_SIZE,
+                    min(total, self.limit),
+                    self.DEFAULT_PAGE_SIZE,
+                )
             ]
 
             for future in as_completed(futures):
@@ -445,35 +482,75 @@ class RealtorScraper(Scraper):
     def process_extra_property_details(self, result: dict) -> dict:
         schools = self.get_key(result, ["nearbySchools", "schools"])
         assessed_value = self.get_key(result, ["taxHistory", 0, "assessment", "total"])
+        tax_history = self.get_key(result, ["taxHistory"])
 
         schools = [school["district"]["name"] for school in schools if school["district"].get("name")]
+
+        # Process tax history
+        latest_tax = None
+        processed_tax_history = None
+        if tax_history and isinstance(tax_history, list):
+            tax_history = sorted(tax_history, key=lambda x: x.get("year", 0), reverse=True)
+
+            if tax_history and "tax" in tax_history[0]:
+                latest_tax = tax_history[0]["tax"]
+
+            processed_tax_history = []
+            for entry in tax_history:
+                if "year" in entry and "tax" in entry:
+                    processed_entry = {
+                        "year": entry["year"],
+                        "tax": entry["tax"],
+                    }
+                    if "assessment" in entry and isinstance(entry["assessment"], dict):
+                        processed_entry["assessment"] = {
+                            "building": entry["assessment"].get("building"),
+                            "land": entry["assessment"].get("land"),
+                            "total": entry["assessment"].get("total"),
+                        }
+                    processed_tax_history.append(processed_entry)
+
         return {
             "schools": schools if schools else None,
             "assessed_value": assessed_value if assessed_value else None,
+            "tax": latest_tax,
+            "tax_history": processed_tax_history,
         }
 
-    def get_prop_details(self, property_id: str) -> dict:
+    @retry(
-        if not self.extra_property_data:
+        retry=retry_if_exception_type(JSONDecodeError),
+        wait=wait_exponential(min=4, max=10),
+        stop=stop_after_attempt(3),
+    )
+    def get_bulk_prop_details(self, property_ids: list[str]) -> dict:
+        """
+        Fetch extra property details for multiple properties in a single GraphQL query.
+        Returns a map of property_id to its details.
+        """
+        if not self.extra_property_data or not property_ids:
             return {}
 
-        query = """query GetHome($property_id: ID!) {
+        property_ids = list(set(property_ids))
-                    home(property_id: $property_id) {
-                        __typename
 
-                        nearbySchools: nearby_schools(radius: 5.0, limit_per_level: 3) {
+        # Construct the bulk query
-                            __typename schools { district { __typename id name } }
+        fragments = "\n".join(
-                        }
+            f'home_{property_id}: home(property_id: {property_id}) {{ ...HomeData }}'
-                        taxHistory: tax_history { __typename tax year assessment { __typename building land total } }
+            for property_id in property_ids
-                    }
+        )
-                }"""
+        query = f"""{HOME_FRAGMENT}
         
-        variables = {"property_id": property_id}
+        query GetHomes {{
-        response = self.session.post(self.PROPERTY_GQL, json={"query": query, "variables": variables})
+            {fragments}
+        }}"""
 
+        response = self.session.post(self.SEARCH_GQL_URL, json={"query": query})
         data = response.json()
-        property_details = data["data"]["home"]
 
-        return self.process_extra_property_details(property_details)
+        if "data" not in data:
+            return {}
+
+        properties = data["data"]
+        return {data.replace('home_', ''): properties[data] for data in properties if properties[data]}
 
     @staticmethod
     def _parse_neighborhoods(result: dict) -> Optional[str]:
@@ -535,20 +612,22 @@ class RealtorScraper(Scraper):
             style = style.upper()
 
         primary_photo = ""
-        if (primary_photo_info := result.get('primary_photo')) and (primary_photo_href := primary_photo_info.get("href")):
+        if (primary_photo_info := result.get("primary_photo")) and (
+            primary_photo_href := primary_photo_info.get("href")
+        ):
             primary_photo = primary_photo_href.replace("s.jpg", "od-w480_h360_x2.webp?w=1080&q=75")
 
         return Description(
             primary_photo=primary_photo,
             alt_photos=RealtorScraper.process_alt_photos(result.get("photos", [])),
-            style=PropertyType.__getitem__(style) if style and style in PropertyType.__members__ else None,
+            style=(PropertyType.__getitem__(style) if style and style in PropertyType.__members__ else None),
             beds=description_data.get("beds"),
             baths_full=description_data.get("baths_full"),
             baths_half=description_data.get("baths_half"),
             sqft=description_data.get("sqft"),
             lot_sqft=description_data.get("lot_sqft"),
             sold_price=(
-                result.get('last_sold_price') or description_data.get("sold_price")
+                result.get("last_sold_price") or description_data.get("sold_price")
                 if result.get("last_sold_date") or result["list_price"] != description_data.get("sold_price")
                 else None
             ),  #: has a sold date or list and sold price are different
@@ -582,4 +661,8 @@ class RealtorScraper(Scraper):
         if not photos_info:
             return None
 
-        return [photo_info["href"].replace("s.jpg", "od-w480_h360_x2.webp?w=1080&q=75") for photo_info in photos_info if photo_info.get("href")]
+        return [
+            photo_info["href"].replace("s.jpg", "od-w480_h360_x2.webp?w=1080&q=75")
+            for photo_info in photos_info
+            if photo_info.get("href")
+        ]

homeharvest/core/scrapers/realtor/queries.py

@@ -2,6 +2,7 @@ _SEARCH_HOMES_DATA_BASE = """{
     pending_date
     listing_id
     property_id
+    href
     list_date
     status
     last_sold_price
@@ -10,6 +11,34 @@ _SEARCH_HOMES_DATA_BASE = """{
     list_price_max
     list_price_min
     price_per_sqft
+    tags
+    details {
+        category
+        text
+        parent_category
+    }
+    pet_policy {
+        cats
+        dogs
+        dogs_small
+        dogs_large
+        __typename
+    }
+    units {
+        availability {
+          date
+          __typename
+        }
+        description {
+          baths_consolidated
+          baths
+          beds
+          sqft
+          __typename
+        }
+        list_price
+        __typename
+    }
     flags {
         is_contingent
         is_pending
@@ -63,11 +92,14 @@ _SEARCH_HOMES_DATA_BASE = """{
     tax_record {
         public_record_id
     }
-    primary_photo {
+    primary_photo(https: true) {
         href
     }
-    photos {
+    photos(https: true) {
         href
+        tags {
+            label
+        }
     }
     advertisers {
         email
@@ -115,15 +147,63 @@ _SEARCH_HOMES_DATA_BASE = """{
         }
         rental_management {
             name
+            href
             fulfillment_id
         }
     }
     """
 
+
+HOME_FRAGMENT = """
+fragment HomeData on Home {
+    property_id
+    nearbySchools: nearby_schools(radius: 5.0, limit_per_level: 3) {
+        __typename schools { district { __typename id name } }
+    }
+    taxHistory: tax_history { __typename tax year assessment { __typename building land total } }
+    monthly_fees {
+        description
+        display_amount
+    }
+    one_time_fees {
+        description
+        display_amount
+    }
+    parking {
+        unassigned_space_rent
+        assigned_spaces_available
+        description
+        assigned_space_rent
+    }
+    terms {
+        text
+        category
+    }
+}
+"""
+
 HOMES_DATA = """%s
                 nearbySchools: nearby_schools(radius: 5.0, limit_per_level: 3) {
                             __typename schools { district { __typename id name } }
                         }
+                monthly_fees {
+                    description
+                    display_amount
+                }
+                one_time_fees {
+                    description
+                    display_amount
+                }
+                parking {
+                    unassigned_space_rent
+                    assigned_spaces_available
+                    description
+                    assigned_space_rent
+                }
+                terms {
+                    text
+                    category
+                }
                 taxHistory: tax_history { __typename tax year assessment { __typename building land total } }
                 estimates {
                     __typename
@@ -140,7 +220,7 @@ HOMES_DATA = """%s
 }""" % _SEARCH_HOMES_DATA_BASE
 
 SEARCH_HOMES_DATA = """%s
-                    current_estimates {
+current_estimates {
     __typename
     source {
         __typename
@@ -152,7 +232,7 @@ SEARCH_HOMES_DATA = """%s
     estimateLow: estimate_low
     date
     isBestHomeValue: isbest_homevalue
-                        }
+}
 }""" % _SEARCH_HOMES_DATA_BASE
 
 GENERAL_RESULTS_QUERY = """{

homeharvest/utils.py

@@ -6,6 +6,8 @@ from .exceptions import InvalidListingType, InvalidDate
 
 ordered_properties = [
     "property_url",
+    "property_id",
+    "listing_id",
     "mls",
     "mls_id",
     "status",
@@ -31,6 +33,8 @@ ordered_properties = [
     "last_sold_date",
     "assessed_value",
     "estimated_value",
+    "tax",
+    "tax_history",
     "new_construction",
     "lot_sqft",
     "price_per_sqft",
@@ -53,6 +57,7 @@ ordered_properties = [
     "builder_id",
     "builder_name",
     "office_id",
+    "office_mls_set",
     "office_name",
     "office_email",
     "office_phones",
@@ -102,6 +107,7 @@ def process_result(result: Property) -> pd.DataFrame:
             prop_data["office_name"] = office_data.name
             prop_data["office_email"] = office_data.email
             prop_data["office_phones"] = office_data.phones
+            prop_data["office_mls_set"] = office_data.mls_set
 
     prop_data["price_per_sqft"] = prop_data["prc_sqft"]
     prop_data["nearby_schools"] = filter(None, prop_data["nearby_schools"]) if prop_data["nearby_schools"] else None
@@ -111,8 +117,11 @@ def process_result(result: Property) -> pd.DataFrame:
     if description:
         prop_data["primary_photo"] = description.primary_photo
         prop_data["alt_photos"] = ", ".join(description.alt_photos) if description.alt_photos else None
-        prop_data["style"] = description.style if isinstance(description.style,
+        prop_data["style"] = (
-                                                             str) else description.style.value if description.style else None
+            description.style
+            if isinstance(description.style, str)
+            else description.style.value if description.style else None
+        )
         prop_data["beds"] = description.beds
         prop_data["full_baths"] = description.baths_full
         prop_data["half_baths"] = description.baths_half

poetry.lock

@@ -1,4 +1,4 @@
-# This file is automatically @generated by Poetry 1.8.2 and should not be changed by hand.
+# This file is automatically @generated by Poetry 1.8.4 and should not be changed by hand.
 
 [[package]]
 name = "annotated-types"
@@ -667,6 +667,21 @@ files = [
     {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"},
 ]
 
+[[package]]
+name = "tenacity"
+version = "9.0.0"
+description = "Retry code until it succeeds"
+optional = false
+python-versions = ">=3.8"
+files = [
+    {file = "tenacity-9.0.0-py3-none-any.whl", hash = "sha256:93de0c98785b27fcf659856aa9f54bfbd399e29969b0621bc7f762bd441b4539"},
+    {file = "tenacity-9.0.0.tar.gz", hash = "sha256:807f37ca97d62aa361264d497b0e31e92b8027044942bfa756160d908320d73b"},
+]
+
+[package.extras]
+doc = ["reno", "sphinx"]
+test = ["pytest", "tornado (>=4.5)", "typeguard"]
+
 [[package]]
 name = "tomli"
 version = "2.0.1"
@@ -740,4 +755,4 @@ test = ["covdefaults (>=2.3)", "coverage (>=7.2.7)", "coverage-enable-subprocess
 [metadata]
 lock-version = "2.0"
 python-versions = ">=3.9,<3.13"
-content-hash = "21ef9cfb35c446a375a2b74c37691d7031afb1e4f66a8b63cb7c1669470689d2"
+content-hash = "cefc11b1bf5ad99d628f6d08f6f03003522cc1b6e48b519230d99d716a5c165c"

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "homeharvest"
-version = "0.4.1"
+version = "0.4.7"
 description = "Real estate scraping library"
 authors = ["Zachary Hampton <zachary@bunsly.com>", "Cullen Watson <cullen@bunsly.com>"]
 homepage = "https://github.com/Bunsly/HomeHarvest"
@@ -14,6 +14,7 @@ python = ">=3.9,<3.13"
 requests = "^2.31.0"
 pandas = "^2.1.1"
 pydantic = "^2.7.4"
+tenacity = "^9.0.0"
 
 
 [tool.poetry.group.dev.dependencies]

tests/test_realtor.py

@@ -1,4 +1,5 @@
-from homeharvest import scrape_property
+from homeharvest import scrape_property, Property
+import pandas as pd
 
 
 def test_realtor_pending_or_contingent():
@@ -105,8 +106,12 @@ def test_realtor():
             location="2530 Al Lipscomb Way",
             listing_type="for_sale",
         ),
-        scrape_property(location="Phoenix, AZ", listing_type="for_rent", limit=1000),  #: does not support "city, state, USA" format
+        scrape_property(
-        scrape_property(location="Dallas, TX", listing_type="sold", limit=1000),  #: does not support "city, state, USA" format
+            location="Phoenix, AZ", listing_type="for_rent", limit=1000
+        ),  #: does not support "city, state, USA" format
+        scrape_property(
+            location="Dallas, TX", listing_type="sold", limit=1000
+        ),  #: does not support "city, state, USA" format
         scrape_property(location="85281"),
     ]
 
@@ -114,11 +119,13 @@ def test_realtor():
 
 
 def test_realtor_city():
-    results = scrape_property(
+    results = scrape_property(location="Atlanta, GA", listing_type="for_sale", limit=1000)
-        location="Atlanta, GA",
+
-        listing_type="for_sale",
+    assert results is not None and len(results) > 0
-        limit=1000
+
-    )
+
+def test_realtor_land():
+    results = scrape_property(location="Atlanta, GA", listing_type="for_sale", property_type=["land"], limit=1000)
 
     assert results is not None and len(results) > 0
 
@@ -128,6 +135,7 @@ def test_realtor_bad_address():
         location="abceefg ju098ot498hh9",
         listing_type="for_sale",
     )
+
     if len(bad_results) == 0:
         assert True
 
@@ -240,9 +248,10 @@ def test_apartment_list_price():
     results = results[results["style"] == "APARTMENT"]
 
     #: get percentage of results with atleast 1 of any column not none, list_price, list_price_min, list_price_max
-    assert len(results[results[["list_price", "list_price_min", "list_price_max"]].notnull().any(axis=1)]) / len(
+    assert (
-        results
+        len(results[results[["list_price", "list_price_min", "list_price_max"]].notnull().any(axis=1)]) / len(results)
-    ) > 0.5
+        > 0.5
+    )
 
 
 def test_builder_exists():
@@ -253,3 +262,41 @@ def test_builder_exists():
 
     assert listing is not None
     assert listing["builder_name"].nunique() > 0
+
+
+def test_phone_number_matching():
+    searches = [
+        scrape_property(
+            location="Phoenix, AZ",
+            listing_type="for_sale",
+            limit=100,
+        ),
+        scrape_property(
+            location="Phoenix, AZ",
+            listing_type="for_sale",
+            limit=100,
+        ),
+    ]
+
+    assert all([search is not None for search in searches])
+
+    #: random row
+    row = searches[0][searches[0]["agent_phones"].notnull()].sample()
+
+    #: find matching row
+    matching_row = searches[1].loc[searches[1]["property_url"] == row["property_url"].values[0]]
+
+    #: assert phone numbers are the same
+    assert row["agent_phones"].values[0] == matching_row["agent_phones"].values[0]
+
+
+def test_return_type():
+    results = {
+        "pandas": scrape_property(location="Surprise, AZ", listing_type="for_rent", limit=100),
+        "pydantic": scrape_property(location="Surprise, AZ", listing_type="for_rent", limit=100, return_type="pydantic"),
+        "raw": scrape_property(location="Surprise, AZ", listing_type="for_rent", limit=100, return_type="raw"),
+    }
+
+    assert isinstance(results["pandas"], pd.DataFrame)
+    assert isinstance(results["pydantic"][0], Property)
+    assert isinstance(results["raw"][0], dict)