Update README.md

- new hrefs
- property_id, listing_id data points

.github/FUNDING.yml

@@ -0,0 +1 @@
+github: Bunsly

README.md

@@ -2,10 +2,6 @@
 
 **HomeHarvest** is a real estate scraping library that extracts and formats data in the style of MLS listings.
 
-**Not technical?** Try out the web scraping tool on our site at [tryhomeharvest.com](https://tryhomeharvest.com).
-
-*Looking to build a data-focused software product?* **[Book a call](https://bunsly.com)** *to work with us.*
-
 ## HomeHarvest Features
 
 - **Source**: Fetches properties directly from **Realtor.com**.
@@ -40,6 +36,7 @@ properties = scrape_property(
   listing_type="sold",  # or (for_sale, for_rent, pending)
   past_days=30,  # sold in last 30 days - listed in last 30 days if (for_sale, for_rent)
 
+  # property_type=['single_family','multi_family'],
   # date_from="2023-05-01", # alternative to past_days
   # date_to="2023-05-28",
   # foreclosure=True
@@ -68,13 +65,25 @@ print(properties.head())
 ```
 Required
 ├── location (str): The address in various formats - this could be just a zip code, a full address, or city/state, etc.
-└── listing_type (option): Choose the type of listing.
+├── listing_type (option): Choose the type of listing.
     - 'for_rent'
     - 'for_sale'
     - 'sold'
-    - 'pending'
+    - 'pending' (for pending/contingent sales)
 
 Optional
+├── property_type (list): Choose the type of properties.
+    - 'single_family'
+    - 'multi_family'
+    - 'condos'
+    - 'condo_townhome_rowhome_coop'
+    - 'condo_townhome'
+    - 'townhomes'
+    - 'duplex_triplex'
+    - 'farm'
+    - 'land'
+    - 'mobile'
+
 ├── radius (decimal): Radius in miles to find comparable properties based on individual addresses.
 │    Example: 5.5 (fetches properties within a 5.5-mile radius if location is set to a specific address; otherwise, ignored)
 │
@@ -94,7 +103,7 @@ Optional
 │
 ├── extra_property_data (True/False): Increases requests by O(n). If set, this fetches additional property data for general searches (e.g. schools, tax appraisals etc.)
 │
-├── exclude_pending (True/False): If set, excludes pending properties from the results unless listing_type is 'pending'
+├── exclude_pending (True/False): If set, excludes 'pending' properties from the 'for_sale' results unless listing_type is 'pending'
 │
 └── limit (integer): Limit the number of properties to fetch. Max & default is 10000.
 ```
@@ -104,6 +113,8 @@ Optional
 Property
 ├── Basic Information:
 │ ├── property_url
+│ ├── property_id
+│ ├── listing_id
 │ ├── mls
 │ ├── mls_id
 │ └── status
@@ -139,6 +150,14 @@ Property
 │ ├── new_construction
 │ └── hoa_fee
 
+├── Tax Information:
+│  ├── year
+│  ├── tax
+│  ├── assessment
+│  │   ├── building
+│  │   ├── land
+│  │   └── total
+
 ├── Location Details:
 │ ├── latitude
 │ ├── longitude

examples/HomeHarvest_Demo.ipynb

@@ -1,141 +0,0 @@
-{
- "cells": [
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "cb48903e-5021-49fe-9688-45cd0bc05d0f",
-   "metadata": {
-    "is_executing": true
-   },
-   "outputs": [],
-   "source": [
-    "from homeharvest import scrape_property\n",
-    "import pandas as pd"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "156488ce-0d5f-43c5-87f4-c33e9c427860",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "pd.set_option('display.max_columns', None)  # Show all columns\n",
-    "pd.set_option('display.max_rows', None)     # Show all rows\n",
-    "pd.set_option('display.width', None)        # Auto-adjust display width to fit console\n",
-    "pd.set_option('display.max_colwidth', 50)   # Limit max column width to 50 characters"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "1c8b9744-8606-4e9b-8add-b90371a249a7",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# check for sale properties\n",
-    "scrape_property(\n",
-    "    location=\"dallas\",\n",
-    "    listing_type=\"for_sale\"\n",
-    ")"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "aaf86093",
-   "metadata": {
-    "collapsed": false,
-    "jupyter": {
-     "outputs_hidden": false
-    }
-   },
-   "outputs": [],
-   "source": [
-    "# search a specific address\n",
-    "scrape_property(\n",
-    "    location=\"2530 Al Lipscomb Way\",\n",
-    "    listing_type=\"for_sale\"\n",
-    ")"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "ab7b4c21-da1d-4713-9df4-d7425d8ce21e",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# check rentals\n",
-    "scrape_property(\n",
-    "    location=\"chicago, illinois\",\n",
-    "    listing_type=\"for_rent\"\n",
-    ")"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "af280cd3",
-   "metadata": {
-    "collapsed": false,
-    "jupyter": {
-     "outputs_hidden": false
-    }
-   },
-   "outputs": [],
-   "source": [
-    "# check sold properties\n",
-    "properties = scrape_property(\n",
-    "    location=\"90210\",\n",
-    "    listing_type=\"sold\",\n",
-    "    past_days=10\n",
-    ")\n",
-    "display(properties)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "628c1ce2",
-   "metadata": {
-    "collapsed": false,
-    "is_executing": true,
-    "jupyter": {
-     "outputs_hidden": false
-    }
-   },
-   "outputs": [],
-   "source": [
-    "# display clickable URLs\n",
-    "from IPython.display import display, HTML\n",
-    "properties['property_url'] = '<a href=\"' + properties['property_url'] + '\" target=\"_blank\">' + properties['property_url'] + '</a>'\n",
-    "\n",
-    "html = properties.to_html(escape=False)\n",
-    "truncate_width = f'<style>.dataframe td {{ max-width: 200px; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; }}</style>{html}'\n",
-    "display(HTML(truncate_width))"
-   ]
-  }
- ],
- "metadata": {
-  "kernelspec": {
-   "display_name": "Python 3 (ipykernel)",
-   "language": "python",
-   "name": "python3"
-  },
-  "language_info": {
-   "codemirror_mode": {
-    "name": "ipython",
-    "version": 3
-   },
-   "file_extension": ".py",
-   "mimetype": "text/x-python",
-   "name": "python",
-   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.10.11"
-  }
- },
- "nbformat": 4,
- "nbformat_minor": 5
-}

examples/HomeHarvest_Demo.py

@@ -1,20 +0,0 @@
-from homeharvest import scrape_property
-from datetime import datetime
-
-# Generate filename based on current timestamp
-current_timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
-filename = f"HomeHarvest_{current_timestamp}.csv"
-
-properties = scrape_property(
-    location="San Diego, CA",
-    listing_type="sold",  # or (for_sale, for_rent)
-    past_days=30,  # sold in last 30 days - listed in last x days if (for_sale, for_rent)
-    # pending_or_contingent=True # use on for_sale listings to find pending / contingent listings
-    # mls_only=True,  # only fetch MLS listings
-    # proxy="http://user:pass@host:port"  # use a proxy to change your IP address
-)
-print(f"Number of properties: {len(properties)}")
-
-# Export to csv
-properties.to_csv(filename, index=False)
-print(properties.head())

examples/price_of_land.py

@@ -0,0 +1,104 @@
+"""
+This script scrapes sold and pending sold land listings in past year for a list of zip codes and saves the data to individual Excel files.
+It adds two columns to the data: 'lot_acres' and 'ppa' (price per acre) for user to analyze average price of land in a zip code.
+"""
+
+import os
+import pandas as pd
+from homeharvest import scrape_property
+
+
+def get_property_details(zip: str, listing_type):
+    properties = scrape_property(location=zip, listing_type=listing_type, property_type=["land"], past_days=365)
+    if not properties.empty:
+        properties["lot_acres"] = properties["lot_sqft"].apply(lambda x: x / 43560 if pd.notnull(x) else None)
+
+        properties = properties[properties["sqft"].isnull()]
+        properties["ppa"] = properties.apply(
+            lambda row: (
+                int(
+                    (
+                        row["sold_price"]
+                        if (pd.notnull(row["sold_price"]) and row["status"] == "SOLD")
+                        else row["list_price"]
+                    )
+                    / row["lot_acres"]
+                )
+                if pd.notnull(row["lot_acres"])
+                and row["lot_acres"] > 0
+                and (pd.notnull(row["sold_price"]) or pd.notnull(row["list_price"]))
+                else None
+            ),
+            axis=1,
+        )
+        properties["ppa"] = properties["ppa"].astype("Int64")
+        selected_columns = [
+            "property_url",
+            "property_id",
+            "style",
+            "status",
+            "street",
+            "city",
+            "state",
+            "zip_code",
+            "county",
+            "list_date",
+            "last_sold_date",
+            "list_price",
+            "sold_price",
+            "lot_sqft",
+            "lot_acres",
+            "ppa",
+        ]
+        properties = properties[selected_columns]
+    return properties
+
+
+def output_to_excel(zip_code, sold_df, pending_df):
+    root_folder = os.getcwd()
+    zip_folder = os.path.join(root_folder, "zips", zip_code)
+
+    # Create zip code folder if it doesn't exist
+    os.makedirs(zip_folder, exist_ok=True)
+
+    # Define file paths
+    sold_file = os.path.join(zip_folder, f"{zip_code}_sold.xlsx")
+    pending_file = os.path.join(zip_folder, f"{zip_code}_pending.xlsx")
+
+    # Save individual sold and pending files
+    sold_df.to_excel(sold_file, index=False)
+    pending_df.to_excel(pending_file, index=False)
+
+
+zip_codes = map(
+    str,
+    [
+        22920,
+        77024,
+        78028,
+        24553,
+        22967,
+        22971,
+        22922,
+        22958,
+        22969,
+        22949,
+        22938,
+        24599,
+        24562,
+        22976,
+        24464,
+        22964,
+        24581,
+    ],
+)
+
+combined_df = pd.DataFrame()
+for zip in zip_codes:
+    sold_df = get_property_details(zip, "sold")
+    pending_df = get_property_details(zip, "pending")
+    combined_df = pd.concat([combined_df, sold_df, pending_df], ignore_index=True)
+    output_to_excel(zip, sold_df, pending_df)
+
+combined_file = os.path.join(os.getcwd(), "zips", "combined.xlsx")
+combined_df.to_excel(combined_file, index=False)

homeharvest/__init__.py

@@ -3,12 +3,13 @@ import pandas as pd
 from .core.scrapers import ScraperInput
 from .utils import process_result, ordered_properties, validate_input, validate_dates, validate_limit
 from .core.scrapers.realtor import RealtorScraper
-from .core.scrapers.models import ListingType
+from .core.scrapers.models import ListingType, SearchPropertyType
 
 
 def scrape_property(
     location: str,
     listing_type: str = "for_sale",
+    property_type: list[str] | None = None,
     radius: float = None,
     mls_only: bool = False,
     past_days: int = None,
@@ -24,6 +25,7 @@ def scrape_property(
     Scrape properties from Realtor.com based on a given location and listing type.
     :param location: Location to search (e.g. "Dallas, TX", "85281", "2530 Al Lipscomb Way")
     :param listing_type: Listing Type (for_sale, for_rent, sold, pending)
+    :param property_type: Property Type (single_family, multi_family, condos, condo_townhome_rowhome_coop, condo_townhome, townhomes, duplex_triplex, farm, land, mobile)
     :param radius: Get properties within _ (e.g. 1.0) miles. Only applicable for individual addresses.
     :param mls_only: If set, fetches only listings with MLS IDs.
     :param proxy: Proxy to use for scraping
@@ -41,6 +43,7 @@ def scrape_property(
     scraper_input = ScraperInput(
         location=location,
         listing_type=ListingType[listing_type.upper()],
+        property_type=[SearchPropertyType[prop.upper()] for prop in property_type] if property_type else None,
         proxy=proxy,
         radius=radius,
         mls_only=mls_only,
@@ -63,4 +66,6 @@ def scrape_property(
     with warnings.catch_warnings():
         warnings.simplefilter("ignore", category=FutureWarning)
 
-        return pd.concat(properties_dfs, ignore_index=True, axis=0)[ordered_properties].replace({"None": pd.NA, None: pd.NA, "": pd.NA})
+        return pd.concat(properties_dfs, ignore_index=True, axis=0)[ordered_properties].replace(
+            {"None": pd.NA, None: pd.NA, "": pd.NA}
+        )

homeharvest/core/scrapers/__init__.py

@@ -5,7 +5,7 @@ from requests.adapters import HTTPAdapter
 from urllib3.util.retry import Retry
 import uuid
 from ...exceptions import AuthenticationError
-from .models import Property, ListingType, SiteName
+from .models import Property, ListingType, SiteName, SearchPropertyType
 import json
 
 
@@ -13,6 +13,7 @@ import json
 class ScraperInput:
     location: str
     listing_type: ListingType
+    property_type: list[SearchPropertyType] | None = None
     radius: float | None = None
     mls_only: bool | None = False
     proxy: str | None = None
@@ -34,11 +35,12 @@ class Scraper:
     ):
         self.location = scraper_input.location
         self.listing_type = scraper_input.listing_type
+        self.property_type = scraper_input.property_type
 
         if not self.session:
             Scraper.session = requests.Session()
             retries = Retry(
-                total=3, backoff_factor=3, status_forcelist=[429, 403], allowed_methods=frozenset(["GET", "POST"])
+                total=3, backoff_factor=4, status_forcelist=[429, 403], allowed_methods=frozenset(["GET", "POST"])
             )
 
             adapter = HTTPAdapter(max_retries=retries)
@@ -46,8 +48,21 @@ class Scraper:
             Scraper.session.mount("https://", adapter)
             Scraper.session.headers.update(
                 {
-                    "auth": f"Bearer {self.get_access_token()}",
-                    "apollographql-client-name": "com.move.Realtor-apollo-ios",
+                    "accept": "application/json, text/javascript",
+                    "accept-language": "en-US,en;q=0.9",
+                    "cache-control": "no-cache",
+                    "content-type": "application/json",
+                    "origin": "https://www.realtor.com",
+                    "pragma": "no-cache",
+                    "priority": "u=1, i",
+                    "rdc-ab-tests": "commute_travel_time_variation:v1",
+                    "sec-ch-ua": '"Not)A;Brand";v="99", "Google Chrome";v="127", "Chromium";v="127"',
+                    "sec-ch-ua-mobile": "?0",
+                    "sec-ch-ua-platform": '"Windows"',
+                    "sec-fetch-dest": "empty",
+                    "sec-fetch-mode": "cors",
+                    "sec-fetch-site": "same-origin",
+                    "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36",
                 }
             )
 
@@ -81,27 +96,29 @@ class Scraper:
         response = requests.post(
             "https://graph.realtor.com/auth/token",
             headers={
-                'Host': 'graph.realtor.com',
-                'Accept': '*/*',
-                'Content-Type': 'Application/json',
-                'X-Client-ID': 'rdc_mobile_native,iphone',
-                'X-Visitor-ID': device_id,
-                'X-Client-Version': '24.21.23.679885',
-                'Accept-Language': 'en-US,en;q=0.9',
-                'User-Agent': 'Realtor.com/24.21.23.679885 CFNetwork/1494.0.7 Darwin/23.4.0',
+                "Host": "graph.realtor.com",
+                "Accept": "*/*",
+                "Content-Type": "Application/json",
+                "X-Client-ID": "rdc_mobile_native,iphone",
+                "X-Visitor-ID": device_id,
+                "X-Client-Version": "24.21.23.679885",
+                "Accept-Language": "en-US,en;q=0.9",
+                "User-Agent": "Realtor.com/24.21.23.679885 CFNetwork/1494.0.7 Darwin/23.4.0",
             },
-            data=json.dumps({
-                "grant_type": "device_mobile",
-                "device_id": device_id,
-                "client_app_id": "rdc_mobile_native,24.21.23.679885,iphone"
-            }))
+            data=json.dumps(
+                {
+                    "grant_type": "device_mobile",
+                    "device_id": device_id,
+                    "client_app_id": "rdc_mobile_native,24.21.23.679885,iphone",
+                }
+            ),
+        )
 
         data = response.json()
 
         if not (access_token := data.get("access_token")):
             raise AuthenticationError(
-                "Failed to get access token, use a proxy/vpn or wait a moment and try again.",
-                response=response
+                "Failed to get access token, use a proxy/vpn or wait a moment and try again.", response=response
             )
 
         return access_token

homeharvest/core/scrapers/models.py

@@ -17,6 +17,19 @@ class SiteName(Enum):
         raise ValueError(f"{value} not found in {cls}")
 
 
+class SearchPropertyType(Enum):
+    SINGLE_FAMILY = "single_family"
+    CONDOS = "condos"
+    CONDO_TOWNHOME_ROWHOME_COOP = "condo_townhome_rowhome_coop"
+    CONDO_TOWNHOME = "condo_townhome"
+    TOWNHOMES = "townhomes"
+    DUPLEX_TRIPLEX = "duplex_triplex"
+    FARM = "farm"
+    LAND = "land"
+    MULTI_FAMILY = "multi_family"
+    MOBILE = "mobile"
+
+
 class ListingType(Enum):
     FOR_SALE = "FOR_SALE"
     FOR_RENT = "FOR_RENT"
@@ -97,6 +110,8 @@ class Entity:
 
 @dataclass
 class Agent(Entity):
+    mls_set: str | None = None
+    nrds_id: str | None = None
     phones: list[dict] | AgentPhone | None = None
     email: str | None = None
     href: str | None = None
@@ -104,6 +119,7 @@ class Agent(Entity):
 
 @dataclass
 class Office(Entity):
+    mls_set: str | None = None
     email: str | None = None
     href: str | None = None
     phones: list[dict] | AgentPhone | None = None
@@ -130,6 +146,10 @@ class Advertisers:
 @dataclass
 class Property:
     property_url: str
+
+    property_id: str
+    listing_id: str | None = None
+
     mls: str | None = None
     mls_id: str | None = None
     status: str | None = None
@@ -156,5 +176,7 @@ class Property:
     nearby_schools: list[str] = None
     assessed_value: int | None = None
     estimated_value: int | None = None
+    tax: int | None = None
+    tax_history: list[dict] | None = None
 
     advertisers: Advertisers | None = None

homeharvest/core/scrapers/realtor/__init__.py

@@ -6,12 +6,33 @@ This module implements the scraper for realtor.com
 """
 
 from __future__ import annotations
+
+import json
 from concurrent.futures import ThreadPoolExecutor, as_completed
 from datetime import datetime
+from json import JSONDecodeError
 from typing import Dict, Union, Optional
 
+from tenacity import (
+    retry,
+    retry_if_exception_type,
+    wait_exponential,
+    stop_after_attempt,
+)
+
 from .. import Scraper
-from ..models import Property, Address, ListingType, Description, PropertyType, Agent, Broker, Builder, Advertisers, Office
+from ..models import (
+    Property,
+    Address,
+    ListingType,
+    Description,
+    PropertyType,
+    Agent,
+    Broker,
+    Builder,
+    Advertisers,
+    Office,
+)
 from .queries import GENERAL_RESULTS_QUERY, SEARCH_HOMES_DATA, HOMES_DATA
 
 
@@ -81,9 +102,12 @@ class RealtorScraper(Scraper):
             return property_info["listings"][0]["listing_id"]
 
     def handle_home(self, property_id: str) -> list[Property]:
-        query = """query Home($property_id: ID!) {
+        query = (
+            """query Home($property_id: ID!) {
                     home(property_id: $property_id) %s
-                }""" % HOMES_DATA
+                }"""
+            % HOMES_DATA
+        )
 
         variables = {"property_id": property_id}
         payload = {
@@ -96,9 +120,7 @@ class RealtorScraper(Scraper):
 
         property_info = response_json["data"]["home"]
 
-        return [
-            self.process_property(property_info, "home")
-        ]
+        return [self.process_property(property_info, "home")]
 
     @staticmethod
     def process_advertisers(advertisers: list[dict] | None) -> Advertisers | None:
@@ -114,13 +136,15 @@ class RealtorScraper(Scraper):
             advertiser_type = advertiser.get("type")
             if advertiser_type == "seller":  #: agent
                 processed_advertisers.agent = Agent(
-                    uuid=advertiser.get("mls_set"),
+                    uuid=_parse_fulfillment_id(advertiser.get("fulfillment_id")),
+                    nrds_id=advertiser.get("nrds_id"),
+                    mls_set=advertiser.get("mls_set"),
                     name=advertiser.get("name"),
                     email=advertiser.get("email"),
                     phones=advertiser.get("phones"),
                 )
 
-                if advertiser.get('broker') and advertiser["broker"].get('name'):  #: has a broker
+                if advertiser.get("broker") and advertiser["broker"].get("name"):  #: has a broker
                     processed_advertisers.broker = Broker(
                         uuid=_parse_fulfillment_id(advertiser["broker"].get("fulfillment_id")),
                         name=advertiser["broker"].get("name"),
@@ -128,7 +152,8 @@ class RealtorScraper(Scraper):
 
                 if advertiser.get("office"):  #: has an office
                     processed_advertisers.office = Office(
-                        uuid=_parse_fulfillment_id(advertiser["office"].get("fulfillment_id")) or advertiser["office"].get("mls_set"),
+                        uuid=_parse_fulfillment_id(advertiser["office"].get("fulfillment_id")),
+                        mls_set=advertiser["office"].get("mls_set"),
                         name=advertiser["office"].get("name"),
                         email=advertiser["office"].get("email"),
                         phones=advertiser["office"].get("phones"),
@@ -150,15 +175,16 @@ class RealtorScraper(Scraper):
             return
 
         able_to_get_lat_long = (
-                result
-                and result.get("location")
-                and result["location"].get("address")
-                and result["location"]["address"].get("coordinate")
+            result
+            and result.get("location")
+            and result["location"].get("address")
+            and result["location"]["address"].get("coordinate")
         )
 
-        is_pending = result["flags"].get("is_pending") or result["flags"].get("is_contingent")
+        is_pending = result["flags"].get("is_pending")
+        is_contingent = result["flags"].get("is_contingent")
 
-        if is_pending and (self.exclude_pending and self.listing_type != ListingType.PENDING):
+        if (is_pending or is_contingent) and (self.exclude_pending and self.listing_type != ListingType.PENDING):
             return
 
         property_id = result["property_id"]
@@ -178,32 +204,32 @@ class RealtorScraper(Scraper):
                 if "source" in result and isinstance(result["source"], dict)
                 else None
             ),
-            property_url=(
-                f"{self.PROPERTY_URL}{property_id}"
-                if self.listing_type != ListingType.FOR_RENT
-                else f"{self.PROPERTY_URL}M{property_id}?listing_status=rental"
-            ),
-            status="PENDING" if is_pending else result["status"].upper(),
+            property_url=result["href"],
+            property_id=property_id,
+            listing_id=result.get("listing_id"),
+            status=("PENDING" if is_pending else "CONTINGENT" if is_contingent else result["status"].upper()),
             list_price=result["list_price"],
             list_price_min=result["list_price_min"],
             list_price_max=result["list_price_max"],
-            list_date=result["list_date"].split("T")[0] if result.get("list_date") else None,
+            list_date=(result["list_date"].split("T")[0] if result.get("list_date") else None),
             prc_sqft=result.get("price_per_sqft"),
             last_sold_date=result.get("last_sold_date"),
             new_construction=result["flags"].get("is_new_construction") is True,
-            hoa_fee=result["hoa"]["fee"] if result.get("hoa") and isinstance(result["hoa"], dict) else None,
-            latitude=result["location"]["address"]["coordinate"].get("lat") if able_to_get_lat_long else None,
-            longitude=result["location"]["address"]["coordinate"].get("lon") if able_to_get_lat_long else None,
+            hoa_fee=(result["hoa"]["fee"] if result.get("hoa") and isinstance(result["hoa"], dict) else None),
+            latitude=(result["location"]["address"]["coordinate"].get("lat") if able_to_get_lat_long else None),
+            longitude=(result["location"]["address"]["coordinate"].get("lon") if able_to_get_lat_long else None),
             address=self._parse_address(result, search_type="general_search"),
             description=self._parse_description(result),
             neighborhoods=self._parse_neighborhoods(result),
-            county=result["location"]["county"].get("name") if result["location"]["county"] else None,
-            fips_code=result["location"]["county"].get("fips_code") if result["location"]["county"] else None,
+            county=(result["location"]["county"].get("name") if result["location"]["county"] else None),
+            fips_code=(result["location"]["county"].get("fips_code") if result["location"]["county"] else None),
             days_on_mls=self.calculate_days_on_mls(result),
             nearby_schools=prop_details.get("schools"),
             assessed_value=prop_details.get("assessed_value"),
             estimated_value=estimated_value if estimated_value else None,
             advertisers=advertisers,
+            tax=prop_details.get("tax"),
+            tax_history=prop_details.get("tax_history"),
         )
         return realty_property
 
@@ -224,6 +250,11 @@ class RealtorScraper(Scraper):
             elif self.last_x_days:
                 date_param = f'list_date: {{ min: "$today-{self.last_x_days}D" }}'
 
+        property_type_param = ""
+        if self.property_type:
+            property_types = [pt.value for pt in self.property_type]
+            property_type_param = f"type: {json.dumps(property_types)}"
+
         sort_param = (
             "sort: [{ field: sold_date, direction: desc }]"
             if self.listing_type == ListingType.SOLD
@@ -258,6 +289,7 @@ class RealtorScraper(Scraper):
                                 status: %s
                                 %s
                                 %s
+                                %s
                             }
                             %s
                             limit: 200
@@ -267,6 +299,7 @@ class RealtorScraper(Scraper):
                 is_foreclosure,
                 listing_type.value.lower(),
                 date_param,
+                property_type_param,
                 pending_or_contingent_param,
                 sort_param,
                 GENERAL_RESULTS_QUERY,
@@ -289,6 +322,7 @@ class RealtorScraper(Scraper):
                                         status: %s
                                         %s
                                         %s
+                                        %s
                                     }
                                     %s
                                     limit: 200
@@ -298,13 +332,14 @@ class RealtorScraper(Scraper):
                 is_foreclosure,
                 listing_type.value.lower(),
                 date_param,
+                property_type_param,
                 pending_or_contingent_param,
                 sort_param,
                 GENERAL_RESULTS_QUERY,
             )
         else:  #: general search, came from an address
             query = (
-                    """query Property_search(
+                """query Property_search(
                         $property_id: [ID]!
                         $offset: Int!,
                     ) {
@@ -314,9 +349,9 @@ class RealtorScraper(Scraper):
                             }
                             limit: 1
                             offset: $offset
-                        ) %s 
+                        ) %s
                     }"""
-                    % GENERAL_RESULTS_QUERY
+                % GENERAL_RESULTS_QUERY
             )
 
         payload = {
@@ -331,12 +366,12 @@ class RealtorScraper(Scraper):
         properties: list[Property] = []
 
         if (
-                response_json is None
-                or "data" not in response_json
-                or response_json["data"] is None
-                or search_key not in response_json["data"]
-                or response_json["data"][search_key] is None
-                or "results" not in response_json["data"][search_key]
+            response_json is None
+            or "data" not in response_json
+            or response_json["data"] is None
+            or search_key not in response_json["data"]
+            or response_json["data"][search_key] is None
+            or "results" not in response_json["data"][search_key]
         ):
             return {"total": 0, "properties": []}
 
@@ -346,12 +381,10 @@ class RealtorScraper(Scraper):
 
         #: limit the number of properties to be processed
         #: example, if your offset is 200, and your limit is 250, return 50
-        properties_list = properties_list[:self.limit - offset]
+        properties_list = properties_list[: self.limit - offset]
 
         with ThreadPoolExecutor(max_workers=self.NUM_PROPERTY_WORKERS) as executor:
-            futures = [
-                executor.submit(self.process_property, result, search_key) for result in properties_list
-            ]
+            futures = [executor.submit(self.process_property, result, search_key) for result in properties_list]
 
             for future in as_completed(futures):
                 result = future.result()
@@ -421,7 +454,11 @@ class RealtorScraper(Scraper):
                     variables=search_variables | {"offset": i},
                     search_type=search_type,
                 )
-                for i in range(self.DEFAULT_PAGE_SIZE, min(total, self.limit), self.DEFAULT_PAGE_SIZE)
+                for i in range(
+                    self.DEFAULT_PAGE_SIZE,
+                    min(total, self.limit),
+                    self.DEFAULT_PAGE_SIZE,
+                )
             ]
 
             for future in as_completed(futures):
@@ -443,13 +480,46 @@ class RealtorScraper(Scraper):
     def process_extra_property_details(self, result: dict) -> dict:
         schools = self.get_key(result, ["nearbySchools", "schools"])
         assessed_value = self.get_key(result, ["taxHistory", 0, "assessment", "total"])
+        tax_history = self.get_key(result, ["taxHistory"])
 
         schools = [school["district"]["name"] for school in schools if school["district"].get("name")]
+
+        # Process tax history
+        latest_tax = None
+        processed_tax_history = None
+        if tax_history and isinstance(tax_history, list):
+            tax_history = sorted(tax_history, key=lambda x: x.get("year", 0), reverse=True)
+
+            if tax_history and "tax" in tax_history[0]:
+                latest_tax = tax_history[0]["tax"]
+
+            processed_tax_history = []
+            for entry in tax_history:
+                if "year" in entry and "tax" in entry:
+                    processed_entry = {
+                        "year": entry["year"],
+                        "tax": entry["tax"],
+                    }
+                    if "assessment" in entry and isinstance(entry["assessment"], dict):
+                        processed_entry["assessment"] = {
+                            "building": entry["assessment"].get("building"),
+                            "land": entry["assessment"].get("land"),
+                            "total": entry["assessment"].get("total"),
+                        }
+                    processed_tax_history.append(processed_entry)
+
         return {
             "schools": schools if schools else None,
             "assessed_value": assessed_value if assessed_value else None,
+            "tax": latest_tax,
+            "tax_history": processed_tax_history,
         }
 
+    @retry(
+        retry=retry_if_exception_type(JSONDecodeError),
+        wait=wait_exponential(min=4, max=10),
+        stop=stop_after_attempt(3),
+    )
     def get_prop_details(self, property_id: str) -> dict:
         if not self.extra_property_data:
             return {}
@@ -466,7 +536,7 @@ class RealtorScraper(Scraper):
                 }"""
 
         variables = {"property_id": property_id}
-        response = self.session.post(self.PROPERTY_GQL, json={"query": query, "variables": variables})
+        response = self.session.post(self.SEARCH_GQL_URL, json={"query": query, "variables": variables})
 
         data = response.json()
         property_details = data["data"]["home"]
@@ -533,20 +603,22 @@ class RealtorScraper(Scraper):
             style = style.upper()
 
         primary_photo = ""
-        if (primary_photo_info := result.get('primary_photo')) and (primary_photo_href := primary_photo_info.get("href")):
+        if (primary_photo_info := result.get("primary_photo")) and (
+            primary_photo_href := primary_photo_info.get("href")
+        ):
             primary_photo = primary_photo_href.replace("s.jpg", "od-w480_h360_x2.webp?w=1080&q=75")
 
         return Description(
             primary_photo=primary_photo,
             alt_photos=RealtorScraper.process_alt_photos(result.get("photos", [])),
-            style=PropertyType.__getitem__(style) if style and style in PropertyType.__members__ else None,
+            style=(PropertyType.__getitem__(style) if style and style in PropertyType.__members__ else None),
             beds=description_data.get("beds"),
             baths_full=description_data.get("baths_full"),
             baths_half=description_data.get("baths_half"),
             sqft=description_data.get("sqft"),
             lot_sqft=description_data.get("lot_sqft"),
             sold_price=(
-                result.get('last_sold_price') or description_data.get("sold_price")
+                result.get("last_sold_price") or description_data.get("sold_price")
                 if result.get("last_sold_date") or result["list_price"] != description_data.get("sold_price")
                 else None
             ),  #: has a sold date or list and sold price are different
@@ -580,4 +652,8 @@ class RealtorScraper(Scraper):
         if not photos_info:
             return None
 
-        return [photo_info["href"].replace("s.jpg", "od-w480_h360_x2.webp?w=1080&q=75") for photo_info in photos_info if photo_info.get("href")]
+        return [
+            photo_info["href"].replace("s.jpg", "od-w480_h360_x2.webp?w=1080&q=75")
+            for photo_info in photos_info
+            if photo_info.get("href")
+        ]

homeharvest/core/scrapers/realtor/queries.py

@@ -2,6 +2,7 @@ _SEARCH_HOMES_DATA_BASE = """{
     pending_date
     listing_id
     property_id
+    href
     list_date
     status
     last_sold_price
@@ -109,6 +110,7 @@ _SEARCH_HOMES_DATA_BASE = """{
             fulfillment_id
         }
         mls_set
+        nrds_id
         rental_corporation {
             fulfillment_id
         }

homeharvest/utils.py

@@ -6,6 +6,8 @@ from .exceptions import InvalidListingType, InvalidDate
 
 ordered_properties = [
     "property_url",
+    "property_id",
+    "listing_id",
     "mls",
     "mls_id",
     "status",
@@ -31,6 +33,8 @@ ordered_properties = [
     "last_sold_date",
     "assessed_value",
     "estimated_value",
+    "tax",
+    "tax_history",
     "new_construction",
     "lot_sqft",
     "price_per_sqft",
@@ -46,11 +50,14 @@ ordered_properties = [
     "agent_name",
     "agent_email",
     "agent_phones",
+    "agent_mls_set",
+    "agent_nrds_id",
     "broker_id",
     "broker_name",
     "builder_id",
     "builder_name",
     "office_id",
+    "office_mls_set",
     "office_name",
     "office_email",
     "office_phones",
@@ -81,6 +88,8 @@ def process_result(result: Property) -> pd.DataFrame:
             prop_data["agent_name"] = agent_data.name
             prop_data["agent_email"] = agent_data.email
             prop_data["agent_phones"] = agent_data.phones
+            prop_data["agent_mls_set"] = agent_data.mls_set
+            prop_data["agent_nrds_id"] = agent_data.nrds_id
 
         if advertiser_data.broker:
             broker_data = advertiser_data.broker
@@ -98,6 +107,7 @@ def process_result(result: Property) -> pd.DataFrame:
             prop_data["office_name"] = office_data.name
             prop_data["office_email"] = office_data.email
             prop_data["office_phones"] = office_data.phones
+            prop_data["office_mls_set"] = office_data.mls_set
 
     prop_data["price_per_sqft"] = prop_data["prc_sqft"]
     prop_data["nearby_schools"] = filter(None, prop_data["nearby_schools"]) if prop_data["nearby_schools"] else None
@@ -107,8 +117,11 @@ def process_result(result: Property) -> pd.DataFrame:
     if description:
         prop_data["primary_photo"] = description.primary_photo
         prop_data["alt_photos"] = ", ".join(description.alt_photos) if description.alt_photos else None
-        prop_data["style"] = description.style if isinstance(description.style,
-                                                             str) else description.style.value if description.style else None
+        prop_data["style"] = (
+            description.style
+            if isinstance(description.style, str)
+            else description.style.value if description.style else None
+        )
         prop_data["beds"] = description.beds
         prop_data["full_baths"] = description.baths_full
         prop_data["half_baths"] = description.baths_half

poetry.lock

@@ -1,4 +1,4 @@
-# This file is automatically @generated by Poetry 1.8.2 and should not be changed by hand.
+# This file is automatically @generated by Poetry 1.8.4 and should not be changed by hand.
 
 [[package]]
 name = "annotated-types"
@@ -667,6 +667,21 @@ files = [
     {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"},
 ]
 
+[[package]]
+name = "tenacity"
+version = "9.0.0"
+description = "Retry code until it succeeds"
+optional = false
+python-versions = ">=3.8"
+files = [
+    {file = "tenacity-9.0.0-py3-none-any.whl", hash = "sha256:93de0c98785b27fcf659856aa9f54bfbd399e29969b0621bc7f762bd441b4539"},
+    {file = "tenacity-9.0.0.tar.gz", hash = "sha256:807f37ca97d62aa361264d497b0e31e92b8027044942bfa756160d908320d73b"},
+]
+
+[package.extras]
+doc = ["reno", "sphinx"]
+test = ["pytest", "tornado (>=4.5)", "typeguard"]
+
 [[package]]
 name = "tomli"
 version = "2.0.1"
@@ -740,4 +755,4 @@ test = ["covdefaults (>=2.3)", "coverage (>=7.2.7)", "coverage-enable-subprocess
 [metadata]
 lock-version = "2.0"
 python-versions = ">=3.9,<3.13"
-content-hash = "21ef9cfb35c446a375a2b74c37691d7031afb1e4f66a8b63cb7c1669470689d2"
+content-hash = "cefc11b1bf5ad99d628f6d08f6f03003522cc1b6e48b519230d99d716a5c165c"

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "homeharvest"
-version = "0.4.0"
+version = "0.4.5"
 description = "Real estate scraping library"
 authors = ["Zachary Hampton <zachary@bunsly.com>", "Cullen Watson <cullen@bunsly.com>"]
 homepage = "https://github.com/Bunsly/HomeHarvest"
@@ -14,6 +14,7 @@ python = ">=3.9,<3.13"
 requests = "^2.31.0"
 pandas = "^2.1.1"
 pydantic = "^2.7.4"
+tenacity = "^9.0.0"
 
 
 [tool.poetry.group.dev.dependencies]

tests/test_realtor.py

@@ -105,8 +105,12 @@ def test_realtor():
             location="2530 Al Lipscomb Way",
             listing_type="for_sale",
         ),
-        scrape_property(location="Phoenix, AZ", listing_type="for_rent", limit=1000),  #: does not support "city, state, USA" format
-        scrape_property(location="Dallas, TX", listing_type="sold", limit=1000),  #: does not support "city, state, USA" format
+        scrape_property(
+            location="Phoenix, AZ", listing_type="for_rent", limit=1000
+        ),  #: does not support "city, state, USA" format
+        scrape_property(
+            location="Dallas, TX", listing_type="sold", limit=1000
+        ),  #: does not support "city, state, USA" format
         scrape_property(location="85281"),
     ]
 
@@ -114,11 +118,13 @@ def test_realtor():
 
 
 def test_realtor_city():
-    results = scrape_property(
-        location="Atlanta, GA",
-        listing_type="for_sale",
-        limit=1000
-    )
+    results = scrape_property(location="Atlanta, GA", listing_type="for_sale", limit=1000)
+
+    assert results is not None and len(results) > 0
+
+
+def test_realtor_land():
+    results = scrape_property(location="Atlanta, GA", listing_type="for_sale", property_type=["land"], limit=1000)
 
     assert results is not None and len(results) > 0
 
@@ -128,6 +134,7 @@ def test_realtor_bad_address():
         location="abceefg ju098ot498hh9",
         listing_type="for_sale",
     )
+
     if len(bad_results) == 0:
         assert True
 
@@ -240,6 +247,43 @@ def test_apartment_list_price():
     results = results[results["style"] == "APARTMENT"]
 
     #: get percentage of results with atleast 1 of any column not none, list_price, list_price_min, list_price_max
-    assert len(results[results[["list_price", "list_price_min", "list_price_max"]].notnull().any(axis=1)]) / len(
-        results
-    ) > 0.5
+    assert (
+        len(results[results[["list_price", "list_price_min", "list_price_max"]].notnull().any(axis=1)]) / len(results)
+        > 0.5
+    )
+
+
+def test_builder_exists():
+    listing = scrape_property(
+        location="18149 W Poston Dr, Surprise, AZ 85387",
+        extra_property_data=False,
+    )
+
+    assert listing is not None
+    assert listing["builder_name"].nunique() > 0
+
+
+def test_phone_number_matching():
+    searches = [
+        scrape_property(
+            location="Phoenix, AZ",
+            listing_type="for_sale",
+            limit=100,
+        ),
+        scrape_property(
+            location="Phoenix, AZ",
+            listing_type="for_sale",
+            limit=100,
+        ),
+    ]
+
+    assert all([search is not None for search in searches])
+
+    #: random row
+    row = searches[0][searches[0]["agent_phones"].notnull()].sample()
+
+    #: find matching row
+    matching_row = searches[1].loc[searches[1]["property_url"] == row["property_url"].values[0]]
+
+    #: assert phone numbers are the same
+    assert row["agent_phones"].values[0] == matching_row["agent_phones"].values[0]