Version bump to 0.8.17

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>

homeharvest/core/scrapers/__init__.py

@@ -73,35 +73,37 @@ class Scraper:
         if not self.session:
             Scraper.session = requests.Session()
             retries = Retry(
-                total=3, backoff_factor=4, status_forcelist=[429, 403], allowed_methods=frozenset(["GET", "POST"])
+                total=3, backoff_factor=4, status_forcelist=[429], allowed_methods=frozenset(["GET", "POST"])
             )
 
-            adapter = HTTPAdapter(max_retries=retries)
+            adapter = HTTPAdapter(max_retries=retries, pool_connections=10, pool_maxsize=20)
             Scraper.session.mount("http://", adapter)
             Scraper.session.mount("https://", adapter)
             Scraper.session.headers.update(
                 {
-                    'sec-ch-ua-platform': '"macOS"',
-                    'rdc-client-name': 'rdc-search-for-sale-desktop',
-                    'sec-ch-ua': '"Chromium";v="142", "Google Chrome";v="142", "Not_A Brand";v="99"',
+                    'Content-Type': 'application/json',
+                    'Accept': '*/*',
+                    'Accept-Language': 'en-US,en;q=0.9',
+                    'Cache-Control': 'no-cache',
+                    'Origin': 'https://www.realtor.com',
+                    'Pragma': 'no-cache',
+                    'Referer': 'https://www.realtor.com/',
+                    'rdc-client-name': 'RDC_WEB_SRP_FS_PAGE',
+                    'rdc-client-version': '3.0.2515',
+                    'sec-ch-ua': '"Google Chrome";v="135", "Not-A.Brand";v="8", "Chromium";v="135"',
                     'sec-ch-ua-mobile': '?0',
-                    'rdc-client-version': '0.1.0',
-                    'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/142.0.0.0 Safari/537.36',
-                    'accept': 'application/json',
-                    'content-type': 'application/json',
-                    'origin': 'https://www.realtor.com',
-                    'sec-fetch-site': 'same-site',
-                    'sec-fetch-mode': 'cors',
+                    'sec-ch-ua-platform': '"macOS"',
                     'sec-fetch-dest': 'empty',
-                    'referer': 'https://www.realtor.com/',
-                    'accept-language': 'en-US,en;q=0.9',
-                    'priority': 'u=1, i',
+                    'sec-fetch-mode': 'cors',
+                    'sec-fetch-site': 'same-site',
+                    'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/135.0.0.0 Safari/537.36',
+                    'x-is-bot': 'false',
                 }
             )
 
-        if scraper_input.proxy:
-            proxy_url = scraper_input.proxy
-            proxies = {"http": proxy_url, "https": proxy_url}
+        self.proxy = scraper_input.proxy
+        if self.proxy:
+            proxies = {"http": self.proxy, "https": self.proxy}
             self.session.proxies.update(proxies)
 
         self.listing_type = scraper_input.listing_type

homeharvest/core/scrapers/realtor/__init__.py

@@ -8,6 +8,7 @@ This module implements the scraper for realtor.com
 from __future__ import annotations
 
 import json
+import re
 from concurrent.futures import ThreadPoolExecutor, as_completed
 from datetime import datetime
 from json import JSONDecodeError
@@ -16,17 +17,19 @@ from typing import Dict, Union
 from tenacity import (
     retry,
     retry_if_exception_type,
+    retry_if_not_exception_type,
     wait_exponential,
     stop_after_attempt,
 )
 
 from .. import Scraper
+from ....exceptions import AuthenticationError
 from ..models import (
     Property,
     ListingType,
     ReturnType
 )
-from .queries import GENERAL_RESULTS_QUERY, SEARCH_HOMES_DATA, HOMES_DATA, HOME_FRAGMENT
+from .queries import GENERAL_RESULTS_QUERY, HOMES_DATA, SEARCH_SUGGESTIONS_QUERY
 from .processors import (
     process_property,
     process_extra_property_details,
@@ -35,47 +38,63 @@ from .processors import (
 
 
 class RealtorScraper(Scraper):
-    SEARCH_GQL_URL = "https://api.frontdoor.realtor.com/graphql"
+    SEARCH_GQL_URL = "https://www.realtor.com/frontdoor/graphql"
     NUM_PROPERTY_WORKERS = 20
     DEFAULT_PAGE_SIZE = 200
 
     def __init__(self, scraper_input):
         super().__init__(scraper_input)
 
-    def handle_location(self):
-        query = """query Search_suggestions($searchInput: SearchSuggestionsInput!) {
-            search_suggestions(search_input: $searchInput) {
-                geo_results {
-                    type
-                    text
-                    geo {
-                        _id
-                        area_type
-                        city
-                        state_code
-                        postal_code
-                        county
-                        centroid { lat lon }
-                        slug_id
-                        geo_id
-                    }
-                }
-            }
-        }"""
+    @staticmethod
+    def _minify_query(query: str) -> str:
+        """Minify GraphQL query by collapsing whitespace to single spaces."""
+        # Split on whitespace, filter empty strings, join with single space
+        return ' '.join(query.split())
 
+    def _graphql_post(self, query: str, variables: dict, operation_name: str) -> dict:
+        """
+        Execute a GraphQL query.
+
+        Args:
+            query: GraphQL query string (must include operationName matching operation_name param)
+            variables: Query variables dictionary
+            operation_name: Name of the GraphQL operation
+
+        Returns:
+            Response JSON dictionary
+        """
+        payload = {
+            "operationName": operation_name,
+            "query": self._minify_query(query),
+            "variables": variables,
+        }
+
+        response = self.session.post(self.SEARCH_GQL_URL, data=json.dumps(payload, separators=(',', ':')))
+
+        if response.status_code == 403:
+            if not self.proxy:
+                raise AuthenticationError(
+                    "Received 403 Forbidden from Realtor.com API.",
+                    response=response
+                )
+            else:
+                raise Exception("Received 403 Forbidden, retrying...")
+
+        return response.json()
+
+    @retry(
+        retry=retry_if_exception_type(Exception),
+        wait=wait_exponential(multiplier=1, min=1, max=4),
+        stop=stop_after_attempt(3),
+    )
+    def handle_location(self):
         variables = {
             "searchInput": {
                 "search_term": self.location
             }
         }
 
-        payload = {
-            "query": query,
-            "variables": variables,
-        }
-
-        response = self.session.post(self.SEARCH_GQL_URL, json=payload)
-        response_json = response.json()
+        response_json = self._graphql_post(SEARCH_SUGGESTIONS_QUERY, variables, "Search_suggestions")
 
         if (
             response_json is None
@@ -86,6 +105,11 @@ class RealtorScraper(Scraper):
             or "geo_results" not in response_json["data"]["search_suggestions"]
             or not response_json["data"]["search_suggestions"]["geo_results"]
         ):
+            # If we got a 400 error with "Required parameter is missing", raise to trigger retry
+            if response_json and "errors" in response_json:
+                error_msgs = [e.get("message", "") for e in response_json.get("errors", [])]
+                if any("Required parameter is missing" in msg for msg in error_msgs):
+                    raise Exception(f"Transient API error: {error_msgs}")
             return None
 
         geo_result = response_json["data"]["search_suggestions"]["geo_results"][0]
@@ -102,31 +126,34 @@ class RealtorScraper(Scraper):
         }
 
         if geo.get("area_type") == "address":
-            geo_id = geo.get("_id", "")
-            if geo_id.startswith("addr:"):
-                result["mpr_id"] = geo_id.replace("addr:", "")
+            # Try to get mpr_id directly from API response first
+            if geo.get("mpr_id"):
+                result["mpr_id"] = geo.get("mpr_id")
+            else:
+                # Fallback: extract from _id field if it has addr: prefix
+                geo_id = geo.get("_id", "")
+                if geo_id.startswith("addr:"):
+                    result["mpr_id"] = geo_id.replace("addr:", "")
 
         return result
 
     def get_latest_listing_id(self, property_id: str) -> str | None:
-        query = """query Property($property_id: ID!) {
+        query = """
+                fragment ListingFragment on Listing {
+                    listing_id
+                    primary
+                }
+                query GetPropertyListingId($property_id: ID!) {
                     property(id: $property_id) {
                         listings {
-                            listing_id
-                            primary
+                            ...ListingFragment
                         }
                     }
                 }
                 """
 
         variables = {"property_id": property_id}
-        payload = {
-            "query": query,
-            "variables": variables,
-        }
-
-        response = self.session.post(self.SEARCH_GQL_URL, json=payload)
-        response_json = response.json()
+        response_json = self._graphql_post(query, variables, "GetPropertyListingId")
 
         property_info = response_json["data"]["property"]
         if property_info["listings"] is None:
@@ -144,18 +171,16 @@ class RealtorScraper(Scraper):
     def handle_home(self, property_id: str) -> list[Property]:
         """Fetch single home with proper error handling."""
         query = (
-            """query Home($property_id: ID!) {
+            """query GetHomeDetails($property_id: ID!) {
                     home(property_id: $property_id) %s
                 }"""
             % HOMES_DATA
         )
 
         variables = {"property_id": property_id}
-        payload = {"query": query, "variables": variables}
 
         try:
-            response = self.session.post(self.SEARCH_GQL_URL, json=payload)
-            data = response.json()
+            data = self._graphql_post(query, variables, "GetHomeDetails")
 
             # Check for errors or missing data
             if "errors" in data or "data" not in data:
@@ -374,12 +399,12 @@ class RealtorScraper(Scraper):
             is_foreclosure = "foreclosure: false"
 
         if search_type == "comps":  #: comps search, came from an address
-            query = """query Property_search(
+            query = """query GetHomeSearch(
                     $coordinates: [Float]!
                     $radius: String!
                     $offset: Int!,
                     ) {
-                        home_search(
+                        homeSearch: home_search(
                             query: {
                                 %s
                                 nearby: {
@@ -407,11 +432,11 @@ class RealtorScraper(Scraper):
                 GENERAL_RESULTS_QUERY,
             )
         elif search_type == "area":  #: general search, came from a general location
-            query = """query Home_search(
+            query = """query GetHomeSearch(
                                 $search_location: SearchLocation,
-                                $offset: Int,
+                                $offset: Int
                             ) {
-                                home_search(
+                                homeSearch: home_search(
                                     query: {
                                         %s
                                         search_location: $search_location
@@ -439,11 +464,11 @@ class RealtorScraper(Scraper):
             )
         else:  #: general search, came from an address
             query = (
-                """query Property_search(
+                """query GetHomeSearch(
                         $property_id: [ID]!
                         $offset: Int!,
                     ) {
-                        home_search(
+                        homeSearch: home_search(
                             query: {
                                 property_id: $property_id
                             }
@@ -454,14 +479,8 @@ class RealtorScraper(Scraper):
                 % GENERAL_RESULTS_QUERY
             )
 
-        payload = {
-            "query": query,
-            "variables": variables,
-        }
-
-        response = self.session.post(self.SEARCH_GQL_URL, json=payload)
-        response_json = response.json()
-        search_key = "home_search" if "home_search" in query else "property_search"
+        response_json = self._graphql_post(query, variables, "GetHomeSearch")
+        search_key = "homeSearch"
 
         properties: list[Union[Property, dict]] = []
 
@@ -1081,8 +1100,8 @@ class RealtorScraper(Scraper):
 
 
     @retry(
-        retry=retry_if_exception_type(JSONDecodeError),
-        wait=wait_exponential(min=4, max=10),
+        retry=retry_if_exception_type((JSONDecodeError, Exception)) & retry_if_not_exception_type(AuthenticationError),
+        wait=wait_exponential(multiplier=1, min=1, max=10),
         stop=stop_after_attempt(3),
     )
     def get_bulk_prop_details(self, property_ids: list[str]) -> dict:
@@ -1095,24 +1114,25 @@ class RealtorScraper(Scraper):
 
         property_ids = list(set(property_ids))
 
-        # Construct the bulk query
         fragments = "\n".join(
-            f'home_{property_id}: home(property_id: {property_id}) {{ ...HomeData }}'
+            f'home_{property_id}: home(property_id: {property_id}) {HOMES_DATA}'
             for property_id in property_ids
         )
-        query = f"""{HOME_FRAGMENT}
+        query = f"""query GetHome {{
+    {fragments}
+}}"""
 
-        query GetHomes {{
-            {fragments}
-        }}"""
+        data = self._graphql_post(query, {}, "GetHome")
 
-        response = self.session.post(self.SEARCH_GQL_URL, json={"query": query})
-        data = response.json()
-
-        if "data" not in data:
+        if "data" not in data or data["data"] is None:
+            # If we got a 400 error with "Required parameter is missing", raise to trigger retry
+            if data and "errors" in data:
+                error_msgs = [e.get("message", "") for e in data.get("errors", [])]
+                if any("Required parameter is missing" in msg for msg in error_msgs):
+                    raise Exception(f"Transient API error: {error_msgs}")
             return {}
 
         properties = data["data"]
-        return {data.replace('home_', ''): properties[data] for data in properties if properties[data]}
+        return {key.replace('home_', ''): properties[key] for key in properties if properties[key]}
 
 

homeharvest/core/scrapers/realtor/queries.py

@@ -1,3 +1,193 @@
+SEARCH_RESULTS_FRAGMENT = """
+fragment PropertyResult on SearchHome {
+    __typename
+    pending_date
+    listing_id
+    property_id
+    href
+    permalink
+    list_date
+    status
+    mls_status
+    last_sold_price
+    last_sold_date
+    last_status_change_date
+    last_update_date
+    list_price
+    list_price_max
+    list_price_min
+    price_per_sqft
+    tags
+    open_houses {
+        start_date
+        end_date
+        description
+        time_zone
+        dst
+        href
+        methods
+    }
+    details {
+        category
+        text
+        parent_category
+    }
+    pet_policy {
+        cats
+        dogs
+        dogs_small
+        dogs_large
+        __typename
+    }
+    units {
+        availability {
+          date
+          __typename
+        }
+        description {
+          baths_consolidated
+          baths
+          beds
+          sqft
+          __typename
+        }
+        photos(https: true) {
+            title
+            href
+            tags {
+                label
+            }
+        }
+        list_price
+        __typename
+    }
+    flags {
+        is_contingent
+        is_pending
+        is_new_construction
+    }
+    description {
+        type
+        sqft
+        beds
+        baths_full
+        baths_half
+        lot_sqft
+        year_built
+        garage
+        type
+        name
+        stories
+        text
+    }
+    source {
+        id
+        listing_id
+    }
+    hoa {
+        fee
+    }
+    location {
+        address {
+            street_direction
+            street_number
+            street_name
+            street_suffix
+            line
+            unit
+            city
+            state_code
+            postal_code
+            coordinate {
+                lon
+                lat
+            }
+        }
+        county {
+            name
+            fips_code
+        }
+        neighborhoods {
+            name
+        }
+    }
+    tax_record {
+        cl_id
+        public_record_id
+        last_update_date
+        apn
+        tax_parcel_id
+    }
+    primary_photo(https: true) {
+        href
+    }
+    advertisers {
+        email
+        broker {
+            name
+            fulfillment_id
+        }
+        type
+        name
+        fulfillment_id
+        builder {
+            name
+            fulfillment_id
+        }
+        phones {
+            ext
+            primary
+            type
+            number
+        }
+        office {
+            name
+            email
+            fulfillment_id
+            href
+            phones {
+                number
+                type
+                primary
+                ext
+            }
+            mls_set
+        }
+        corporation {
+            specialties
+            name
+            bio
+            href
+            fulfillment_id
+        }
+        mls_set
+        nrds_id
+        state_license
+        rental_corporation {
+            fulfillment_id
+        }
+        rental_management {
+            name
+            href
+            fulfillment_id
+        }
+    }
+    current_estimates {
+        __typename
+        source {
+            __typename
+            type
+            name
+        }
+        estimate
+        estimateHigh: estimate_high
+        estimateLow: estimate_low
+        date
+        isBestHomeValue: isbest_homevalue
+    }
+}
+"""
+
 _SEARCH_HOMES_DATA_BASE = """{
     pending_date
     listing_id
@@ -181,8 +371,189 @@ _SEARCH_HOMES_DATA_BASE = """{
 
 
 HOME_FRAGMENT = """
-fragment HomeData on Home {
+fragment PropertyResult on Home {
+    __typename
+    pending_date
+    listing_id
     property_id
+    href
+    permalink
+    list_date
+    status
+    mls_status
+    last_sold_price
+    last_sold_date
+    last_status_change_date
+    last_update_date
+    list_price
+    list_price_max
+    list_price_min
+    price_per_sqft
+    tags
+    open_houses {
+        start_date
+        end_date
+        description
+        time_zone
+        dst
+        href
+        methods
+    }
+    details {
+        category
+        text
+        parent_category
+    }
+    pet_policy {
+        cats
+        dogs
+        dogs_small
+        dogs_large
+        __typename
+    }
+    units {
+        availability {
+          date
+          __typename
+        }
+        description {
+          baths_consolidated
+          baths
+          beds
+          sqft
+          __typename
+        }
+        photos(https: true) {
+            title
+            href
+            tags {
+                label
+            }
+        }
+        list_price
+        __typename
+    }
+    flags {
+        is_contingent
+        is_pending
+        is_new_construction
+    }
+    description {
+        type
+        sqft
+        beds
+        baths_full
+        baths_half
+        lot_sqft
+        year_built
+        garage
+        type
+        name
+        stories
+        text
+    }
+    source {
+        id
+        listing_id
+    }
+    hoa {
+        fee
+    }
+    location {
+        address {
+            street_direction
+            street_number
+            street_name
+            street_suffix
+            line
+            unit
+            city
+            state_code
+            postal_code
+            coordinate {
+                lon
+                lat
+            }
+        }
+        county {
+            name
+            fips_code
+        }
+        neighborhoods {
+            name
+        }
+        parcel {
+            parcel_id
+        }
+    }
+    tax_record {
+        cl_id
+        public_record_id
+        last_update_date
+        apn
+        tax_parcel_id
+    }
+    primary_photo(https: true) {
+        href
+    }
+    photos(https: true) {
+        title
+        href
+        tags {
+            label
+        }
+    }
+    advertisers {
+        email
+        broker {
+            name
+            fulfillment_id
+        }
+        type
+        name
+        fulfillment_id
+        builder {
+            name
+            fulfillment_id
+        }
+        phones {
+            ext
+            primary
+            type
+            number
+        }
+        office {
+            name
+            email
+            fulfillment_id
+            href
+            phones {
+                number
+                type
+                primary
+                ext
+            }
+            mls_set
+        }
+        corporation {
+            specialties
+            name
+            bio
+            href
+            fulfillment_id
+        }
+        mls_set
+        nrds_id
+        state_license
+        rental_corporation {
+            fulfillment_id
+        }
+        rental_management {
+            name
+            href
+            fulfillment_id
+        }
+    }
     nearbySchools: nearby_schools(radius: 5.0, limit_per_level: 3) {
         __typename schools { district { __typename id name } }
     }
@@ -198,11 +569,6 @@ fragment HomeData on Home {
             last_n_days
         }
     }
-    location {
-        parcel {
-            parcel_id
-        }
-    }
     taxHistory: tax_history { __typename tax year assessment { __typename building land total } }
     property_history {
         date
@@ -227,6 +593,18 @@ fragment HomeData on Home {
         text
         category
     }
+    estimates {
+        __typename
+        currentValues: current_values {
+            __typename
+            source { __typename type name }
+            estimate
+            estimateHigh: estimate_high
+            estimateLow: estimate_low
+            date
+            isBestHomeValue: isbest_homevalue
+        }
+    }
 }
 """
 
@@ -300,8 +678,128 @@ current_estimates {
 }
 }""" % _SEARCH_HOMES_DATA_BASE
 
-GENERAL_RESULTS_QUERY = """{
+# Query body using inline fields (kept for backward compatibility)
+GENERAL_RESULTS_QUERY_BODY = """{
                             count
                             total
                             results %s
                         }""" % SEARCH_HOMES_DATA
+
+GENERAL_RESULTS_QUERY = """{
+                            __typename
+                            count
+                            total
+                            results %s
+                        }""" % SEARCH_HOMES_DATA
+
+LISTING_PHOTOS_FRAGMENT = """
+fragment ListingPhotosFragment on SearchHome {
+    __typename
+    photos(https: true) {
+        __typename
+        title
+        href
+        tags {
+            __typename
+            label
+            probability
+        }
+    }
+}
+"""
+
+SEARCH_SUGGESTIONS_QUERY = """query Search_suggestions($searchInput: SearchSuggestionsInput!) {
+  search_suggestions(search_input: $searchInput) {
+    raw_input_parser_result
+    typeahead_results {
+      display_string
+      display_geo
+      geo {
+        _id
+        _score
+        mpr_id
+        area_type
+        city
+        state_code
+        state
+        postal_code
+        country
+        lat
+        lon
+        county
+        counties {
+          name
+          fips
+          state_code
+        }
+        slug_id
+        geo_id
+        score
+        name
+        city_slug_id
+        centroid {
+          lat
+          lon
+        }
+        county_needed_for_uniq
+        street
+        line
+        school
+        school_id
+        school_district
+        has_catchment
+        university
+        university_id
+        neighborhood
+        park
+      }
+      url
+    }
+    geo_results {
+      type
+      text
+      geo {
+        _id
+        _score
+        mpr_id
+        area_type
+        city
+        state_code
+        state
+        postal_code
+        country
+        lat
+        lon
+        county
+        counties {
+          name
+          fips
+          state_code
+        }
+        slug_id
+        geo_id
+        score
+        name
+        city_slug_id
+        centroid {
+          lat
+          lon
+        }
+        county_needed_for_uniq
+        street
+        line
+        school
+        school_id
+        school_district
+        has_catchment
+        university
+        university_id
+        neighborhood
+        park
+      }
+    }
+    no_matches
+    has_results
+    original_string
+  }
+}"""

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "homeharvest"
-version = "0.8.6b"
+version = "0.8.17"
 description = "Real estate scraping library"
 authors = ["Zachary Hampton <zachary@bunsly.com>", "Cullen Watson <cullen@bunsly.com>"]
 homepage = "https://github.com/ZacharyHampton/HomeHarvest"

tests/test_realtor.py

@@ -1,4 +1,5 @@
 import pytz
+from concurrent.futures import ThreadPoolExecutor, as_completed
 
 from homeharvest import scrape_property, Property
 import pandas as pd
@@ -307,6 +308,30 @@ def test_phone_number_matching():
     assert row["agent_phones"].values[0] == matching_row["agent_phones"].values[0]
 
 
+def test_parallel_search_consistency():
+    """Test that the same search executed 3 times in parallel returns consistent results"""
+    def search_task():
+        return scrape_property(
+            location="Phoenix, AZ",
+            listing_type="for_sale",
+            limit=100
+        )
+
+    with ThreadPoolExecutor(max_workers=3) as executor:
+        futures = [executor.submit(search_task) for _ in range(3)]
+        results = [future.result() for future in as_completed(futures)]
+
+    # Verify all results are valid
+    assert all([result is not None for result in results])
+    assert all([isinstance(result, pd.DataFrame) for result in results])
+    assert all([len(result) > 0 for result in results])
+
+    # Verify all results have the same length (primary consistency check)
+    lengths = [len(result) for result in results]
+    assert len(set(lengths)) == 1, \
+        f"All parallel searches should return same number of results, got lengths: {lengths}"
+
+
 def test_return_type():
     results = {
         "pandas": [scrape_property(location="Surprise, AZ", listing_type="for_rent", limit=100)],