Refactor scraper to use direct requests and bump to 0.8.18

- Replace session-based approach with direct requests calls
- Move headers to module-level DEFAULT_HEADERS constant
- Temporarily disable extra_property_data feature

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>

homeharvest/core/scrapers/__init__.py

@@ -2,8 +2,6 @@ from __future__ import annotations
 from typing import Union
 
 import requests
-from requests.adapters import HTTPAdapter
-from urllib3.util.retry import Retry
 import uuid
 from ...exceptions import AuthenticationError
 from .models import Property, ListingType, SiteName, SearchPropertyType, ReturnType
@@ -11,6 +9,27 @@ import json
 from pydantic import BaseModel
 
 
+DEFAULT_HEADERS = {
+    'Content-Type': 'application/json',
+    'Accept': '*/*',
+    'Accept-Language': 'en-US,en;q=0.9',
+    'Cache-Control': 'no-cache',
+    'Origin': 'https://www.realtor.com',
+    'Pragma': 'no-cache',
+    'Referer': 'https://www.realtor.com/',
+    'rdc-client-name': 'RDC_WEB_SRP_FS_PAGE',
+    'rdc-client-version': '3.0.2515',
+    'sec-ch-ua': '"Google Chrome";v="135", "Not-A.Brand";v="8", "Chromium";v="135"',
+    'sec-ch-ua-mobile': '?0',
+    'sec-ch-ua-platform': '"macOS"',
+    'sec-fetch-dest': 'empty',
+    'sec-fetch-mode': 'cors',
+    'sec-fetch-site': 'same-site',
+    'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/135.0.0.0 Safari/537.36',
+    'x-is-bot': 'false',
+}
+
+
 class ScraperInput(BaseModel):
     location: str
     listing_type: ListingType | list[ListingType] | None
@@ -60,8 +79,6 @@ class ScraperInput(BaseModel):
 
 
 class Scraper:
-    session = None
-
     def __init__(
         self,
         scraper_input: ScraperInput,
@@ -69,38 +86,8 @@ class Scraper:
         self.location = scraper_input.location
         self.listing_type = scraper_input.listing_type
         self.property_type = scraper_input.property_type
-
-        if not self.session:
-            Scraper.session = requests.Session()
-            retries = Retry(
-                total=3, backoff_factor=4, status_forcelist=[429, 403], allowed_methods=frozenset(["GET", "POST"])
-            )
-
-            adapter = HTTPAdapter(max_retries=retries, pool_connections=10, pool_maxsize=20)
-            Scraper.session.mount("http://", adapter)
-            Scraper.session.mount("https://", adapter)
-            Scraper.session.headers.update(
-                {
-                    'Host': 'api.frontdoor.realtor.com',
-                    'rdc-ab-test-client': 'ios_for_sale',
-                    'Content-Type': 'application/json',
-                    'apollographql-client-version': '26.9.25-26.9.25.0774600',
-                    'Accept': '*/*',
-                    'Accept-Language': 'en-US,en;q=0.9',
-                    'rdc-client-version': '26.9.25',
-                    'X-APOLLO-OPERATION-TYPE': 'query',
-                    'rdc-client-name': 'RDC_NATIVE_MOBILE-iPhone-com.move.Realtor',
-                    'apollographql-client-name': 'com.move.Realtor-apollo-ios',
-                    'newrelic': '',
-                    'transparent': '',
-                    'User-Agent': 'Realtor.com/26.9.25.0774600 CFNetwork/3860.200.71 Darwin/25.1.0',
-                }
-            )
-
-        if scraper_input.proxy:
-            proxy_url = scraper_input.proxy
-            proxies = {"http": proxy_url, "https": proxy_url}
-            self.session.proxies.update(proxies)
+        self.proxy = scraper_input.proxy
+        self.proxies = {"http": self.proxy, "https": self.proxy} if self.proxy else None
 
         self.listing_type = scraper_input.listing_type
         self.radius = scraper_input.radius
@@ -111,7 +98,7 @@ class Scraper:
         self.date_from_precision = scraper_input.date_from_precision
         self.date_to_precision = scraper_input.date_to_precision
         self.foreclosure = scraper_input.foreclosure
-        self.extra_property_data = scraper_input.extra_property_data
+        self.extra_property_data = False  # TODO: temporarily disabled
         self.exclude_pending = scraper_input.exclude_pending
         self.limit = scraper_input.limit
         self.offset = scraper_input.offset

homeharvest/core/scrapers/realtor/__init__.py

@@ -8,25 +8,27 @@ This module implements the scraper for realtor.com
 from __future__ import annotations
 
 import json
+import requests
 from concurrent.futures import ThreadPoolExecutor, as_completed
-from datetime import datetime
 from json import JSONDecodeError
 from typing import Dict, Union
 
 from tenacity import (
     retry,
     retry_if_exception_type,
+    retry_if_not_exception_type,
     wait_exponential,
     stop_after_attempt,
 )
 
-from .. import Scraper
+from .. import Scraper, DEFAULT_HEADERS
+from ....exceptions import AuthenticationError
 from ..models import (
     Property,
     ListingType,
     ReturnType
 )
-from .queries import GENERAL_RESULTS_QUERY, SEARCH_HOMES_DATA, HOMES_DATA, HOME_FRAGMENT, SEARCH_RESULTS_FRAGMENT
+from .queries import GENERAL_RESULTS_QUERY, HOMES_DATA, SEARCH_SUGGESTIONS_QUERY
 from .processors import (
     process_property,
     process_extra_property_details,
@@ -35,35 +37,53 @@ from .processors import (
 
 
 class RealtorScraper(Scraper):
-    SEARCH_GQL_URL = "https://api.frontdoor.realtor.com/graphql"
+    SEARCH_GQL_URL = "https://www.realtor.com/frontdoor/graphql"
     NUM_PROPERTY_WORKERS = 20
     DEFAULT_PAGE_SIZE = 200
 
     def __init__(self, scraper_input):
         super().__init__(scraper_input)
 
+    @staticmethod
+    def _minify_query(query: str) -> str:
+        """Minify GraphQL query by collapsing whitespace to single spaces."""
+        # Split on whitespace, filter empty strings, join with single space
+        return ' '.join(query.split())
+
     def _graphql_post(self, query: str, variables: dict, operation_name: str) -> dict:
         """
-        Execute a GraphQL query with operation-specific headers.
+        Execute a GraphQL query.
 
         Args:
             query: GraphQL query string (must include operationName matching operation_name param)
             variables: Query variables dictionary
-            operation_name: Name of the GraphQL operation for Apollo headers
+            operation_name: Name of the GraphQL operation
 
         Returns:
             Response JSON dictionary
         """
-        # Set operation-specific header (must match query's operationName)
-        self.session.headers['X-APOLLO-OPERATION-NAME'] = operation_name
-
         payload = {
-            "operationName": operation_name,  # Include in payload
-            "query": query,
+            "operationName": operation_name,
+            "query": self._minify_query(query),
             "variables": variables,
         }
 
-        response = self.session.post(self.SEARCH_GQL_URL, json=payload)
+        response = requests.post(
+            self.SEARCH_GQL_URL,
+            headers=DEFAULT_HEADERS,
+            data=json.dumps(payload, separators=(',', ':')),
+            proxies=self.proxies
+        )
+
+        if response.status_code == 403:
+            if not self.proxy:
+                raise AuthenticationError(
+                    "Received 403 Forbidden from Realtor.com API.",
+                    response=response
+                )
+            else:
+                raise Exception("Received 403 Forbidden, retrying...")
+
         return response.json()
 
     @retry(
@@ -72,37 +92,13 @@ class RealtorScraper(Scraper):
         stop=stop_after_attempt(3),
     )
     def handle_location(self):
-        query = """
-        fragment SuggestionFragment on SearchSuggestionGeoResult {
-            type
-            text
-            geo {
-                _id
-                area_type
-                city
-                state_code
-                postal_code
-                county
-                centroid { lat lon }
-                slug_id
-                geo_id
-            }
-        }
-        query SearchSuggestions($searchInput: SearchSuggestionsInput!) {
-            search_suggestions(search_input: $searchInput) {
-                geo_results {
-                    ...SuggestionFragment
-                }
-            }
-        }"""
-
         variables = {
             "searchInput": {
                 "search_term": self.location
             }
         }
 
-        response_json = self._graphql_post(query, variables, "SearchSuggestions")
+        response_json = self._graphql_post(SEARCH_SUGGESTIONS_QUERY, variables, "Search_suggestions")
 
         if (
             response_json is None
@@ -134,9 +130,14 @@ class RealtorScraper(Scraper):
         }
 
         if geo.get("area_type") == "address":
-            geo_id = geo.get("_id", "")
-            if geo_id.startswith("addr:"):
-                result["mpr_id"] = geo_id.replace("addr:", "")
+            # Try to get mpr_id directly from API response first
+            if geo.get("mpr_id"):
+                result["mpr_id"] = geo.get("mpr_id")
+            else:
+                # Fallback: extract from _id field if it has addr: prefix
+                geo_id = geo.get("_id", "")
+                if geo_id.startswith("addr:"):
+                    result["mpr_id"] = geo_id.replace("addr:", "")
 
         return result
 
@@ -174,13 +175,10 @@ class RealtorScraper(Scraper):
     def handle_home(self, property_id: str) -> list[Property]:
         """Fetch single home with proper error handling."""
         query = (
-            """%s
-                query GetHomeDetails($property_id: ID!) {
-                    home(property_id: $property_id) {
-                        ...HomeDetailsFragment
-                    }
+            """query GetHomeDetails($property_id: ID!) {
+                    home(property_id: $property_id) %s
                 }"""
-            % HOME_FRAGMENT
+            % HOMES_DATA
         )
 
         variables = {"property_id": property_id}
@@ -405,8 +403,7 @@ class RealtorScraper(Scraper):
             is_foreclosure = "foreclosure: false"
 
         if search_type == "comps":  #: comps search, came from an address
-            query = """%s
-                query GetHomeSearch(
+            query = """query GetHomeSearch(
                     $coordinates: [Float]!
                     $radius: String!
                     $offset: Int!,
@@ -429,7 +426,6 @@ class RealtorScraper(Scraper):
                             offset: $offset
                     ) %s
                 }""" % (
-                SEARCH_RESULTS_FRAGMENT,
                 is_foreclosure,
                 status_param,
                 date_param,
@@ -440,10 +436,9 @@ class RealtorScraper(Scraper):
                 GENERAL_RESULTS_QUERY,
             )
         elif search_type == "area":  #: general search, came from a general location
-            query = """%s
-                            query GetHomeSearch(
+            query = """query GetHomeSearch(
                                 $search_location: SearchLocation,
-                                $offset: Int,
+                                $offset: Int
                             ) {
                                 homeSearch: home_search(
                                     query: {
@@ -461,7 +456,6 @@ class RealtorScraper(Scraper):
                                     offset: $offset
                                 ) %s
                             }""" % (
-                SEARCH_RESULTS_FRAGMENT,
                 is_foreclosure,
                 status_param,
                 date_param,
@@ -474,8 +468,7 @@ class RealtorScraper(Scraper):
             )
         else:  #: general search, came from an address
             query = (
-                """%s
-                    query GetHomeSearch(
+                """query GetHomeSearch(
                         $property_id: [ID]!
                         $offset: Int!,
                     ) {
@@ -487,7 +480,7 @@ class RealtorScraper(Scraper):
                             offset: $offset
                         ) %s
                     }"""
-                % (SEARCH_RESULTS_FRAGMENT, GENERAL_RESULTS_QUERY)
+                % GENERAL_RESULTS_QUERY
             )
 
         response_json = self._graphql_post(query, variables, "GetHomeSearch")
@@ -1111,7 +1104,7 @@ class RealtorScraper(Scraper):
 
 
     @retry(
-        retry=retry_if_exception_type((JSONDecodeError, Exception)),
+        retry=retry_if_exception_type((JSONDecodeError, Exception)) & retry_if_not_exception_type(AuthenticationError),
         wait=wait_exponential(multiplier=1, min=1, max=10),
         stop=stop_after_attempt(3),
     )
@@ -1126,16 +1119,14 @@ class RealtorScraper(Scraper):
         property_ids = list(set(property_ids))
 
         fragments = "\n".join(
-            f'home_{property_id}: home(property_id: {property_id}) {{ ...HomeDetailsFragment }}'
+            f'home_{property_id}: home(property_id: {property_id}) {HOMES_DATA}'
             for property_id in property_ids
         )
-        query = f"""{HOME_FRAGMENT}
-
-query GetHomeDetails {{
+        query = f"""query GetHome {{
     {fragments}
 }}"""
 
-        data = self._graphql_post(query, {}, "GetHomeDetails")
+        data = self._graphql_post(query, {}, "GetHome")
 
         if "data" not in data or data["data"] is None:
             # If we got a 400 error with "Required parameter is missing", raise to trigger retry

homeharvest/core/scrapers/realtor/queries.py

@@ -1,5 +1,5 @@
 SEARCH_RESULTS_FRAGMENT = """
-fragment SearchFragment on SearchHome {
+fragment PropertyResult on SearchHome {
     __typename
     pending_date
     listing_id
@@ -121,13 +121,6 @@ fragment SearchFragment on SearchHome {
     primary_photo(https: true) {
         href
     }
-    photos(https: true) {
-        title
-        href
-        tags {
-            label
-        }
-    }
     advertisers {
         email
         broker {
@@ -378,7 +371,7 @@ _SEARCH_HOMES_DATA_BASE = """{
 
 
 HOME_FRAGMENT = """
-fragment HomeDetailsFragment on Home {
+fragment PropertyResult on Home {
     __typename
     pending_date
     listing_id
@@ -696,8 +689,117 @@ GENERAL_RESULTS_QUERY = """{
                             __typename
                             count
                             total
-                            results {
-                                __typename
-                                ...SearchFragment
-                            }
-                        }"""
+                            results %s
+                        }""" % SEARCH_HOMES_DATA
+
+LISTING_PHOTOS_FRAGMENT = """
+fragment ListingPhotosFragment on SearchHome {
+    __typename
+    photos(https: true) {
+        __typename
+        title
+        href
+        tags {
+            __typename
+            label
+            probability
+        }
+    }
+}
+"""
+
+SEARCH_SUGGESTIONS_QUERY = """query Search_suggestions($searchInput: SearchSuggestionsInput!) {
+  search_suggestions(search_input: $searchInput) {
+    raw_input_parser_result
+    typeahead_results {
+      display_string
+      display_geo
+      geo {
+        _id
+        _score
+        mpr_id
+        area_type
+        city
+        state_code
+        state
+        postal_code
+        country
+        lat
+        lon
+        county
+        counties {
+          name
+          fips
+          state_code
+        }
+        slug_id
+        geo_id
+        score
+        name
+        city_slug_id
+        centroid {
+          lat
+          lon
+        }
+        county_needed_for_uniq
+        street
+        line
+        school
+        school_id
+        school_district
+        has_catchment
+        university
+        university_id
+        neighborhood
+        park
+      }
+      url
+    }
+    geo_results {
+      type
+      text
+      geo {
+        _id
+        _score
+        mpr_id
+        area_type
+        city
+        state_code
+        state
+        postal_code
+        country
+        lat
+        lon
+        county
+        counties {
+          name
+          fips
+          state_code
+        }
+        slug_id
+        geo_id
+        score
+        name
+        city_slug_id
+        centroid {
+          lat
+          lon
+        }
+        county_needed_for_uniq
+        street
+        line
+        school
+        school_id
+        school_district
+        has_catchment
+        university
+        university_id
+        neighborhood
+        park
+      }
+    }
+    no_matches
+    has_results
+    original_string
+  }
+}"""

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "homeharvest"
-version = "0.8.9"
+version = "0.8.18"
 description = "Real estate scraping library"
 authors = ["Zachary Hampton <zachary@bunsly.com>", "Cullen Watson <cullen@bunsly.com>"]
 homepage = "https://github.com/ZacharyHampton/HomeHarvest"