diff --git a/homeharvest/core/scrapers/__init__.py b/homeharvest/core/scrapers/__init__.py index 5967864..aef6cad 100644 --- a/homeharvest/core/scrapers/__init__.py +++ b/homeharvest/core/scrapers/__init__.py @@ -73,7 +73,7 @@ class Scraper: if not self.session: Scraper.session = requests.Session() retries = Retry( - total=3, backoff_factor=4, status_forcelist=[429, 403], allowed_methods=frozenset(["GET", "POST"]) + total=3, backoff_factor=4, status_forcelist=[429], allowed_methods=frozenset(["GET", "POST"]) ) adapter = HTTPAdapter(max_retries=retries, pool_connections=10, pool_maxsize=20) @@ -81,25 +81,21 @@ class Scraper: Scraper.session.mount("https://", adapter) Scraper.session.headers.update( { - 'Host': 'api.frontdoor.realtor.com', - 'rdc-ab-test-client': 'ios_for_sale', 'Content-Type': 'application/json', - 'apollographql-client-version': '26.9.25-26.9.25.0774600', + 'apollographql-client-version': '26.11.0-26.11.0.0249758', 'Accept': '*/*', 'Accept-Language': 'en-US,en;q=0.9', - 'rdc-client-version': '26.9.25', + 'rdc-client-version': '26.11.0', 'X-APOLLO-OPERATION-TYPE': 'query', 'rdc-client-name': 'RDC_NATIVE_MOBILE-iPhone-com.move.Realtor', 'apollographql-client-name': 'com.move.Realtor-apollo-ios', - 'newrelic': '', - 'transparent': '', - 'User-Agent': 'Realtor.com/26.9.25.0774600 CFNetwork/3860.200.71 Darwin/25.1.0', + 'User-Agent': 'Realtor.com/26.11.0.0249758 CFNetwork/3860.200.71 Darwin/25.1.0', } ) - if scraper_input.proxy: - proxy_url = scraper_input.proxy - proxies = {"http": proxy_url, "https": proxy_url} + self.proxy = scraper_input.proxy + if self.proxy: + proxies = {"http": self.proxy, "https": self.proxy} self.session.proxies.update(proxies) self.listing_type = scraper_input.listing_type diff --git a/homeharvest/core/scrapers/realtor/__init__.py b/homeharvest/core/scrapers/realtor/__init__.py index 40d5f88..7ad40ca 100644 --- a/homeharvest/core/scrapers/realtor/__init__.py +++ b/homeharvest/core/scrapers/realtor/__init__.py @@ -8,6 +8,7 @@ This module implements the scraper for realtor.com from __future__ import annotations import json +import re from concurrent.futures import ThreadPoolExecutor, as_completed from datetime import datetime from json import JSONDecodeError @@ -21,12 +22,13 @@ from tenacity import ( ) from .. import Scraper +from ....exceptions import AuthenticationError from ..models import ( Property, ListingType, ReturnType ) -from .queries import GENERAL_RESULTS_QUERY, SEARCH_HOMES_DATA, HOMES_DATA, HOME_FRAGMENT, SEARCH_RESULTS_FRAGMENT +from .queries import GENERAL_RESULTS_QUERY, SEARCH_HOMES_DATA, HOMES_DATA, HOME_FRAGMENT, SEARCH_RESULTS_FRAGMENT, LISTING_PHOTOS_FRAGMENT, MORPHEUS_SUGGESTIONS_QUERY from .processors import ( process_property, process_extra_property_details, @@ -42,6 +44,12 @@ class RealtorScraper(Scraper): def __init__(self, scraper_input): super().__init__(scraper_input) + @staticmethod + def _minify_query(query: str) -> str: + """Minify GraphQL query by collapsing whitespace to single spaces.""" + # Split on whitespace, filter empty strings, join with single space + return ' '.join(query.split()) + def _graphql_post(self, query: str, variables: dict, operation_name: str) -> dict: """ Execute a GraphQL query with operation-specific headers. @@ -59,11 +67,21 @@ class RealtorScraper(Scraper): payload = { "operationName": operation_name, # Include in payload - "query": query, + "query": self._minify_query(query), "variables": variables, } response = self.session.post(self.SEARCH_GQL_URL, json=payload) + + if response.status_code == 403: + if not self.proxy: + raise AuthenticationError( + "Received 403 Forbidden from Realtor.com API.", + response=response + ) + else: + raise Exception("Received 403 Forbidden, retrying...") + return response.json() @retry( @@ -72,37 +90,13 @@ class RealtorScraper(Scraper): stop=stop_after_attempt(3), ) def handle_location(self): - query = """ - fragment SuggestionFragment on SearchSuggestionGeoResult { - type - text - geo { - _id - area_type - city - state_code - postal_code - county - centroid { lat lon } - slug_id - geo_id - } - } - query SearchSuggestions($searchInput: SearchSuggestionsInput!) { - search_suggestions(search_input: $searchInput) { - geo_results { - ...SuggestionFragment - } - } - }""" - variables = { "searchInput": { "search_term": self.location } } - response_json = self._graphql_post(query, variables, "SearchSuggestions") + response_json = self._graphql_post(MORPHEUS_SUGGESTIONS_QUERY, variables, "GetMorpheusSuggestions") if ( response_json is None @@ -405,8 +399,7 @@ class RealtorScraper(Scraper): is_foreclosure = "foreclosure: false" if search_type == "comps": #: comps search, came from an address - query = """%s - query GetHomeSearch( + query = """query GetHomeSearch( $coordinates: [Float]! $radius: String! $offset: Int!, @@ -428,8 +421,9 @@ class RealtorScraper(Scraper): limit: 200 offset: $offset ) %s - }""" % ( - SEARCH_RESULTS_FRAGMENT, + } + %s + %s""" % ( is_foreclosure, status_param, date_param, @@ -438,10 +432,11 @@ class RealtorScraper(Scraper): pending_or_contingent_param, sort_param, GENERAL_RESULTS_QUERY, + SEARCH_RESULTS_FRAGMENT, + LISTING_PHOTOS_FRAGMENT, ) elif search_type == "area": #: general search, came from a general location - query = """%s - query GetHomeSearch( + query = """query GetHomeSearch( $search_location: SearchLocation, $offset: Int, ) { @@ -460,8 +455,9 @@ class RealtorScraper(Scraper): limit: 200 offset: $offset ) %s - }""" % ( - SEARCH_RESULTS_FRAGMENT, + } + %s + %s""" % ( is_foreclosure, status_param, date_param, @@ -471,11 +467,12 @@ class RealtorScraper(Scraper): bucket_param, sort_param, GENERAL_RESULTS_QUERY, + SEARCH_RESULTS_FRAGMENT, + LISTING_PHOTOS_FRAGMENT, ) else: #: general search, came from an address query = ( - """%s - query GetHomeSearch( + """query GetHomeSearch( $property_id: [ID]! $offset: Int!, ) { @@ -486,8 +483,10 @@ class RealtorScraper(Scraper): limit: 1 offset: $offset ) %s - }""" - % (SEARCH_RESULTS_FRAGMENT, GENERAL_RESULTS_QUERY) + } + %s + %s""" + % (GENERAL_RESULTS_QUERY, SEARCH_RESULTS_FRAGMENT, LISTING_PHOTOS_FRAGMENT) ) response_json = self._graphql_post(query, variables, "GetHomeSearch") diff --git a/homeharvest/core/scrapers/realtor/queries.py b/homeharvest/core/scrapers/realtor/queries.py index 1258824..7e6b604 100644 --- a/homeharvest/core/scrapers/realtor/queries.py +++ b/homeharvest/core/scrapers/realtor/queries.py @@ -121,13 +121,6 @@ fragment SearchFragment on SearchHome { primary_photo(https: true) { href } - photos(https: true) { - title - href - tags { - label - } - } advertisers { email broker { @@ -699,5 +692,24 @@ GENERAL_RESULTS_QUERY = """{ results { __typename ...SearchFragment + ...ListingPhotosFragment } }""" + +LISTING_PHOTOS_FRAGMENT = """ +fragment ListingPhotosFragment on SearchHome { + __typename + photos(https: true) { + __typename + title + href + tags { + __typename + label + probability + } + } +} +""" + +MORPHEUS_SUGGESTIONS_QUERY = """query GetMorpheusSuggestions($searchInput: SearchSuggestionsInput!) { search_suggestions(search_input: $searchInput) { __typename geo_results { __typename type text geo { __typename _id _score mpr_id area_type city state_code postal_code country lat lon county counties { __typename name fips state_code } slug_id geo_id score name city_slug_id centroid { __typename lat lon } county_needed_for_uniq street line school school_id school_district school_district_id has_catchment university university_id neighborhood park } } no_matches has_results filter_criteria { __typename property_type { __typename type } price { __typename min max pattern } bed { __typename min max pattern } bath { __typename min max pattern } feature_tags { __typename tags } listing_status { __typename new_construction existing_homes foreclosures recently_sold fifty_five_plus open_house hide_new_construction hide_existing_homes hide_foreclosures hide_recently_sold hide_fifty_five_plus hide_open_house virtual_tour three_d_tour contingent hide_contingent pending hide_pending } keyword { __typename keywords } garage { __typename min max pattern } age { __typename min max pattern } stories { __typename min max pattern } lot_size { __typename min max pattern } square_feet { __typename min max pattern } home_size { __typename min max pattern } basement finished_basement pool waterfront fireplace detached_garage expand { __typename radius } hoa { __typename type fee } } message_data { __typename property_type pool waterfront fireplace basement finished_basement detached_garage listing_status { __typename new_construction existing_homes foreclosures recently_sold fifty_five_plus open_house hide_new_construction hide_existing_homes hide_foreclosures hide_recently_sold hide_fifty_five_plus hide_open_house } keywords price { __typename min max pattern } bed { __typename min max pattern } bath { __typename min max pattern } garage { __typename min max pattern } stories { __typename min max pattern } age { __typename min max pattern } lot_size { __typename min max pattern } square_feet { __typename min max pattern } } original_string morpheus_context } }""" diff --git a/pyproject.toml b/pyproject.toml index 6f0c219..e563455 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "homeharvest" -version = "0.8.9" +version = "0.8.10" description = "Real estate scraping library" authors = ["Zachary Hampton ", "Cullen Watson "] homepage = "https://github.com/ZacharyHampton/HomeHarvest"