turtlebasket/HomeHarvest
1
0
Fork 0
mirror of https://github.com/Bunsly/HomeHarvest.git synced 2026-03-04 19:44:29 -08:00
Code Issues Packages Projects Releases Wiki Activity

Version bump to 0.8.10

Browse Source 
🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
Zachary Hampton
2025-12-16 18:32:33 -08:00
parent 9406c92a66
commit ef6db606fd
4 changed files with 65 additions and 58 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
homeharvest/core/scrapers/__init__.py
View File

@@ -73,7 +73,7 @@ class Scraper:
if not self.session:
Scraper.session = requests.Session()
retries = Retry(
total=3, backoff_factor=4, status_forcelist=[429, 403], allowed_methods=frozenset(["GET", "POST"])
total=3, backoff_factor=4, status_forcelist=[429], allowed_methods=frozenset(["GET", "POST"])
)
adapter = HTTPAdapter(max_retries=retries, pool_connections=10, pool_maxsize=20)
@@ -81,25 +81,21 @@ class Scraper:
Scraper.session.mount("https://", adapter)
Scraper.session.headers.update(
{
'Host': 'api.frontdoor.realtor.com',
'rdc-ab-test-client': 'ios_for_sale',
'Content-Type': 'application/json',
'apollographql-client-version': '26.9.25-26.9.25.0774600',
'apollographql-client-version': '26.11.0-26.11.0.0249758',
'Accept': '*/*',
'Accept-Language': 'en-US,en;q=0.9',
'rdc-client-version': '26.9.25',
'rdc-client-version': '26.11.0',
'X-APOLLO-OPERATION-TYPE': 'query',
'rdc-client-name': 'RDC_NATIVE_MOBILE-iPhone-com.move.Realtor',
'apollographql-client-name': 'com.move.Realtor-apollo-ios',
'newrelic': '',
'transparent': '',
'User-Agent': 'Realtor.com/26.9.25.0774600 CFNetwork/3860.200.71 Darwin/25.1.0',
'User-Agent': 'Realtor.com/26.11.0.0249758 CFNetwork/3860.200.71 Darwin/25.1.0',
}
)
if scraper_input.proxy:
proxy_url = scraper_input.proxy
proxies = {"http": proxy_url, "https": proxy_url}
self.proxy = scraper_input.proxy
if self.proxy:
proxies = {"http": self.proxy, "https": self.proxy}
self.session.proxies.update(proxies)
self.listing_type = scraper_input.listing_type

77
homeharvest/core/scrapers/realtor/__init__.py
View File

@@ -8,6 +8,7 @@ This module implements the scraper for realtor.com
from __future__ import annotations
import json
import re
from concurrent.futures import ThreadPoolExecutor, as_completed
from datetime import datetime
from json import JSONDecodeError
@@ -21,12 +22,13 @@ from tenacity import (
)
from .. import Scraper
from ....exceptions import AuthenticationError
from ..models import (
Property,
ListingType,
ReturnType
)
from .queries import GENERAL_RESULTS_QUERY, SEARCH_HOMES_DATA, HOMES_DATA, HOME_FRAGMENT, SEARCH_RESULTS_FRAGMENT
from .queries import GENERAL_RESULTS_QUERY, SEARCH_HOMES_DATA, HOMES_DATA, HOME_FRAGMENT, SEARCH_RESULTS_FRAGMENT, LISTING_PHOTOS_FRAGMENT, MORPHEUS_SUGGESTIONS_QUERY
from .processors import (
process_property,
process_extra_property_details,
@@ -42,6 +44,12 @@ class RealtorScraper(Scraper):
def __init__(self, scraper_input):
super().__init__(scraper_input)
@staticmethod
def _minify_query(query: str) -> str:
"""Minify GraphQL query by collapsing whitespace to single spaces."""
# Split on whitespace, filter empty strings, join with single space
return ' '.join(query.split())
def _graphql_post(self, query: str, variables: dict, operation_name: str) -> dict:
"""
Execute a GraphQL query with operation-specific headers.
@@ -59,11 +67,21 @@ class RealtorScraper(Scraper):
payload = {
"operationName": operation_name, # Include in payload
"query": query,
"query": self._minify_query(query),
"variables": variables,
}
response = self.session.post(self.SEARCH_GQL_URL, json=payload)
if response.status_code == 403:
if not self.proxy:
raise AuthenticationError(
"Received 403 Forbidden from Realtor.com API.",
response=response
)
else:
raise Exception("Received 403 Forbidden, retrying...")
return response.json()
@retry(
@@ -72,37 +90,13 @@ class RealtorScraper(Scraper):
stop=stop_after_attempt(3),
)
def handle_location(self):
query = """
fragment SuggestionFragment on SearchSuggestionGeoResult {
type
text
geo {
_id
area_type
city
state_code
postal_code
county
centroid { lat lon }
slug_id
geo_id
}
}
query SearchSuggestions($searchInput: SearchSuggestionsInput!) {
search_suggestions(search_input: $searchInput) {
geo_results {
...SuggestionFragment
}
}
}"""
variables = {
"searchInput": {
"search_term": self.location
}
}
response_json = self._graphql_post(query, variables, "SearchSuggestions")
response_json = self._graphql_post(MORPHEUS_SUGGESTIONS_QUERY, variables, "GetMorpheusSuggestions")
if (
response_json is None
@@ -405,8 +399,7 @@ class RealtorScraper(Scraper):
is_foreclosure = "foreclosure: false"
if search_type == "comps": #: comps search, came from an address
query = """%s
query GetHomeSearch(
query = """query GetHomeSearch(
$coordinates: [Float]!
$radius: String!
$offset: Int!,
@@ -428,8 +421,9 @@ class RealtorScraper(Scraper):
limit: 200
offset: $offset
) %s
}""" % (
SEARCH_RESULTS_FRAGMENT,
}
%s
%s""" % (
is_foreclosure,
status_param,
date_param,
@@ -438,10 +432,11 @@ class RealtorScraper(Scraper):
pending_or_contingent_param,
sort_param,
GENERAL_RESULTS_QUERY,
SEARCH_RESULTS_FRAGMENT,
LISTING_PHOTOS_FRAGMENT,
)
elif search_type == "area": #: general search, came from a general location
query = """%s
query GetHomeSearch(
query = """query GetHomeSearch(
$search_location: SearchLocation,
$offset: Int,
) {
@@ -460,8 +455,9 @@ class RealtorScraper(Scraper):
limit: 200
offset: $offset
) %s
}""" % (
SEARCH_RESULTS_FRAGMENT,
}
%s
%s""" % (
is_foreclosure,
status_param,
date_param,
@@ -471,11 +467,12 @@ class RealtorScraper(Scraper):
bucket_param,
sort_param,
GENERAL_RESULTS_QUERY,
SEARCH_RESULTS_FRAGMENT,
LISTING_PHOTOS_FRAGMENT,
)
else: #: general search, came from an address
query = (
"""%s
query GetHomeSearch(
"""query GetHomeSearch(
$property_id: [ID]!
$offset: Int!,
) {
@@ -486,8 +483,10 @@ class RealtorScraper(Scraper):
limit: 1
offset: $offset
) %s
}"""
% (SEARCH_RESULTS_FRAGMENT, GENERAL_RESULTS_QUERY)
}
%s
%s"""
% (GENERAL_RESULTS_QUERY, SEARCH_RESULTS_FRAGMENT, LISTING_PHOTOS_FRAGMENT)
)
response_json = self._graphql_post(query, variables, "GetHomeSearch")

26
homeharvest/core/scrapers/realtor/queries.py
View File

@@ -121,13 +121,6 @@ fragment SearchFragment on SearchHome {
primary_photo(https: true) {
href
}
photos(https: true) {
title
href
tags {
label
}
}
advertisers {
email
broker {
@@ -699,5 +692,24 @@ GENERAL_RESULTS_QUERY = """{
results {
__typename
...SearchFragment
...ListingPhotosFragment
}
}"""
LISTING_PHOTOS_FRAGMENT = """
fragment ListingPhotosFragment on SearchHome {
__typename
photos(https: true) {
__typename
title
href
tags {
__typename
label
probability
}
}
}
"""
MORPHEUS_SUGGESTIONS_QUERY = """query GetMorpheusSuggestions($searchInput: SearchSuggestionsInput!) { search_suggestions(search_input: $searchInput) { __typename geo_results { __typename type text geo { __typename _id _score mpr_id area_type city state_code postal_code country lat lon county counties { __typename name fips state_code } slug_id geo_id score name city_slug_id centroid { __typename lat lon } county_needed_for_uniq street line school school_id school_district school_district_id has_catchment university university_id neighborhood park } } no_matches has_results filter_criteria { __typename property_type { __typename type } price { __typename min max pattern } bed { __typename min max pattern } bath { __typename min max pattern } feature_tags { __typename tags } listing_status { __typename new_construction existing_homes foreclosures recently_sold fifty_five_plus open_house hide_new_construction hide_existing_homes hide_foreclosures hide_recently_sold hide_fifty_five_plus hide_open_house virtual_tour three_d_tour contingent hide_contingent pending hide_pending } keyword { __typename keywords } garage { __typename min max pattern } age { __typename min max pattern } stories { __typename min max pattern } lot_size { __typename min max pattern } square_feet { __typename min max pattern } home_size { __typename min max pattern } basement finished_basement pool waterfront fireplace detached_garage expand { __typename radius } hoa { __typename type fee } } message_data { __typename property_type pool waterfront fireplace basement finished_basement detached_garage listing_status { __typename new_construction existing_homes foreclosures recently_sold fifty_five_plus open_house hide_new_construction hide_existing_homes hide_foreclosures hide_recently_sold hide_fifty_five_plus hide_open_house } keywords price { __typename min max pattern } bed { __typename min max pattern } bath { __typename min max pattern } garage { __typename min max pattern } stories { __typename min max pattern } age { __typename min max pattern } lot_size { __typename min max pattern } square_feet { __typename min max pattern } } original_string morpheus_context } }"""