turtlebasket/HomeHarvest
1
0
Fork 0
mirror of https://github.com/Bunsly/HomeHarvest.git synced 2026-03-04 19:44:29 -08:00
Code Issues Packages Projects Releases Wiki Activity

Fix 403 error from Realtor.com API changes

Browse Source 
- Update GraphQL endpoint to api.frontdoor.realtor.com
- Update HTTP headers with newer Chrome version and correct client name/version
- Improve error handling in handle_home method
- Fix response validation for missing/null data

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
Zachary Hampton
2025-12-04 18:56:10 -08:00
parent 8bf1f9e24b
commit 0b283e18bd
2 changed files with 40 additions and 31 deletions
Download Patch File Download Diff File Expand all files Collapse all files

30
homeharvest/core/scrapers/__init__.py
View File

@@ -81,21 +81,21 @@ class Scraper:
Scraper.session.mount("https://", adapter)
Scraper.session.headers.update(
{
"accept": "application/json, text/javascript",
"accept-language": "en-US,en;q=0.9",
"cache-control": "no-cache",
"content-type": "application/json",
"origin": "https://www.realtor.com",
"pragma": "no-cache",
"priority": "u=1, i",
"rdc-ab-tests": "commute_travel_time_variation:v1",
"sec-ch-ua": '"Not)A;Brand";v="99", "Google Chrome";v="127", "Chromium";v="127"',
"sec-ch-ua-mobile": "?0",
"sec-ch-ua-platform": '"Windows"',
"sec-fetch-dest": "empty",
"sec-fetch-mode": "cors",
"sec-fetch-site": "same-origin",
"user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36",
'sec-ch-ua-platform': '"macOS"',
'rdc-client-name': 'rdc-search-for-sale-desktop',
'sec-ch-ua': '"Chromium";v="142", "Google Chrome";v="142", "Not_A Brand";v="99"',
'sec-ch-ua-mobile': '?0',
'rdc-client-version': '0.1.0',
'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/142.0.0.0 Safari/537.36',
'accept': 'application/json',
'content-type': 'application/json',
'origin': 'https://www.realtor.com',
'sec-fetch-site': 'same-site',
'sec-fetch-mode': 'cors',
'sec-fetch-dest': 'empty',
'referer': 'https://www.realtor.com/',
'accept-language': 'en-US,en;q=0.9',
'priority': 'u=1, i',
}
)

41
homeharvest/core/scrapers/realtor/__init__.py
View File

@@ -35,9 +35,7 @@ from .processors import (
class RealtorScraper(Scraper):
SEARCH_GQL_URL = "https://www.realtor.com/api/v1/rdc_search_srp?client_id=rdc-search-new-communities&schema=vesta"
PROPERTY_URL = "https://www.realtor.com/realestateandhomes-detail/"
PROPERTY_GQL = "https://graph.realtor.com/graphql"
SEARCH_GQL_URL = "https://api.frontdoor.realtor.com/graphql"
ADDRESS_AUTOCOMPLETE_URL = "https://parser-external.geo.moveaws.com/suggest"
NUM_PROPERTY_WORKERS = 20
DEFAULT_PAGE_SIZE = 200
@@ -108,6 +106,7 @@ class RealtorScraper(Scraper):
return property_info["listings"][0]["listing_id"]
def handle_home(self, property_id: str) -> list[Property]:
"""Fetch single home with proper error handling."""
query = (
"""query Home($property_id: ID!) {
home(property_id: $property_id) %s
@@ -116,23 +115,33 @@ class RealtorScraper(Scraper):
)
variables = {"property_id": property_id}
payload = {
"query": query,
"variables": variables,
}
payload = {"query": query, "variables": variables}
response = self.session.post(self.SEARCH_GQL_URL, json=payload)
response_json = response.json()
try:
response = self.session.post(self.SEARCH_GQL_URL, json=payload)
data = response.json()
property_info = response_json["data"]["home"]
# Check for errors or missing data
if "errors" in data or "data" not in data:
return []
if self.return_type != ReturnType.raw:
return [process_property(property_info, self.mls_only, self.extra_property_data,
self.exclude_pending, self.listing_type, get_key, process_extra_property_details)]
else:
return [property_info]
if data["data"] is None or "home" not in data["data"]:
return []
property_info = data["data"]["home"]
if property_info is None:
return []
# Process based on return type
if self.return_type != ReturnType.raw:
return [process_property(property_info, self.mls_only, self.extra_property_data,
self.exclude_pending, self.listing_type, get_key,
process_extra_property_details)]
else:
return [property_info]
except Exception:
return []
def general_search(self, variables: dict, search_type: str) -> Dict[str, Union[int, Union[list[Property], list[dict]]]]:
"""
@@ -366,7 +375,7 @@ class RealtorScraper(Scraper):
$city: String,
$county: [String],
$state_code: String,
$postal_code: String
$postal_code: String,
$offset: Int,
) {
home_search(