Version bump to 0.8.17

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
zacharyhampton
2025-12-21 19:11:10 -07:00
parent 9a0cac650e
commit 129ab37dff
4 changed files with 21 additions and 37 deletions

View File

@@ -85,9 +85,11 @@ class Scraper:
'Accept': '*/*', 'Accept': '*/*',
'Accept-Language': 'en-US,en;q=0.9', 'Accept-Language': 'en-US,en;q=0.9',
'Cache-Control': 'no-cache', 'Cache-Control': 'no-cache',
'Origin': 'https://www.realtor.com',
'Pragma': 'no-cache', 'Pragma': 'no-cache',
'rdc-client-name': 'rdc-home', 'Referer': 'https://www.realtor.com/',
'rdc-client-version': '2.68.0', 'rdc-client-name': 'RDC_WEB_SRP_FS_PAGE',
'rdc-client-version': '3.0.2515',
'sec-ch-ua': '"Google Chrome";v="135", "Not-A.Brand";v="8", "Chromium";v="135"', 'sec-ch-ua': '"Google Chrome";v="135", "Not-A.Brand";v="8", "Chromium";v="135"',
'sec-ch-ua-mobile': '?0', 'sec-ch-ua-mobile': '?0',
'sec-ch-ua-platform': '"macOS"', 'sec-ch-ua-platform': '"macOS"',
@@ -95,6 +97,7 @@ class Scraper:
'sec-fetch-mode': 'cors', 'sec-fetch-mode': 'cors',
'sec-fetch-site': 'same-site', 'sec-fetch-site': 'same-site',
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/135.0.0.0 Safari/537.36', 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/135.0.0.0 Safari/537.36',
'x-is-bot': 'false',
} }
) )

View File

@@ -29,7 +29,7 @@ from ..models import (
ListingType, ListingType,
ReturnType ReturnType
) )
from .queries import GENERAL_RESULTS_QUERY, SEARCH_HOMES_DATA, HOMES_DATA, HOME_FRAGMENT, SEARCH_RESULTS_FRAGMENT, LISTING_PHOTOS_FRAGMENT, SEARCH_SUGGESTIONS_QUERY from .queries import GENERAL_RESULTS_QUERY, HOMES_DATA, SEARCH_SUGGESTIONS_QUERY
from .processors import ( from .processors import (
process_property, process_property,
process_extra_property_details, process_extra_property_details,
@@ -171,13 +171,10 @@ class RealtorScraper(Scraper):
def handle_home(self, property_id: str) -> list[Property]: def handle_home(self, property_id: str) -> list[Property]:
"""Fetch single home with proper error handling.""" """Fetch single home with proper error handling."""
query = ( query = (
"""%s """query GetHomeDetails($property_id: ID!) {
query GetHomeDetails($property_id: ID!) { home(property_id: $property_id) %s
home(property_id: $property_id) {
...SearchFragment
}
}""" }"""
% HOME_FRAGMENT % HOMES_DATA
) )
variables = {"property_id": property_id} variables = {"property_id": property_id}
@@ -424,9 +421,7 @@ class RealtorScraper(Scraper):
limit: 200 limit: 200
offset: $offset offset: $offset
) %s ) %s
} }""" % (
%s
%s""" % (
is_foreclosure, is_foreclosure,
status_param, status_param,
date_param, date_param,
@@ -435,13 +430,11 @@ class RealtorScraper(Scraper):
pending_or_contingent_param, pending_or_contingent_param,
sort_param, sort_param,
GENERAL_RESULTS_QUERY, GENERAL_RESULTS_QUERY,
SEARCH_RESULTS_FRAGMENT,
LISTING_PHOTOS_FRAGMENT,
) )
elif search_type == "area": #: general search, came from a general location elif search_type == "area": #: general search, came from a general location
query = """query GetHomeSearch( query = """query GetHomeSearch(
$search_location: SearchLocation, $search_location: SearchLocation,
$offset: Int, $offset: Int
) { ) {
homeSearch: home_search( homeSearch: home_search(
query: { query: {
@@ -458,9 +451,7 @@ class RealtorScraper(Scraper):
limit: 200 limit: 200
offset: $offset offset: $offset
) %s ) %s
} }""" % (
%s
%s""" % (
is_foreclosure, is_foreclosure,
status_param, status_param,
date_param, date_param,
@@ -470,8 +461,6 @@ class RealtorScraper(Scraper):
bucket_param, bucket_param,
sort_param, sort_param,
GENERAL_RESULTS_QUERY, GENERAL_RESULTS_QUERY,
SEARCH_RESULTS_FRAGMENT,
LISTING_PHOTOS_FRAGMENT,
) )
else: #: general search, came from an address else: #: general search, came from an address
query = ( query = (
@@ -486,10 +475,8 @@ class RealtorScraper(Scraper):
limit: 1 limit: 1
offset: $offset offset: $offset
) %s ) %s
} }"""
%s % GENERAL_RESULTS_QUERY
%s"""
% (GENERAL_RESULTS_QUERY, SEARCH_RESULTS_FRAGMENT, LISTING_PHOTOS_FRAGMENT)
) )
response_json = self._graphql_post(query, variables, "GetHomeSearch") response_json = self._graphql_post(query, variables, "GetHomeSearch")
@@ -1128,12 +1115,10 @@ class RealtorScraper(Scraper):
property_ids = list(set(property_ids)) property_ids = list(set(property_ids))
fragments = "\n".join( fragments = "\n".join(
f'home_{property_id}: home(property_id: {property_id}) {{ ...SearchFragment }}' f'home_{property_id}: home(property_id: {property_id}) {HOMES_DATA}'
for property_id in property_ids for property_id in property_ids
) )
query = f"""{HOME_FRAGMENT} query = f"""query GetHome {{
query GetHome {{
{fragments} {fragments}
}}""" }}"""

View File

@@ -1,5 +1,5 @@
SEARCH_RESULTS_FRAGMENT = """ SEARCH_RESULTS_FRAGMENT = """
fragment SearchFragment on SearchHome { fragment PropertyResult on SearchHome {
__typename __typename
pending_date pending_date
listing_id listing_id
@@ -371,7 +371,7 @@ _SEARCH_HOMES_DATA_BASE = """{
HOME_FRAGMENT = """ HOME_FRAGMENT = """
fragment SearchFragment on Home { fragment PropertyResult on Home {
__typename __typename
pending_date pending_date
listing_id listing_id
@@ -689,12 +689,8 @@ GENERAL_RESULTS_QUERY = """{
__typename __typename
count count
total total
results { results %s
__typename }""" % SEARCH_HOMES_DATA
...SearchFragment
...ListingPhotosFragment
}
}"""
LISTING_PHOTOS_FRAGMENT = """ LISTING_PHOTOS_FRAGMENT = """
fragment ListingPhotosFragment on SearchHome { fragment ListingPhotosFragment on SearchHome {

View File

@@ -1,6 +1,6 @@
[tool.poetry] [tool.poetry]
name = "homeharvest" name = "homeharvest"
version = "0.8.16" version = "0.8.17"
description = "Real estate scraping library" description = "Real estate scraping library"
authors = ["Zachary Hampton <zachary@bunsly.com>", "Cullen Watson <cullen@bunsly.com>"] authors = ["Zachary Hampton <zachary@bunsly.com>", "Cullen Watson <cullen@bunsly.com>"]
homepage = "https://github.com/ZacharyHampton/HomeHarvest" homepage = "https://github.com/ZacharyHampton/HomeHarvest"