mirror of
https://github.com/Bunsly/HomeHarvest.git
synced 2026-03-04 19:44:29 -08:00
Version bump to 0.8.17
🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
@@ -85,9 +85,11 @@ class Scraper:
|
|||||||
'Accept': '*/*',
|
'Accept': '*/*',
|
||||||
'Accept-Language': 'en-US,en;q=0.9',
|
'Accept-Language': 'en-US,en;q=0.9',
|
||||||
'Cache-Control': 'no-cache',
|
'Cache-Control': 'no-cache',
|
||||||
|
'Origin': 'https://www.realtor.com',
|
||||||
'Pragma': 'no-cache',
|
'Pragma': 'no-cache',
|
||||||
'rdc-client-name': 'rdc-home',
|
'Referer': 'https://www.realtor.com/',
|
||||||
'rdc-client-version': '2.68.0',
|
'rdc-client-name': 'RDC_WEB_SRP_FS_PAGE',
|
||||||
|
'rdc-client-version': '3.0.2515',
|
||||||
'sec-ch-ua': '"Google Chrome";v="135", "Not-A.Brand";v="8", "Chromium";v="135"',
|
'sec-ch-ua': '"Google Chrome";v="135", "Not-A.Brand";v="8", "Chromium";v="135"',
|
||||||
'sec-ch-ua-mobile': '?0',
|
'sec-ch-ua-mobile': '?0',
|
||||||
'sec-ch-ua-platform': '"macOS"',
|
'sec-ch-ua-platform': '"macOS"',
|
||||||
@@ -95,6 +97,7 @@ class Scraper:
|
|||||||
'sec-fetch-mode': 'cors',
|
'sec-fetch-mode': 'cors',
|
||||||
'sec-fetch-site': 'same-site',
|
'sec-fetch-site': 'same-site',
|
||||||
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/135.0.0.0 Safari/537.36',
|
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/135.0.0.0 Safari/537.36',
|
||||||
|
'x-is-bot': 'false',
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|||||||
@@ -29,7 +29,7 @@ from ..models import (
|
|||||||
ListingType,
|
ListingType,
|
||||||
ReturnType
|
ReturnType
|
||||||
)
|
)
|
||||||
from .queries import GENERAL_RESULTS_QUERY, SEARCH_HOMES_DATA, HOMES_DATA, HOME_FRAGMENT, SEARCH_RESULTS_FRAGMENT, LISTING_PHOTOS_FRAGMENT, SEARCH_SUGGESTIONS_QUERY
|
from .queries import GENERAL_RESULTS_QUERY, HOMES_DATA, SEARCH_SUGGESTIONS_QUERY
|
||||||
from .processors import (
|
from .processors import (
|
||||||
process_property,
|
process_property,
|
||||||
process_extra_property_details,
|
process_extra_property_details,
|
||||||
@@ -171,13 +171,10 @@ class RealtorScraper(Scraper):
|
|||||||
def handle_home(self, property_id: str) -> list[Property]:
|
def handle_home(self, property_id: str) -> list[Property]:
|
||||||
"""Fetch single home with proper error handling."""
|
"""Fetch single home with proper error handling."""
|
||||||
query = (
|
query = (
|
||||||
"""%s
|
"""query GetHomeDetails($property_id: ID!) {
|
||||||
query GetHomeDetails($property_id: ID!) {
|
home(property_id: $property_id) %s
|
||||||
home(property_id: $property_id) {
|
|
||||||
...SearchFragment
|
|
||||||
}
|
|
||||||
}"""
|
}"""
|
||||||
% HOME_FRAGMENT
|
% HOMES_DATA
|
||||||
)
|
)
|
||||||
|
|
||||||
variables = {"property_id": property_id}
|
variables = {"property_id": property_id}
|
||||||
@@ -424,9 +421,7 @@ class RealtorScraper(Scraper):
|
|||||||
limit: 200
|
limit: 200
|
||||||
offset: $offset
|
offset: $offset
|
||||||
) %s
|
) %s
|
||||||
}
|
}""" % (
|
||||||
%s
|
|
||||||
%s""" % (
|
|
||||||
is_foreclosure,
|
is_foreclosure,
|
||||||
status_param,
|
status_param,
|
||||||
date_param,
|
date_param,
|
||||||
@@ -435,13 +430,11 @@ class RealtorScraper(Scraper):
|
|||||||
pending_or_contingent_param,
|
pending_or_contingent_param,
|
||||||
sort_param,
|
sort_param,
|
||||||
GENERAL_RESULTS_QUERY,
|
GENERAL_RESULTS_QUERY,
|
||||||
SEARCH_RESULTS_FRAGMENT,
|
|
||||||
LISTING_PHOTOS_FRAGMENT,
|
|
||||||
)
|
)
|
||||||
elif search_type == "area": #: general search, came from a general location
|
elif search_type == "area": #: general search, came from a general location
|
||||||
query = """query GetHomeSearch(
|
query = """query GetHomeSearch(
|
||||||
$search_location: SearchLocation,
|
$search_location: SearchLocation,
|
||||||
$offset: Int,
|
$offset: Int
|
||||||
) {
|
) {
|
||||||
homeSearch: home_search(
|
homeSearch: home_search(
|
||||||
query: {
|
query: {
|
||||||
@@ -458,9 +451,7 @@ class RealtorScraper(Scraper):
|
|||||||
limit: 200
|
limit: 200
|
||||||
offset: $offset
|
offset: $offset
|
||||||
) %s
|
) %s
|
||||||
}
|
}""" % (
|
||||||
%s
|
|
||||||
%s""" % (
|
|
||||||
is_foreclosure,
|
is_foreclosure,
|
||||||
status_param,
|
status_param,
|
||||||
date_param,
|
date_param,
|
||||||
@@ -470,8 +461,6 @@ class RealtorScraper(Scraper):
|
|||||||
bucket_param,
|
bucket_param,
|
||||||
sort_param,
|
sort_param,
|
||||||
GENERAL_RESULTS_QUERY,
|
GENERAL_RESULTS_QUERY,
|
||||||
SEARCH_RESULTS_FRAGMENT,
|
|
||||||
LISTING_PHOTOS_FRAGMENT,
|
|
||||||
)
|
)
|
||||||
else: #: general search, came from an address
|
else: #: general search, came from an address
|
||||||
query = (
|
query = (
|
||||||
@@ -486,10 +475,8 @@ class RealtorScraper(Scraper):
|
|||||||
limit: 1
|
limit: 1
|
||||||
offset: $offset
|
offset: $offset
|
||||||
) %s
|
) %s
|
||||||
}
|
}"""
|
||||||
%s
|
% GENERAL_RESULTS_QUERY
|
||||||
%s"""
|
|
||||||
% (GENERAL_RESULTS_QUERY, SEARCH_RESULTS_FRAGMENT, LISTING_PHOTOS_FRAGMENT)
|
|
||||||
)
|
)
|
||||||
|
|
||||||
response_json = self._graphql_post(query, variables, "GetHomeSearch")
|
response_json = self._graphql_post(query, variables, "GetHomeSearch")
|
||||||
@@ -1128,12 +1115,10 @@ class RealtorScraper(Scraper):
|
|||||||
property_ids = list(set(property_ids))
|
property_ids = list(set(property_ids))
|
||||||
|
|
||||||
fragments = "\n".join(
|
fragments = "\n".join(
|
||||||
f'home_{property_id}: home(property_id: {property_id}) {{ ...SearchFragment }}'
|
f'home_{property_id}: home(property_id: {property_id}) {HOMES_DATA}'
|
||||||
for property_id in property_ids
|
for property_id in property_ids
|
||||||
)
|
)
|
||||||
query = f"""{HOME_FRAGMENT}
|
query = f"""query GetHome {{
|
||||||
|
|
||||||
query GetHome {{
|
|
||||||
{fragments}
|
{fragments}
|
||||||
}}"""
|
}}"""
|
||||||
|
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
SEARCH_RESULTS_FRAGMENT = """
|
SEARCH_RESULTS_FRAGMENT = """
|
||||||
fragment SearchFragment on SearchHome {
|
fragment PropertyResult on SearchHome {
|
||||||
__typename
|
__typename
|
||||||
pending_date
|
pending_date
|
||||||
listing_id
|
listing_id
|
||||||
@@ -371,7 +371,7 @@ _SEARCH_HOMES_DATA_BASE = """{
|
|||||||
|
|
||||||
|
|
||||||
HOME_FRAGMENT = """
|
HOME_FRAGMENT = """
|
||||||
fragment SearchFragment on Home {
|
fragment PropertyResult on Home {
|
||||||
__typename
|
__typename
|
||||||
pending_date
|
pending_date
|
||||||
listing_id
|
listing_id
|
||||||
@@ -689,12 +689,8 @@ GENERAL_RESULTS_QUERY = """{
|
|||||||
__typename
|
__typename
|
||||||
count
|
count
|
||||||
total
|
total
|
||||||
results {
|
results %s
|
||||||
__typename
|
}""" % SEARCH_HOMES_DATA
|
||||||
...SearchFragment
|
|
||||||
...ListingPhotosFragment
|
|
||||||
}
|
|
||||||
}"""
|
|
||||||
|
|
||||||
LISTING_PHOTOS_FRAGMENT = """
|
LISTING_PHOTOS_FRAGMENT = """
|
||||||
fragment ListingPhotosFragment on SearchHome {
|
fragment ListingPhotosFragment on SearchHome {
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
[tool.poetry]
|
[tool.poetry]
|
||||||
name = "homeharvest"
|
name = "homeharvest"
|
||||||
version = "0.8.16"
|
version = "0.8.17"
|
||||||
description = "Real estate scraping library"
|
description = "Real estate scraping library"
|
||||||
authors = ["Zachary Hampton <zachary@bunsly.com>", "Cullen Watson <cullen@bunsly.com>"]
|
authors = ["Zachary Hampton <zachary@bunsly.com>", "Cullen Watson <cullen@bunsly.com>"]
|
||||||
homepage = "https://github.com/ZacharyHampton/HomeHarvest"
|
homepage = "https://github.com/ZacharyHampton/HomeHarvest"
|
||||||
|
|||||||
Reference in New Issue
Block a user