mirror of
https://github.com/Bunsly/HomeHarvest.git
synced 2026-03-04 19:44:29 -08:00
Compare commits
9 Commits
fix/realto
...
v0.8.13
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
cab0216f29 | ||
|
|
8ee720ce5c | ||
|
|
8eb138ee1a | ||
|
|
ef6db606fd | ||
|
|
9406c92a66 | ||
|
|
fefacdd264 | ||
|
|
3579c10196 | ||
|
|
f5784e0191 | ||
|
|
57093f5d17 |
@@ -5,6 +5,7 @@ import requests
|
||||
from requests.adapters import HTTPAdapter
|
||||
from urllib3.util.retry import Retry
|
||||
import uuid
|
||||
import secrets
|
||||
from ...exceptions import AuthenticationError
|
||||
from .models import Property, ListingType, SiteName, SearchPropertyType, ReturnType
|
||||
import json
|
||||
@@ -73,35 +74,30 @@ class Scraper:
|
||||
if not self.session:
|
||||
Scraper.session = requests.Session()
|
||||
retries = Retry(
|
||||
total=3, backoff_factor=4, status_forcelist=[429, 403], allowed_methods=frozenset(["GET", "POST"])
|
||||
total=3, backoff_factor=4, status_forcelist=[429], allowed_methods=frozenset(["GET", "POST"])
|
||||
)
|
||||
|
||||
adapter = HTTPAdapter(max_retries=retries)
|
||||
adapter = HTTPAdapter(max_retries=retries, pool_connections=10, pool_maxsize=20)
|
||||
Scraper.session.mount("http://", adapter)
|
||||
Scraper.session.mount("https://", adapter)
|
||||
Scraper.session.headers.update(
|
||||
{
|
||||
'sec-ch-ua-platform': '"macOS"',
|
||||
'rdc-client-name': 'rdc-search-for-sale-desktop',
|
||||
'sec-ch-ua': '"Chromium";v="142", "Google Chrome";v="142", "Not_A Brand";v="99"',
|
||||
'sec-ch-ua-mobile': '?0',
|
||||
'rdc-client-version': '0.1.0',
|
||||
'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/142.0.0.0 Safari/537.36',
|
||||
'accept': 'application/json',
|
||||
'content-type': 'application/json',
|
||||
'origin': 'https://www.realtor.com',
|
||||
'sec-fetch-site': 'same-site',
|
||||
'sec-fetch-mode': 'cors',
|
||||
'sec-fetch-dest': 'empty',
|
||||
'referer': 'https://www.realtor.com/',
|
||||
'accept-language': 'en-US,en;q=0.9',
|
||||
'priority': 'u=1, i',
|
||||
'Content-Type': 'application/json',
|
||||
'apollographql-client-version': '26.11.1-26.11.1.1106489',
|
||||
'Accept': '*/*',
|
||||
'Accept-Language': 'en-US,en;q=0.9',
|
||||
'rdc-client-version': '26.11.1',
|
||||
'X-APOLLO-OPERATION-TYPE': 'query',
|
||||
'X-APOLLO-OPERATION-ID': secrets.token_hex(32),
|
||||
'rdc-client-name': 'RDC_NATIVE_MOBILE-iPhone-com.move.Realtor',
|
||||
'apollographql-client-name': 'com.move.Realtor-apollo-ios',
|
||||
'User-Agent': 'Realtor.com/26.11.1.1106489 CFNetwork/3860.200.71 Darwin/25.1.0',
|
||||
}
|
||||
)
|
||||
|
||||
if scraper_input.proxy:
|
||||
proxy_url = scraper_input.proxy
|
||||
proxies = {"http": proxy_url, "https": proxy_url}
|
||||
self.proxy = scraper_input.proxy
|
||||
if self.proxy:
|
||||
proxies = {"http": self.proxy, "https": self.proxy}
|
||||
self.session.proxies.update(proxies)
|
||||
|
||||
self.listing_type = scraper_input.listing_type
|
||||
|
||||
@@ -8,6 +8,7 @@ This module implements the scraper for realtor.com
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from datetime import datetime
|
||||
from json import JSONDecodeError
|
||||
@@ -21,12 +22,13 @@ from tenacity import (
|
||||
)
|
||||
|
||||
from .. import Scraper
|
||||
from ....exceptions import AuthenticationError
|
||||
from ..models import (
|
||||
Property,
|
||||
ListingType,
|
||||
ReturnType
|
||||
)
|
||||
from .queries import GENERAL_RESULTS_QUERY, SEARCH_HOMES_DATA, HOMES_DATA, HOME_FRAGMENT
|
||||
from .queries import GENERAL_RESULTS_QUERY, SEARCH_HOMES_DATA, HOMES_DATA, HOME_FRAGMENT, SEARCH_RESULTS_FRAGMENT, LISTING_PHOTOS_FRAGMENT, MORPHEUS_SUGGESTIONS_QUERY
|
||||
from .processors import (
|
||||
process_property,
|
||||
process_extra_property_details,
|
||||
@@ -42,40 +44,59 @@ class RealtorScraper(Scraper):
|
||||
def __init__(self, scraper_input):
|
||||
super().__init__(scraper_input)
|
||||
|
||||
def handle_location(self):
|
||||
query = """query Search_suggestions($searchInput: SearchSuggestionsInput!) {
|
||||
search_suggestions(search_input: $searchInput) {
|
||||
geo_results {
|
||||
type
|
||||
text
|
||||
geo {
|
||||
_id
|
||||
area_type
|
||||
city
|
||||
state_code
|
||||
postal_code
|
||||
county
|
||||
centroid { lat lon }
|
||||
slug_id
|
||||
geo_id
|
||||
}
|
||||
}
|
||||
}
|
||||
}"""
|
||||
@staticmethod
|
||||
def _minify_query(query: str) -> str:
|
||||
"""Minify GraphQL query by collapsing whitespace to single spaces."""
|
||||
# Split on whitespace, filter empty strings, join with single space
|
||||
return ' '.join(query.split())
|
||||
|
||||
def _graphql_post(self, query: str, variables: dict, operation_name: str) -> dict:
|
||||
"""
|
||||
Execute a GraphQL query with operation-specific headers.
|
||||
|
||||
Args:
|
||||
query: GraphQL query string (must include operationName matching operation_name param)
|
||||
variables: Query variables dictionary
|
||||
operation_name: Name of the GraphQL operation for Apollo headers
|
||||
|
||||
Returns:
|
||||
Response JSON dictionary
|
||||
"""
|
||||
# Set operation-specific header (must match query's operationName)
|
||||
self.session.headers['X-APOLLO-OPERATION-NAME'] = operation_name
|
||||
|
||||
payload = {
|
||||
"operationName": operation_name, # Include in payload
|
||||
"query": self._minify_query(query),
|
||||
"variables": variables,
|
||||
}
|
||||
|
||||
response = self.session.post(self.SEARCH_GQL_URL, data=json.dumps(payload, separators=(',', ':')))
|
||||
|
||||
if response.status_code == 403:
|
||||
if not self.proxy:
|
||||
raise AuthenticationError(
|
||||
"Received 403 Forbidden from Realtor.com API.",
|
||||
response=response
|
||||
)
|
||||
else:
|
||||
raise Exception("Received 403 Forbidden, retrying...")
|
||||
|
||||
return response.json()
|
||||
|
||||
@retry(
|
||||
retry=retry_if_exception_type(Exception),
|
||||
wait=wait_exponential(multiplier=1, min=1, max=4),
|
||||
stop=stop_after_attempt(3),
|
||||
)
|
||||
def handle_location(self):
|
||||
variables = {
|
||||
"searchInput": {
|
||||
"search_term": self.location
|
||||
}
|
||||
}
|
||||
|
||||
payload = {
|
||||
"query": query,
|
||||
"variables": variables,
|
||||
}
|
||||
|
||||
response = self.session.post(self.SEARCH_GQL_URL, json=payload)
|
||||
response_json = response.json()
|
||||
response_json = self._graphql_post(MORPHEUS_SUGGESTIONS_QUERY, variables, "GetMorpheusSuggestions")
|
||||
|
||||
if (
|
||||
response_json is None
|
||||
@@ -86,6 +107,11 @@ class RealtorScraper(Scraper):
|
||||
or "geo_results" not in response_json["data"]["search_suggestions"]
|
||||
or not response_json["data"]["search_suggestions"]["geo_results"]
|
||||
):
|
||||
# If we got a 400 error with "Required parameter is missing", raise to trigger retry
|
||||
if response_json and "errors" in response_json:
|
||||
error_msgs = [e.get("message", "") for e in response_json.get("errors", [])]
|
||||
if any("Required parameter is missing" in msg for msg in error_msgs):
|
||||
raise Exception(f"Transient API error: {error_msgs}")
|
||||
return None
|
||||
|
||||
geo_result = response_json["data"]["search_suggestions"]["geo_results"][0]
|
||||
@@ -109,24 +135,22 @@ class RealtorScraper(Scraper):
|
||||
return result
|
||||
|
||||
def get_latest_listing_id(self, property_id: str) -> str | None:
|
||||
query = """query Property($property_id: ID!) {
|
||||
query = """
|
||||
fragment ListingFragment on Listing {
|
||||
listing_id
|
||||
primary
|
||||
}
|
||||
query GetPropertyListingId($property_id: ID!) {
|
||||
property(id: $property_id) {
|
||||
listings {
|
||||
listing_id
|
||||
primary
|
||||
...ListingFragment
|
||||
}
|
||||
}
|
||||
}
|
||||
"""
|
||||
|
||||
variables = {"property_id": property_id}
|
||||
payload = {
|
||||
"query": query,
|
||||
"variables": variables,
|
||||
}
|
||||
|
||||
response = self.session.post(self.SEARCH_GQL_URL, json=payload)
|
||||
response_json = response.json()
|
||||
response_json = self._graphql_post(query, variables, "GetPropertyListingId")
|
||||
|
||||
property_info = response_json["data"]["property"]
|
||||
if property_info["listings"] is None:
|
||||
@@ -144,18 +168,19 @@ class RealtorScraper(Scraper):
|
||||
def handle_home(self, property_id: str) -> list[Property]:
|
||||
"""Fetch single home with proper error handling."""
|
||||
query = (
|
||||
"""query Home($property_id: ID!) {
|
||||
home(property_id: $property_id) %s
|
||||
"""%s
|
||||
query GetHomeDetails($property_id: ID!) {
|
||||
home(property_id: $property_id) {
|
||||
...HomeDetailsFragment
|
||||
}
|
||||
}"""
|
||||
% HOMES_DATA
|
||||
% HOME_FRAGMENT
|
||||
)
|
||||
|
||||
variables = {"property_id": property_id}
|
||||
payload = {"query": query, "variables": variables}
|
||||
|
||||
try:
|
||||
response = self.session.post(self.SEARCH_GQL_URL, json=payload)
|
||||
data = response.json()
|
||||
data = self._graphql_post(query, variables, "GetHomeDetails")
|
||||
|
||||
# Check for errors or missing data
|
||||
if "errors" in data or "data" not in data:
|
||||
@@ -374,12 +399,12 @@ class RealtorScraper(Scraper):
|
||||
is_foreclosure = "foreclosure: false"
|
||||
|
||||
if search_type == "comps": #: comps search, came from an address
|
||||
query = """query Property_search(
|
||||
query = """query GetHomeSearch(
|
||||
$coordinates: [Float]!
|
||||
$radius: String!
|
||||
$offset: Int!,
|
||||
) {
|
||||
home_search(
|
||||
homeSearch: home_search(
|
||||
query: {
|
||||
%s
|
||||
nearby: {
|
||||
@@ -396,7 +421,9 @@ class RealtorScraper(Scraper):
|
||||
limit: 200
|
||||
offset: $offset
|
||||
) %s
|
||||
}""" % (
|
||||
}
|
||||
%s
|
||||
%s""" % (
|
||||
is_foreclosure,
|
||||
status_param,
|
||||
date_param,
|
||||
@@ -405,13 +432,15 @@ class RealtorScraper(Scraper):
|
||||
pending_or_contingent_param,
|
||||
sort_param,
|
||||
GENERAL_RESULTS_QUERY,
|
||||
SEARCH_RESULTS_FRAGMENT,
|
||||
LISTING_PHOTOS_FRAGMENT,
|
||||
)
|
||||
elif search_type == "area": #: general search, came from a general location
|
||||
query = """query Home_search(
|
||||
query = """query GetHomeSearch(
|
||||
$search_location: SearchLocation,
|
||||
$offset: Int,
|
||||
) {
|
||||
home_search(
|
||||
homeSearch: home_search(
|
||||
query: {
|
||||
%s
|
||||
search_location: $search_location
|
||||
@@ -426,7 +455,9 @@ class RealtorScraper(Scraper):
|
||||
limit: 200
|
||||
offset: $offset
|
||||
) %s
|
||||
}""" % (
|
||||
}
|
||||
%s
|
||||
%s""" % (
|
||||
is_foreclosure,
|
||||
status_param,
|
||||
date_param,
|
||||
@@ -436,32 +467,30 @@ class RealtorScraper(Scraper):
|
||||
bucket_param,
|
||||
sort_param,
|
||||
GENERAL_RESULTS_QUERY,
|
||||
SEARCH_RESULTS_FRAGMENT,
|
||||
LISTING_PHOTOS_FRAGMENT,
|
||||
)
|
||||
else: #: general search, came from an address
|
||||
query = (
|
||||
"""query Property_search(
|
||||
"""query GetHomeSearch(
|
||||
$property_id: [ID]!
|
||||
$offset: Int!,
|
||||
) {
|
||||
home_search(
|
||||
homeSearch: home_search(
|
||||
query: {
|
||||
property_id: $property_id
|
||||
}
|
||||
limit: 1
|
||||
offset: $offset
|
||||
) %s
|
||||
}"""
|
||||
% GENERAL_RESULTS_QUERY
|
||||
}
|
||||
%s
|
||||
%s"""
|
||||
% (GENERAL_RESULTS_QUERY, SEARCH_RESULTS_FRAGMENT, LISTING_PHOTOS_FRAGMENT)
|
||||
)
|
||||
|
||||
payload = {
|
||||
"query": query,
|
||||
"variables": variables,
|
||||
}
|
||||
|
||||
response = self.session.post(self.SEARCH_GQL_URL, json=payload)
|
||||
response_json = response.json()
|
||||
search_key = "home_search" if "home_search" in query else "property_search"
|
||||
response_json = self._graphql_post(query, variables, "GetHomeSearch")
|
||||
search_key = "homeSearch"
|
||||
|
||||
properties: list[Union[Property, dict]] = []
|
||||
|
||||
@@ -1081,8 +1110,8 @@ class RealtorScraper(Scraper):
|
||||
|
||||
|
||||
@retry(
|
||||
retry=retry_if_exception_type(JSONDecodeError),
|
||||
wait=wait_exponential(min=4, max=10),
|
||||
retry=retry_if_exception_type((JSONDecodeError, Exception)),
|
||||
wait=wait_exponential(multiplier=1, min=1, max=10),
|
||||
stop=stop_after_attempt(3),
|
||||
)
|
||||
def get_bulk_prop_details(self, property_ids: list[str]) -> dict:
|
||||
@@ -1095,24 +1124,27 @@ class RealtorScraper(Scraper):
|
||||
|
||||
property_ids = list(set(property_ids))
|
||||
|
||||
# Construct the bulk query
|
||||
fragments = "\n".join(
|
||||
f'home_{property_id}: home(property_id: {property_id}) {{ ...HomeData }}'
|
||||
f'home_{property_id}: home(property_id: {property_id}) {{ ...HomeDetailsFragment }}'
|
||||
for property_id in property_ids
|
||||
)
|
||||
query = f"""{HOME_FRAGMENT}
|
||||
|
||||
query GetHomes {{
|
||||
{fragments}
|
||||
}}"""
|
||||
|
||||
response = self.session.post(self.SEARCH_GQL_URL, json={"query": query})
|
||||
data = response.json()
|
||||
query GetHomeDetails {{
|
||||
{fragments}
|
||||
}}"""
|
||||
|
||||
if "data" not in data:
|
||||
data = self._graphql_post(query, {}, "GetHomeDetails")
|
||||
|
||||
if "data" not in data or data["data"] is None:
|
||||
# If we got a 400 error with "Required parameter is missing", raise to trigger retry
|
||||
if data and "errors" in data:
|
||||
error_msgs = [e.get("message", "") for e in data.get("errors", [])]
|
||||
if any("Required parameter is missing" in msg for msg in error_msgs):
|
||||
raise Exception(f"Transient API error: {error_msgs}")
|
||||
return {}
|
||||
|
||||
properties = data["data"]
|
||||
return {data.replace('home_', ''): properties[data] for data in properties if properties[data]}
|
||||
return {key.replace('home_', ''): properties[key] for key in properties if properties[key]}
|
||||
|
||||
|
||||
|
||||
@@ -1,3 +1,193 @@
|
||||
SEARCH_RESULTS_FRAGMENT = """
|
||||
fragment SearchFragment on SearchHome {
|
||||
__typename
|
||||
pending_date
|
||||
listing_id
|
||||
property_id
|
||||
href
|
||||
permalink
|
||||
list_date
|
||||
status
|
||||
mls_status
|
||||
last_sold_price
|
||||
last_sold_date
|
||||
last_status_change_date
|
||||
last_update_date
|
||||
list_price
|
||||
list_price_max
|
||||
list_price_min
|
||||
price_per_sqft
|
||||
tags
|
||||
open_houses {
|
||||
start_date
|
||||
end_date
|
||||
description
|
||||
time_zone
|
||||
dst
|
||||
href
|
||||
methods
|
||||
}
|
||||
details {
|
||||
category
|
||||
text
|
||||
parent_category
|
||||
}
|
||||
pet_policy {
|
||||
cats
|
||||
dogs
|
||||
dogs_small
|
||||
dogs_large
|
||||
__typename
|
||||
}
|
||||
units {
|
||||
availability {
|
||||
date
|
||||
__typename
|
||||
}
|
||||
description {
|
||||
baths_consolidated
|
||||
baths
|
||||
beds
|
||||
sqft
|
||||
__typename
|
||||
}
|
||||
photos(https: true) {
|
||||
title
|
||||
href
|
||||
tags {
|
||||
label
|
||||
}
|
||||
}
|
||||
list_price
|
||||
__typename
|
||||
}
|
||||
flags {
|
||||
is_contingent
|
||||
is_pending
|
||||
is_new_construction
|
||||
}
|
||||
description {
|
||||
type
|
||||
sqft
|
||||
beds
|
||||
baths_full
|
||||
baths_half
|
||||
lot_sqft
|
||||
year_built
|
||||
garage
|
||||
type
|
||||
name
|
||||
stories
|
||||
text
|
||||
}
|
||||
source {
|
||||
id
|
||||
listing_id
|
||||
}
|
||||
hoa {
|
||||
fee
|
||||
}
|
||||
location {
|
||||
address {
|
||||
street_direction
|
||||
street_number
|
||||
street_name
|
||||
street_suffix
|
||||
line
|
||||
unit
|
||||
city
|
||||
state_code
|
||||
postal_code
|
||||
coordinate {
|
||||
lon
|
||||
lat
|
||||
}
|
||||
}
|
||||
county {
|
||||
name
|
||||
fips_code
|
||||
}
|
||||
neighborhoods {
|
||||
name
|
||||
}
|
||||
}
|
||||
tax_record {
|
||||
cl_id
|
||||
public_record_id
|
||||
last_update_date
|
||||
apn
|
||||
tax_parcel_id
|
||||
}
|
||||
primary_photo(https: true) {
|
||||
href
|
||||
}
|
||||
advertisers {
|
||||
email
|
||||
broker {
|
||||
name
|
||||
fulfillment_id
|
||||
}
|
||||
type
|
||||
name
|
||||
fulfillment_id
|
||||
builder {
|
||||
name
|
||||
fulfillment_id
|
||||
}
|
||||
phones {
|
||||
ext
|
||||
primary
|
||||
type
|
||||
number
|
||||
}
|
||||
office {
|
||||
name
|
||||
email
|
||||
fulfillment_id
|
||||
href
|
||||
phones {
|
||||
number
|
||||
type
|
||||
primary
|
||||
ext
|
||||
}
|
||||
mls_set
|
||||
}
|
||||
corporation {
|
||||
specialties
|
||||
name
|
||||
bio
|
||||
href
|
||||
fulfillment_id
|
||||
}
|
||||
mls_set
|
||||
nrds_id
|
||||
state_license
|
||||
rental_corporation {
|
||||
fulfillment_id
|
||||
}
|
||||
rental_management {
|
||||
name
|
||||
href
|
||||
fulfillment_id
|
||||
}
|
||||
}
|
||||
current_estimates {
|
||||
__typename
|
||||
source {
|
||||
__typename
|
||||
type
|
||||
name
|
||||
}
|
||||
estimate
|
||||
estimateHigh: estimate_high
|
||||
estimateLow: estimate_low
|
||||
date
|
||||
isBestHomeValue: isbest_homevalue
|
||||
}
|
||||
}
|
||||
"""
|
||||
|
||||
_SEARCH_HOMES_DATA_BASE = """{
|
||||
pending_date
|
||||
listing_id
|
||||
@@ -181,8 +371,189 @@ _SEARCH_HOMES_DATA_BASE = """{
|
||||
|
||||
|
||||
HOME_FRAGMENT = """
|
||||
fragment HomeData on Home {
|
||||
fragment HomeDetailsFragment on Home {
|
||||
__typename
|
||||
pending_date
|
||||
listing_id
|
||||
property_id
|
||||
href
|
||||
permalink
|
||||
list_date
|
||||
status
|
||||
mls_status
|
||||
last_sold_price
|
||||
last_sold_date
|
||||
last_status_change_date
|
||||
last_update_date
|
||||
list_price
|
||||
list_price_max
|
||||
list_price_min
|
||||
price_per_sqft
|
||||
tags
|
||||
open_houses {
|
||||
start_date
|
||||
end_date
|
||||
description
|
||||
time_zone
|
||||
dst
|
||||
href
|
||||
methods
|
||||
}
|
||||
details {
|
||||
category
|
||||
text
|
||||
parent_category
|
||||
}
|
||||
pet_policy {
|
||||
cats
|
||||
dogs
|
||||
dogs_small
|
||||
dogs_large
|
||||
__typename
|
||||
}
|
||||
units {
|
||||
availability {
|
||||
date
|
||||
__typename
|
||||
}
|
||||
description {
|
||||
baths_consolidated
|
||||
baths
|
||||
beds
|
||||
sqft
|
||||
__typename
|
||||
}
|
||||
photos(https: true) {
|
||||
title
|
||||
href
|
||||
tags {
|
||||
label
|
||||
}
|
||||
}
|
||||
list_price
|
||||
__typename
|
||||
}
|
||||
flags {
|
||||
is_contingent
|
||||
is_pending
|
||||
is_new_construction
|
||||
}
|
||||
description {
|
||||
type
|
||||
sqft
|
||||
beds
|
||||
baths_full
|
||||
baths_half
|
||||
lot_sqft
|
||||
year_built
|
||||
garage
|
||||
type
|
||||
name
|
||||
stories
|
||||
text
|
||||
}
|
||||
source {
|
||||
id
|
||||
listing_id
|
||||
}
|
||||
hoa {
|
||||
fee
|
||||
}
|
||||
location {
|
||||
address {
|
||||
street_direction
|
||||
street_number
|
||||
street_name
|
||||
street_suffix
|
||||
line
|
||||
unit
|
||||
city
|
||||
state_code
|
||||
postal_code
|
||||
coordinate {
|
||||
lon
|
||||
lat
|
||||
}
|
||||
}
|
||||
county {
|
||||
name
|
||||
fips_code
|
||||
}
|
||||
neighborhoods {
|
||||
name
|
||||
}
|
||||
parcel {
|
||||
parcel_id
|
||||
}
|
||||
}
|
||||
tax_record {
|
||||
cl_id
|
||||
public_record_id
|
||||
last_update_date
|
||||
apn
|
||||
tax_parcel_id
|
||||
}
|
||||
primary_photo(https: true) {
|
||||
href
|
||||
}
|
||||
photos(https: true) {
|
||||
title
|
||||
href
|
||||
tags {
|
||||
label
|
||||
}
|
||||
}
|
||||
advertisers {
|
||||
email
|
||||
broker {
|
||||
name
|
||||
fulfillment_id
|
||||
}
|
||||
type
|
||||
name
|
||||
fulfillment_id
|
||||
builder {
|
||||
name
|
||||
fulfillment_id
|
||||
}
|
||||
phones {
|
||||
ext
|
||||
primary
|
||||
type
|
||||
number
|
||||
}
|
||||
office {
|
||||
name
|
||||
email
|
||||
fulfillment_id
|
||||
href
|
||||
phones {
|
||||
number
|
||||
type
|
||||
primary
|
||||
ext
|
||||
}
|
||||
mls_set
|
||||
}
|
||||
corporation {
|
||||
specialties
|
||||
name
|
||||
bio
|
||||
href
|
||||
fulfillment_id
|
||||
}
|
||||
mls_set
|
||||
nrds_id
|
||||
state_license
|
||||
rental_corporation {
|
||||
fulfillment_id
|
||||
}
|
||||
rental_management {
|
||||
name
|
||||
href
|
||||
fulfillment_id
|
||||
}
|
||||
}
|
||||
nearbySchools: nearby_schools(radius: 5.0, limit_per_level: 3) {
|
||||
__typename schools { district { __typename id name } }
|
||||
}
|
||||
@@ -198,11 +569,6 @@ fragment HomeData on Home {
|
||||
last_n_days
|
||||
}
|
||||
}
|
||||
location {
|
||||
parcel {
|
||||
parcel_id
|
||||
}
|
||||
}
|
||||
taxHistory: tax_history { __typename tax year assessment { __typename building land total } }
|
||||
property_history {
|
||||
date
|
||||
@@ -227,6 +593,18 @@ fragment HomeData on Home {
|
||||
text
|
||||
category
|
||||
}
|
||||
estimates {
|
||||
__typename
|
||||
currentValues: current_values {
|
||||
__typename
|
||||
source { __typename type name }
|
||||
estimate
|
||||
estimateHigh: estimate_high
|
||||
estimateLow: estimate_low
|
||||
date
|
||||
isBestHomeValue: isbest_homevalue
|
||||
}
|
||||
}
|
||||
}
|
||||
"""
|
||||
|
||||
@@ -300,8 +678,38 @@ current_estimates {
|
||||
}
|
||||
}""" % _SEARCH_HOMES_DATA_BASE
|
||||
|
||||
GENERAL_RESULTS_QUERY = """{
|
||||
# Query body using inline fields (kept for backward compatibility)
|
||||
GENERAL_RESULTS_QUERY_BODY = """{
|
||||
count
|
||||
total
|
||||
results %s
|
||||
}""" % SEARCH_HOMES_DATA
|
||||
|
||||
GENERAL_RESULTS_QUERY = """{
|
||||
__typename
|
||||
count
|
||||
total
|
||||
results {
|
||||
__typename
|
||||
...SearchFragment
|
||||
...ListingPhotosFragment
|
||||
}
|
||||
}"""
|
||||
|
||||
LISTING_PHOTOS_FRAGMENT = """
|
||||
fragment ListingPhotosFragment on SearchHome {
|
||||
__typename
|
||||
photos(https: true) {
|
||||
__typename
|
||||
title
|
||||
href
|
||||
tags {
|
||||
__typename
|
||||
label
|
||||
probability
|
||||
}
|
||||
}
|
||||
}
|
||||
"""
|
||||
|
||||
MORPHEUS_SUGGESTIONS_QUERY = """query GetMorpheusSuggestions($searchInput: SearchSuggestionsInput!) { search_suggestions(search_input: $searchInput) { __typename geo_results { __typename type text geo { __typename _id _score mpr_id area_type city state_code postal_code country lat lon county counties { __typename name fips state_code } slug_id geo_id score name city_slug_id centroid { __typename lat lon } county_needed_for_uniq street line school school_id school_district school_district_id has_catchment university university_id neighborhood park } } no_matches has_results filter_criteria { __typename property_type { __typename type } price { __typename min max pattern } bed { __typename min max pattern } bath { __typename min max pattern } feature_tags { __typename tags } listing_status { __typename new_construction existing_homes foreclosures recently_sold fifty_five_plus open_house hide_new_construction hide_existing_homes hide_foreclosures hide_recently_sold hide_fifty_five_plus hide_open_house virtual_tour three_d_tour contingent hide_contingent pending hide_pending } keyword { __typename keywords } garage { __typename min max pattern } age { __typename min max pattern } stories { __typename min max pattern } lot_size { __typename min max pattern } square_feet { __typename min max pattern } home_size { __typename min max pattern } basement finished_basement pool waterfront fireplace detached_garage expand { __typename radius } hoa { __typename type fee } } message_data { __typename property_type pool waterfront fireplace basement finished_basement detached_garage listing_status { __typename new_construction existing_homes foreclosures recently_sold fifty_five_plus open_house hide_new_construction hide_existing_homes hide_foreclosures hide_recently_sold hide_fifty_five_plus hide_open_house } keywords price { __typename min max pattern } bed { __typename min max pattern } bath { __typename min max pattern } garage { __typename min max pattern } stories { __typename min max pattern } age { __typename min max pattern } lot_size { __typename min max pattern } square_feet { __typename min max pattern } } original_string morpheus_context } }"""
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
[tool.poetry]
|
||||
name = "homeharvest"
|
||||
version = "0.8.6b"
|
||||
version = "0.8.13"
|
||||
description = "Real estate scraping library"
|
||||
authors = ["Zachary Hampton <zachary@bunsly.com>", "Cullen Watson <cullen@bunsly.com>"]
|
||||
homepage = "https://github.com/ZacharyHampton/HomeHarvest"
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import pytz
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
|
||||
from homeharvest import scrape_property, Property
|
||||
import pandas as pd
|
||||
@@ -307,6 +308,30 @@ def test_phone_number_matching():
|
||||
assert row["agent_phones"].values[0] == matching_row["agent_phones"].values[0]
|
||||
|
||||
|
||||
def test_parallel_search_consistency():
|
||||
"""Test that the same search executed 3 times in parallel returns consistent results"""
|
||||
def search_task():
|
||||
return scrape_property(
|
||||
location="Phoenix, AZ",
|
||||
listing_type="for_sale",
|
||||
limit=100
|
||||
)
|
||||
|
||||
with ThreadPoolExecutor(max_workers=3) as executor:
|
||||
futures = [executor.submit(search_task) for _ in range(3)]
|
||||
results = [future.result() for future in as_completed(futures)]
|
||||
|
||||
# Verify all results are valid
|
||||
assert all([result is not None for result in results])
|
||||
assert all([isinstance(result, pd.DataFrame) for result in results])
|
||||
assert all([len(result) > 0 for result in results])
|
||||
|
||||
# Verify all results have the same length (primary consistency check)
|
||||
lengths = [len(result) for result in results]
|
||||
assert len(set(lengths)) == 1, \
|
||||
f"All parallel searches should return same number of results, got lengths: {lengths}"
|
||||
|
||||
|
||||
def test_return_type():
|
||||
results = {
|
||||
"pandas": [scrape_property(location="Surprise, AZ", listing_type="for_rent", limit=100)],
|
||||
|
||||
Reference in New Issue
Block a user