mirror of
https://github.com/Bunsly/HomeHarvest.git
synced 2026-03-04 19:44:29 -08:00
Compare commits
3 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
6f3faceb27 | ||
|
|
cab0216f29 | ||
|
|
8ee720ce5c |
@@ -5,6 +5,7 @@ import requests
|
||||
from requests.adapters import HTTPAdapter
|
||||
from urllib3.util.retry import Retry
|
||||
import uuid
|
||||
import secrets
|
||||
from ...exceptions import AuthenticationError
|
||||
from .models import Property, ListingType, SiteName, SearchPropertyType, ReturnType
|
||||
import json
|
||||
@@ -87,6 +88,7 @@ class Scraper:
|
||||
'Accept-Language': 'en-US,en;q=0.9',
|
||||
'rdc-client-version': '26.11.1',
|
||||
'X-APOLLO-OPERATION-TYPE': 'query',
|
||||
'X-APOLLO-OPERATION-ID': secrets.token_hex(32),
|
||||
'rdc-client-name': 'RDC_NATIVE_MOBILE-iPhone-com.move.Realtor',
|
||||
'apollographql-client-name': 'com.move.Realtor-apollo-ios',
|
||||
'User-Agent': 'Realtor.com/26.11.1.1106489 CFNetwork/3860.200.71 Darwin/25.1.0',
|
||||
|
||||
@@ -17,6 +17,7 @@ from typing import Dict, Union
|
||||
from tenacity import (
|
||||
retry,
|
||||
retry_if_exception_type,
|
||||
retry_if_not_exception_type,
|
||||
wait_exponential,
|
||||
stop_after_attempt,
|
||||
)
|
||||
@@ -1110,7 +1111,7 @@ class RealtorScraper(Scraper):
|
||||
|
||||
|
||||
@retry(
|
||||
retry=retry_if_exception_type((JSONDecodeError, Exception)),
|
||||
retry=retry_if_exception_type((JSONDecodeError, Exception)) & retry_if_not_exception_type(AuthenticationError),
|
||||
wait=wait_exponential(multiplier=1, min=1, max=10),
|
||||
stop=stop_after_attempt(3),
|
||||
)
|
||||
@@ -1125,16 +1126,16 @@ class RealtorScraper(Scraper):
|
||||
property_ids = list(set(property_ids))
|
||||
|
||||
fragments = "\n".join(
|
||||
f'home_{property_id}: home(property_id: {property_id}) {{ ...HomeDetailsFragment }}'
|
||||
f'home_{property_id}: home(property_id: {property_id}) {{ ...SearchFragment }}'
|
||||
for property_id in property_ids
|
||||
)
|
||||
query = f"""{HOME_FRAGMENT}
|
||||
|
||||
query GetHomeDetails {{
|
||||
query GetHome {{
|
||||
{fragments}
|
||||
}}"""
|
||||
|
||||
data = self._graphql_post(query, {}, "GetHomeDetails")
|
||||
data = self._graphql_post(query, {}, "GetHome")
|
||||
|
||||
if "data" not in data or data["data"] is None:
|
||||
# If we got a 400 error with "Required parameter is missing", raise to trigger retry
|
||||
|
||||
@@ -371,7 +371,7 @@ _SEARCH_HOMES_DATA_BASE = """{
|
||||
|
||||
|
||||
HOME_FRAGMENT = """
|
||||
fragment HomeDetailsFragment on Home {
|
||||
fragment SearchFragment on Home {
|
||||
__typename
|
||||
pending_date
|
||||
listing_id
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
[tool.poetry]
|
||||
name = "homeharvest"
|
||||
version = "0.8.11"
|
||||
version = "0.8.14"
|
||||
description = "Real estate scraping library"
|
||||
authors = ["Zachary Hampton <zachary@bunsly.com>", "Cullen Watson <cullen@bunsly.com>"]
|
||||
homepage = "https://github.com/ZacharyHampton/HomeHarvest"
|
||||
|
||||
Reference in New Issue
Block a user