mirror of
https://github.com/Bunsly/HomeHarvest.git
synced 2026-03-04 19:44:29 -08:00
Compare commits
1 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
6f3faceb27 |
@@ -17,6 +17,7 @@ from typing import Dict, Union
|
|||||||
from tenacity import (
|
from tenacity import (
|
||||||
retry,
|
retry,
|
||||||
retry_if_exception_type,
|
retry_if_exception_type,
|
||||||
|
retry_if_not_exception_type,
|
||||||
wait_exponential,
|
wait_exponential,
|
||||||
stop_after_attempt,
|
stop_after_attempt,
|
||||||
)
|
)
|
||||||
@@ -1110,7 +1111,7 @@ class RealtorScraper(Scraper):
|
|||||||
|
|
||||||
|
|
||||||
@retry(
|
@retry(
|
||||||
retry=retry_if_exception_type((JSONDecodeError, Exception)),
|
retry=retry_if_exception_type((JSONDecodeError, Exception)) & retry_if_not_exception_type(AuthenticationError),
|
||||||
wait=wait_exponential(multiplier=1, min=1, max=10),
|
wait=wait_exponential(multiplier=1, min=1, max=10),
|
||||||
stop=stop_after_attempt(3),
|
stop=stop_after_attempt(3),
|
||||||
)
|
)
|
||||||
@@ -1125,16 +1126,16 @@ class RealtorScraper(Scraper):
|
|||||||
property_ids = list(set(property_ids))
|
property_ids = list(set(property_ids))
|
||||||
|
|
||||||
fragments = "\n".join(
|
fragments = "\n".join(
|
||||||
f'home_{property_id}: home(property_id: {property_id}) {{ ...HomeDetailsFragment }}'
|
f'home_{property_id}: home(property_id: {property_id}) {{ ...SearchFragment }}'
|
||||||
for property_id in property_ids
|
for property_id in property_ids
|
||||||
)
|
)
|
||||||
query = f"""{HOME_FRAGMENT}
|
query = f"""{HOME_FRAGMENT}
|
||||||
|
|
||||||
query GetHomeDetails {{
|
query GetHome {{
|
||||||
{fragments}
|
{fragments}
|
||||||
}}"""
|
}}"""
|
||||||
|
|
||||||
data = self._graphql_post(query, {}, "GetHomeDetails")
|
data = self._graphql_post(query, {}, "GetHome")
|
||||||
|
|
||||||
if "data" not in data or data["data"] is None:
|
if "data" not in data or data["data"] is None:
|
||||||
# If we got a 400 error with "Required parameter is missing", raise to trigger retry
|
# If we got a 400 error with "Required parameter is missing", raise to trigger retry
|
||||||
|
|||||||
@@ -371,7 +371,7 @@ _SEARCH_HOMES_DATA_BASE = """{
|
|||||||
|
|
||||||
|
|
||||||
HOME_FRAGMENT = """
|
HOME_FRAGMENT = """
|
||||||
fragment HomeDetailsFragment on Home {
|
fragment SearchFragment on Home {
|
||||||
__typename
|
__typename
|
||||||
pending_date
|
pending_date
|
||||||
listing_id
|
listing_id
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
[tool.poetry]
|
[tool.poetry]
|
||||||
name = "homeharvest"
|
name = "homeharvest"
|
||||||
version = "0.8.13"
|
version = "0.8.14"
|
||||||
description = "Real estate scraping library"
|
description = "Real estate scraping library"
|
||||||
authors = ["Zachary Hampton <zachary@bunsly.com>", "Cullen Watson <cullen@bunsly.com>"]
|
authors = ["Zachary Hampton <zachary@bunsly.com>", "Cullen Watson <cullen@bunsly.com>"]
|
||||||
homepage = "https://github.com/ZacharyHampton/HomeHarvest"
|
homepage = "https://github.com/ZacharyHampton/HomeHarvest"
|
||||||
|
|||||||
Reference in New Issue
Block a user