- location bug fix

pull/65/head v0.3.13
Zachary Hampton 2024-03-06 16:31:06 -07:00
parent 5ea9a6f6b6
commit d775540afd
3 changed files with 38 additions and 27 deletions

View File

@ -158,7 +158,8 @@ class RealtorScraper(Scraper):
else None,
address=self._parse_address(property_info, search_type="handle_listing"),
description=Description(
primary_photo=property_info["primary_photo"].get("href", "").replace("s.jpg", "od-w480_h360_x2.webp?w=1080&q=75"),
primary_photo=property_info["primary_photo"].get("href", "").replace("s.jpg",
"od-w480_h360_x2.webp?w=1080&q=75"),
alt_photos=self.process_alt_photos(property_info.get("photos", [])),
style=property_info["basic"].get("type", "").upper(),
beds=property_info["basic"].get("beds"),
@ -288,7 +289,7 @@ class RealtorScraper(Scraper):
]
def general_search(
self, variables: dict, search_type: str
self, variables: dict, search_type: str
) -> Dict[str, Union[int, list[Property]]]:
"""
Handles a location area & returns a list of properties
@ -381,16 +382,15 @@ class RealtorScraper(Scraper):
if self.listing_type == ListingType.PENDING
else ""
)
listing_type = ListingType.FOR_SALE if self.listing_type == ListingType.PENDING else self.listing_type
is_foreclosure = ""
if 'foreclosure' in variables and variables['foreclosure'] == True:
is_foreclosure = "foreclosure: true"
if 'foreclosure' in variables and variables['foreclosure'] == False:
is_foreclosure = "foreclosure: false"
if variables.get('foreclosure') is True:
is_foreclosure = "foreclosure: true"
elif variables.get('foreclosure') is False:
is_foreclosure = "foreclosure: false"
if search_type == "comps": #: comps search, came from an address
query = """query Property_search(
$coordinates: [Float]!
@ -412,7 +412,7 @@ class RealtorScraper(Scraper):
limit: 200
offset: $offset
) %s""" % (
is_foreclosure,
is_foreclosure,
listing_type.value.lower(),
date_param,
pending_or_contingent_param,
@ -451,7 +451,7 @@ class RealtorScraper(Scraper):
)
else: #: general search, came from an address
query = (
"""query Property_search(
"""query Property_search(
$property_id: [ID]!
$offset: Int!,
) {
@ -462,7 +462,7 @@ class RealtorScraper(Scraper):
limit: 1
offset: $offset
) %s"""
% results_query
% results_query
)
payload = {
@ -478,12 +478,12 @@ class RealtorScraper(Scraper):
properties: list[Property] = []
if (
response_json is None
or "data" not in response_json
or response_json["data"] is None
or search_key not in response_json["data"]
or response_json["data"][search_key] is None
or "results" not in response_json["data"][search_key]
response_json is None
or "data" not in response_json
or response_json["data"] is None
or search_key not in response_json["data"]
or response_json["data"][search_key] is None
or "results" not in response_json["data"][search_key]
):
return {"total": 0, "properties": []}
@ -498,10 +498,10 @@ class RealtorScraper(Scraper):
continue
able_to_get_lat_long = (
result
and result.get("location")
and result["location"].get("address")
and result["location"]["address"].get("coordinate")
result
and result.get("location")
and result["location"].get("address")
and result["location"]["address"].get("coordinate")
)
is_pending = result["flags"].get("is_pending") or result["flags"].get("is_contingent")
@ -552,7 +552,7 @@ class RealtorScraper(Scraper):
search_variables = {
"offset": 0,
}
search_type = (
"comps"
if self.radius and location_type == "address"
@ -578,6 +578,9 @@ class RealtorScraper(Scraper):
return gql_results["properties"]
else: #: general search, comps (radius)
if not location_info.get("centroid"):
return []
coordinates = list(location_info["centroid"].values())
search_variables |= {
"coordinates": coordinates,
@ -597,7 +600,7 @@ class RealtorScraper(Scraper):
"postal_code": location_info.get("postal_code"),
}
if self.foreclosure:
if self.foreclosure:
search_variables['foreclosure'] = self.foreclosure
result = self.general_search(search_variables, search_type=search_type)
@ -692,7 +695,6 @@ class RealtorScraper(Scraper):
stories=description_data.get("stories"),
)
@staticmethod
def calculate_days_on_mls(result: dict) -> Optional[int]:
list_date_str = result.get("list_date")

View File

@ -1,6 +1,6 @@
[tool.poetry]
name = "homeharvest"
version = "0.3.12"
version = "0.3.13"
description = "Real estate scraping library"
authors = ["Zachary Hampton <zachary@bunsly.com>", "Cullen Watson <cullen@bunsly.com>"]
homepage = "https://github.com/Bunsly/HomeHarvest"

View File

@ -131,6 +131,15 @@ def test_realtor():
assert all([result is not None for result in results])
def test_realtor_city():
results = scrape_property(
location="Atlanta, GA",
listing_type="for_sale",
)
assert results is not None and len(results) > 0
def test_realtor_bad_address():
bad_results = scrape_property(
location="abceefg ju098ot498hh9",