- location bug fix

pull/65/head v0.3.13
Zachary Hampton 2024-03-06 16:31:06 -07:00
parent 5ea9a6f6b6
commit d775540afd
3 changed files with 38 additions and 27 deletions

View File

@ -158,7 +158,8 @@ class RealtorScraper(Scraper):
else None, else None,
address=self._parse_address(property_info, search_type="handle_listing"), address=self._parse_address(property_info, search_type="handle_listing"),
description=Description( description=Description(
primary_photo=property_info["primary_photo"].get("href", "").replace("s.jpg", "od-w480_h360_x2.webp?w=1080&q=75"), primary_photo=property_info["primary_photo"].get("href", "").replace("s.jpg",
"od-w480_h360_x2.webp?w=1080&q=75"),
alt_photos=self.process_alt_photos(property_info.get("photos", [])), alt_photos=self.process_alt_photos(property_info.get("photos", [])),
style=property_info["basic"].get("type", "").upper(), style=property_info["basic"].get("type", "").upper(),
beds=property_info["basic"].get("beds"), beds=property_info["basic"].get("beds"),
@ -288,7 +289,7 @@ class RealtorScraper(Scraper):
] ]
def general_search( def general_search(
self, variables: dict, search_type: str self, variables: dict, search_type: str
) -> Dict[str, Union[int, list[Property]]]: ) -> Dict[str, Union[int, list[Property]]]:
""" """
Handles a location area & returns a list of properties Handles a location area & returns a list of properties
@ -385,11 +386,10 @@ class RealtorScraper(Scraper):
listing_type = ListingType.FOR_SALE if self.listing_type == ListingType.PENDING else self.listing_type listing_type = ListingType.FOR_SALE if self.listing_type == ListingType.PENDING else self.listing_type
is_foreclosure = "" is_foreclosure = ""
if 'foreclosure' in variables and variables['foreclosure'] == True: if variables.get('foreclosure') is True:
is_foreclosure = "foreclosure: true" is_foreclosure = "foreclosure: true"
elif variables.get('foreclosure') is False:
if 'foreclosure' in variables and variables['foreclosure'] == False: is_foreclosure = "foreclosure: false"
is_foreclosure = "foreclosure: false"
if search_type == "comps": #: comps search, came from an address if search_type == "comps": #: comps search, came from an address
query = """query Property_search( query = """query Property_search(
@ -451,7 +451,7 @@ class RealtorScraper(Scraper):
) )
else: #: general search, came from an address else: #: general search, came from an address
query = ( query = (
"""query Property_search( """query Property_search(
$property_id: [ID]! $property_id: [ID]!
$offset: Int!, $offset: Int!,
) { ) {
@ -462,7 +462,7 @@ class RealtorScraper(Scraper):
limit: 1 limit: 1
offset: $offset offset: $offset
) %s""" ) %s"""
% results_query % results_query
) )
payload = { payload = {
@ -478,12 +478,12 @@ class RealtorScraper(Scraper):
properties: list[Property] = [] properties: list[Property] = []
if ( if (
response_json is None response_json is None
or "data" not in response_json or "data" not in response_json
or response_json["data"] is None or response_json["data"] is None
or search_key not in response_json["data"] or search_key not in response_json["data"]
or response_json["data"][search_key] is None or response_json["data"][search_key] is None
or "results" not in response_json["data"][search_key] or "results" not in response_json["data"][search_key]
): ):
return {"total": 0, "properties": []} return {"total": 0, "properties": []}
@ -498,10 +498,10 @@ class RealtorScraper(Scraper):
continue continue
able_to_get_lat_long = ( able_to_get_lat_long = (
result result
and result.get("location") and result.get("location")
and result["location"].get("address") and result["location"].get("address")
and result["location"]["address"].get("coordinate") and result["location"]["address"].get("coordinate")
) )
is_pending = result["flags"].get("is_pending") or result["flags"].get("is_contingent") is_pending = result["flags"].get("is_pending") or result["flags"].get("is_contingent")
@ -578,6 +578,9 @@ class RealtorScraper(Scraper):
return gql_results["properties"] return gql_results["properties"]
else: #: general search, comps (radius) else: #: general search, comps (radius)
if not location_info.get("centroid"):
return []
coordinates = list(location_info["centroid"].values()) coordinates = list(location_info["centroid"].values())
search_variables |= { search_variables |= {
"coordinates": coordinates, "coordinates": coordinates,
@ -692,7 +695,6 @@ class RealtorScraper(Scraper):
stories=description_data.get("stories"), stories=description_data.get("stories"),
) )
@staticmethod @staticmethod
def calculate_days_on_mls(result: dict) -> Optional[int]: def calculate_days_on_mls(result: dict) -> Optional[int]:
list_date_str = result.get("list_date") list_date_str = result.get("list_date")

View File

@ -1,6 +1,6 @@
[tool.poetry] [tool.poetry]
name = "homeharvest" name = "homeharvest"
version = "0.3.12" version = "0.3.13"
description = "Real estate scraping library" description = "Real estate scraping library"
authors = ["Zachary Hampton <zachary@bunsly.com>", "Cullen Watson <cullen@bunsly.com>"] authors = ["Zachary Hampton <zachary@bunsly.com>", "Cullen Watson <cullen@bunsly.com>"]
homepage = "https://github.com/Bunsly/HomeHarvest" homepage = "https://github.com/Bunsly/HomeHarvest"

View File

@ -131,6 +131,15 @@ def test_realtor():
assert all([result is not None for result in results]) assert all([result is not None for result in results])
def test_realtor_city():
results = scrape_property(
location="Atlanta, GA",
listing_type="for_sale",
)
assert results is not None and len(results) > 0
def test_realtor_bad_address(): def test_realtor_bad_address():
bad_results = scrape_property( bad_results = scrape_property(
location="abceefg ju098ot498hh9", location="abceefg ju098ot498hh9",