enh: assessed/estimated value (#77)

README.md

@@ -134,6 +134,13 @@ Property
 ├── Location Details:
 │ ├── latitude
 │ ├── longitude
+│ ├── nearby_schools
+
+
+├── Agent Info:
+│ ├── agent
+│ ├── broker
+│ └── broker_phone
 
 ├── Agent Info:
 │ ├── agent

homeharvest/core/scrapers/__init__.py

@@ -1,5 +1,7 @@
 from dataclasses import dataclass
 import requests
+from requests.adapters import HTTPAdapter
+from urllib3.util.retry import Retry
 import uuid
 from .models import Property, ListingType, SiteName
 
@@ -18,24 +20,30 @@ class ScraperInput:
 
 
 class Scraper:
+    session = None
+
     def __init__(
         self,
         scraper_input: ScraperInput,
-        session: requests.Session = None,
     ):
         self.location = scraper_input.location
         self.listing_type = scraper_input.listing_type
 
-        if not session:
+        if not self.session:
-            self.session = requests.Session()
+            Scraper.session = requests.Session()
-            self.session.headers.update(
+            retries = Retry(
+                total=3, backoff_factor=3, status_forcelist=[429, 403], allowed_methods=frozenset(["GET", "POST"])
+            )
+
+            adapter = HTTPAdapter(max_retries=retries)
+            Scraper.session.mount("http://", adapter)
+            Scraper.session.mount("https://", adapter)
+            Scraper.session.headers.update(
                 {
                     "auth": f"Bearer {self.get_access_token()}",
                     "apollographql-client-name": "com.move.Realtor-apollo-ios",
                 }
             )
-        else:
-            self.session = session
 
         if scraper_input.proxy:
             proxy_url = scraper_input.proxy
@@ -72,4 +80,8 @@ class Scraper:
         response = requests.post(url, headers=headers, data=payload)
 
         data = response.json()
-        return data["access_token"]
+        try:
+            access_token = data["access_token"]
+        except Exception:
+            raise Exception("Could not get access token, use a proxy/vpn or wait")
+        return access_token

homeharvest/core/scrapers/models.py

@@ -23,6 +23,12 @@ class ListingType(Enum):
     SOLD = "SOLD"
 
 
+@dataclass
+class Agent:
+    name: str | None = None
+    phone: str | None = None
+
+
 class PropertyType(Enum):
     APARTMENT = "APARTMENT"
     BUILDING = "BUILDING"
@@ -67,6 +73,7 @@ class Description:
     year_built: int | None = None
     garage: float | None = None
     stories: int | None = None
+    text: str | None = None
 
 
 @dataclass
@@ -95,5 +102,9 @@ class Property:
     latitude: float | None = None
     longitude: float | None = None
     neighborhoods: Optional[str] = None
-
+    county: Optional[str] = None
+    fips_code: Optional[str] = None
     agents: list[Agent] = None
+    nearby_schools: list[str] = None
+    assessed_value: int | None = None
+    estimated_value: int | None = None

homeharvest/core/scrapers/realtor/__init__.py

@@ -5,9 +5,9 @@ homeharvest.realtor.__init__
 This module implements the scraper for realtor.com
 """
 
+from concurrent.futures import ThreadPoolExecutor, as_completed
 from datetime import datetime
 from typing import Dict, Union, Optional
-from concurrent.futures import ThreadPoolExecutor, as_completed
 
 from .. import Scraper
 from ..models import Property, Address, ListingType, Description, PropertyType, Agent
@@ -142,7 +142,7 @@ class RealtorScraper(Scraper):
                 days_on_mls = None
 
         property_id = property_info["details"]["permalink"]
-        agents = self.get_agents(property_id)
+        prop_details = self.get_prop_details(property_id)
         listing = Property(
             mls=mls,
             mls_id=(
@@ -176,9 +176,13 @@ class RealtorScraper(Scraper):
                 year_built=property_info["details"].get("year_built"),
                 garage=property_info["details"].get("garage"),
                 stories=property_info["details"].get("stories"),
+                text=property_info.get("description", {}).get("text"),
             ),
             days_on_mls=days_on_mls,
-            agents=agents,
+            agents=prop_details.get("agents"),
+            nearby_schools=prop_details.get("schools"),
+            assessed_value=prop_details.get("assessed_value"),
+            estimated_value=prop_details.get("estimated_value"),
         )
 
         return [listing]
@@ -272,7 +276,7 @@ class RealtorScraper(Scraper):
                 }"""
 
         variables = {"property_id": property_id}
-        agents = self.get_agents(property_id)
+        prop_details = self.get_prop_details(property_id)
 
         payload = {
             "query": query,
@@ -290,7 +294,10 @@ class RealtorScraper(Scraper):
                 property_url=f"{self.PROPERTY_URL}{property_info['details']['permalink']}",
                 address=self._parse_address(property_info, search_type="handle_address"),
                 description=self._parse_description(property_info),
-                agents=agents,
+                agents=prop_details.get("agents"),
+                nearby_schools=prop_details.get("schools"),
+                assessed_value=prop_details.get("assessed_value"),
+                estimated_value=prop_details.get("estimated_value"),
             )
         ]
 
@@ -328,6 +335,7 @@ class RealtorScraper(Scraper):
                                     type
                                     name
                                     stories
+                                    text
                                 }
                                 source {
                                     id
@@ -351,10 +359,17 @@ class RealtorScraper(Scraper):
                                             lat
                                         }
                                     }
+                                    county {
+                                        name
+                                        fips_code
+                                    }
                                     neighborhoods {
                                         name
                                     }
                                 }
+                                tax_record {
+                                    public_record_id
+                                }
                                 primary_photo {
                                     href
                                 }
@@ -475,7 +490,6 @@ class RealtorScraper(Scraper):
         }
 
         response = self.session.post(self.SEARCH_GQL_URL, json=payload)
-        response.raise_for_status()
         response_json = response.json()
         search_key = "home_search" if "home_search" in query else "property_search"
 
@@ -510,7 +524,7 @@ class RealtorScraper(Scraper):
                 return
 
             property_id = result["property_id"]
-            agents = self.get_agents(property_id)
+            prop_details = self.get_prop_details(property_id)
 
             realty_property = Property(
                 mls=mls,
@@ -534,8 +548,14 @@ class RealtorScraper(Scraper):
                 longitude=result["location"]["address"]["coordinate"].get("lon") if able_to_get_lat_long else None,
                 address=self._parse_address(result, search_type="general_search"),
                 description=self._parse_description(result),
+                neighborhoods=self._parse_neighborhoods(result),
+                county=result["location"]["county"].get("name") if result["location"]["county"] else None,
+                fips_code=result["location"]["county"].get("fips_code") if result["location"]["county"] else None,
                 days_on_mls=self.calculate_days_on_mls(result),
-                agents=agents,
+                agents=prop_details.get("agents"),
+                nearby_schools=prop_details.get("schools"),
+                assessed_value=prop_details.get("assessed_value"),
+                estimated_value=prop_details.get("estimated_value"),
             )
             return realty_property
 
@@ -630,18 +650,33 @@ class RealtorScraper(Scraper):
 
         return homes
 
-    def get_agents(self, property_id: str) -> list[Agent]:
+    def get_prop_details(self, property_id: str) -> dict:
-        payload = f'{{"query":"query GetHome($property_id: ID!) {{\\n  home(property_id: $property_id) {{\\n    __typename\\n\\n    consumerAdvertisers: consumer_advertisers {{\\n      __typename\\n      type\\n      advertiserId: advertiser_id\\n      name\\n      phone\\n      type\\n      href\\n      slogan\\n      photo {{\\n        __typename\\n        href\\n      }}\\n      showRealtorLogo: show_realtor_logo\\n      hours\\n    }}\\n\\n\\n  }}\\n}}\\n","variables":{{"property_id":"{property_id}"}}}}'
+        payload = f'{{"query":"query GetHome($property_id: ID!) {{\\n  home(property_id: $property_id) {{\\n    __typename\\n\\n    consumerAdvertisers: consumer_advertisers {{\\n      __typename\\n      type\\n      advertiserId: advertiser_id\\n      name\\n      phone\\n      type\\n      href\\n      slogan\\n      photo {{\\n        __typename\\n        href\\n      }}\\n      showRealtorLogo: show_realtor_logo\\n      hours\\n    }}\\n\\n\\n  nearbySchools: nearby_schools(radius: 5.0, limit_per_level: 3) {{ __typename schools {{ district {{ __typename id name }} }} }} taxHistory: tax_history {{ __typename tax year assessment {{ __typename building land total }} }}estimates {{ __typename currentValues: current_values {{ __typename source {{ __typename type name }} estimate estimateHigh: estimate_high estimateLow: estimate_low date isBestHomeValue: isbest_homevalue }} }}    }}\\n}}\\n","variables":{{"property_id":"{property_id}"}}}}'
         response = self.session.post(self.PROPERTY_GQL, data=payload)
 
-        data = response.json()
+        def get_key(keys: list):
-        try:
+            try:
-            ads = data["data"]["home"]["consumerAdvertisers"]
+                data = response.json()
-        except (KeyError, TypeError):
+                for key in keys:
-            return []
+                    data = data[key]
+                return data
+            except (KeyError, TypeError):
+                return {}
+
+        ads = get_key(["data", "home", "consumerAdvertisers"])
+        schools = get_key(["data", "home", "nearbySchools", "schools"])
+        assessed_value = get_key(["data", "home", "taxHistory", 0, "assessment", "total"])
+        estimated_value = get_key(["data", "home", "estimates", "currentValues", 0, "estimate"])
 
         agents = [Agent(name=ad["name"], phone=ad["phone"]) for ad in ads]
-        return agents
+
+        schools = [school["district"]["name"] for school in schools]
+        return {
+            "agents": agents if agents else None,
+            "schools": schools if schools else None,
+            "assessed_value": assessed_value if assessed_value else None,
+            "estimated_value": estimated_value if estimated_value else None,
+        }
 
     @staticmethod
     def _parse_neighborhoods(result: dict) -> Optional[str]:
@@ -715,6 +750,7 @@ class RealtorScraper(Scraper):
             year_built=description_data.get("year_built"),
             garage=description_data.get("garage"),
             stories=description_data.get("stories"),
+            text=description_data.get("text"),
         )
 
     @staticmethod

homeharvest/utils.py

@@ -8,6 +8,7 @@ ordered_properties = [
     "mls",
     "mls_id",
     "status",
+    "text",
     "style",
     "street",
     "unit",
@@ -24,16 +25,22 @@ ordered_properties = [
     "list_date",
     "sold_price",
     "last_sold_date",
+    "assessed_value",
+    "estimated_value",
     "lot_sqft",
     "price_per_sqft",
     "latitude",
     "longitude",
+    "neighborhoods",
+    "county",
+    "fips_code",
     "stories",
     "hoa_fee",
     "parking_garage",
     "agent",
     "broker",
     "broker_phone",
+    "nearby_schools",
     "primary_photo",
     "alt_photos",
 ]
@@ -60,11 +67,13 @@ def process_result(result: Property) -> pd.DataFrame:
                 prop_data["broker_phone"] = agents[1].phone
 
     prop_data["price_per_sqft"] = prop_data["prc_sqft"]
+    prop_data["nearby_schools"] = filter(None, prop_data["nearby_schools"]) if prop_data["nearby_schools"] else None
+    prop_data["nearby_schools"] = ", ".join(set(prop_data["nearby_schools"])) if prop_data["nearby_schools"] else None
 
     description = result.description
     prop_data["primary_photo"] = description.primary_photo
     prop_data["alt_photos"] = ", ".join(description.alt_photos)
-    prop_data["style"] = description.style.value
+    prop_data["style"] = description.style if type(description.style) == str else description.style.value
     prop_data["beds"] = description.beds
     prop_data["full_baths"] = description.baths_full
     prop_data["half_baths"] = description.baths_half
@@ -74,6 +83,7 @@ def process_result(result: Property) -> pd.DataFrame:
     prop_data["year_built"] = description.year_built
     prop_data["parking_garage"] = description.garage
     prop_data["stories"] = description.stories
+    prop_data["text"] = description.text
 
     properties_df = pd.DataFrame([prop_data])
     properties_df = properties_df.reindex(columns=ordered_properties)

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "homeharvest"
-version = "0.3.17"
+version = "0.3.20"
 description = "Real estate scraping library"
 authors = ["Zachary Hampton <zachary@bunsly.com>", "Cullen Watson <cullen@bunsly.com>"]
 homepage = "https://github.com/Bunsly/HomeHarvest"