Schools (#69)

README.md

@@ -134,6 +134,13 @@ Property
 ├── Location Details:
 │ ├── latitude
 │ ├── longitude
+│ ├── nearby_schools
+
+
+├── Agent Info:
+│ ├── agent
+│ ├── broker
+│ └── broker_phone
 
 ├── Agent Info:
 │ ├── agent

homeharvest/core/scrapers/__init__.py

@@ -1,3 +1,4 @@
+import uuid
 from dataclasses import dataclass
 import requests
 import uuid

homeharvest/core/scrapers/models.py

@@ -23,6 +23,12 @@ class ListingType(Enum):
     SOLD = "SOLD"
 
 
+@dataclass
+class Agent:
+    name: str | None = None
+    phone: str | None = None
+
+
 class PropertyType(Enum):
     APARTMENT = "APARTMENT"
     BUILDING = "BUILDING"
@@ -97,3 +103,4 @@ class Property:
     neighborhoods: Optional[str] = None
 
     agents: list[Agent] = None
+    nearby_schools: list[str] = None

homeharvest/core/scrapers/realtor/__init__.py

@@ -142,7 +142,7 @@ class RealtorScraper(Scraper):
                 days_on_mls = None
 
         property_id = property_info["details"]["permalink"]
-        agents = self.get_agents(property_id)
+        agents_schools = self.get_agents_schools(property_id)
         listing = Property(
             mls=mls,
             mls_id=(
@@ -178,7 +178,8 @@ class RealtorScraper(Scraper):
                 stories=property_info["details"].get("stories"),
             ),
             days_on_mls=days_on_mls,
-            agents=agents,
+            agents=agents_schools["agents"],
+            nearby_schools=agents_schools["schools"],
         )
 
         return [listing]
@@ -272,7 +273,7 @@ class RealtorScraper(Scraper):
                 }"""
 
         variables = {"property_id": property_id}
-        agents = self.get_agents(property_id)
+        agents_schools = self.get_agents_schools(property_id)
 
         payload = {
             "query": query,
@@ -290,7 +291,8 @@ class RealtorScraper(Scraper):
                 property_url=f"{self.PROPERTY_URL}{property_info['details']['permalink']}",
                 address=self._parse_address(property_info, search_type="handle_address"),
                 description=self._parse_description(property_info),
-                agents=agents,
+                agents=agents_schools["agents"],
+                nearby_schools=agents_schools["schools"],
             )
         ]
 
@@ -510,7 +512,7 @@ class RealtorScraper(Scraper):
                 return
 
             property_id = result["property_id"]
-            agents = self.get_agents(property_id)
+            agents_schools = self.get_agents_schools(property_id)
 
             realty_property = Property(
                 mls=mls,
@@ -535,7 +537,8 @@ class RealtorScraper(Scraper):
                 address=self._parse_address(result, search_type="general_search"),
                 description=self._parse_description(result),
                 days_on_mls=self.calculate_days_on_mls(result),
-                agents=agents,
+                agents=agents_schools["agents"],
+                nearby_schools=agents_schools["schools"],
             )
             return realty_property
 
@@ -630,18 +633,25 @@ class RealtorScraper(Scraper):
 
         return homes
 
-    def get_agents(self, property_id: str) -> list[Agent]:
+    def get_agents_schools(self, property_id: str) -> dict:
-        payload = f'{{"query":"query GetHome($property_id: ID!) {{\\n  home(property_id: $property_id) {{\\n    __typename\\n\\n    consumerAdvertisers: consumer_advertisers {{\\n      __typename\\n      type\\n      advertiserId: advertiser_id\\n      name\\n      phone\\n      type\\n      href\\n      slogan\\n      photo {{\\n        __typename\\n        href\\n      }}\\n      showRealtorLogo: show_realtor_logo\\n      hours\\n    }}\\n\\n\\n  }}\\n}}\\n","variables":{{"property_id":"{property_id}"}}}}'
+        payload = f'{{"query":"query GetHome($property_id: ID!) {{\\n  home(property_id: $property_id) {{\\n    __typename\\n\\n    consumerAdvertisers: consumer_advertisers {{\\n      __typename\\n      type\\n      advertiserId: advertiser_id\\n      name\\n      phone\\n      type\\n      href\\n      slogan\\n      photo {{\\n        __typename\\n        href\\n      }}\\n      showRealtorLogo: show_realtor_logo\\n      hours\\n    }}\\n\\n\\n  nearbySchools: nearby_schools(radius: 5.0, limit_per_level: 3) {{ __typename schools {{ district {{ __typename id name }} }} }}}}\\n}}\\n","variables":{{"property_id":"{property_id}"}}}}'
         response = self.session.post(self.PROPERTY_GQL, data=payload)
 
-        data = response.json()
+        def get_key(keys: list):
-        try:
+            try:
-            ads = data["data"]["home"]["consumerAdvertisers"]
+                data = response.json()
-        except (KeyError, TypeError):
+                for key in keys:
-            return []
+                    data = data[key]
+                return data
+            except (KeyError, TypeError):
+                return []
+
+        ads = get_key(["data", "home", "consumerAdvertisers"])
+        schools = get_key(["data", "home", "nearbySchools", "schools"])
 
         agents = [Agent(name=ad["name"], phone=ad["phone"]) for ad in ads]
-        return agents
+        schools = [school["district"]["name"] for school in schools]
+        return {"agents": agents, "schools": schools}
 
     @staticmethod
     def _parse_neighborhoods(result: dict) -> Optional[str]:

homeharvest/utils.py

@@ -34,6 +34,7 @@ ordered_properties = [
     "agent",
     "broker",
     "broker_phone",
+    "nearby_schools",
     "primary_photo",
     "alt_photos",
 ]
@@ -60,6 +61,8 @@ def process_result(result: Property) -> pd.DataFrame:
                 prop_data["broker_phone"] = agents[1].phone
 
     prop_data["price_per_sqft"] = prop_data["prc_sqft"]
+    prop_data["nearby_schools"] = filter(None, prop_data["nearby_schools"]) if prop_data["nearby_schools"] else None
+    prop_data["nearby_schools"] = ", ".join(set(prop_data["nearby_schools"])) if prop_data["nearby_schools"] else None
 
     description = result.description
     prop_data["primary_photo"] = description.primary_photo

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "homeharvest"
-version = "0.3.17"
+version = "0.3.18"
 description = "Real estate scraping library"
 authors = ["Zachary Hampton <zachary@bunsly.com>", "Cullen Watson <cullen@bunsly.com>"]
 homepage = "https://github.com/Bunsly/HomeHarvest"