chore: version

README.md

@@ -134,6 +134,13 @@ Property
 ├── Location Details:
 │ ├── latitude
 │ ├── longitude
+│ ├── nearby_schools
+
+
+├── Agent Info:
+│ ├── agent
+│ ├── broker
+│ └── broker_phone
 
 ├── Agent Info:
 │ ├── agent

homeharvest/core/scrapers/__init__.py

@@ -1,3 +1,4 @@
+import uuid
 from dataclasses import dataclass
 import requests
 import uuid

homeharvest/core/scrapers/models.py

@@ -23,6 +23,12 @@ class ListingType(Enum):
     SOLD = "SOLD"
 
 
+@dataclass
+class Agent:
+    name: str | None = None
+    phone: str | None = None
+
+
 class PropertyType(Enum):
     APARTMENT = "APARTMENT"
     BUILDING = "BUILDING"
@@ -67,6 +73,7 @@ class Description:
     year_built: int | None = None
     garage: float | None = None
     stories: int | None = None
+    text: str | None = None
 
 
 @dataclass
@@ -95,5 +102,7 @@ class Property:
     latitude: float | None = None
     longitude: float | None = None
     neighborhoods: Optional[str] = None
-
+    county: Optional[str] = None
+    fips_code: Optional[str] = None
     agents: list[Agent] = None
+    nearby_schools: list[str] = None

homeharvest/core/scrapers/realtor/__init__.py

@@ -142,7 +142,7 @@ class RealtorScraper(Scraper):
                 days_on_mls = None
 
         property_id = property_info["details"]["permalink"]
-        agents = self.get_agents(property_id)
+        agents_schools = self.get_agents_schools(property_id)
         listing = Property(
             mls=mls,
             mls_id=(
@@ -176,9 +176,11 @@ class RealtorScraper(Scraper):
                 year_built=property_info["details"].get("year_built"),
                 garage=property_info["details"].get("garage"),
                 stories=property_info["details"].get("stories"),
+                text=property_info["description"].get("text"),
             ),
             days_on_mls=days_on_mls,
-            agents=agents,
+            agents=agents_schools["agents"],
+            nearby_schools=agents_schools["schools"],
         )
 
         return [listing]
@@ -272,7 +274,7 @@ class RealtorScraper(Scraper):
                 }"""
 
         variables = {"property_id": property_id}
-        agents = self.get_agents(property_id)
+        agents_schools = self.get_agents_schools(property_id)
 
         payload = {
             "query": query,
@@ -290,7 +292,8 @@ class RealtorScraper(Scraper):
                 property_url=f"{self.PROPERTY_URL}{property_info['details']['permalink']}",
                 address=self._parse_address(property_info, search_type="handle_address"),
                 description=self._parse_description(property_info),
-                agents=agents,
+                agents=agents_schools["agents"],
+                nearby_schools=agents_schools["schools"],
             )
         ]
 
@@ -328,6 +331,7 @@ class RealtorScraper(Scraper):
                                     type
                                     name
                                     stories
+                                    text
                                 }
                                 source {
                                     id
@@ -351,10 +355,17 @@ class RealtorScraper(Scraper):
                                             lat
                                         }
                                     }
+                                    county {
+                                        name
+                                        fips_code
+                                    }
                                     neighborhoods {
                                         name
                                     }
                                 }
+                                tax_record {
+                                    public_record_id
+                                }
                                 primary_photo {
                                     href
                                 }
@@ -510,7 +521,7 @@ class RealtorScraper(Scraper):
                 return
 
             property_id = result["property_id"]
-            agents = self.get_agents(property_id)
+            agents_schools = self.get_agents_schools(property_id)
 
             realty_property = Property(
                 mls=mls,
@@ -534,8 +545,12 @@ class RealtorScraper(Scraper):
                 longitude=result["location"]["address"]["coordinate"].get("lon") if able_to_get_lat_long else None,
                 address=self._parse_address(result, search_type="general_search"),
                 description=self._parse_description(result),
+                neighborhoods=self._parse_neighborhoods(result),
+                county=result["location"]["county"].get("name"),
+                fips_code=result["location"]["county"].get("fips_code"),
                 days_on_mls=self.calculate_days_on_mls(result),
-                agents=agents,
+                agents=agents_schools["agents"],
+                nearby_schools=agents_schools["schools"],
             )
             return realty_property
 
@@ -630,18 +645,25 @@ class RealtorScraper(Scraper):
 
         return homes
 
-    def get_agents(self, property_id: str) -> list[Agent]:
-        payload = f'{{"query":"query GetHome($property_id: ID!) {{\\n  home(property_id: $property_id) {{\\n    __typename\\n\\n    consumerAdvertisers: consumer_advertisers {{\\n      __typename\\n      type\\n      advertiserId: advertiser_id\\n      name\\n      phone\\n      type\\n      href\\n      slogan\\n      photo {{\\n        __typename\\n        href\\n      }}\\n      showRealtorLogo: show_realtor_logo\\n      hours\\n    }}\\n\\n\\n  }}\\n}}\\n","variables":{{"property_id":"{property_id}"}}}}'
+    def get_agents_schools(self, property_id: str) -> dict:
+        payload = f'{{"query":"query GetHome($property_id: ID!) {{\\n  home(property_id: $property_id) {{\\n    __typename\\n\\n    consumerAdvertisers: consumer_advertisers {{\\n      __typename\\n      type\\n      advertiserId: advertiser_id\\n      name\\n      phone\\n      type\\n      href\\n      slogan\\n      photo {{\\n        __typename\\n        href\\n      }}\\n      showRealtorLogo: show_realtor_logo\\n      hours\\n    }}\\n\\n\\n  nearbySchools: nearby_schools(radius: 5.0, limit_per_level: 3) {{ __typename schools {{ district {{ __typename id name }} }} }}}}\\n}}\\n","variables":{{"property_id":"{property_id}"}}}}'
         response = self.session.post(self.PROPERTY_GQL, data=payload)
 
-        data = response.json()
-        try:
-            ads = data["data"]["home"]["consumerAdvertisers"]
-        except (KeyError, TypeError):
-            return []
+        def get_key(keys: list):
+            try:
+                data = response.json()
+                for key in keys:
+                    data = data[key]
+                return data
+            except (KeyError, TypeError):
+                return []
+
+        ads = get_key(["data", "home", "consumerAdvertisers"])
+        schools = get_key(["data", "home", "nearbySchools", "schools"])
 
         agents = [Agent(name=ad["name"], phone=ad["phone"]) for ad in ads]
-        return agents
+        schools = [school["district"]["name"] for school in schools]
+        return {"agents": agents, "schools": schools}
 
     @staticmethod
     def _parse_neighborhoods(result: dict) -> Optional[str]:
@@ -715,6 +737,7 @@ class RealtorScraper(Scraper):
             year_built=description_data.get("year_built"),
             garage=description_data.get("garage"),
             stories=description_data.get("stories"),
+            text=description_data.get("text"),
         )
 
     @staticmethod

homeharvest/utils.py

@@ -8,6 +8,7 @@ ordered_properties = [
     "mls",
     "mls_id",
     "status",
+    "text",
     "style",
     "street",
     "unit",
@@ -28,12 +29,16 @@ ordered_properties = [
     "price_per_sqft",
     "latitude",
     "longitude",
+    "neighborhoods",
+    "county",
+    "fips_code",
     "stories",
     "hoa_fee",
     "parking_garage",
     "agent",
     "broker",
     "broker_phone",
+    "nearby_schools",
     "primary_photo",
     "alt_photos",
 ]
@@ -60,6 +65,8 @@ def process_result(result: Property) -> pd.DataFrame:
                 prop_data["broker_phone"] = agents[1].phone
 
     prop_data["price_per_sqft"] = prop_data["prc_sqft"]
+    prop_data["nearby_schools"] = filter(None, prop_data["nearby_schools"]) if prop_data["nearby_schools"] else None
+    prop_data["nearby_schools"] = ", ".join(set(prop_data["nearby_schools"])) if prop_data["nearby_schools"] else None
 
     description = result.description
     prop_data["primary_photo"] = description.primary_photo
@@ -74,6 +81,8 @@ def process_result(result: Property) -> pd.DataFrame:
     prop_data["year_built"] = description.year_built
     prop_data["parking_garage"] = description.garage
     prop_data["stories"] = description.stories
+    prop_data["text"] = description.text
+
 
     properties_df = pd.DataFrame([prop_data])
     properties_df = properties_df.reindex(columns=ordered_properties)

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "homeharvest"
-version = "0.3.17"
+version = "0.3.19"
 description = "Real estate scraping library"
 authors = ["Zachary Hampton <zachary@bunsly.com>", "Cullen Watson <cullen@bunsly.com>"]
 homepage = "https://github.com/Bunsly/HomeHarvest"