turtlebasket/HomeHarvest
1
0
Fork 0
mirror of https://github.com/Bunsly/HomeHarvest.git synced 2026-03-06 12:34:29 -08:00
Code Issues Packages Projects Releases Wiki Activity

- pydantic conversion

Browse Source
This commit is contained in:
Zachary Hampton
2025-07-15 12:25:43 -07:00
parent 8311f4dfbc
commit 79082090cb
3 changed files with 64 additions and 61 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
homeharvest/core/scrapers/__init__.py
View File

@@ -1,5 +1,4 @@
from __future__ import annotations from __future__ import annotations
from dataclasses import dataclass
from typing import Union from typing import Union
import requests import requests
@@ -9,10 +8,10 @@ import uuid
from ...exceptions import AuthenticationError from ...exceptions import AuthenticationError
from .models import Property, ListingType, SiteName, SearchPropertyType, ReturnType from .models import Property, ListingType, SiteName, SearchPropertyType, ReturnType
import json import json
from pydantic import BaseModel
@dataclass class ScraperInput(BaseModel):
class ScraperInput:
location: str location: str
listing_type: ListingType listing_type: ListingType
property_type: list[SearchPropertyType] | None = None property_type: list[SearchPropertyType] | None = None

56
homeharvest/core/scrapers/models.py
View File

@@ -1,7 +1,7 @@
from __future__ import annotations from __future__ import annotations
from dataclasses import dataclass
from enum import Enum from enum import Enum
from typing import Optional from typing import Optional
from pydantic import BaseModel, computed_field
class ReturnType(Enum): class ReturnType(Enum):
@@ -44,12 +44,6 @@ class ListingType(Enum):
SOLD = "SOLD" SOLD = "SOLD"
@dataclass
class Agent:
name: str | None = None
phone: str | None = None
class PropertyType(Enum): class PropertyType(Enum):
APARTMENT = "APARTMENT" APARTMENT = "APARTMENT"
BUILDING = "BUILDING" BUILDING = "BUILDING"
@@ -74,9 +68,7 @@ class PropertyType(Enum):
OTHER = "OTHER" OTHER = "OTHER"
@dataclass class Address(BaseModel):
class Address:
formatted_address: str | None = None
full_line: str | None = None full_line: str | None = None
street: str | None = None street: str | None = None
unit: str | None = None unit: str | None = None
@@ -84,11 +76,32 @@ class Address:
state: str | None = None state: str | None = None
zip: str | None = None zip: str | None = None
@computed_field
@property
def formatted_address(self) -> str | None:
"""Computed property that combines full_line, city, state, and zip into a formatted address."""
parts = []
if self.full_line:
parts.append(self.full_line)
city_state_zip = []
if self.city:
city_state_zip.append(self.city)
if self.state:
city_state_zip.append(self.state)
if self.zip:
city_state_zip.append(self.zip)
if city_state_zip:
parts.append(", ".join(city_state_zip))
return ", ".join(parts) if parts else None
@dataclass
class Description: class Description(BaseModel):
primary_photo: str | None = None primary_photo: str | None = None
alt_photos: list[str] | None = None alt_photos: list[str] | None = None
style: PropertyType | None = None style: PropertyType | None = None
@@ -104,21 +117,18 @@ class Description:
text: str | None = None text: str | None = None
@dataclass class AgentPhone(BaseModel):
class AgentPhone: #: For documentation purposes only (at the moment)
number: str | None = None number: str | None = None
type: str | None = None type: str | None = None
primary: bool | None = None primary: bool | None = None
ext: str | None = None ext: str | None = None
@dataclass class Entity(BaseModel):
class Entity:
name: str name: str
uuid: str | None = None uuid: str | None = None
@dataclass
class Agent(Entity): class Agent(Entity):
mls_set: str | None = None mls_set: str | None = None
nrds_id: str | None = None nrds_id: str | None = None
@@ -127,7 +137,6 @@ class Agent(Entity):
href: str | None = None href: str | None = None
@dataclass
class Office(Entity): class Office(Entity):
mls_set: str | None = None mls_set: str | None = None
email: str | None = None email: str | None = None
@@ -135,28 +144,23 @@ class Office(Entity):
phones: list[dict] | AgentPhone | None = None phones: list[dict] | AgentPhone | None = None
@dataclass
class Broker(Entity): class Broker(Entity):
pass pass
@dataclass
class Builder(Entity): class Builder(Entity):
pass pass
@dataclass class Advertisers(BaseModel):
class Advertisers:
agent: Agent | None = None agent: Agent | None = None
broker: Broker | None = None broker: Broker | None = None
builder: Builder | None = None builder: Builder | None = None
office: Office | None = None office: Office | None = None
@dataclass class Property(BaseModel):
class Property:
property_url: str property_url: str
property_id: str property_id: str
#: allows_cats: bool #: allows_cats: bool
#: allows_dogs: bool #: allows_dogs: bool
@@ -188,7 +192,7 @@ class Property:
neighborhoods: Optional[str] = None neighborhoods: Optional[str] = None
county: Optional[str] = None county: Optional[str] = None
fips_code: Optional[str] = None fips_code: Optional[str] = None
nearby_schools: list[str] = None nearby_schools: list[str] | None = None
assessed_value: int | None = None assessed_value: int | None = None
estimated_value: int | None = None estimated_value: int | None = None
tax: int | None = None tax: int | None = None

64
homeharvest/utils.py
View File

@@ -69,45 +69,45 @@ ordered_properties = [
def process_result(result: Property) -> pd.DataFrame: def process_result(result: Property) -> pd.DataFrame:
prop_data = {prop: None for prop in ordered_properties} prop_data = {prop: None for prop in ordered_properties}
prop_data.update(result.__dict__) prop_data.update(result.model_dump())
if "address" in prop_data: if "address" in prop_data and prop_data["address"]:
address_data = prop_data["address"] address_data = prop_data["address"]
prop_data["full_street_line"] = address_data.full_line prop_data["full_street_line"] = address_data.get("full_line")
prop_data["street"] = address_data.street prop_data["street"] = address_data.get("street")
prop_data["unit"] = address_data.unit prop_data["unit"] = address_data.get("unit")
prop_data["city"] = address_data.city prop_data["city"] = address_data.get("city")
prop_data["state"] = address_data.state prop_data["state"] = address_data.get("state")
prop_data["zip_code"] = address_data.zip prop_data["zip_code"] = address_data.get("zip")
if "advertisers" in prop_data and prop_data.get("advertisers"): if "advertisers" in prop_data and prop_data.get("advertisers"):
advertiser_data: Advertisers | None = prop_data["advertisers"] advertiser_data = prop_data["advertisers"]
if advertiser_data.agent: if advertiser_data.get("agent"):
agent_data = advertiser_data.agent agent_data = advertiser_data["agent"]
prop_data["agent_id"] = agent_data.uuid prop_data["agent_id"] = agent_data.get("uuid")
prop_data["agent_name"] = agent_data.name prop_data["agent_name"] = agent_data.get("name")
prop_data["agent_email"] = agent_data.email prop_data["agent_email"] = agent_data.get("email")
prop_data["agent_phones"] = agent_data.phones prop_data["agent_phones"] = agent_data.get("phones")
prop_data["agent_mls_set"] = agent_data.mls_set prop_data["agent_mls_set"] = agent_data.get("mls_set")
prop_data["agent_nrds_id"] = agent_data.nrds_id prop_data["agent_nrds_id"] = agent_data.get("nrds_id")
if advertiser_data.broker: if advertiser_data.get("broker"):
broker_data = advertiser_data.broker broker_data = advertiser_data["broker"]
prop_data["broker_id"] = broker_data.uuid prop_data["broker_id"] = broker_data.get("uuid")
prop_data["broker_name"] = broker_data.name prop_data["broker_name"] = broker_data.get("name")
if advertiser_data.builder: if advertiser_data.get("builder"):
builder_data = advertiser_data.builder builder_data = advertiser_data["builder"]
prop_data["builder_id"] = builder_data.uuid prop_data["builder_id"] = builder_data.get("uuid")
prop_data["builder_name"] = builder_data.name prop_data["builder_name"] = builder_data.get("name")
if advertiser_data.office: if advertiser_data.get("office"):
office_data = advertiser_data.office office_data = advertiser_data["office"]
prop_data["office_id"] = office_data.uuid prop_data["office_id"] = office_data.get("uuid")
prop_data["office_name"] = office_data.name prop_data["office_name"] = office_data.get("name")
prop_data["office_email"] = office_data.email prop_data["office_email"] = office_data.get("email")
prop_data["office_phones"] = office_data.phones prop_data["office_phones"] = office_data.get("phones")
prop_data["office_mls_set"] = office_data.mls_set prop_data["office_mls_set"] = office_data.get("mls_set")
prop_data["price_per_sqft"] = prop_data["prc_sqft"] prop_data["price_per_sqft"] = prop_data["prc_sqft"]
prop_data["nearby_schools"] = filter(None, prop_data["nearby_schools"]) if prop_data["nearby_schools"] else None prop_data["nearby_schools"] = filter(None, prop_data["nearby_schools"]) if prop_data["nearby_schools"] else None