pull/71/head^2 v0.3.18
Cullen Watson 2024-04-16 20:01:20 -05:00 committed by GitHub
parent 6076b0f961
commit 7246703999
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
6 changed files with 43 additions and 15 deletions

View File

@ -134,6 +134,13 @@ Property
├── Location Details: ├── Location Details:
│ ├── latitude │ ├── latitude
│ ├── longitude │ ├── longitude
│ ├── nearby_schools
├── Agent Info:
│ ├── agent
│ ├── broker
│ └── broker_phone
├── Agent Info: ├── Agent Info:
│ ├── agent │ ├── agent

View File

@ -1,3 +1,4 @@
import uuid
from dataclasses import dataclass from dataclasses import dataclass
import requests import requests
import uuid import uuid

View File

@ -23,6 +23,12 @@ class ListingType(Enum):
SOLD = "SOLD" SOLD = "SOLD"
@dataclass
class Agent:
name: str | None = None
phone: str | None = None
class PropertyType(Enum): class PropertyType(Enum):
APARTMENT = "APARTMENT" APARTMENT = "APARTMENT"
BUILDING = "BUILDING" BUILDING = "BUILDING"
@ -97,3 +103,4 @@ class Property:
neighborhoods: Optional[str] = None neighborhoods: Optional[str] = None
agents: list[Agent] = None agents: list[Agent] = None
nearby_schools: list[str] = None

View File

@ -142,7 +142,7 @@ class RealtorScraper(Scraper):
days_on_mls = None days_on_mls = None
property_id = property_info["details"]["permalink"] property_id = property_info["details"]["permalink"]
agents = self.get_agents(property_id) agents_schools = self.get_agents_schools(property_id)
listing = Property( listing = Property(
mls=mls, mls=mls,
mls_id=( mls_id=(
@ -178,7 +178,8 @@ class RealtorScraper(Scraper):
stories=property_info["details"].get("stories"), stories=property_info["details"].get("stories"),
), ),
days_on_mls=days_on_mls, days_on_mls=days_on_mls,
agents=agents, agents=agents_schools["agents"],
nearby_schools=agents_schools["schools"],
) )
return [listing] return [listing]
@ -272,7 +273,7 @@ class RealtorScraper(Scraper):
}""" }"""
variables = {"property_id": property_id} variables = {"property_id": property_id}
agents = self.get_agents(property_id) agents_schools = self.get_agents_schools(property_id)
payload = { payload = {
"query": query, "query": query,
@ -290,7 +291,8 @@ class RealtorScraper(Scraper):
property_url=f"{self.PROPERTY_URL}{property_info['details']['permalink']}", property_url=f"{self.PROPERTY_URL}{property_info['details']['permalink']}",
address=self._parse_address(property_info, search_type="handle_address"), address=self._parse_address(property_info, search_type="handle_address"),
description=self._parse_description(property_info), description=self._parse_description(property_info),
agents=agents, agents=agents_schools["agents"],
nearby_schools=agents_schools["schools"],
) )
] ]
@ -510,7 +512,7 @@ class RealtorScraper(Scraper):
return return
property_id = result["property_id"] property_id = result["property_id"]
agents = self.get_agents(property_id) agents_schools = self.get_agents_schools(property_id)
realty_property = Property( realty_property = Property(
mls=mls, mls=mls,
@ -535,7 +537,8 @@ class RealtorScraper(Scraper):
address=self._parse_address(result, search_type="general_search"), address=self._parse_address(result, search_type="general_search"),
description=self._parse_description(result), description=self._parse_description(result),
days_on_mls=self.calculate_days_on_mls(result), days_on_mls=self.calculate_days_on_mls(result),
agents=agents, agents=agents_schools["agents"],
nearby_schools=agents_schools["schools"],
) )
return realty_property return realty_property
@ -630,18 +633,25 @@ class RealtorScraper(Scraper):
return homes return homes
def get_agents(self, property_id: str) -> list[Agent]: def get_agents_schools(self, property_id: str) -> dict:
payload = f'{{"query":"query GetHome($property_id: ID!) {{\\n home(property_id: $property_id) {{\\n __typename\\n\\n consumerAdvertisers: consumer_advertisers {{\\n __typename\\n type\\n advertiserId: advertiser_id\\n name\\n phone\\n type\\n href\\n slogan\\n photo {{\\n __typename\\n href\\n }}\\n showRealtorLogo: show_realtor_logo\\n hours\\n }}\\n\\n\\n }}\\n}}\\n","variables":{{"property_id":"{property_id}"}}}}' payload = f'{{"query":"query GetHome($property_id: ID!) {{\\n home(property_id: $property_id) {{\\n __typename\\n\\n consumerAdvertisers: consumer_advertisers {{\\n __typename\\n type\\n advertiserId: advertiser_id\\n name\\n phone\\n type\\n href\\n slogan\\n photo {{\\n __typename\\n href\\n }}\\n showRealtorLogo: show_realtor_logo\\n hours\\n }}\\n\\n\\n nearbySchools: nearby_schools(radius: 5.0, limit_per_level: 3) {{ __typename schools {{ district {{ __typename id name }} }} }}}}\\n}}\\n","variables":{{"property_id":"{property_id}"}}}}'
response = self.session.post(self.PROPERTY_GQL, data=payload) response = self.session.post(self.PROPERTY_GQL, data=payload)
data = response.json() def get_key(keys: list):
try: try:
ads = data["data"]["home"]["consumerAdvertisers"] data = response.json()
for key in keys:
data = data[key]
return data
except (KeyError, TypeError): except (KeyError, TypeError):
return [] return []
ads = get_key(["data", "home", "consumerAdvertisers"])
schools = get_key(["data", "home", "nearbySchools", "schools"])
agents = [Agent(name=ad["name"], phone=ad["phone"]) for ad in ads] agents = [Agent(name=ad["name"], phone=ad["phone"]) for ad in ads]
return agents schools = [school["district"]["name"] for school in schools]
return {"agents": agents, "schools": schools}
@staticmethod @staticmethod
def _parse_neighborhoods(result: dict) -> Optional[str]: def _parse_neighborhoods(result: dict) -> Optional[str]:

View File

@ -34,6 +34,7 @@ ordered_properties = [
"agent", "agent",
"broker", "broker",
"broker_phone", "broker_phone",
"nearby_schools",
"primary_photo", "primary_photo",
"alt_photos", "alt_photos",
] ]
@ -60,6 +61,8 @@ def process_result(result: Property) -> pd.DataFrame:
prop_data["broker_phone"] = agents[1].phone prop_data["broker_phone"] = agents[1].phone
prop_data["price_per_sqft"] = prop_data["prc_sqft"] prop_data["price_per_sqft"] = prop_data["prc_sqft"]
prop_data["nearby_schools"] = filter(None, prop_data["nearby_schools"]) if prop_data["nearby_schools"] else None
prop_data["nearby_schools"] = ", ".join(set(prop_data["nearby_schools"])) if prop_data["nearby_schools"] else None
description = result.description description = result.description
prop_data["primary_photo"] = description.primary_photo prop_data["primary_photo"] = description.primary_photo

View File

@ -1,6 +1,6 @@
[tool.poetry] [tool.poetry]
name = "homeharvest" name = "homeharvest"
version = "0.3.17" version = "0.3.18"
description = "Real estate scraping library" description = "Real estate scraping library"
authors = ["Zachary Hampton <zachary@bunsly.com>", "Cullen Watson <cullen@bunsly.com>"] authors = ["Zachary Hampton <zachary@bunsly.com>", "Cullen Watson <cullen@bunsly.com>"]
homepage = "https://github.com/Bunsly/HomeHarvest" homepage = "https://github.com/Bunsly/HomeHarvest"