mirror of
https://github.com/Bunsly/HomeHarvest.git
synced 2026-03-05 12:04:31 -08:00
Compare commits
9 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
aacd168545 | ||
|
|
0d70007000 | ||
|
|
018d3fbac4 | ||
|
|
803fd618e9 | ||
|
|
b23b55ca80 | ||
|
|
3458a08383 | ||
|
|
c3e24a4ce0 | ||
|
|
46985dcee4 | ||
|
|
04ae968716 |
17
README.md
17
README.md
@@ -21,7 +21,7 @@
|
||||
```bash
|
||||
pip install -U homeharvest
|
||||
```
|
||||
_Python version >= [3.10](https://www.python.org/downloads/release/python-3100/) required_
|
||||
_Python version >= [3.9](https://www.python.org/downloads/release/python-3100/) required_
|
||||
|
||||
## Usage
|
||||
|
||||
@@ -43,7 +43,6 @@ properties = scrape_property(
|
||||
# date_from="2023-05-01", # alternative to past_days
|
||||
# date_to="2023-05-28",
|
||||
# foreclosure=True
|
||||
|
||||
# mls_only=True, # only fetch MLS listings
|
||||
)
|
||||
print(f"Number of properties: {len(properties)}")
|
||||
@@ -92,6 +91,8 @@ Optional
|
||||
├── foreclosure (True/False): If set, fetches only foreclosures
|
||||
│
|
||||
└── proxy (string): In format 'http://user:pass@host:port'
|
||||
│
|
||||
└── extra_property_data (bool): Increases requests by O(n). If set, this fetches additional property data (e.g. agent, broker, property evaluations etc.)
|
||||
```
|
||||
|
||||
### Property Schema
|
||||
@@ -139,17 +140,13 @@ Property
|
||||
|
||||
├── Agent Info:
|
||||
│ ├── agent
|
||||
│ ├── broker
|
||||
│ └── broker_phone
|
||||
|
||||
├── Agent Info:
|
||||
│ ├── agent
|
||||
│ ├── broker
|
||||
│ └── broker_phone
|
||||
│ ├── agent_email
|
||||
│ └── agent_phone
|
||||
```
|
||||
|
||||
### Exceptions
|
||||
The following exceptions may be raised when using HomeHarvest:
|
||||
|
||||
- `InvalidListingType` - valid options: `for_sale`, `for_rent`, `sold`
|
||||
- `InvalidDate` - date_from or date_to is not in the format YYYY-MM-DD
|
||||
- `InvalidDate` - date_from or date_to is not in the format YYYY-MM-DD.
|
||||
- `AuthenticationError` - Realtor.com token request failed.
|
||||
|
||||
@@ -13,9 +13,10 @@ def scrape_property(
|
||||
mls_only: bool = False,
|
||||
past_days: int = None,
|
||||
proxy: str = None,
|
||||
date_from: str = None,
|
||||
date_from: str = None, #: TODO: Switch to one parameter, Date, with date_from and date_to, pydantic validation
|
||||
date_to: str = None,
|
||||
foreclosure: bool = None,
|
||||
extra_property_data: bool = True,
|
||||
) -> pd.DataFrame:
|
||||
"""
|
||||
Scrape properties from Realtor.com based on a given location and listing type.
|
||||
@@ -23,9 +24,11 @@ def scrape_property(
|
||||
:param listing_type: Listing Type (for_sale, for_rent, sold)
|
||||
:param radius: Get properties within _ (e.g. 1.0) miles. Only applicable for individual addresses.
|
||||
:param mls_only: If set, fetches only listings with MLS IDs.
|
||||
:param proxy: Proxy to use for scraping
|
||||
:param past_days: Get properties sold or listed (dependent on your listing_type) in the last _ days.
|
||||
:param date_from, date_to: Get properties sold or listed (dependent on your listing_type) between these dates. format: 2021-01-28
|
||||
:param proxy: Proxy to use for scraping
|
||||
:param foreclosure: If set, fetches only foreclosure listings.
|
||||
:param extra_property_data: Increases requests by O(n). If set, this fetches additional property data (e.g. agent, broker, property evaluations etc.)
|
||||
"""
|
||||
validate_input(listing_type)
|
||||
validate_dates(date_from, date_to)
|
||||
@@ -40,6 +43,7 @@ def scrape_property(
|
||||
date_from=date_from,
|
||||
date_to=date_to,
|
||||
foreclosure=foreclosure,
|
||||
extra_property_data=extra_property_data,
|
||||
)
|
||||
|
||||
site = RealtorScraper(scraper_input)
|
||||
@@ -49,6 +53,9 @@ def scrape_property(
|
||||
if not properties_dfs:
|
||||
return pd.DataFrame()
|
||||
|
||||
properties_dfs = [df for df in properties_dfs if not df.empty]
|
||||
|
||||
with warnings.catch_warnings():
|
||||
warnings.simplefilter("ignore", category=FutureWarning)
|
||||
return pd.concat(properties_dfs, ignore_index=True, axis=0)[ordered_properties]
|
||||
|
||||
return pd.concat(properties_dfs, ignore_index=True, axis=0)[ordered_properties].replace({"None": pd.NA, None: pd.NA, "": pd.NA})
|
||||
|
||||
@@ -1,7 +1,10 @@
|
||||
import uuid
|
||||
from __future__ import annotations
|
||||
from dataclasses import dataclass
|
||||
import requests
|
||||
from requests.adapters import HTTPAdapter
|
||||
from urllib3.util.retry import Retry
|
||||
import uuid
|
||||
from ...exceptions import AuthenticationError
|
||||
from .models import Property, ListingType, SiteName
|
||||
|
||||
|
||||
@@ -10,33 +13,40 @@ class ScraperInput:
|
||||
location: str
|
||||
listing_type: ListingType
|
||||
radius: float | None = None
|
||||
mls_only: bool | None = None
|
||||
mls_only: bool | None = False
|
||||
proxy: str | None = None
|
||||
last_x_days: int | None = None
|
||||
date_from: str | None = None
|
||||
date_to: str | None = None
|
||||
foreclosure: bool | None = None
|
||||
foreclosure: bool | None = False
|
||||
extra_property_data: bool | None = True
|
||||
|
||||
|
||||
class Scraper:
|
||||
session = None
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
scraper_input: ScraperInput,
|
||||
session: requests.Session = None,
|
||||
):
|
||||
self.location = scraper_input.location
|
||||
self.listing_type = scraper_input.listing_type
|
||||
|
||||
if not session:
|
||||
self.session = requests.Session()
|
||||
self.session.headers.update(
|
||||
if not self.session:
|
||||
Scraper.session = requests.Session()
|
||||
retries = Retry(
|
||||
total=3, backoff_factor=3, status_forcelist=[429, 403], allowed_methods=frozenset(["GET", "POST"])
|
||||
)
|
||||
|
||||
adapter = HTTPAdapter(max_retries=retries)
|
||||
Scraper.session.mount("http://", adapter)
|
||||
Scraper.session.mount("https://", adapter)
|
||||
Scraper.session.headers.update(
|
||||
{
|
||||
"auth": f"Bearer {self.get_access_token()}",
|
||||
"apollographql-client-name": "com.move.Realtor-apollo-ios",
|
||||
}
|
||||
)
|
||||
else:
|
||||
self.session = session
|
||||
|
||||
if scraper_input.proxy:
|
||||
proxy_url = scraper_input.proxy
|
||||
@@ -50,6 +60,7 @@ class Scraper:
|
||||
self.date_from = scraper_input.date_from
|
||||
self.date_to = scraper_input.date_to
|
||||
self.foreclosure = scraper_input.foreclosure
|
||||
self.extra_property_data = scraper_input.extra_property_data
|
||||
|
||||
def search(self) -> list[Property]: ...
|
||||
|
||||
@@ -58,7 +69,8 @@ class Scraper:
|
||||
|
||||
def handle_location(self): ...
|
||||
|
||||
def get_access_token(self):
|
||||
@staticmethod
|
||||
def get_access_token():
|
||||
url = "https://graph.realtor.com/auth/token"
|
||||
|
||||
payload = f'{{"client_app_id":"rdc_mobile_native,24.20.4.149916,iphone","device_id":"{str(uuid.uuid4()).upper()}","grant_type":"device_mobile"}}'
|
||||
@@ -73,4 +85,11 @@ class Scraper:
|
||||
response = requests.post(url, headers=headers, data=payload)
|
||||
|
||||
data = response.json()
|
||||
return data["access_token"]
|
||||
|
||||
if not (access_token := data.get("access_token")):
|
||||
raise AuthenticationError(
|
||||
"Failed to get access token, use a proxy/vpn or wait a moment and try again.",
|
||||
response=response
|
||||
)
|
||||
|
||||
return access_token
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
from __future__ import annotations
|
||||
from dataclasses import dataclass
|
||||
from enum import Enum
|
||||
from typing import Optional
|
||||
@@ -36,6 +37,7 @@ class PropertyType(Enum):
|
||||
CONDO_TOWNHOME = "CONDO_TOWNHOME"
|
||||
CONDO_TOWNHOME_ROWHOME_COOP = "CONDO_TOWNHOME_ROWHOME_COOP"
|
||||
CONDO = "CONDO"
|
||||
CONDOP = "CONDOP"
|
||||
CONDOS = "CONDOS"
|
||||
COOP = "COOP"
|
||||
DUPLEX_TRIPLEX = "DUPLEX_TRIPLEX"
|
||||
@@ -52,6 +54,7 @@ class PropertyType(Enum):
|
||||
|
||||
@dataclass
|
||||
class Address:
|
||||
full_line: str | None = None
|
||||
street: str | None = None
|
||||
unit: str | None = None
|
||||
city: str | None = None
|
||||
@@ -76,10 +79,27 @@ class Description:
|
||||
text: str | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class AgentPhone: #: For documentation purposes only (at the moment)
|
||||
number: str | None = None
|
||||
type: str | None = None
|
||||
primary: bool | None = None
|
||||
ext: str | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class Agent:
|
||||
name: str | None = None
|
||||
phones: list[dict] | AgentPhone | None = None
|
||||
email: str | None = None
|
||||
href: str | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class Broker:
|
||||
name: str | None = None
|
||||
phone: str | None = None
|
||||
website: str | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -104,5 +124,8 @@ class Property:
|
||||
neighborhoods: Optional[str] = None
|
||||
county: Optional[str] = None
|
||||
fips_code: Optional[str] = None
|
||||
agents: list[Agent] = None
|
||||
agents: list[Agent] | None = None
|
||||
brokers: list[Broker] | None = None
|
||||
nearby_schools: list[str] = None
|
||||
assessed_value: int | None = None
|
||||
estimated_value: int | None = None
|
||||
|
||||
@@ -4,13 +4,13 @@ homeharvest.realtor.__init__
|
||||
|
||||
This module implements the scraper for realtor.com
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from datetime import datetime
|
||||
from typing import Dict, Union, Optional
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
|
||||
from .. import Scraper
|
||||
from ..models import Property, Address, ListingType, Description, PropertyType, Agent
|
||||
from ..models import Property, Address, ListingType, Description, PropertyType, Agent, Broker
|
||||
|
||||
|
||||
class RealtorScraper(Scraper):
|
||||
@@ -52,6 +52,7 @@ class RealtorScraper(Scraper):
|
||||
listing_id
|
||||
}
|
||||
address {
|
||||
line
|
||||
street_direction
|
||||
street_number
|
||||
street_name
|
||||
@@ -142,7 +143,7 @@ class RealtorScraper(Scraper):
|
||||
days_on_mls = None
|
||||
|
||||
property_id = property_info["details"]["permalink"]
|
||||
agents_schools = self.get_agents_schools(property_id)
|
||||
prop_details = self.get_prop_details(property_id)
|
||||
listing = Property(
|
||||
mls=mls,
|
||||
mls_id=(
|
||||
@@ -165,7 +166,7 @@ class RealtorScraper(Scraper):
|
||||
longitude=property_info["address"]["location"]["coordinate"].get("lon") if able_to_get_lat_long else None,
|
||||
address=self._parse_address(property_info, search_type="handle_listing"),
|
||||
description=Description(
|
||||
alt_photos=self.process_alt_photos(property_info.get("media", {}).get("photos", [])),
|
||||
alt_photos=self.process_alt_photos(property_info["media"].get("photos", [])) if property_info.get("media") else None,
|
||||
style=property_info["basic"].get("type", "").upper(),
|
||||
beds=property_info["basic"].get("beds"),
|
||||
baths_full=property_info["basic"].get("baths_full"),
|
||||
@@ -176,11 +177,14 @@ class RealtorScraper(Scraper):
|
||||
year_built=property_info["details"].get("year_built"),
|
||||
garage=property_info["details"].get("garage"),
|
||||
stories=property_info["details"].get("stories"),
|
||||
text=property_info["description"].get("text"),
|
||||
text=property_info.get("description", {}).get("text"),
|
||||
),
|
||||
days_on_mls=days_on_mls,
|
||||
agents=agents_schools["agents"],
|
||||
nearby_schools=agents_schools["schools"],
|
||||
agents=prop_details.get("agents"),
|
||||
brokers=prop_details.get("brokers"),
|
||||
nearby_schools=prop_details.get("schools"),
|
||||
assessed_value=prop_details.get("assessed_value"),
|
||||
estimated_value=prop_details.get("estimated_value"),
|
||||
)
|
||||
|
||||
return [listing]
|
||||
@@ -233,6 +237,7 @@ class RealtorScraper(Scraper):
|
||||
stories
|
||||
}
|
||||
address {
|
||||
line
|
||||
street_direction
|
||||
street_number
|
||||
street_name
|
||||
@@ -274,7 +279,7 @@ class RealtorScraper(Scraper):
|
||||
}"""
|
||||
|
||||
variables = {"property_id": property_id}
|
||||
agents_schools = self.get_agents_schools(property_id)
|
||||
prop_details = self.get_prop_details(property_id)
|
||||
|
||||
payload = {
|
||||
"query": query,
|
||||
@@ -292,8 +297,11 @@ class RealtorScraper(Scraper):
|
||||
property_url=f"{self.PROPERTY_URL}{property_info['details']['permalink']}",
|
||||
address=self._parse_address(property_info, search_type="handle_address"),
|
||||
description=self._parse_description(property_info),
|
||||
agents=agents_schools["agents"],
|
||||
nearby_schools=agents_schools["schools"],
|
||||
agents=prop_details.get("agents"),
|
||||
brokers=prop_details.get("brokers"),
|
||||
nearby_schools=prop_details.get("schools"),
|
||||
assessed_value=prop_details.get("assessed_value"),
|
||||
estimated_value=prop_details.get("estimated_value"),
|
||||
)
|
||||
]
|
||||
|
||||
@@ -346,6 +354,7 @@ class RealtorScraper(Scraper):
|
||||
street_number
|
||||
street_name
|
||||
street_suffix
|
||||
line
|
||||
unit
|
||||
city
|
||||
state_code
|
||||
@@ -486,7 +495,6 @@ class RealtorScraper(Scraper):
|
||||
}
|
||||
|
||||
response = self.session.post(self.SEARCH_GQL_URL, json=payload)
|
||||
response.raise_for_status()
|
||||
response_json = response.json()
|
||||
search_key = "home_search" if "home_search" in query else "property_search"
|
||||
|
||||
@@ -521,7 +529,7 @@ class RealtorScraper(Scraper):
|
||||
return
|
||||
|
||||
property_id = result["property_id"]
|
||||
agents_schools = self.get_agents_schools(property_id)
|
||||
prop_details = self.get_prop_details(property_id)
|
||||
|
||||
realty_property = Property(
|
||||
mls=mls,
|
||||
@@ -546,11 +554,14 @@ class RealtorScraper(Scraper):
|
||||
address=self._parse_address(result, search_type="general_search"),
|
||||
description=self._parse_description(result),
|
||||
neighborhoods=self._parse_neighborhoods(result),
|
||||
county=result["location"]["county"].get("name"),
|
||||
fips_code=result["location"]["county"].get("fips_code"),
|
||||
county=result["location"]["county"].get("name") if result["location"]["county"] else None,
|
||||
fips_code=result["location"]["county"].get("fips_code") if result["location"]["county"] else None,
|
||||
days_on_mls=self.calculate_days_on_mls(result),
|
||||
agents=agents_schools["agents"],
|
||||
nearby_schools=agents_schools["schools"],
|
||||
agents=prop_details.get("agents"),
|
||||
brokers=prop_details.get("brokers"),
|
||||
nearby_schools=prop_details.get("schools"),
|
||||
assessed_value=prop_details.get("assessed_value"),
|
||||
estimated_value=prop_details.get("estimated_value"),
|
||||
)
|
||||
return realty_property
|
||||
|
||||
@@ -645,25 +656,91 @@ class RealtorScraper(Scraper):
|
||||
|
||||
return homes
|
||||
|
||||
def get_agents_schools(self, property_id: str) -> dict:
|
||||
payload = f'{{"query":"query GetHome($property_id: ID!) {{\\n home(property_id: $property_id) {{\\n __typename\\n\\n consumerAdvertisers: consumer_advertisers {{\\n __typename\\n type\\n advertiserId: advertiser_id\\n name\\n phone\\n type\\n href\\n slogan\\n photo {{\\n __typename\\n href\\n }}\\n showRealtorLogo: show_realtor_logo\\n hours\\n }}\\n\\n\\n nearbySchools: nearby_schools(radius: 5.0, limit_per_level: 3) {{ __typename schools {{ district {{ __typename id name }} }} }}}}\\n}}\\n","variables":{{"property_id":"{property_id}"}}}}'
|
||||
response = self.session.post(self.PROPERTY_GQL, data=payload)
|
||||
def get_prop_details(self, property_id: str) -> dict:
|
||||
if not self.extra_property_data:
|
||||
return {}
|
||||
|
||||
#: TODO: migrate "advertisers" and "estimates" to general query
|
||||
|
||||
query = """query GetHome($property_id: ID!) {
|
||||
home(property_id: $property_id) {
|
||||
__typename
|
||||
|
||||
advertisers {
|
||||
__typename
|
||||
type
|
||||
name
|
||||
email
|
||||
phones { number type ext primary }
|
||||
}
|
||||
|
||||
consumer_advertisers {
|
||||
name
|
||||
phone
|
||||
href
|
||||
type
|
||||
}
|
||||
|
||||
nearbySchools: nearby_schools(radius: 5.0, limit_per_level: 3) {
|
||||
__typename schools { district { __typename id name } }
|
||||
}
|
||||
taxHistory: tax_history { __typename tax year assessment { __typename building land total } }
|
||||
estimates {
|
||||
__typename
|
||||
currentValues: current_values {
|
||||
__typename
|
||||
source { __typename type name }
|
||||
estimate
|
||||
estimateHigh: estimate_high
|
||||
estimateLow: estimate_low
|
||||
date
|
||||
isBestHomeValue: isbest_homevalue
|
||||
}
|
||||
}
|
||||
}
|
||||
}"""
|
||||
|
||||
variables = {"property_id": property_id}
|
||||
response = self.session.post(self.PROPERTY_GQL, json={"query": query, "variables": variables})
|
||||
data = response.json()
|
||||
|
||||
def get_key(keys: list):
|
||||
try:
|
||||
data = response.json()
|
||||
value = data
|
||||
for key in keys:
|
||||
data = data[key]
|
||||
return data
|
||||
except (KeyError, TypeError):
|
||||
return []
|
||||
value = value[key]
|
||||
|
||||
return value or {}
|
||||
except (KeyError, TypeError, IndexError):
|
||||
return {}
|
||||
|
||||
agents = get_key(["data", "home", "advertisers"])
|
||||
advertisers = get_key(["data", "home", "consumer_advertisers"])
|
||||
|
||||
ads = get_key(["data", "home", "consumerAdvertisers"])
|
||||
schools = get_key(["data", "home", "nearbySchools", "schools"])
|
||||
assessed_value = get_key(["data", "home", "taxHistory", 0, "assessment", "total"])
|
||||
estimated_value = get_key(["data", "home", "estimates", "currentValues", 0, "estimate"])
|
||||
|
||||
agents = [Agent(name=ad["name"], phone=ad["phone"]) for ad in ads]
|
||||
schools = [school["district"]["name"] for school in schools]
|
||||
return {"agents": agents, "schools": schools}
|
||||
agents = [Agent(
|
||||
name=ad["name"],
|
||||
email=ad["email"],
|
||||
phones=ad["phones"]
|
||||
) for ad in agents]
|
||||
|
||||
brokers = [Broker(
|
||||
name=ad["name"],
|
||||
phone=ad["phone"],
|
||||
website=ad["href"]
|
||||
) for ad in advertisers if ad.get("type") != "Agent"]
|
||||
|
||||
schools = [school["district"]["name"] for school in schools if school['district'].get('name')]
|
||||
return {
|
||||
"agents": agents if agents else None,
|
||||
"brokers": brokers if brokers else None,
|
||||
"schools": schools if schools else None,
|
||||
"assessed_value": assessed_value if assessed_value else None,
|
||||
"estimated_value": estimated_value if estimated_value else None,
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _parse_neighborhoods(result: dict) -> Optional[str]:
|
||||
@@ -685,20 +762,22 @@ class RealtorScraper(Scraper):
|
||||
|
||||
return address_part
|
||||
|
||||
def _parse_address(self, result: dict, search_type):
|
||||
@staticmethod
|
||||
def _parse_address(result: dict, search_type):
|
||||
if search_type == "general_search":
|
||||
address = result["location"]["address"]
|
||||
else:
|
||||
address = result["address"]
|
||||
|
||||
return Address(
|
||||
full_line=address.get("line"),
|
||||
street=" ".join(
|
||||
[
|
||||
self.handle_none_safely(address.get("street_number")),
|
||||
self.handle_none_safely(address.get("street_direction")),
|
||||
self.handle_none_safely(address.get("street_name")),
|
||||
self.handle_none_safely(address.get("street_suffix")),
|
||||
]
|
||||
part for part in [
|
||||
address.get("street_number"),
|
||||
address.get("street_direction"),
|
||||
address.get("street_name"),
|
||||
address.get("street_suffix"),
|
||||
] if part is not None
|
||||
).strip(),
|
||||
unit=address["unit"],
|
||||
city=address["city"],
|
||||
@@ -726,14 +805,14 @@ class RealtorScraper(Scraper):
|
||||
|
||||
return Description(
|
||||
primary_photo=primary_photo,
|
||||
alt_photos=RealtorScraper.process_alt_photos(result.get("photos")),
|
||||
style=PropertyType(style) if style else None,
|
||||
alt_photos=RealtorScraper.process_alt_photos(result.get("photos", [])),
|
||||
style=PropertyType.__getitem__(style) if style and style in PropertyType.__members__ else None,
|
||||
beds=description_data.get("beds"),
|
||||
baths_full=description_data.get("baths_full"),
|
||||
baths_half=description_data.get("baths_half"),
|
||||
sqft=description_data.get("sqft"),
|
||||
lot_sqft=description_data.get("lot_sqft"),
|
||||
sold_price=description_data.get("sold_price"),
|
||||
sold_price=description_data.get("sold_price") if result.get('last_sold_date') or result["list_price"] != description_data.get("sold_price") else None, #: has a sold date or list and sold price are different
|
||||
year_built=description_data.get("year_built"),
|
||||
garage=description_data.get("garage"),
|
||||
stories=description_data.get("stories"),
|
||||
|
||||
@@ -4,3 +4,11 @@ class InvalidListingType(Exception):
|
||||
|
||||
class InvalidDate(Exception):
|
||||
"""Raised when only one of date_from or date_to is provided or not in the correct format. ex: 2023-10-23"""
|
||||
|
||||
|
||||
class AuthenticationError(Exception):
|
||||
"""Raised when there is an issue with the authentication process."""
|
||||
def __init__(self, *args, response):
|
||||
super().__init__(*args)
|
||||
|
||||
self.response = response
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
from __future__ import annotations
|
||||
import pandas as pd
|
||||
from datetime import datetime
|
||||
from .core.scrapers.models import Property, ListingType
|
||||
from .core.scrapers.models import Property, ListingType, Agent
|
||||
from .exceptions import InvalidListingType, InvalidDate
|
||||
|
||||
ordered_properties = [
|
||||
@@ -10,6 +11,7 @@ ordered_properties = [
|
||||
"status",
|
||||
"text",
|
||||
"style",
|
||||
"full_street_line",
|
||||
"street",
|
||||
"unit",
|
||||
"city",
|
||||
@@ -25,6 +27,8 @@ ordered_properties = [
|
||||
"list_date",
|
||||
"sold_price",
|
||||
"last_sold_date",
|
||||
"assessed_value",
|
||||
"estimated_value",
|
||||
"lot_sqft",
|
||||
"price_per_sqft",
|
||||
"latitude",
|
||||
@@ -36,8 +40,11 @@ ordered_properties = [
|
||||
"hoa_fee",
|
||||
"parking_garage",
|
||||
"agent",
|
||||
"agent_email",
|
||||
"agent_phones",
|
||||
"broker",
|
||||
"broker_phone",
|
||||
"broker_website",
|
||||
"nearby_schools",
|
||||
"primary_photo",
|
||||
"alt_photos",
|
||||
@@ -50,6 +57,7 @@ def process_result(result: Property) -> pd.DataFrame:
|
||||
|
||||
if "address" in prop_data:
|
||||
address_data = prop_data["address"]
|
||||
prop_data["full_street_line"] = address_data.full_line
|
||||
prop_data["street"] = address_data.street
|
||||
prop_data["unit"] = address_data.unit
|
||||
prop_data["city"] = address_data.city
|
||||
@@ -57,12 +65,18 @@ def process_result(result: Property) -> pd.DataFrame:
|
||||
prop_data["zip_code"] = address_data.zip
|
||||
|
||||
if "agents" in prop_data:
|
||||
agents = prop_data["agents"]
|
||||
agents: list[Agent] | None = prop_data["agents"]
|
||||
if agents:
|
||||
prop_data["agent"] = agents[0].name
|
||||
if len(agents) > 1:
|
||||
prop_data["broker"] = agents[1].name
|
||||
prop_data["broker_phone"] = agents[1].phone
|
||||
prop_data["agent_email"] = agents[0].email
|
||||
prop_data["agent_phones"] = agents[0].phones
|
||||
|
||||
if "brokers" in prop_data:
|
||||
brokers = prop_data["brokers"]
|
||||
if brokers:
|
||||
prop_data["broker"] = brokers[0].name
|
||||
prop_data["broker_phone"] = brokers[0].phone
|
||||
prop_data["broker_website"] = brokers[0].website
|
||||
|
||||
prop_data["price_per_sqft"] = prop_data["prc_sqft"]
|
||||
prop_data["nearby_schools"] = filter(None, prop_data["nearby_schools"]) if prop_data["nearby_schools"] else None
|
||||
@@ -70,8 +84,8 @@ def process_result(result: Property) -> pd.DataFrame:
|
||||
|
||||
description = result.description
|
||||
prop_data["primary_photo"] = description.primary_photo
|
||||
prop_data["alt_photos"] = ", ".join(description.alt_photos)
|
||||
prop_data["style"] = description.style.value
|
||||
prop_data["alt_photos"] = ", ".join(description.alt_photos) if description.alt_photos else None
|
||||
prop_data["style"] = description.style if type(description.style) == str else description.style.value
|
||||
prop_data["beds"] = description.beds
|
||||
prop_data["full_baths"] = description.baths_full
|
||||
prop_data["half_baths"] = description.baths_half
|
||||
@@ -83,7 +97,6 @@ def process_result(result: Property) -> pd.DataFrame:
|
||||
prop_data["stories"] = description.stories
|
||||
prop_data["text"] = description.text
|
||||
|
||||
|
||||
properties_df = pd.DataFrame([prop_data])
|
||||
properties_df = properties_df.reindex(columns=ordered_properties)
|
||||
|
||||
@@ -106,5 +119,5 @@ def validate_dates(date_from: str | None, date_to: str | None) -> None:
|
||||
|
||||
if date_to_obj < date_from_obj:
|
||||
raise InvalidDate("date_to must be after date_from.")
|
||||
except ValueError as e:
|
||||
except ValueError:
|
||||
raise InvalidDate(f"Invalid date format or range")
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
[tool.poetry]
|
||||
name = "homeharvest"
|
||||
version = "0.3.19"
|
||||
version = "0.3.27"
|
||||
description = "Real estate scraping library"
|
||||
authors = ["Zachary Hampton <zachary@bunsly.com>", "Cullen Watson <cullen@bunsly.com>"]
|
||||
homepage = "https://github.com/Bunsly/HomeHarvest"
|
||||
@@ -10,7 +10,7 @@ readme = "README.md"
|
||||
homeharvest = "homeharvest.cli:main"
|
||||
|
||||
[tool.poetry.dependencies]
|
||||
python = ">=3.10,<3.13"
|
||||
python = ">=3.9,<3.13"
|
||||
requests = "^2.31.0"
|
||||
pandas = "^2.1.1"
|
||||
|
||||
|
||||
@@ -142,3 +142,26 @@ def test_realtor_foreclosed():
|
||||
def test_realtor_agent():
|
||||
scraped = scrape_property(location="Detroit, MI", listing_type="for_sale")
|
||||
assert scraped["agent"].nunique() > 1
|
||||
|
||||
|
||||
def test_realtor_without_extra_details():
|
||||
results = [
|
||||
scrape_property(
|
||||
location="15509 N 172nd Dr, Surprise, AZ 85388",
|
||||
extra_property_data=False,
|
||||
),
|
||||
scrape_property(
|
||||
location="15509 N 172nd Dr, Surprise, AZ 85388",
|
||||
),
|
||||
]
|
||||
|
||||
assert not results[0].equals(results[1])
|
||||
|
||||
|
||||
def test_pr_zip_code():
|
||||
results = scrape_property(
|
||||
location="00741",
|
||||
listing_type="for_sale",
|
||||
)
|
||||
|
||||
assert results is not None and len(results) > 0
|
||||
|
||||
Reference in New Issue
Block a user