added county, fips, and text desciption fields (#72)

pull/76/head
joecryptotoo 2024-04-20 15:44:28 -07:00 committed by GitHub
parent 7246703999
commit 7a525caeb8
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
3 changed files with 22 additions and 1 deletions

View File

@ -73,6 +73,7 @@ class Description:
year_built: int | None = None
garage: float | None = None
stories: int | None = None
text: str | None = None
@dataclass
@ -101,6 +102,7 @@ class Property:
latitude: float | None = None
longitude: float | None = None
neighborhoods: Optional[str] = None
county: Optional[str] = None
fips_code: Optional[str] = None
agents: list[Agent] = None
nearby_schools: list[str] = None

View File

@ -176,6 +176,7 @@ class RealtorScraper(Scraper):
year_built=property_info["details"].get("year_built"),
garage=property_info["details"].get("garage"),
stories=property_info["details"].get("stories"),
text=property_info["description"].get("text"),
),
days_on_mls=days_on_mls,
agents=agents_schools["agents"],
@ -330,6 +331,7 @@ class RealtorScraper(Scraper):
type
name
stories
text
}
source {
id
@ -353,10 +355,17 @@ class RealtorScraper(Scraper):
lat
}
}
county {
name
fips_code
}
neighborhoods {
name
}
}
tax_record {
public_record_id
}
primary_photo {
href
}
@ -536,6 +545,9 @@ class RealtorScraper(Scraper):
longitude=result["location"]["address"]["coordinate"].get("lon") if able_to_get_lat_long else None,
address=self._parse_address(result, search_type="general_search"),
description=self._parse_description(result),
neighborhoods=self._parse_neighborhoods(result),
county=result["location"]["county"].get("name"),
fips_code=result["location"]["county"].get("fips_code"),
days_on_mls=self.calculate_days_on_mls(result),
agents=agents_schools["agents"],
nearby_schools=agents_schools["schools"],
@ -725,6 +737,7 @@ class RealtorScraper(Scraper):
year_built=description_data.get("year_built"),
garage=description_data.get("garage"),
stories=description_data.get("stories"),
text=description_data.get("text"),
)
@staticmethod

View File

@ -8,6 +8,7 @@ ordered_properties = [
"mls",
"mls_id",
"status",
"text",
"style",
"street",
"unit",
@ -28,6 +29,9 @@ ordered_properties = [
"price_per_sqft",
"latitude",
"longitude",
"neighborhoods",
"county",
"fips_code",
"stories",
"hoa_fee",
"parking_garage",
@ -77,6 +81,8 @@ def process_result(result: Property) -> pd.DataFrame:
prop_data["year_built"] = description.year_built
prop_data["parking_garage"] = description.garage
prop_data["stories"] = description.stories
prop_data["text"] = description.text
properties_df = pd.DataFrame([prop_data])
properties_df = properties_df.reindex(columns=ordered_properties)