- realtor radius

This commit is contained in:
Zachary Hampton
2023-10-02 13:58:47 -07:00
parent 1f1ca8068f
commit 40bbf76db1
4 changed files with 123 additions and 82 deletions

View File

@@ -106,7 +106,7 @@ def _process_result(result: Property) -> pd.DataFrame:
return properties_df
def _scrape_single_site(location: str, site_name: str, listing_type: str, proxy: str = None) -> pd.DataFrame:
def _scrape_single_site(location: str, site_name: str, listing_type: str, radius: float, proxy: str = None) -> pd.DataFrame:
"""
Helper function to scrape a single site.
"""
@@ -117,6 +117,7 @@ def _scrape_single_site(location: str, site_name: str, listing_type: str, proxy:
listing_type=ListingType[listing_type.upper()],
site_name=SiteName.get_by_value(site_name.lower()),
proxy=proxy,
radius=radius,
)
site = _scrapers[site_name.lower()](scraper_input)
@@ -134,12 +135,14 @@ def scrape_property(
location: str,
site_name: Union[str, list[str]] = "realtor.com",
listing_type: str = "for_sale",
radius: float = None,
proxy: str = None,
keep_duplicates: bool = False
) -> pd.DataFrame:
"""
Scrape property from various sites from a given location and listing type.
:param radius: Radius in miles to find comparable properties on individual addresses
:param keep_duplicates:
:param proxy:
:param location: US Location (e.g. 'San Francisco, CA', 'Cook County, IL', '85281', '2530 Al Lipscomb Way')
@@ -157,12 +160,12 @@ def scrape_property(
results = []
if len(site_name) == 1:
final_df = _scrape_single_site(location, site_name[0], listing_type, proxy)
final_df = _scrape_single_site(location, site_name[0], listing_type, radius, proxy)
results.append(final_df)
else:
with ThreadPoolExecutor() as executor:
futures = {
executor.submit(_scrape_single_site, location, s_name, listing_type, proxy): s_name
executor.submit(_scrape_single_site, location, s_name, listing_type, radius, proxy): s_name
for s_name in site_name
}