feat: keep duplicates flag

This commit is contained in:
Cullen Watson
2023-09-20 20:24:18 -05:00
parent e9ddc6df92
commit 644f16b25b
2 changed files with 11 additions and 2 deletions

View File

@@ -119,6 +119,7 @@ def scrape_property(
site_name: Union[str, list[str]] = None,
listing_type: str = "for_sale",
proxy: str = None,
keep_duplicates: bool = False
) -> pd.DataFrame:
"""
Scrape property from various sites from a given location and listing type.
@@ -165,5 +166,6 @@ def scrape_property(
if col not in final_df.columns:
final_df[col] = None
final_df = final_df.drop_duplicates(subset=columns_to_track, keep="first")
if not keep_duplicates:
final_df = final_df.drop_duplicates(subset=columns_to_track, keep="first")
return final_df