turtlebasket
/
HomeHarvest
mirror of https://github.com/Bunsly/HomeHarvest.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

fix: drop dups if cols exist

pull/8/head
Cullen Watson 2023-09-18 22:24:14 -05:00
parent e1917009ae
commit 42e8ac4de9
1 changed files with 4 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
homeharvest/__init__.py
View File

@ -157,5 +157,8 @@ def scrape_property(
return pd.DataFrame() return pd.DataFrame()
final_df = pd.concat(results, ignore_index=True) final_df = pd.concat(results, ignore_index=True)
final_df = final_df.drop_duplicates(subset=["street_address", "city", "unit"], keep="first") dupe_check_columns = ["street_address", "city", "unit"]
if all(col in final_df.columns for col in dupe_check_columns):
final_df = final_df.drop_duplicates(subset=dupe_check_columns, keep="first")
return final_df return final_df