homeharvest/core/scrapers/models.py

@@ -97,8 +97,6 @@ class Entity:
 
 @dataclass
 class Agent(Entity):
-    mls_set: str | None = None
-    nrds_id: str | None = None
     phones: list[dict] | AgentPhone | None = None
     email: str | None = None
     href: str | None = None
@@ -106,7 +104,6 @@ class Agent(Entity):
 
 @dataclass
 class Office(Entity):
-    mls_set: str | None = None
     email: str | None = None
     href: str | None = None
     phones: list[dict] | AgentPhone | None = None

homeharvest/core/scrapers/realtor/__init__.py

@@ -114,9 +114,7 @@ class RealtorScraper(Scraper):
             advertiser_type = advertiser.get("type")
             if advertiser_type == "seller":  #: agent
                 processed_advertisers.agent = Agent(
-                    uuid=_parse_fulfillment_id(advertiser.get("fulfillment_id")),
+                    uuid=advertiser.get("mls_set"),
-                    nrds_id=advertiser.get("nrds_id"),
-                    mls_set=advertiser.get("mls_set"),
                     name=advertiser.get("name"),
                     email=advertiser.get("email"),
                     phones=advertiser.get("phones"),
@@ -130,8 +128,7 @@ class RealtorScraper(Scraper):
 
                 if advertiser.get("office"):  #: has an office
                     processed_advertisers.office = Office(
-                        uuid=_parse_fulfillment_id(advertiser["office"].get("fulfillment_id")),
+                        uuid=_parse_fulfillment_id(advertiser["office"].get("fulfillment_id")) or advertiser["office"].get("mls_set"),
-                        mls_set=advertiser["office"].get("mls_set"),
                         name=advertiser["office"].get("name"),
                         email=advertiser["office"].get("email"),
                         phones=advertiser["office"].get("phones"),

homeharvest/core/scrapers/realtor/queries.py

@@ -109,7 +109,6 @@ _SEARCH_HOMES_DATA_BASE = """{
             fulfillment_id
         }
         mls_set
-        nrds_id
         rental_corporation {
             fulfillment_id
         }

homeharvest/utils.py

@@ -46,14 +46,11 @@ ordered_properties = [
     "agent_name",
     "agent_email",
     "agent_phones",
-    "agent_mls_set",
-    "agent_nrds_id",
     "broker_id",
     "broker_name",
     "builder_id",
     "builder_name",
     "office_id",
-    "office_mls_set",
     "office_name",
     "office_email",
     "office_phones",
@@ -84,8 +81,6 @@ def process_result(result: Property) -> pd.DataFrame:
             prop_data["agent_name"] = agent_data.name
             prop_data["agent_email"] = agent_data.email
             prop_data["agent_phones"] = agent_data.phones
-            prop_data["agent_mls_set"] = agent_data.mls_set
-            prop_data["agent_nrds_id"] = agent_data.nrds_id
 
         if advertiser_data.broker:
             broker_data = advertiser_data.broker
@@ -103,7 +98,6 @@ def process_result(result: Property) -> pd.DataFrame:
             prop_data["office_name"] = office_data.name
             prop_data["office_email"] = office_data.email
             prop_data["office_phones"] = office_data.phones
-            prop_data["office_mls_set"] = office_data.mls_set
 
     prop_data["price_per_sqft"] = prop_data["prc_sqft"]
     prop_data["nearby_schools"] = filter(None, prop_data["nearby_schools"]) if prop_data["nearby_schools"] else None

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "homeharvest"
-version = "0.4.2"
+version = "0.4.0"
 description = "Real estate scraping library"
 authors = ["Zachary Hampton <zachary@bunsly.com>", "Cullen Watson <cullen@bunsly.com>"]
 homepage = "https://github.com/Bunsly/HomeHarvest"

tests/test_realtor.py

@@ -243,13 +243,3 @@ def test_apartment_list_price():
     assert len(results[results[["list_price", "list_price_min", "list_price_max"]].notnull().any(axis=1)]) / len(
         results
     ) > 0.5
-
-
-def test_builder_exists():
-    listing = scrape_property(
-        location="18149 W Poston Dr, Surprise, AZ 85387",
-        extra_property_data=False,
-    )
-
-    assert listing is not None
-    assert listing["builder_name"].nunique() > 0