- switch eps

- new hrefs
- property_id, listing_id data points

README.md

@@ -104,6 +104,8 @@ Optional
 Property
 ├── Basic Information:
 │ ├── property_url
+│ ├── property_id
+│ ├── listing_id
 │ ├── mls
 │ ├── mls_id
 │ └── status

homeharvest/core/scrapers/__init__.py

@@ -46,8 +46,21 @@ class Scraper:
             Scraper.session.mount("https://", adapter)
             Scraper.session.headers.update(
                 {
-                    "auth": f"Bearer {self.get_access_token()}",
-                    "apollographql-client-name": "com.move.Realtor-apollo-ios",
+                    'accept': 'application/json, text/javascript',
+                    'accept-language': 'en-US,en;q=0.9',
+                    'cache-control': 'no-cache',
+                    'content-type': 'application/json',
+                    'origin': 'https://www.realtor.com',
+                    'pragma': 'no-cache',
+                    'priority': 'u=1, i',
+                    'rdc-ab-tests': 'commute_travel_time_variation:v1',
+                    'sec-ch-ua': '"Not)A;Brand";v="99", "Google Chrome";v="127", "Chromium";v="127"',
+                    'sec-ch-ua-mobile': '?0',
+                    'sec-ch-ua-platform': '"Windows"',
+                    'sec-fetch-dest': 'empty',
+                    'sec-fetch-mode': 'cors',
+                    'sec-fetch-site': 'same-origin',
+                    'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36',
                 }
             )
 

homeharvest/core/scrapers/models.py

@@ -97,6 +97,8 @@ class Entity:
 
 @dataclass
 class Agent(Entity):
+    mls_set: str | None = None
+    nrds_id: str | None = None
     phones: list[dict] | AgentPhone | None = None
     email: str | None = None
     href: str | None = None
@@ -104,6 +106,7 @@ class Agent(Entity):
 
 @dataclass
 class Office(Entity):
+    mls_set: str | None = None
     email: str | None = None
     href: str | None = None
     phones: list[dict] | AgentPhone | None = None
@@ -130,6 +133,10 @@ class Advertisers:
 @dataclass
 class Property:
     property_url: str
+
+    property_id: str
+    listing_id: str | None = None
+
     mls: str | None = None
     mls_id: str | None = None
     status: str | None = None

homeharvest/core/scrapers/realtor/__init__.py

@@ -114,7 +114,9 @@ class RealtorScraper(Scraper):
             advertiser_type = advertiser.get("type")
             if advertiser_type == "seller":  #: agent
                 processed_advertisers.agent = Agent(
-                    uuid=advertiser.get("mls_set"),
+                    uuid=_parse_fulfillment_id(advertiser.get("fulfillment_id")),
+                    nrds_id=advertiser.get("nrds_id"),
+                    mls_set=advertiser.get("mls_set"),
                     name=advertiser.get("name"),
                     email=advertiser.get("email"),
                     phones=advertiser.get("phones"),
@@ -128,7 +130,8 @@ class RealtorScraper(Scraper):
 
                 if advertiser.get("office"):  #: has an office
                     processed_advertisers.office = Office(
-                        uuid=_parse_fulfillment_id(advertiser["office"].get("fulfillment_id")) or advertiser["office"].get("mls_set"),
+                        uuid=_parse_fulfillment_id(advertiser["office"].get("fulfillment_id")),
+                        mls_set=advertiser["office"].get("mls_set"),
                         name=advertiser["office"].get("name"),
                         email=advertiser["office"].get("email"),
                         phones=advertiser["office"].get("phones"),
@@ -178,11 +181,9 @@ class RealtorScraper(Scraper):
                 if "source" in result and isinstance(result["source"], dict)
                 else None
             ),
-            property_url=(
-                f"{self.PROPERTY_URL}{property_id}"
-                if self.listing_type != ListingType.FOR_RENT
-                else f"{self.PROPERTY_URL}M{property_id}?listing_status=rental"
-            ),
+            property_url=result["href"],
+            property_id=property_id,
+            listing_id=result.get("listing_id"),
             status="PENDING" if is_pending else result["status"].upper(),
             list_price=result["list_price"],
             list_price_min=result["list_price_min"],
@@ -466,7 +467,7 @@ class RealtorScraper(Scraper):
                 }"""
 
         variables = {"property_id": property_id}
-        response = self.session.post(self.PROPERTY_GQL, json={"query": query, "variables": variables})
+        response = self.session.post(self.SEARCH_GQL_URL, json={"query": query, "variables": variables})
 
         data = response.json()
         property_details = data["data"]["home"]

homeharvest/core/scrapers/realtor/queries.py

@@ -2,6 +2,7 @@ _SEARCH_HOMES_DATA_BASE = """{
     pending_date
     listing_id
     property_id
+    href
     list_date
     status
     last_sold_price
@@ -109,6 +110,7 @@ _SEARCH_HOMES_DATA_BASE = """{
             fulfillment_id
         }
         mls_set
+        nrds_id
         rental_corporation {
             fulfillment_id
         }

homeharvest/utils.py

@@ -6,6 +6,8 @@ from .exceptions import InvalidListingType, InvalidDate
 
 ordered_properties = [
     "property_url",
+    "property_id",
+    "listing_id",
     "mls",
     "mls_id",
     "status",
@@ -46,11 +48,14 @@ ordered_properties = [
     "agent_name",
     "agent_email",
     "agent_phones",
+    "agent_mls_set",
+    "agent_nrds_id",
     "broker_id",
     "broker_name",
     "builder_id",
     "builder_name",
     "office_id",
+    "office_mls_set",
     "office_name",
     "office_email",
     "office_phones",
@@ -81,6 +86,8 @@ def process_result(result: Property) -> pd.DataFrame:
             prop_data["agent_name"] = agent_data.name
             prop_data["agent_email"] = agent_data.email
             prop_data["agent_phones"] = agent_data.phones
+            prop_data["agent_mls_set"] = agent_data.mls_set
+            prop_data["agent_nrds_id"] = agent_data.nrds_id
 
         if advertiser_data.broker:
             broker_data = advertiser_data.broker
@@ -98,6 +105,7 @@ def process_result(result: Property) -> pd.DataFrame:
             prop_data["office_name"] = office_data.name
             prop_data["office_email"] = office_data.email
             prop_data["office_phones"] = office_data.phones
+            prop_data["office_mls_set"] = office_data.mls_set
 
     prop_data["price_per_sqft"] = prop_data["prc_sqft"]
     prop_data["nearby_schools"] = filter(None, prop_data["nearby_schools"]) if prop_data["nearby_schools"] else None

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "homeharvest"
-version = "0.4.0"
+version = "0.4.3"
 description = "Real estate scraping library"
 authors = ["Zachary Hampton <zachary@bunsly.com>", "Cullen Watson <cullen@bunsly.com>"]
 homepage = "https://github.com/Bunsly/HomeHarvest"

tests/test_realtor.py

@@ -128,6 +128,7 @@ def test_realtor_bad_address():
         location="abceefg ju098ot498hh9",
         listing_type="for_sale",
     )
+
     if len(bad_results) == 0:
         assert True
 
@@ -243,3 +244,39 @@ def test_apartment_list_price():
     assert len(results[results[["list_price", "list_price_min", "list_price_max"]].notnull().any(axis=1)]) / len(
         results
     ) > 0.5
+
+
+def test_builder_exists():
+    listing = scrape_property(
+        location="18149 W Poston Dr, Surprise, AZ 85387",
+        extra_property_data=False,
+    )
+
+    assert listing is not None
+    assert listing["builder_name"].nunique() > 0
+
+
+def test_phone_number_matching():
+    searches = [
+        scrape_property(
+            location="Phoenix, AZ",
+            listing_type="for_sale",
+            limit=100,
+        ),
+        scrape_property(
+            location="Phoenix, AZ",
+            listing_type="for_sale",
+            limit=100,
+        ),
+    ]
+
+    assert all([search is not None for search in searches])
+
+    #: random row
+    row = searches[0][searches[0]["agent_phones"].notnull()].sample()
+
+    #: find matching row
+    matching_row = searches[1].loc[searches[1]["property_url"] == row["property_url"].values[0]]
+
+    #: assert phone numbers are the same
+    assert row["agent_phones"].values[0] == matching_row["agent_phones"].values[0]