- switch eps

- new hrefs
- property_id, listing_id data points

README.md

@@ -104,6 +104,8 @@ Optional
 Property
 ├── Basic Information:
 │ ├── property_url
+│ ├── property_id
+│ ├── listing_id
 │ ├── mls
 │ ├── mls_id
 │ └── status

homeharvest/core/scrapers/__init__.py

@@ -46,8 +46,21 @@ class Scraper:
             Scraper.session.mount("https://", adapter)
             Scraper.session.headers.update(
                 {
-                    "auth": f"Bearer {self.get_access_token()}",
-                    "apollographql-client-name": "com.move.Realtor-apollo-ios",
+                    'accept': 'application/json, text/javascript',
+                    'accept-language': 'en-US,en;q=0.9',
+                    'cache-control': 'no-cache',
+                    'content-type': 'application/json',
+                    'origin': 'https://www.realtor.com',
+                    'pragma': 'no-cache',
+                    'priority': 'u=1, i',
+                    'rdc-ab-tests': 'commute_travel_time_variation:v1',
+                    'sec-ch-ua': '"Not)A;Brand";v="99", "Google Chrome";v="127", "Chromium";v="127"',
+                    'sec-ch-ua-mobile': '?0',
+                    'sec-ch-ua-platform': '"Windows"',
+                    'sec-fetch-dest': 'empty',
+                    'sec-fetch-mode': 'cors',
+                    'sec-fetch-site': 'same-origin',
+                    'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36',
                 }
             )
 

homeharvest/core/scrapers/models.py

@@ -106,6 +106,7 @@ class Agent(Entity):
 
 @dataclass
 class Office(Entity):
+    mls_set: str | None = None
     email: str | None = None
     href: str | None = None
     phones: list[dict] | AgentPhone | None = None
@@ -132,6 +133,10 @@ class Advertisers:
 @dataclass
 class Property:
     property_url: str
+
+    property_id: str
+    listing_id: str | None = None
+
     mls: str | None = None
     mls_id: str | None = None
     status: str | None = None

homeharvest/core/scrapers/realtor/__init__.py

@@ -130,7 +130,8 @@ class RealtorScraper(Scraper):
 
                 if advertiser.get("office"):  #: has an office
                     processed_advertisers.office = Office(
-                        uuid=_parse_fulfillment_id(advertiser["office"].get("fulfillment_id")) or advertiser["office"].get("mls_set"),
+                        uuid=_parse_fulfillment_id(advertiser["office"].get("fulfillment_id")),
+                        mls_set=advertiser["office"].get("mls_set"),
                         name=advertiser["office"].get("name"),
                         email=advertiser["office"].get("email"),
                         phones=advertiser["office"].get("phones"),
@@ -180,11 +181,9 @@ class RealtorScraper(Scraper):
                 if "source" in result and isinstance(result["source"], dict)
                 else None
             ),
-            property_url=(
-                f"{self.PROPERTY_URL}{property_id}"
-                if self.listing_type != ListingType.FOR_RENT
-                else f"{self.PROPERTY_URL}M{property_id}?listing_status=rental"
-            ),
+            property_url=result["href"],
+            property_id=property_id,
+            listing_id=result.get("listing_id"),
             status="PENDING" if is_pending else result["status"].upper(),
             list_price=result["list_price"],
             list_price_min=result["list_price_min"],
@@ -468,7 +467,7 @@ class RealtorScraper(Scraper):
                 }"""
 
         variables = {"property_id": property_id}
-        response = self.session.post(self.PROPERTY_GQL, json={"query": query, "variables": variables})
+        response = self.session.post(self.SEARCH_GQL_URL, json={"query": query, "variables": variables})
 
         data = response.json()
         property_details = data["data"]["home"]

homeharvest/core/scrapers/realtor/queries.py

@@ -2,6 +2,7 @@ _SEARCH_HOMES_DATA_BASE = """{
     pending_date
     listing_id
     property_id
+    href
     list_date
     status
     last_sold_price

homeharvest/utils.py

@@ -6,6 +6,8 @@ from .exceptions import InvalidListingType, InvalidDate
 
 ordered_properties = [
     "property_url",
+    "property_id",
+    "listing_id",
     "mls",
     "mls_id",
     "status",
@@ -53,6 +55,7 @@ ordered_properties = [
     "builder_id",
     "builder_name",
     "office_id",
+    "office_mls_set",
     "office_name",
     "office_email",
     "office_phones",
@@ -102,6 +105,7 @@ def process_result(result: Property) -> pd.DataFrame:
             prop_data["office_name"] = office_data.name
             prop_data["office_email"] = office_data.email
             prop_data["office_phones"] = office_data.phones
+            prop_data["office_mls_set"] = office_data.mls_set
 
     prop_data["price_per_sqft"] = prop_data["prc_sqft"]
     prop_data["nearby_schools"] = filter(None, prop_data["nearby_schools"]) if prop_data["nearby_schools"] else None

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "homeharvest"
-version = "0.4.1"
+version = "0.4.3"
 description = "Real estate scraping library"
 authors = ["Zachary Hampton <zachary@bunsly.com>", "Cullen Watson <cullen@bunsly.com>"]
 homepage = "https://github.com/Bunsly/HomeHarvest"

tests/test_realtor.py

@@ -128,6 +128,7 @@ def test_realtor_bad_address():
         location="abceefg ju098ot498hh9",
         listing_type="for_sale",
     )
+
     if len(bad_results) == 0:
         assert True
 
@@ -253,3 +254,29 @@ def test_builder_exists():
 
     assert listing is not None
     assert listing["builder_name"].nunique() > 0
+
+
+def test_phone_number_matching():
+    searches = [
+        scrape_property(
+            location="Phoenix, AZ",
+            listing_type="for_sale",
+            limit=100,
+        ),
+        scrape_property(
+            location="Phoenix, AZ",
+            listing_type="for_sale",
+            limit=100,
+        ),
+    ]
+
+    assert all([search is not None for search in searches])
+
+    #: random row
+    row = searches[0][searches[0]["agent_phones"].notnull()].sample()
+
+    #: find matching row
+    matching_row = searches[1].loc[searches[1]["property_url"] == row["property_url"].values[0]]
+
+    #: assert phone numbers are the same
+    assert row["agent_phones"].values[0] == matching_row["agent_phones"].values[0]