- switch eps

- new hrefs
- property_id, listing_id data points

homeharvest/core/scrapers/__init__.py

@@ -46,8 +46,21 @@ class Scraper:
             Scraper.session.mount("https://", adapter)
             Scraper.session.headers.update(
                 {
-                    "auth": f"Bearer {self.get_access_token()}",
-                    "apollographql-client-name": "com.move.Realtor-apollo-ios",
+                    'accept': 'application/json, text/javascript',
+                    'accept-language': 'en-US,en;q=0.9',
+                    'cache-control': 'no-cache',
+                    'content-type': 'application/json',
+                    'origin': 'https://www.realtor.com',
+                    'pragma': 'no-cache',
+                    'priority': 'u=1, i',
+                    'rdc-ab-tests': 'commute_travel_time_variation:v1',
+                    'sec-ch-ua': '"Not)A;Brand";v="99", "Google Chrome";v="127", "Chromium";v="127"',
+                    'sec-ch-ua-mobile': '?0',
+                    'sec-ch-ua-platform': '"Windows"',
+                    'sec-fetch-dest': 'empty',
+                    'sec-fetch-mode': 'cors',
+                    'sec-fetch-site': 'same-origin',
+                    'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36',
                 }
             )
 

homeharvest/core/scrapers/models.py

@@ -133,6 +133,10 @@ class Advertisers:
 @dataclass
 class Property:
     property_url: str
+
+    property_id: str
+    listing_id: str | None = None
+
     mls: str | None = None
     mls_id: str | None = None
     status: str | None = None

homeharvest/core/scrapers/realtor/__init__.py

@@ -181,11 +181,9 @@ class RealtorScraper(Scraper):
                 if "source" in result and isinstance(result["source"], dict)
                 else None
             ),
-            property_url=(
-                f"{self.PROPERTY_URL}{property_id}"
-                if self.listing_type != ListingType.FOR_RENT
-                else f"{self.PROPERTY_URL}M{property_id}?listing_status=rental"
-            ),
+            property_url=result["href"],
+            property_id=property_id,
+            listing_id=result.get("listing_id"),
             status="PENDING" if is_pending else result["status"].upper(),
             list_price=result["list_price"],
             list_price_min=result["list_price_min"],
@@ -469,7 +467,7 @@ class RealtorScraper(Scraper):
                 }"""
 
         variables = {"property_id": property_id}
-        response = self.session.post(self.PROPERTY_GQL, json={"query": query, "variables": variables})
+        response = self.session.post(self.SEARCH_GQL_URL, json={"query": query, "variables": variables})
 
         data = response.json()
         property_details = data["data"]["home"]

homeharvest/core/scrapers/realtor/queries.py

@@ -2,6 +2,7 @@ _SEARCH_HOMES_DATA_BASE = """{
     pending_date
     listing_id
     property_id
+    href
     list_date
     status
     last_sold_price