- pending listing support

- removal of pending_or_contingent param

homeharvest/__init__.py

@@ -13,7 +13,6 @@ def scrape_property(
     radius: float = None,
     mls_only: bool = False,
     past_days: int = None,
-    pending_or_contingent: bool = False,
     proxy: str = None,
 ) -> pd.DataFrame:
     """
@@ -23,7 +22,6 @@ def scrape_property(
     :param radius: Get properties within _ (e.g. 1.0) miles. Only applicable for individual addresses.
     :param mls_only: If set, fetches only listings with MLS IDs.
     :param past_days: Get properties sold or listed (dependent on your listing_type) in the last _ days.
-    :param pending_or_contingent: If set, fetches only pending or contingent listings. Only applicable for for_sale listings from general area searches.
     :param proxy: Proxy to use for scraping
     """
     validate_input(listing_type)
@@ -35,7 +33,6 @@ def scrape_property(
         radius=radius,
         mls_only=mls_only,
         last_x_days=past_days,
-        pending_or_contingent=pending_or_contingent,
     )
 
     site = RealtorScraper(scraper_input)

homeharvest/cli.py

@@ -14,7 +14,7 @@ def main():
         "--listing_type",
         type=str,
         default="for_sale",
-        choices=["for_sale", "for_rent", "sold"],
+        choices=["for_sale", "for_rent", "sold", "pending"],
         help="Listing type to scrape",
     )
 
@@ -60,13 +60,6 @@ def main():
         help="If set, fetches only MLS listings.",
     )
 
-    parser.add_argument(
-        "-c",
-        "--pending_or_contingent",
-        action="store_true",
-        help="If set, fetches only pending or contingent listings. Only applicable for for_sale listings from general area searches.",
-    )
-
     args = parser.parse_args()
 
     result = scrape_property(
@@ -76,7 +69,6 @@ def main():
         proxy=args.proxy,
         mls_only=args.mls_only,
         past_days=args.days,
-        pending_or_contingent=args.pending_or_contingent,
     )
 
     if not args.filename:

homeharvest/core/scrapers/__init__.py

@@ -11,7 +11,6 @@ class ScraperInput:
     mls_only: bool | None = None
     proxy: str | None = None
     last_x_days: int | None = None
-    pending_or_contingent: bool | None = None
 
 
 class Scraper:
@@ -37,7 +36,6 @@ class Scraper:
         self.radius = scraper_input.radius
         self.last_x_days = scraper_input.last_x_days
         self.mls_only = scraper_input.mls_only
-        self.pending_or_contingent = scraper_input.pending_or_contingent
 
     def search(self) -> list[Property]:
         ...

homeharvest/core/scrapers/models.py

@@ -19,6 +19,7 @@ class SiteName(Enum):
 class ListingType(Enum):
     FOR_SALE = "FOR_SALE"
     FOR_RENT = "FOR_RENT"
+    PENDING = "PENDING"
     SOLD = "SOLD"
 
 

homeharvest/core/scrapers/realtor/__init__.py

@@ -18,7 +18,6 @@ class RealtorScraper(Scraper):
     ADDRESS_AUTOCOMPLETE_URL = "https://parser-external.geo.moveaws.com/suggest"
 
     def __init__(self, scraper_input):
-        self.counter = 1
         super().__init__(scraper_input)
 
     def handle_location(self):
@@ -274,6 +273,10 @@ class RealtorScraper(Scraper):
                                 last_sold_date
                                 list_price
                                 price_per_sqft
+                                flags {
+                                    is_contingent
+                                    is_pending
+                                }
                                 description {
                                     sqft
                                     beds
@@ -335,17 +338,19 @@ class RealtorScraper(Scraper):
 
         pending_or_contingent_param = (
             "or_filters: { contingent: true, pending: true }"
-            if self.pending_or_contingent
+            if self.listing_type == ListingType.PENDING
             else ""
         )
 
+        listing_type = ListingType.FOR_SALE if self.listing_type == ListingType.PENDING else self.listing_type
+
         if search_type == "comps":  #: comps search, came from an address
             query = """query Property_search(
                     $coordinates: [Float]!
                     $radius: String!
                     $offset: Int!,
                     ) {
-                        property_search(
+                        home_search(
                             query: { 
                                 nearby: {
                                     coordinates: $coordinates
@@ -353,13 +358,15 @@ class RealtorScraper(Scraper):
                                 }
                                 status: %s
                                 %s
+                                %s
                             }
                             %s
                             limit: 200
                             offset: $offset
                     ) %s""" % (
-                self.listing_type.value.lower(),
+                listing_type.value.lower(),
                 date_param,
+                pending_or_contingent_param,
                 sort_param,
                 results_query,
             )
@@ -385,7 +392,7 @@ class RealtorScraper(Scraper):
                                     limit: 200
                                     offset: $offset
                                 ) %s""" % (
-                self.listing_type.value.lower(),
+                listing_type.value.lower(),
                 date_param,
                 pending_or_contingent_param,
                 sort_param,
@@ -415,7 +422,7 @@ class RealtorScraper(Scraper):
         response = self.session.post(self.SEARCH_GQL_URL, json=payload)
         response.raise_for_status()
         response_json = response.json()
-        search_key = "home_search" if search_type == "area" else "property_search"
+        search_key = "home_search" if "home_search" in query else "property_search"
 
         properties: list[Property] = []
 
@@ -430,7 +437,6 @@ class RealtorScraper(Scraper):
             return {"total": 0, "properties": []}
 
         for result in response_json["data"][search_key]["results"]:
-            self.counter += 1
             mls = (
                 result["source"].get("id")
                 if "source" in result and isinstance(result["source"], dict)
@@ -447,13 +453,15 @@ class RealtorScraper(Scraper):
                 and result["location"]["address"].get("coordinate")
             )
 
+            is_pending = result["flags"].get("is_pending") or result["flags"].get("is_contingent")
+
             realty_property = Property(
                 mls=mls,
                 mls_id=result["source"].get("listing_id")
                 if "source" in result and isinstance(result["source"], dict)
                 else None,
                 property_url=f"{self.PROPERTY_URL}{result['property_id']}",
-                status=result["status"].upper(),
+                status="PENDING" if is_pending else result["status"].upper(),
                 list_price=result["list_price"],
                 list_date=result["list_date"].split("T")[0]
                 if result.get("list_date")