Update pyproject.toml

[enh] days_on_mls attr

README.md

@@ -6,9 +6,9 @@
 
 **Not technical?** Try out the web scraping tool on our site at [tryhomeharvest.com](https://tryhomeharvest.com).
 
-*Looking to build a data-focused software product?* **[Book a call](https://calendly.com/zachary-products/15min)** *to work with us.*
+*Looking to build a data-focused software product?* **[Book a call](https://calendly.com/bunsly/15min)** *to work with us.*
 
-Check out another project we wrote: ***[JobSpy](https://github.com/cullenwatson/JobSpy)** – a Python package for job scraping*
+Check out another project we wrote: ***[JobSpy](https://github.com/Bunsly/JobSpy)** – a Python package for job scraping*
 
 ## HomeHarvest Features
 
@@ -20,7 +20,7 @@ Check out another project we wrote: ***[JobSpy](https://github.com/cullenwatson/
   - **CLI**: For users who prefer command-line operations.
 
 
-[Video Guide for HomeHarvest](https://youtu.be/JnV7eR2Ve2o) - _updated for release v0.2.7_
+[Video Guide for HomeHarvest](https://youtu.be/J1qgNPgmSLI) - _updated for release v0.3.4_
 
 ![homeharvest](https://github.com/ZacharyHampton/HomeHarvest/assets/78247585/b3d5d727-e67b-4a9f-85d8-1e65fd18620a)
 
@@ -45,9 +45,8 @@ filename = f"HomeHarvest_{current_timestamp}.csv"
 
 properties = scrape_property(
   location="San Diego, CA",
-  listing_type="sold",  # or (for_sale, for_rent)
+  listing_type="sold",  # or (for_sale, for_rent, pending)
   past_days=30,  # sold in last 30 days - listed in last x days if (for_sale, for_rent)
-  # pending_or_contingent=True # use on for_sale listings to find pending / contingent listings
   # mls_only=True,  # only fetch MLS listings
   # proxy="http://user:pass@host:port"  # use a proxy to change your IP address
 )
@@ -69,7 +68,7 @@ positional arguments:
   location              Location to scrape (e.g., San Francisco, CA)                                                          
                                                                                                                              
 options:                                                                                                                      
-  -l {for_sale,for_rent,sold}, --listing_type {for_sale,for_rent,sold}                                                        
+  -l {for_sale,for_rent,sold,pending}, --listing_type {for_sale,for_rent,sold,pending}                                                        
                         Listing type to scrape                                                                                
   -o {excel,csv}, --output {excel,csv}                                                                                        
                         Output format                                                                                         
@@ -81,9 +80,6 @@ options:
   -r RADIUS, --radius RADIUS                                                                                                  
                         Get comparable properties within _ (e.g., 0.0) miles. Only applicable for individual addresses.        
   -m, --mls_only        If set, fetches only MLS listings.                                                                    
-  -c, --pending_or_contingent
-                        If set, fetches only pending or contingent listings. Only applicable for for_sale listings from general area searches.
-
 ```
 ```bash
 homeharvest "San Francisco, CA" -l for_rent -o excel -f HomeHarvest
@@ -110,6 +106,7 @@ Required
     - 'for_rent'
     - 'for_sale'
     - 'sold'
+    - 'pending'
 
 Optional
 ├── radius (decimal): Radius in miles to find comparable properties based on individual addresses.
@@ -117,8 +114,6 @@ Optional
 │
 ├── past_days (integer): Number of past days to filter properties. Utilizes 'last_sold_date' for 'sold' listing types, and 'list_date' for others (for_rent, for_sale).
 │    Example: 30 (fetches properties listed/sold in the last 30 days)
-|
-├── pending_or_contingent (True/False): If set, fetches only pending or contingent listings. Only applicable for `for_sale listings` from general area searches.
 │
 ├── mls_only (True/False): If set, fetches only MLS listings (mainly applicable to 'sold' listings)
 │
@@ -152,6 +147,7 @@ Property
 │ └── lot_sqft
 
 ├── Property Listing Details:
+│ ├── days_on_mls
 │ ├── list_price
 │ ├── list_date
 │ ├── sold_price

examples/HomeHarvest_Demo.ipynb

@@ -4,7 +4,9 @@
    "cell_type": "code",
    "execution_count": null,
    "id": "cb48903e-5021-49fe-9688-45cd0bc05d0f",
-   "metadata": {},
+   "metadata": {
+    "is_executing": true
+   },
    "outputs": [],
    "source": [
     "from homeharvest import scrape_property\n",
@@ -84,10 +86,34 @@
    "outputs": [],
    "source": [
     "# check sold properties\n",
-    "scrape_property(\n",
+    "properties = scrape_property(\n",
     "    location=\"90210\",\n",
-    "    listing_type=\"sold\"\n",
+    "    listing_type=\"sold\",\n",
-    ")"
+    "    past_days=10\n",
+    ")\n",
+    "display(properties)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "628c1ce2",
+   "metadata": {
+    "collapsed": false,
+    "is_executing": true,
+    "jupyter": {
+     "outputs_hidden": false
+    }
+   },
+   "outputs": [],
+   "source": [
+    "# display clickable URLs\n",
+    "from IPython.display import display, HTML\n",
+    "properties['property_url'] = '<a href=\"' + properties['property_url'] + '\" target=\"_blank\">' + properties['property_url'] + '</a>'\n",
+    "\n",
+    "html = properties.to_html(escape=False)\n",
+    "truncate_width = f'<style>.dataframe td {{ max-width: 200px; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; }}</style>{html}'\n",
+    "display(HTML(truncate_width))"
    ]
   }
  ],

homeharvest/__init__.py

@@ -13,7 +13,6 @@ def scrape_property(
     radius: float = None,
     mls_only: bool = False,
     past_days: int = None,
-    pending_or_contingent: bool = False,
     proxy: str = None,
 ) -> pd.DataFrame:
     """
@@ -23,7 +22,6 @@ def scrape_property(
     :param radius: Get properties within _ (e.g. 1.0) miles. Only applicable for individual addresses.
     :param mls_only: If set, fetches only listings with MLS IDs.
     :param past_days: Get properties sold or listed (dependent on your listing_type) in the last _ days.
-    :param pending_or_contingent: If set, fetches only pending or contingent listings. Only applicable for for_sale listings from general area searches.
     :param proxy: Proxy to use for scraping
     """
     validate_input(listing_type)
@@ -35,7 +33,6 @@ def scrape_property(
         radius=radius,
         mls_only=mls_only,
         last_x_days=past_days,
-        pending_or_contingent=pending_or_contingent,
     )
 
     site = RealtorScraper(scraper_input)

homeharvest/cli.py

@@ -14,7 +14,7 @@ def main():
         "--listing_type",
         type=str,
         default="for_sale",
-        choices=["for_sale", "for_rent", "sold"],
+        choices=["for_sale", "for_rent", "sold", "pending"],
         help="Listing type to scrape",
     )
 
@@ -60,13 +60,6 @@ def main():
         help="If set, fetches only MLS listings.",
     )
 
-    parser.add_argument(
-        "-c",
-        "--pending_or_contingent",
-        action="store_true",
-        help="If set, fetches only pending or contingent listings. Only applicable for for_sale listings from general area searches.",
-    )
-
     args = parser.parse_args()
 
     result = scrape_property(
@@ -76,7 +69,6 @@ def main():
         proxy=args.proxy,
         mls_only=args.mls_only,
         past_days=args.days,
-        pending_or_contingent=args.pending_or_contingent,
     )
 
     if not args.filename:

homeharvest/core/scrapers/__init__.py

@@ -11,7 +11,6 @@ class ScraperInput:
     mls_only: bool | None = None
     proxy: str | None = None
     last_x_days: int | None = None
-    pending_or_contingent: bool | None = None
 
 
 class Scraper:
@@ -37,7 +36,6 @@ class Scraper:
         self.radius = scraper_input.radius
         self.last_x_days = scraper_input.last_x_days
         self.mls_only = scraper_input.mls_only
-        self.pending_or_contingent = scraper_input.pending_or_contingent
 
     def search(self) -> list[Property]:
         ...

homeharvest/core/scrapers/models.py

@@ -19,6 +19,7 @@ class SiteName(Enum):
 class ListingType(Enum):
     FOR_SALE = "FOR_SALE"
     FOR_RENT = "FOR_RENT"
+    PENDING = "PENDING"
     SOLD = "SOLD"
 
 
@@ -58,6 +59,7 @@ class Property:
     last_sold_date: str | None = None
     prc_sqft: int | None = None
     hoa_fee: int | None = None
+    days_on_mls: int | None = None
     description: Description | None = None
 
     latitude: float | None = None

homeharvest/core/scrapers/realtor/__init__.py

@@ -4,6 +4,7 @@ homeharvest.realtor.__init__
 
 This module implements the scraper for realtor.com
 """
+from datetime import datetime
 from typing import Dict, Union, Optional
 from concurrent.futures import ThreadPoolExecutor, as_completed
 
@@ -18,7 +19,6 @@ class RealtorScraper(Scraper):
     ADDRESS_AUTOCOMPLETE_URL = "https://parser-external.geo.moveaws.com/suggest"
 
     def __init__(self, scraper_input):
-        self.counter = 1
         super().__init__(scraper_input)
 
     def handle_location(self):
@@ -105,11 +105,29 @@ class RealtorScraper(Scraper):
         )
 
         able_to_get_lat_long = (
-            property_info
+                property_info
-            and property_info.get("address")
+                and property_info.get("address")
-            and property_info["address"].get("location")
+                and property_info["address"].get("location")
-            and property_info["address"]["location"].get("coordinate")
+                and property_info["address"]["location"].get("coordinate")
         )
+        list_date_str = property_info["basic"]["list_date"].split("T")[0] if property_info["basic"].get(
+            "list_date") else None
+        last_sold_date_str = property_info["basic"]["sold_date"].split("T")[0] if property_info["basic"].get(
+            "sold_date") else None
+
+        list_date = datetime.strptime(list_date_str, "%Y-%m-%d") if list_date_str else None
+        last_sold_date = datetime.strptime(last_sold_date_str, "%Y-%m-%d") if last_sold_date_str else None
+        today = datetime.now()
+
+        days_on_mls = None
+        status = property_info["basic"]["status"].lower()
+        if list_date:
+            if status == "sold" and last_sold_date:
+                days_on_mls = (last_sold_date - list_date).days
+            elif status in ('for_sale', 'for_rent'):
+                days_on_mls = (today - list_date).days
+            if days_on_mls and days_on_mls < 0:
+                days_on_mls = None
 
         listing = Property(
             mls=mls,
@@ -119,17 +137,13 @@ class RealtorScraper(Scraper):
             property_url=f"{self.PROPERTY_URL}{property_info['details']['permalink']}",
             status=property_info["basic"]["status"].upper(),
             list_price=property_info["basic"]["price"],
-            list_date=property_info["basic"]["list_date"].split("T")[0]
+            list_date=list_date,
-            if property_info["basic"].get("list_date")
-            else None,
             prc_sqft=property_info["basic"].get("price")
-            / property_info["basic"].get("sqft")
+                     / property_info["basic"].get("sqft")
             if property_info["basic"].get("price")
-            and property_info["basic"].get("sqft")
+               and property_info["basic"].get("sqft")
-            else None,
-            last_sold_date=property_info["basic"]["sold_date"].split("T")[0]
-            if property_info["basic"].get("sold_date")
             else None,
+            last_sold_date=last_sold_date,
             latitude=property_info["address"]["location"]["coordinate"].get("lat")
             if able_to_get_lat_long
             else None,
@@ -149,6 +163,7 @@ class RealtorScraper(Scraper):
                 garage=property_info["details"].get("garage"),
                 stories=property_info["details"].get("stories"),
             ),
+            days_on_mls=days_on_mls
         )
 
         return [listing]
@@ -274,6 +289,10 @@ class RealtorScraper(Scraper):
                                 last_sold_date
                                 list_price
                                 price_per_sqft
+                                flags {
+                                    is_contingent
+                                    is_pending
+                                }
                                 description {
                                     sqft
                                     beds
@@ -335,17 +354,19 @@ class RealtorScraper(Scraper):
 
         pending_or_contingent_param = (
             "or_filters: { contingent: true, pending: true }"
-            if self.pending_or_contingent
+            if self.listing_type == ListingType.PENDING
             else ""
         )
 
+        listing_type = ListingType.FOR_SALE if self.listing_type == ListingType.PENDING else self.listing_type
+
         if search_type == "comps":  #: comps search, came from an address
             query = """query Property_search(
                     $coordinates: [Float]!
                     $radius: String!
                     $offset: Int!,
                     ) {
-                        property_search(
+                        home_search(
                             query: { 
                                 nearby: {
                                     coordinates: $coordinates
@@ -353,13 +374,15 @@ class RealtorScraper(Scraper):
                                 }
                                 status: %s
                                 %s
+                                %s
                             }
                             %s
                             limit: 200
                             offset: $offset
                     ) %s""" % (
-                self.listing_type.value.lower(),
+                listing_type.value.lower(),
                 date_param,
+                pending_or_contingent_param,
                 sort_param,
                 results_query,
             )
@@ -385,7 +408,7 @@ class RealtorScraper(Scraper):
                                     limit: 200
                                     offset: $offset
                                 ) %s""" % (
-                self.listing_type.value.lower(),
+                listing_type.value.lower(),
                 date_param,
                 pending_or_contingent_param,
                 sort_param,
@@ -415,7 +438,7 @@ class RealtorScraper(Scraper):
         response = self.session.post(self.SEARCH_GQL_URL, json=payload)
         response.raise_for_status()
         response_json = response.json()
-        search_key = "home_search" if search_type == "area" else "property_search"
+        search_key = "home_search" if "home_search" in query else "property_search"
 
         properties: list[Property] = []
 
@@ -430,7 +453,6 @@ class RealtorScraper(Scraper):
             return {"total": 0, "properties": []}
 
         for result in response_json["data"][search_key]["results"]:
-            self.counter += 1
             mls = (
                 result["source"].get("id")
                 if "source" in result and isinstance(result["source"], dict)
@@ -447,13 +469,15 @@ class RealtorScraper(Scraper):
                 and result["location"]["address"].get("coordinate")
             )
 
+            is_pending = result["flags"].get("is_pending") or result["flags"].get("is_contingent")
+
             realty_property = Property(
                 mls=mls,
                 mls_id=result["source"].get("listing_id")
                 if "source" in result and isinstance(result["source"], dict)
                 else None,
                 property_url=f"{self.PROPERTY_URL}{result['property_id']}",
-                status=result["status"].upper(),
+                status="PENDING" if is_pending else result["status"].upper(),
                 list_price=result["list_price"],
                 list_date=result["list_date"].split("T")[0]
                 if result.get("list_date")
@@ -470,8 +494,8 @@ class RealtorScraper(Scraper):
                 if able_to_get_lat_long
                 else None,
                 address=self._parse_address(result, search_type="general_search"),
-                #: neighborhoods=self._parse_neighborhoods(result),
                 description=self._parse_description(result),
+                days_on_mls=self.calculate_days_on_mls(result)
             )
             properties.append(realty_property)
 
@@ -582,7 +606,6 @@ class RealtorScraper(Scraper):
         description_data = result.get("description", {})
 
         if description_data is None or not isinstance(description_data, dict):
-            print("Warning: description_data is invalid!")
             description_data = {}
 
         style = description_data.get("type", "")
@@ -601,3 +624,22 @@ class RealtorScraper(Scraper):
             garage=description_data.get("garage"),
             stories=description_data.get("stories"),
         )
+
+    @staticmethod
+    def calculate_days_on_mls(result: dict) -> Optional[int]:
+        list_date_str = result.get("list_date")
+        list_date = datetime.strptime(list_date_str.split("T")[0], "%Y-%m-%d") if list_date_str else None
+        last_sold_date_str = result.get("last_sold_date")
+        last_sold_date = datetime.strptime(last_sold_date_str, "%Y-%m-%d") if last_sold_date_str else None
+        today = datetime.now()
+
+        if list_date:
+            if result["status"] == 'sold':
+                if last_sold_date:
+                    days = (last_sold_date - list_date).days
+                    if days >= 0:
+                        return days
+            elif result["status"] in ('for_sale', 'for_rent'):
+                days = (today - list_date).days
+                if days >= 0:
+                    return days

homeharvest/utils.py

@@ -18,6 +18,7 @@ ordered_properties = [
     "half_baths",
     "sqft",
     "year_built",
+    "days_on_mls",
     "list_price",
     "list_date",
     "sold_price",

pyproject.toml

@@ -1,9 +1,9 @@
 [tool.poetry]
 name = "homeharvest"
-version = "0.3.3"
+version = "0.3.6"
 description = "Real estate scraping library supporting Zillow, Realtor.com & Redfin."
 authors = ["Zachary Hampton <zachary@zacharysproducts.com>", "Cullen Watson <cullen@cullen.ai>"]
-homepage = "https://github.com/ZacharyHampton/HomeHarvest"
+homepage = "https://github.com/Bunsly/HomeHarvest"
 readme = "README.md"
 
 [tool.poetry.scripts]

tests/test_realtor.py

@@ -7,14 +7,10 @@ from homeharvest.exceptions import (
 
 def test_realtor_pending_or_contingent():
     pending_or_contingent_result = scrape_property(
-        location="Surprise, AZ",
+        location="Surprise, AZ", listing_type="pending"
-        pending_or_contingent=True,
     )
 
-    regular_result = scrape_property(
+    regular_result = scrape_property(location="Surprise, AZ", listing_type="for_sale")
-        location="Surprise, AZ",
-        pending_or_contingent=False,
-    )
 
     assert all(
         [
@@ -25,6 +21,35 @@ def test_realtor_pending_or_contingent():
     assert len(pending_or_contingent_result) != len(regular_result)
 
 
+def test_realtor_pending_comps():
+    pending_comps = scrape_property(
+        location="2530 Al Lipscomb Way",
+        radius=5,
+        past_days=180,
+        listing_type="pending",
+    )
+
+    for_sale_comps = scrape_property(
+        location="2530 Al Lipscomb Way",
+        radius=5,
+        past_days=180,
+        listing_type="for_sale",
+    )
+
+    sold_comps = scrape_property(
+        location="2530 Al Lipscomb Way",
+        radius=5,
+        past_days=180,
+        listing_type="sold",
+    )
+
+    results = [pending_comps, for_sale_comps, sold_comps]
+    assert all([result is not None for result in results])
+
+    #: assert all lengths are different
+    assert len(set([len(result) for result in results])) == len(results)
+
+
 def test_realtor_comps():
     result = scrape_property(
         location="2530 Al Lipscomb Way",