Merge pull request #43 from Bunsly/add_photos

Add photos

README.md

@@ -6,7 +6,7 @@
 
 **Not technical?** Try out the web scraping tool on our site at [tryhomeharvest.com](https://tryhomeharvest.com).
 
-*Looking to build a data-focused software product?* **[Book a call](https://calendly.com/bunsly/15min)** *to work with us.*
+*Looking to build a data-focused software product?* **[Book a call](https://bunsly.com)** *to work with us.*
 
 Check out another project we wrote: ***[JobSpy](https://github.com/Bunsly/JobSpy)** – a Python package for job scraping*
 
@@ -91,15 +91,16 @@ Optional
 ├── past_days (integer): Number of past days to filter properties. Utilizes 'last_sold_date' for 'sold' listing types, and 'list_date' for others (for_rent, for_sale).
 │    Example: 30 (fetches properties listed/sold in the last 30 days)
 │
-├── date_range (string tuple): Start and end dates to filter properties listed or sold, both dates are required.
+├── date_from, date_to (string): Start and end dates to filter properties listed or sold, both dates are required.
-}    (use this to get properties in chunks as there's a 10k result limit)
+|    (use this to get properties in chunks as there's a 10k result limit)
 │    Format for both must be "YYYY-MM-DD". 
-│    Example: ("2023-05-01", "2023-05-15") (fetches properties listed/sold between these dates)
+│    Example: "2023-05-01", "2023-05-15" (fetches properties listed/sold between these dates)
 │
 ├── mls_only (True/False): If set, fetches only MLS listings (mainly applicable to 'sold' listings)
 │
 └── proxy (string): In format 'http://user:pass@host:port'
 
+
 ```
 
 ### CLI 
@@ -177,7 +178,7 @@ Property
 The following exceptions may be raised when using HomeHarvest:
 
 - `InvalidListingType` - valid options: `for_sale`, `for_rent`, `sold`
-- `NoResultsFound` - no properties found from your search
+- `InvalidDate` - date_from or date_to is not in the format YYYY-MM-DD
   
   
 ## Frequently Asked Questions

homeharvest/core/scrapers/models.py

@@ -34,6 +34,8 @@ class Address:
 
 @dataclass
 class Description:
+    primary_photo: str | None = None
+    alt_photos: list[str] | None = None
     style: str | None = None
     beds: int | None = None
     baths_full: int | None = None

homeharvest/core/scrapers/realtor/__init__.py

@@ -49,6 +49,7 @@ class RealtorScraper(Scraper):
                             listing_id
                         }
                         address {
+                            street_direction
                             street_number
                             street_name
                             street_suffix
@@ -83,6 +84,12 @@ class RealtorScraper(Scraper):
                             garage
                             permalink
                         }
+                        primary_photo {
+                            href
+                        }
+                        photos {
+                            href
+                        }
                     }
                 }"""
 
@@ -151,6 +158,8 @@ class RealtorScraper(Scraper):
             else None,
             address=self._parse_address(property_info, search_type="handle_listing"),
             description=Description(
+                primary_photo=property_info["primary_photo"].get("href", "").replace("s.jpg", "od-w480_h360_x2.webp?w=1080&q=75"),
+                alt_photos=self.process_alt_photos(property_info.get("photos", [])),
                 style=property_info["basic"].get("type", "").upper(),
                 beds=property_info["basic"].get("beds"),
                 baths_full=property_info["basic"].get("baths_full"),
@@ -215,6 +224,7 @@ class RealtorScraper(Scraper):
                             stories
                         }
                         address {
+                            street_direction
                             street_number
                             street_name
                             street_suffix
@@ -245,6 +255,12 @@ class RealtorScraper(Scraper):
                             units
                             year_built
                         }
+                        primary_photo {
+                            href
+                        }
+                        photos {
+                            href
+                        }
                     }
                 }"""
 
@@ -315,6 +331,7 @@ class RealtorScraper(Scraper):
                                 }
                                 location {
                                     address {
+                                        street_direction
                                         street_number
                                         street_name
                                         street_suffix
@@ -331,6 +348,12 @@ class RealtorScraper(Scraper):
                                         name
                                     }
                                 }
+                                primary_photo {
+                                    href
+                                }
+                                photos {
+                                    href
+                                }
                             }
                         }
                     }"""
@@ -606,6 +629,7 @@ class RealtorScraper(Scraper):
         return Address(
             street=" ".join([
                 self.handle_none_safely(address.get('street_number')),
+                self.handle_none_safely(address.get('street_direction')),
                 self.handle_none_safely(address.get('street_name')),
                 self.handle_none_safely(address.get('street_suffix')),
             ]).strip(),
@@ -617,6 +641,7 @@ class RealtorScraper(Scraper):
 
     @staticmethod
     def _parse_description(result: dict) -> Description:
+
         description_data = result.get("description", {})
 
         if description_data is None or not isinstance(description_data, dict):
@@ -626,7 +651,16 @@ class RealtorScraper(Scraper):
         if style is not None:
             style = style.upper()
 
+        primary_photo = ""
+        if result and "primary_photo" in result:
+            primary_photo_info = result["primary_photo"]
+            if primary_photo_info and "href" in primary_photo_info:
+                primary_photo_href = primary_photo_info["href"]
+                primary_photo = primary_photo_href.replace("s.jpg", "od-w480_h360_x2.webp?w=1080&q=75")
+
         return Description(
+            primary_photo=primary_photo,
+            alt_photos=RealtorScraper.process_alt_photos(result.get("photos")),
             style=style,
             beds=description_data.get("beds"),
             baths_full=description_data.get("baths_full"),
@@ -639,6 +673,7 @@ class RealtorScraper(Scraper):
             stories=description_data.get("stories"),
         )
 
+
     @staticmethod
     def calculate_days_on_mls(result: dict) -> Optional[int]:
         list_date_str = result.get("list_date")
@@ -657,3 +692,16 @@ class RealtorScraper(Scraper):
                 days = (today - list_date).days
                 if days >= 0:
                     return days
+
+    @staticmethod
+    def process_alt_photos(photos_info):
+        try:
+            alt_photos = []
+            if photos_info:
+                for photo_info in photos_info:
+                    href = photo_info.get("href", "")
+                    alt_photo_href = href.replace("s.jpg", "od-w480_h360_x2.webp?w=1080&q=75")
+                    alt_photos.append(alt_photo_href)
+            return alt_photos
+        except Exception:
+            pass

homeharvest/utils.py

@@ -5,6 +5,8 @@ from .exceptions import InvalidListingType, InvalidDate
 
 ordered_properties = [
     "property_url",
+    "primary_photo",
+    "alt_photos",
     "mls",
     "mls_id",
     "status",
@@ -49,6 +51,8 @@ def process_result(result: Property) -> pd.DataFrame:
     prop_data["price_per_sqft"] = prop_data["prc_sqft"]
 
     description = result.description
+    prop_data["primary_photo"] = description.primary_photo
+    prop_data["alt_photos"] = ", ".join(description.alt_photos)
     prop_data["style"] = description.style
     prop_data["beds"] = description.beds
     prop_data["full_baths"] = description.baths_full

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "homeharvest"
-version = "0.3.8"
+version = "0.3.10"
 description = "Real estate scraping library supporting Zillow, Realtor.com & Redfin."
 authors = ["Zachary Hampton <zachary@zacharysproducts.com>", "Cullen Watson <cullen@cullen.ai>"]
 homepage = "https://github.com/Bunsly/HomeHarvest"