Merge pull request #43 from Bunsly/add_photos

Add photos

README.md

@@ -92,7 +92,7 @@ Optional
 │    Example: 30 (fetches properties listed/sold in the last 30 days)
 │
 ├── date_from, date_to (string): Start and end dates to filter properties listed or sold, both dates are required.
-}    (use this to get properties in chunks as there's a 10k result limit)
+|    (use this to get properties in chunks as there's a 10k result limit)
 │    Format for both must be "YYYY-MM-DD". 
 │    Example: "2023-05-01", "2023-05-15" (fetches properties listed/sold between these dates)
 │

homeharvest/core/scrapers/models.py

@@ -34,6 +34,8 @@ class Address:
 
 @dataclass
 class Description:
+    primary_photo: str | None = None
+    alt_photos: list[str] | None = None
     style: str | None = None
     beds: int | None = None
     baths_full: int | None = None

homeharvest/core/scrapers/realtor/__init__.py

@@ -84,6 +84,12 @@ class RealtorScraper(Scraper):
                             garage
                             permalink
                         }
+                        primary_photo {
+                            href
+                        }
+                        photos {
+                            href
+                        }
                     }
                 }"""
 
@@ -152,6 +158,8 @@ class RealtorScraper(Scraper):
             else None,
             address=self._parse_address(property_info, search_type="handle_listing"),
             description=Description(
+                primary_photo=property_info["primary_photo"].get("href", "").replace("s.jpg", "od-w480_h360_x2.webp?w=1080&q=75"),
+                alt_photos=self.process_alt_photos(property_info.get("photos", [])),
                 style=property_info["basic"].get("type", "").upper(),
                 beds=property_info["basic"].get("beds"),
                 baths_full=property_info["basic"].get("baths_full"),
@@ -247,6 +255,12 @@ class RealtorScraper(Scraper):
                             units
                             year_built
                         }
+                        primary_photo {
+                            href
+                        }
+                        photos {
+                            href
+                        }
                     }
                 }"""
 
@@ -334,6 +348,12 @@ class RealtorScraper(Scraper):
                                         name
                                     }
                                 }
+                                primary_photo {
+                                    href
+                                }
+                                photos {
+                                    href
+                                }
                             }
                         }
                     }"""
@@ -621,6 +641,7 @@ class RealtorScraper(Scraper):
 
     @staticmethod
     def _parse_description(result: dict) -> Description:
+
         description_data = result.get("description", {})
 
         if description_data is None or not isinstance(description_data, dict):
@@ -630,7 +651,16 @@ class RealtorScraper(Scraper):
         if style is not None:
             style = style.upper()
 
+        primary_photo = ""
+        if result and "primary_photo" in result:
+            primary_photo_info = result["primary_photo"]
+            if primary_photo_info and "href" in primary_photo_info:
+                primary_photo_href = primary_photo_info["href"]
+                primary_photo = primary_photo_href.replace("s.jpg", "od-w480_h360_x2.webp?w=1080&q=75")
+
         return Description(
+            primary_photo=primary_photo,
+            alt_photos=RealtorScraper.process_alt_photos(result.get("photos")),
             style=style,
             beds=description_data.get("beds"),
             baths_full=description_data.get("baths_full"),
@@ -643,6 +673,7 @@ class RealtorScraper(Scraper):
             stories=description_data.get("stories"),
         )
 
+
     @staticmethod
     def calculate_days_on_mls(result: dict) -> Optional[int]:
         list_date_str = result.get("list_date")
@@ -661,3 +692,16 @@ class RealtorScraper(Scraper):
                 days = (today - list_date).days
                 if days >= 0:
                     return days
+
+    @staticmethod
+    def process_alt_photos(photos_info):
+        try:
+            alt_photos = []
+            if photos_info:
+                for photo_info in photos_info:
+                    href = photo_info.get("href", "")
+                    alt_photo_href = href.replace("s.jpg", "od-w480_h360_x2.webp?w=1080&q=75")
+                    alt_photos.append(alt_photo_href)
+            return alt_photos
+        except Exception:
+            pass

homeharvest/utils.py

@@ -5,6 +5,8 @@ from .exceptions import InvalidListingType, InvalidDate
 
 ordered_properties = [
     "property_url",
+    "primary_photo",
+    "alt_photos",
     "mls",
     "mls_id",
     "status",
@@ -49,6 +51,8 @@ def process_result(result: Property) -> pd.DataFrame:
     prop_data["price_per_sqft"] = prop_data["prc_sqft"]
 
     description = result.description
+    prop_data["primary_photo"] = description.primary_photo
+    prop_data["alt_photos"] = ", ".join(description.alt_photos)
     prop_data["style"] = description.style
     prop_data["beds"] = description.beds
     prop_data["full_baths"] = description.baths_full

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "homeharvest"
-version = "0.3.9"
+version = "0.3.10"
 description = "Real estate scraping library supporting Zillow, Realtor.com & Redfin."
 authors = ["Zachary Hampton <zachary@zacharysproducts.com>", "Cullen Watson <cullen@cullen.ai>"]
 homepage = "https://github.com/Bunsly/HomeHarvest"