- style error bug fix

README.md

@@ -90,9 +90,11 @@ Optional
 │
 ├── foreclosure (True/False): If set, fetches only foreclosures
 │
-└── proxy (string): In format 'http://user:pass@host:port'
+├── proxy (string): In format 'http://user:pass@host:port'
 │
-└── extra_property_data (bool): Increases requests by O(n). If set, this fetches additional property data (e.g. agent, broker, property evaluations etc.)
+├── extra_property_data (True/False): Increases requests by O(n). If set, this fetches additional property data (e.g. agent, broker, property evaluations etc.)
+│
+└── exclude_pending (True/False): If set, excludes pending properties from the results unless listing_type is 'pending'
 ```
 
 ### Property Schema
@@ -142,6 +144,11 @@ Property
 │ ├── agent
 │ ├── agent_email
 │ └── agent_phone
+
+├── Broker Info:
+│ ├── broker
+│ ├── broker_email
+│ └── broker_website
 ```
 
 ### Exceptions

homeharvest/__init__.py

@@ -17,11 +17,12 @@ def scrape_property(
     date_to: str = None,
     foreclosure: bool = None,
     extra_property_data: bool = True,
+    exclude_pending: bool = False,
 ) -> pd.DataFrame:
     """
     Scrape properties from Realtor.com based on a given location and listing type.
     :param location: Location to search (e.g. "Dallas, TX", "85281", "2530 Al Lipscomb Way")
-    :param listing_type: Listing Type (for_sale, for_rent, sold)
+    :param listing_type: Listing Type (for_sale, for_rent, sold, pending)
     :param radius: Get properties within _ (e.g. 1.0) miles. Only applicable for individual addresses.
     :param mls_only: If set, fetches only listings with MLS IDs.
     :param proxy: Proxy to use for scraping
@@ -29,6 +30,7 @@ def scrape_property(
     :param date_from, date_to: Get properties sold or listed (dependent on your listing_type) between these dates. format: 2021-01-28
     :param foreclosure: If set, fetches only foreclosure listings.
     :param extra_property_data: Increases requests by O(n). If set, this fetches additional property data (e.g. agent, broker, property evaluations etc.)
+    :param exclude_pending: If true, this excludes pending or contingent properties from the results, unless listing type is pending.
     """
     validate_input(listing_type)
     validate_dates(date_from, date_to)
@@ -44,6 +46,7 @@ def scrape_property(
         date_to=date_to,
         foreclosure=foreclosure,
         extra_property_data=extra_property_data,
+        exclude_pending=exclude_pending,
     )
 
     site = RealtorScraper(scraper_input)

homeharvest/core/scrapers/__init__.py

@@ -21,6 +21,7 @@ class ScraperInput:
     date_to: str | None = None
     foreclosure: bool | None = False
     extra_property_data: bool | None = True
+    exclude_pending: bool | None = False
 
 
 class Scraper:
@@ -62,6 +63,7 @@ class Scraper:
         self.date_to = scraper_input.date_to
         self.foreclosure = scraper_input.foreclosure
         self.extra_property_data = scraper_input.extra_property_data
+        self.exclude_pending = scraper_input.exclude_pending
 
     def search(self) -> list[Property]: ...
 

homeharvest/core/scrapers/realtor/__init__.py

@@ -525,7 +525,7 @@ class RealtorScraper(Scraper):
 
             is_pending = result["flags"].get("is_pending") or result["flags"].get("is_contingent")
 
-            if is_pending and self.listing_type != ListingType.PENDING:
+            if is_pending and (self.exclude_pending and self.listing_type != ListingType.PENDING):
                 return
 
             property_id = result["property_id"]

homeharvest/utils.py

@@ -83,19 +83,20 @@ def process_result(result: Property) -> pd.DataFrame:
     prop_data["nearby_schools"] = ", ".join(set(prop_data["nearby_schools"])) if prop_data["nearby_schools"] else None
 
     description = result.description
-    prop_data["primary_photo"] = description.primary_photo
+    if description:
-    prop_data["alt_photos"] = ", ".join(description.alt_photos) if description.alt_photos else None
+        prop_data["primary_photo"] = description.primary_photo
-    prop_data["style"] = description.style if type(description.style) == str else description.style.value
+        prop_data["alt_photos"] = ", ".join(description.alt_photos) if description.alt_photos else None
-    prop_data["beds"] = description.beds
+        prop_data["style"] = description.style if isinstance(description.style, str) else description.style.value if description.style else None
-    prop_data["full_baths"] = description.baths_full
+        prop_data["beds"] = description.beds
-    prop_data["half_baths"] = description.baths_half
+        prop_data["full_baths"] = description.baths_full
-    prop_data["sqft"] = description.sqft
+        prop_data["half_baths"] = description.baths_half
-    prop_data["lot_sqft"] = description.lot_sqft
+        prop_data["sqft"] = description.sqft
-    prop_data["sold_price"] = description.sold_price
+        prop_data["lot_sqft"] = description.lot_sqft
-    prop_data["year_built"] = description.year_built
+        prop_data["sold_price"] = description.sold_price
-    prop_data["parking_garage"] = description.garage
+        prop_data["year_built"] = description.year_built
-    prop_data["stories"] = description.stories
+        prop_data["parking_garage"] = description.garage
-    prop_data["text"] = description.text
+        prop_data["stories"] = description.stories
+        prop_data["text"] = description.text
 
     properties_df = pd.DataFrame([prop_data])
     properties_df = properties_df.reindex(columns=ordered_properties)

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "homeharvest"
-version = "0.3.28"
+version = "0.3.30"
 description = "Real estate scraping library"
 authors = ["Zachary Hampton <zachary@bunsly.com>", "Cullen Watson <cullen@bunsly.com>"]
 homepage = "https://github.com/Bunsly/HomeHarvest"

tests/test_realtor.py

@@ -4,7 +4,7 @@ from homeharvest import scrape_property
 def test_realtor_pending_or_contingent():
     pending_or_contingent_result = scrape_property(location="Surprise, AZ", listing_type="pending")
 
-    regular_result = scrape_property(location="Surprise, AZ", listing_type="for_sale")
+    regular_result = scrape_property(location="Surprise, AZ", listing_type="for_sale", exclude_pending=True)
 
     assert all([result is not None for result in [pending_or_contingent_result, regular_result]])
     assert len(pending_or_contingent_result) != len(regular_result)
@@ -165,3 +165,23 @@ def test_pr_zip_code():
     )
 
     assert results is not None and len(results) > 0
+
+
+def test_exclude_pending():
+    results = scrape_property(
+        location="33567",
+        listing_type="pending",
+        exclude_pending=True,
+    )
+
+    assert results is not None and len(results) > 0
+
+
+def test_style_value_error():
+    results = scrape_property(
+        location="Alaska, AK",
+        listing_type="sold",
+        extra_property_data=False,
+    )
+
+    assert results is not None and len(results) > 0