- version bump

- Replace /suggest REST endpoint with GraphQL Search_suggestions query
- Use search_location field instead of individual city/county/state/postal_code fields
- Fix coordinate order to [lon, lat] (GeoJSON standard) for radius searches
- Extract mpr_id from addr: prefix for single address lookups

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>

README.md

@@ -84,7 +84,7 @@ properties = scrape_property(
 #### Sorting & Listing Types
 ```py
 # Sort options: list_price, list_date, sqft, beds, baths, last_update_date
-# Listing types: "for_sale", "for_rent", "sold", "pending", list, or None (all)
+# Listing types: "for_sale", "for_rent", "sold", "pending", "off_market", list, or None (common types)
 properties = scrape_property(
     location="Miami, FL",
     listing_type=["for_sale", "pending"],  # Single string, list, or None
@@ -94,6 +94,17 @@ properties = scrape_property(
 )
 ```
 
+#### Pagination Control
+```py
+# Sequential mode with early termination (more efficient for narrow filters)
+properties = scrape_property(
+    location="Los Angeles, CA",
+    listing_type="for_sale",
+    updated_in_past_hours=2,  # Narrow time window
+    parallel=False  # Fetch pages sequentially, stop when filters no longer match
+)
+```
+
 ## Output
 ```plaintext
 >>> properties.head()
@@ -129,30 +140,38 @@ for prop in properties[:5]:
 ```
 Required
 ├── location (str): Flexible location search - accepts any of these formats:
-    - ZIP code: "92104"
+│    - ZIP code: "92104"
-    - City: "San Diego" or "San Francisco"
+│    - City: "San Diego" or "San Francisco"
-    - City, State (abbreviated or full): "San Diego, CA" or "San Diego, California"
+│    - City, State (abbreviated or full): "San Diego, CA" or "San Diego, California"
-    - Full address: "1234 Main St, San Diego, CA 92104"
+│    - Full address: "1234 Main St, San Diego, CA 92104"
-    - Neighborhood: "Downtown San Diego"
+│    - Neighborhood: "Downtown San Diego"
-    - County: "San Diego County"
+│    - County: "San Diego County"
-├── listing_type (option): Choose the type of listing.
+│    - State (no support for abbreviated): "California"
-    - 'for_rent'
+│
-    - 'for_sale'
+├── listing_type (str | list[str] | None): Choose the type of listing.
-    - 'sold'
+│    - 'for_sale'
-    - 'pending' (for pending/contingent sales)
+│    - 'for_rent'
-
+│    - 'sold'
+│    - 'pending'
+│    - 'off_market'
+│    - 'new_community'
+│    - 'other'
+│    - 'ready_to_build'
+│    - List of strings returns properties matching ANY status: ['for_sale', 'pending']
+│    - None returns common listing types (for_sale, for_rent, sold, pending, off_market)
+│
 Optional
 ├── property_type (list): Choose the type of properties.
-    - 'single_family'
+│    - 'single_family'
-    - 'multi_family'
+│    - 'multi_family'
-    - 'condos'
+│    - 'condos'
-    - 'condo_townhome_rowhome_coop'
+│    - 'condo_townhome_rowhome_coop'
-    - 'condo_townhome'
+│    - 'condo_townhome'
-    - 'townhomes'
+│    - 'townhomes'
-    - 'duplex_triplex'
+│    - 'duplex_triplex'
-    - 'farm'
+│    - 'farm'
-    - 'land'
+│    - 'land'
-    - 'mobile'
+│    - 'mobile'
 │
 ├── return_type (option): Choose the return type.
 │    - 'pandas' (default)
@@ -165,12 +184,12 @@ Optional
 ├── past_days (integer): Number of past days to filter properties. Utilizes 'last_sold_date' for 'sold' listing types, and 'list_date' for others (for_rent, for_sale).
 │    Example: 30 (fetches properties listed/sold in the last 30 days)
 │
-├── past_hours (integer): Number of past hours to filter properties (more precise than past_days). Uses client-side filtering.
+├── past_hours (integer | timedelta): Number of past hours to filter properties (more precise than past_days). Uses client-side filtering.
-│    Example: 24 (fetches properties from the last 24 hours)
+│    Example: 24 or timedelta(hours=24) (fetches properties from the last 24 hours)
 │    Note: Cannot be used together with past_days or date_from/date_to
 │
 ├── date_from, date_to (string): Start and end dates to filter properties listed or sold, both dates are required.
-|    (use this to get properties in chunks as there's a 10k result limit)
+│    (use this to get properties in chunks as there's a 10k result limit)
 │    Accepts multiple formats with automatic precision detection:
 │    - Date strings: "YYYY-MM-DD" (day precision)
 │    - Datetime strings: "YYYY-MM-DDTHH:MM:SS" (hour precision, uses client-side filtering)
@@ -180,6 +199,14 @@ Optional
 │      Day precision: "2023-05-01", "2023-05-15"
 │      Hour precision: "2025-01-20T09:00:00", "2025-01-20T17:00:00"
 │
+├── updated_since (datetime | str): Filter properties updated since a specific date/time (based on last_update_date field)
+│    Accepts datetime objects or ISO 8601 strings
+│    Example: updated_since=datetime(2025, 11, 10, 9, 0) or "2025-11-10T09:00:00"
+│
+├── updated_in_past_hours (integer | timedelta): Filter properties updated in the past X hours (based on last_update_date field)
+│    Accepts integer (hours) or timedelta object
+│    Example: updated_in_past_hours=24 or timedelta(hours=24)
+│
 ├── beds_min, beds_max (integer): Filter by number of bedrooms
 │    Example: beds_min=2, beds_max=4 (2-4 bedrooms)
 │
@@ -199,7 +226,7 @@ Optional
 │    Example: year_built_min=2000, year_built_max=2024 (built between 2000-2024)
 │
 ├── sort_by (string): Sort results by field
-│    Options: 'list_date', 'sold_date', 'list_price', 'sqft', 'beds', 'baths'
+│    Options: 'list_date', 'sold_date', 'list_price', 'sqft', 'beds', 'baths', 'last_update_date'
 │    Example: sort_by='list_price'
 │
 ├── sort_direction (string): Sort direction, default is 'desc'
@@ -218,7 +245,9 @@ Optional
 │
 ├── limit (integer): Limit the number of properties to fetch. Max & default is 10000.
 │
-└── offset (integer): Starting position for pagination within the 10k limit. Use with limit to fetch results in chunks.
+├── offset (integer): Starting position for pagination within the 10k limit. Use with limit to fetch results in chunks.
+│
+└── parallel (True/False): Controls pagination strategy. Default is True (fetch pages in parallel for speed). Set to False for sequential fetching with early termination (useful for rate limiting or narrow time windows).
 ```
 
 ### Property Schema
@@ -265,6 +294,7 @@ Property
 │ ├── sold_price
 │ ├── last_sold_date  # datetime (full timestamp: YYYY-MM-DD HH:MM:SS)
 │ ├── last_status_change_date  # datetime (full timestamp: YYYY-MM-DD HH:MM:SS)
+│ ├── last_update_date  # datetime (full timestamp: YYYY-MM-DD HH:MM:SS)
 │ ├── last_sold_price
 │ ├── price_per_sqft
 │ ├── new_construction

homeharvest/__init__.py

@@ -48,6 +48,8 @@ def scrape_property(
     # New sorting parameters
     sort_by: str = None,
     sort_direction: str = "desc",
+    # Pagination control
+    parallel: bool = True,
 ) -> Union[pd.DataFrame, list[dict], list[Property]]:
     """
     Scrape properties from Realtor.com based on a given location and listing type.
@@ -72,6 +74,8 @@ def scrape_property(
         - date objects: date(2025, 1, 20) (day-level precision)
         - datetime objects: datetime(2025, 1, 20, 14, 30) (hour-level precision)
         The precision is automatically detected based on the input format.
+        Timezone handling: Naive datetimes are treated as local time and automatically converted to UTC.
+        Timezone-aware datetimes are converted to UTC. For best results, use timezone-aware datetimes.
     :param foreclosure: If set, fetches only foreclosure listings.
     :param extra_property_data: Increases requests by O(n). If set, this fetches additional property data (e.g. agent, broker, property evaluations etc.)
     :param exclude_pending: If true, this excludes pending or contingent properties from the results, unless listing type is pending.
@@ -80,7 +84,11 @@ def scrape_property(
 
     New parameters:
     :param past_hours: Get properties in the last _ hours (requires client-side filtering). Accepts int or timedelta.
-    :param updated_since: Filter by last_update_date (when property was last updated). Accepts datetime object or ISO 8601 string (client-side filtering)
+    :param updated_since: Filter by last_update_date (when property was last updated). Accepts datetime object or ISO 8601 string (client-side filtering).
+        Timezone handling: Naive datetimes (like datetime.now()) are treated as local time and automatically converted to UTC.
+        Timezone-aware datetimes are converted to UTC. Examples:
+        - datetime.now() - uses your local timezone
+        - datetime.now(timezone.utc) - uses UTC explicitly
     :param updated_in_past_hours: Filter by properties updated in the last _ hours. Accepts int or timedelta (client-side filtering)
     :param beds_min, beds_max: Filter by number of bedrooms
     :param baths_min, baths_max: Filter by number of bathrooms
@@ -90,6 +98,9 @@ def scrape_property(
     :param year_built_min, year_built_max: Filter by year built
     :param sort_by: Sort results by field (list_date, sold_date, list_price, sqft, beds, baths, last_update_date)
     :param sort_direction: Sort direction (asc, desc)
+    :param parallel: Controls pagination strategy. True (default) = fetch all pages in parallel for maximum speed.
+        False = fetch pages sequentially with early termination checks (useful for rate limiting or narrow time windows).
+        Sequential mode will stop paginating as soon as time-based filters indicate no more matches are possible.
 
     Note: past_days and past_hours also accept timedelta objects for more Pythonic usage.
     """
@@ -129,6 +140,22 @@ def scrape_property(
     converted_updated_since = convert_to_datetime_string(updated_since)
     converted_updated_in_past_hours = extract_timedelta_hours(updated_in_past_hours)
 
+    # Auto-apply optimal sort for time-based filters (unless user specified different sort)
+    if (converted_updated_since or converted_updated_in_past_hours) and not sort_by:
+        sort_by = "last_update_date"
+        if not sort_direction:
+            sort_direction = "desc"  # Most recent first
+
+    # Auto-apply optimal sort for PENDING listings with date filters
+    # PENDING API filtering is broken, so we rely on client-side filtering
+    # Sorting by pending_date ensures efficient pagination with early termination
+    elif (converted_listing_type == ListingType.PENDING and
+          (converted_past_days or converted_past_hours or converted_date_from) and
+          not sort_by):
+        sort_by = "pending_date"
+        if not sort_direction:
+            sort_direction = "desc"  # Most recent first
+
     scraper_input = ScraperInput(
         location=location,
         listing_type=converted_listing_type,
@@ -168,6 +195,8 @@ def scrape_property(
         # New sorting
         sort_by=sort_by,
         sort_direction=sort_direction,
+        # Pagination control
+        parallel=parallel,
     )
 
     site = RealtorScraper(scraper_input)

homeharvest/core/scrapers/__init__.py

@@ -55,6 +55,9 @@ class ScraperInput(BaseModel):
     sort_by: str | None = None
     sort_direction: str = "desc"
 
+    # Pagination control
+    parallel: bool = True
+
 
 class Scraper:
     session = None
@@ -78,21 +81,21 @@ class Scraper:
             Scraper.session.mount("https://", adapter)
             Scraper.session.headers.update(
                 {
-                    "accept": "application/json, text/javascript",
+                    'sec-ch-ua-platform': '"macOS"',
-                    "accept-language": "en-US,en;q=0.9",
+                    'rdc-client-name': 'rdc-search-for-sale-desktop',
-                    "cache-control": "no-cache",
+                    'sec-ch-ua': '"Chromium";v="142", "Google Chrome";v="142", "Not_A Brand";v="99"',
-                    "content-type": "application/json",
+                    'sec-ch-ua-mobile': '?0',
-                    "origin": "https://www.realtor.com",
+                    'rdc-client-version': '0.1.0',
-                    "pragma": "no-cache",
+                    'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/142.0.0.0 Safari/537.36',
-                    "priority": "u=1, i",
+                    'accept': 'application/json',
-                    "rdc-ab-tests": "commute_travel_time_variation:v1",
+                    'content-type': 'application/json',
-                    "sec-ch-ua": '"Not)A;Brand";v="99", "Google Chrome";v="127", "Chromium";v="127"',
+                    'origin': 'https://www.realtor.com',
-                    "sec-ch-ua-mobile": "?0",
+                    'sec-fetch-site': 'same-site',
-                    "sec-ch-ua-platform": '"Windows"',
+                    'sec-fetch-mode': 'cors',
-                    "sec-fetch-dest": "empty",
+                    'sec-fetch-dest': 'empty',
-                    "sec-fetch-mode": "cors",
+                    'referer': 'https://www.realtor.com/',
-                    "sec-fetch-site": "same-origin",
+                    'accept-language': 'en-US,en;q=0.9',
-                    "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36",
+                    'priority': 'u=1, i',
                 }
             )
 
@@ -141,6 +144,9 @@ class Scraper:
         self.sort_by = scraper_input.sort_by
         self.sort_direction = scraper_input.sort_direction
 
+        # Pagination control
+        self.parallel = scraper_input.parallel
+
     def search(self) -> list[Union[Property | dict]]: ...
 
     @staticmethod

homeharvest/core/scrapers/realtor/__init__.py

@@ -35,10 +35,7 @@ from .processors import (
 
 
 class RealtorScraper(Scraper):
-    SEARCH_GQL_URL = "https://www.realtor.com/api/v1/rdc_search_srp?client_id=rdc-search-new-communities&schema=vesta"
+    SEARCH_GQL_URL = "https://api.frontdoor.realtor.com/graphql"
-    PROPERTY_URL = "https://www.realtor.com/realestateandhomes-detail/"
-    PROPERTY_GQL = "https://graph.realtor.com/graphql"
-    ADDRESS_AUTOCOMPLETE_URL = "https://parser-external.geo.moveaws.com/suggest"
     NUM_PROPERTY_WORKERS = 20
     DEFAULT_PAGE_SIZE = 200
 
@@ -46,33 +43,70 @@ class RealtorScraper(Scraper):
         super().__init__(scraper_input)
 
     def handle_location(self):
-        # Get client_id from listing_type
+        query = """query Search_suggestions($searchInput: SearchSuggestionsInput!) {
-        if self.listing_type is None:
+            search_suggestions(search_input: $searchInput) {
-            client_id = "for-sale"
+                geo_results {
-        elif isinstance(self.listing_type, list):
+                    type
-            client_id = self.listing_type[0].value.lower().replace("_", "-") if self.listing_type else "for-sale"
+                    text
-        else:
+                    geo {
-            client_id = self.listing_type.value.lower().replace("_", "-")
+                        _id
+                        area_type
+                        city
+                        state_code
+                        postal_code
+                        county
+                        centroid { lat lon }
+                        slug_id
+                        geo_id
+                    }
+                }
+            }
+        }"""
 
-        params = {
+        variables = {
-            "input": self.location,
+            "searchInput": {
-            "client_id": client_id,
+                "search_term": self.location
-            "limit": "1",
+            }
-            "area_types": "city,state,county,postal_code,address,street,neighborhood,school,school_district,university,park",
         }
 
-        response = self.session.get(
+        payload = {
-            self.ADDRESS_AUTOCOMPLETE_URL,
+            "query": query,
-            params=params,
+            "variables": variables,
-        )
+        }
+
+        response = self.session.post(self.SEARCH_GQL_URL, json=payload)
         response_json = response.json()
 
-        result = response_json["autocomplete"]
+        if (
-
+            response_json is None
-        if not result:
+            or "data" not in response_json
+            or response_json["data"] is None
+            or "search_suggestions" not in response_json["data"]
+            or response_json["data"]["search_suggestions"] is None
+            or "geo_results" not in response_json["data"]["search_suggestions"]
+            or not response_json["data"]["search_suggestions"]["geo_results"]
+        ):
             return None
 
-        return result[0]
+        geo_result = response_json["data"]["search_suggestions"]["geo_results"][0]
+        geo = geo_result.get("geo", {})
+
+        result = {
+            "text": geo_result.get("text"),
+            "area_type": geo.get("area_type"),
+            "city": geo.get("city"),
+            "state_code": geo.get("state_code"),
+            "postal_code": geo.get("postal_code"),
+            "county": geo.get("county"),
+            "centroid": geo.get("centroid"),
+        }
+
+        if geo.get("area_type") == "address":
+            geo_id = geo.get("_id", "")
+            if geo_id.startswith("addr:"):
+                result["mpr_id"] = geo_id.replace("addr:", "")
+
+        return result
 
     def get_latest_listing_id(self, property_id: str) -> str | None:
         query = """query Property($property_id: ID!) {
@@ -108,6 +142,7 @@ class RealtorScraper(Scraper):
             return property_info["listings"][0]["listing_id"]
 
     def handle_home(self, property_id: str) -> list[Property]:
+        """Fetch single home with proper error handling."""
         query = (
             """query Home($property_id: ID!) {
                     home(property_id: $property_id) %s
@@ -116,23 +151,33 @@ class RealtorScraper(Scraper):
         )
 
         variables = {"property_id": property_id}
-        payload = {
+        payload = {"query": query, "variables": variables}
-            "query": query,
-            "variables": variables,
-        }
 
-        response = self.session.post(self.SEARCH_GQL_URL, json=payload)
+        try:
-        response_json = response.json()
+            response = self.session.post(self.SEARCH_GQL_URL, json=payload)
+            data = response.json()
 
-        property_info = response_json["data"]["home"]
+            # Check for errors or missing data
+            if "errors" in data or "data" not in data:
+                return []
 
-        if self.return_type != ReturnType.raw:
+            if data["data"] is None or "home" not in data["data"]:
-            return [process_property(property_info, self.mls_only, self.extra_property_data, 
+                return []
-                                   self.exclude_pending, self.listing_type, get_key, process_extra_property_details)]
-        else:
-            return [property_info]
 
+            property_info = data["data"]["home"]
+            if property_info is None:
+                return []
 
+            # Process based on return type
+            if self.return_type != ReturnType.raw:
+                return [process_property(property_info, self.mls_only, self.extra_property_data,
+                                       self.exclude_pending, self.listing_type, get_key,
+                                       process_extra_property_details)]
+            else:
+                return [property_info]
+
+        except Exception:
+            return []
 
     def general_search(self, variables: dict, search_type: str) -> Dict[str, Union[int, Union[list[Property], list[dict]]]]:
         """
@@ -144,7 +189,15 @@ class RealtorScraper(Scraper):
         # Determine date field based on listing type
         # Convert listing_type to list for uniform handling
         if self.listing_type is None:
-            listing_types = []
+            # When None, return all common listing types as documented
+            # Note: NEW_COMMUNITY, OTHER, and READY_TO_BUILD are excluded as they typically return no results
+            listing_types = [
+                ListingType.FOR_SALE,
+                ListingType.FOR_RENT,
+                ListingType.SOLD,
+                ListingType.PENDING,
+                ListingType.OFF_MARKET,
+            ]
             date_field = None  # When no listing_type is specified, skip date filtering
         elif isinstance(self.listing_type, list):
             listing_types = self.listing_type
@@ -277,10 +330,14 @@ class RealtorScraper(Scraper):
         else:
             sort_param = ""  #: prioritize normal fractal sort from realtor
 
-        # Handle PENDING with or_filters (applies if PENDING is in the list or is the single type)
+        # Handle PENDING with or_filters
+        # Only use or_filters when PENDING is the only type or mixed only with FOR_SALE
+        # Using or_filters with other types (SOLD, FOR_RENT, etc.) will exclude those types
         has_pending = ListingType.PENDING in listing_types
+        other_types = [lt for lt in listing_types if lt not in [ListingType.PENDING, ListingType.FOR_SALE]]
+        use_or_filters = has_pending and len(other_types) == 0
         pending_or_contingent_param = (
-            "or_filters: { contingent: true, pending: true }" if has_pending else ""
+            "or_filters: { contingent: true, pending: true }" if use_or_filters else ""
         )
 
         # Build bucket parameter (only use fractal sort if no custom sort is specified)
@@ -351,19 +408,13 @@ class RealtorScraper(Scraper):
             )
         elif search_type == "area":  #: general search, came from a general location
             query = """query Home_search(
-                                $city: String,
+                                $search_location: SearchLocation,
-                                $county: [String],
-                                $state_code: String,
-                                $postal_code: String
                                 $offset: Int,
                             ) {
                                 home_search(
                                     query: {
                                         %s
-                                        city: $city
+                                        search_location: $search_location
-                                        county: $county
-                                        postal_code: $postal_code
-                                        state_code: $state_code
                                         %s
                                         %s
                                         %s
@@ -499,24 +550,16 @@ class RealtorScraper(Scraper):
                 if not location_info.get("centroid"):
                     return []
 
-                coordinates = list(location_info["centroid"].values())
+                centroid = location_info["centroid"]
+                coordinates = [centroid["lon"], centroid["lat"]]  # GeoJSON order: [lon, lat]
                 search_variables |= {
                     "coordinates": coordinates,
                     "radius": "{}mi".format(self.radius),
                 }
 
-        elif location_type == "postal_code":
+        else:  #: general search (city, county, postal_code, etc.)
             search_variables |= {
-                "postal_code": location_info.get("postal_code"),
+                "search_location": {"location": location_info.get("text")},
-            }
-
-        else:  #: general search, location
-            search_variables |= {
-                "city": location_info.get("city"),
-                "county": location_info.get("county"),
-                "state_code": location_info.get("state_code"),
-                "postal_code": location_info.get("postal_code"),
-
             }
 
         if self.foreclosure:
@@ -526,31 +569,49 @@ class RealtorScraper(Scraper):
         total = result["total"]
         homes = result["properties"]
 
-        with ThreadPoolExecutor() as executor:
+        # Fetch remaining pages based on parallel parameter
-            # Store futures with their offsets to maintain proper sort order
+        if self.offset + self.DEFAULT_PAGE_SIZE < min(total, self.offset + self.limit):
-            # Start from offset + page_size and go up to offset + limit
+            if self.parallel:
-            futures_with_offsets = [
+                # Parallel mode: Fetch all remaining pages in parallel
-                (i, executor.submit(
+                with ThreadPoolExecutor() as executor:
-                    self.general_search,
+                    futures_with_offsets = [
-                    variables=search_variables | {"offset": i},
+                        (i, executor.submit(
-                    search_type=search_type,
+                            self.general_search,
-                ))
+                            variables=search_variables | {"offset": i},
-                for i in range(
+                            search_type=search_type,
+                        ))
+                        for i in range(
+                            self.offset + self.DEFAULT_PAGE_SIZE,
+                            min(total, self.offset + self.limit),
+                            self.DEFAULT_PAGE_SIZE,
+                        )
+                    ]
+
+                    # Collect results and sort by offset to preserve API sort order
+                    results = []
+                    for offset, future in futures_with_offsets:
+                        results.append((offset, future.result()["properties"]))
+
+                    results.sort(key=lambda x: x[0])
+                    for offset, properties in results:
+                        homes.extend(properties)
+            else:
+                # Sequential mode: Fetch pages one by one with early termination checks
+                for current_offset in range(
                     self.offset + self.DEFAULT_PAGE_SIZE,
                     min(total, self.offset + self.limit),
                     self.DEFAULT_PAGE_SIZE,
-                )
+                ):
-            ]
+                    # Check if we should continue based on time-based filters
+                    if not self._should_fetch_more_pages(homes):
+                        break
 
-            # Collect results and sort by offset to preserve API sort order across pages
+                    result = self.general_search(
-            results = []
+                        variables=search_variables | {"offset": current_offset},
-            for offset, future in futures_with_offsets:
+                        search_type=search_type,
-                results.append((offset, future.result()["properties"]))
+                    )
-
+                    page_properties = result["properties"]
-            # Sort by offset and concatenate in correct order
+                    homes.extend(page_properties)
-            results.sort(key=lambda x: x[0])
-            for offset, properties in results:
-                homes.extend(properties)
 
         # Apply client-side hour-based filtering if needed
         # (API only supports day-level filtering, so we post-filter for hour precision)
@@ -747,13 +808,14 @@ class RealtorScraper(Scraper):
         if not homes:
             return homes
 
-        from datetime import datetime, timedelta
+        from datetime import datetime, timedelta, timezone
 
         # Determine date range for last_update_date filtering
         date_range = None
 
         if self.updated_in_past_hours:
-            cutoff_datetime = datetime.now() - timedelta(hours=self.updated_in_past_hours)
+            # Use UTC now, strip timezone to match naive property dates
+            cutoff_datetime = (datetime.now(timezone.utc) - timedelta(hours=self.updated_in_past_hours)).replace(tzinfo=None)
             date_range = {'type': 'since', 'date': cutoff_datetime}
         elif self.updated_since:
             try:
@@ -784,15 +846,19 @@ class RealtorScraper(Scraper):
 
     def _get_date_range(self):
         """Get the date range for filtering based on instance parameters."""
-        from datetime import datetime, timedelta
+        from datetime import datetime, timedelta, timezone
-        
+
         if self.last_x_days:
-            cutoff_date = datetime.now() - timedelta(days=self.last_x_days)
+            # Use UTC now, strip timezone to match naive property dates
+            cutoff_date = (datetime.now(timezone.utc) - timedelta(days=self.last_x_days)).replace(tzinfo=None)
             return {'type': 'since', 'date': cutoff_date}
         elif self.date_from and self.date_to:
             try:
-                from_date = datetime.fromisoformat(self.date_from)
+                # Parse and strip timezone to match naive property dates
-                to_date = datetime.fromisoformat(self.date_to)
+                from_date_str = self.date_from.replace('Z', '+00:00') if self.date_from.endswith('Z') else self.date_from
+                to_date_str = self.date_to.replace('Z', '+00:00') if self.date_to.endswith('Z') else self.date_to
+                from_date = datetime.fromisoformat(from_date_str).replace(tzinfo=None)
+                to_date = datetime.fromisoformat(to_date_str).replace(tzinfo=None)
                 return {'type': 'range', 'from_date': from_date, 'to_date': to_date}
             except ValueError:
                 return None
@@ -844,6 +910,74 @@ class RealtorScraper(Scraper):
             return date_range['from_date'] <= date_obj <= date_range['to_date']
         return False
 
+    def _should_fetch_more_pages(self, first_page):
+        """Determine if we should continue pagination based on first page results.
+
+        This optimization prevents unnecessary API calls when using time-based filters
+        with date sorting. If the last property on page 1 is already outside the time
+        window, all future pages will also be outside (due to sort order).
+
+        Args:
+            first_page: List of properties from the first page
+
+        Returns:
+            bool: True if we should continue pagination, False to stop early
+        """
+        from datetime import datetime, timedelta, timezone
+
+        # Check for last_update_date filters
+        if (self.updated_since or self.updated_in_past_hours) and self.sort_by == "last_update_date":
+            if not first_page:
+                return False
+
+            last_property = first_page[-1]
+            last_date = self._extract_date_from_home(last_property, 'last_update_date')
+
+            if not last_date:
+                return True
+
+            # Build date range for last_update_date filter
+            if self.updated_since:
+                try:
+                    cutoff_datetime = datetime.fromisoformat(self.updated_since.replace('Z', '+00:00') if self.updated_since.endswith('Z') else self.updated_since)
+                    # Strip timezone to match naive datetimes from _parse_date_value
+                    cutoff_datetime = cutoff_datetime.replace(tzinfo=None)
+                    date_range = {'type': 'since', 'date': cutoff_datetime}
+                except ValueError:
+                    return True
+            elif self.updated_in_past_hours:
+                # Use UTC now, strip timezone to match naive property dates
+                cutoff_datetime = (datetime.now(timezone.utc) - timedelta(hours=self.updated_in_past_hours)).replace(tzinfo=None)
+                date_range = {'type': 'since', 'date': cutoff_datetime}
+            else:
+                return True
+
+            return self._is_datetime_in_range(last_date, date_range)
+
+        # Check for PENDING date filters
+        if (self.listing_type == ListingType.PENDING and
+            (self.last_x_days or self.past_hours or self.date_from) and
+            self.sort_by == "pending_date"):
+
+            if not first_page:
+                return False
+
+            last_property = first_page[-1]
+            last_date = self._extract_date_from_home(last_property, 'pending_date')
+
+            if not last_date:
+                return True
+
+            # Build date range for pending date filter
+            date_range = self._get_date_range()
+            if not date_range:
+                return True
+
+            return self._is_datetime_in_range(last_date, date_range)
+
+        # No optimization applicable, continue pagination
+        return True
+
     def _apply_sort(self, homes):
         """Apply client-side sorting to ensure results are properly ordered.
 
@@ -862,6 +996,8 @@ class RealtorScraper(Scraper):
 
         def get_sort_key(home):
             """Extract the sort field value from a home (handles both dict and Property object)."""
+            from datetime import datetime
+
             if isinstance(home, dict):
                 value = home.get(self.sort_by)
             else:
@@ -877,20 +1013,23 @@ class RealtorScraper(Scraper):
             if self.sort_by in ['list_date', 'sold_date', 'pending_date', 'last_update_date']:
                 if isinstance(value, str):
                     try:
-                        from datetime import datetime
                         # Handle timezone indicators
                         date_value = value
                         if date_value.endswith('Z'):
                             date_value = date_value[:-1] + '+00:00'
                         parsed_date = datetime.fromisoformat(date_value)
-                        return (0, parsed_date)
+                        # Normalize to timezone-naive for consistent comparison
+                        return 0, parsed_date.replace(tzinfo=None)
                     except (ValueError, AttributeError):
                         # If parsing fails, treat as None
                         return (1, 0) if self.sort_direction == "desc" else (1, float('inf'))
-                return (0, value)
+                # Handle datetime objects directly (normalize timezone)
+                if isinstance(value, datetime):
+                    return 0, value.replace(tzinfo=None)
+                return 0, value
 
             # For numeric fields, ensure we can compare
-            return (0, value)
+            return 0, value
 
         # Sort the homes
         reverse = (self.sort_direction == "desc")

homeharvest/utils.py

@@ -331,15 +331,26 @@ def validate_sort(sort_by: str | None, sort_direction: str | None = "desc") -> N
 
 def convert_to_datetime_string(value) -> str | None:
     """
-    Convert datetime object or string to ISO 8601 string format.
+    Convert datetime object or string to ISO 8601 string format with UTC timezone.
 
     Accepts:
-    - datetime.datetime objects
+    - datetime.datetime objects (naive or timezone-aware)
-    - datetime.date objects
+      - Naive datetimes are treated as local time and converted to UTC
+      - Timezone-aware datetimes are converted to UTC
+    - datetime.date objects (treated as midnight UTC)
     - ISO 8601 strings (returned as-is)
     - None (returns None)
 
-    Returns ISO 8601 formatted string or None.
+    Returns ISO 8601 formatted string with UTC timezone or None.
+
+    Examples:
+        >>> # Naive datetime (treated as local time)
+        >>> convert_to_datetime_string(datetime(2025, 1, 20, 14, 30))
+        '2025-01-20T22:30:00+00:00'  # Assuming PST (UTC-8)
+
+        >>> # Timezone-aware datetime
+        >>> convert_to_datetime_string(datetime(2025, 1, 20, 14, 30, tzinfo=timezone.utc))
+        '2025-01-20T14:30:00+00:00'
     """
     if value is None:
         return None
@@ -349,13 +360,23 @@ def convert_to_datetime_string(value) -> str | None:
         return value
 
     # datetime.datetime object
-    from datetime import datetime, date
+    from datetime import datetime, date, timezone
     if isinstance(value, datetime):
-        return value.isoformat()
+        # Handle naive datetime - treat as local time and convert to UTC
+        if value.tzinfo is None:
+            # Convert naive datetime to aware local time, then to UTC
+            local_aware = value.astimezone()
+            utc_aware = local_aware.astimezone(timezone.utc)
+            return utc_aware.isoformat()
+        else:
+            # Already timezone-aware, convert to UTC
+            utc_aware = value.astimezone(timezone.utc)
+            return utc_aware.isoformat()
 
-    # datetime.date object (convert to datetime at midnight)
+    # datetime.date object (convert to datetime at midnight UTC)
     if isinstance(value, date):
-        return datetime.combine(value, datetime.min.time()).isoformat()
+        utc_datetime = datetime.combine(value, datetime.min.time()).replace(tzinfo=timezone.utc)
+        return utc_datetime.isoformat()
 
     raise ValueError(
         f"Invalid datetime value. Expected datetime object, date object, or ISO 8601 string. "

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "homeharvest"
-version = "0.8.0"
+version = "0.8.6b"
 description = "Real estate scraping library"
 authors = ["Zachary Hampton <zachary@bunsly.com>", "Cullen Watson <cullen@bunsly.com>"]
 homepage = "https://github.com/ZacharyHampton/HomeHarvest"

tests/test_realtor.py

@@ -1,3 +1,5 @@
+import pytz
+
 from homeharvest import scrape_property, Property
 import pandas as pd
 
@@ -85,6 +87,25 @@ def test_realtor_date_range_sold():
     )
 
 
+def test_listing_type_none_includes_sold():
+    """Test that listing_type=None includes sold listings (issue #142)"""
+    # Get properties with listing_type=None (should include all common types)
+    result_none = scrape_property(
+        location="Warren, MI",
+        listing_type=None
+    )
+
+    # Verify we got results
+    assert result_none is not None and len(result_none) > 0
+
+    # Verify sold listings are included
+    status_types = set(result_none['status'].unique())
+    assert 'SOLD' in status_types, "SOLD listings should be included when listing_type=None"
+
+    # Verify we get multiple listing types (not just one)
+    assert len(status_types) > 1, "Should return multiple listing types when listing_type=None"
+
+
 def test_realtor_single_property():
     results = [
         scrape_property(
@@ -1357,4 +1378,238 @@ def test_combined_filters_with_raw_data():
         mls_id = source.get('id') if source else None
 
         assert mls_id is not None and mls_id != "", \
-            f"Property {prop.get('property_id')} should have an MLS ID (source.id)"
+            f"Property {prop.get('property_id')} should have an MLS ID (source.id)"
+
+
+def test_updated_since_filtering():
+    """Test the updated_since parameter for filtering by last_update_date"""
+    from datetime import datetime, timedelta
+
+    # Test 1: Filter by last update in past 10 minutes (user's example)
+    cutoff_time = datetime.now() - timedelta(minutes=10)
+    result_10min = scrape_property(
+        location="California",
+        updated_since=cutoff_time,
+        sort_by="last_update_date",
+        sort_direction="desc",
+        limit=100
+    )
+
+    assert result_10min is not None
+    print(f"\n10-minute window returned {len(result_10min)} properties")
+
+    # Test 2: Verify all results have last_update_date within range
+    if len(result_10min) > 0:
+        for idx in range(min(10, len(result_10min))):
+            update_date_str = result_10min.iloc[idx]["last_update_date"]
+            if pd.notna(update_date_str):
+                try:
+                    # Handle timezone-aware datetime strings
+                    date_str = str(update_date_str)
+                    if '+' in date_str or date_str.endswith('Z'):
+                        # Remove timezone for comparison with naive cutoff_time
+                        date_str = date_str.replace('+00:00', '').replace('Z', '')
+                    update_date = datetime.strptime(date_str, "%Y-%m-%d %H:%M:%S")
+
+                    assert update_date >= cutoff_time, \
+                        f"Property last_update_date {update_date} should be >= {cutoff_time}"
+                    print(f"Property {idx}: last_update_date = {update_date} (valid)")
+                except (ValueError, TypeError) as e:
+                    print(f"Warning: Could not parse date {update_date_str}: {e}")
+
+    # Test 3: Compare different time windows
+    result_1hour = scrape_property(
+        location="California",
+        updated_since=datetime.now() - timedelta(hours=1),
+        limit=50
+    )
+
+    result_24hours = scrape_property(
+        location="California",
+        updated_since=datetime.now() - timedelta(hours=24),
+        limit=50
+    )
+
+    print(f"1-hour window: {len(result_1hour)} properties")
+    print(f"24-hour window: {len(result_24hours)} properties")
+
+    # Longer time window should return same or more results
+    if len(result_1hour) > 0 and len(result_24hours) > 0:
+        assert len(result_1hour) <= len(result_24hours), \
+            "1-hour filter should return <= 24-hour results"
+
+    # Test 4: Verify sorting works with filtering
+    if len(result_10min) > 1:
+        # Get non-null dates
+        dates = []
+        for idx in range(len(result_10min)):
+            date_str = result_10min.iloc[idx]["last_update_date"]
+            if pd.notna(date_str):
+                try:
+                    # Handle timezone-aware datetime strings
+                    clean_date_str = str(date_str)
+                    if '+' in clean_date_str or clean_date_str.endswith('Z'):
+                        clean_date_str = clean_date_str.replace('+00:00', '').replace('Z', '')
+                    dates.append(datetime.strptime(clean_date_str, "%Y-%m-%d %H:%M:%S"))
+                except (ValueError, TypeError):
+                    pass
+
+        if len(dates) > 1:
+            # Check if sorted descending
+            for i in range(len(dates) - 1):
+                assert dates[i] >= dates[i + 1], \
+                    f"Results should be sorted by last_update_date descending: {dates[i]} >= {dates[i+1]}"
+
+
+def test_updated_since_optimization():
+    """Test that updated_since optimization works (auto-sort + early termination)"""
+    from datetime import datetime, timedelta
+    import time
+
+    # Test 1: Verify auto-sort is applied when using updated_since without explicit sort
+    start_time = time.time()
+    result = scrape_property(
+        location="California",
+        updated_since=datetime.now() - timedelta(minutes=5),
+        # NO sort_by specified - should auto-apply sort_by="last_update_date"
+        limit=50
+    )
+    elapsed_time = time.time() - start_time
+
+    print(f"\nAuto-sort test: {len(result)} properties in {elapsed_time:.2f}s")
+
+    # Should complete quickly due to early termination optimization (<5 seconds)
+    assert elapsed_time < 5.0, f"Query should be fast with optimization, took {elapsed_time:.2f}s"
+
+    # Verify results are sorted by last_update_date (proving auto-sort worked)
+    if len(result) > 1:
+        dates = []
+        for idx in range(min(10, len(result))):
+            date_str = result.iloc[idx]["last_update_date"]
+            if pd.notna(date_str):
+                try:
+                    clean_date_str = str(date_str)
+                    if '+' in clean_date_str or clean_date_str.endswith('Z'):
+                        clean_date_str = clean_date_str.replace('+00:00', '').replace('Z', '')
+                    dates.append(datetime.strptime(clean_date_str, "%Y-%m-%d %H:%M:%S"))
+                except (ValueError, TypeError):
+                    pass
+
+        if len(dates) > 1:
+            # Verify descending order (most recent first)
+            for i in range(len(dates) - 1):
+                assert dates[i] >= dates[i + 1], \
+                    "Auto-applied sort should order by last_update_date descending"
+
+    print("Auto-sort optimization verified ✓")
+
+
+def test_pending_date_optimization():
+    """Test that PENDING + date filters get auto-sort and early termination"""
+    from datetime import datetime, timedelta
+    import time
+
+    # Test: Verify auto-sort is applied for PENDING with past_days
+    start_time = time.time()
+    result = scrape_property(
+        location="California",
+        listing_type="pending",
+        past_days=7,
+        # NO sort_by specified - should auto-apply sort_by="pending_date"
+        limit=50
+    )
+    elapsed_time = time.time() - start_time
+
+    print(f"\nPENDING auto-sort test: {len(result)} properties in {elapsed_time:.2f}s")
+
+    # Should complete quickly due to optimization (<10 seconds)
+    assert elapsed_time < 10.0, f"PENDING query should be fast with optimization, took {elapsed_time:.2f}s"
+
+    # Verify results are sorted by pending_date (proving auto-sort worked)
+    if len(result) > 1:
+        dates = []
+        for idx in range(min(10, len(result))):
+            date_str = result.iloc[idx]["pending_date"]
+            if pd.notna(date_str):
+                try:
+                    clean_date_str = str(date_str)
+                    if '+' in clean_date_str or clean_date_str.endswith('Z'):
+                        clean_date_str = clean_date_str.replace('+00:00', '').replace('Z', '')
+                    dates.append(datetime.strptime(clean_date_str, "%Y-%m-%d %H:%M:%S"))
+                except (ValueError, TypeError):
+                    pass
+
+        if len(dates) > 1:
+            # Verify descending order (most recent first)
+            for i in range(len(dates) - 1):
+                assert dates[i] >= dates[i + 1], \
+                    "PENDING auto-applied sort should order by pending_date descending"
+
+    print("PENDING optimization verified ✓")
+
+
+def test_basic_last_update_date():
+    from datetime import datetime, timedelta
+
+    # Test with naive datetime (treated as local time)
+    now = datetime.now()
+
+    properties = scrape_property(
+        "California",
+        updated_since=now - timedelta(minutes=10),
+        sort_by="last_update_date",
+        sort_direction="desc"
+    )
+
+    # Convert now to timezone-aware for comparison with UTC dates in DataFrame
+    now_utc = now.astimezone(tz=pytz.timezone("UTC"))
+
+    # Check all last_update_date values are <= now
+    assert (properties["last_update_date"] <= now_utc).all()
+
+    # Verify we got some results
+    assert len(properties) > 0
+
+
+def test_timezone_aware_last_update_date():
+    """Test that timezone-aware datetimes work correctly for updated_since"""
+    from datetime import datetime, timedelta, timezone
+
+    # Test with timezone-aware datetime (explicit UTC)
+    now_utc = datetime.now(timezone.utc)
+
+    properties = scrape_property(
+        "California",
+        updated_since=now_utc - timedelta(minutes=10),
+        sort_by="last_update_date",
+        sort_direction="desc"
+    )
+
+    # Check all last_update_date values are <= now
+    assert (properties["last_update_date"] <= now_utc).all()
+
+    # Verify we got some results
+    assert len(properties) > 0
+
+
+def test_timezone_handling_date_range():
+    """Test timezone handling for date_from and date_to parameters"""
+    from datetime import datetime, timedelta
+
+    # Test with naive datetimes for date range (PENDING properties)
+    now = datetime.now()
+    three_days_ago = now - timedelta(days=3)
+
+    properties = scrape_property(
+        "California",
+        listing_type="pending",
+        date_from=three_days_ago,
+        date_to=now
+    )
+
+    # Verify we got results and they're within the date range
+    if len(properties) > 0:
+        # Convert now to UTC for comparison
+        now_utc = now.astimezone(tz=pytz.timezone("UTC"))
+        assert (properties["pending_date"] <= now_utc).all()
+