From 9b61a89c778c15d440827cf28dcffb9e56f23724 Mon Sep 17 00:00:00 2001 From: Zachary Hampton Date: Tue, 11 Nov 2025 17:40:21 -0800 Subject: [PATCH] Fix timezone handling for all date parameters MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Treat naive datetimes as local time and convert to UTC automatically - Support both naive and timezone-aware datetimes for updated_since, date_from, date_to - Fix timezone comparison bug that caused incorrect filtering with naive datetimes - Update documentation with clear timezone handling examples - Add comprehensive timezone tests for naive and aware datetimes - Bump version to 0.8.3 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- homeharvest/__init__.py | 8 ++- homeharvest/core/scrapers/realtor/__init__.py | 39 ++++++---- homeharvest/utils.py | 37 +++++++--- pyproject.toml | 2 +- tests/test_realtor.py | 71 ++++++++++++++++++- 5 files changed, 133 insertions(+), 24 deletions(-) diff --git a/homeharvest/__init__.py b/homeharvest/__init__.py index 155fef2..1436e10 100644 --- a/homeharvest/__init__.py +++ b/homeharvest/__init__.py @@ -72,6 +72,8 @@ def scrape_property( - date objects: date(2025, 1, 20) (day-level precision) - datetime objects: datetime(2025, 1, 20, 14, 30) (hour-level precision) The precision is automatically detected based on the input format. + Timezone handling: Naive datetimes are treated as local time and automatically converted to UTC. + Timezone-aware datetimes are converted to UTC. For best results, use timezone-aware datetimes. :param foreclosure: If set, fetches only foreclosure listings. :param extra_property_data: Increases requests by O(n). If set, this fetches additional property data (e.g. agent, broker, property evaluations etc.) :param exclude_pending: If true, this excludes pending or contingent properties from the results, unless listing type is pending. @@ -80,7 +82,11 @@ def scrape_property( New parameters: :param past_hours: Get properties in the last _ hours (requires client-side filtering). Accepts int or timedelta. - :param updated_since: Filter by last_update_date (when property was last updated). Accepts datetime object or ISO 8601 string (client-side filtering) + :param updated_since: Filter by last_update_date (when property was last updated). Accepts datetime object or ISO 8601 string (client-side filtering). + Timezone handling: Naive datetimes (like datetime.now()) are treated as local time and automatically converted to UTC. + Timezone-aware datetimes are converted to UTC. Examples: + - datetime.now() - uses your local timezone + - datetime.now(timezone.utc) - uses UTC explicitly :param updated_in_past_hours: Filter by properties updated in the last _ hours. Accepts int or timedelta (client-side filtering) :param beds_min, beds_max: Filter by number of bedrooms :param baths_min, baths_max: Filter by number of bathrooms diff --git a/homeharvest/core/scrapers/realtor/__init__.py b/homeharvest/core/scrapers/realtor/__init__.py index 8f06822..ea3a63b 100644 --- a/homeharvest/core/scrapers/realtor/__init__.py +++ b/homeharvest/core/scrapers/realtor/__init__.py @@ -755,13 +755,14 @@ class RealtorScraper(Scraper): if not homes: return homes - from datetime import datetime, timedelta + from datetime import datetime, timedelta, timezone # Determine date range for last_update_date filtering date_range = None if self.updated_in_past_hours: - cutoff_datetime = datetime.now() - timedelta(hours=self.updated_in_past_hours) + # Use UTC now, strip timezone to match naive property dates + cutoff_datetime = (datetime.now(timezone.utc) - timedelta(hours=self.updated_in_past_hours)).replace(tzinfo=None) date_range = {'type': 'since', 'date': cutoff_datetime} elif self.updated_since: try: @@ -792,15 +793,19 @@ class RealtorScraper(Scraper): def _get_date_range(self): """Get the date range for filtering based on instance parameters.""" - from datetime import datetime, timedelta - + from datetime import datetime, timedelta, timezone + if self.last_x_days: - cutoff_date = datetime.now() - timedelta(days=self.last_x_days) + # Use UTC now, strip timezone to match naive property dates + cutoff_date = (datetime.now(timezone.utc) - timedelta(days=self.last_x_days)).replace(tzinfo=None) return {'type': 'since', 'date': cutoff_date} elif self.date_from and self.date_to: try: - from_date = datetime.fromisoformat(self.date_from) - to_date = datetime.fromisoformat(self.date_to) + # Parse and strip timezone to match naive property dates + from_date_str = self.date_from.replace('Z', '+00:00') if self.date_from.endswith('Z') else self.date_from + to_date_str = self.date_to.replace('Z', '+00:00') if self.date_to.endswith('Z') else self.date_to + from_date = datetime.fromisoformat(from_date_str).replace(tzinfo=None) + to_date = datetime.fromisoformat(to_date_str).replace(tzinfo=None) return {'type': 'range', 'from_date': from_date, 'to_date': to_date} except ValueError: return None @@ -865,7 +870,7 @@ class RealtorScraper(Scraper): Returns: bool: True if we should continue pagination, False to stop early """ - from datetime import datetime, timedelta + from datetime import datetime, timedelta, timezone # Check for last_update_date filters if (self.updated_since or self.updated_in_past_hours) and self.sort_by == "last_update_date": @@ -882,11 +887,14 @@ class RealtorScraper(Scraper): if self.updated_since: try: cutoff_datetime = datetime.fromisoformat(self.updated_since.replace('Z', '+00:00') if self.updated_since.endswith('Z') else self.updated_since) + # Strip timezone to match naive datetimes from _parse_date_value + cutoff_datetime = cutoff_datetime.replace(tzinfo=None) date_range = {'type': 'since', 'date': cutoff_datetime} except ValueError: return True elif self.updated_in_past_hours: - cutoff_datetime = datetime.now() - timedelta(hours=self.updated_in_past_hours) + # Use UTC now, strip timezone to match naive property dates + cutoff_datetime = (datetime.now(timezone.utc) - timedelta(hours=self.updated_in_past_hours)).replace(tzinfo=None) date_range = {'type': 'since', 'date': cutoff_datetime} else: return True @@ -935,6 +943,8 @@ class RealtorScraper(Scraper): def get_sort_key(home): """Extract the sort field value from a home (handles both dict and Property object).""" + from datetime import datetime + if isinstance(home, dict): value = home.get(self.sort_by) else: @@ -950,20 +960,23 @@ class RealtorScraper(Scraper): if self.sort_by in ['list_date', 'sold_date', 'pending_date', 'last_update_date']: if isinstance(value, str): try: - from datetime import datetime # Handle timezone indicators date_value = value if date_value.endswith('Z'): date_value = date_value[:-1] + '+00:00' parsed_date = datetime.fromisoformat(date_value) - return (0, parsed_date) + # Normalize to timezone-naive for consistent comparison + return 0, parsed_date.replace(tzinfo=None) except (ValueError, AttributeError): # If parsing fails, treat as None return (1, 0) if self.sort_direction == "desc" else (1, float('inf')) - return (0, value) + # Handle datetime objects directly (normalize timezone) + if isinstance(value, datetime): + return 0, value.replace(tzinfo=None) + return 0, value # For numeric fields, ensure we can compare - return (0, value) + return 0, value # Sort the homes reverse = (self.sort_direction == "desc") diff --git a/homeharvest/utils.py b/homeharvest/utils.py index ee57475..53d9a35 100644 --- a/homeharvest/utils.py +++ b/homeharvest/utils.py @@ -331,15 +331,26 @@ def validate_sort(sort_by: str | None, sort_direction: str | None = "desc") -> N def convert_to_datetime_string(value) -> str | None: """ - Convert datetime object or string to ISO 8601 string format. + Convert datetime object or string to ISO 8601 string format with UTC timezone. Accepts: - - datetime.datetime objects - - datetime.date objects + - datetime.datetime objects (naive or timezone-aware) + - Naive datetimes are treated as local time and converted to UTC + - Timezone-aware datetimes are converted to UTC + - datetime.date objects (treated as midnight UTC) - ISO 8601 strings (returned as-is) - None (returns None) - Returns ISO 8601 formatted string or None. + Returns ISO 8601 formatted string with UTC timezone or None. + + Examples: + >>> # Naive datetime (treated as local time) + >>> convert_to_datetime_string(datetime(2025, 1, 20, 14, 30)) + '2025-01-20T22:30:00+00:00' # Assuming PST (UTC-8) + + >>> # Timezone-aware datetime + >>> convert_to_datetime_string(datetime(2025, 1, 20, 14, 30, tzinfo=timezone.utc)) + '2025-01-20T14:30:00+00:00' """ if value is None: return None @@ -349,13 +360,23 @@ def convert_to_datetime_string(value) -> str | None: return value # datetime.datetime object - from datetime import datetime, date + from datetime import datetime, date, timezone if isinstance(value, datetime): - return value.isoformat() + # Handle naive datetime - treat as local time and convert to UTC + if value.tzinfo is None: + # Convert naive datetime to aware local time, then to UTC + local_aware = value.astimezone() + utc_aware = local_aware.astimezone(timezone.utc) + return utc_aware.isoformat() + else: + # Already timezone-aware, convert to UTC + utc_aware = value.astimezone(timezone.utc) + return utc_aware.isoformat() - # datetime.date object (convert to datetime at midnight) + # datetime.date object (convert to datetime at midnight UTC) if isinstance(value, date): - return datetime.combine(value, datetime.min.time()).isoformat() + utc_datetime = datetime.combine(value, datetime.min.time()).replace(tzinfo=timezone.utc) + return utc_datetime.isoformat() raise ValueError( f"Invalid datetime value. Expected datetime object, date object, or ISO 8601 string. " diff --git a/pyproject.toml b/pyproject.toml index 671f8f9..09e160f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "homeharvest" -version = "0.8.2" +version = "0.8.3" description = "Real estate scraping library" authors = ["Zachary Hampton ", "Cullen Watson "] homepage = "https://github.com/ZacharyHampton/HomeHarvest" diff --git a/tests/test_realtor.py b/tests/test_realtor.py index 7e19617..cdfa8d5 100644 --- a/tests/test_realtor.py +++ b/tests/test_realtor.py @@ -1,3 +1,5 @@ +import pytz + from homeharvest import scrape_property, Property import pandas as pd @@ -1524,4 +1526,71 @@ def test_pending_date_optimization(): assert dates[i] >= dates[i + 1], \ "PENDING auto-applied sort should order by pending_date descending" - print("PENDING optimization verified ✓") \ No newline at end of file + print("PENDING optimization verified ✓") + + +def test_basic_last_update_date(): + from datetime import datetime, timedelta + + # Test with naive datetime (treated as local time) + now = datetime.now() + + properties = scrape_property( + "California", + updated_since=now - timedelta(minutes=10), + sort_by="last_update_date", + sort_direction="desc" + ) + + # Convert now to timezone-aware for comparison with UTC dates in DataFrame + now_utc = now.astimezone(tz=pytz.timezone("UTC")) + + # Check all last_update_date values are <= now + assert (properties["last_update_date"] <= now_utc).all() + + # Verify we got some results + assert len(properties) > 0 + + +def test_timezone_aware_last_update_date(): + """Test that timezone-aware datetimes work correctly for updated_since""" + from datetime import datetime, timedelta, timezone + + # Test with timezone-aware datetime (explicit UTC) + now_utc = datetime.now(timezone.utc) + + properties = scrape_property( + "California", + updated_since=now_utc - timedelta(minutes=10), + sort_by="last_update_date", + sort_direction="desc" + ) + + # Check all last_update_date values are <= now + assert (properties["last_update_date"] <= now_utc).all() + + # Verify we got some results + assert len(properties) > 0 + + +def test_timezone_handling_date_range(): + """Test timezone handling for date_from and date_to parameters""" + from datetime import datetime, timedelta + + # Test with naive datetimes for date range (PENDING properties) + now = datetime.now() + three_days_ago = now - timedelta(days=3) + + properties = scrape_property( + "California", + listing_type="pending", + date_from=three_days_ago, + date_to=now + ) + + # Verify we got results and they're within the date range + if len(properties) > 0: + # Convert now to UTC for comparison + now_utc = now.astimezone(tz=pytz.timezone("UTC")) + assert (properties["pending_date"] <= now_utc).all() +