turtlebasket/HomeHarvest
1
0
Fork 0
mirror of https://github.com/Bunsly/HomeHarvest.git synced 2026-03-06 04:24:29 -08:00
Code Issues Packages Projects Releases Wiki Activity

Consolidate date_from/date_to parameters - remove datetime_from/datetime_to

Browse Source 
Simplified the time filtering interface by consolidating datetime_from/datetime_to
into date_from/date_to with automatic precision detection.

Changes:
- Remove datetime_from and datetime_to parameters (confusing to have both)
- Update date_from/date_to to accept multiple formats:
  - Date strings: "2025-01-20" (day precision)
  - Datetime strings: "2025-01-20T14:30:00" (hour precision)
  - date objects: date(2025, 1, 20) (day precision)
  - datetime objects: datetime(2025, 1, 20, 9, 0) (hour precision)
- Add detect_precision_and_convert() helper to automatically detect precision
- Add date_from_precision and date_to_precision fields to track precision level
- Update filtering logic to use precision fields instead of separate parameters
- Update README to remove datetime_from/datetime_to examples
- Update validation to accept ISO datetime strings

Benefits:
- Single, intuitive parameter name (date_from/date_to)
- Automatic precision detection based on input format
- Reduced API surface area and cognitive load
- More Pythonic - accept multiple input types

All changes are backward compatible for existing date_from/date_to string usage.

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
Zachary Hampton
2025-11-11 12:19:15 -08:00
parent 940b663011
commit c7a0d6d398
5 changed files with 108 additions and 53 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
homeharvest/core/scrapers/__init__.py
View File

@@ -21,6 +21,8 @@ class ScraperInput(BaseModel):
last_x_days: int | None = None
date_from: str | None = None
date_to: str | None = None
date_from_precision: str | None = None # "day" or "hour"
date_to_precision: str | None = None # "day" or "hour"
foreclosure: bool | None = False
extra_property_data: bool | None = True
exclude_pending: bool | None = False
@@ -30,8 +32,6 @@ class ScraperInput(BaseModel):
# New date/time filtering parameters
past_hours: int | None = None
datetime_from: str | None = None
datetime_to: str | None = None
# New last_update_date filtering parameters
updated_since: str | None = None
@@ -107,6 +107,8 @@ class Scraper:
self.mls_only = scraper_input.mls_only
self.date_from = scraper_input.date_from
self.date_to = scraper_input.date_to
self.date_from_precision = scraper_input.date_from_precision
self.date_to_precision = scraper_input.date_to_precision
self.foreclosure = scraper_input.foreclosure
self.extra_property_data = scraper_input.extra_property_data
self.exclude_pending = scraper_input.exclude_pending
@@ -116,8 +118,6 @@ class Scraper:
# New date/time filtering
self.past_hours = scraper_input.past_hours
self.datetime_from = scraper_input.datetime_from
self.datetime_to = scraper_input.datetime_to
# New last_update_date filtering
self.updated_since = scraper_input.updated_since

28
homeharvest/core/scrapers/realtor/__init__.py
View File

@@ -164,23 +164,26 @@ class RealtorScraper(Scraper):
# Build date parameter (expand to full days if hour-based filtering is used)
if date_field:
if self.datetime_from or self.datetime_to:
# Check if we have hour precision (need to extract date part for API, then filter client-side)
has_hour_precision = (self.date_from_precision == "hour" or self.date_to_precision == "hour")
if has_hour_precision and (self.date_from or self.date_to):
# Hour-based datetime filtering: extract date parts for API, client-side filter by hours
from datetime import datetime
min_date = None
max_date = None
if self.datetime_from:
if self.date_from:
try:
dt_from = datetime.fromisoformat(self.datetime_from.replace('Z', '+00:00'))
dt_from = datetime.fromisoformat(self.date_from.replace('Z', '+00:00'))
min_date = dt_from.strftime("%Y-%m-%d")
except (ValueError, AttributeError):
pass
if self.datetime_to:
if self.date_to:
try:
dt_to = datetime.fromisoformat(self.datetime_to.replace('Z', '+00:00'))
dt_to = datetime.fromisoformat(self.date_to.replace('Z', '+00:00'))
max_date = dt_to.strftime("%Y-%m-%d")
except (ValueError, AttributeError):
pass
@@ -551,7 +554,8 @@ class RealtorScraper(Scraper):
# Apply client-side hour-based filtering if needed
# (API only supports day-level filtering, so we post-filter for hour precision)
if self.past_hours or self.datetime_from or self.datetime_to:
has_hour_precision = (self.date_from_precision == "hour" or self.date_to_precision == "hour")
if self.past_hours or has_hour_precision:
homes = self._apply_hour_based_date_filter(homes)
# Apply client-side date filtering for PENDING properties
# (server-side filters are broken in the API)
@@ -577,7 +581,7 @@ class RealtorScraper(Scraper):
def _apply_hour_based_date_filter(self, homes):
"""Apply client-side hour-based date filtering for all listing types.
This is used when past_hours, datetime_from, or datetime_to are specified,
This is used when past_hours or date_from/date_to have hour precision,
since the API only supports day-level filtering.
"""
if not homes:
@@ -591,17 +595,17 @@ class RealtorScraper(Scraper):
if self.past_hours:
cutoff_datetime = datetime.now() - timedelta(hours=self.past_hours)
date_range = {'type': 'since', 'date': cutoff_datetime}
elif self.datetime_from or self.datetime_to:
elif self.date_from or self.date_to:
try:
from_datetime = None
to_datetime = None
if self.datetime_from:
from_datetime_str = self.datetime_from.replace('Z', '+00:00') if self.datetime_from.endswith('Z') else self.datetime_from
if self.date_from:
from_datetime_str = self.date_from.replace('Z', '+00:00') if self.date_from.endswith('Z') else self.date_from
from_datetime = datetime.fromisoformat(from_datetime_str).replace(tzinfo=None)
if self.datetime_to:
to_datetime_str = self.datetime_to.replace('Z', '+00:00') if self.datetime_to.endswith('Z') else self.datetime_to
if self.date_to:
to_datetime_str = self.date_to.replace('Z', '+00:00') if self.date_to.endswith('Z') else self.date_to
to_datetime = datetime.fromisoformat(to_datetime_str).replace(tzinfo=None)
if from_datetime and to_datetime: