mirror of
https://github.com/Bunsly/HomeHarvest.git
synced 2026-03-04 19:44:29 -08:00
Add pagination offset support for API queries
Implements offset parameter to enable pagination within the 10k API limit. Users can now fetch results in chunks (e.g., offset=200, limit=200 for results 200-399). Includes validation to ensure offset + limit doesn't exceed API maximum. Also fixes multi-page result sorting to preserve correct order across page boundaries. Fixes #139 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
@@ -1,5 +1,6 @@
|
||||
from __future__ import annotations
|
||||
import pandas as pd
|
||||
import warnings
|
||||
from datetime import datetime
|
||||
from .core.scrapers.models import Property, ListingType, Advertisers
|
||||
from .exceptions import InvalidListingType, InvalidDate
|
||||
@@ -182,6 +183,36 @@ def validate_limit(limit: int) -> None:
|
||||
raise ValueError("Property limit must be between 1 and 10,000.")
|
||||
|
||||
|
||||
def validate_offset(offset: int, limit: int = 10000) -> None:
|
||||
"""Validate offset parameter for pagination.
|
||||
|
||||
Args:
|
||||
offset: Starting position for results pagination
|
||||
limit: Maximum number of results to fetch
|
||||
|
||||
Raises:
|
||||
ValueError: If offset is invalid or if offset + limit exceeds API limit
|
||||
"""
|
||||
if offset is not None and offset < 0:
|
||||
raise ValueError("Offset must be non-negative (>= 0).")
|
||||
|
||||
# Check if offset + limit exceeds API's hard limit of 10,000
|
||||
if offset is not None and limit is not None and (offset + limit) > 10000:
|
||||
raise ValueError(
|
||||
f"offset ({offset}) + limit ({limit}) = {offset + limit} exceeds API maximum of 10,000. "
|
||||
f"The API cannot return results beyond position 10,000. "
|
||||
f"To fetch more results, narrow your search."
|
||||
)
|
||||
|
||||
# Warn if offset is not a multiple of 200 (API page size)
|
||||
if offset is not None and offset > 0 and offset % 200 != 0:
|
||||
warnings.warn(
|
||||
f"Offset should be a multiple of 200 (page size) for optimal performance. "
|
||||
f"Using offset {offset} may result in less efficient pagination.",
|
||||
UserWarning
|
||||
)
|
||||
|
||||
|
||||
def validate_datetime(datetime_str: str | None) -> None:
|
||||
"""Validate ISO 8601 datetime format."""
|
||||
if not datetime_str:
|
||||
|
||||
Reference in New Issue
Block a user