mirror of
https://github.com/Bunsly/HomeHarvest.git
synced 2026-03-04 19:44:29 -08:00
Add configurable parallel/sequential pagination with parallel parameter
- Add `parallel: bool = True` parameter to control pagination strategy - Parallel mode (default): Fetches all pages in parallel for maximum speed - Sequential mode: Fetches pages one-by-one with early termination checks - Early termination stops pagination when time-based filters indicate no more matches - Useful for rate limiting and narrow time windows - Simplified pagination logic by removing hybrid first-page pre-check - Updated README with usage example and parameter documentation - Version bump to 0.8.4 - All 54 tests passing 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
@@ -48,6 +48,8 @@ def scrape_property(
|
||||
# New sorting parameters
|
||||
sort_by: str = None,
|
||||
sort_direction: str = "desc",
|
||||
# Pagination control
|
||||
parallel: bool = True,
|
||||
) -> Union[pd.DataFrame, list[dict], list[Property]]:
|
||||
"""
|
||||
Scrape properties from Realtor.com based on a given location and listing type.
|
||||
@@ -96,6 +98,9 @@ def scrape_property(
|
||||
:param year_built_min, year_built_max: Filter by year built
|
||||
:param sort_by: Sort results by field (list_date, sold_date, list_price, sqft, beds, baths, last_update_date)
|
||||
:param sort_direction: Sort direction (asc, desc)
|
||||
:param parallel: Controls pagination strategy. True (default) = fetch all pages in parallel for maximum speed.
|
||||
False = fetch pages sequentially with early termination checks (useful for rate limiting or narrow time windows).
|
||||
Sequential mode will stop paginating as soon as time-based filters indicate no more matches are possible.
|
||||
|
||||
Note: past_days and past_hours also accept timedelta objects for more Pythonic usage.
|
||||
"""
|
||||
@@ -190,6 +195,8 @@ def scrape_property(
|
||||
# New sorting
|
||||
sort_by=sort_by,
|
||||
sort_direction=sort_direction,
|
||||
# Pagination control
|
||||
parallel=parallel,
|
||||
)
|
||||
|
||||
site = RealtorScraper(scraper_input)
|
||||
|
||||
Reference in New Issue
Block a user