mirror of
https://github.com/Bunsly/HomeHarvest.git
synced 2026-03-04 19:44:29 -08:00
Add configurable parallel/sequential pagination with parallel parameter
- Add `parallel: bool = True` parameter to control pagination strategy - Parallel mode (default): Fetches all pages in parallel for maximum speed - Sequential mode: Fetches pages one-by-one with early termination checks - Early termination stops pagination when time-based filters indicate no more matches - Useful for rate limiting and narrow time windows - Simplified pagination logic by removing hybrid first-page pre-check - Updated README with usage example and parameter documentation - Version bump to 0.8.4 - All 54 tests passing 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
@@ -55,6 +55,9 @@ class ScraperInput(BaseModel):
|
||||
sort_by: str | None = None
|
||||
sort_direction: str = "desc"
|
||||
|
||||
# Pagination control
|
||||
parallel: bool = True
|
||||
|
||||
|
||||
class Scraper:
|
||||
session = None
|
||||
@@ -141,6 +144,9 @@ class Scraper:
|
||||
self.sort_by = scraper_input.sort_by
|
||||
self.sort_direction = scraper_input.sort_direction
|
||||
|
||||
# Pagination control
|
||||
self.parallel = scraper_input.parallel
|
||||
|
||||
def search(self) -> list[Union[Property | dict]]: ...
|
||||
|
||||
@staticmethod
|
||||
|
||||
Reference in New Issue
Block a user