mirror of
https://github.com/Bunsly/HomeHarvest.git
synced 2026-03-05 20:14:30 -08:00
Refactor scraper to use direct requests and bump to 0.8.18
- Replace session-based approach with direct requests calls - Move headers to module-level DEFAULT_HEADERS constant - Temporarily disable extra_property_data feature 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
@@ -8,9 +8,8 @@ This module implements the scraper for realtor.com
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
import requests
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from datetime import datetime
|
||||
from json import JSONDecodeError
|
||||
from typing import Dict, Union
|
||||
|
||||
@@ -22,7 +21,7 @@ from tenacity import (
|
||||
stop_after_attempt,
|
||||
)
|
||||
|
||||
from .. import Scraper
|
||||
from .. import Scraper, DEFAULT_HEADERS
|
||||
from ....exceptions import AuthenticationError
|
||||
from ..models import (
|
||||
Property,
|
||||
@@ -69,7 +68,12 @@ class RealtorScraper(Scraper):
|
||||
"variables": variables,
|
||||
}
|
||||
|
||||
response = self.session.post(self.SEARCH_GQL_URL, data=json.dumps(payload, separators=(',', ':')))
|
||||
response = requests.post(
|
||||
self.SEARCH_GQL_URL,
|
||||
headers=DEFAULT_HEADERS,
|
||||
data=json.dumps(payload, separators=(',', ':')),
|
||||
proxies=self.proxies
|
||||
)
|
||||
|
||||
if response.status_code == 403:
|
||||
if not self.proxy:
|
||||
|
||||
Reference in New Issue
Block a user