Refactor scraper to use direct requests and bump to 0.8.18

- Replace session-based approach with direct requests calls
- Move headers to module-level DEFAULT_HEADERS constant
- Temporarily disable extra_property_data feature

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
zacharyhampton
2025-12-26 00:29:53 -07:00
parent 129ab37dff
commit 8a6ac96db4
3 changed files with 32 additions and 45 deletions

View File

@@ -8,9 +8,8 @@ This module implements the scraper for realtor.com
from __future__ import annotations
import json
import re
import requests
from concurrent.futures import ThreadPoolExecutor, as_completed
from datetime import datetime
from json import JSONDecodeError
from typing import Dict, Union
@@ -22,7 +21,7 @@ from tenacity import (
stop_after_attempt,
)
from .. import Scraper
from .. import Scraper, DEFAULT_HEADERS
from ....exceptions import AuthenticationError
from ..models import (
Property,
@@ -69,7 +68,12 @@ class RealtorScraper(Scraper):
"variables": variables,
}
response = self.session.post(self.SEARCH_GQL_URL, data=json.dumps(payload, separators=(',', ':')))
response = requests.post(
self.SEARCH_GQL_URL,
headers=DEFAULT_HEADERS,
data=json.dumps(payload, separators=(',', ':')),
proxies=self.proxies
)
if response.status_code == 403:
if not self.proxy: