mirror of
https://github.com/Bunsly/HomeHarvest.git
synced 2026-03-05 03:54:29 -08:00
Compare commits
3 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
0d70007000 | ||
|
|
018d3fbac4 | ||
|
|
803fd618e9 |
@@ -21,7 +21,7 @@
|
||||
```bash
|
||||
pip install -U homeharvest
|
||||
```
|
||||
_Python version >= [3.10](https://www.python.org/downloads/release/python-3100/) required_
|
||||
_Python version >= [3.9](https://www.python.org/downloads/release/python-3100/) required_
|
||||
|
||||
## Usage
|
||||
|
||||
|
||||
@@ -53,7 +53,9 @@ def scrape_property(
|
||||
if not properties_dfs:
|
||||
return pd.DataFrame()
|
||||
|
||||
properties_dfs = [df for df in properties_dfs if not df.empty]
|
||||
|
||||
with warnings.catch_warnings():
|
||||
warnings.simplefilter("ignore", category=FutureWarning)
|
||||
|
||||
return pd.concat(properties_dfs, ignore_index=True, axis=0)[ordered_properties].replace({"None": "", None: ""})
|
||||
return pd.concat(properties_dfs, ignore_index=True, axis=0)[ordered_properties].replace({"None": pd.NA, None: pd.NA, "": pd.NA})
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
from __future__ import annotations
|
||||
from dataclasses import dataclass
|
||||
import requests
|
||||
from requests.adapters import HTTPAdapter
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
from __future__ import annotations
|
||||
from dataclasses import dataclass
|
||||
from enum import Enum
|
||||
from typing import Optional
|
||||
@@ -36,6 +37,7 @@ class PropertyType(Enum):
|
||||
CONDO_TOWNHOME = "CONDO_TOWNHOME"
|
||||
CONDO_TOWNHOME_ROWHOME_COOP = "CONDO_TOWNHOME_ROWHOME_COOP"
|
||||
CONDO = "CONDO"
|
||||
CONDOP = "CONDOP"
|
||||
CONDOS = "CONDOS"
|
||||
COOP = "COOP"
|
||||
DUPLEX_TRIPLEX = "DUPLEX_TRIPLEX"
|
||||
|
||||
@@ -4,7 +4,7 @@ homeharvest.realtor.__init__
|
||||
|
||||
This module implements the scraper for realtor.com
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from datetime import datetime
|
||||
from typing import Dict, Union, Optional
|
||||
@@ -166,7 +166,7 @@ class RealtorScraper(Scraper):
|
||||
longitude=property_info["address"]["location"]["coordinate"].get("lon") if able_to_get_lat_long else None,
|
||||
address=self._parse_address(property_info, search_type="handle_listing"),
|
||||
description=Description(
|
||||
alt_photos=self.process_alt_photos(property_info.get("media", {}).get("photos", [])),
|
||||
alt_photos=self.process_alt_photos(property_info["media"].get("photos", [])) if property_info.get("media") else None,
|
||||
style=property_info["basic"].get("type", "").upper(),
|
||||
beds=property_info["basic"].get("beds"),
|
||||
baths_full=property_info["basic"].get("baths_full"),
|
||||
@@ -805,8 +805,8 @@ class RealtorScraper(Scraper):
|
||||
|
||||
return Description(
|
||||
primary_photo=primary_photo,
|
||||
alt_photos=RealtorScraper.process_alt_photos(result.get("photos")),
|
||||
style=PropertyType(style) if style else None,
|
||||
alt_photos=RealtorScraper.process_alt_photos(result.get("photos", [])),
|
||||
style=PropertyType.__getitem__(style) if style and style in PropertyType.__members__ else None,
|
||||
beds=description_data.get("beds"),
|
||||
baths_full=description_data.get("baths_full"),
|
||||
baths_half=description_data.get("baths_half"),
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
from __future__ import annotations
|
||||
import pandas as pd
|
||||
from datetime import datetime
|
||||
from .core.scrapers.models import Property, ListingType, Agent
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
[tool.poetry]
|
||||
name = "homeharvest"
|
||||
version = "0.3.23"
|
||||
version = "0.3.26"
|
||||
description = "Real estate scraping library"
|
||||
authors = ["Zachary Hampton <zachary@bunsly.com>", "Cullen Watson <cullen@bunsly.com>"]
|
||||
homepage = "https://github.com/Bunsly/HomeHarvest"
|
||||
@@ -10,7 +10,7 @@ readme = "README.md"
|
||||
homeharvest = "homeharvest.cli:main"
|
||||
|
||||
[tool.poetry.dependencies]
|
||||
python = ">=3.10,<3.13"
|
||||
python = ">=3.9,<3.13"
|
||||
requests = "^2.31.0"
|
||||
pandas = "^2.1.1"
|
||||
|
||||
|
||||
@@ -155,7 +155,7 @@ def test_realtor_without_extra_details():
|
||||
),
|
||||
]
|
||||
|
||||
assert results[0] != results[1]
|
||||
assert not results[0].equals(results[1])
|
||||
|
||||
|
||||
def test_pr_zip_code():
|
||||
|
||||
Reference in New Issue
Block a user