mirror of
https://github.com/Bunsly/HomeHarvest.git
synced 2026-03-05 12:04:31 -08:00
Compare commits
2 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
018d3fbac4 | ||
|
|
803fd618e9 |
@@ -21,7 +21,7 @@
|
|||||||
```bash
|
```bash
|
||||||
pip install -U homeharvest
|
pip install -U homeharvest
|
||||||
```
|
```
|
||||||
_Python version >= [3.10](https://www.python.org/downloads/release/python-3100/) required_
|
_Python version >= [3.9](https://www.python.org/downloads/release/python-3100/) required_
|
||||||
|
|
||||||
## Usage
|
## Usage
|
||||||
|
|
||||||
|
|||||||
@@ -53,7 +53,9 @@ def scrape_property(
|
|||||||
if not properties_dfs:
|
if not properties_dfs:
|
||||||
return pd.DataFrame()
|
return pd.DataFrame()
|
||||||
|
|
||||||
|
properties_dfs = [df for df in properties_dfs if not df.empty]
|
||||||
|
|
||||||
with warnings.catch_warnings():
|
with warnings.catch_warnings():
|
||||||
warnings.simplefilter("ignore", category=FutureWarning)
|
warnings.simplefilter("ignore", category=FutureWarning)
|
||||||
|
|
||||||
return pd.concat(properties_dfs, ignore_index=True, axis=0)[ordered_properties].replace({"None": "", None: ""})
|
return pd.concat(properties_dfs, ignore_index=True, axis=0)[ordered_properties].replace({"None": pd.NA, None: pd.NA, "": pd.NA})
|
||||||
|
|||||||
@@ -1,3 +1,4 @@
|
|||||||
|
from __future__ import annotations
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
import requests
|
import requests
|
||||||
from requests.adapters import HTTPAdapter
|
from requests.adapters import HTTPAdapter
|
||||||
|
|||||||
@@ -1,3 +1,4 @@
|
|||||||
|
from __future__ import annotations
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
from enum import Enum
|
from enum import Enum
|
||||||
from typing import Optional
|
from typing import Optional
|
||||||
@@ -36,6 +37,7 @@ class PropertyType(Enum):
|
|||||||
CONDO_TOWNHOME = "CONDO_TOWNHOME"
|
CONDO_TOWNHOME = "CONDO_TOWNHOME"
|
||||||
CONDO_TOWNHOME_ROWHOME_COOP = "CONDO_TOWNHOME_ROWHOME_COOP"
|
CONDO_TOWNHOME_ROWHOME_COOP = "CONDO_TOWNHOME_ROWHOME_COOP"
|
||||||
CONDO = "CONDO"
|
CONDO = "CONDO"
|
||||||
|
CONDOP = "CONDOP"
|
||||||
CONDOS = "CONDOS"
|
CONDOS = "CONDOS"
|
||||||
COOP = "COOP"
|
COOP = "COOP"
|
||||||
DUPLEX_TRIPLEX = "DUPLEX_TRIPLEX"
|
DUPLEX_TRIPLEX = "DUPLEX_TRIPLEX"
|
||||||
|
|||||||
@@ -4,7 +4,7 @@ homeharvest.realtor.__init__
|
|||||||
|
|
||||||
This module implements the scraper for realtor.com
|
This module implements the scraper for realtor.com
|
||||||
"""
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
from typing import Dict, Union, Optional
|
from typing import Dict, Union, Optional
|
||||||
@@ -806,7 +806,7 @@ class RealtorScraper(Scraper):
|
|||||||
return Description(
|
return Description(
|
||||||
primary_photo=primary_photo,
|
primary_photo=primary_photo,
|
||||||
alt_photos=RealtorScraper.process_alt_photos(result.get("photos")),
|
alt_photos=RealtorScraper.process_alt_photos(result.get("photos")),
|
||||||
style=PropertyType(style) if style else None,
|
style=PropertyType.__getitem__(style) if style and style in PropertyType.__members__ else None,
|
||||||
beds=description_data.get("beds"),
|
beds=description_data.get("beds"),
|
||||||
baths_full=description_data.get("baths_full"),
|
baths_full=description_data.get("baths_full"),
|
||||||
baths_half=description_data.get("baths_half"),
|
baths_half=description_data.get("baths_half"),
|
||||||
|
|||||||
@@ -1,3 +1,4 @@
|
|||||||
|
from __future__ import annotations
|
||||||
import pandas as pd
|
import pandas as pd
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
from .core.scrapers.models import Property, ListingType, Agent
|
from .core.scrapers.models import Property, ListingType, Agent
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
[tool.poetry]
|
[tool.poetry]
|
||||||
name = "homeharvest"
|
name = "homeharvest"
|
||||||
version = "0.3.23"
|
version = "0.3.25"
|
||||||
description = "Real estate scraping library"
|
description = "Real estate scraping library"
|
||||||
authors = ["Zachary Hampton <zachary@bunsly.com>", "Cullen Watson <cullen@bunsly.com>"]
|
authors = ["Zachary Hampton <zachary@bunsly.com>", "Cullen Watson <cullen@bunsly.com>"]
|
||||||
homepage = "https://github.com/Bunsly/HomeHarvest"
|
homepage = "https://github.com/Bunsly/HomeHarvest"
|
||||||
@@ -10,7 +10,7 @@ readme = "README.md"
|
|||||||
homeharvest = "homeharvest.cli:main"
|
homeharvest = "homeharvest.cli:main"
|
||||||
|
|
||||||
[tool.poetry.dependencies]
|
[tool.poetry.dependencies]
|
||||||
python = ">=3.10,<3.13"
|
python = ">=3.9,<3.13"
|
||||||
requests = "^2.31.0"
|
requests = "^2.31.0"
|
||||||
pandas = "^2.1.1"
|
pandas = "^2.1.1"
|
||||||
|
|
||||||
|
|||||||
@@ -155,7 +155,7 @@ def test_realtor_without_extra_details():
|
|||||||
),
|
),
|
||||||
]
|
]
|
||||||
|
|
||||||
assert results[0] != results[1]
|
assert not results[0].equals(results[1])
|
||||||
|
|
||||||
|
|
||||||
def test_pr_zip_code():
|
def test_pr_zip_code():
|
||||||
|
|||||||
Reference in New Issue
Block a user