Merge pull request #12 from ZacharyHampton/proxy_bug

fix: proxy add to session correctly

README.md

@@ -22,6 +22,23 @@ pip install --force-reinstall homeharvest
   _Python version >= [3.10](https://www.python.org/downloads/release/python-3100/) required_ 
 
 ## Usage
+
+### CLI 
+
+```bash
+homeharvest "San Francisco, CA" -s zillow realtor.com redfin -l for_rent -o excel -f HomeHarvest
+```
+
+This will scrape properties from the specified sites for the given location and listing type, and save the results to an Excel file named `HomeHarvest.xlsx`.
+
+By default:
+- If `-s` or `--site_name` is not provided, it will scrape from all available sites.
+- If `-l` or `--listing_type` is left blank, the default is `for_sale`. Other options are `for_rent` or `sold`.
+- The `-o` or `--output` default format is `excel`. Options are `csv` or `excel`.
+- If `-f` or `--filename` is left blank, the default is `HomeHarvest_<current_timestamp>`.
+- If `-p` or `--proxy` is not provided, the scraper uses the local IP.
+### Python 
+
 ```py
 from homeharvest import scrape_property
 import pandas as pd
@@ -35,16 +52,17 @@ properties: pd.DataFrame = scrape_property(
 #: Note, to export to CSV or Excel, use properties.to_csv() or properties.to_excel().
 print(properties)
 ```
+
 ## Output
 ```py
 >>> properties.head()
-                           street   city  ... mls_id description
+                                        property_url site_name listing_type  apt_min_price  apt_max_price   ...  
-0                 420 N  Scottsdale Rd  Tempe  ...    NaN         NaN
+0  https://www.redfin.com/AZ/Tempe/1003-W-Washing...    redfin     for_rent         1666.0         2750.0   ... 
-1                1255 E  University Dr  Tempe  ...    NaN         NaN
+1  https://www.redfin.com/AZ/Tempe/VELA-at-Town-L...    redfin     for_rent         1665.0         3763.0   ...  
-2              1979 E  Rio Salado Pkwy  Tempe  ...    NaN         NaN
+2  https://www.redfin.com/AZ/Tempe/Camden-Tempe/a...    redfin     for_rent         1939.0         3109.0   ...  
-3                      548 S Wilson St  Tempe  ...   None        None
+3  https://www.redfin.com/AZ/Tempe/Emerson-Park/a...    redfin     for_rent         1185.0         1817.0   ... 
-4  945 E  Playa Del Norte Dr Unit 4027  Tempe  ...    NaN         NaN
+4  https://www.redfin.com/AZ/Tempe/Rio-Paradiso-A...    redfin     for_rent         1470.0         2235.0   ...   
-[5 rows x 23 columns]
+[5 rows x 41 columns]
 ```
 
 ### Parameters for `scrape_properties()`
@@ -54,6 +72,7 @@ Required
 └── listing_type (enum): for_rent, for_sale, sold
 Optional
 ├── site_name (List[enum], default=all three sites): zillow, realtor.com, redfin
+├── proxy (str): in format 'http://user:pass@host:port' or [https, socks]
 ```
 
 ### Property Schema
@@ -104,7 +123,14 @@ Property
 │   └── bldg_min_area (int)
 
 └── Apartment Details (for property type: apartment):
-    └── apt_min_price (int)
+    ├── apt_min_beds: int
+    ├── apt_max_beds: int
+    ├── apt_min_baths: float
+    ├── apt_max_baths: float
+    ├── apt_min_price: int
+    ├── apt_max_price: int
+    ├── apt_min_sqft: int
+    ├── apt_max_sqft: int
 ```
 ## Supported Countries for Property Scraping
 

homeharvest/__init__.py

@@ -18,7 +18,7 @@ _scrapers = {
 }
 
 
-def validate_input(site_name: str, listing_type: str) -> None:
+def _validate_input(site_name: str, listing_type: str) -> None:
     if site_name.lower() not in _scrapers:
         raise InvalidSite(f"Provided site, '{site_name}', does not exist.")
 
@@ -28,7 +28,7 @@ def validate_input(site_name: str, listing_type: str) -> None:
         )
 
 
-def get_ordered_properties(result: Property) -> list[str]:
+def _get_ordered_properties(result: Property) -> list[str]:
     return [
         "property_url",
         "site_name",
@@ -68,14 +68,14 @@ def get_ordered_properties(result: Property) -> list[str]:
         "year_built",
         "agent_name",
         "mls_id",
-        "description",
         "img_src",
         "latitude",
         "longitude",
+        "description",
     ]
 
 
-def process_result(result: Property) -> pd.DataFrame:
+def _process_result(result: Property) -> pd.DataFrame:
     prop_data = result.__dict__
 
     prop_data["site_name"] = prop_data["site_name"].value
@@ -96,29 +96,30 @@ def process_result(result: Property) -> pd.DataFrame:
         del prop_data["address"]
 
     properties_df = pd.DataFrame([prop_data])
-    properties_df = properties_df[get_ordered_properties(result)]
+    properties_df = properties_df[_get_ordered_properties(result)]
 
     return properties_df
 
 
 def _scrape_single_site(
-    location: str, site_name: str, listing_type: str
+    location: str, site_name: str, listing_type: str, proxy: str = None
 ) -> pd.DataFrame:
     """
     Helper function to scrape a single site.
     """
-    validate_input(site_name, listing_type)
+    _validate_input(site_name, listing_type)
 
     scraper_input = ScraperInput(
         location=location,
         listing_type=ListingType[listing_type.upper()],
         site_name=SiteName.get_by_value(site_name.lower()),
+        proxy=proxy,
     )
 
     site = _scrapers[site_name.lower()](scraper_input)
     results = site.search()
 
-    properties_dfs = [process_result(result) for result in results]
+    properties_dfs = [_process_result(result) for result in results]
     properties_dfs = [
         df.dropna(axis=1, how="all") for df in properties_dfs if not df.empty
     ]
@@ -132,6 +133,7 @@ def scrape_property(
     location: str,
     site_name: Union[str, list[str]] = None,
     listing_type: str = "for_sale",
+    proxy: str = None,
 ) -> pd.DataFrame:
     """
     Scrape property from various sites from a given location and listing type.
@@ -151,13 +153,13 @@ def scrape_property(
     results = []
 
     if len(site_name) == 1:
-        final_df = _scrape_single_site(location, site_name[0], listing_type)
+        final_df = _scrape_single_site(location, site_name[0], listing_type, proxy)
         results.append(final_df)
     else:
         with ThreadPoolExecutor() as executor:
             futures = {
                 executor.submit(
-                    _scrape_single_site, location, s_name, listing_type
+                    _scrape_single_site, location, s_name, listing_type, proxy
                 ): s_name
                 for s_name in site_name
             }

homeharvest/cli.py

@@ -0,0 +1,72 @@
+import argparse
+import datetime
+from homeharvest import scrape_property
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Home Harvest Property Scraper")
+    parser.add_argument(
+        "location", type=str, help="Location to scrape (e.g., San Francisco, CA)"
+    )
+
+    parser.add_argument(
+        "-s",
+        "--site_name",
+        type=str,
+        nargs="*",
+        default=None,
+        help="Site name(s) to scrape from (e.g., realtor, zillow)",
+    )
+
+    parser.add_argument(
+        "-l",
+        "--listing_type",
+        type=str,
+        default="for_sale",
+        choices=["for_sale", "for_rent", "sold"],
+        help="Listing type to scrape",
+    )
+
+    parser.add_argument(
+        "-o",
+        "--output",
+        type=str,
+        default="excel",
+        choices=["excel", "csv"],
+        help="Output format",
+    )
+
+    parser.add_argument(
+        "-f",
+        "--filename",
+        type=str,
+        default=None,
+        help="Name of the output file (without extension)",
+    )
+
+    parser.add_argument(
+        "-p", "--proxy", type=str, default=None, help="Proxy to use for scraping"
+    )
+
+    args = parser.parse_args()
+
+    result = scrape_property(
+        args.location, args.site_name, args.listing_type, proxy=args.proxy
+    )
+
+    if not args.filename:
+        timestamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S")
+        args.filename = f"HomeHarvest_{timestamp}"
+
+    if args.output == "excel":
+        output_filename = f"{args.filename}.xlsx"
+        result.to_excel(output_filename, index=False)
+        print(f"Excel file saved as {output_filename}")
+    elif args.output == "csv":
+        output_filename = f"{args.filename}.csv"
+        result.to_csv(output_filename, index=False)
+        print(f"CSV file saved as {output_filename}")
+
+
+if __name__ == "__main__":
+    main()

homeharvest/core/scrapers/__init__.py

@@ -8,7 +8,7 @@ class ScraperInput:
     location: str
     listing_type: ListingType
     site_name: SiteName
-    proxy_url: str | None = None
+    proxy: str | None = None
 
 
 class Scraper:
@@ -17,15 +17,16 @@ class Scraper:
         self.listing_type = scraper_input.listing_type
 
         self.session = requests.Session()
+        if scraper_input.proxy:
+            proxy_url = scraper_input.proxy
+            proxies = {
+                "http": proxy_url,
+                "https": proxy_url
+            }
+            self.session.proxies.update(proxies)
         self.listing_type = scraper_input.listing_type
         self.site_name = scraper_input.site_name
 
-        if scraper_input.proxy_url:
-            self.session.proxies = {
-                "http": scraper_input.proxy_url,
-                "https": scraper_input.proxy_url,
-            }
-
     def search(self) -> list[Property]:
         ...
 

homeharvest/core/scrapers/zillow/__init__.py

@@ -1,6 +1,5 @@
 import re
 import json
-import string
 from .. import Scraper
 from ....utils import parse_address_two, parse_unit
 from ....exceptions import GeoCoordsNotFound, NoResultsFound
@@ -32,7 +31,9 @@ class ZillowScraper(Scraper):
         return response.json()["results"] != []
 
     def search(self):
-        resp = self.session.get(self.url, headers=self._get_headers())
+        resp = self.session.get(
+            self.url, headers=self._get_headers()
+        )
         resp.raise_for_status()
         content = resp.text
 
@@ -129,7 +130,9 @@ class ZillowScraper(Scraper):
             "wants": {"cat1": ["mapResults"]},
             "isDebugRequest": False,
         }
-        resp = self.session.put(url, headers=self._get_headers(), json=payload)
+        resp = self.session.put(
+            url, headers=self._get_headers(), json=payload
+        )
         resp.raise_for_status()
         a = resp.json()
         return self._parse_properties(resp.json())
@@ -188,7 +191,9 @@ class ZillowScraper(Scraper):
                     else None,
                     "img_src": result.get("imgSrc"),
                     "price_per_sqft": int(home_info["price"] // home_info["livingArea"])
-                    if "livingArea" in home_info and "price" in home_info
+                    if "livingArea" in home_info
+                    and home_info["livingArea"] != 0
+                    and "price" in home_info
                     else None,
                 }
                 property_obj = Property(**property_data)

pyproject.toml

@@ -1,11 +1,14 @@
 [tool.poetry]
 name = "homeharvest"
-version = "0.2.2"
+version = "0.2.5"
 description = "Real estate scraping library supporting Zillow, Realtor.com & Redfin."
 authors = ["Zachary Hampton <zachary@zacharysproducts.com>", "Cullen Watson <cullen@cullen.ai>"]
 homepage = "https://github.com/ZacharyHampton/HomeHarvest"
 readme = "README.md"
 
+[tool.poetry.scripts]
+homeharvest = "homeharvest.cli:main"
+
 [tool.poetry.dependencies]
 python = "^3.10"
 requests = "^2.31.0"