Google sheets integration (#22)

api/core/formatters/__init__.py

@@ -4,3 +4,4 @@ from enum import Enum
 class OutputFormat(Enum):
     CSV = "csv"
     JSON = "json"
+    GSHEET = "gsheet"

api/core/formatters/csv/__init__.py

@@ -1,19 +1,50 @@
+import gspread
+from oauth2client.service_account import ServiceAccountCredentials
+
 import csv
 from io import StringIO
 from datetime import datetime
 
 from ...jobs import *
 from ...scrapers import *
-
-
-def generate_filename() -> str:
-    timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
-    return f"JobSpy_results_{timestamp}.csv"
+from settings import *
 
 
 class CSVFormatter:
     @staticmethod
-    def format(jobs: ScraperResponse) -> StringIO:
+    def upload_to_google_sheet(csv_data: str):
+        try:
+            scope = [
+                "https://www.googleapis.com/auth/spreadsheets",
+                "https://www.googleapis.com/auth/drive.file",
+                "https://www.googleapis.com/auth/drive",
+            ]
+            credentials = ServiceAccountCredentials.from_json_keyfile_name(
+                GSHEET_JSON_KEY_PATH, scope
+            )
+            gc = gspread.authorize(credentials)
+            sh = gc.open(GSHEET_NAME)
+
+            worksheet = sh.get_worksheet(0)
+            data_string = csv_data.getvalue()
+            reader = csv.reader(StringIO(data_string))
+
+            rows = list(reader)
+
+            for i, row in enumerate(rows):
+                if i == 0:
+                    continue
+                worksheet.append_row(row)
+        except Exception as e:
+            raise e
+
+    @staticmethod
+    def generate_filename() -> str:
+        timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+        return f"JobSpy_results_{timestamp}.csv"
+
+    @staticmethod
+    def format(jobs: CommonResponse) -> StringIO:
         """
         Transfomr the jobs objects into csv
         :param jobs:
@@ -41,7 +72,7 @@ class CSVFormatter:
         writer.writerow(headers)
 
         for site, job_response in jobs.dict().items():
-            if job_response and job_response.get("success"):
+            if isinstance(job_response, dict) and job_response.get("success"):
                 for job in job_response["jobs"]:
                     writer.writerow(
                         [

api/core/jobs/__init__.py

@@ -55,12 +55,13 @@ class JobResponse(BaseModel):
     success: bool
     error: str = None
 
+    total_results: int = None
+
     jobs: list[JobPost] = []
 
-    total_results: int = None
     returned_results: int = None
 
-    @validator("returned_results")
+    @validator("returned_results", pre=True, always=True)
     def set_returned_results(cls, v, values):
         if v is None and values.get("jobs"):
             return len(values["jobs"])

api/core/scrapers/__init__.py

@@ -1,6 +1,6 @@
 from ..jobs import *
 from ..formatters import OutputFormat
-from typing import List, Dict, Optional
+from typing import List, Dict, Optional, Any
 
 
 class StatusException(Exception):
@@ -28,10 +28,12 @@ class ScraperInput(BaseModel):
     results_wanted: int = 15
 
 
-class ScraperResponse(BaseModel):
-    linkedin: Optional[JobResponse]
-    indeed: Optional[JobResponse]
-    zip_recruiter: Optional[JobResponse]
+class CommonResponse(BaseModel):
+    status: Optional[str]
+    error: Optional[str]
+    linkedin: Optional[Any] = None
+    indeed: Optional[Any] = None
+    zip_recruiter: Optional[Any] = None
 
 
 class Scraper: