From 2a96fdc8a557bdf14dedc2f28ac55fa53099f827 Mon Sep 17 00:00:00 2001
From: Cullen Watson <cullen@cullenwatson.com>
Date: Tue, 11 Jul 2023 08:51:19 -0500
Subject: [PATCH] fix(jobs): add 1 to loop counter

---
 api/core/scrapers/indeed/__init__.py       | 6 +++++-
 api/core/scrapers/linkedin/__init__.py     | 6 +++++-
 api/core/scrapers/ziprecruiter/__init__.py | 5 ++++-
 3 files changed, 14 insertions(+), 3 deletions(-)

diff --git a/api/core/scrapers/indeed/__init__.py b/api/core/scrapers/indeed/__init__.py
index c9b8ae9..3aa6a88 100644
--- a/api/core/scrapers/indeed/__init__.py
+++ b/api/core/scrapers/indeed/__init__.py
@@ -89,6 +89,7 @@ class IndeedScraper(Scraper):
                 )
 
             for job in jobs["metaData"]["mosaicProviderJobCardsModel"]["results"]:
+                processed_jobs += 1
                 job_url = f'{self.job_url}{job["jobkey"]}'
                 if job_url in seen_urls:
                     continue
@@ -135,7 +136,10 @@ class IndeedScraper(Scraper):
                     job_url=job_url,
                 )
                 job_list.append(job_post)
-                if len(job_list) >= scraper_input.results_wanted:
+                if (
+                    len(job_list) >= scraper_input.results_wanted
+                    or processed_jobs >= total_num_jobs
+                ):
                     break
 
             if (
diff --git a/api/core/scrapers/linkedin/__init__.py b/api/core/scrapers/linkedin/__init__.py
index f3cd68d..fd26bb5 100644
--- a/api/core/scrapers/linkedin/__init__.py
+++ b/api/core/scrapers/linkedin/__init__.py
@@ -74,6 +74,7 @@ class LinkedInScraper(Scraper):
                     "div",
                     class_="base-card relative w-full hover:no-underline focus:no-underline base-card--link base-search-card base-search-card--link job-search-card",
                 ):
+                    processed_jobs += 1
                     data_entity_urn = job_card.get("data-entity-urn", "")
                     job_id = (
                         data_entity_urn.split(":")[-1] if data_entity_urn else "N/A"
@@ -113,7 +114,10 @@ class LinkedInScraper(Scraper):
                         job_url=job_url,
                     )
                     job_list.append(job_post)
-                    if len(job_list) >= scraper_input.results_wanted:
+                    if (
+                        len(job_list) >= scraper_input.results_wanted
+                        or processed_jobs >= job_count
+                    ):
                         break
                 if (
                     len(job_list) >= scraper_input.results_wanted
diff --git a/api/core/scrapers/ziprecruiter/__init__.py b/api/core/scrapers/ziprecruiter/__init__.py
index 117d4c4..ff7780e 100644
--- a/api/core/scrapers/ziprecruiter/__init__.py
+++ b/api/core/scrapers/ziprecruiter/__init__.py
@@ -114,7 +114,10 @@ class ZipRecruiterScraper(Scraper):
                     job_url=job_url,
                 )
                 job_list.append(job_post)
-                if len(job_list) >= scraper_input.results_wanted:
+                if (
+                    len(job_list) >= scraper_input.results_wanted
+                    or processed_jobs >= job_count
+                ):
                     break
 
             if (