error log

pull/53/head
yiwen 2023-10-01 16:41:28 +08:00
parent 6d2cdcf813
commit 1834889d9d
2 changed files with 19 additions and 13 deletions

View File

@ -21,6 +21,7 @@ pd.set_option('display.max_colwidth', 50) # set to 0 to see full job url / desc
# fetch jobs for each location
locations = read_location_list('location.json')
for location in locations:
try:
jobs: pd.DataFrame = scrape_jobs(
# site_name=["indeed", "linkedin", "zip_recruiter"],
site_name=["indeed"],
@ -33,7 +34,11 @@ for location in locations:
proxy="http://34.120.172.140:8123",
# proxy="http://crawler-gost-proxy.jobright-internal.com:8080",
)
except Exception as e:
print(f'Error when process: {location}')
print(e)
continue
print(f'{location}: {jobs.shape[0]} rows append.')
if os.path.isfile('./jobs.csv'):
jobs.to_csv('./jobs.csv', index=False, mode='a', header=False)
else:

View File

@ -28,6 +28,7 @@ from ...jobs import (
)
from .. import Scraper, ScraperInput, Site
def extract_emails_from_text(text: str) -> Optional[list[str]]:
if not text:
return None