turtlebasket
/
JobSpy
mirror of https://github.com/Bunsly/JobSpy
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

updated insert many method

pull/231/head
Yariv Menachem 2024-12-24 18:34:19 +02:00
parent e50766f756
commit 80fef6d67b
1 changed files with 4 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
src/jobspy/db/job_repository.py
View File

@ -75,20 +75,9 @@ class JobRepository:
pymongo.errors.BulkWriteError: If an error occurs during the bulk insert. pymongo.errors.BulkWriteError: If an error occurs during the bulk insert.
""" """
new_jobs = [] new_jobs = []
operations = [ result = self.collection.insert_many(jobs)
UpdateOne({"id": job.id}, {"$setOnInsert": job.model_dump( new_jobs = [jobs[i] for i, _ in enumerate(
exclude={"date_posted"})}, upsert=True) result.inserted_ids) if result.inserted_ids]
for job in jobs print(f"Inserted Jobs: {len(new_jobs)}")
]
try:
result = self.collection.bulk_write(operations)
new_jobs = [jobs[i] for i in range(
result.inserted_count)] if result.inserted_count > 0 else []
print(f"Inserted Jobs: {len(new_jobs)}")
except pymongo.errors.BulkWriteError as e:
# Handle bulk write errors gracefully, e.g., log details
print(f"Bulk Insert Error: {e}")
return new_jobs return new_jobs