turtlebasket
/
JobSpy
mirror of https://github.com/Bunsly/JobSpy
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

all data saved to db at the last step, 

cache works !
pull/231/head
Yariv Menachem 2025-01-05 17:41:48 +02:00
parent a2f02d7efd
commit 4873082147
5 changed files with 53 additions and 51 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

BIN
requirements.txt
View File

Binary file not shown.

17
src/config/cache_manager.py Normal file
View File

@ -0,0 +1,17 @@
from cachebox import LRUCache
class CacheboxCacheManager:
def __init__(self):
self._cache = LRUCache(50)
def find(self, cache_id: str):
"""Finding cached data by id, else None"""
return self._cache.get(cache_id)
def save(self, cache_id: str, data):
"""Finding cached data by id, else None"""
self._cache.insert(cache_id, data)
cache_manager = CacheboxCacheManager()

3
src/model/User.py
View File

@ -9,4 +9,7 @@ class User(BaseModel):
full_name: str
username: str
chat_id: Union[int, str] = None
experience: Union[int, str] = None
field: Optional[Position] = None
cities: Optional[list[str]] = None
title_filters: Optional[list[str]] = None

17
src/model/user_repository.py
View File

@ -4,6 +4,7 @@ from cachebox import LRUCache
from dotenv import load_dotenv
from pymongo import UpdateOne
from config.cache_manager import cache_manager
from jobspy import create_logger
from .User import User
from .monogo_db import mongo_client
@ -13,7 +14,6 @@ load_dotenv()
class UserRepository:
def __init__(self):
self._cache = LRUCache(50)
self._logger = create_logger("UserRepository")
self._collection = mongo_client.get_collection('user')
self._collection.create_index('username', unique=True)
@ -29,15 +29,15 @@ class UserRepository:
The user document if found, otherwise None.
"""
user = None
cached_user = self._cache[user_id]
cached_user = cache_manager.find(user_id)
if cached_user:
return User(**cached_user)
return cached_user
result = self._collection.find_one({"id": user_id})
if result:
user = User(**result)
self._cache[user_id] = user
cache_manager.save(user_id, user)
return user
@ -52,15 +52,15 @@ class UserRepository:
The user document if found, otherwise None.
"""
user = None
cached_user = self._cache.get(username)
cached_user = cache_manager.find(username)
if cached_user:
return cached_user
self._logger.info("Find user by username")
result = self._collection.find_one({"username": username})
self._logger.info("find user by usernameeeeeeee")
if result:
user = User(**result)
self._cache[username] = user
cache_manager.save(username, user)
return user
@ -74,7 +74,7 @@ class UserRepository:
Returns:
True if the update was successful, False otherwise.
"""
result = self._collection.update_one({"id": user.id}, {"$set": user.model_dump()})
result = self._collection.update_one({"username": user.username}, {"$set": user.model_dump()})
return result.modified_count > 0
def insert_user(self, user: User):
@ -88,6 +88,7 @@ class UserRepository:
Exception: If an error occurs during insertion.
"""
self._collection.insert_one(user.model_dump())
cache_manager.save(user.username, user)
self._logger.info(f"Inserted new user with username {user.username}.")
def insert_many_if_not_found(self, users: list[User]) -> tuple[list[User], list[User]]:

67
src/telegram_handler/telegram_start_handler.py
View File

@ -5,6 +5,7 @@ from telegram.ext import (
ContextTypes, ConversationHandler, CommandHandler, MessageHandler, filters,
)
from config.cache_manager import cache_manager
from jobspy.scrapers.utils import create_logger
from model.Position import Position
from model.User import User
@ -13,7 +14,6 @@ from telegram_bot import TelegramBot
from telegram_handler.start_handler_constats import START_MESSAGE, POSITION_MESSAGE, POSITION_NOT_FOUND, \
LOCATION_MESSAGE, EXPERIENCE_MESSAGE, FILTER_TILE_MESSAGE, THANK_YOU_MESSAGE, BYE_MESSAGE, VERIFY_MESSAGE, \
SEARCH_MESSAGE
from telegram_handler.telegram_handler import TelegramHandler
class Flow(Enum):
@ -26,14 +26,11 @@ class Flow(Enum):
SKIP_FILTERS = 6
class TelegramStartHandler(TelegramHandler):
class TelegramStartHandler:
def __init__(self):
self.filters = None
self.telegram_bot = TelegramBot()
self.logger = create_logger("TelegramStartHandler")
self.temp_user = None
self.cities = None
async def start(self, update: Update, context: ContextTypes.DEFAULT_TYPE) -> int:
"""Starts the conversation and asks the user about their position."""
@ -70,18 +67,23 @@ class TelegramStartHandler(TelegramHandler):
reply_markup=reply_markup,
)
return Flow.POSITION.value
cached_user: User = cache_manager.find(user.username)
cached_user.field = position
cache_manager.save(cached_user.username, cached_user)
await update.message.reply_text(LOCATION_MESSAGE)
return Flow.ADDRESS.value
async def address(self, update: Update, context: ContextTypes.DEFAULT_TYPE) -> int:
"""Asks for a location."""
user = update.message.from_user
self.cities = update.message.text.split(",")
cities = update.message.text.split(",")
reply_markup = ReplyKeyboardMarkup([[KeyboardButton("Yes"), KeyboardButton("No")]], one_time_keyboard=True,
input_field_placeholder=Flow.VERIFY_ADDRESS.name)
await update.message.reply_text(VERIFY_MESSAGE % self.cities, reply_markup=reply_markup)
await update.message.reply_text(VERIFY_MESSAGE % cities, reply_markup=reply_markup)
cached_user: User = cache_manager.find(update.message.from_user.username)
cached_user.cities = cities
cache_manager.save(cached_user.username, cached_user)
return Flow.VERIFY_ADDRESS.value
@ -103,17 +105,23 @@ class TelegramStartHandler(TelegramHandler):
"""Asks for a experience."""
user = update.message.from_user
self.logger.info("Experience of %s: %s", user.first_name, update.message.text)
cached_user: User = cache_manager.find(update.message.from_user.username)
cached_user.experience = update.message.text
cache_manager.save(cached_user.username, cached_user)
await update.message.reply_text(
FILTER_TILE_MESSAGE)
return Flow.FILTERS.value
async def filters_flow(self, update: Update, context: ContextTypes.DEFAULT_TYPE) -> int:
"""Asks for a filters_flow."""
self.filters = update.message.text.split(",")
title_filters = update.message.text.split(",")
reply_markup = ReplyKeyboardMarkup([[KeyboardButton("Yes"), KeyboardButton("No")]], one_time_keyboard=True,
input_field_placeholder=Flow.VERIFY_FILTERS.name)
await update.message.reply_text(VERIFY_MESSAGE % self.filters, reply_markup=reply_markup)
await update.message.reply_text(VERIFY_MESSAGE % title_filters, reply_markup=reply_markup)
cached_user: User = cache_manager.find(update.message.from_user.username)
cached_user.title_filters = title_filters
cache_manager.save(cached_user.username, cached_user)
return Flow.VERIFY_FILTERS.value
@ -125,7 +133,8 @@ class TelegramStartHandler(TelegramHandler):
await update.message.reply_text(THANK_YOU_MESSAGE)
await update.message.reply_text(SEARCH_MESSAGE)
cached_user: User = cache_manager.find(update.message.from_user.username)
user_repository.update(cached_user)
return ConversationHandler.END
async def skip_filter(self, update: Update, context: ContextTypes.DEFAULT_TYPE) -> int:
@ -144,38 +153,10 @@ class TelegramStartHandler(TelegramHandler):
await update.message.reply_text(
BYE_MESSAGE, reply_markup=ReplyKeyboardRemove()
)
cached_user: User = cache_manager.find(user.username)
user_repository.update(cached_user.username, cached_user)
return ConversationHandler.END
async def handle(self, update: Update, context: ContextTypes.DEFAULT_TYPE):
self.logger.info("start handling")
# chat: Chat = update.message.chat
# chat.id - 368620919
# chat.username - 'Qw1zeR'
# chat.full_name - 'Qw1zeR'
# user = User(full_name=chat.full_name, username=chat.username, chat_id=chat.id)
# user_repository.insert_user(user)
# fields = field_repository.find_all() # Get all fields from the database
# buttons = [[KeyboardButton(field.name)] for field in fields]
# reply_markup = ReplyKeyboardMarkup(buttons, one_time_keyboard=True)
#
# await update.message.reply_text("Please select your field:", reply_markup=reply_markup)
# await self.telegram_bot.set_message_reaction(
# update.message.message_id, ReactionEmoji.FIRE)
# site_names = [site.name for site in self.sites_to_scrap]
# site_names_print = ", ".join(site_names)
# await self.telegram_bot.send_text(
# f"Start scarping: {site_names_print}")
# self.logger.info(f"Found {len(jobs)} jobs")
# self.jobRepository.insert_many_if_not_found(filtered_out_jobs)
# old_jobs, new_jobs = self.jobRepository.insert_many_if_not_found(jobs)
# for newJob in new_jobs:
# await self.telegram_bot.send_job(newJob)
# self.logger.info(f"Found {len(old_jobs)} old jobs")
# await self.telegram_bot.send_text(
# f"Finished scarping: {site_names_print}")
self.logger.info("finished handling")
start_handler = TelegramStartHandler()
start_conv_handler = ConversationHandler(