diff --git a/.env.example b/.env.example index 694300e..81e50e1 100644 --- a/.env.example +++ b/.env.example @@ -5,6 +5,18 @@ # https://makersuite.google.com/app/apikey GOOGLE_API_KEY= +# https://platform.openai.com/settings/organization/api-keys +OPENAI_API_KEY= + +# https://admin.mistral.ai/organization/api-keys +MISTRAL_API_KEY= + +# https://platform.deepseek.com/api_keys +DEEPSEEK_API_KEY= + +# https://console.x.ai/team/%TEAM_ID%/api-keys +XAI_API_KEY= + ############################################################################### # Configurazioni per gli agenti di mercato @@ -42,13 +54,7 @@ CRYPTOPANIC_API_KEY= REDDIT_API_CLIENT_ID= REDDIT_API_CLIENT_SECRET= -# Per ottenere questa API è necessario seguire i seguenti passaggi: -# - Installare l'estensione su chrome X Auth Helper -# - Dargli il permesso di girare in incognito -# - Andare in incognito ed entrare sul proprio account X -# - Aprire l'estensione e fare "get key" -# - Chiudere chrome -# Dovrebbe funzionare per 5 anni o finchè non si si fa il log out, in ogni caso si può ricreare +# https://www.npmjs.com/package/rettiwt-api X_API_KEY= diff --git a/Dockerfile b/Dockerfile index 3a354bb..17e3234 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,10 +2,9 @@ FROM debian:bookworm-slim # Installiamo le dipendenze di sistema -RUN apt-get update && \ - apt-get install -y curl npm && \ +RUN apt update && \ + apt install -y curl && \ rm -rf /var/lib/apt/lists/* -RUN npm install -g rettiwt-api # Installiamo uv RUN curl -LsSf https://astral.sh/uv/install.sh | sh @@ -20,6 +19,11 @@ COPY uv.lock ./ RUN uv sync --frozen --no-dev ENV PYTHONPATH="./src" +# Installiamo le dipendenze per X (rettiwt, nodejs e npm) +RUN curl -fsSL https://deb.nodesource.com/setup_22.x | bash - +RUN apt install -y nodejs && rm -rf /var/lib/apt/lists/* +RUN npm install -g rettiwt-api + # Copiamo i file del progetto COPY LICENSE ./ COPY src/ ./src/ diff --git a/configs.yaml b/configs.yaml index f83ae9e..615fcf8 100644 --- a/configs.yaml +++ b/configs.yaml @@ -19,6 +19,18 @@ models: label: Gemini # - name: gemini-2.0-pro # TODO Non funziona, ha un nome diverso # label: Gemini Pro + gpt: + - name: gpt-4o + label: OpenAIChat + deepseek: + - name: deepseek-chat + label: DeepSeek + xai: + - name: grok-3 + label: xAI + mistral: + - name: mistral-large-latest + label: Mistral ollama: - name: gpt-oss:latest label: Ollama GPT diff --git a/demos/api_socials_providers.py b/demos/api_socials_providers.py new file mode 100644 index 0000000..aa4b14f --- /dev/null +++ b/demos/api_socials_providers.py @@ -0,0 +1,17 @@ +from dotenv import load_dotenv +from app.api.tools import SocialAPIsTool + +def main(): + api = SocialAPIsTool() + articles_aggregated = api.get_top_crypto_posts_aggregated(limit_per_wrapper=2) + for provider, posts in articles_aggregated.items(): + print("===================================") + print(f"Provider: {provider}") + for post in posts: + print(f"== [{post.timestamp}] - {post.title} ==") + print(f" {post.description}") + print(f" {len(post.comments)}") + +if __name__ == "__main__": + load_dotenv() + main() diff --git a/pyproject.toml b/pyproject.toml index e7b2209..5e3bdb7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,6 +24,10 @@ dependencies = [ # Modelli supportati e installati (aggiungere qui sotto quelli che si vogliono usare) "google-genai", "ollama", + "openai", + "mistralai", + "deepseek", + "xai", # API di exchange di criptovalute "coinbase-advanced-py", diff --git a/src/app/api/core/social.py b/src/app/api/core/social.py index fe4d5bf..05953a3 100644 --- a/src/app/api/core/social.py +++ b/src/app/api/core/social.py @@ -9,25 +9,25 @@ class SocialPost(BaseModel): """ Represents a social media post with time, title, description, and comments. """ - time: str = "" + timestamp: str = "" title: str = "" description: str = "" comments: list["SocialComment"] = [] def set_timestamp(self, timestamp_ms: int | None = None, timestamp_s: int | None = None) -> None: """ Use the unified_timestamp function to set the time.""" - self.time = unified_timestamp(timestamp_ms, timestamp_s) + self.timestamp = unified_timestamp(timestamp_ms, timestamp_s) class SocialComment(BaseModel): """ Represents a comment on a social media post. """ - time: str = "" + timestamp: str = "" description: str = "" def set_timestamp(self, timestamp_ms: int | None = None, timestamp_s: int | None = None) -> None: """ Use the unified_timestamp function to set the time.""" - self.time = unified_timestamp(timestamp_ms, timestamp_s) + self.timestamp = unified_timestamp(timestamp_ms, timestamp_s) class SocialWrapper: diff --git a/src/app/api/social/chan.py b/src/app/api/social/chan.py index a39e517..66efdb0 100644 --- a/src/app/api/social/chan.py +++ b/src/app/api/social/chan.py @@ -1,15 +1,20 @@ -''' -Usiamo le API di 4chan per ottenere un catalogo di threads dalla board /biz/ -''' import re import html import requests -from bs4 import BeautifulSoup +import warnings +from bs4 import BeautifulSoup, MarkupResemblesLocatorWarning from datetime import datetime from app.api.core.social import * +# Ignora i warning di BeautifulSoup quando incontra HTML malformato o un link, mentre si aspetta un HTML completo +warnings.filterwarnings("ignore", category=MarkupResemblesLocatorWarning) + class ChanWrapper(SocialWrapper): + """ + Wrapper per l'API di 4chan, in particolare per la board /biz/ (Business & Finance) + Fonte API: https://a.4cdn.org/biz/catalog.json + """ def __init__(self): super().__init__() diff --git a/src/app/api/social/reddit.py b/src/app/api/social/reddit.py index 201166c..306e49e 100644 --- a/src/app/api/social/reddit.py +++ b/src/app/api/social/reddit.py @@ -23,13 +23,13 @@ SUBREDDITS = [ def extract_post(post: Submission) -> SocialPost: social = SocialPost() - social.set_timestamp(timestamp_ms=post.created) + social.set_timestamp(timestamp_s=post.created) social.title = post.title social.description = post.selftext for top_comment in post.comments: comment = SocialComment() - comment.set_timestamp(timestamp_ms=top_comment.created) + comment.set_timestamp(timestamp_s=top_comment.created) comment.description = top_comment.body social.comments.append(comment) diff --git a/src/app/api/social/x.py b/src/app/api/social/x.py index a1b1bd4..c14f5b6 100644 --- a/src/app/api/social/x.py +++ b/src/app/api/social/x.py @@ -2,6 +2,7 @@ import os import json import subprocess from shutil import which +from datetime import datetime from app.api.core.social import SocialWrapper, SocialPost @@ -28,19 +29,20 @@ class XWrapper(SocialWrapper): def get_top_crypto_posts(self, limit:int = 5) -> list[SocialPost]: - social_posts: list[SocialPost] = [] + posts: list[SocialPost] = [] for user in X_USERS: - process = subprocess.run(f"rettiwt -k {self.api_key} tweet search -f {str(user)}", capture_output=True) + cmd = f"rettiwt -k {self.api_key} tweet search {limit} -f {str(user)}" + process = subprocess.run(cmd, capture_output=True, shell=True) results = process.stdout.decode() json_result = json.loads(results) - tweets = json_result['list'] - for tweet in tweets[:limit]: + for tweet in json_result.get('list', []): + time = datetime.fromisoformat(tweet['createdAt']) social_post = SocialPost() - social_post.time = tweet['createdAt'] - social_post.title = str(user) + " tweeted: " + social_post.set_timestamp(timestamp_s=int(time.timestamp())) + social_post.title = f"{user} tweeted: " social_post.description = tweet['fullText'] - social_posts.append(social_post) + posts.append(social_post) - return social_posts + return posts diff --git a/src/app/configs.py b/src/app/configs.py index 45b5b01..edd3e39 100644 --- a/src/app/configs.py +++ b/src/app/configs.py @@ -10,6 +10,10 @@ from agno.tools import Toolkit from agno.models.base import Model from agno.models.google import Gemini from agno.models.ollama import Ollama +from agno.models.openai import OpenAIChat +from agno.models.mistral import MistralChat +from agno.models.deepseek import DeepSeek +from agno.models.xai import xAI log = logging.getLogger(__name__) @@ -19,7 +23,6 @@ class AppModel(BaseModel): name: str = "gemini-2.0-flash" label: str = "Gemini" model: type[Model] | None = None - def get_model(self, instructions: str) -> Model: """ Restituisce un'istanza del modello specificato. @@ -54,22 +57,84 @@ class AppModel(BaseModel): output_schema=output_schema ) + + class APIConfig(BaseModel): retry_attempts: int = 3 retry_delay_seconds: int = 2 + + class Strategy(BaseModel): name: str = "Conservative" label: str = "Conservative" description: str = "Focus on low-risk investments with steady returns." + + class ModelsConfig(BaseModel): gemini: list[AppModel] = [AppModel()] + gpt: list[AppModel] = [AppModel(name="gpt-4o", label="OpenAIChat")] + mistral: list[AppModel] = [AppModel(name="mistral-large-latest", label="Mistral")] + deepseek: list[AppModel] = [AppModel(name="deepseek-chat", label="DeepSeek")] + xai: list[AppModel] = [AppModel(name="grok-3", label="xAI")] ollama: list[AppModel] = [] @property def all_models(self) -> list[AppModel]: - return self.gemini + self.ollama + return self.gemini + self.ollama + self.gpt + self.mistral + self.deepseek + self.xai + + def validate_models(self) -> None: + """ + Validate the configured models for each provider. + """ + self.__validate_online_models(self.gemini, clazz=Gemini, key="GOOGLE_API_KEY") + self.__validate_online_models(self.gpt, clazz=OpenAIChat, key="OPENAI_API_KEY") + self.__validate_online_models(self.mistral, clazz=MistralChat, key="MISTRAL_API_KEY") + self.__validate_online_models(self.deepseek, clazz=DeepSeek, key="DEEPSEEK_API_KEY") + self.__validate_online_models(self.xai, clazz=xAI, key="XAI_API_KEY") + + self.__validate_ollama_models() + + def __validate_online_models(self, models: list[AppModel], clazz: type[Model], key: str | None = None) -> None: + """ + Validate models for online providers like Gemini. + Args: + models: list of AppModel instances to validate + clazz: class of the model (e.g. Gemini) + key: API key required for the provider (optional) + """ + if key and os.getenv(key) is None: + log.warning(f"No {key} set in environment variables for provider.") + models.clear() + return + + for model in models: + model.model = clazz + + def __validate_ollama_models(self) -> None: + """ + Validate models for the Ollama provider. + """ + try: + models_list = ollama.list() + availables = {model['model'] for model in models_list['models']} + not_availables: list[str] = [] + + for model in self.ollama: + if model.name in availables: + model.model = Ollama + else: + not_availables.append(model.name) + if not_availables: + log.warning(f"Ollama models not available: {not_availables}") + + self.ollama = [model for model in self.ollama if model.model] + + except Exception as e: + log.warning(f"Ollama is not running or not reachable: {e}") + + class AgentsConfigs(BaseModel): strategy: str = "Conservative" @@ -118,7 +183,7 @@ class AppConfig(BaseModel): super().__init__(*args, **kwargs) self.set_logging_level() - self.validate_models() + self.models.validate_models() self._initialized = True def get_model_by_name(self, name: str) -> AppModel: @@ -186,53 +251,3 @@ class AppConfig(BaseModel): logger = logging.getLogger(logger_name) logger.handlers.clear() logger.propagate = True - - def validate_models(self) -> None: - """ - Validate the configured models for each provider. - """ - self.__validate_online_models("gemini", clazz=Gemini, key="GOOGLE_API_KEY") - self.__validate_ollama_models() - - def __validate_online_models(self, provider: str, clazz: type[Model], key: str | None = None) -> None: - """ - Validate models for online providers like Gemini. - Args: - provider: name of the provider (e.g. "gemini") - clazz: class of the model (e.g. Gemini) - key: API key required for the provider (optional) - """ - if getattr(self.models, provider) is None: - log.warning(f"No models configured for provider '{provider}'.") - - models: list[AppModel] = getattr(self.models, provider) - if key and os.getenv(key) is None: - log.warning(f"No {key} set in environment variables for {provider}.") - models.clear() - return - - for model in models: - model.model = clazz - - def __validate_ollama_models(self) -> None: - """ - Validate models for the Ollama provider. - """ - try: - models_list = ollama.list() - availables = {model['model'] for model in models_list['models']} - not_availables: list[str] = [] - - for model in self.models.ollama: - if model.name in availables: - model.model = Ollama - else: - not_availables.append(model.name) - if not_availables: - log.warning(f"Ollama models not available: {not_availables}") - - self.models.ollama = [model for model in self.models.ollama if model.model] - - except Exception as e: - log.warning(f"Ollama is not running or not reachable: {e}") - diff --git a/src/app/interface/chat.py b/src/app/interface/chat.py index 150197b..37529f4 100644 --- a/src/app/interface/chat.py +++ b/src/app/interface/chat.py @@ -83,13 +83,15 @@ class ChatManager: label="Modello da usare" ) provider.change(fn=self.inputs.choose_team_leader, inputs=provider, outputs=None) + provider.value = self.inputs.team_leader_model.label - style = gr.Dropdown( + strategy = gr.Dropdown( choices=self.inputs.list_strategies_names(), type="index", label="Stile di investimento" ) - style.change(fn=self.inputs.choose_strategy, inputs=style, outputs=None) + strategy.change(fn=self.inputs.choose_strategy, inputs=strategy, outputs=None) + strategy.value = self.inputs.strategy.label chat = gr.ChatInterface( fn=self.gradio_respond diff --git a/tests/api/test_social_4chan.py b/tests/api/test_social_4chan.py index b39a36d..dcf42d2 100644 --- a/tests/api/test_social_4chan.py +++ b/tests/api/test_social_4chan.py @@ -16,7 +16,7 @@ class TestChanWrapper: assert len(posts) == 2 for post in posts: assert post.title != "" - assert post.time != "" - assert re.match(r'\d{4}-\d{2}-\d{2}', post.time) + assert post.timestamp != "" + assert re.match(r'\d{4}-\d{2}-\d{2}', post.timestamp) assert isinstance(post.comments, list) diff --git a/tests/api/test_social_reddit.py b/tests/api/test_social_reddit.py index a83fe8a..adb4e13 100644 --- a/tests/api/test_social_reddit.py +++ b/tests/api/test_social_reddit.py @@ -19,7 +19,7 @@ class TestRedditWrapper: assert len(posts) == 2 for post in posts: assert post.title != "" - assert re.match(r'\d{4}-\d{2}-\d{2}', post.time) + assert re.match(r'\d{4}-\d{2}-\d{2}', post.timestamp) assert isinstance(post.comments, list) assert len(post.comments) <= MAX_COMMENTS diff --git a/tests/api/test_social_x_api.py b/tests/api/test_social_x_api.py index 15f39c3..39f75f9 100644 --- a/tests/api/test_social_x_api.py +++ b/tests/api/test_social_x_api.py @@ -1,11 +1,13 @@ import os import re import pytest +from shutil import which from app.api.social.x import XWrapper @pytest.mark.social @pytest.mark.api @pytest.mark.skipif(not os.getenv("X_API_KEY"), reason="X_API_KEY not set in environment variables") +@pytest.mark.skipif(which('rettiwt') is None, reason="rettiwt not installed") class TestXWrapper: def test_initialization(self): wrapper = XWrapper() @@ -18,5 +20,5 @@ class TestXWrapper: assert len(posts) == 2 for post in posts: assert post.title != "" - assert re.match(r'\d{4}-\d{2}-\d{2}', post.time) + assert re.match(r'\d{4}-\d{2}-\d{2}', post.timestamp) assert isinstance(post.comments, list) diff --git a/tests/tools/test_socials_tool.py b/tests/tools/test_socials_tool.py index c021a90..3a481f7 100644 --- a/tests/tools/test_socials_tool.py +++ b/tests/tools/test_socials_tool.py @@ -17,7 +17,7 @@ class TestSocialAPIsTool: assert len(result) > 0 for post in result: assert post.title is not None - assert post.time is not None + assert post.timestamp is not None def test_social_api_tool_get_top__all_results(self): tool = SocialAPIsTool() @@ -27,4 +27,4 @@ class TestSocialAPIsTool: for _provider, posts in result.items(): for post in posts: assert post.title is not None - assert post.time is not None + assert post.timestamp is not None