WIP: Fix social timestamps #49
20
.env.example
20
.env.example
@@ -5,6 +5,18 @@
|
||||
# https://makersuite.google.com/app/apikey
|
||||
GOOGLE_API_KEY=
|
||||
|
||||
# https://platform.openai.com/settings/organization/api-keys
|
||||
OPENAI_API_KEY=
|
||||
|
||||
# https://admin.mistral.ai/organization/api-keys
|
||||
MISTRAL_API_KEY=
|
||||
|
||||
# https://platform.deepseek.com/api_keys
|
||||
DEEPSEEK_API_KEY=
|
||||
|
||||
# https://console.x.ai/team/%TEAM_ID%/api-keys
|
||||
XAI_API_KEY=
|
||||
|
||||
|
||||
###############################################################################
|
||||
# Configurazioni per gli agenti di mercato
|
||||
@@ -42,13 +54,7 @@ CRYPTOPANIC_API_KEY=
|
||||
REDDIT_API_CLIENT_ID=
|
||||
REDDIT_API_CLIENT_SECRET=
|
||||
|
||||
# Per ottenere questa API è necessario seguire i seguenti passaggi:
|
||||
# - Installare l'estensione su chrome X Auth Helper
|
||||
# - Dargli il permesso di girare in incognito
|
||||
# - Andare in incognito ed entrare sul proprio account X
|
||||
# - Aprire l'estensione e fare "get key"
|
||||
# - Chiudere chrome
|
||||
# Dovrebbe funzionare per 5 anni o finchè non si si fa il log out, in ogni caso si può ricreare
|
||||
# https://www.npmjs.com/package/rettiwt-api
|
||||
X_API_KEY=
|
||||
|
||||
|
||||
|
||||
10
Dockerfile
10
Dockerfile
@@ -2,10 +2,9 @@
|
||||
FROM debian:bookworm-slim
|
||||
|
||||
# Installiamo le dipendenze di sistema
|
||||
RUN apt-get update && \
|
||||
apt-get install -y curl npm && \
|
||||
RUN apt update && \
|
||||
apt install -y curl && \
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
RUN npm install -g rettiwt-api
|
||||
|
||||
# Installiamo uv
|
||||
RUN curl -LsSf https://astral.sh/uv/install.sh | sh
|
||||
@@ -20,6 +19,11 @@ COPY uv.lock ./
|
||||
RUN uv sync --frozen --no-dev
|
||||
ENV PYTHONPATH="./src"
|
||||
|
||||
# Installiamo le dipendenze per X (rettiwt, nodejs e npm)
|
||||
RUN curl -fsSL https://deb.nodesource.com/setup_22.x | bash -
|
||||
RUN apt install -y nodejs && rm -rf /var/lib/apt/lists/*
|
||||
RUN npm install -g rettiwt-api
|
||||
|
||||
# Copiamo i file del progetto
|
||||
COPY LICENSE ./
|
||||
COPY src/ ./src/
|
||||
|
||||
12
configs.yaml
12
configs.yaml
@@ -19,6 +19,18 @@ models:
|
||||
label: Gemini
|
||||
# - name: gemini-2.0-pro # TODO Non funziona, ha un nome diverso
|
||||
# label: Gemini Pro
|
||||
gpt:
|
||||
- name: gpt-4o
|
||||
label: OpenAIChat
|
||||
deepseek:
|
||||
- name: deepseek-chat
|
||||
label: DeepSeek
|
||||
xai:
|
||||
- name: grok-3
|
||||
label: xAI
|
||||
mistral:
|
||||
- name: mistral-large-latest
|
||||
label: Mistral
|
||||
ollama:
|
||||
- name: gpt-oss:latest
|
||||
label: Ollama GPT
|
||||
|
||||
17
demos/api_socials_providers.py
Normal file
17
demos/api_socials_providers.py
Normal file
@@ -0,0 +1,17 @@
|
||||
from dotenv import load_dotenv
|
||||
from app.api.tools import SocialAPIsTool
|
||||
|
||||
def main():
|
||||
api = SocialAPIsTool()
|
||||
articles_aggregated = api.get_top_crypto_posts_aggregated(limit_per_wrapper=2)
|
||||
for provider, posts in articles_aggregated.items():
|
||||
print("===================================")
|
||||
print(f"Provider: {provider}")
|
||||
for post in posts:
|
||||
print(f"== [{post.timestamp}] - {post.title} ==")
|
||||
print(f" {post.description}")
|
||||
print(f" {len(post.comments)}")
|
||||
|
||||
if __name__ == "__main__":
|
||||
load_dotenv()
|
||||
main()
|
||||
@@ -24,6 +24,10 @@ dependencies = [
|
||||
# Modelli supportati e installati (aggiungere qui sotto quelli che si vogliono usare)
|
||||
"google-genai",
|
||||
"ollama",
|
||||
"openai",
|
||||
"mistralai",
|
||||
"deepseek",
|
||||
"xai",
|
||||
|
||||
# API di exchange di criptovalute
|
||||
"coinbase-advanced-py",
|
||||
|
||||
@@ -9,25 +9,25 @@ class SocialPost(BaseModel):
|
||||
"""
|
||||
Represents a social media post with time, title, description, and comments.
|
||||
"""
|
||||
time: str = ""
|
||||
timestamp: str = ""
|
||||
title: str = ""
|
||||
description: str = ""
|
||||
comments: list["SocialComment"] = []
|
||||
|
||||
def set_timestamp(self, timestamp_ms: int | None = None, timestamp_s: int | None = None) -> None:
|
||||
""" Use the unified_timestamp function to set the time."""
|
||||
self.time = unified_timestamp(timestamp_ms, timestamp_s)
|
||||
self.timestamp = unified_timestamp(timestamp_ms, timestamp_s)
|
||||
|
||||
class SocialComment(BaseModel):
|
||||
"""
|
||||
Represents a comment on a social media post.
|
||||
"""
|
||||
time: str = ""
|
||||
timestamp: str = ""
|
||||
description: str = ""
|
||||
|
||||
def set_timestamp(self, timestamp_ms: int | None = None, timestamp_s: int | None = None) -> None:
|
||||
""" Use the unified_timestamp function to set the time."""
|
||||
self.time = unified_timestamp(timestamp_ms, timestamp_s)
|
||||
self.timestamp = unified_timestamp(timestamp_ms, timestamp_s)
|
||||
|
||||
|
||||
class SocialWrapper:
|
||||
|
||||
@@ -1,15 +1,20 @@
|
||||
'''
|
||||
Usiamo le API di 4chan per ottenere un catalogo di threads dalla board /biz/
|
||||
'''
|
||||
import re
|
||||
import html
|
||||
import requests
|
||||
from bs4 import BeautifulSoup
|
||||
import warnings
|
||||
from bs4 import BeautifulSoup, MarkupResemblesLocatorWarning
|
||||
from datetime import datetime
|
||||
from app.api.core.social import *
|
||||
|
||||
# Ignora i warning di BeautifulSoup quando incontra HTML malformato o un link, mentre si aspetta un HTML completo
|
||||
warnings.filterwarnings("ignore", category=MarkupResemblesLocatorWarning)
|
||||
|
||||
|
||||
class ChanWrapper(SocialWrapper):
|
||||
"""
|
||||
Wrapper per l'API di 4chan, in particolare per la board /biz/ (Business & Finance)
|
||||
Fonte API: https://a.4cdn.org/biz/catalog.json
|
||||
"""
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
|
||||
|
||||
@@ -23,13 +23,13 @@ SUBREDDITS = [
|
||||
|
||||
def extract_post(post: Submission) -> SocialPost:
|
||||
social = SocialPost()
|
||||
social.set_timestamp(timestamp_ms=post.created)
|
||||
social.set_timestamp(timestamp_s=post.created)
|
||||
social.title = post.title
|
||||
social.description = post.selftext
|
||||
|
||||
for top_comment in post.comments:
|
||||
comment = SocialComment()
|
||||
comment.set_timestamp(timestamp_ms=top_comment.created)
|
||||
comment.set_timestamp(timestamp_s=top_comment.created)
|
||||
comment.description = top_comment.body
|
||||
social.comments.append(comment)
|
||||
|
||||
|
||||
@@ -2,6 +2,7 @@ import os
|
||||
import json
|
||||
import subprocess
|
||||
from shutil import which
|
||||
from datetime import datetime
|
||||
from app.api.core.social import SocialWrapper, SocialPost
|
||||
|
||||
|
||||
@@ -28,19 +29,20 @@ class XWrapper(SocialWrapper):
|
||||
|
||||
|
||||
def get_top_crypto_posts(self, limit:int = 5) -> list[SocialPost]:
|
||||
social_posts: list[SocialPost] = []
|
||||
posts: list[SocialPost] = []
|
||||
|
||||
for user in X_USERS:
|
||||
process = subprocess.run(f"rettiwt -k {self.api_key} tweet search -f {str(user)}", capture_output=True)
|
||||
cmd = f"rettiwt -k {self.api_key} tweet search {limit} -f {str(user)}"
|
||||
process = subprocess.run(cmd, capture_output=True, shell=True)
|
||||
results = process.stdout.decode()
|
||||
json_result = json.loads(results)
|
||||
|
||||
tweets = json_result['list']
|
||||
for tweet in tweets[:limit]:
|
||||
for tweet in json_result.get('list', []):
|
||||
time = datetime.fromisoformat(tweet['createdAt'])
|
||||
social_post = SocialPost()
|
||||
social_post.time = tweet['createdAt']
|
||||
social_post.title = str(user) + " tweeted: "
|
||||
social_post.set_timestamp(timestamp_s=int(time.timestamp()))
|
||||
social_post.title = f"{user} tweeted: "
|
||||
social_post.description = tweet['fullText']
|
||||
social_posts.append(social_post)
|
||||
posts.append(social_post)
|
||||
|
||||
return social_posts
|
||||
return posts
|
||||
|
||||
@@ -10,6 +10,10 @@ from agno.tools import Toolkit
|
||||
from agno.models.base import Model
|
||||
from agno.models.google import Gemini
|
||||
from agno.models.ollama import Ollama
|
||||
from agno.models.openai import OpenAIChat
|
||||
from agno.models.mistral import MistralChat
|
||||
from agno.models.deepseek import DeepSeek
|
||||
from agno.models.xai import xAI
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@@ -19,7 +23,6 @@ class AppModel(BaseModel):
|
||||
name: str = "gemini-2.0-flash"
|
||||
label: str = "Gemini"
|
||||
model: type[Model] | None = None
|
||||
|
||||
def get_model(self, instructions: str) -> Model:
|
||||
"""
|
||||
Restituisce un'istanza del modello specificato.
|
||||
@@ -54,22 +57,84 @@ class AppModel(BaseModel):
|
||||
output_schema=output_schema
|
||||
)
|
||||
|
||||
|
||||
|
||||
class APIConfig(BaseModel):
|
||||
retry_attempts: int = 3
|
||||
retry_delay_seconds: int = 2
|
||||
|
||||
|
||||
|
||||
class Strategy(BaseModel):
|
||||
name: str = "Conservative"
|
||||
label: str = "Conservative"
|
||||
description: str = "Focus on low-risk investments with steady returns."
|
||||
|
||||
|
||||
|
||||
class ModelsConfig(BaseModel):
|
||||
gemini: list[AppModel] = [AppModel()]
|
||||
gpt: list[AppModel] = [AppModel(name="gpt-4o", label="OpenAIChat")]
|
||||
mistral: list[AppModel] = [AppModel(name="mistral-large-latest", label="Mistral")]
|
||||
deepseek: list[AppModel] = [AppModel(name="deepseek-chat", label="DeepSeek")]
|
||||
xai: list[AppModel] = [AppModel(name="grok-3", label="xAI")]
|
||||
ollama: list[AppModel] = []
|
||||
|
||||
@property
|
||||
def all_models(self) -> list[AppModel]:
|
||||
return self.gemini + self.ollama
|
||||
return self.gemini + self.ollama + self.gpt + self.mistral + self.deepseek + self.xai
|
||||
|
||||
def validate_models(self) -> None:
|
||||
"""
|
||||
Validate the configured models for each provider.
|
||||
"""
|
||||
self.__validate_online_models(self.gemini, clazz=Gemini, key="GOOGLE_API_KEY")
|
||||
self.__validate_online_models(self.gpt, clazz=OpenAIChat, key="OPENAI_API_KEY")
|
||||
self.__validate_online_models(self.mistral, clazz=MistralChat, key="MISTRAL_API_KEY")
|
||||
self.__validate_online_models(self.deepseek, clazz=DeepSeek, key="DEEPSEEK_API_KEY")
|
||||
self.__validate_online_models(self.xai, clazz=xAI, key="XAI_API_KEY")
|
||||
|
||||
self.__validate_ollama_models()
|
||||
|
||||
def __validate_online_models(self, models: list[AppModel], clazz: type[Model], key: str | None = None) -> None:
|
||||
"""
|
||||
Validate models for online providers like Gemini.
|
||||
Args:
|
||||
models: list of AppModel instances to validate
|
||||
clazz: class of the model (e.g. Gemini)
|
||||
key: API key required for the provider (optional)
|
||||
"""
|
||||
if key and os.getenv(key) is None:
|
||||
log.warning(f"No {key} set in environment variables for provider.")
|
||||
models.clear()
|
||||
return
|
||||
|
||||
for model in models:
|
||||
model.model = clazz
|
||||
|
||||
def __validate_ollama_models(self) -> None:
|
||||
"""
|
||||
Validate models for the Ollama provider.
|
||||
"""
|
||||
try:
|
||||
models_list = ollama.list()
|
||||
availables = {model['model'] for model in models_list['models']}
|
||||
not_availables: list[str] = []
|
||||
|
||||
for model in self.ollama:
|
||||
if model.name in availables:
|
||||
model.model = Ollama
|
||||
else:
|
||||
not_availables.append(model.name)
|
||||
if not_availables:
|
||||
log.warning(f"Ollama models not available: {not_availables}")
|
||||
|
||||
self.ollama = [model for model in self.ollama if model.model]
|
||||
|
||||
except Exception as e:
|
||||
log.warning(f"Ollama is not running or not reachable: {e}")
|
||||
|
||||
|
||||
|
||||
class AgentsConfigs(BaseModel):
|
||||
strategy: str = "Conservative"
|
||||
@@ -118,7 +183,7 @@ class AppConfig(BaseModel):
|
||||
|
||||
super().__init__(*args, **kwargs)
|
||||
self.set_logging_level()
|
||||
self.validate_models()
|
||||
self.models.validate_models()
|
||||
self._initialized = True
|
||||
|
||||
def get_model_by_name(self, name: str) -> AppModel:
|
||||
@@ -186,53 +251,3 @@ class AppConfig(BaseModel):
|
||||
logger = logging.getLogger(logger_name)
|
||||
logger.handlers.clear()
|
||||
logger.propagate = True
|
||||
|
||||
def validate_models(self) -> None:
|
||||
"""
|
||||
Validate the configured models for each provider.
|
||||
"""
|
||||
self.__validate_online_models("gemini", clazz=Gemini, key="GOOGLE_API_KEY")
|
||||
self.__validate_ollama_models()
|
||||
|
||||
def __validate_online_models(self, provider: str, clazz: type[Model], key: str | None = None) -> None:
|
||||
"""
|
||||
Validate models for online providers like Gemini.
|
||||
Args:
|
||||
provider: name of the provider (e.g. "gemini")
|
||||
clazz: class of the model (e.g. Gemini)
|
||||
key: API key required for the provider (optional)
|
||||
"""
|
||||
if getattr(self.models, provider) is None:
|
||||
log.warning(f"No models configured for provider '{provider}'.")
|
||||
|
||||
models: list[AppModel] = getattr(self.models, provider)
|
||||
if key and os.getenv(key) is None:
|
||||
log.warning(f"No {key} set in environment variables for {provider}.")
|
||||
models.clear()
|
||||
return
|
||||
|
||||
for model in models:
|
||||
model.model = clazz
|
||||
|
||||
def __validate_ollama_models(self) -> None:
|
||||
"""
|
||||
Validate models for the Ollama provider.
|
||||
"""
|
||||
try:
|
||||
models_list = ollama.list()
|
||||
availables = {model['model'] for model in models_list['models']}
|
||||
not_availables: list[str] = []
|
||||
|
||||
for model in self.models.ollama:
|
||||
if model.name in availables:
|
||||
model.model = Ollama
|
||||
else:
|
||||
not_availables.append(model.name)
|
||||
if not_availables:
|
||||
log.warning(f"Ollama models not available: {not_availables}")
|
||||
|
||||
self.models.ollama = [model for model in self.models.ollama if model.model]
|
||||
|
||||
except Exception as e:
|
||||
log.warning(f"Ollama is not running or not reachable: {e}")
|
||||
|
||||
|
||||
@@ -83,13 +83,15 @@ class ChatManager:
|
||||
label="Modello da usare"
|
||||
)
|
||||
provider.change(fn=self.inputs.choose_team_leader, inputs=provider, outputs=None)
|
||||
provider.value = self.inputs.team_leader_model.label
|
||||
|
||||
style = gr.Dropdown(
|
||||
strategy = gr.Dropdown(
|
||||
choices=self.inputs.list_strategies_names(),
|
||||
type="index",
|
||||
label="Stile di investimento"
|
||||
)
|
||||
style.change(fn=self.inputs.choose_strategy, inputs=style, outputs=None)
|
||||
strategy.change(fn=self.inputs.choose_strategy, inputs=strategy, outputs=None)
|
||||
strategy.value = self.inputs.strategy.label
|
||||
|
||||
chat = gr.ChatInterface(
|
||||
fn=self.gradio_respond
|
||||
|
||||
@@ -16,7 +16,7 @@ class TestChanWrapper:
|
||||
assert len(posts) == 2
|
||||
for post in posts:
|
||||
assert post.title != ""
|
||||
assert post.time != ""
|
||||
assert re.match(r'\d{4}-\d{2}-\d{2}', post.time)
|
||||
assert post.timestamp != ""
|
||||
assert re.match(r'\d{4}-\d{2}-\d{2}', post.timestamp)
|
||||
assert isinstance(post.comments, list)
|
||||
|
||||
|
||||
@@ -19,7 +19,7 @@ class TestRedditWrapper:
|
||||
assert len(posts) == 2
|
||||
for post in posts:
|
||||
assert post.title != ""
|
||||
assert re.match(r'\d{4}-\d{2}-\d{2}', post.time)
|
||||
assert re.match(r'\d{4}-\d{2}-\d{2}', post.timestamp)
|
||||
|
||||
assert isinstance(post.comments, list)
|
||||
assert len(post.comments) <= MAX_COMMENTS
|
||||
|
||||
@@ -1,11 +1,13 @@
|
||||
import os
|
||||
import re
|
||||
import pytest
|
||||
from shutil import which
|
||||
from app.api.social.x import XWrapper
|
||||
|
||||
@pytest.mark.social
|
||||
@pytest.mark.api
|
||||
@pytest.mark.skipif(not os.getenv("X_API_KEY"), reason="X_API_KEY not set in environment variables")
|
||||
@pytest.mark.skipif(which('rettiwt') is None, reason="rettiwt not installed")
|
||||
class TestXWrapper:
|
||||
def test_initialization(self):
|
||||
wrapper = XWrapper()
|
||||
@@ -18,5 +20,5 @@ class TestXWrapper:
|
||||
assert len(posts) == 2
|
||||
for post in posts:
|
||||
assert post.title != ""
|
||||
assert re.match(r'\d{4}-\d{2}-\d{2}', post.time)
|
||||
assert re.match(r'\d{4}-\d{2}-\d{2}', post.timestamp)
|
||||
assert isinstance(post.comments, list)
|
||||
|
||||
@@ -17,7 +17,7 @@ class TestSocialAPIsTool:
|
||||
assert len(result) > 0
|
||||
for post in result:
|
||||
assert post.title is not None
|
||||
assert post.time is not None
|
||||
assert post.timestamp is not None
|
||||
|
||||
def test_social_api_tool_get_top__all_results(self):
|
||||
tool = SocialAPIsTool()
|
||||
@@ -27,4 +27,4 @@ class TestSocialAPIsTool:
|
||||
for _provider, posts in result.items():
|
||||
for post in posts:
|
||||
assert post.title is not None
|
||||
assert post.time is not None
|
||||
assert post.timestamp is not None
|
||||
|
||||
Reference in New Issue
Block a user