WIP: Fix social timestamps #49

Closed
Berack96 wants to merge 6 commits from fix-socials-timestamp into demos
15 changed files with 158 additions and 89 deletions

View File

@@ -5,6 +5,18 @@
# https://makersuite.google.com/app/apikey
GOOGLE_API_KEY=
# https://platform.openai.com/settings/organization/api-keys
OPENAI_API_KEY=
# https://admin.mistral.ai/organization/api-keys
MISTRAL_API_KEY=
# https://platform.deepseek.com/api_keys
DEEPSEEK_API_KEY=
# https://console.x.ai/team/%TEAM_ID%/api-keys
XAI_API_KEY=
###############################################################################
# Configurazioni per gli agenti di mercato
@@ -42,13 +54,7 @@ CRYPTOPANIC_API_KEY=
REDDIT_API_CLIENT_ID=
REDDIT_API_CLIENT_SECRET=
# Per ottenere questa API è necessario seguire i seguenti passaggi:
# - Installare l'estensione su chrome X Auth Helper
# - Dargli il permesso di girare in incognito
# - Andare in incognito ed entrare sul proprio account X
# - Aprire l'estensione e fare "get key"
# - Chiudere chrome
# Dovrebbe funzionare per 5 anni o finchè non si si fa il log out, in ogni caso si può ricreare
# https://www.npmjs.com/package/rettiwt-api
X_API_KEY=

View File

@@ -2,10 +2,9 @@
FROM debian:bookworm-slim
# Installiamo le dipendenze di sistema
RUN apt-get update && \
apt-get install -y curl npm && \
RUN apt update && \
apt install -y curl && \
rm -rf /var/lib/apt/lists/*
RUN npm install -g rettiwt-api
# Installiamo uv
RUN curl -LsSf https://astral.sh/uv/install.sh | sh
@@ -20,6 +19,11 @@ COPY uv.lock ./
RUN uv sync --frozen --no-dev
ENV PYTHONPATH="./src"
# Installiamo le dipendenze per X (rettiwt, nodejs e npm)
RUN curl -fsSL https://deb.nodesource.com/setup_22.x | bash -
RUN apt install -y nodejs && rm -rf /var/lib/apt/lists/*
RUN npm install -g rettiwt-api
# Copiamo i file del progetto
COPY LICENSE ./
COPY src/ ./src/

View File

@@ -19,6 +19,18 @@ models:
label: Gemini
# - name: gemini-2.0-pro # TODO Non funziona, ha un nome diverso
# label: Gemini Pro
gpt:
- name: gpt-4o
label: OpenAIChat
deepseek:
- name: deepseek-chat
label: DeepSeek
xai:
- name: grok-3
label: xAI
mistral:
- name: mistral-large-latest
label: Mistral
ollama:
- name: gpt-oss:latest
label: Ollama GPT

View File

@@ -0,0 +1,17 @@
from dotenv import load_dotenv
from app.api.tools import SocialAPIsTool
def main():
api = SocialAPIsTool()
articles_aggregated = api.get_top_crypto_posts_aggregated(limit_per_wrapper=2)
for provider, posts in articles_aggregated.items():
print("===================================")
print(f"Provider: {provider}")
for post in posts:
print(f"== [{post.timestamp}] - {post.title} ==")
print(f" {post.description}")
print(f" {len(post.comments)}")
if __name__ == "__main__":
load_dotenv()
main()

View File

@@ -24,6 +24,10 @@ dependencies = [
# Modelli supportati e installati (aggiungere qui sotto quelli che si vogliono usare)
"google-genai",
"ollama",
"openai",
"mistralai",
"deepseek",
"xai",
# API di exchange di criptovalute
"coinbase-advanced-py",

View File

@@ -9,25 +9,25 @@ class SocialPost(BaseModel):
"""
Represents a social media post with time, title, description, and comments.
"""
time: str = ""
timestamp: str = ""
title: str = ""
description: str = ""
comments: list["SocialComment"] = []
def set_timestamp(self, timestamp_ms: int | None = None, timestamp_s: int | None = None) -> None:
""" Use the unified_timestamp function to set the time."""
self.time = unified_timestamp(timestamp_ms, timestamp_s)
self.timestamp = unified_timestamp(timestamp_ms, timestamp_s)
class SocialComment(BaseModel):
"""
Represents a comment on a social media post.
"""
time: str = ""
timestamp: str = ""
description: str = ""
def set_timestamp(self, timestamp_ms: int | None = None, timestamp_s: int | None = None) -> None:
""" Use the unified_timestamp function to set the time."""
self.time = unified_timestamp(timestamp_ms, timestamp_s)
self.timestamp = unified_timestamp(timestamp_ms, timestamp_s)
class SocialWrapper:

View File

@@ -1,15 +1,20 @@
'''
Usiamo le API di 4chan per ottenere un catalogo di threads dalla board /biz/
'''
import re
import html
import requests
from bs4 import BeautifulSoup
import warnings
from bs4 import BeautifulSoup, MarkupResemblesLocatorWarning
from datetime import datetime
from app.api.core.social import *
# Ignora i warning di BeautifulSoup quando incontra HTML malformato o un link, mentre si aspetta un HTML completo
warnings.filterwarnings("ignore", category=MarkupResemblesLocatorWarning)
class ChanWrapper(SocialWrapper):
"""
Wrapper per l'API di 4chan, in particolare per la board /biz/ (Business & Finance)
Fonte API: https://a.4cdn.org/biz/catalog.json
"""
def __init__(self):
super().__init__()

View File

@@ -23,13 +23,13 @@ SUBREDDITS = [
def extract_post(post: Submission) -> SocialPost:
social = SocialPost()
social.set_timestamp(timestamp_ms=post.created)
social.set_timestamp(timestamp_s=post.created)
social.title = post.title
social.description = post.selftext
for top_comment in post.comments:
comment = SocialComment()
comment.set_timestamp(timestamp_ms=top_comment.created)
comment.set_timestamp(timestamp_s=top_comment.created)
comment.description = top_comment.body
social.comments.append(comment)

View File

@@ -2,6 +2,7 @@ import os
import json
import subprocess
from shutil import which
from datetime import datetime
from app.api.core.social import SocialWrapper, SocialPost
@@ -28,19 +29,20 @@ class XWrapper(SocialWrapper):
def get_top_crypto_posts(self, limit:int = 5) -> list[SocialPost]:
social_posts: list[SocialPost] = []
posts: list[SocialPost] = []
for user in X_USERS:
process = subprocess.run(f"rettiwt -k {self.api_key} tweet search -f {str(user)}", capture_output=True)
cmd = f"rettiwt -k {self.api_key} tweet search {limit} -f {str(user)}"
process = subprocess.run(cmd, capture_output=True, shell=True)
results = process.stdout.decode()
json_result = json.loads(results)
tweets = json_result['list']
for tweet in tweets[:limit]:
for tweet in json_result.get('list', []):
time = datetime.fromisoformat(tweet['createdAt'])
social_post = SocialPost()
social_post.time = tweet['createdAt']
social_post.title = str(user) + " tweeted: "
social_post.set_timestamp(timestamp_s=int(time.timestamp()))
social_post.title = f"{user} tweeted: "
social_post.description = tweet['fullText']
social_posts.append(social_post)
posts.append(social_post)
return social_posts
return posts

View File

@@ -10,6 +10,10 @@ from agno.tools import Toolkit
from agno.models.base import Model
from agno.models.google import Gemini
from agno.models.ollama import Ollama
from agno.models.openai import OpenAIChat
from agno.models.mistral import MistralChat
from agno.models.deepseek import DeepSeek
from agno.models.xai import xAI
log = logging.getLogger(__name__)
@@ -19,7 +23,6 @@ class AppModel(BaseModel):
name: str = "gemini-2.0-flash"
label: str = "Gemini"
model: type[Model] | None = None
def get_model(self, instructions: str) -> Model:
"""
Restituisce un'istanza del modello specificato.
@@ -54,22 +57,84 @@ class AppModel(BaseModel):
output_schema=output_schema
)
class APIConfig(BaseModel):
retry_attempts: int = 3
retry_delay_seconds: int = 2
class Strategy(BaseModel):
name: str = "Conservative"
label: str = "Conservative"
description: str = "Focus on low-risk investments with steady returns."
class ModelsConfig(BaseModel):
gemini: list[AppModel] = [AppModel()]
gpt: list[AppModel] = [AppModel(name="gpt-4o", label="OpenAIChat")]
mistral: list[AppModel] = [AppModel(name="mistral-large-latest", label="Mistral")]
deepseek: list[AppModel] = [AppModel(name="deepseek-chat", label="DeepSeek")]
xai: list[AppModel] = [AppModel(name="grok-3", label="xAI")]
ollama: list[AppModel] = []
@property
def all_models(self) -> list[AppModel]:
return self.gemini + self.ollama
return self.gemini + self.ollama + self.gpt + self.mistral + self.deepseek + self.xai
def validate_models(self) -> None:
"""
Validate the configured models for each provider.
"""
self.__validate_online_models(self.gemini, clazz=Gemini, key="GOOGLE_API_KEY")
self.__validate_online_models(self.gpt, clazz=OpenAIChat, key="OPENAI_API_KEY")
self.__validate_online_models(self.mistral, clazz=MistralChat, key="MISTRAL_API_KEY")
self.__validate_online_models(self.deepseek, clazz=DeepSeek, key="DEEPSEEK_API_KEY")
self.__validate_online_models(self.xai, clazz=xAI, key="XAI_API_KEY")
self.__validate_ollama_models()
def __validate_online_models(self, models: list[AppModel], clazz: type[Model], key: str | None = None) -> None:
"""
Validate models for online providers like Gemini.
Args:
models: list of AppModel instances to validate
clazz: class of the model (e.g. Gemini)
key: API key required for the provider (optional)
"""
if key and os.getenv(key) is None:
log.warning(f"No {key} set in environment variables for provider.")
models.clear()
return
for model in models:
model.model = clazz
def __validate_ollama_models(self) -> None:
"""
Validate models for the Ollama provider.
"""
try:
models_list = ollama.list()
availables = {model['model'] for model in models_list['models']}
not_availables: list[str] = []
for model in self.ollama:
if model.name in availables:
model.model = Ollama
else:
not_availables.append(model.name)
if not_availables:
log.warning(f"Ollama models not available: {not_availables}")
self.ollama = [model for model in self.ollama if model.model]
except Exception as e:
log.warning(f"Ollama is not running or not reachable: {e}")
class AgentsConfigs(BaseModel):
strategy: str = "Conservative"
@@ -118,7 +183,7 @@ class AppConfig(BaseModel):
super().__init__(*args, **kwargs)
self.set_logging_level()
self.validate_models()
self.models.validate_models()
self._initialized = True
def get_model_by_name(self, name: str) -> AppModel:
@@ -186,53 +251,3 @@ class AppConfig(BaseModel):
logger = logging.getLogger(logger_name)
logger.handlers.clear()
logger.propagate = True
def validate_models(self) -> None:
"""
Validate the configured models for each provider.
"""
self.__validate_online_models("gemini", clazz=Gemini, key="GOOGLE_API_KEY")
self.__validate_ollama_models()
def __validate_online_models(self, provider: str, clazz: type[Model], key: str | None = None) -> None:
"""
Validate models for online providers like Gemini.
Args:
provider: name of the provider (e.g. "gemini")
clazz: class of the model (e.g. Gemini)
key: API key required for the provider (optional)
"""
if getattr(self.models, provider) is None:
log.warning(f"No models configured for provider '{provider}'.")
models: list[AppModel] = getattr(self.models, provider)
if key and os.getenv(key) is None:
log.warning(f"No {key} set in environment variables for {provider}.")
models.clear()
return
for model in models:
model.model = clazz
def __validate_ollama_models(self) -> None:
"""
Validate models for the Ollama provider.
"""
try:
models_list = ollama.list()
availables = {model['model'] for model in models_list['models']}
not_availables: list[str] = []
for model in self.models.ollama:
if model.name in availables:
model.model = Ollama
else:
not_availables.append(model.name)
if not_availables:
log.warning(f"Ollama models not available: {not_availables}")
self.models.ollama = [model for model in self.models.ollama if model.model]
except Exception as e:
log.warning(f"Ollama is not running or not reachable: {e}")

View File

@@ -83,13 +83,15 @@ class ChatManager:
label="Modello da usare"
)
provider.change(fn=self.inputs.choose_team_leader, inputs=provider, outputs=None)
provider.value = self.inputs.team_leader_model.label
style = gr.Dropdown(
strategy = gr.Dropdown(
choices=self.inputs.list_strategies_names(),
type="index",
label="Stile di investimento"
)
style.change(fn=self.inputs.choose_strategy, inputs=style, outputs=None)
strategy.change(fn=self.inputs.choose_strategy, inputs=strategy, outputs=None)
strategy.value = self.inputs.strategy.label
chat = gr.ChatInterface(
fn=self.gradio_respond

View File

@@ -16,7 +16,7 @@ class TestChanWrapper:
assert len(posts) == 2
for post in posts:
assert post.title != ""
assert post.time != ""
assert re.match(r'\d{4}-\d{2}-\d{2}', post.time)
assert post.timestamp != ""
assert re.match(r'\d{4}-\d{2}-\d{2}', post.timestamp)
assert isinstance(post.comments, list)

View File

@@ -19,7 +19,7 @@ class TestRedditWrapper:
assert len(posts) == 2
for post in posts:
assert post.title != ""
assert re.match(r'\d{4}-\d{2}-\d{2}', post.time)
assert re.match(r'\d{4}-\d{2}-\d{2}', post.timestamp)
assert isinstance(post.comments, list)
assert len(post.comments) <= MAX_COMMENTS

View File

@@ -1,11 +1,13 @@
import os
import re
import pytest
from shutil import which
from app.api.social.x import XWrapper
@pytest.mark.social
@pytest.mark.api
@pytest.mark.skipif(not os.getenv("X_API_KEY"), reason="X_API_KEY not set in environment variables")
@pytest.mark.skipif(which('rettiwt') is None, reason="rettiwt not installed")
class TestXWrapper:
def test_initialization(self):
wrapper = XWrapper()
@@ -18,5 +20,5 @@ class TestXWrapper:
assert len(posts) == 2
for post in posts:
assert post.title != ""
assert re.match(r'\d{4}-\d{2}-\d{2}', post.time)
assert re.match(r'\d{4}-\d{2}-\d{2}', post.timestamp)
assert isinstance(post.comments, list)

View File

@@ -17,7 +17,7 @@ class TestSocialAPIsTool:
assert len(result) > 0
for post in result:
assert post.title is not None
assert post.time is not None
assert post.timestamp is not None
def test_social_api_tool_get_top__all_results(self):
tool = SocialAPIsTool()
@@ -27,4 +27,4 @@ class TestSocialAPIsTool:
for _provider, posts in result.items():
for post in posts:
assert post.title is not None
assert post.time is not None
assert post.timestamp is not None