first commit
This commit is contained in:
@@ -0,0 +1,372 @@
|
||||
"""Volatility analyzer service - analyzes price volatility using AI to detect leading signals."""
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
from openai import OpenAI
|
||||
|
||||
from src.config import get_settings
|
||||
from src.models.leading_signal import LeadingSignal, SignalType
|
||||
from src.services.price_monitor import VolatilityAlert
|
||||
from src.services.twitter_search import TwitterSearchService
|
||||
from src.prompts.volatility_analyzer import VolatilityAnalyzerPrompts
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Directory for storing leading signals dataset
|
||||
LEADING_SIGNALS_DIR = Path(__file__).parent.parent.parent / "leading_signals"
|
||||
|
||||
|
||||
class VolatilityAnalyzer:
|
||||
"""
|
||||
Analyzes price volatility events using LLM to detect "price leads news" signals.
|
||||
|
||||
Uses Tavily web search and Twitter to verify whether a price movement
|
||||
preceded public news, building a dataset of leading signals.
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
self.settings = get_settings()
|
||||
|
||||
# Configure OpenAI-compatible API client
|
||||
self.client = OpenAI(
|
||||
base_url=self.settings.llm_base_url,
|
||||
api_key=self.settings.llm_api_key,
|
||||
)
|
||||
|
||||
self.prompts = VolatilityAnalyzerPrompts()
|
||||
self.twitter_search = TwitterSearchService(api_key=self.settings.twitter_api_key)
|
||||
from src.services.web_search import WebSearchService
|
||||
self.web_search = WebSearchService(
|
||||
tavily_api_key=self.settings.tavily_api_key,
|
||||
serper_api_key=self.settings.serper_api_key,
|
||||
)
|
||||
|
||||
# Ensure storage directory exists
|
||||
LEADING_SIGNALS_DIR.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
def _extract_json_from_response(self, response: str) -> Optional[dict]:
|
||||
"""
|
||||
Extract JSON from LLM response.
|
||||
|
||||
Args:
|
||||
response: The LLM response text
|
||||
|
||||
Returns:
|
||||
Parsed JSON dict or None
|
||||
"""
|
||||
# Try to find JSON in code blocks
|
||||
json_pattern = r"```(?:json)?\s*([\s\S]*?)```"
|
||||
matches = re.findall(json_pattern, response)
|
||||
|
||||
for match in matches:
|
||||
try:
|
||||
return json.loads(match.strip())
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
|
||||
# Try to find raw JSON
|
||||
try:
|
||||
start = response.find("{")
|
||||
end = response.rfind("}") + 1
|
||||
if start >= 0 and end > start:
|
||||
return json.loads(response[start:end])
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
|
||||
return None
|
||||
|
||||
def _parse_signal_type(self, type_str: str) -> SignalType:
|
||||
"""Parse signal type string to enum."""
|
||||
try:
|
||||
return SignalType(type_str.upper())
|
||||
except ValueError:
|
||||
return SignalType.SPECULATION
|
||||
|
||||
def _store_leading_signal(self, signal: LeadingSignal) -> str:
|
||||
"""
|
||||
Store a leading signal to the dataset.
|
||||
|
||||
Args:
|
||||
signal: The leading signal to store
|
||||
|
||||
Returns:
|
||||
Path to the stored file
|
||||
"""
|
||||
# Create filename with timestamp and market info
|
||||
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
|
||||
market_slug = re.sub(r'[^\w\s-]', '', signal.market_question)[:40]
|
||||
market_slug = re.sub(r'\s+', '_', market_slug)
|
||||
|
||||
filename = f"{timestamp}_{signal.signal_type.value}_{market_slug}.json"
|
||||
filepath = LEADING_SIGNALS_DIR / filename
|
||||
|
||||
with open(filepath, 'w', encoding='utf-8') as f:
|
||||
json.dump(signal.to_dict(), f, ensure_ascii=False, indent=2)
|
||||
|
||||
return str(filepath)
|
||||
|
||||
def _store_all_signals_index(self, signal: LeadingSignal) -> None:
|
||||
"""
|
||||
Append signal to the master index file for easy querying.
|
||||
|
||||
Args:
|
||||
signal: The signal to append
|
||||
"""
|
||||
index_file = LEADING_SIGNALS_DIR / "signals_index.jsonl"
|
||||
|
||||
with open(index_file, 'a', encoding='utf-8') as f:
|
||||
f.write(json.dumps(signal.to_dict(), ensure_ascii=False) + "\n")
|
||||
|
||||
async def analyze_volatility(self, alert: VolatilityAlert) -> Optional[LeadingSignal]:
|
||||
"""
|
||||
Analyze a price volatility event to determine if it's a leading signal.
|
||||
|
||||
Args:
|
||||
alert: The volatility alert to analyze
|
||||
|
||||
Returns:
|
||||
LeadingSignal if analysis successful, None otherwise
|
||||
"""
|
||||
logger.info(
|
||||
f"Analyzing volatility: {alert.market_question[:50]}... "
|
||||
f"{alert.direction} {abs(alert.price_change_percent):.1%}"
|
||||
)
|
||||
|
||||
# Search web (Tavily) for news verification
|
||||
web_search_context = ""
|
||||
if self.web_search.is_available():
|
||||
logger.info(f"Searching web for: {alert.market_question[:50]}...")
|
||||
web_result = self.web_search.search_for_market(
|
||||
market_question=alert.market_question,
|
||||
max_results=5,
|
||||
)
|
||||
if web_result and "unavailable" not in web_result.lower():
|
||||
web_search_context = web_result
|
||||
logger.info("Web search (Tavily) completed")
|
||||
|
||||
# Search Twitter for social sentiment
|
||||
twitter_context = ""
|
||||
if self.twitter_search.is_available():
|
||||
logger.info(f"Searching Twitter for: {alert.market_question[:50]}...")
|
||||
twitter_result = self.twitter_search.search_for_market(
|
||||
market_question=alert.market_question,
|
||||
limit=10,
|
||||
)
|
||||
if twitter_result and "unavailable" not in twitter_result.lower():
|
||||
twitter_context = twitter_result
|
||||
logger.info("Twitter search completed")
|
||||
|
||||
# Build prompts
|
||||
system_prompt = self.prompts.system_prompt()
|
||||
user_prompt = self.prompts.analyze_volatility(
|
||||
market_question=alert.market_question,
|
||||
price_change_percent=alert.price_change_percent,
|
||||
direction=alert.direction,
|
||||
start_price=alert.start_price,
|
||||
end_price=alert.end_price,
|
||||
window_seconds=alert.window_seconds,
|
||||
detected_at=alert.detected_at,
|
||||
twitter_context=twitter_context,
|
||||
web_search_context=web_search_context,
|
||||
)
|
||||
|
||||
try:
|
||||
# Call LLM API
|
||||
response = self.client.chat.completions.create(
|
||||
model=self.settings.llm_model,
|
||||
messages=[
|
||||
{"role": "system", "content": system_prompt},
|
||||
{"role": "user", "content": user_prompt},
|
||||
],
|
||||
)
|
||||
|
||||
analysis_text = response.choices[0].message.content
|
||||
logger.debug(f"LLM response: {analysis_text[:500]}...")
|
||||
|
||||
# Extract JSON from response
|
||||
json_data = self._extract_json_from_response(analysis_text)
|
||||
|
||||
if not json_data:
|
||||
logger.warning("Could not parse LLM response as JSON")
|
||||
return None
|
||||
|
||||
# Create LeadingSignal from response
|
||||
signal_id = f"vol_{alert.market_id}_{int(datetime.now().timestamp())}"
|
||||
|
||||
signal = LeadingSignal(
|
||||
id=signal_id,
|
||||
market_id=alert.market_id,
|
||||
market_question=alert.market_question,
|
||||
price_change_percent=alert.price_change_percent,
|
||||
direction=alert.direction,
|
||||
start_price=alert.start_price,
|
||||
end_price=alert.end_price,
|
||||
window_seconds=alert.window_seconds,
|
||||
detected_at=datetime.utcnow().isoformat(),
|
||||
volatility_detected_at=alert.detected_at,
|
||||
signal_type=self._parse_signal_type(json_data.get("signal_type", "SPECULATION")),
|
||||
confidence=float(json_data.get("confidence", 0.0)),
|
||||
is_leading_signal=bool(json_data.get("is_leading_signal", False)),
|
||||
news_found=bool(json_data.get("news_found", False)),
|
||||
earliest_news_time=json_data.get("earliest_news_time"),
|
||||
key_news_headlines=json_data.get("key_news_headlines", []),
|
||||
earliest_social_time=json_data.get("earliest_social_time"),
|
||||
key_social_posts=json_data.get("key_social_posts", []),
|
||||
time_advantage_minutes=int(json_data.get("time_advantage_minutes", 0)),
|
||||
reasoning=str(json_data.get("reasoning", "")),
|
||||
potential_information_source=str(json_data.get("potential_information_source", "")),
|
||||
full_analysis=analysis_text,
|
||||
)
|
||||
|
||||
# Store the signal
|
||||
filepath = self._store_leading_signal(signal)
|
||||
self._store_all_signals_index(signal)
|
||||
|
||||
# Log result
|
||||
if signal.is_leading_signal:
|
||||
logger.warning(
|
||||
f"🚨 LEADING SIGNAL DETECTED: {alert.market_question[:50]}... "
|
||||
f"Time advantage: {signal.time_advantage_minutes} minutes"
|
||||
)
|
||||
else:
|
||||
logger.info(
|
||||
f"Volatility analyzed: {signal.signal_type.value} "
|
||||
f"(confidence: {signal.confidence:.1%})"
|
||||
)
|
||||
|
||||
logger.info(f"Signal stored: {filepath}")
|
||||
|
||||
return signal
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error analyzing volatility: {e}")
|
||||
return None
|
||||
|
||||
def format_signal_report(self, signal: LeadingSignal) -> str:
|
||||
"""
|
||||
Format a leading signal as a readable report.
|
||||
|
||||
Args:
|
||||
signal: The signal to format
|
||||
|
||||
Returns:
|
||||
Formatted report string
|
||||
"""
|
||||
direction_label = "Up" if signal.direction == "UP" else "Down"
|
||||
signal_type_label = {
|
||||
SignalType.LEADING_SIGNAL: "Leading Signal (Price Preceded News)",
|
||||
SignalType.NEWS_DRIVEN: "News-Driven",
|
||||
SignalType.SOCIAL_DRIVEN: "Social-Driven",
|
||||
SignalType.SPECULATION: "Speculative Volatility",
|
||||
}
|
||||
|
||||
news_headlines = "\n".join([f" - {h}" for h in signal.key_news_headlines]) or " None"
|
||||
social_posts = "\n".join([f" - {p}" for p in signal.key_social_posts]) or " None"
|
||||
|
||||
report = f"""
|
||||
{'='*70}
|
||||
# Price Volatility Analysis Report
|
||||
{'='*70}
|
||||
|
||||
**Analysis Time**: {signal.detected_at}
|
||||
|
||||
## Volatility Details
|
||||
|
||||
| Field | Details |
|
||||
|-------|---------|
|
||||
| **Market** | {signal.market_question} |
|
||||
| **Price Change** | {direction_label} {abs(signal.price_change_percent):.1%} |
|
||||
| **Start Price** | {signal.start_price:.2%} |
|
||||
| **End Price** | {signal.end_price:.2%} |
|
||||
| **Time Window** | {signal.window_seconds // 60} min |
|
||||
|
||||
{'='*70}
|
||||
## Analysis Results
|
||||
{'='*70}
|
||||
|
||||
| Field | Result |
|
||||
|-------|--------|
|
||||
| **Signal Type** | {signal_type_label.get(signal.signal_type, 'Unknown')} |
|
||||
| **Confidence** | {signal.confidence:.1%} |
|
||||
| **Is Leading Signal** | {'Yes' if signal.is_leading_signal else 'No'} |
|
||||
| **Time Advantage** | {signal.time_advantage_minutes} min |
|
||||
|
||||
**Earliest News Time**: {signal.earliest_news_time or 'N/A'}
|
||||
**Earliest Social Time**: {signal.earliest_social_time or 'N/A'}
|
||||
|
||||
## Key News
|
||||
{news_headlines}
|
||||
|
||||
## Key Social Posts
|
||||
{social_posts}
|
||||
|
||||
## Reasoning
|
||||
{signal.reasoning}
|
||||
|
||||
## Suspected Information Source
|
||||
{signal.potential_information_source or 'Unknown'}
|
||||
|
||||
{'='*70}
|
||||
{signal.full_analysis}
|
||||
{'='*70}
|
||||
"""
|
||||
return report
|
||||
|
||||
def get_leading_signals_stats(self) -> dict:
|
||||
"""
|
||||
Get statistics about collected leading signals.
|
||||
|
||||
Returns:
|
||||
Dictionary with stats
|
||||
"""
|
||||
index_file = LEADING_SIGNALS_DIR / "signals_index.jsonl"
|
||||
|
||||
if not index_file.exists():
|
||||
return {
|
||||
"total_signals": 0,
|
||||
"leading_signals": 0,
|
||||
"news_driven": 0,
|
||||
"social_driven": 0,
|
||||
"speculation": 0,
|
||||
}
|
||||
|
||||
stats = {
|
||||
"total_signals": 0,
|
||||
"leading_signals": 0,
|
||||
"news_driven": 0,
|
||||
"social_driven": 0,
|
||||
"speculation": 0,
|
||||
"avg_time_advantage_minutes": 0,
|
||||
}
|
||||
|
||||
time_advantages = []
|
||||
|
||||
try:
|
||||
with open(index_file, 'r', encoding='utf-8') as f:
|
||||
for line in f:
|
||||
if line.strip():
|
||||
data = json.loads(line)
|
||||
stats["total_signals"] += 1
|
||||
|
||||
signal_type = data.get("signal_type", "SPECULATION")
|
||||
if signal_type == "LEADING_SIGNAL":
|
||||
stats["leading_signals"] += 1
|
||||
time_advantages.append(data.get("time_advantage_minutes", 0))
|
||||
elif signal_type == "NEWS_DRIVEN":
|
||||
stats["news_driven"] += 1
|
||||
elif signal_type == "SOCIAL_DRIVEN":
|
||||
stats["social_driven"] += 1
|
||||
else:
|
||||
stats["speculation"] += 1
|
||||
|
||||
if time_advantages:
|
||||
stats["avg_time_advantage_minutes"] = sum(time_advantages) / len(time_advantages)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error reading signals index: {e}")
|
||||
|
||||
return stats
|
||||
Reference in New Issue
Block a user