Update to tiered market coverage (700+ markets), refactor anomaly detection and trade monitoring
- Expand market coverage from 50 trending to 700+ active markets with three volume tiers - Refactor anomaly detector with improved scoring logic - Simplify trade monitor architecture - Add tiered market fetching in market_fetcher - Update prompts, settings, and etherscan service - Remove requirements.txt (using other dependency management) - Update README to reflect new capabilities
This commit is contained in:
SII-leiyu
+9
-11
@@ -15,13 +15,6 @@ class Settings(BaseSettings):
|
||||
gemini_api_key: str = Field(default="", alias="GEMINI_API_KEY")
|
||||
llm_base_url: str = Field(default="http://apicz.boyuerichdata.com/v1/", alias="LLM_BASE_URL")
|
||||
|
||||
# Trade data API mode: "internal" (private API) or "official" (Polymarket data-api)
|
||||
trade_api_mode: str = Field(default="official", alias="TRADE_API_MODE")
|
||||
|
||||
# Internal trade data API (only used when TRADE_API_MODE=internal)
|
||||
internal_api_url: str = Field(default="http://103.197.25.170:18088", alias="INTERNAL_API_URL")
|
||||
internal_api_key: str = Field(default="", alias="INTERNAL_API_KEY")
|
||||
|
||||
# Twitter API (for social sentiment search)
|
||||
twitter_api_key: str = Field(default="", alias="TWITTER_API_KEY")
|
||||
|
||||
@@ -49,8 +42,6 @@ class Settings(BaseSettings):
|
||||
telegram_session_string: str = Field(default="", alias="TELEGRAM_SESSION_STRING")
|
||||
telegram_channels: str = Field(default="", alias="TELEGRAM_CHANNELS")
|
||||
|
||||
# Polygon Wallet
|
||||
polygon_wallet_private_key: str = Field(default="", alias="POLYGON_WALLET_PRIVATE_KEY")
|
||||
|
||||
# MongoDB
|
||||
mongodb_uri: str = Field(default="mongodb://localhost:27017/whale_watcher", alias="MONGODB_URI")
|
||||
@@ -67,12 +58,19 @@ class Settings(BaseSettings):
|
||||
fetch_interval_seconds: int = Field(default=15, alias="FETCH_INTERVAL_SECONDS")
|
||||
trending_markets_limit: int = Field(default=50, alias="TRENDING_MARKETS_LIMIT")
|
||||
|
||||
# Tiered market monitoring (full-coverage mode)
|
||||
full_market_scan: bool = Field(default=True, alias="FULL_MARKET_SCAN")
|
||||
tier1_volume_min: float = Field(default=500_000, alias="TIER1_VOLUME_MIN")
|
||||
tier2_volume_min: float = Field(default=10_000, alias="TIER2_VOLUME_MIN")
|
||||
tier3_volume_min: float = Field(default=1_000, alias="TIER3_VOLUME_MIN")
|
||||
tier1_poll_interval: int = Field(default=15, alias="TIER1_POLL_INTERVAL")
|
||||
tier2_poll_interval: int = Field(default=60, alias="TIER2_POLL_INTERVAL")
|
||||
tier3_poll_interval: int = Field(default=300, alias="TIER3_POLL_INTERVAL")
|
||||
|
||||
# LLM Settings
|
||||
llm_model: str = Field(default="gemini-3-flash-preview", alias="LLM_MODEL")
|
||||
llm_temperature: float = Field(default=0.0, alias="LLM_TEMPERATURE")
|
||||
|
||||
# Trade Execution
|
||||
enable_trade_execution: bool = Field(default=False, alias="ENABLE_TRADE_EXECUTION")
|
||||
|
||||
# Email notification
|
||||
email_smtp_server: str = Field(default="smtp.qq.com", alias="EMAIL_SMTP_SERVER")
|
||||
|
||||
+44
-3
@@ -230,17 +230,58 @@ class WhaleWatcher:
|
||||
|
||||
async def refresh_markets(self) -> None:
|
||||
"""Fetch and update the list of monitored markets."""
|
||||
if self.settings.full_market_scan:
|
||||
await self._refresh_markets_tiered()
|
||||
else:
|
||||
await self._refresh_markets_legacy()
|
||||
|
||||
async def _refresh_markets_tiered(self) -> None:
|
||||
"""Full-coverage tiered monitoring (mirrors options flow passive approach)."""
|
||||
logger.info("Fetching ALL active markets for tiered monitoring...")
|
||||
|
||||
tiers = self.market_fetcher.get_tiered_markets()
|
||||
|
||||
# Also merge token launch markets into appropriate tiers
|
||||
existing_ids = set()
|
||||
for tier_markets in tiers.values():
|
||||
for tm in tier_markets:
|
||||
existing_ids.add(tm.market.id)
|
||||
|
||||
token_markets = self.market_fetcher.get_token_launch_markets()
|
||||
token_added = 0
|
||||
for tm in token_markets:
|
||||
if tm.market.id not in existing_ids:
|
||||
# Assign to tier based on volume
|
||||
vol = tm.volume_24hr
|
||||
if vol >= self.settings.tier1_volume_min:
|
||||
tiers["tier1"].append(tm)
|
||||
elif vol >= self.settings.tier2_volume_min:
|
||||
tiers["tier2"].append(tm)
|
||||
else:
|
||||
tiers["tier3"].append(tm)
|
||||
existing_ids.add(tm.market.id)
|
||||
token_added += 1
|
||||
|
||||
if token_added:
|
||||
logger.info(f"Added {token_added} token launch markets to tiers")
|
||||
|
||||
total = sum(len(v) for v in tiers.values())
|
||||
if total > 0:
|
||||
self.trade_monitor.set_tiered_markets(tiers)
|
||||
logger.info(f"Tiered monitoring active: {total} markets total")
|
||||
else:
|
||||
logger.error("Failed to fetch any markets")
|
||||
|
||||
async def _refresh_markets_legacy(self) -> None:
|
||||
"""Original Top-N trending markets mode."""
|
||||
logger.info("Fetching trending markets...")
|
||||
|
||||
trending_markets = self.market_fetcher.get_trending_markets(
|
||||
limit=self.settings.trending_markets_limit
|
||||
)
|
||||
|
||||
# Additionally scan for specialized market categories
|
||||
# that may not be in the top trending list
|
||||
existing_ids = {tm.market.id for tm in trending_markets}
|
||||
|
||||
# 1. Token launch / crypto project markets
|
||||
token_markets = self.market_fetcher.get_token_launch_markets()
|
||||
token_added = 0
|
||||
for tm in token_markets:
|
||||
|
||||
@@ -59,14 +59,14 @@ You can call the following tools to obtain real-time information (all results ar
|
||||
**Tool usage principles**:
|
||||
- Based on market type and trade characteristics, decide which tools to call
|
||||
- You may call one, multiple, or zero tools
|
||||
- You may call the same tool multiple times with different keywords
|
||||
- If trade size is very large or information asymmetry suspicion is high, search more aggressively
|
||||
- **You have a maximum of 3 tool-call rounds. Budget wisely: use Round 1 for broad search (web + twitter + telegram in parallel), Round 2 for targeted follow-up if needed, then produce your final analysis. Do NOT use all rounds just searching — reserve capacity for your final answer.**
|
||||
- Do NOT call the same tool (e.g. search_web) more than 3 times total across all rounds
|
||||
- If the first search already covers the topic well, stop searching and analyze
|
||||
|
||||
**Tool collaboration and cross-verification (important)**:
|
||||
- Information from different tools must be **cross-verified** — do not draw conclusions from a single source. Example: if web search finds a policy rumor, verify with Twitter for public reaction and corroborate with economic data
|
||||
- If you discover new leads or keywords while using one tool, **immediately call other tools to follow up**. Example: if news search reveals an official's resignation, search for that person's name for more details and check Twitter for unreported information
|
||||
- When multiple tools return **contradictory results**, explicitly note the discrepancy and lower confidence — do not cherry-pick
|
||||
- Encourage "search chain" investigation: first-round search → discover leads → targeted second-round → deep third-round, progressing layer by layer rather than skimming the surface
|
||||
- Information from different tools should be **cross-verified** — but 2-3 sources are sufficient, do not over-search
|
||||
- When multiple tools return **contradictory results**, explicitly note the discrepancy and lower confidence
|
||||
- Prefer breadth (web + twitter + domain-specific tool) over depth (web × 8 with slightly different keywords)
|
||||
|
||||
## Polymarket Trading Mechanics
|
||||
|
||||
|
||||
+175
-138
@@ -1,9 +1,21 @@
|
||||
"""Anomaly detection service - multi-dimensional scoring for whale trades."""
|
||||
"""
|
||||
Anomaly detection service — confidence scoring for whale trades.
|
||||
|
||||
Mirrors the options flow confidence scoring from llm-trading-agent:
|
||||
Base confidence: 0.50
|
||||
+ Premium-to-threshold ratio: +0.20 (sqrt-scaled by market liquidity)
|
||||
+ Signal cleanliness (ask ratio): +0.10
|
||||
+ Volume/OI equivalent (depth ratio): +0.10
|
||||
+ Alert rule / cluster tier: +0.10
|
||||
|
||||
Total max = 1.0. Pre-filter threshold = 0.60 (matches options flow pipeline).
|
||||
"""
|
||||
import logging
|
||||
import math
|
||||
import time
|
||||
from collections import defaultdict, deque
|
||||
from datetime import datetime
|
||||
from typing import Dict, List, Optional, Tuple
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
from src.config import get_settings
|
||||
from src.models.trade import WhaleTrade, TradeActivity, TraderHistory
|
||||
@@ -15,51 +27,29 @@ logger = logging.getLogger(__name__)
|
||||
|
||||
class AnomalyDetector:
|
||||
"""
|
||||
Multi-dimensional anomaly detection for whale trades.
|
||||
Confidence scoring for whale trades, mirroring options flow pipeline.
|
||||
|
||||
Scoring dimensions:
|
||||
1. Size relative to market (trade vs market 24h volume)
|
||||
2. Price uncertainty (closer to 0.5 = more uncertain = more interesting)
|
||||
3. Time-of-day (off-peak hours = more suspicious)
|
||||
4. Trader deviation (trade size vs trader's historical average)
|
||||
5. Cluster signal (multiple same-direction trades in short window)
|
||||
5-factor scoring (same structure as OptionsFlowSignalProvider._calculate_confidence):
|
||||
1. Base confidence: 0.50
|
||||
2. Premium-to-threshold ratio: +0.20 (trade_size vs dynamic threshold)
|
||||
3. Signal cleanliness: +0.10 (conviction / price displacement)
|
||||
4. Depth ratio: +0.10 (trade_size vs market liquidity, like Volume/OI)
|
||||
5. Cluster tier: +0.10 (repeated same-direction trades, like alert_rule)
|
||||
"""
|
||||
|
||||
# --- Time-of-day weights (US Eastern Time) ---
|
||||
# Polymarket is US-dominated, so we use ET to judge trading hour anomaly.
|
||||
# Higher weight = more unusual trading hour = more suspicious.
|
||||
_ET_HOUR_WEIGHTS = {
|
||||
# ET 0-5 (midnight to 5am) — very unusual, most suspicious
|
||||
0: 0.6, 1: 0.7, 2: 0.8, 3: 0.9, 4: 0.8, 5: 0.6,
|
||||
# ET 6-8 (early morning) — some early traders
|
||||
6: 0.4, 7: 0.3, 8: 0.2,
|
||||
# ET 9-17 (US business hours) — peak activity, least suspicious
|
||||
9: 0.1, 10: 0.0, 11: 0.0, 12: 0.0, 13: 0.0,
|
||||
14: 0.0, 15: 0.0, 16: 0.0, 17: 0.1,
|
||||
# ET 18-20 (evening) — moderate
|
||||
18: 0.2, 19: 0.2, 20: 0.3,
|
||||
# ET 21-23 (late night) — unusual
|
||||
21: 0.4, 22: 0.5, 23: 0.5,
|
||||
}
|
||||
# UTC offset for US Eastern: -5 (EST) or -4 (EDT).
|
||||
# Use -4 as default (EDT covers ~Mar-Nov, most of the year).
|
||||
_ET_UTC_OFFSET = -4
|
||||
|
||||
# Cluster detection: track recent trades per market
|
||||
# Key: market_id, Value: deque of (timestamp, side, usdc_size)
|
||||
# Cluster detection
|
||||
_CLUSTER_WINDOW_SECONDS = 300 # 5 minutes
|
||||
_CLUSTER_MIN_COUNT = 3 # minimum trades for cluster signal
|
||||
_CLUSTER_MIN_COUNT = 3
|
||||
|
||||
def __init__(self):
|
||||
self.settings = get_settings()
|
||||
self.trader_profiler = TraderProfiler()
|
||||
# Recent trades for cluster detection: market_id -> deque
|
||||
self._recent_trades: Dict[str, deque] = defaultdict(
|
||||
lambda: deque(maxlen=50)
|
||||
)
|
||||
|
||||
# ================================================================
|
||||
# Core scoring
|
||||
# Core scoring — mirrors OptionsFlowSignalProvider._calculate_confidence
|
||||
# ================================================================
|
||||
|
||||
def get_anomaly_score(
|
||||
@@ -70,81 +60,38 @@ class AnomalyDetector:
|
||||
market_id: str = "",
|
||||
) -> Tuple[float, dict]:
|
||||
"""
|
||||
Calculate multi-dimensional anomaly score.
|
||||
Calculate confidence score using options-flow-style 5-factor model.
|
||||
|
||||
Returns:
|
||||
(total_score, breakdown_dict) where breakdown has per-dimension scores.
|
||||
(total_score, breakdown_dict) — total in [0.0, 1.0].
|
||||
"""
|
||||
breakdown = {}
|
||||
|
||||
# --- 1. Size score (absolute) ---
|
||||
# $5k=0.1, $20k=0.25, $50k=0.4, $100k+=0.5
|
||||
raw_size = min(0.5, 0.1 + (activity.usdc_size - 5000) / 250000)
|
||||
breakdown["size_abs"] = max(0.0, raw_size)
|
||||
# --- 1. Base confidence: 0.50 ---
|
||||
breakdown["base"] = 0.50
|
||||
|
||||
# --- 2. Size relative to market volume ---
|
||||
if market and market.volume_24hr > 0:
|
||||
# What fraction of 24h volume is this single trade?
|
||||
ratio = activity.usdc_size / market.volume_24hr
|
||||
# ratio 0.001=noise, 0.01=notable, 0.05=significant, 0.1+=massive
|
||||
rel_score = min(0.3, ratio * 6.0) # 0.05 ratio -> 0.3
|
||||
breakdown["size_relative"] = rel_score
|
||||
else:
|
||||
breakdown["size_relative"] = 0.15 # unknown market volume, use neutral
|
||||
# --- 2. Premium-to-threshold ratio: +0.20 max ---
|
||||
# Mirrors _premium_ratio_bonus: min(0.20, (sqrt(ratio) - 1) * 0.20)
|
||||
# where ratio = trade_size / dynamic_threshold
|
||||
# dynamic_threshold = base_size * sqrt(market_volume / baseline_volume)
|
||||
breakdown["premium_ratio"] = self._premium_ratio_bonus(activity, market)
|
||||
|
||||
# --- 3. Price uncertainty ---
|
||||
# Price is taker's buy price (no normalization).
|
||||
# Lower price = more uncertain/risky bet = more interesting.
|
||||
# 0.5 -> 0.2, 0.3/0.7 -> 0.1, 0.1/0.9 -> 0.0
|
||||
dist = abs(activity.price - 0.5)
|
||||
if dist <= 0.3:
|
||||
price_score = 0.2 * (1 - dist / 0.3)
|
||||
else:
|
||||
price_score = 0.0
|
||||
breakdown["price_uncertainty"] = price_score
|
||||
# --- 3. Signal cleanliness (conviction): +0.10 max ---
|
||||
# Mirrors _ask_ratio_bonus: measures taker aggressiveness.
|
||||
# In options: ASK-side ratio > 0.8 = full bonus.
|
||||
# In Polymarket: price displacement from market mid = conviction.
|
||||
breakdown["signal_clean"] = self._signal_cleanliness_bonus(activity, market)
|
||||
|
||||
# --- 4. Time-of-day ---
|
||||
utc_hour = datetime.utcfromtimestamp(activity.timestamp).hour
|
||||
et_hour = (utc_hour + self._ET_UTC_OFFSET) % 24
|
||||
breakdown["time_of_day"] = self._ET_HOUR_WEIGHTS.get(et_hour, 0.1) * 0.15
|
||||
# --- 4. Depth ratio (like Volume/OI): +0.10 max ---
|
||||
# Mirrors _volume_oi_bonus: trade_size / market_liquidity.
|
||||
# High ratio = new significant positioning.
|
||||
breakdown["depth_ratio"] = self._depth_ratio_bonus(activity, market)
|
||||
|
||||
# --- 5. Trader deviation ---
|
||||
if trader_history and trader_history.avg_trade_size > 0:
|
||||
# How many X of their average is this trade?
|
||||
multiple = activity.usdc_size / trader_history.avg_trade_size
|
||||
# 1x=normal, 2x=notable, 5x=very unusual, 10x+=extreme
|
||||
if multiple >= 5:
|
||||
deviation_score = 0.15
|
||||
elif multiple >= 2:
|
||||
deviation_score = 0.05 + (multiple - 2) / 3 * 0.10
|
||||
else:
|
||||
deviation_score = 0.0
|
||||
breakdown["trader_deviation"] = deviation_score
|
||||
else:
|
||||
# Unknown trader history — slightly suspicious
|
||||
breakdown["trader_deviation"] = 0.05
|
||||
|
||||
# --- 6. Cluster signal ---
|
||||
cluster_score = self._get_cluster_score(activity, market_id=market_id)
|
||||
breakdown["cluster"] = cluster_score
|
||||
|
||||
# --- 7. Niche market bonus ---
|
||||
# Small/niche markets have higher information asymmetry value.
|
||||
# Large political/macro markets (volume > $5M/day) are noisy;
|
||||
# small markets (< $500k/day) are where insider signals matter most.
|
||||
if market and market.volume_24hr > 0:
|
||||
vol = market.volume_24hr
|
||||
if vol < 100_000:
|
||||
niche_score = 0.15 # very niche
|
||||
elif vol < 500_000:
|
||||
niche_score = 0.10
|
||||
elif vol < 2_000_000:
|
||||
niche_score = 0.05
|
||||
else:
|
||||
niche_score = 0.0 # large/macro market, no bonus
|
||||
breakdown["niche_market"] = niche_score
|
||||
else:
|
||||
breakdown["niche_market"] = 0.05
|
||||
# --- 5. Cluster tier (like alert_rule): +0.10 max ---
|
||||
# Mirrors _alert_rule_bonus: repeated same-direction activity.
|
||||
# In options: RepeatedHitsAscendingFill = 0.10, RepeatedHits = 0.07.
|
||||
# In Polymarket: multiple same-direction trades in 5-min window.
|
||||
breakdown["cluster_tier"] = self._cluster_tier_bonus(activity, market_id)
|
||||
|
||||
# --- Total ---
|
||||
total = sum(breakdown.values())
|
||||
@@ -152,22 +99,113 @@ class AnomalyDetector:
|
||||
|
||||
return total, breakdown
|
||||
|
||||
def record_trade(self, activity: TradeActivity, market_id: str):
|
||||
"""Record a trade for cluster detection. Call for every trade, not just whales."""
|
||||
self._recent_trades[market_id].append((
|
||||
activity.timestamp,
|
||||
activity.side,
|
||||
activity.usdc_size,
|
||||
))
|
||||
|
||||
def _get_cluster_score(self, activity: TradeActivity, market_id: str = "") -> float:
|
||||
@staticmethod
|
||||
def _premium_ratio_bonus(activity: TradeActivity, market: Optional[Market]) -> float:
|
||||
"""
|
||||
Check if there are multiple same-direction trades in a short window.
|
||||
Trade size vs dynamic threshold, sqrt-scaled.
|
||||
|
||||
A cluster of BUY or SELL in the same market in 5 minutes suggests
|
||||
coordinated or informed trading.
|
||||
Mirrors OptionsFlowSignalProvider._premium_ratio_bonus:
|
||||
threshold = 100K * sqrt(market_cap / 40B)
|
||||
bonus = min(0.20, (sqrt(premium / threshold) - 1) * 0.20)
|
||||
|
||||
Polymarket mapping:
|
||||
threshold = base_size * sqrt(market_volume / baseline_volume)
|
||||
base_size = $5,000, baseline_volume = $1,000,000
|
||||
"""
|
||||
max_bonus = 0.20
|
||||
trade_size = activity.usdc_size
|
||||
|
||||
if trade_size <= 0:
|
||||
return 0.0
|
||||
|
||||
# Dynamic threshold based on market volume (like market-cap scaling)
|
||||
base_size = 10_000.0
|
||||
baseline_volume = 1_000_000.0
|
||||
|
||||
if market and market.volume > 0:
|
||||
threshold = base_size * math.sqrt(market.volume / baseline_volume)
|
||||
threshold = max(5_000.0, threshold) # floor $5K
|
||||
else:
|
||||
threshold = base_size
|
||||
|
||||
ratio = trade_size / threshold
|
||||
if ratio <= 1.0:
|
||||
return 0.0
|
||||
|
||||
bonus = (math.sqrt(ratio) - 1) * max_bonus
|
||||
return min(max_bonus, max(0.0, bonus))
|
||||
|
||||
@staticmethod
|
||||
def _signal_cleanliness_bonus(activity: TradeActivity, market: Optional[Market]) -> float:
|
||||
"""
|
||||
Taker conviction / price displacement from market mid.
|
||||
|
||||
Mirrors _ask_ratio_bonus logic:
|
||||
ASK ratio > 0.8 → 0.10 (full), > 0.6 → 0.05 (half)
|
||||
|
||||
Polymarket mapping:
|
||||
A buyer paying significantly above market mid = aggressive taker (like ASK-side).
|
||||
Displacement > 5% → 0.10, > 2% → 0.05.
|
||||
"""
|
||||
if not market or not market.outcome_prices:
|
||||
return 0.0
|
||||
|
||||
# Get market mid price for the outcome the trader bought
|
||||
if activity.outcome == "Yes":
|
||||
market_mid = market.outcome_prices[0]
|
||||
elif len(market.outcome_prices) > 1:
|
||||
market_mid = market.outcome_prices[1]
|
||||
else:
|
||||
market_mid = 1.0 - market.outcome_prices[0]
|
||||
|
||||
displacement = activity.price - market_mid
|
||||
|
||||
if displacement > 0.05:
|
||||
return 0.10 # strong conviction (like ASK ratio > 0.8)
|
||||
if displacement > 0.02:
|
||||
return 0.05 # moderate conviction (like ASK ratio > 0.6)
|
||||
return 0.0
|
||||
|
||||
@staticmethod
|
||||
def _depth_ratio_bonus(activity: TradeActivity, market: Optional[Market]) -> float:
|
||||
"""
|
||||
Trade size vs market liquidity (like Volume/OI).
|
||||
|
||||
Mirrors _volume_oi_bonus:
|
||||
V/OI > 3.0 → 0.10, > 1.5 → 0.07, > 1.0 → 0.03
|
||||
|
||||
Polymarket mapping:
|
||||
depth_ratio = trade_size / market_liquidity
|
||||
> 0.10 → 0.10, > 0.05 → 0.07, > 0.02 → 0.03
|
||||
"""
|
||||
if not market or not market.liquidity or market.liquidity <= 0:
|
||||
return 0.0
|
||||
|
||||
ratio = activity.usdc_size / market.liquidity
|
||||
|
||||
if ratio > 0.10:
|
||||
return 0.10
|
||||
if ratio > 0.05:
|
||||
return 0.07
|
||||
if ratio > 0.02:
|
||||
return 0.03
|
||||
return 0.0
|
||||
|
||||
def _cluster_tier_bonus(self, activity: TradeActivity, market_id: str = "") -> float:
|
||||
"""
|
||||
Cluster of same-direction trades in short window (like alert_rule tiers).
|
||||
|
||||
Mirrors _alert_rule_bonus tier structure:
|
||||
RepeatedHitsAscendingFill → 0.10
|
||||
RepeatedHits → 0.07
|
||||
SweepsFollowedByFloor → 0.05
|
||||
Single sweep → 0.03
|
||||
|
||||
Polymarket mapping:
|
||||
5+ same-direction trades in 5min → 0.10
|
||||
3-4 trades → 0.07
|
||||
2 trades with large volume → 0.03
|
||||
"""
|
||||
# Use the same market_id key as record_trade()
|
||||
key = market_id or activity.condition_id
|
||||
recent = self._recent_trades.get(key)
|
||||
if not recent:
|
||||
@@ -176,7 +214,6 @@ class AnomalyDetector:
|
||||
now = activity.timestamp
|
||||
cutoff = now - self._CLUSTER_WINDOW_SECONDS
|
||||
|
||||
# Count same-direction trades in window
|
||||
same_dir_count = 0
|
||||
same_dir_volume = 0.0
|
||||
for ts, side, size in recent:
|
||||
@@ -184,14 +221,22 @@ class AnomalyDetector:
|
||||
same_dir_count += 1
|
||||
same_dir_volume += size
|
||||
|
||||
if same_dir_count >= 5:
|
||||
return 0.10
|
||||
if same_dir_count >= self._CLUSTER_MIN_COUNT:
|
||||
# 3 trades = 0.05, 5+ = 0.10, volume also matters
|
||||
count_score = min(0.10, 0.02 * same_dir_count)
|
||||
vol_bonus = min(0.05, same_dir_volume / 500000)
|
||||
return count_score + vol_bonus
|
||||
|
||||
return 0.07
|
||||
if same_dir_count >= 2 and same_dir_volume > 20_000:
|
||||
return 0.03
|
||||
return 0.0
|
||||
|
||||
def record_trade(self, activity: TradeActivity, market_id: str):
|
||||
"""Record a trade for cluster detection. Call for every trade, not just whales."""
|
||||
self._recent_trades[market_id].append((
|
||||
activity.timestamp,
|
||||
activity.side,
|
||||
activity.usdc_size,
|
||||
))
|
||||
|
||||
# ================================================================
|
||||
# Pre-filter (before LLM)
|
||||
# ================================================================
|
||||
@@ -202,13 +247,13 @@ class AnomalyDetector:
|
||||
market: Optional[Market] = None,
|
||||
trader_history: Optional[TraderHistory] = None,
|
||||
market_id: str = "",
|
||||
min_score: float = 0.40,
|
||||
min_score: float = 0.65,
|
||||
) -> Tuple[bool, float, dict]:
|
||||
"""
|
||||
Decide whether a whale trade warrants LLM analysis.
|
||||
|
||||
Returns:
|
||||
(should_analyze, score, breakdown)
|
||||
Threshold 0.65: requires at least base (0.50) + one strong factor
|
||||
to trigger LLM analysis.
|
||||
"""
|
||||
score, breakdown = self.get_anomaly_score(
|
||||
activity, market, trader_history, market_id=market_id,
|
||||
@@ -230,9 +275,9 @@ class AnomalyDetector:
|
||||
def filter_whale_trades(
|
||||
self,
|
||||
trades: List[WhaleTrade],
|
||||
min_score: float = 0.5,
|
||||
min_score: float = 0.65,
|
||||
) -> List[WhaleTrade]:
|
||||
"""Filter whale trades by anomaly score."""
|
||||
"""Filter whale trades by confidence score."""
|
||||
filtered = []
|
||||
for trade in trades:
|
||||
score, _ = self.get_anomaly_score(trade.trade)
|
||||
@@ -248,16 +293,13 @@ class AnomalyDetector:
|
||||
"""Analyze the context of a whale trade for LLM input."""
|
||||
trade = whale_trade.trade
|
||||
|
||||
# Direction interpretation (only BUY trades, no normalization)
|
||||
if trade.outcome == "Yes":
|
||||
direction_meaning = f"Trader bought Yes Token @ {trade.price:.4f} — Bullish (believes event will occur)"
|
||||
else:
|
||||
direction_meaning = f"Trader bought No Token @ {trade.price:.4f} — Bearish (believes event will NOT occur)"
|
||||
|
||||
# Buy price directly reflects taker's conviction — lower price = higher odds bet
|
||||
implied_prob = trade.price
|
||||
|
||||
# Market state
|
||||
market_state = "uncertain"
|
||||
if whale_trade.market_outcome_prices:
|
||||
max_price = max(whale_trade.market_outcome_prices)
|
||||
@@ -266,7 +308,6 @@ class AnomalyDetector:
|
||||
elif max_price < 0.6:
|
||||
market_state = "highly uncertain"
|
||||
|
||||
# Multi-dimensional anomaly score
|
||||
score, breakdown = self.get_anomaly_score(trade)
|
||||
|
||||
return {
|
||||
@@ -289,12 +330,10 @@ class AnomalyDetector:
|
||||
context = self.analyze_trade_context(whale_trade)
|
||||
trade = whale_trade.trade
|
||||
|
||||
# Build outcome prices string
|
||||
prices_str = ""
|
||||
for outcome, price in zip(context["market_outcomes"], context["current_prices"]):
|
||||
prices_str += f" - {outcome}: {price:.2%}\n"
|
||||
|
||||
# Trader profile
|
||||
trader_profile = self.trader_profiler.generate_profile(
|
||||
wallet_address=trade.proxy_wallet or "Unknown",
|
||||
ranking=whale_trade.trader_ranking,
|
||||
@@ -302,15 +341,13 @@ class AnomalyDetector:
|
||||
)
|
||||
trader_profile_str = self.trader_profiler.format_profile_for_llm(trader_profile)
|
||||
|
||||
# Anomaly breakdown string
|
||||
bd = context["anomaly_breakdown"]
|
||||
breakdown_str = (
|
||||
f" Absolute size: {bd.get('size_abs', 0):.2f} | "
|
||||
f"Relative to market: {bd.get('size_relative', 0):.2f} | "
|
||||
f"Price uncertainty: {bd.get('price_uncertainty', 0):.2f} | "
|
||||
f"Time of day: {bd.get('time_of_day', 0):.2f} | "
|
||||
f"Trader deviation: {bd.get('trader_deviation', 0):.2f} | "
|
||||
f"Cluster signal: {bd.get('cluster', 0):.2f}"
|
||||
f" Base: {bd.get('base', 0):.2f} | "
|
||||
f"Premium ratio: {bd.get('premium_ratio', 0):.2f} | "
|
||||
f"Signal clean: {bd.get('signal_clean', 0):.2f} | "
|
||||
f"Depth ratio: {bd.get('depth_ratio', 0):.2f} | "
|
||||
f"Cluster tier: {bd.get('cluster_tier', 0):.2f}"
|
||||
)
|
||||
|
||||
return f"""
|
||||
@@ -323,7 +360,7 @@ class AnomalyDetector:
|
||||
- **Trade time**: {datetime.fromtimestamp(trade.timestamp).strftime('%Y-%m-%d %H:%M:%S UTC')}
|
||||
- **Trader wallet**: {trade.proxy_wallet or 'Unknown'}
|
||||
|
||||
### Anomaly Score
|
||||
### Confidence Score
|
||||
- **Overall score**: {context['anomaly_score']:.2f}/1.00
|
||||
- **Score breakdown**:
|
||||
{breakdown_str}
|
||||
|
||||
+36
-17
@@ -7,19 +7,25 @@ import httpx
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
ETHERSCAN_API = "https://api.etherscan.io/api"
|
||||
ETHERSCAN_API_V2 = "https://api.etherscan.io/v2/api"
|
||||
|
||||
# Well-known ERC-20 token contracts on Ethereum mainnet
|
||||
# Default chain: Polygon (137) where Polymarket operates.
|
||||
# Ethereum mainnet = 1, can be overridden per-call.
|
||||
DEFAULT_CHAIN_ID = 137
|
||||
|
||||
# Well-known ERC-20 token contracts on Polygon
|
||||
TOKEN_CONTRACTS = {
|
||||
"USDC": "0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48",
|
||||
"USDT": "0xdac17f958d2ee523a2206206994597c13d831ec7",
|
||||
"WETH": "0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2",
|
||||
"DAI": "0x6b175474e89094c44da98b954eedeac495271d0f",
|
||||
"USDC": "0x3c499c542cef5e3811e1192ce70d8cc03d5c3359", # USDC native on Polygon
|
||||
"USDC.e": "0x2791bca1f2de4661ed88a30c99a7a9449aa84174", # USDC.e (bridged) on Polygon
|
||||
"USDT": "0xc2132d05d31c914a87c6611c10748aeb04b58e8f",
|
||||
"WETH": "0x7ceb23fd6bc0add59e62ac25578270cff1b9f619",
|
||||
"DAI": "0x8f3cf7ad23cd3cadbd9735aff958023239c6a063",
|
||||
}
|
||||
|
||||
# Decimals per token (used for converting raw amounts)
|
||||
TOKEN_DECIMALS = {
|
||||
"USDC": 6,
|
||||
"USDC.e": 6,
|
||||
"USDT": 6,
|
||||
"WETH": 18,
|
||||
"DAI": 18,
|
||||
@@ -65,10 +71,11 @@ class EtherscanService:
|
||||
def is_available(self) -> bool:
|
||||
return bool(self.api_key and self.api_key.strip())
|
||||
|
||||
def _get(self, params: dict) -> dict:
|
||||
"""Make authenticated GET request to Etherscan API."""
|
||||
def _get(self, params: dict, chain_id: int = DEFAULT_CHAIN_ID) -> dict:
|
||||
"""Make authenticated GET request to Etherscan V2 API."""
|
||||
params["apikey"] = self.api_key
|
||||
resp = self._client.get(ETHERSCAN_API, params=params)
|
||||
params["chainid"] = chain_id
|
||||
resp = self._client.get(ETHERSCAN_API_V2, params=params)
|
||||
resp.raise_for_status()
|
||||
return resp.json()
|
||||
|
||||
@@ -111,18 +118,30 @@ class EtherscanService:
|
||||
|
||||
# Filter by token if not "ALL"
|
||||
if token_upper != "ALL":
|
||||
contract = TOKEN_CONTRACTS.get(token_upper, "").lower()
|
||||
if contract:
|
||||
# For USDC, match both native and bridged (USDC.e) contracts
|
||||
if token_upper == "USDC":
|
||||
allowed = {
|
||||
TOKEN_CONTRACTS.get("USDC", "").lower(),
|
||||
TOKEN_CONTRACTS.get("USDC.e", "").lower(),
|
||||
}
|
||||
allowed.discard("")
|
||||
transfers = [
|
||||
tx for tx in transfers
|
||||
if tx.get("contractAddress", "").lower() == contract
|
||||
if tx.get("contractAddress", "").lower() in allowed
|
||||
]
|
||||
else:
|
||||
# Try matching by symbol in the response
|
||||
transfers = [
|
||||
tx for tx in transfers
|
||||
if tx.get("tokenSymbol", "").upper() == token_upper
|
||||
]
|
||||
contract = TOKEN_CONTRACTS.get(token_upper, "").lower()
|
||||
if contract:
|
||||
transfers = [
|
||||
tx for tx in transfers
|
||||
if tx.get("contractAddress", "").lower() == contract
|
||||
]
|
||||
else:
|
||||
# Try matching by symbol in the response
|
||||
transfers = [
|
||||
tx for tx in transfers
|
||||
if tx.get("tokenSymbol", "").upper() == token_upper
|
||||
]
|
||||
|
||||
if not transfers:
|
||||
return f"No {token_upper} transfers found for {_short_address(addr)} in the last 20 token transactions."
|
||||
|
||||
@@ -32,7 +32,7 @@ logger = logging.getLogger(__name__)
|
||||
# Maximum tool-use rounds to prevent infinite loops
|
||||
# 14 tools available; LLM can call multiple per round but may need
|
||||
# several rounds for chain-of-investigation (search → discover → verify)
|
||||
MAX_TOOL_ROUNDS = 5
|
||||
MAX_TOOL_ROUNDS = 3
|
||||
|
||||
|
||||
class LLMAnalyzer:
|
||||
@@ -259,12 +259,25 @@ class LLMAnalyzer:
|
||||
|
||||
# Tool-use loop
|
||||
for round_idx in range(MAX_TOOL_ROUNDS + 1):
|
||||
# Last round: no tools, force final answer
|
||||
is_last_round = (round_idx == MAX_TOOL_ROUNDS)
|
||||
if is_last_round:
|
||||
messages.append({
|
||||
"role": "user",
|
||||
"content": (
|
||||
"You have used all available tool rounds. "
|
||||
"Based on all information gathered, provide your final "
|
||||
"analysis and output the JSON assessment now."
|
||||
),
|
||||
})
|
||||
|
||||
# Call LLM
|
||||
call_kwargs = {
|
||||
"model": self.settings.llm_model,
|
||||
"messages": messages,
|
||||
"max_tokens": 4096,
|
||||
}
|
||||
if tool_schemas and round_idx < MAX_TOOL_ROUNDS:
|
||||
if tool_schemas and not is_last_round:
|
||||
call_kwargs["tools"] = tool_schemas
|
||||
call_kwargs["tool_choice"] = "auto"
|
||||
|
||||
@@ -289,10 +302,13 @@ class LLMAnalyzer:
|
||||
|
||||
# No tool calls — final response
|
||||
analysis_text = msg.content or ""
|
||||
finish_reason = response.choices[0].finish_reason
|
||||
logger.info(
|
||||
f"Analysis complete after {round_idx + 1} round(s) "
|
||||
f"({len(analysis_text)} chars)"
|
||||
f"({len(analysis_text)} chars, finish={finish_reason})"
|
||||
)
|
||||
if len(analysis_text) < 200:
|
||||
logger.warning(f"Suspiciously short response: {analysis_text[:200]}")
|
||||
break
|
||||
|
||||
# Parse JSON decision from final response
|
||||
|
||||
@@ -372,6 +372,59 @@ class MarketFetcher:
|
||||
logger.error(f"Error fetching niche markets: {e}")
|
||||
return niche_markets
|
||||
|
||||
def get_tiered_markets(self) -> dict[str, list[TrendingMarket]]:
|
||||
"""
|
||||
Fetch ALL active markets and classify into 3 tiers by 24h volume.
|
||||
|
||||
Returns dict with keys "tier1", "tier2", "tier3", each a list of TrendingMarket.
|
||||
Mirrors options flow's "passive receive all signals" approach:
|
||||
scan everything, filter later.
|
||||
"""
|
||||
settings = self.settings
|
||||
all_markets = self.get_all_current_markets()
|
||||
|
||||
tiers: dict[str, list[TrendingMarket]] = {"tier1": [], "tier2": [], "tier3": []}
|
||||
|
||||
for market in all_markets:
|
||||
# Apply standard filters
|
||||
raw = {
|
||||
"question": market.question,
|
||||
"description": market.description,
|
||||
"slug": market.slug,
|
||||
}
|
||||
if self._should_filter_market(raw):
|
||||
continue
|
||||
|
||||
if not market.clob_token_ids or not market.active or market.closed:
|
||||
continue
|
||||
|
||||
vol = market.volume_24hr
|
||||
|
||||
tm = TrendingMarket(
|
||||
market=market,
|
||||
volume_24hr=vol,
|
||||
liquidity=market.liquidity,
|
||||
)
|
||||
|
||||
if vol >= settings.tier1_volume_min:
|
||||
tiers["tier1"].append(tm)
|
||||
elif vol >= settings.tier2_volume_min:
|
||||
tiers["tier2"].append(tm)
|
||||
elif vol >= settings.tier3_volume_min:
|
||||
tiers["tier3"].append(tm)
|
||||
# vol < tier3_volume_min → dead market, skip
|
||||
|
||||
# Sort each tier by volume descending
|
||||
for tier in tiers.values():
|
||||
tier.sort(key=lambda t: t.volume_24hr, reverse=True)
|
||||
|
||||
logger.info(
|
||||
f"Tiered markets: Tier1={len(tiers['tier1'])} (>{settings.tier1_volume_min/1000:.0f}K), "
|
||||
f"Tier2={len(tiers['tier2'])} (>{settings.tier2_volume_min/1000:.0f}K), "
|
||||
f"Tier3={len(tiers['tier3'])} (>{settings.tier3_volume_min/1000:.0f}K)"
|
||||
)
|
||||
return tiers
|
||||
|
||||
def get_market_by_id(self, market_id: str) -> Optional[Market]:
|
||||
"""
|
||||
Fetch a single market by ID.
|
||||
|
||||
@@ -167,11 +167,20 @@ class TelegramSearchService:
|
||||
# Calculate per-channel limit
|
||||
limit_per_channel = max(3, limit // len(self.channels))
|
||||
|
||||
# Run async search (nest_asyncio allows nested run_until_complete)
|
||||
loop = asyncio.get_event_loop()
|
||||
messages = loop.run_until_complete(
|
||||
self._search_all_channels(query, limit_per_channel)
|
||||
)
|
||||
# Run async search — handle both sync and async calling contexts
|
||||
coro = self._search_all_channels(query, limit_per_channel)
|
||||
try:
|
||||
loop = asyncio.get_running_loop()
|
||||
# Already inside an async event loop — use a new thread
|
||||
import concurrent.futures
|
||||
with concurrent.futures.ThreadPoolExecutor() as pool:
|
||||
messages = pool.submit(
|
||||
asyncio.run, coro
|
||||
).result(timeout=30)
|
||||
except RuntimeError:
|
||||
# No running loop — safe to use run_until_complete
|
||||
loop = asyncio.get_event_loop()
|
||||
messages = loop.run_until_complete(coro)
|
||||
|
||||
# Trim to total limit
|
||||
messages = messages[:limit]
|
||||
|
||||
+118
-213
@@ -2,7 +2,7 @@
|
||||
Trade monitoring service - per-market parallel architecture.
|
||||
|
||||
Each market runs its own independent async task that:
|
||||
1. Polls the internal API for new trades (incremental via start_ts)
|
||||
1. Polls the official Polymarket data-api for new trades
|
||||
2. Detects whale trades
|
||||
3. Fetches trader ranking + history in parallel
|
||||
4. Fires the whale callback (LLM report generation) without blocking other markets
|
||||
@@ -12,6 +12,7 @@ Modeled after paper_trading/paper_trading.py's _market_loop pattern.
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import random
|
||||
import time as _time
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
@@ -32,7 +33,7 @@ logger = logging.getLogger(__name__)
|
||||
# Gamma API for fetching latest market prices
|
||||
GAMMA_API_URL = "https://gamma-api.polymarket.com/markets"
|
||||
|
||||
# Internal API for trade data (more stable than official data-api)
|
||||
# Official Polymarket data-api for trade data
|
||||
# URL and key loaded from settings (.env)
|
||||
|
||||
# File to persist processed transaction hashes
|
||||
@@ -52,32 +53,27 @@ class TradeMonitor:
|
||||
):
|
||||
self.settings = get_settings()
|
||||
|
||||
# Official API (for trader ranking/history queries only)
|
||||
# Official Polymarket data-api
|
||||
self.data_api_url = "https://data-api.polymarket.com"
|
||||
self.trades_endpoint = f"{self.data_api_url}/trades"
|
||||
self.leaderboard_endpoint = f"{self.data_api_url}/v1/leaderboard"
|
||||
self._client = httpx.AsyncClient(timeout=30.0)
|
||||
|
||||
# Internal API client for trade data
|
||||
self._internal_api_url = self.settings.internal_api_url
|
||||
self._internal_client = httpx.AsyncClient(
|
||||
timeout=30.0,
|
||||
headers={
|
||||
"X-API-Key": self.settings.internal_api_key,
|
||||
"Accept": "application/json",
|
||||
"Accept-Encoding": "gzip",
|
||||
},
|
||||
self._client = httpx.AsyncClient(
|
||||
timeout=httpx.Timeout(30.0, pool=120.0),
|
||||
limits=httpx.Limits(
|
||||
max_connections=50,
|
||||
max_keepalive_connections=20,
|
||||
keepalive_expiry=30,
|
||||
),
|
||||
)
|
||||
|
||||
# Per-market last-fetch timestamps for incremental polling
|
||||
self._market_last_ts: Dict[str, int] = {}
|
||||
|
||||
# Global rate limiter for internal API (matches paper_trading: 5 QPS max)
|
||||
# NOTE: Lock created lazily in run() to avoid "attached to different loop" error
|
||||
# Rate limiter: Lock + Semaphore created lazily in run() to avoid "attached to different loop" error
|
||||
self._api_lock: Optional[asyncio.Lock] = None
|
||||
self._api_sem: Optional[asyncio.Semaphore] = None # concurrency limiter
|
||||
self._api_last_request: float = 0.0
|
||||
self._api_global_interval: float = 1.0 # min 1s between requests = 1 QPS
|
||||
self._api_global_interval: float = 0.2 # min 0.2s between requests = 5 QPS
|
||||
|
||||
# Cache for trader rankings to avoid repeated API calls
|
||||
self._trader_ranking_cache: Dict[str, TraderRanking] = {}
|
||||
@@ -143,7 +139,6 @@ class TradeMonitor:
|
||||
"""Cleanup resources."""
|
||||
self._save_processed_txns()
|
||||
await self._client.aclose()
|
||||
await self._internal_client.aclose()
|
||||
|
||||
# ================================================================
|
||||
# Market list management
|
||||
@@ -157,28 +152,46 @@ class TradeMonitor:
|
||||
self._monitored_markets[tm.market.id] = tm.market
|
||||
logger.info(f"Now monitoring {len(self._monitored_markets)} markets")
|
||||
|
||||
def set_tiered_markets(self, tiers: dict[str, list]) -> None:
|
||||
"""
|
||||
Set markets with per-tier poll intervals.
|
||||
|
||||
Stores poll_interval per market_id in _market_poll_intervals dict.
|
||||
"""
|
||||
self._monitored_markets = {}
|
||||
self._market_poll_intervals: dict[str, int] = {}
|
||||
|
||||
tier_intervals = {
|
||||
"tier1": self.settings.tier1_poll_interval,
|
||||
"tier2": self.settings.tier2_poll_interval,
|
||||
"tier3": self.settings.tier3_poll_interval,
|
||||
}
|
||||
|
||||
for tier_name, markets in tiers.items():
|
||||
interval = tier_intervals.get(tier_name, self.settings.fetch_interval_seconds)
|
||||
for tm in markets:
|
||||
if tm.market.id:
|
||||
self._monitored_markets[tm.market.id] = tm.market
|
||||
self._market_poll_intervals[tm.market.id] = interval
|
||||
|
||||
tier_counts = {k: len(v) for k, v in tiers.items()}
|
||||
logger.info(
|
||||
f"Tiered monitoring: {tier_counts} "
|
||||
f"(intervals: {tier_intervals}s), total={len(self._monitored_markets)}"
|
||||
)
|
||||
|
||||
# ================================================================
|
||||
# Trade fetching: dispatches to internal or official API
|
||||
# Trade fetching
|
||||
# ================================================================
|
||||
|
||||
_MAX_RETRIES = 3
|
||||
_RETRY_BACKOFF = [1, 2, 4] # seconds between retries
|
||||
|
||||
async def fetch_market_trades(self, market_id: str) -> List[TradeActivity]:
|
||||
"""
|
||||
Fetch recent trades for a market. Dispatches to internal or official API
|
||||
based on TRADE_API_MODE setting.
|
||||
"""
|
||||
if self.settings.trade_api_mode == "internal":
|
||||
return await self._fetch_trades_internal(market_id)
|
||||
else:
|
||||
return await self._fetch_trades_official(market_id)
|
||||
_MAX_RETRIES = 4
|
||||
_RETRY_BACKOFF = [2, 5, 10, 20] # seconds between retries (with jitter)
|
||||
|
||||
# ================================================================
|
||||
# Official Polymarket data-api: fetch trades
|
||||
# ================================================================
|
||||
|
||||
async def _fetch_trades_official(self, market_id: str) -> List[TradeActivity]:
|
||||
async def fetch_market_trades(self, market_id: str) -> List[TradeActivity]:
|
||||
"""
|
||||
Fetch recent trades using the official Polymarket data-api /trades endpoint.
|
||||
|
||||
@@ -200,13 +213,9 @@ class TradeMonitor:
|
||||
|
||||
params: Dict[str, object] = {
|
||||
"market": condition_id,
|
||||
"limit": 50 if last_ts is None else 500,
|
||||
"limit": 50,
|
||||
}
|
||||
|
||||
# Incremental polling: only fetch trades after last seen timestamp
|
||||
if last_ts is not None:
|
||||
params["after"] = last_ts + 1
|
||||
|
||||
sem = self._api_sem or asyncio.Semaphore(20)
|
||||
last_err: Optional[Exception] = None
|
||||
async with sem:
|
||||
@@ -238,11 +247,11 @@ class TradeMonitor:
|
||||
except httpx.HTTPError as e:
|
||||
last_err = e
|
||||
if attempt < self._MAX_RETRIES - 1:
|
||||
delay = self._RETRY_BACKOFF[attempt]
|
||||
delay = self._RETRY_BACKOFF[attempt] + random.uniform(0, 2)
|
||||
logger.debug(
|
||||
f"Official API retry for {market_id} "
|
||||
f"(attempt {attempt + 1}/{self._MAX_RETRIES}): "
|
||||
f"{type(e).__name__}, retrying in {delay}s"
|
||||
f"{type(e).__name__}, retrying in {delay:.1f}s"
|
||||
)
|
||||
await asyncio.sleep(delay)
|
||||
else:
|
||||
@@ -324,153 +333,6 @@ class TradeMonitor:
|
||||
logger.warning(f"Error fetching official trades for {market_id}: {type(e).__name__}: {e}")
|
||||
return []
|
||||
|
||||
# ================================================================
|
||||
# Internal API: fetch trades
|
||||
# ================================================================
|
||||
|
||||
async def _fetch_trades_internal(self, market_id: str) -> List[TradeActivity]:
|
||||
"""
|
||||
Fetch recent taker trades for a market using the internal /flows API.
|
||||
|
||||
/flows returns one record per taker per transaction (already aggregated
|
||||
across maker fills), with accurate usd_amount and real execution price.
|
||||
Uses incremental polling via start_ts.
|
||||
Retries up to _MAX_RETRIES times on connection/timeout errors.
|
||||
"""
|
||||
try:
|
||||
last_ts = self._market_last_ts.get(market_id)
|
||||
|
||||
params: Dict[str, object] = {
|
||||
"market_id": market_id,
|
||||
"role": "taker",
|
||||
# First poll: only fetch recent 50 trades to record txn hashes
|
||||
# Subsequent polls: incremental via start_ts, small data
|
||||
"limit": 50 if last_ts is None else 500,
|
||||
"desc": True,
|
||||
}
|
||||
|
||||
if last_ts is not None:
|
||||
params["start_ts"] = last_ts + 1
|
||||
|
||||
# Semaphore limits concurrent requests; Lock enforces per-request interval
|
||||
sem = self._api_sem or asyncio.Semaphore(20)
|
||||
last_err: Optional[Exception] = None
|
||||
async with sem:
|
||||
for attempt in range(self._MAX_RETRIES):
|
||||
try:
|
||||
# Global rate limit
|
||||
async with self._api_lock:
|
||||
now = _time.monotonic()
|
||||
wait = self._api_global_interval - (now - self._api_last_request)
|
||||
if wait > 0:
|
||||
await asyncio.sleep(wait)
|
||||
self._api_last_request = _time.monotonic()
|
||||
|
||||
response = await self._internal_client.get(
|
||||
f"{self._internal_api_url}/flows", params=params,
|
||||
)
|
||||
response.raise_for_status()
|
||||
break # success
|
||||
except httpx.HTTPStatusError as e:
|
||||
if e.response.status_code in (502, 503, 504) and attempt < self._MAX_RETRIES - 1:
|
||||
delay = self._RETRY_BACKOFF[attempt]
|
||||
logger.debug(
|
||||
f"Internal API {e.response.status_code} for {market_id} "
|
||||
f"(attempt {attempt + 1}/{self._MAX_RETRIES}), "
|
||||
f"retrying in {delay}s"
|
||||
)
|
||||
await asyncio.sleep(delay)
|
||||
continue
|
||||
raise # don't retry other HTTP errors
|
||||
except httpx.HTTPError as e:
|
||||
last_err = e
|
||||
if attempt < self._MAX_RETRIES - 1:
|
||||
delay = self._RETRY_BACKOFF[attempt]
|
||||
logger.debug(
|
||||
f"Internal API retry for {market_id} "
|
||||
f"(attempt {attempt + 1}/{self._MAX_RETRIES}): "
|
||||
f"{type(e).__name__}, retrying in {delay}s"
|
||||
)
|
||||
await asyncio.sleep(delay)
|
||||
else:
|
||||
logger.warning(
|
||||
f"Internal API connection error for {market_id} "
|
||||
f"(attempt {attempt + 1}/{self._MAX_RETRIES}, giving up): "
|
||||
f"{type(e).__name__}: {e}"
|
||||
)
|
||||
return []
|
||||
else:
|
||||
# All retries exhausted (shouldn't reach here, but just in case)
|
||||
return []
|
||||
|
||||
data = response.json()
|
||||
if not data:
|
||||
return []
|
||||
|
||||
activities = []
|
||||
max_ts = last_ts or 0
|
||||
|
||||
for item in data:
|
||||
try:
|
||||
raw_direction = item.get("direction", "")
|
||||
|
||||
# Only track BUY trades (new positions).
|
||||
# SELL may just be exiting a position, not a directional signal.
|
||||
if raw_direction != "BUY":
|
||||
continue
|
||||
|
||||
token_amount = float(item.get("token_amount", 0) or 0)
|
||||
raw_price = float(item.get("price", 0) or 0)
|
||||
usdc_size = float(item.get("usd_amount", 0) or 0)
|
||||
|
||||
# No normalization — keep real price and outcome:
|
||||
# - nonusdc_side=token1: BUY Yes token at raw_price
|
||||
# - nonusdc_side=token2: BUY No token at raw_price
|
||||
nonusdc_side = item.get("nonusdc_side", "token1")
|
||||
outcome = "Yes" if nonusdc_side == "token1" else "No"
|
||||
|
||||
ts = int(item.get("timestamp", 0) or 0)
|
||||
|
||||
if ts > max_ts:
|
||||
max_ts = ts
|
||||
|
||||
activity = TradeActivity(
|
||||
transaction_hash=f"{item.get('transaction_hash', '')}-{item.get('log_index', '')}",
|
||||
timestamp=ts,
|
||||
condition_id=item.get("condition_id", market_id),
|
||||
asset=item.get("condition_id", ""),
|
||||
side="BUY",
|
||||
size=token_amount,
|
||||
usdc_size=usdc_size,
|
||||
price=raw_price,
|
||||
outcome=outcome,
|
||||
outcome_index=0 if outcome == "Yes" else 1,
|
||||
title="",
|
||||
slug=None,
|
||||
event_slug=None,
|
||||
proxy_wallet=item.get("address"),
|
||||
name=None,
|
||||
)
|
||||
activities.append(activity)
|
||||
except Exception as e:
|
||||
logger.debug(f"Failed to parse /flows trade: {e}")
|
||||
continue
|
||||
|
||||
if max_ts > 0:
|
||||
self._market_last_ts[market_id] = max_ts
|
||||
|
||||
return activities
|
||||
|
||||
except httpx.HTTPStatusError as e:
|
||||
logger.warning(
|
||||
f"Flows API HTTP {e.response.status_code} for {market_id}: "
|
||||
f"{e.response.text[:200]}"
|
||||
)
|
||||
return []
|
||||
except Exception as e:
|
||||
logger.warning(f"Error fetching flows for {market_id}: {type(e).__name__}: {e}")
|
||||
return []
|
||||
|
||||
# ================================================================
|
||||
# Official API: trader info (ranking + history)
|
||||
# ================================================================
|
||||
@@ -761,37 +623,78 @@ class TradeMonitor:
|
||||
|
||||
def _is_whale_trade(self, activity: TradeActivity, market: Optional[Market] = None) -> bool:
|
||||
"""
|
||||
Check if a trade qualifies as a whale trade.
|
||||
Multi-layer pre-filter mirroring options flow SignalFilter._check_signal.
|
||||
|
||||
Uses a dynamic size threshold based on market volume:
|
||||
- Large markets (24h vol > $1M): standard threshold (MIN_TRADE_SIZE_USD)
|
||||
- Small markets (24h vol < $100k): lowered to $1,000
|
||||
- In between: linearly interpolated
|
||||
Filter chain (early rejection, same order as options flow):
|
||||
1. Price range — like moneyness filter (OTM/ITM range)
|
||||
2. Direction — BUY only (like enabled direction_filters)
|
||||
3. Resolution window — like DTE filter (3-60 days sweet spot)
|
||||
4. Size — like premium filter ($250K+ minimum)
|
||||
5. Dynamic size — like dynamic_premium (base × √(vol / baseline))
|
||||
6. Signal strength — like ask_ratio filter (conviction check)
|
||||
"""
|
||||
# Price filter: only BUY trades remain, price is the taker's buy price.
|
||||
# Low price = cheap bet with high upside, high price = expensive/certain.
|
||||
# Filter to [MIN_PRICE, MAX_PRICE] range (e.g. 0-0.7).
|
||||
import math
|
||||
from datetime import datetime as _dt
|
||||
|
||||
# --- 1. Price range (like moneyness: OTM 0-20%) ---
|
||||
# Price 0.2-0.8 = uncertain outcome = tradeable
|
||||
# Price < 0.2 or > 0.8 = near-consensus = no edge
|
||||
if not (self.settings.min_price <= activity.price <= self.settings.max_price):
|
||||
return False
|
||||
|
||||
# Dynamic threshold based on market total volume:
|
||||
# - Tiny markets ($10k-$100k vol): $1,000 (niche, info asymmetry high)
|
||||
# - Medium markets ($100k-$5M vol): $5,000 (standard)
|
||||
# - Large markets ($5M+ vol): $10,000 (macro, noise high)
|
||||
if market and market.volume > 0:
|
||||
vol = market.volume # total volume, not 24hr
|
||||
if vol <= 10_000:
|
||||
threshold = 500
|
||||
elif vol <= 100_000:
|
||||
threshold = 1_000
|
||||
elif vol <= 5_000_000:
|
||||
threshold = 5_000
|
||||
else:
|
||||
threshold = 10_000
|
||||
else:
|
||||
threshold = 5_000
|
||||
# --- 2. Direction: BUY only (like direction_filters.enabled) ---
|
||||
# Already enforced upstream (only BUY trades reach here)
|
||||
|
||||
return activity.usdc_size >= threshold
|
||||
# --- 3. Resolution window (like DTE min=3, max=60) ---
|
||||
# Markets resolving < 6 hours = price already settled (like DTE < 3)
|
||||
# Markets resolving > 90 days = too far out, edge diluted (like DTE > 60)
|
||||
if market and market.end_date:
|
||||
try:
|
||||
end_dt = _dt.fromisoformat(market.end_date.replace("Z", "+00:00"))
|
||||
now_dt = _dt.utcnow().replace(tzinfo=end_dt.tzinfo) if end_dt.tzinfo else _dt.utcnow()
|
||||
hours_to_resolution = max(0, (end_dt - now_dt).total_seconds() / 3600)
|
||||
if hours_to_resolution < 6:
|
||||
return False # too close, like DTE < 3
|
||||
if hours_to_resolution > 90 * 24:
|
||||
return False # too far, like DTE > 60
|
||||
except (ValueError, TypeError):
|
||||
pass # unknown end date, don't reject
|
||||
|
||||
# --- 4. Size (like premium min=$250K) ---
|
||||
# Base minimum: $5,000 (Polymarket scale vs options $250K)
|
||||
if activity.usdc_size < 5_000:
|
||||
return False
|
||||
|
||||
# --- 5. Dynamic size (like dynamic_premium = base × √(mcap / baseline)) ---
|
||||
# Larger markets require proportionally larger trades to be meaningful
|
||||
base_size = 10_000.0
|
||||
baseline_volume = 1_000_000.0
|
||||
|
||||
if market and market.volume > 0:
|
||||
threshold = base_size * math.sqrt(market.volume / baseline_volume)
|
||||
threshold = max(5_000.0, min(threshold, 100_000.0)) # floor $5K, cap $100K
|
||||
else:
|
||||
threshold = base_size
|
||||
|
||||
if activity.usdc_size < threshold:
|
||||
return False
|
||||
|
||||
# --- 6. Signal strength (like ask_ratio > 70%) ---
|
||||
# In Polymarket: buyer paying above market mid = conviction
|
||||
# Reject trades at or below market mid (no conviction, possibly hedging)
|
||||
if market and market.outcome_prices:
|
||||
if activity.outcome == "Yes":
|
||||
market_mid = market.outcome_prices[0]
|
||||
elif len(market.outcome_prices) > 1:
|
||||
market_mid = market.outcome_prices[1]
|
||||
else:
|
||||
market_mid = 1.0 - market.outcome_prices[0]
|
||||
|
||||
# Must pay above market mid (no discount buys = no conviction)
|
||||
if activity.price < market_mid + 0.01:
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
async def _handle_whale(self, activity: TradeActivity, market_id: str, market: Market):
|
||||
"""
|
||||
@@ -885,7 +788,9 @@ class TradeMonitor:
|
||||
if not market:
|
||||
return
|
||||
|
||||
poll_interval = self.settings.fetch_interval_seconds
|
||||
# Per-market interval (from tiered monitoring) or global default
|
||||
poll_intervals = getattr(self, '_market_poll_intervals', {})
|
||||
poll_interval = poll_intervals.get(market_id, self.settings.fetch_interval_seconds)
|
||||
# If we already have a last_ts for this market, it means the loop was
|
||||
# restarted (e.g. after a market list refresh) — skip the silent
|
||||
# first-poll window to avoid missing trades.
|
||||
@@ -955,7 +860,7 @@ class TradeMonitor:
|
||||
|
||||
# Create lock/semaphore inside event loop (avoids "attached to different loop" error)
|
||||
self._api_lock = asyncio.Lock()
|
||||
self._api_sem = asyncio.Semaphore(5) # max 5 concurrent API requests
|
||||
self._api_sem = asyncio.Semaphore(10) # max 10 concurrent API requests
|
||||
|
||||
logger.info(
|
||||
f"Starting parallel trade monitor "
|
||||
|
||||
Reference in New Issue
Block a user