From 9eab3485a141993f79855e9bb8680a4e713b08a9 Mon Sep 17 00:00:00 2001 From: SII-leiyu <115807123+chaoleiyv@users.noreply.github.com> Date: Sat, 2 May 2026 18:59:12 +0800 Subject: [PATCH] Update to tiered market coverage (700+ markets), refactor anomaly detection and trade monitoring - Expand market coverage from 50 trending to 700+ active markets with three volume tiers - Refactor anomaly detector with improved scoring logic - Simplify trade monitor architecture - Add tiered market fetching in market_fetcher - Update prompts, settings, and etherscan service - Remove requirements.txt (using other dependency management) - Update README to reflect new capabilities --- README.md | 280 +++++++++++++------------- data/signals.db | Bin 933888 -> 946176 bytes data/signals.db-shm | Bin 32768 -> 0 bytes data/signals.db-wal | 0 requirements.txt | 22 -- src/config/settings.py | 20 +- src/main.py | 47 ++++- src/prompts/whale_analyzer.py | 12 +- src/services/anomaly_detector.py | 313 ++++++++++++++++------------- src/services/etherscan.py | 53 +++-- src/services/llm_analyzer.py | 22 +- src/services/market_fetcher.py | 53 +++++ src/services/telegram_search.py | 19 +- src/services/trade_monitor.py | 331 +++++++++++-------------------- 14 files changed, 621 insertions(+), 551 deletions(-) delete mode 100644 data/signals.db-shm delete mode 100644 data/signals.db-wal delete mode 100644 requirements.txt diff --git a/README.md b/README.md index 2a2ac2e..59c0afb 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ [![License: MIT](https://img.shields.io/badge/License-MIT-green.svg)](LICENSE) [![Polymarket](https://img.shields.io/badge/Polymarket-Official_API-purple.svg)](https://docs.polymarket.com/) -Real-time monitoring | Multi-dimensional anomaly detection | LLM investigation with 14 autonomous tools | Signal accuracy tracking +Real-time monitoring | Tiered market coverage | LLM investigation with 14 autonomous tools | Signal accuracy tracking [Quick Start](#-quick-start) | [How It Works](#-how-it-works) | [Features](#-features) | [Configuration](#-configuration) | [Dashboard](#-dashboard) @@ -18,11 +18,11 @@ Real-time monitoring | Multi-dimensional anomaly detection | LLM investigation w ## What It Does -Whale Watcher continuously monitors 50+ trending Polymarket markets, detects large trades with anomalous patterns, and uses an LLM agent with 14 autonomous research tools to investigate whether the trader may possess an information advantage. It tracks signal accuracy over time, achieving **63.5% win rate** with **+28.3% average ROI** on high-confidence signals. +Whale Watcher continuously monitors **700+ active Polymarket markets** across three volume tiers, detects large trades with anomalous patterns, and uses an LLM agent with **14 autonomous research tools** to investigate whether the trader may possess an information advantage. It tracks signal accuracy over time, generates daily briefings, and sends real-time email alerts for high-confidence signals. ``` -Trending Markets → Whale Detection → Anomaly Scoring → LLM Investigation → Signal Tracking - (50+) ($1k-$100k+) (5 dimensions) (14 tools, 5 rounds) (win rate, ROI) +Tiered Markets → Whale Detection → Anomaly Scoring → LLM Investigation → Signal Tracking + (700+) ($5k-$100k+) (5 dimensions) (14 tools, 3 rounds) (win rate, ROI) ``` --- @@ -35,35 +35,36 @@ Trending Markets → Whale Detection → Anomaly Scoring → LLM Investigation ``` $ python -m src.main run -╭──────────────────────────────────────────────────────────╮ -│ 🐋 Polymarket Whale Watcher │ -│ │ -│ Markets Monitored: 50 │ -│ Polling Interval: 15s │ -│ Min Trade Size: $1,000 │ -│ Price Range: 0 - 0.7 │ -╰──────────────────────────────────────────────────────────╯ +============================================================ +WHALE WATCHER STARTED +============================================================ +Monitoring: 765 markets +Interval: 10 seconds +Min Trade Size: $10,000 USD +Price Range: 0.1 - 0.9 +============================================================ -[14:22:51] 🐋 WHALE DETECTED on "Will MegaETH launch a token by June 30, 2026?" - BUY Yes @ 0.4200 | $92,336 USDC | Wallet: 0x7a3b...f91e - Anomaly Score: 0.78/1.00 +Tiered monitoring: Tier1=8 (>500K), Tier2=198 (>10K), Tier3=559 (>1K) -[14:22:53] 🤖 LLM Analysis started (model: gemini-3-flash-preview) - → Tool call: search_web("MegaETH token launch date 2026") - → Tool call: search_twitter("MegaETH $METH token TGE") - → Tool call: get_protocol_tvl("megaeth") - → Tool call: get_contract_info("0x4f9b...2a1c") - → Tool call: search_telegram("MegaETH launch") +[22:05:14] WHALE TRADE DETECTED! + Amount: $5,000.00 USDC + Side: BUY Yes + Price: 0.7962 + Market: Over $20M committed to the Printr public sale? -[14:23:07] ✅ Analysis complete - Information Asymmetry Score: 0.72 (HIGH) - Recommendation: BUY Yes | Confidence: 0.75 - Report saved: reports/20260415/... + Generating analysis report... + Round 1: LLM requested 3 tool call(s) + → search_web("Printr public sale Sonar raise commitments") + → search_twitter("Printr PRINT token sale commitments") + → search_telegram("Printr public sale Sonar raise") + Round 2: LLM requested 2 tool call(s) + → search_web("Printr PRINT token sale total raised April 28") + → search_twitter("Printr sale oversubscribed April 28") -[15:00:00] 📊 Resolution check: 3 markets resolved - → "EdgeX FDV above 400M" resolved YES — Signal CORRECT (ROI: +142%) - → "Will Trump talk to Rutte" resolved NO — Signal INCORRECT - → "Over 9M committed to P2P" resolved YES — Signal CORRECT (ROI: +67%) + Analysis complete after 3 round(s) + Information Asymmetry Score: 0.22 (LOW) + Trader Credibility: LOW (#2650076, PnL: -$207K) + Report saved: reports/20260428/... ``` @@ -75,14 +76,11 @@ Each whale trade generates a detailed markdown report: - **Trade details** — amount, direction, price, trader wallet - **Trader profile** — leaderboard rank, PnL, history, recent trades +- **Event positions** — whale's positions across related markets (hedge detection) +- **Market top holders** — Top 5 buyers/sellers with rankings - **LLM investigation** — autonomous tool calls (web, Twitter, Telegram, on-chain, DeFi) - **Information asymmetry assessment** — score, evidence, reasoning -Example finding: -> *"New ERC-20 contract deployed by MegaETH deployer wallet 6 hours before trade — not yet publicly announced. KOL tweets about insider knowledge preceded the trade by ~3 hours."* -> -> **Information Asymmetry Score: 0.72** | Trader Credibility: HIGH - See full example: [docs/examples/sample_report.md](docs/examples/sample_report.md) @@ -90,17 +88,12 @@ See full example: [docs/examples/sample_report.md](docs/examples/sample_report.m
Daily Briefing — Automated intelligence summary -Daily briefings include: -- High-confidence signals with analysis +Daily briefings are generated at 10:00 AM local time and emailed automatically. They include: +- High-confidence signals (IAS >= 60%) with full analysis +- Fallback: top 5 signals by score if none reach 60% - Price volatility alerts - Historical signal performance (win rate, ROI by confidence tier) -| Metric | Value | -|--------|-------| -| Win Rate | **63.5%** | -| Avg ROI | **+28.3%** | -| Signals with IAS >= 60% | 3 today | - See full example: [docs/examples/sample_briefing.md](docs/examples/sample_briefing.md)
@@ -130,7 +123,7 @@ Then add your API key and start: echo "GEMINI_API_KEY=your_key_here" >> .env # Activate the environment and run -source venv/bin/activate +source .venv/bin/activate python -m src.main run ``` @@ -143,33 +136,20 @@ docker build -t whale-watcher . docker run --env-file .env -v ./data:/app/data -v ./reports:/app/reports whale-watcher ``` -### Make Commands - -```bash -make setup # One-click setup -make run # Start monitoring -make run-debug # Start with debug logging -make dashboard # Start web dashboard -make markets # View trending markets -make briefing # Generate today's briefing -make help # Show all commands -``` - --- ## Features | Feature | Description | |---------|-------------| -| **Real-Time Monitoring** | Parallel per-market polling of 50+ trending markets | -| **Anomaly Detection** | 5-dimensional scoring: size, price uncertainty, time-of-day, trader deviation, cluster signals | -| **Trader Profiling** | Leaderboard ranking, PnL, trading history, recent behavior | -| **LLM Analysis** | 14 autonomous tools across 5 rounds of investigation | -| **Signal Tracking** | Automatic market resolution checking, win rate stats by confidence tier | -| **Daily Briefing** | Automated 10:00 AM summary with high-confidence signals | +| **Tiered Market Monitoring** | 700+ markets across 3 tiers: Tier1 (>$500K, 15s), Tier2 (>$10K, 60s), Tier3 (>$1K, 300s) | +| **Anomaly Detection** | 5-factor scoring: premium ratio, signal cleanliness, depth ratio, cluster signals, base confidence | +| **Trader Profiling** | Leaderboard ranking, PnL, trading history, event positions, market top holders | +| **LLM Analysis** | 14 autonomous tools across up to 3 rounds of investigation | +| **Signal Tracking** | Automatic market resolution checking every 30 min, win rate stats by confidence tier | +| **Daily Briefing** | Automated 10:00 AM summary with high-confidence signals, emailed to configured recipients | | **Email Alerts** | Real-time notifications for high information-asymmetry signals (>= 60%) | | **Web Dashboard** | FastAPI-based signal performance dashboard | -| **Official API** | Works with Polymarket's public API — no private API access needed | ### 14 LLM Research Tools @@ -178,11 +158,10 @@ The LLM agent autonomously selects and uses these tools during investigation: | Category | Tools | |----------|-------| | **Social Sentiment** | `search_twitter`, `search_telegram`, `search_web` | -| **Crypto Data** | `get_crypto_price`, `get_defi_metrics`, `get_token_unlocks` | -| **Financial Data** | `get_stock_price`, `get_economic_indicators` | -| **On-Chain** | `get_wallet_transactions`, `get_contract_info` | -| **Legislation** | `search_congress` | -| **Market Data** | `get_market_history`, `get_trader_positions` | +| **Crypto Data** | `get_crypto_price`, `get_crypto_market_overview`, `get_protocol_tvl`, `get_token_unlocks`, `get_protocol_revenue` | +| **Financial Data** | `get_stock_price`, `get_stock_news`, `get_economic_data` | +| **On-Chain** | `get_wallet_transfers`, `get_contract_info` | +| **Legislation** | `get_bill_status`, `get_recent_legislation` | --- @@ -191,12 +170,12 @@ The LLM agent autonomously selects and uses these tools during investigation: ```mermaid flowchart LR A[Polymarket API] --> B[Market Fetcher] - B --> C[50+ Trending Markets] + B --> C[700+ Tiered Markets] C --> D[Trade Monitor] D --> E{Whale Trade?} E -->|No| D E -->|Yes| F[Anomaly Detector] - F --> G{Score > Threshold?} + F --> G{Score >= 0.65?} G -->|No| D G -->|Yes| H[LLM Analyzer] H --> I[14 Research Tools] @@ -207,20 +186,28 @@ flowchart LR ### Pipeline -1. **Market Selection** — Fetches top trending markets by 24h volume from Polymarket, filters out sports/weather/short-term price markets, refreshes every 15 minutes +1. **Market Selection** — Fetches all active markets from Polymarket Gamma API, classifies into 3 tiers by 24h volume, adds token launch markets. Refreshes every 15 minutes. -2. **Trade Monitoring** — Runs parallel async tasks per market, polls for new trades incrementally, deduplicates by transaction hash +2. **Trade Monitoring** — Runs parallel async tasks per market (one task per market), polls official Polymarket data-api for new taker BUY trades, deduplicates by transaction hash. Connection pool tuned for 700+ concurrent markets. -3. **Anomaly Detection** — Multi-dimensional scoring on 5 axes: - - **Size** — Trade size relative to market 24h volume - - **Price uncertainty** — Closer to 0.5 = more interesting - - **Time-of-day** — ET hour-based suspicion weights - - **Trader deviation** — Trade size vs trader's historical average - - **Cluster signal** — Same-direction trades within 5-minute window +3. **Whale Pre-filter** — Multi-layer filter chain: + - Price range: 0.10 - 0.90 + - Minimum size: $5,000 hard floor + - Dynamic threshold: $10K base, scaled by market volume (floor $5K, cap $100K) + - Conviction check: must pay above market mid price + - Resolution window: 6 hours to 90 days -4. **LLM Investigation** — Builds rich context (trade + trader profile + market data), LLM autonomously uses tools to investigate (up to 5 rounds), produces structured recommendation with information asymmetry score +4. **Anomaly Scoring** — 5-factor model (max 1.0): + - Base confidence: 0.50 + - Premium-to-threshold ratio: +0.20 + - Signal cleanliness (conviction): +0.10 + - Depth ratio (size vs liquidity): +0.10 + - Cluster tier (repeated same-direction): +0.10 + - Threshold: >= 0.65 to trigger LLM analysis -5. **Signal Tracking** — Resolution tracker checks every 30 minutes for resolved markets, validates signal correctness, computes theoretical ROI, aggregates win rates by confidence tier +5. **LLM Investigation** — Builds rich context (trade + trader profile + event positions + market top holders + historical signals), LLM autonomously uses tools for up to 3 rounds, produces structured assessment with information asymmetry score (0-1). + +6. **Signal Tracking** — Resolution tracker checks every 30 minutes for resolved markets, validates signal correctness, computes theoretical ROI. Daily briefings generated at 10:00 AM and emailed. --- @@ -232,35 +219,57 @@ Copy `.env.example` to `.env` and configure: | Variable | Description | Get It | |----------|-------------|--------| -| `GEMINI_API_KEY` | Gemini API key for LLM analysis | [Google AI Studio](https://aistudio.google.com/apikey) | - -### Trade Data Source - -| Variable | Default | Description | -|----------|---------|-------------| -| `TRADE_API_MODE` | `official` | `official` = Polymarket public API (no auth needed), `internal` = private API | - -The bot works out of the box with Polymarket's official public API. No private API access required. +| `GEMINI_API_KEY` | LLM API key for analysis | [Google AI Studio](https://aistudio.google.com/apikey) | ### Optional (enhances analysis) | Variable | Description | Get It | |----------|-------------|--------| | `TAVILY_API_KEY` | Web search (primary) | [tavily.com](https://tavily.com) | -| `TWITTER_API_KEY` | Twitter sentiment search | [Twitter Developer](https://developer.twitter.com) | +| `SERPER_API_KEY` | Web search (fallback) | [serper.dev](https://serper.dev) | +| `TWITTER_API_KEY` | Twitter sentiment search | [twitterapi.io](https://twitterapi.io) | | `POLYGON_API_KEY` | Stock/ETF/forex data | [polygon.io](https://polygon.io) | | `FRED_API_KEY` | Economic indicators | [FRED](https://fred.stlouisfed.org/docs/api/api_key.html) | -| `ETHERSCAN_API_KEY` | On-chain wallet analysis | [etherscan.io](https://etherscan.io/apis) | +| `ETHERSCAN_API_KEY` | On-chain wallet analysis (Polygon V2) | [etherscan.io](https://etherscan.io/apis) | | `CONGRESS_API_KEY` | US legislation data | [congress.gov](https://api.congress.gov/) | +| `TELEGRAM_API_ID` / `TELEGRAM_API_HASH` | Telegram channel monitoring | [my.telegram.org](https://my.telegram.org) | + +### LLM Settings + +| Variable | Default | Description | +|----------|---------|-------------| +| `LLM_MODEL` | `gemini-3-flash-preview` | Model name (any OpenAI-compatible) | +| `LLM_BASE_URL` | Google AI endpoint | OpenAI-compatible API base URL | +| `LLM_TEMPERATURE` | `0` | LLM temperature | ### Whale Detection Tuning | Variable | Default | Description | |----------|---------|-------------| -| `MIN_TRADE_SIZE_USD` | `1000` | Minimum trade size to consider | -| `MIN_PRICE` / `MAX_PRICE` | `0` / `0.7` | Price range filter | -| `FETCH_INTERVAL_SECONDS` | `15` | Polling interval per market | -| `TRENDING_MARKETS_LIMIT` | `50` | Number of markets to monitor | +| `MIN_TRADE_SIZE_USD` | `10000` | Minimum trade size to consider | +| `MIN_PRICE` / `MAX_PRICE` | `0.10` / `0.90` | Price range filter | +| `FETCH_INTERVAL_SECONDS` | `10` | Default polling interval | + +### Tiered Market Monitoring + +| Variable | Default | Description | +|----------|---------|-------------| +| `FULL_MARKET_SCAN` | `true` | Enable tiered monitoring (all active markets) | +| `TIER1_VOLUME_MIN` | `500000` | Tier 1 volume threshold | +| `TIER2_VOLUME_MIN` | `10000` | Tier 2 volume threshold | +| `TIER3_VOLUME_MIN` | `1000` | Tier 3 volume threshold | +| `TIER1_POLL_INTERVAL` | `15` | Tier 1 polling interval (seconds) | +| `TIER2_POLL_INTERVAL` | `60` | Tier 2 polling interval (seconds) | +| `TIER3_POLL_INTERVAL` | `300` | Tier 3 polling interval (seconds) | + +### Email Alerts + +| Variable | Default | Description | +|----------|---------|-------------| +| `EMAIL_ENABLED` | `false` | Enable email notifications | +| `EMAIL_SENDER` | — | Sender email address | +| `EMAIL_PASSWORD` | — | Sender email password (app password) | +| `EMAIL_RECIPIENT` | — | Comma-separated recipient emails | --- @@ -270,6 +279,8 @@ The bot works out of the box with Polymarket's official public API. No private A # Core python -m src.main run [--debug] # Start monitoring python -m src.main check-markets --limit 20 # View trending markets + +# Analysis python -m src.main test-analyze # Test LLM on a specific market # Reports @@ -312,20 +323,28 @@ src/ │ ├── trade.py # TradeActivity, WhaleTrade, TraderRanking │ ├── decision.py # TradeRecommendation, LLMDecision │ └── anomaly_signal.py # AnomalySignal (stored signal) -├── services/ # Business logic (23 modules) -│ ├── market_fetcher.py # Polymarket Gamma API -│ ├── trade_monitor.py # Per-market parallel monitoring (official + internal API) -│ ├── anomaly_detector.py # Multi-dimensional anomaly scoring -│ ├── llm_analyzer.py # LLM with tool-use (14 tools, 5 rounds) +├── services/ # Business logic +│ ├── market_fetcher.py # Polymarket Gamma API (tiered market selection) +│ ├── trade_monitor.py # Per-market parallel monitoring (official API) +│ ├── anomaly_detector.py # 5-factor anomaly scoring +│ ├── llm_analyzer.py # LLM with tool-use (14 tools, 3 rounds) │ ├── tools.py # Tool registry │ ├── resolution_tracker.py # Market resolution checking │ ├── stats_engine.py # Performance statistics -│ ├── daily_briefing.py # Daily summary generation -│ └── [data services] # Twitter, Telegram, CoinGecko, DeFiLlama, -│ # FRED, Polygon, Etherscan, Congress, web search +│ ├── daily_briefing.py # Daily summary generation + email +│ ├── twitter_search.py # Twitter API search +│ ├── telegram_search.py # Telegram channel monitoring +│ ├── web_search.py # Tavily/Serper/DuckDuckGo search +│ ├── coingecko.py # Crypto prices and market data +│ ├── defillama.py # DeFi TVL, revenue, token unlocks +│ ├── fred.py # FRED macroeconomic data +│ ├── polygon.py # Stock/ETF prices and news +│ ├── etherscan.py # On-chain data (Polygon, Etherscan V2) +│ └── congress.py # US legislation data +├── prompts/ # LLM system prompts +│ ├── whale_analyzer.py # Whale trade analysis prompt +│ └── volatility_analyzer.py # Price volatility analysis prompt ├── db/database.py # SQLite signal storage -├── prompts/ # LLM system prompts & tool schemas -├── dashboard.py # FastAPI web dashboard └── main.py # CLI entry point (Typer) ``` @@ -334,59 +353,54 @@ src/ ## Architecture ``` - ┌─────────────────────────────┐ - │ Polymarket Gamma API │ - │ (trending markets, prices) │ - └──────────────┬──────────────┘ + ┌──────────────────────────────┐ + │ Polymarket Gamma API │ + │ (all active markets) │ + └──────────────┬───────────────┘ │ - ┌──────────────▼──────────────┐ - │ Market Fetcher │ - │ (filter sports/weather/etc) │ - └──────────────┬──────────────┘ + ┌──────────────▼───────────────┐ + │ Market Fetcher │ + │ Tier1: >$500K (15s poll) │ + │ Tier2: >$10K (60s poll) │ + │ Tier3: >$1K (300s poll) │ + └──────────────┬───────────────┘ │ ┌────────────────────▼────────────────────┐ - │ Trade Monitor (async) │ - │ 50+ parallel market polling tasks │ - │ Official API ←→ Internal API (switch) │ + │ Trade Monitor (async, 700+) │ + │ Official Polymarket data-api │ + │ Per-market parallel tasks │ + │ Pool: 50 connections, 120s timeout │ └────────────────────┬────────────────────┘ │ ┌────────────────────▼────────────────────┐ - │ Anomaly Detector │ - │ 5-axis scoring: size, price, time, │ - │ trader deviation, cluster signals │ + │ Pre-filter + Scoring │ + │ $5K+ size, 0.10-0.90 price, conviction │ + │ 5-factor anomaly score >= 0.65 │ └────────────────────┬────────────────────┘ │ ┌────────────────────▼────────────────────┐ - │ LLM Analyzer (Gemini) │ - │ 14 tools × 5 rounds of investigation │ + │ LLM Analyzer (14 tools) │ + │ Up to 3 rounds of investigation │ + │ max_tokens: 4096 │ ├─────────┬────────┬────────┬─────────────┤ │ Twitter │ Web │ DeFi │ On-Chain │ - │Telegram │ Search │ Crypto │ Congress │ + │Telegram │ Search │ Crypto │ Legislation│ └─────────┴───┬────┴────────┴─────────────┘ │ ┌─────────────▼──────────────────────────┐ │ Signal Storage (SQLite) │ │ → Resolution Tracker (every 30min) │ - │ → Stats Engine (win rate, ROI) │ - │ → Dashboard (FastAPI) │ + │ → Daily Briefing (10:00 AM + email) │ │ → Email Alerts (IAS >= 60%) │ + │ → Dashboard (FastAPI) │ └────────────────────────────────────────┘ ``` --- -## Safety - -- Trade execution **disabled by default** (`ENABLE_TRADE_EXECUTION=false`) -- Position size capped at 20% of balance if enabled -- Minimum 60% confidence threshold for execution -- Price range filter avoids obvious outcomes -- All decisions logged for audit trail -- Rate limiting on all external APIs - ## Disclaimer -This system is for research and educational purposes. Prediction market trading involves significant risk. Never trade with funds you cannot afford to lose. Always verify recommendations independently. +This system is for research and educational purposes. Prediction market trading involves significant risk. The information asymmetry scores and analyses are AI-generated estimates, not financial advice. Always verify independently. --- diff --git a/data/signals.db b/data/signals.db index d107ff643f7a28f6c160fea516166bd1e2a524c9..cd5bd2234546acc307894100ab91e879139c09d6 100644 GIT binary patch delta 8109 zcmbtZdwdl2wV$1toqf#gZX%$>5Hch|f@IfuKcX}VJ`nOCHK>F_BQ3x3i!I`x4VI0Qu)J> z&o48-`OW#A^F8N$&zT>aHtCy=>br`a?_e1A{@rX1T!ijA^LS)m!2Jf@0+ZPLLrg&w zr&kYm7c6m*^eVb#_;BGKkN*$--SG6?{tYnJ)DB0A?<8FB)xphIYKA|Y>no&AlOBw` zc-YO|n`U6--b=!O>+mE%H3hANU>` zI#au#4`)7R{=xi-`91SC^BVIUbBOr~^CYvM`3`eGvxnKm+`(*NZfBBAlxbtW#;jzT znCqD(%vFrSaLnb*TxK?N5yLPPgV87H|D`{mkJE3_uhKuGe@Z_^KSBRHy_eoi@1VES z1N1lO^>h~4kOU(C~Ze|TrL7$<==ojfn>3#HC+BL`fsrOgj zXThxp+zXt98^=y($5Ywy!|eDEC*i@dzh|#c!u8OvmtW!V3_0pcU@Wa~gfUhx`(Vvj zcFoIZ&13Z?E|~c_x_+g8A*}f%J1%IrvIyo6WLG|x9rtI)AEA}K4M_5TG^|F_cEb#g z4Wao<8V7IkIC@FK?bt{X9!C$l>PA-^(6Am288nwhLkbN^G`9|2b)l;Sx{9N#7`lq0 zp%ZS0PBfOmmb04Bs5PxB@Hm#EXUouVJzDT^)9jGPu>{@Mq4#UiRSg=xf`-Lt?kaS3 zB^s*HEAKajXL%g+(R~FP%F$4Uo~&N60`_@q#cUW~Sg`=c@f9Aa|`;wm%s*%SMyHKc`+7QDWi z#O;~(zmbmo`=dV~9XHd{2oOWMCh3ZR1dJ3oD2{v$^Sm}%xQqCe2fuT^{n@gd2#;&D zbh+c0bN*BvOM;;5f~ZF{L(@&Q zO*9PCR3l25GZbFdjaGw?C}t~mx`FI3!tGC4|8=7W%7~j9$p(yUfIXIwm)pEv3K%t> zg=mJc#dm+HYm2^lHGnk3$&>t z&99AYA}IEI)Fy_&yRWhzPmvdUx2s!d1ijladhH{`(@e<}=tU`{$Qt}MB|aNDK#;84 zK`qAMw*>U%snNx+5-)h1o+Fs&2nv`VL%>wqrDFtg1QJ^_MtthLa;jJ*#0!!tfk!{{ zL5ab!IT$v_4Oi#?>@@7FPGyBb0*u~riukVU0s;KJ>T$yTBuR0j)B*2)!m-?bLnO0d z0r7Q4Bq?SZwA7HGtCECLsJ1MYdS#}*d!&}63P!0~>H)$&)RL8e>^a&(veUt&hIBy{ zge**JNNSw?C5HbJAUcAV?d5Kn>e77)*I>`)#+*w>XSO*1JdIw$q|B`OD6@9w;Ii<@ z8Yi`ez%lozG47`D;jzO1bK*OD?S(HmC#+odf^*b0)yngveKPW9Y~;UNV0K^WI?hLxN77uvx7+l>=J#!(gPk;3IXlLuOkxdj;K)!)dZ-DeM3csh^KD~#U z2)QDgdfz=Yk|iM|aw0DYD3bRf0b>|3Hd@HKpC{%8*PW-H3#|6szu2{{XyDeodY3#& z{c)N^I}g;4*F8UH-SZ_QhbW4p!Wi)>79KiWcr(6jw*A;RRfK39g7BB))J??FsVOlz zWeY0LSo8c}v->ii*K1(@?qb(nuB}CRy{J^1irKIVQ%v}z2i zCXRq!Az~Y~!9Flf&GcNmxil~CAmh4*xIpZjDhd4L=c}F_Jwmu%cas=KM8FMeh&`;w z=_K#K$UES66N&Fh*^ke3O$6nyXS%MK20w9f>Mh76{|M^>?<~d0r66cAS!OG}=c7<) z3hArw2ORx4W+R z;up(CBbP5-`3NFG2hocWy`a}R0{^zdzAu#%1m+z~xvya^Kzt*c+(0{n7&(Y6Bg^+} z=&$%ZY+C{kl(@F^^~-tKo=bbGF9h4jCJ*!!eVDTk%2S)4wUA6#S#rvzzw-I5_bzWC!Rm=QNz9pvFhkP?;PZ-$q~Trj}vkGH)BmSWJ?47WFUNGeBSL-O`yYC8$7E z?4{qjkfgTV^j_!4J{nj`#yq!TGI(WS#XS}MOLW0*N@TD zlIvteZ+)Hq0Y$CF+>bccf}&q{;5%gd?qgYF5djV#qsOVKxJhbAl%dPhP!;yR@~`^I z`i9X*KA^)J$h$D&E|9dDsQ6Z*f7Mha7e3%}Z7m<@op5;=KI6W4gPMr~*u_wX$Q}&u z0r?3WzmKr@3BHL0J}mfZd{YHYCfC}+*_USCJ+y74)<@Nn-5Avkg5nfDMA{E+^&$Tv zlMiq8)p(`~nrw2-X=;}RzazVRkZ%sm;TSqx_#ys4uU+}KvvQXI%{St|fSi+mU*})+ z-l8Y{K%$Rg_@kiDcW``fv;B*U{1fr^my7%}d|w=IBdnh+pe}V^j=3)%b#s2!J<$N` z5kcV%JuGR^>GEP&=IOih(*Ev}{&~f{86f7u95D~f-_h&7 z)>oJ96Y~cRtGVtkt506ijiHthYsAx5Fl;3I(q@!xPbM<)2%G3KlSbM~#91rOHk#d5 zIKj5|u*)-XlNGs|Oe#3>7Q7V9y%aqAQqYPAW7(I2tv$iD@KR9ZxNcJt+ca5?w6$`) zu5kiqN?|F?$tJIIl417g3a|2dgaagnR=`wg6%;_7gMTuwhehSf+iktyy?@D#t7<;@ zN9C&x2cD_@@zG;TA86bH>Bn{3(r-!ls*zMq)zqr8u6VSpIw(sb;v<>Y1d-D?S*|Ku zpNM8+W?8igM?iua;xwKUMf-u@lD7`+J2Im{<#@=YYZxO!CN$IANU>Z%FoK#*`#$=ZpJt=m5D#X?$t&NGe z!Cqq-u|z!5YJ@vew&Nr*8)-q^$m>-s&sED@b7K(wm75!L+b%JpX)BgUn(T_Cm57=- zY4t`tWu(}vt#l82L!!rsreNbHa|637k?e#xN@thZ1>c}&p4((GvP`I!kYxgFRhQqQ z#fXDiW_yBdHX_OV4h@METNiIP(T>@_Hb2`@w8!TQ^-1tK9@11<1HUM$sDTTURu~+` zh2ZN^lsPD@0(enW!HH=r9fe0_jUd{Juq(REcoqBAj1jfktZ+8+tJ!O#>n(t$9wyE8 z2EfI(WHy3Q4Mxn2Bg+5*GnF=CU1in0>QWU&lQhA;`z1$-OHl*~l(Fyr2%mKg929Vp zrYgED$fBm4+x@)!`{MK)GTtm%4i>1Q&fRwFZPyuA)Qqs;&n_dGGTDZPMiwyYO2kv< zgx>`XotFrFHN>#Oak{8yTQ2Nb40{%c_Nl%HtNVLOfp9D{PdKRJFMTNJ+F3JLl%t&d zK_^hompuc>OIoqsOtR4WG_qm)xl?#SUvtumr;}_~rZsAX*_07AYZ9;~C~&c0I1!6k z>9iRMrW3(*hZzKq!s1|VaS$!$y2C~|66SzL)Cl-D91)F3STsyU=h{R?YZH{P%(WS& z-YRopRT5MaNW4`w6~0Y_1ntv7#Fw?r8vp5BL@R&Xxb~+X9oal%dzMh7a9EHzPB8ga zIl}2-K4P?TP)H)JrXY%y@Bi@!Y&6p8~k9524b?2>xBG8zm zhjbt(T~griX2@}RMs_wN>E~wc!^PhDcF$$BY;S+cSF&doN!fA(UT7~qg_qcyhT-;= zgN|ADfe-PEg)9MG-vb3BYOTvaF2Pk>A{m3a!WvdGl1y}=(s7L$He;>edy%)F9%Tf3 z>`BsPuRZ7p+3y~7%l65iJ1Shc;82GSf3wmqSDmx%P^CBIm7r8WXDmsIZ9L^IDxU}; zuAV&`ASe?cK-lYml2WE=ANvR|aj6`f8flWvZzM{tgvOeOmQBi{1%T;@yy*S7iV772=XT&(N1T z6`mIqyY5QY+(o!_m zz!L@hh5oNWwm;d(TtZE&k*?Sd-8ol58-=pAiIX={c}=*# zqB&`;W;&fn!X~LMD{ZnJR(l5<&cr4j=cpmXA}wP^veQf>m0&~Dhz3!qn#{5J%z;9d zMka+jkxhtCLph2K^+7ve_JGzMCKUNxo@G}@jI@CS=QtRwU2mnV)+iv8fRyinx7g#G zcC+yWSeCL7R#qJGL9U%NBI}`ZG}_Iqh(s#;YEI`Yvg`)vAG)GOxDTou&#@heOfm%$ z^64E#s3J8{j}u93V2w0F4aEgp$%RxX^a}xgBMExJj^Pf3A>xHRAYoJ=GElfXj1gjEC5!L>lz#smeM7MJ~V{T!;ECXccogh1`73SAA)= zGMxaQz`&OCRlv1XtkIfEMA>qwikH-dtO0=u>Ld5AXl`E7vaD$-?8Bm delta 2435 zcmZ9Oe^3Eg7Xw{gLA)2J9C539# zCh|*Yq7n6gRSJZJG%<;YaCWy-69aM5s?F$`=7&;8rO|5BWV})>X{)C1dvZ4>_s3^; z-}n3cc%OZqcdq(MT6J~Wo*3aG0O0CgM;3jUwek0TPjzY17vw3r2)^`MZR>qoZI5uK z-6{82d{4)`1jVduy0ysV8=a)s(I`p;KH0gC^K~Yd^IpGwk}ol3m?VtK2GH(kquPk} zjdn@9pna;H)lO?4X#?6Z&CvR^Ltck#vgHYFxmKbTXpebwTuF&n)ywLT`kCri|D^sw z9aP^_O|@U`Q4gx!YNvWYZC0DSwXVdhd8$*LsZLYl)mU|+I!;woNrlQiOuJS5%6ip$$$H+KW8aeZ zB>4#`A^Bu6aglVALS~WaB!R?|3B*QZg5-PhxAKVmwfr~viaaELDxZ;0${)(_$%b}A z`%)=Za+C$ufb|pW$G$=PfD|J@i3g)W{yWIqLEfVDo?9}Tlf0Zej&icQfO3O-oGYmz_6a7oWeU&kU&F57BVg< zTw#;AC2TK`;V~w7tuShq#67~^vl(0rSqzI9G8r-$7BZw$c&`_hMM~UM_CAFnks*QY zc#4W>oZg}+%9BM&l!2o0a1>&nSbUkYS)#B@y+kfcSKxWLPdi4xg^laROEb|B5QgZR zU*MY)y~kT(1c^1c?@|kr`@8TO)Cf=`-G2e`t()|6#El8eorqgb3>$}?=q|9_WpnT^ z3%?8KLwRA%fAhFu!6@eN0Dc`azyxX*1>u2J8G2tOnjrsb zem%(!*1??K4UbukBM>Yg7s57l9Khq$?kKg(TEgtMKVVlM*}hq9 zjIVW%*_~L599Clu!T~e__z`M%gXi0bezrsimF#AT@HLePv?r2)ltd+QQbNJ;0hBLqYfDVQwO!Ei4K|Y5vBE{bDOxsUKc9*TaV%l zt5J*vhaklRDW1lfNDK6Yes41lr8DVy{Jv;ZV|vn2fJdq18_e&+`o-3;e09VPs%m3kK#6B3}EcUJ%HZn zJ>Eh4YttJlH^n`8Zh8m+#QMy-v4SSgOHE6iGvA}fPldVWCjl%%USHXIE1a9-7CuI- z3T-7Go&eT%c!H*9Lh}R^FGzetX-kF}eSbiHyHd=7GE;J>7oc9M`wbn&LVeeGF;t90 zN67898fV!!(11Pa|Ma`7& z*VkTJuXV(U_1kMg9dWSAl1_)uLOZDg2p!bWOK%^KUUSV7s|0K1dVB-6?#?=B$AK|`xJ7T&{N`Kd!%W6f7chp%b>xQL+@nrC=8=JPsC~&WuOj+MtBIv^OYO7GsY>zs)qj zO`~37In=SNrc&QqW{rq!UoP(U>=4HWCDplLPFk`@_q<7R&8mMYPpd~&+JdJ5e~KD@ xK*P_~506trzWw9W2*q%y$cCb&N?^)P)gjS;Ms9Xt$8h@Dr8w98V1b$~{vTh}eHj1% diff --git a/data/signals.db-shm b/data/signals.db-shm deleted file mode 100644 index 36f17debe825a8557e2eec0ef42a63efe743cfbe..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 32768 zcmeI*F-}5J5Cu>XM4&K~bT+oX0heQCLr>ufU>~;DB<_K2sD%x*lrV#k&{B}R_|8pk zKKUV!d1nE`=&QNOwEj9A#PcWm@$fs`t`^(w+h+dx_&%OZU!IohW;wd>`*~g_pS!ah zULN1)=luP4v=hCHcB7N%bM!U(9nHCf009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkL{KMCa42_Zm$ z0D-y$ZtJ?P+Y&ev$X#1PpiY6@GA9J;6v+LGLVy4P0tCtw$PK_kpiY7NI=0.27.0 -pydantic>=2.0.0 -pydantic-settings>=2.0.0 -python-dotenv>=1.0.0 - -# AI/LLM (OpenAI-compatible API) -openai>=1.0.0 - -# Web search fallback -duckduckgo-search>=7.0.0 - -# CLI and formatting -rich>=13.0.0 -typer>=0.9.0 - -# Telegram channel monitoring -telethon>=1.36.0 - -# Dashboard -fastapi>=0.100.0 -uvicorn>=0.20.0 diff --git a/src/config/settings.py b/src/config/settings.py index b75b616..df8f7b9 100644 --- a/src/config/settings.py +++ b/src/config/settings.py @@ -15,13 +15,6 @@ class Settings(BaseSettings): gemini_api_key: str = Field(default="", alias="GEMINI_API_KEY") llm_base_url: str = Field(default="http://apicz.boyuerichdata.com/v1/", alias="LLM_BASE_URL") - # Trade data API mode: "internal" (private API) or "official" (Polymarket data-api) - trade_api_mode: str = Field(default="official", alias="TRADE_API_MODE") - - # Internal trade data API (only used when TRADE_API_MODE=internal) - internal_api_url: str = Field(default="http://103.197.25.170:18088", alias="INTERNAL_API_URL") - internal_api_key: str = Field(default="", alias="INTERNAL_API_KEY") - # Twitter API (for social sentiment search) twitter_api_key: str = Field(default="", alias="TWITTER_API_KEY") @@ -49,8 +42,6 @@ class Settings(BaseSettings): telegram_session_string: str = Field(default="", alias="TELEGRAM_SESSION_STRING") telegram_channels: str = Field(default="", alias="TELEGRAM_CHANNELS") - # Polygon Wallet - polygon_wallet_private_key: str = Field(default="", alias="POLYGON_WALLET_PRIVATE_KEY") # MongoDB mongodb_uri: str = Field(default="mongodb://localhost:27017/whale_watcher", alias="MONGODB_URI") @@ -67,12 +58,19 @@ class Settings(BaseSettings): fetch_interval_seconds: int = Field(default=15, alias="FETCH_INTERVAL_SECONDS") trending_markets_limit: int = Field(default=50, alias="TRENDING_MARKETS_LIMIT") + # Tiered market monitoring (full-coverage mode) + full_market_scan: bool = Field(default=True, alias="FULL_MARKET_SCAN") + tier1_volume_min: float = Field(default=500_000, alias="TIER1_VOLUME_MIN") + tier2_volume_min: float = Field(default=10_000, alias="TIER2_VOLUME_MIN") + tier3_volume_min: float = Field(default=1_000, alias="TIER3_VOLUME_MIN") + tier1_poll_interval: int = Field(default=15, alias="TIER1_POLL_INTERVAL") + tier2_poll_interval: int = Field(default=60, alias="TIER2_POLL_INTERVAL") + tier3_poll_interval: int = Field(default=300, alias="TIER3_POLL_INTERVAL") + # LLM Settings llm_model: str = Field(default="gemini-3-flash-preview", alias="LLM_MODEL") llm_temperature: float = Field(default=0.0, alias="LLM_TEMPERATURE") - # Trade Execution - enable_trade_execution: bool = Field(default=False, alias="ENABLE_TRADE_EXECUTION") # Email notification email_smtp_server: str = Field(default="smtp.qq.com", alias="EMAIL_SMTP_SERVER") diff --git a/src/main.py b/src/main.py index 4056d70..123858e 100644 --- a/src/main.py +++ b/src/main.py @@ -230,17 +230,58 @@ class WhaleWatcher: async def refresh_markets(self) -> None: """Fetch and update the list of monitored markets.""" + if self.settings.full_market_scan: + await self._refresh_markets_tiered() + else: + await self._refresh_markets_legacy() + + async def _refresh_markets_tiered(self) -> None: + """Full-coverage tiered monitoring (mirrors options flow passive approach).""" + logger.info("Fetching ALL active markets for tiered monitoring...") + + tiers = self.market_fetcher.get_tiered_markets() + + # Also merge token launch markets into appropriate tiers + existing_ids = set() + for tier_markets in tiers.values(): + for tm in tier_markets: + existing_ids.add(tm.market.id) + + token_markets = self.market_fetcher.get_token_launch_markets() + token_added = 0 + for tm in token_markets: + if tm.market.id not in existing_ids: + # Assign to tier based on volume + vol = tm.volume_24hr + if vol >= self.settings.tier1_volume_min: + tiers["tier1"].append(tm) + elif vol >= self.settings.tier2_volume_min: + tiers["tier2"].append(tm) + else: + tiers["tier3"].append(tm) + existing_ids.add(tm.market.id) + token_added += 1 + + if token_added: + logger.info(f"Added {token_added} token launch markets to tiers") + + total = sum(len(v) for v in tiers.values()) + if total > 0: + self.trade_monitor.set_tiered_markets(tiers) + logger.info(f"Tiered monitoring active: {total} markets total") + else: + logger.error("Failed to fetch any markets") + + async def _refresh_markets_legacy(self) -> None: + """Original Top-N trending markets mode.""" logger.info("Fetching trending markets...") trending_markets = self.market_fetcher.get_trending_markets( limit=self.settings.trending_markets_limit ) - # Additionally scan for specialized market categories - # that may not be in the top trending list existing_ids = {tm.market.id for tm in trending_markets} - # 1. Token launch / crypto project markets token_markets = self.market_fetcher.get_token_launch_markets() token_added = 0 for tm in token_markets: diff --git a/src/prompts/whale_analyzer.py b/src/prompts/whale_analyzer.py index 8d06d00..df33a26 100644 --- a/src/prompts/whale_analyzer.py +++ b/src/prompts/whale_analyzer.py @@ -59,14 +59,14 @@ You can call the following tools to obtain real-time information (all results ar **Tool usage principles**: - Based on market type and trade characteristics, decide which tools to call - You may call one, multiple, or zero tools -- You may call the same tool multiple times with different keywords -- If trade size is very large or information asymmetry suspicion is high, search more aggressively +- **You have a maximum of 3 tool-call rounds. Budget wisely: use Round 1 for broad search (web + twitter + telegram in parallel), Round 2 for targeted follow-up if needed, then produce your final analysis. Do NOT use all rounds just searching — reserve capacity for your final answer.** +- Do NOT call the same tool (e.g. search_web) more than 3 times total across all rounds +- If the first search already covers the topic well, stop searching and analyze **Tool collaboration and cross-verification (important)**: -- Information from different tools must be **cross-verified** — do not draw conclusions from a single source. Example: if web search finds a policy rumor, verify with Twitter for public reaction and corroborate with economic data -- If you discover new leads or keywords while using one tool, **immediately call other tools to follow up**. Example: if news search reveals an official's resignation, search for that person's name for more details and check Twitter for unreported information -- When multiple tools return **contradictory results**, explicitly note the discrepancy and lower confidence — do not cherry-pick -- Encourage "search chain" investigation: first-round search → discover leads → targeted second-round → deep third-round, progressing layer by layer rather than skimming the surface +- Information from different tools should be **cross-verified** — but 2-3 sources are sufficient, do not over-search +- When multiple tools return **contradictory results**, explicitly note the discrepancy and lower confidence +- Prefer breadth (web + twitter + domain-specific tool) over depth (web × 8 with slightly different keywords) ## Polymarket Trading Mechanics diff --git a/src/services/anomaly_detector.py b/src/services/anomaly_detector.py index ec60bd7..00a4b29 100644 --- a/src/services/anomaly_detector.py +++ b/src/services/anomaly_detector.py @@ -1,9 +1,21 @@ -"""Anomaly detection service - multi-dimensional scoring for whale trades.""" +""" +Anomaly detection service — confidence scoring for whale trades. + +Mirrors the options flow confidence scoring from llm-trading-agent: + Base confidence: 0.50 + + Premium-to-threshold ratio: +0.20 (sqrt-scaled by market liquidity) + + Signal cleanliness (ask ratio): +0.10 + + Volume/OI equivalent (depth ratio): +0.10 + + Alert rule / cluster tier: +0.10 + +Total max = 1.0. Pre-filter threshold = 0.60 (matches options flow pipeline). +""" import logging +import math import time from collections import defaultdict, deque +from datetime import datetime from typing import Dict, List, Optional, Tuple -from datetime import datetime, timedelta from src.config import get_settings from src.models.trade import WhaleTrade, TradeActivity, TraderHistory @@ -15,51 +27,29 @@ logger = logging.getLogger(__name__) class AnomalyDetector: """ - Multi-dimensional anomaly detection for whale trades. + Confidence scoring for whale trades, mirroring options flow pipeline. - Scoring dimensions: - 1. Size relative to market (trade vs market 24h volume) - 2. Price uncertainty (closer to 0.5 = more uncertain = more interesting) - 3. Time-of-day (off-peak hours = more suspicious) - 4. Trader deviation (trade size vs trader's historical average) - 5. Cluster signal (multiple same-direction trades in short window) + 5-factor scoring (same structure as OptionsFlowSignalProvider._calculate_confidence): + 1. Base confidence: 0.50 + 2. Premium-to-threshold ratio: +0.20 (trade_size vs dynamic threshold) + 3. Signal cleanliness: +0.10 (conviction / price displacement) + 4. Depth ratio: +0.10 (trade_size vs market liquidity, like Volume/OI) + 5. Cluster tier: +0.10 (repeated same-direction trades, like alert_rule) """ - # --- Time-of-day weights (US Eastern Time) --- - # Polymarket is US-dominated, so we use ET to judge trading hour anomaly. - # Higher weight = more unusual trading hour = more suspicious. - _ET_HOUR_WEIGHTS = { - # ET 0-5 (midnight to 5am) — very unusual, most suspicious - 0: 0.6, 1: 0.7, 2: 0.8, 3: 0.9, 4: 0.8, 5: 0.6, - # ET 6-8 (early morning) — some early traders - 6: 0.4, 7: 0.3, 8: 0.2, - # ET 9-17 (US business hours) — peak activity, least suspicious - 9: 0.1, 10: 0.0, 11: 0.0, 12: 0.0, 13: 0.0, - 14: 0.0, 15: 0.0, 16: 0.0, 17: 0.1, - # ET 18-20 (evening) — moderate - 18: 0.2, 19: 0.2, 20: 0.3, - # ET 21-23 (late night) — unusual - 21: 0.4, 22: 0.5, 23: 0.5, - } - # UTC offset for US Eastern: -5 (EST) or -4 (EDT). - # Use -4 as default (EDT covers ~Mar-Nov, most of the year). - _ET_UTC_OFFSET = -4 - - # Cluster detection: track recent trades per market - # Key: market_id, Value: deque of (timestamp, side, usdc_size) + # Cluster detection _CLUSTER_WINDOW_SECONDS = 300 # 5 minutes - _CLUSTER_MIN_COUNT = 3 # minimum trades for cluster signal + _CLUSTER_MIN_COUNT = 3 def __init__(self): self.settings = get_settings() self.trader_profiler = TraderProfiler() - # Recent trades for cluster detection: market_id -> deque self._recent_trades: Dict[str, deque] = defaultdict( lambda: deque(maxlen=50) ) # ================================================================ - # Core scoring + # Core scoring — mirrors OptionsFlowSignalProvider._calculate_confidence # ================================================================ def get_anomaly_score( @@ -70,81 +60,38 @@ class AnomalyDetector: market_id: str = "", ) -> Tuple[float, dict]: """ - Calculate multi-dimensional anomaly score. + Calculate confidence score using options-flow-style 5-factor model. Returns: - (total_score, breakdown_dict) where breakdown has per-dimension scores. + (total_score, breakdown_dict) — total in [0.0, 1.0]. """ breakdown = {} - # --- 1. Size score (absolute) --- - # $5k=0.1, $20k=0.25, $50k=0.4, $100k+=0.5 - raw_size = min(0.5, 0.1 + (activity.usdc_size - 5000) / 250000) - breakdown["size_abs"] = max(0.0, raw_size) + # --- 1. Base confidence: 0.50 --- + breakdown["base"] = 0.50 - # --- 2. Size relative to market volume --- - if market and market.volume_24hr > 0: - # What fraction of 24h volume is this single trade? - ratio = activity.usdc_size / market.volume_24hr - # ratio 0.001=noise, 0.01=notable, 0.05=significant, 0.1+=massive - rel_score = min(0.3, ratio * 6.0) # 0.05 ratio -> 0.3 - breakdown["size_relative"] = rel_score - else: - breakdown["size_relative"] = 0.15 # unknown market volume, use neutral + # --- 2. Premium-to-threshold ratio: +0.20 max --- + # Mirrors _premium_ratio_bonus: min(0.20, (sqrt(ratio) - 1) * 0.20) + # where ratio = trade_size / dynamic_threshold + # dynamic_threshold = base_size * sqrt(market_volume / baseline_volume) + breakdown["premium_ratio"] = self._premium_ratio_bonus(activity, market) - # --- 3. Price uncertainty --- - # Price is taker's buy price (no normalization). - # Lower price = more uncertain/risky bet = more interesting. - # 0.5 -> 0.2, 0.3/0.7 -> 0.1, 0.1/0.9 -> 0.0 - dist = abs(activity.price - 0.5) - if dist <= 0.3: - price_score = 0.2 * (1 - dist / 0.3) - else: - price_score = 0.0 - breakdown["price_uncertainty"] = price_score + # --- 3. Signal cleanliness (conviction): +0.10 max --- + # Mirrors _ask_ratio_bonus: measures taker aggressiveness. + # In options: ASK-side ratio > 0.8 = full bonus. + # In Polymarket: price displacement from market mid = conviction. + breakdown["signal_clean"] = self._signal_cleanliness_bonus(activity, market) - # --- 4. Time-of-day --- - utc_hour = datetime.utcfromtimestamp(activity.timestamp).hour - et_hour = (utc_hour + self._ET_UTC_OFFSET) % 24 - breakdown["time_of_day"] = self._ET_HOUR_WEIGHTS.get(et_hour, 0.1) * 0.15 + # --- 4. Depth ratio (like Volume/OI): +0.10 max --- + # Mirrors _volume_oi_bonus: trade_size / market_liquidity. + # High ratio = new significant positioning. + breakdown["depth_ratio"] = self._depth_ratio_bonus(activity, market) - # --- 5. Trader deviation --- - if trader_history and trader_history.avg_trade_size > 0: - # How many X of their average is this trade? - multiple = activity.usdc_size / trader_history.avg_trade_size - # 1x=normal, 2x=notable, 5x=very unusual, 10x+=extreme - if multiple >= 5: - deviation_score = 0.15 - elif multiple >= 2: - deviation_score = 0.05 + (multiple - 2) / 3 * 0.10 - else: - deviation_score = 0.0 - breakdown["trader_deviation"] = deviation_score - else: - # Unknown trader history — slightly suspicious - breakdown["trader_deviation"] = 0.05 - - # --- 6. Cluster signal --- - cluster_score = self._get_cluster_score(activity, market_id=market_id) - breakdown["cluster"] = cluster_score - - # --- 7. Niche market bonus --- - # Small/niche markets have higher information asymmetry value. - # Large political/macro markets (volume > $5M/day) are noisy; - # small markets (< $500k/day) are where insider signals matter most. - if market and market.volume_24hr > 0: - vol = market.volume_24hr - if vol < 100_000: - niche_score = 0.15 # very niche - elif vol < 500_000: - niche_score = 0.10 - elif vol < 2_000_000: - niche_score = 0.05 - else: - niche_score = 0.0 # large/macro market, no bonus - breakdown["niche_market"] = niche_score - else: - breakdown["niche_market"] = 0.05 + # --- 5. Cluster tier (like alert_rule): +0.10 max --- + # Mirrors _alert_rule_bonus: repeated same-direction activity. + # In options: RepeatedHitsAscendingFill = 0.10, RepeatedHits = 0.07. + # In Polymarket: multiple same-direction trades in 5-min window. + breakdown["cluster_tier"] = self._cluster_tier_bonus(activity, market_id) # --- Total --- total = sum(breakdown.values()) @@ -152,22 +99,113 @@ class AnomalyDetector: return total, breakdown - def record_trade(self, activity: TradeActivity, market_id: str): - """Record a trade for cluster detection. Call for every trade, not just whales.""" - self._recent_trades[market_id].append(( - activity.timestamp, - activity.side, - activity.usdc_size, - )) - - def _get_cluster_score(self, activity: TradeActivity, market_id: str = "") -> float: + @staticmethod + def _premium_ratio_bonus(activity: TradeActivity, market: Optional[Market]) -> float: """ - Check if there are multiple same-direction trades in a short window. + Trade size vs dynamic threshold, sqrt-scaled. - A cluster of BUY or SELL in the same market in 5 minutes suggests - coordinated or informed trading. + Mirrors OptionsFlowSignalProvider._premium_ratio_bonus: + threshold = 100K * sqrt(market_cap / 40B) + bonus = min(0.20, (sqrt(premium / threshold) - 1) * 0.20) + + Polymarket mapping: + threshold = base_size * sqrt(market_volume / baseline_volume) + base_size = $5,000, baseline_volume = $1,000,000 + """ + max_bonus = 0.20 + trade_size = activity.usdc_size + + if trade_size <= 0: + return 0.0 + + # Dynamic threshold based on market volume (like market-cap scaling) + base_size = 10_000.0 + baseline_volume = 1_000_000.0 + + if market and market.volume > 0: + threshold = base_size * math.sqrt(market.volume / baseline_volume) + threshold = max(5_000.0, threshold) # floor $5K + else: + threshold = base_size + + ratio = trade_size / threshold + if ratio <= 1.0: + return 0.0 + + bonus = (math.sqrt(ratio) - 1) * max_bonus + return min(max_bonus, max(0.0, bonus)) + + @staticmethod + def _signal_cleanliness_bonus(activity: TradeActivity, market: Optional[Market]) -> float: + """ + Taker conviction / price displacement from market mid. + + Mirrors _ask_ratio_bonus logic: + ASK ratio > 0.8 → 0.10 (full), > 0.6 → 0.05 (half) + + Polymarket mapping: + A buyer paying significantly above market mid = aggressive taker (like ASK-side). + Displacement > 5% → 0.10, > 2% → 0.05. + """ + if not market or not market.outcome_prices: + return 0.0 + + # Get market mid price for the outcome the trader bought + if activity.outcome == "Yes": + market_mid = market.outcome_prices[0] + elif len(market.outcome_prices) > 1: + market_mid = market.outcome_prices[1] + else: + market_mid = 1.0 - market.outcome_prices[0] + + displacement = activity.price - market_mid + + if displacement > 0.05: + return 0.10 # strong conviction (like ASK ratio > 0.8) + if displacement > 0.02: + return 0.05 # moderate conviction (like ASK ratio > 0.6) + return 0.0 + + @staticmethod + def _depth_ratio_bonus(activity: TradeActivity, market: Optional[Market]) -> float: + """ + Trade size vs market liquidity (like Volume/OI). + + Mirrors _volume_oi_bonus: + V/OI > 3.0 → 0.10, > 1.5 → 0.07, > 1.0 → 0.03 + + Polymarket mapping: + depth_ratio = trade_size / market_liquidity + > 0.10 → 0.10, > 0.05 → 0.07, > 0.02 → 0.03 + """ + if not market or not market.liquidity or market.liquidity <= 0: + return 0.0 + + ratio = activity.usdc_size / market.liquidity + + if ratio > 0.10: + return 0.10 + if ratio > 0.05: + return 0.07 + if ratio > 0.02: + return 0.03 + return 0.0 + + def _cluster_tier_bonus(self, activity: TradeActivity, market_id: str = "") -> float: + """ + Cluster of same-direction trades in short window (like alert_rule tiers). + + Mirrors _alert_rule_bonus tier structure: + RepeatedHitsAscendingFill → 0.10 + RepeatedHits → 0.07 + SweepsFollowedByFloor → 0.05 + Single sweep → 0.03 + + Polymarket mapping: + 5+ same-direction trades in 5min → 0.10 + 3-4 trades → 0.07 + 2 trades with large volume → 0.03 """ - # Use the same market_id key as record_trade() key = market_id or activity.condition_id recent = self._recent_trades.get(key) if not recent: @@ -176,7 +214,6 @@ class AnomalyDetector: now = activity.timestamp cutoff = now - self._CLUSTER_WINDOW_SECONDS - # Count same-direction trades in window same_dir_count = 0 same_dir_volume = 0.0 for ts, side, size in recent: @@ -184,14 +221,22 @@ class AnomalyDetector: same_dir_count += 1 same_dir_volume += size + if same_dir_count >= 5: + return 0.10 if same_dir_count >= self._CLUSTER_MIN_COUNT: - # 3 trades = 0.05, 5+ = 0.10, volume also matters - count_score = min(0.10, 0.02 * same_dir_count) - vol_bonus = min(0.05, same_dir_volume / 500000) - return count_score + vol_bonus - + return 0.07 + if same_dir_count >= 2 and same_dir_volume > 20_000: + return 0.03 return 0.0 + def record_trade(self, activity: TradeActivity, market_id: str): + """Record a trade for cluster detection. Call for every trade, not just whales.""" + self._recent_trades[market_id].append(( + activity.timestamp, + activity.side, + activity.usdc_size, + )) + # ================================================================ # Pre-filter (before LLM) # ================================================================ @@ -202,13 +247,13 @@ class AnomalyDetector: market: Optional[Market] = None, trader_history: Optional[TraderHistory] = None, market_id: str = "", - min_score: float = 0.40, + min_score: float = 0.65, ) -> Tuple[bool, float, dict]: """ Decide whether a whale trade warrants LLM analysis. - Returns: - (should_analyze, score, breakdown) + Threshold 0.65: requires at least base (0.50) + one strong factor + to trigger LLM analysis. """ score, breakdown = self.get_anomaly_score( activity, market, trader_history, market_id=market_id, @@ -230,9 +275,9 @@ class AnomalyDetector: def filter_whale_trades( self, trades: List[WhaleTrade], - min_score: float = 0.5, + min_score: float = 0.65, ) -> List[WhaleTrade]: - """Filter whale trades by anomaly score.""" + """Filter whale trades by confidence score.""" filtered = [] for trade in trades: score, _ = self.get_anomaly_score(trade.trade) @@ -248,16 +293,13 @@ class AnomalyDetector: """Analyze the context of a whale trade for LLM input.""" trade = whale_trade.trade - # Direction interpretation (only BUY trades, no normalization) if trade.outcome == "Yes": direction_meaning = f"Trader bought Yes Token @ {trade.price:.4f} — Bullish (believes event will occur)" else: direction_meaning = f"Trader bought No Token @ {trade.price:.4f} — Bearish (believes event will NOT occur)" - # Buy price directly reflects taker's conviction — lower price = higher odds bet implied_prob = trade.price - # Market state market_state = "uncertain" if whale_trade.market_outcome_prices: max_price = max(whale_trade.market_outcome_prices) @@ -266,7 +308,6 @@ class AnomalyDetector: elif max_price < 0.6: market_state = "highly uncertain" - # Multi-dimensional anomaly score score, breakdown = self.get_anomaly_score(trade) return { @@ -289,12 +330,10 @@ class AnomalyDetector: context = self.analyze_trade_context(whale_trade) trade = whale_trade.trade - # Build outcome prices string prices_str = "" for outcome, price in zip(context["market_outcomes"], context["current_prices"]): prices_str += f" - {outcome}: {price:.2%}\n" - # Trader profile trader_profile = self.trader_profiler.generate_profile( wallet_address=trade.proxy_wallet or "Unknown", ranking=whale_trade.trader_ranking, @@ -302,15 +341,13 @@ class AnomalyDetector: ) trader_profile_str = self.trader_profiler.format_profile_for_llm(trader_profile) - # Anomaly breakdown string bd = context["anomaly_breakdown"] breakdown_str = ( - f" Absolute size: {bd.get('size_abs', 0):.2f} | " - f"Relative to market: {bd.get('size_relative', 0):.2f} | " - f"Price uncertainty: {bd.get('price_uncertainty', 0):.2f} | " - f"Time of day: {bd.get('time_of_day', 0):.2f} | " - f"Trader deviation: {bd.get('trader_deviation', 0):.2f} | " - f"Cluster signal: {bd.get('cluster', 0):.2f}" + f" Base: {bd.get('base', 0):.2f} | " + f"Premium ratio: {bd.get('premium_ratio', 0):.2f} | " + f"Signal clean: {bd.get('signal_clean', 0):.2f} | " + f"Depth ratio: {bd.get('depth_ratio', 0):.2f} | " + f"Cluster tier: {bd.get('cluster_tier', 0):.2f}" ) return f""" @@ -323,7 +360,7 @@ class AnomalyDetector: - **Trade time**: {datetime.fromtimestamp(trade.timestamp).strftime('%Y-%m-%d %H:%M:%S UTC')} - **Trader wallet**: {trade.proxy_wallet or 'Unknown'} -### Anomaly Score +### Confidence Score - **Overall score**: {context['anomaly_score']:.2f}/1.00 - **Score breakdown**: {breakdown_str} diff --git a/src/services/etherscan.py b/src/services/etherscan.py index 44b4ad3..349e30a 100644 --- a/src/services/etherscan.py +++ b/src/services/etherscan.py @@ -7,19 +7,25 @@ import httpx logger = logging.getLogger(__name__) -ETHERSCAN_API = "https://api.etherscan.io/api" +ETHERSCAN_API_V2 = "https://api.etherscan.io/v2/api" -# Well-known ERC-20 token contracts on Ethereum mainnet +# Default chain: Polygon (137) where Polymarket operates. +# Ethereum mainnet = 1, can be overridden per-call. +DEFAULT_CHAIN_ID = 137 + +# Well-known ERC-20 token contracts on Polygon TOKEN_CONTRACTS = { - "USDC": "0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48", - "USDT": "0xdac17f958d2ee523a2206206994597c13d831ec7", - "WETH": "0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2", - "DAI": "0x6b175474e89094c44da98b954eedeac495271d0f", + "USDC": "0x3c499c542cef5e3811e1192ce70d8cc03d5c3359", # USDC native on Polygon + "USDC.e": "0x2791bca1f2de4661ed88a30c99a7a9449aa84174", # USDC.e (bridged) on Polygon + "USDT": "0xc2132d05d31c914a87c6611c10748aeb04b58e8f", + "WETH": "0x7ceb23fd6bc0add59e62ac25578270cff1b9f619", + "DAI": "0x8f3cf7ad23cd3cadbd9735aff958023239c6a063", } # Decimals per token (used for converting raw amounts) TOKEN_DECIMALS = { "USDC": 6, + "USDC.e": 6, "USDT": 6, "WETH": 18, "DAI": 18, @@ -65,10 +71,11 @@ class EtherscanService: def is_available(self) -> bool: return bool(self.api_key and self.api_key.strip()) - def _get(self, params: dict) -> dict: - """Make authenticated GET request to Etherscan API.""" + def _get(self, params: dict, chain_id: int = DEFAULT_CHAIN_ID) -> dict: + """Make authenticated GET request to Etherscan V2 API.""" params["apikey"] = self.api_key - resp = self._client.get(ETHERSCAN_API, params=params) + params["chainid"] = chain_id + resp = self._client.get(ETHERSCAN_API_V2, params=params) resp.raise_for_status() return resp.json() @@ -111,18 +118,30 @@ class EtherscanService: # Filter by token if not "ALL" if token_upper != "ALL": - contract = TOKEN_CONTRACTS.get(token_upper, "").lower() - if contract: + # For USDC, match both native and bridged (USDC.e) contracts + if token_upper == "USDC": + allowed = { + TOKEN_CONTRACTS.get("USDC", "").lower(), + TOKEN_CONTRACTS.get("USDC.e", "").lower(), + } + allowed.discard("") transfers = [ tx for tx in transfers - if tx.get("contractAddress", "").lower() == contract + if tx.get("contractAddress", "").lower() in allowed ] else: - # Try matching by symbol in the response - transfers = [ - tx for tx in transfers - if tx.get("tokenSymbol", "").upper() == token_upper - ] + contract = TOKEN_CONTRACTS.get(token_upper, "").lower() + if contract: + transfers = [ + tx for tx in transfers + if tx.get("contractAddress", "").lower() == contract + ] + else: + # Try matching by symbol in the response + transfers = [ + tx for tx in transfers + if tx.get("tokenSymbol", "").upper() == token_upper + ] if not transfers: return f"No {token_upper} transfers found for {_short_address(addr)} in the last 20 token transactions." diff --git a/src/services/llm_analyzer.py b/src/services/llm_analyzer.py index 3d32cbf..8c9b6a7 100644 --- a/src/services/llm_analyzer.py +++ b/src/services/llm_analyzer.py @@ -32,7 +32,7 @@ logger = logging.getLogger(__name__) # Maximum tool-use rounds to prevent infinite loops # 14 tools available; LLM can call multiple per round but may need # several rounds for chain-of-investigation (search → discover → verify) -MAX_TOOL_ROUNDS = 5 +MAX_TOOL_ROUNDS = 3 class LLMAnalyzer: @@ -259,12 +259,25 @@ class LLMAnalyzer: # Tool-use loop for round_idx in range(MAX_TOOL_ROUNDS + 1): + # Last round: no tools, force final answer + is_last_round = (round_idx == MAX_TOOL_ROUNDS) + if is_last_round: + messages.append({ + "role": "user", + "content": ( + "You have used all available tool rounds. " + "Based on all information gathered, provide your final " + "analysis and output the JSON assessment now." + ), + }) + # Call LLM call_kwargs = { "model": self.settings.llm_model, "messages": messages, + "max_tokens": 4096, } - if tool_schemas and round_idx < MAX_TOOL_ROUNDS: + if tool_schemas and not is_last_round: call_kwargs["tools"] = tool_schemas call_kwargs["tool_choice"] = "auto" @@ -289,10 +302,13 @@ class LLMAnalyzer: # No tool calls — final response analysis_text = msg.content or "" + finish_reason = response.choices[0].finish_reason logger.info( f"Analysis complete after {round_idx + 1} round(s) " - f"({len(analysis_text)} chars)" + f"({len(analysis_text)} chars, finish={finish_reason})" ) + if len(analysis_text) < 200: + logger.warning(f"Suspiciously short response: {analysis_text[:200]}") break # Parse JSON decision from final response diff --git a/src/services/market_fetcher.py b/src/services/market_fetcher.py index d58fd6e..37770c4 100644 --- a/src/services/market_fetcher.py +++ b/src/services/market_fetcher.py @@ -372,6 +372,59 @@ class MarketFetcher: logger.error(f"Error fetching niche markets: {e}") return niche_markets + def get_tiered_markets(self) -> dict[str, list[TrendingMarket]]: + """ + Fetch ALL active markets and classify into 3 tiers by 24h volume. + + Returns dict with keys "tier1", "tier2", "tier3", each a list of TrendingMarket. + Mirrors options flow's "passive receive all signals" approach: + scan everything, filter later. + """ + settings = self.settings + all_markets = self.get_all_current_markets() + + tiers: dict[str, list[TrendingMarket]] = {"tier1": [], "tier2": [], "tier3": []} + + for market in all_markets: + # Apply standard filters + raw = { + "question": market.question, + "description": market.description, + "slug": market.slug, + } + if self._should_filter_market(raw): + continue + + if not market.clob_token_ids or not market.active or market.closed: + continue + + vol = market.volume_24hr + + tm = TrendingMarket( + market=market, + volume_24hr=vol, + liquidity=market.liquidity, + ) + + if vol >= settings.tier1_volume_min: + tiers["tier1"].append(tm) + elif vol >= settings.tier2_volume_min: + tiers["tier2"].append(tm) + elif vol >= settings.tier3_volume_min: + tiers["tier3"].append(tm) + # vol < tier3_volume_min → dead market, skip + + # Sort each tier by volume descending + for tier in tiers.values(): + tier.sort(key=lambda t: t.volume_24hr, reverse=True) + + logger.info( + f"Tiered markets: Tier1={len(tiers['tier1'])} (>{settings.tier1_volume_min/1000:.0f}K), " + f"Tier2={len(tiers['tier2'])} (>{settings.tier2_volume_min/1000:.0f}K), " + f"Tier3={len(tiers['tier3'])} (>{settings.tier3_volume_min/1000:.0f}K)" + ) + return tiers + def get_market_by_id(self, market_id: str) -> Optional[Market]: """ Fetch a single market by ID. diff --git a/src/services/telegram_search.py b/src/services/telegram_search.py index 0356ecb..8acb1e9 100644 --- a/src/services/telegram_search.py +++ b/src/services/telegram_search.py @@ -167,11 +167,20 @@ class TelegramSearchService: # Calculate per-channel limit limit_per_channel = max(3, limit // len(self.channels)) - # Run async search (nest_asyncio allows nested run_until_complete) - loop = asyncio.get_event_loop() - messages = loop.run_until_complete( - self._search_all_channels(query, limit_per_channel) - ) + # Run async search — handle both sync and async calling contexts + coro = self._search_all_channels(query, limit_per_channel) + try: + loop = asyncio.get_running_loop() + # Already inside an async event loop — use a new thread + import concurrent.futures + with concurrent.futures.ThreadPoolExecutor() as pool: + messages = pool.submit( + asyncio.run, coro + ).result(timeout=30) + except RuntimeError: + # No running loop — safe to use run_until_complete + loop = asyncio.get_event_loop() + messages = loop.run_until_complete(coro) # Trim to total limit messages = messages[:limit] diff --git a/src/services/trade_monitor.py b/src/services/trade_monitor.py index e93a41d..336ac0a 100644 --- a/src/services/trade_monitor.py +++ b/src/services/trade_monitor.py @@ -2,7 +2,7 @@ Trade monitoring service - per-market parallel architecture. Each market runs its own independent async task that: -1. Polls the internal API for new trades (incremental via start_ts) +1. Polls the official Polymarket data-api for new trades 2. Detects whale trades 3. Fetches trader ranking + history in parallel 4. Fires the whale callback (LLM report generation) without blocking other markets @@ -12,6 +12,7 @@ Modeled after paper_trading/paper_trading.py's _market_loop pattern. import asyncio import json import logging +import random import time as _time from datetime import datetime from pathlib import Path @@ -32,7 +33,7 @@ logger = logging.getLogger(__name__) # Gamma API for fetching latest market prices GAMMA_API_URL = "https://gamma-api.polymarket.com/markets" -# Internal API for trade data (more stable than official data-api) +# Official Polymarket data-api for trade data # URL and key loaded from settings (.env) # File to persist processed transaction hashes @@ -52,32 +53,27 @@ class TradeMonitor: ): self.settings = get_settings() - # Official API (for trader ranking/history queries only) + # Official Polymarket data-api self.data_api_url = "https://data-api.polymarket.com" self.trades_endpoint = f"{self.data_api_url}/trades" self.leaderboard_endpoint = f"{self.data_api_url}/v1/leaderboard" - self._client = httpx.AsyncClient(timeout=30.0) - - # Internal API client for trade data - self._internal_api_url = self.settings.internal_api_url - self._internal_client = httpx.AsyncClient( - timeout=30.0, - headers={ - "X-API-Key": self.settings.internal_api_key, - "Accept": "application/json", - "Accept-Encoding": "gzip", - }, + self._client = httpx.AsyncClient( + timeout=httpx.Timeout(30.0, pool=120.0), + limits=httpx.Limits( + max_connections=50, + max_keepalive_connections=20, + keepalive_expiry=30, + ), ) # Per-market last-fetch timestamps for incremental polling self._market_last_ts: Dict[str, int] = {} - # Global rate limiter for internal API (matches paper_trading: 5 QPS max) - # NOTE: Lock created lazily in run() to avoid "attached to different loop" error + # Rate limiter: Lock + Semaphore created lazily in run() to avoid "attached to different loop" error self._api_lock: Optional[asyncio.Lock] = None self._api_sem: Optional[asyncio.Semaphore] = None # concurrency limiter self._api_last_request: float = 0.0 - self._api_global_interval: float = 1.0 # min 1s between requests = 1 QPS + self._api_global_interval: float = 0.2 # min 0.2s between requests = 5 QPS # Cache for trader rankings to avoid repeated API calls self._trader_ranking_cache: Dict[str, TraderRanking] = {} @@ -143,7 +139,6 @@ class TradeMonitor: """Cleanup resources.""" self._save_processed_txns() await self._client.aclose() - await self._internal_client.aclose() # ================================================================ # Market list management @@ -157,28 +152,46 @@ class TradeMonitor: self._monitored_markets[tm.market.id] = tm.market logger.info(f"Now monitoring {len(self._monitored_markets)} markets") + def set_tiered_markets(self, tiers: dict[str, list]) -> None: + """ + Set markets with per-tier poll intervals. + + Stores poll_interval per market_id in _market_poll_intervals dict. + """ + self._monitored_markets = {} + self._market_poll_intervals: dict[str, int] = {} + + tier_intervals = { + "tier1": self.settings.tier1_poll_interval, + "tier2": self.settings.tier2_poll_interval, + "tier3": self.settings.tier3_poll_interval, + } + + for tier_name, markets in tiers.items(): + interval = tier_intervals.get(tier_name, self.settings.fetch_interval_seconds) + for tm in markets: + if tm.market.id: + self._monitored_markets[tm.market.id] = tm.market + self._market_poll_intervals[tm.market.id] = interval + + tier_counts = {k: len(v) for k, v in tiers.items()} + logger.info( + f"Tiered monitoring: {tier_counts} " + f"(intervals: {tier_intervals}s), total={len(self._monitored_markets)}" + ) + # ================================================================ - # Trade fetching: dispatches to internal or official API + # Trade fetching # ================================================================ - _MAX_RETRIES = 3 - _RETRY_BACKOFF = [1, 2, 4] # seconds between retries - - async def fetch_market_trades(self, market_id: str) -> List[TradeActivity]: - """ - Fetch recent trades for a market. Dispatches to internal or official API - based on TRADE_API_MODE setting. - """ - if self.settings.trade_api_mode == "internal": - return await self._fetch_trades_internal(market_id) - else: - return await self._fetch_trades_official(market_id) + _MAX_RETRIES = 4 + _RETRY_BACKOFF = [2, 5, 10, 20] # seconds between retries (with jitter) # ================================================================ # Official Polymarket data-api: fetch trades # ================================================================ - async def _fetch_trades_official(self, market_id: str) -> List[TradeActivity]: + async def fetch_market_trades(self, market_id: str) -> List[TradeActivity]: """ Fetch recent trades using the official Polymarket data-api /trades endpoint. @@ -200,13 +213,9 @@ class TradeMonitor: params: Dict[str, object] = { "market": condition_id, - "limit": 50 if last_ts is None else 500, + "limit": 50, } - # Incremental polling: only fetch trades after last seen timestamp - if last_ts is not None: - params["after"] = last_ts + 1 - sem = self._api_sem or asyncio.Semaphore(20) last_err: Optional[Exception] = None async with sem: @@ -238,11 +247,11 @@ class TradeMonitor: except httpx.HTTPError as e: last_err = e if attempt < self._MAX_RETRIES - 1: - delay = self._RETRY_BACKOFF[attempt] + delay = self._RETRY_BACKOFF[attempt] + random.uniform(0, 2) logger.debug( f"Official API retry for {market_id} " f"(attempt {attempt + 1}/{self._MAX_RETRIES}): " - f"{type(e).__name__}, retrying in {delay}s" + f"{type(e).__name__}, retrying in {delay:.1f}s" ) await asyncio.sleep(delay) else: @@ -324,153 +333,6 @@ class TradeMonitor: logger.warning(f"Error fetching official trades for {market_id}: {type(e).__name__}: {e}") return [] - # ================================================================ - # Internal API: fetch trades - # ================================================================ - - async def _fetch_trades_internal(self, market_id: str) -> List[TradeActivity]: - """ - Fetch recent taker trades for a market using the internal /flows API. - - /flows returns one record per taker per transaction (already aggregated - across maker fills), with accurate usd_amount and real execution price. - Uses incremental polling via start_ts. - Retries up to _MAX_RETRIES times on connection/timeout errors. - """ - try: - last_ts = self._market_last_ts.get(market_id) - - params: Dict[str, object] = { - "market_id": market_id, - "role": "taker", - # First poll: only fetch recent 50 trades to record txn hashes - # Subsequent polls: incremental via start_ts, small data - "limit": 50 if last_ts is None else 500, - "desc": True, - } - - if last_ts is not None: - params["start_ts"] = last_ts + 1 - - # Semaphore limits concurrent requests; Lock enforces per-request interval - sem = self._api_sem or asyncio.Semaphore(20) - last_err: Optional[Exception] = None - async with sem: - for attempt in range(self._MAX_RETRIES): - try: - # Global rate limit - async with self._api_lock: - now = _time.monotonic() - wait = self._api_global_interval - (now - self._api_last_request) - if wait > 0: - await asyncio.sleep(wait) - self._api_last_request = _time.monotonic() - - response = await self._internal_client.get( - f"{self._internal_api_url}/flows", params=params, - ) - response.raise_for_status() - break # success - except httpx.HTTPStatusError as e: - if e.response.status_code in (502, 503, 504) and attempt < self._MAX_RETRIES - 1: - delay = self._RETRY_BACKOFF[attempt] - logger.debug( - f"Internal API {e.response.status_code} for {market_id} " - f"(attempt {attempt + 1}/{self._MAX_RETRIES}), " - f"retrying in {delay}s" - ) - await asyncio.sleep(delay) - continue - raise # don't retry other HTTP errors - except httpx.HTTPError as e: - last_err = e - if attempt < self._MAX_RETRIES - 1: - delay = self._RETRY_BACKOFF[attempt] - logger.debug( - f"Internal API retry for {market_id} " - f"(attempt {attempt + 1}/{self._MAX_RETRIES}): " - f"{type(e).__name__}, retrying in {delay}s" - ) - await asyncio.sleep(delay) - else: - logger.warning( - f"Internal API connection error for {market_id} " - f"(attempt {attempt + 1}/{self._MAX_RETRIES}, giving up): " - f"{type(e).__name__}: {e}" - ) - return [] - else: - # All retries exhausted (shouldn't reach here, but just in case) - return [] - - data = response.json() - if not data: - return [] - - activities = [] - max_ts = last_ts or 0 - - for item in data: - try: - raw_direction = item.get("direction", "") - - # Only track BUY trades (new positions). - # SELL may just be exiting a position, not a directional signal. - if raw_direction != "BUY": - continue - - token_amount = float(item.get("token_amount", 0) or 0) - raw_price = float(item.get("price", 0) or 0) - usdc_size = float(item.get("usd_amount", 0) or 0) - - # No normalization — keep real price and outcome: - # - nonusdc_side=token1: BUY Yes token at raw_price - # - nonusdc_side=token2: BUY No token at raw_price - nonusdc_side = item.get("nonusdc_side", "token1") - outcome = "Yes" if nonusdc_side == "token1" else "No" - - ts = int(item.get("timestamp", 0) or 0) - - if ts > max_ts: - max_ts = ts - - activity = TradeActivity( - transaction_hash=f"{item.get('transaction_hash', '')}-{item.get('log_index', '')}", - timestamp=ts, - condition_id=item.get("condition_id", market_id), - asset=item.get("condition_id", ""), - side="BUY", - size=token_amount, - usdc_size=usdc_size, - price=raw_price, - outcome=outcome, - outcome_index=0 if outcome == "Yes" else 1, - title="", - slug=None, - event_slug=None, - proxy_wallet=item.get("address"), - name=None, - ) - activities.append(activity) - except Exception as e: - logger.debug(f"Failed to parse /flows trade: {e}") - continue - - if max_ts > 0: - self._market_last_ts[market_id] = max_ts - - return activities - - except httpx.HTTPStatusError as e: - logger.warning( - f"Flows API HTTP {e.response.status_code} for {market_id}: " - f"{e.response.text[:200]}" - ) - return [] - except Exception as e: - logger.warning(f"Error fetching flows for {market_id}: {type(e).__name__}: {e}") - return [] - # ================================================================ # Official API: trader info (ranking + history) # ================================================================ @@ -761,37 +623,78 @@ class TradeMonitor: def _is_whale_trade(self, activity: TradeActivity, market: Optional[Market] = None) -> bool: """ - Check if a trade qualifies as a whale trade. + Multi-layer pre-filter mirroring options flow SignalFilter._check_signal. - Uses a dynamic size threshold based on market volume: - - Large markets (24h vol > $1M): standard threshold (MIN_TRADE_SIZE_USD) - - Small markets (24h vol < $100k): lowered to $1,000 - - In between: linearly interpolated + Filter chain (early rejection, same order as options flow): + 1. Price range — like moneyness filter (OTM/ITM range) + 2. Direction — BUY only (like enabled direction_filters) + 3. Resolution window — like DTE filter (3-60 days sweet spot) + 4. Size — like premium filter ($250K+ minimum) + 5. Dynamic size — like dynamic_premium (base × √(vol / baseline)) + 6. Signal strength — like ask_ratio filter (conviction check) """ - # Price filter: only BUY trades remain, price is the taker's buy price. - # Low price = cheap bet with high upside, high price = expensive/certain. - # Filter to [MIN_PRICE, MAX_PRICE] range (e.g. 0-0.7). + import math + from datetime import datetime as _dt + + # --- 1. Price range (like moneyness: OTM 0-20%) --- + # Price 0.2-0.8 = uncertain outcome = tradeable + # Price < 0.2 or > 0.8 = near-consensus = no edge if not (self.settings.min_price <= activity.price <= self.settings.max_price): return False - # Dynamic threshold based on market total volume: - # - Tiny markets ($10k-$100k vol): $1,000 (niche, info asymmetry high) - # - Medium markets ($100k-$5M vol): $5,000 (standard) - # - Large markets ($5M+ vol): $10,000 (macro, noise high) - if market and market.volume > 0: - vol = market.volume # total volume, not 24hr - if vol <= 10_000: - threshold = 500 - elif vol <= 100_000: - threshold = 1_000 - elif vol <= 5_000_000: - threshold = 5_000 - else: - threshold = 10_000 - else: - threshold = 5_000 + # --- 2. Direction: BUY only (like direction_filters.enabled) --- + # Already enforced upstream (only BUY trades reach here) - return activity.usdc_size >= threshold + # --- 3. Resolution window (like DTE min=3, max=60) --- + # Markets resolving < 6 hours = price already settled (like DTE < 3) + # Markets resolving > 90 days = too far out, edge diluted (like DTE > 60) + if market and market.end_date: + try: + end_dt = _dt.fromisoformat(market.end_date.replace("Z", "+00:00")) + now_dt = _dt.utcnow().replace(tzinfo=end_dt.tzinfo) if end_dt.tzinfo else _dt.utcnow() + hours_to_resolution = max(0, (end_dt - now_dt).total_seconds() / 3600) + if hours_to_resolution < 6: + return False # too close, like DTE < 3 + if hours_to_resolution > 90 * 24: + return False # too far, like DTE > 60 + except (ValueError, TypeError): + pass # unknown end date, don't reject + + # --- 4. Size (like premium min=$250K) --- + # Base minimum: $5,000 (Polymarket scale vs options $250K) + if activity.usdc_size < 5_000: + return False + + # --- 5. Dynamic size (like dynamic_premium = base × √(mcap / baseline)) --- + # Larger markets require proportionally larger trades to be meaningful + base_size = 10_000.0 + baseline_volume = 1_000_000.0 + + if market and market.volume > 0: + threshold = base_size * math.sqrt(market.volume / baseline_volume) + threshold = max(5_000.0, min(threshold, 100_000.0)) # floor $5K, cap $100K + else: + threshold = base_size + + if activity.usdc_size < threshold: + return False + + # --- 6. Signal strength (like ask_ratio > 70%) --- + # In Polymarket: buyer paying above market mid = conviction + # Reject trades at or below market mid (no conviction, possibly hedging) + if market and market.outcome_prices: + if activity.outcome == "Yes": + market_mid = market.outcome_prices[0] + elif len(market.outcome_prices) > 1: + market_mid = market.outcome_prices[1] + else: + market_mid = 1.0 - market.outcome_prices[0] + + # Must pay above market mid (no discount buys = no conviction) + if activity.price < market_mid + 0.01: + return False + + return True async def _handle_whale(self, activity: TradeActivity, market_id: str, market: Market): """ @@ -885,7 +788,9 @@ class TradeMonitor: if not market: return - poll_interval = self.settings.fetch_interval_seconds + # Per-market interval (from tiered monitoring) or global default + poll_intervals = getattr(self, '_market_poll_intervals', {}) + poll_interval = poll_intervals.get(market_id, self.settings.fetch_interval_seconds) # If we already have a last_ts for this market, it means the loop was # restarted (e.g. after a market list refresh) — skip the silent # first-poll window to avoid missing trades. @@ -955,7 +860,7 @@ class TradeMonitor: # Create lock/semaphore inside event loop (avoids "attached to different loop" error) self._api_lock = asyncio.Lock() - self._api_sem = asyncio.Semaphore(5) # max 5 concurrent API requests + self._api_sem = asyncio.Semaphore(10) # max 10 concurrent API requests logger.info( f"Starting parallel trade monitor "