diff --git a/CLAUDE.md b/CLAUDE.md index ca16a1b5..02e0038e 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -4,7 +4,7 @@ This file provides guidance to Claude Code (claude.ai/code) when working with co ## Project Overview -PolyWeather Pro — a paid institutional weather-intelligence terminal for temperature settlement markets. 50 monitored cities, DEB multi-model temperature blending, Mu probability calibration, Polymarket CLOB/WS price integration. Next.js 15 + React 19 (Vercel) frontend, FastAPI backend (VPS), Telegram bot. +PolyWeather Pro — a paid institutional weather-intelligence terminal. 50 monitored cities with real-time METAR/AMOS/MADIS observations, DEB multi-model temperature blending, Mu probability calibration, and intraday bias correction. Pure meteorological decision workspace; no market/price layer. Next.js 15 + React 19 (Vercel) frontend, FastAPI backend (VPS), Telegram bot. **Business model**: Paid-only, $10/month, no free tier, no trial. Landing page is public; `/terminal` requires login + active subscription. diff --git a/src/data_collection/polymarket_readonly.py b/src/data_collection/polymarket_readonly.py deleted file mode 100644 index 572c9738..00000000 --- a/src/data_collection/polymarket_readonly.py +++ /dev/null @@ -1,3673 +0,0 @@ -""" -Polymarket read-only market layer. - -P0 scope: -- Market discovery from Gamma REST -- Price / midpoint / spread / orderbook read from CLOB REST -- Optional WebSocket quote acceleration via PolymarketWsQuoteCache -- No signing, no order placement -""" - -from __future__ import annotations - -import json -import math -import os -import re -import threading -import time -import unicodedata -from datetime import datetime, timedelta, timezone -from typing import Any, Dict, List, Optional, Tuple - -import httpx -from loguru import logger - -from src.data_collection.city_registry import ALIASES, CITY_REGISTRY -from src.data_collection.polymarket_ws_cache import PolymarketWsQuoteCache - - -def _safe_float(value: Any) -> Optional[float]: - if value is None: - return None - try: - if isinstance(value, str): - value = value.strip() - if not value: - return None - numeric = float(value) - if math.isnan(numeric) or math.isinf(numeric): - return None - return numeric - except Exception: - return None - - -def _safe_int(value: Any, default: int) -> int: - try: - return int(value) - except Exception: - return default - - -def _safe_bool(value: Any) -> Optional[bool]: - if value is None: - return None - if isinstance(value, bool): - return value - if isinstance(value, (int, float)): - return bool(value) - if isinstance(value, str): - normalized = value.strip().lower() - if normalized in {"1", "true", "yes", "on"}: - return True - if normalized in {"0", "false", "no", "off"}: - return False - return bool(value) - - -def _normalize_text(value: Any) -> str: - text = str(value or "").strip().lower() - if not text: - return "" - text = unicodedata.normalize("NFKD", text) - text = "".join(ch for ch in text if not unicodedata.combining(ch)) - text = text.replace("_", " ").replace("-", " ") - return " ".join(text.split()) - - -def _normalize_city_key(city: Any) -> str: - raw = _normalize_text(city) - if not raw: - return "" - return ALIASES.get(raw, raw) - - -MARKET_CITY_ALIASES: Dict[str, str] = { -} - -MARKET_CITY_SLUG_ALIASES: Dict[str, str] = { - # Polymarket's weather event URL uses the colloquial NYC slug, while - # PolyWeather keeps the canonical registry key as "new york". - "new york": "nyc", - # The tracked station is Buckley/Aurora, but Polymarket lists this market - # under the user-facing Denver city name. - "aurora": "denver", -} - - -def _city_local_date(city_key: str) -> str: - """Return ISO date string (YYYY-MM-DD) for the city's local timezone.""" - city = CITY_REGISTRY.get(city_key, {}) - tz_offset = city.get("tz_offset", 0) - local_dt = datetime.now(timezone.utc) + timedelta(seconds=tz_offset) - return local_dt.strftime("%Y-%m-%d") - - -def _resolve_market_city_key(city_key: str) -> str: - return MARKET_CITY_ALIASES.get(city_key, city_key) - - -def _contains_token(haystack: str, token: str) -> bool: - token = _normalize_text(token) - if not token: - return False - pattern = r"\b" + re.escape(token) + r"\b" - try: - return re.search(pattern, haystack) is not None - except re.error: - return False - - -def _json_or_list(value: Any) -> List[Any]: - if value is None: - return [] - if isinstance(value, list): - return value - if isinstance(value, tuple): - return list(value) - if isinstance(value, str): - text = value.strip() - if not text: - return [] - try: - parsed = json.loads(text) - if isinstance(parsed, list): - return parsed - except Exception: - return [] - return [] - - -def _to_plain_dict(value: Any) -> Dict[str, Any]: - if isinstance(value, dict): - return value - if value is None: - return {} - if hasattr(value, "dict") and callable(value.dict): - try: - data = value.dict() - if isinstance(data, dict): - return data - except Exception: - pass - if hasattr(value, "__dict__"): - try: - data = dict(vars(value)) - if isinstance(data, dict): - return data - except Exception: - pass - return {} - - -def _extract_price(value: Any) -> Optional[float]: - if value is None: - return None - direct = _safe_float(value) - if direct is not None: - return direct - if isinstance(value, dict): - for key in ( - "price", - "mid", - "midpoint", - "value", - "last_trade_price", - "lastPrice", - ): - numeric = _safe_float(value.get(key)) - if numeric is not None: - return numeric - plain = _to_plain_dict(value) - if plain: - for key in ( - "price", - "mid", - "midpoint", - "value", - "last_trade_price", - "lastPrice", - ): - numeric = _safe_float(plain.get(key)) - if numeric is not None: - return numeric - return None - - -def _clamp_probability(value: Optional[float]) -> Optional[float]: - if value is None: - return None - if value < 0.0: - return 0.0 - if value > 1.0: - return 1.0 - return value - - -def _clamp_float(value: Optional[float], lower: float, upper: float) -> Optional[float]: - if value is None: - return None - return max(lower, min(upper, float(value))) - - -def _parse_hhmm_to_minutes(value: Any) -> Optional[int]: - text = str(value or "").strip() - if not text or ":" not in text: - return None - try: - hh, mm = text.split(":", 1) - hour = int(hh) - minute = int(mm[:2]) - except Exception: - return None - if hour < 0 or hour > 23 or minute < 0 or minute > 59: - return None - return hour * 60 + minute - - -def _extract_iso_date(value: Any) -> Optional[str]: - if not value: - return None - text = str(value).strip() - if not text: - return None - if len(text) >= 10 and text[4] == "-" and text[7] == "-": - return text[:10] - # Common API formats from Gamma/CLOB - candidates = ( - text, - text.replace("Z", "+00:00"), - text.split(".")[0] + "Z" if "." in text and "T" in text else text, - ) - for candidate in candidates: - try: - dt = datetime.fromisoformat(candidate.replace("Z", "+00:00")) - return dt.date().isoformat() - except Exception: - continue - return None - - -def _parse_iso_datetime_utc(value: Any) -> Optional[datetime]: - if not value: - return None - text = str(value).strip() - if not text: - return None - # Prefer timestamps that include a time component; plain dates are ambiguous. - if "T" not in text: - return None - try: - dt = datetime.fromisoformat(text.replace("Z", "+00:00")) - except Exception: - return None - if dt.tzinfo is None: - return dt.replace(tzinfo=timezone.utc) - return dt.astimezone(timezone.utc) - - -def _build_city_token_index() -> Dict[str, List[str]]: - result: Dict[str, List[str]] = {} - for key, info in CITY_REGISTRY.items(): - normalized_key = _normalize_text(key) - tokens = {normalized_key, normalized_key.replace(" ", "")} - - display_name = _normalize_text(info.get("name")) - if display_name: - tokens.add(display_name) - tokens.add(display_name.replace(" ", "")) - - for alias, target in ALIASES.items(): - if target != key: - continue - norm_alias = _normalize_text(alias) - if not norm_alias: - continue - # Ignore very short aliases to reduce false-positive matching. - if len(norm_alias) < 3 and norm_alias not in {"nyc"}: - continue - tokens.add(norm_alias) - - if key == "new york": - tokens.update({"central park", "new yorks central park"}) - if key == "sao paulo": - tokens.update({"sao paulo", "sao-paulo", "sao paulo"}) - - result[key] = sorted(tokens, key=len, reverse=True) - return result - - -CITY_TOKEN_INDEX = _build_city_token_index() - -WEATHER_KEYWORDS = ( - "temperature", - "temp", - "high", - "low", - "hotter", - "colder", - "above", - "below", -) - -MONTH_TO_NUM = { - "jan": 1, - "january": 1, - "feb": 2, - "february": 2, - "mar": 3, - "march": 3, - "apr": 4, - "april": 4, - "may": 5, - "jun": 6, - "june": 6, - "jul": 7, - "july": 7, - "aug": 8, - "august": 8, - "sep": 9, - "sept": 9, - "september": 9, - "oct": 10, - "october": 10, - "nov": 11, - "november": 11, - "dec": 12, - "december": 12, -} - - -def _parse_target_date(value: str) -> Optional[datetime]: - try: - return datetime.fromisoformat(value) - except Exception: - return None - - -def _extract_dates_from_text( - text: str, - default_year: Optional[int], -) -> List[str]: - dates: List[str] = [] - - for year, month, day in re.findall(r"\b(20\d{2})[-/](\d{1,2})[-/](\d{1,2})\b", text): - try: - parsed = datetime(int(year), int(month), int(day)).date().isoformat() - dates.append(parsed) - except Exception: - continue - - month_pattern = "|".join(sorted(MONTH_TO_NUM.keys(), key=len, reverse=True)) - for month_name, day_raw, year_raw in re.findall( - rf"\b({month_pattern})\s+(\d{{1,2}})(?:st|nd|rd|th)?(?:\s*(20\d{{2}}))?\b", - text, - ): - year = int(year_raw) if year_raw else default_year - if not year: - continue - try: - parsed = datetime(year, MONTH_TO_NUM[month_name], int(day_raw)).date().isoformat() - dates.append(parsed) - except Exception: - continue - - for day_raw, month_name, year_raw in re.findall( - rf"\b(\d{{1,2}})(?:st|nd|rd|th)?\s+({month_pattern})(?:\s*(20\d{{2}}))?\b", - text, - ): - year = int(year_raw) if year_raw else default_year - if not year: - continue - try: - parsed = datetime(year, MONTH_TO_NUM[month_name], int(day_raw)).date().isoformat() - dates.append(parsed) - except Exception: - continue - - # Deduplicate while preserving order - unique: List[str] = [] - seen = set() - for value in dates: - if value in seen: - continue - seen.add(value) - unique.append(value) - return unique - - -class PolymarketReadOnlyLayer: - def __init__(self) -> None: - self.enabled = ( - str(os.getenv("POLYMARKET_MARKET_SCAN_ENABLED", "true")).strip().lower() - not in {"0", "false", "no", "off"} - ) - self.gamma_url = ( - str(os.getenv("POLYMARKET_GAMMA_URL", "https://gamma-api.polymarket.com")) - .strip() - .rstrip("/") - ) - self.clob_url = ( - str(os.getenv("POLYMARKET_CLOB_URL", "https://clob.polymarket.com")) - .strip() - .rstrip("/") - ) - self.data_url = ( - str(os.getenv("POLYMARKET_DATA_URL", "https://data-api.polymarket.com")) - .strip() - .rstrip("/") - ) - self.http_timeout = _safe_float(os.getenv("POLYMARKET_HTTP_TIMEOUT_SEC")) or 20.0 - self.market_cache_ttl = _safe_int( - os.getenv("POLYMARKET_MARKET_CACHE_TTL_SEC", "60"), - 60, - ) - self.price_cache_ttl = _safe_int( - os.getenv("POLYMARKET_PRICE_CACHE_TTL_SEC", "30"), - 30, - ) - self.discovery_pages = _safe_int( - os.getenv("POLYMARKET_DISCOVERY_PAGES", "6"), - 6, - ) - self.discovery_limit = _safe_int( - os.getenv("POLYMARKET_DISCOVERY_LIMIT", "200"), - 200, - ) - self.min_liquidity_for_signal = ( - _safe_float(os.getenv("POLYMARKET_SIGNAL_MIN_LIQUIDITY")) or 50.0 - ) - self.edge_threshold = _safe_float(os.getenv("POLYMARKET_SIGNAL_EDGE_PCT")) or 2.0 - fast_price_only = _safe_bool(os.getenv("POLYMARKET_FAST_PRICE_ONLY", "false")) - self.fast_price_only = True if fast_price_only is None else bool(fast_price_only) - - self._session = httpx.Client( - timeout=self.http_timeout, - follow_redirects=True, - ) - self._markets_cache: Dict[str, Dict[str, Any]] = {} - self._active_markets_cache: Dict[str, Any] = {"data": [], "t": 0.0} - self._broad_markets_cache: Dict[str, Any] = {"data": [], "t": 0.0} - self._price_cache: Dict[str, Dict[str, Any]] = {} - self._lock = threading.Lock() - - self._ws_cache = PolymarketWsQuoteCache.from_env() - self._ws_cache.start() - - def _market_scan_debug_enabled(self) -> bool: - return ( - str(os.getenv("POLYMARKET_MARKET_SCAN_DEBUG", "false")).strip().lower() - in {"1", "true", "yes", "on"} - ) - - def _debug_market_scan(self, message: str, **payload: Any) -> None: - if not self._market_scan_debug_enabled(): - return - try: - details = json.dumps(payload, ensure_ascii=False, default=str) - except Exception: - details = str(payload) - logger.info(f"POLYMARKET_MARKET_SCAN_DEBUG {message} {details}") - - def build_market_scan( - self, - city: Any, - target_date: Any, - temperature_bucket: Optional[Dict[str, Any]] = None, - model_probability: Optional[float] = None, - probability_distribution: Optional[List[Dict[str, Any]]] = None, - temp_symbol: Optional[str] = None, - fallback_sparkline: Optional[List[float]] = None, - forced_market_slug: Optional[str] = None, - include_related_buckets: bool = True, - scan_filters: Optional[Dict[str, Any]] = None, - scan_context: Optional[Dict[str, Any]] = None, - ) -> Dict[str, Any]: - date_str = _extract_iso_date(target_date) or str(target_date or "") - city_key = _normalize_city_key(city) - market_city_key = _resolve_market_city_key(city_key) - requested_slug = str(forced_market_slug or "").strip().lower() or None - - scan: Dict[str, Any] = { - "available": False, - "reason": None, - "city_key": city_key or None, - "market_city_key": market_city_key or None, - "primary_market": None, - "selected_date": date_str or None, - "selected_condition_id": None, - "selected_slug": requested_slug, - "temperature_bucket": temperature_bucket, - "model_probability": model_probability, - "market_price": None, - "midpoint": None, - "spread": None, - "edge_percent": None, - "signal_label": "MONITOR", - "confidence": "low", - "yes_token": None, - "no_token": None, - "yes_buy": None, - "yes_sell": None, - "yes_midpoint": None, - "yes_spread": None, - "no_buy": None, - "no_sell": None, - "no_midpoint": None, - "no_spread": None, - "last_trade_price": None, - "liquidity": None, - "volume": None, - "quote_source": None, - "quote_age_ms": None, - "price_analysis": None, - "sparkline": fallback_sparkline or [], - "top_buckets": [], - "all_buckets": [], - "recent_trades": [], - "scan_scope": "full" if include_related_buckets else "lite", - "distribution_bias": None, - "window_phase": None, - "window_score": None, - "primary_signal": None, - "signal_status": "no_market", - "candidate_count": 0, - "scan_rows": [], - "resolved_market_type": "maxtemp", - "websocket": { - "enabled": False, - "status": "disabled_rest_only", - }, - } - - if not self.enabled: - scan["reason"] = "Market scan disabled by POLYMARKET_MARKET_SCAN_ENABLED." - self._debug_market_scan("disabled", city=city_key, date=date_str) - return scan - - if not city_key or city_key not in CITY_REGISTRY: - scan["reason"] = "City is not supported by the Polymarket market layer." - self._debug_market_scan("unsupported_city", city=city, normalized=city_key) - return scan - - if not market_city_key or market_city_key not in CITY_REGISTRY: - scan["reason"] = "Mapped market city is not supported by the Polymarket market layer." - self._debug_market_scan( - "unsupported_market_city", - city=city_key, - market_city=market_city_key, - ) - return scan - - if not date_str: - scan["reason"] = "Missing target date for market discovery." - self._debug_market_scan("missing_date", city=city_key, market_city=market_city_key) - return scan - - try: - preferred_temp = None - if isinstance(temperature_bucket, dict): - preferred_temp = _safe_float(temperature_bucket.get("temp")) - market, reason = self._find_primary_market( - market_city_key, - date_str, - forced_market_slug=requested_slug, - preferred_temp=preferred_temp, - ) - except Exception as exc: - logger.warning( - f"Polymarket market discovery failed ({city_key}->{market_city_key}): {exc}" - ) - scan["reason"] = "Market discovery failed." - self._debug_market_scan( - "discovery_exception", - city=city_key, - market_city=market_city_key, - error=str(exc), - ) - return scan - - if not market: - scan["reason"] = reason or "No active Polymarket market matched city/date." - self._debug_market_scan( - "no_market", - city=city_key, - market_city=market_city_key, - date=date_str, - forced_slug=requested_slug, - reason=scan["reason"], - ) - return scan - - market_date = self._extract_market_date(market) - condition_id = str( - market.get("conditionId") - or market.get("condition_id") - or market.get("conditionID") - or "" - ).strip() or None - market_slug = str(market.get("slug") or "").strip() or None - liquidity = _extract_price( - market.get("liquidityNum") - or market.get("liquidity") - or market.get("liquidityClob") - ) - volume = _extract_price( - market.get("volumeNum") - or market.get("volume") - or market.get("volume24hr") - ) - trade_state = self._market_trade_state(market) - primary_market_payload = { - "id": market.get("id"), - "question": market.get("question") or market.get("title"), - "slug": market_slug, - "condition_id": condition_id, - "end_date": market_date, - "active": trade_state.get("active"), - "closed": trade_state.get("closed"), - "accepting_orders": trade_state.get("accepting_orders"), - "ended_at_utc": trade_state.get("ended_at_utc"), - "tradable": trade_state.get("tradable"), - "tradable_reason": trade_state.get("reason"), - "liquidity": liquidity, - "volume": volume, - } - if not trade_state.get("tradable"): - scan["reason"] = ( - "Matched market is not tradable." - + ( - f" reason={trade_state.get('reason')}" - if trade_state.get("reason") - else "" - ) - ) - scan["primary_market"] = primary_market_payload - scan["selected_condition_id"] = condition_id - scan["selected_slug"] = market_slug - scan["liquidity"] = liquidity - scan["volume"] = volume - self._debug_market_scan( - "not_tradable", - city=city_key, - market_city=market_city_key, - date=date_str, - slug=market_slug, - trade_state=trade_state, - ) - return scan - - tokens = self._extract_market_tokens(market) - yes_token, no_token = self._resolve_yes_no_tokens(tokens) - if not yes_token or not no_token: - scan["reason"] = "Matched market has no resolvable YES/NO token pair." - scan["primary_market"] = primary_market_payload - scan["selected_condition_id"] = condition_id - scan["selected_slug"] = market_slug - scan["liquidity"] = liquidity - scan["volume"] = volume - self._debug_market_scan( - "missing_tokens", - city=city_key, - market_city=market_city_key, - date=date_str, - slug=market_slug, - token_count=len(tokens), - ) - return scan - - yes_prices = self._merge_market_quote_fallback( - self._get_token_market_data(str(yes_token.get("token_id"))), - market, - "yes", - ) - no_prices = self._merge_market_quote_fallback( - self._get_token_market_data(str(no_token.get("token_id"))), - market, - "no", - ) - - if liquidity is None: - liquidity = _extract_price(yes_prices.get("book_liquidity")) - last_trade_price = _extract_price(yes_prices.get("last_trade_price")) - market_price = ( - _extract_price(yes_prices.get("midpoint")) - or _extract_price(yes_prices.get("buy")) - or _extract_price(yes_token.get("implied_probability")) - ) - distribution_model_probability = self._aggregate_distribution_probability_for_market( - market=market, - probability_distribution=probability_distribution, - temp_symbol=temp_symbol, - ) - if distribution_model_probability is not None: - model_probability = distribution_model_probability - - edge_percent = None - if model_probability is not None and market_price is not None: - edge_percent = (model_probability - market_price) * 100.0 - - signal_label, confidence = self._derive_signal(edge_percent, liquidity) - - top_buckets: List[Dict[str, Any]] = [] - all_buckets: List[Dict[str, Any]] = [] - if include_related_buckets: - top_bucket_limit = max( - 1, - _safe_int(os.getenv("POLYMARKET_TOP_BUCKET_LIMIT", "4"), 4), - ) - all_bucket_limit = max( - top_bucket_limit, - _safe_int(os.getenv("POLYMARKET_ALL_BUCKET_LIMIT", "8"), 8), - ) - all_buckets = self._build_top_temperature_buckets( - city_key=market_city_key, - target_date=date_str, - primary_market=market, - probability_distribution=probability_distribution, - temp_symbol=temp_symbol, - limit=all_bucket_limit, - ) - top_buckets = list(all_buckets[:top_bucket_limit]) - - yes_payload = { - "outcome": yes_token.get("outcome") or "Yes", - "token_id": yes_token.get("token_id"), - "implied_probability": _extract_price(yes_token.get("implied_probability")), - "buy_price": _extract_price(yes_prices.get("buy")), - "sell_price": _extract_price(yes_prices.get("sell")), - "midpoint": _extract_price(yes_prices.get("midpoint")), - "last_trade_price": _extract_price(yes_prices.get("last_trade_price")), - "quote_source": yes_prices.get("quote_source"), - "quote_age_ms": _safe_int(yes_prices.get("quote_age_ms"), 0), - "book": yes_prices.get("book"), - } - no_payload = { - "outcome": no_token.get("outcome") or "No", - "token_id": no_token.get("token_id"), - "implied_probability": _extract_price(no_token.get("implied_probability")), - "buy_price": _extract_price(no_prices.get("buy")), - "sell_price": _extract_price(no_prices.get("sell")), - "midpoint": _extract_price(no_prices.get("midpoint")), - "last_trade_price": _extract_price(no_prices.get("last_trade_price")), - "quote_source": no_prices.get("quote_source"), - "quote_age_ms": _safe_int(no_prices.get("quote_age_ms"), 0), - "book": no_prices.get("book"), - } - yes_midpoint = _extract_price(yes_payload.get("midpoint")) - no_midpoint = _extract_price(no_payload.get("midpoint")) - yes_buy = _extract_price(yes_payload.get("buy_price")) - yes_sell = _extract_price(yes_payload.get("sell_price")) - no_buy = _extract_price(no_payload.get("buy_price")) - no_sell = _extract_price(no_payload.get("sell_price")) - yes_spread = ( - max(0.0, float(yes_buy) - float(yes_sell)) - if yes_buy is not None and yes_sell is not None - else None - ) - no_spread = ( - max(0.0, float(no_buy) - float(no_sell)) - if no_buy is not None and no_sell is not None - else None - ) - price_analysis = self._build_price_analysis( - model_probability=model_probability, - yes_buy=yes_buy, - yes_sell=yes_sell, - no_buy=no_buy, - no_sell=no_sell, - ) - distribution_scan = self._build_distribution_scan_pack( - city_key=market_city_key, - target_date=date_str, - primary_market=market, - probability_distribution=probability_distribution, - temp_symbol=temp_symbol, - scan_context=scan_context, - scan_filters=scan_filters, - ) - primary_signal = distribution_scan.get("primary_signal") - if isinstance(primary_signal, dict): - signal_label = str(primary_signal.get("action") or signal_label or "").strip() or signal_label - signal_score = _safe_float(primary_signal.get("final_score")) - if signal_score is not None and signal_score >= 85.0: - confidence = "high" - elif signal_score is not None and signal_score >= 70.0: - confidence = "medium" - elif signal_score is not None: - confidence = "low" - signal_edge = _safe_float(primary_signal.get("edge_percent")) - if signal_edge is not None: - edge_percent = signal_edge - - sparkline_values: List[float] = [] - for candidate in ( - _extract_price(yes_payload.get("sell_price")), - _extract_price(yes_payload.get("buy_price")), - market_price, - model_probability, - ): - if candidate is None: - continue - sparkline_values.append(round(candidate * 100.0, 2)) - if not sparkline_values: - sparkline_values = fallback_sparkline or [] - - market_url = self._build_market_url(market) - scan.update( - { - "available": True, - "reason": None, - "primary_market": primary_market_payload, - "selected_condition_id": condition_id, - "selected_slug": market_slug, - "model_probability": model_probability, - "market_price": market_price, - "midpoint": yes_midpoint if yes_midpoint is not None else market_price, - "spread": yes_spread, - "edge_percent": edge_percent, - "signal_label": signal_label, - "confidence": confidence, - "yes_token": yes_payload, - "no_token": no_payload, - "yes_buy": yes_buy, - "yes_sell": yes_sell, - "yes_midpoint": yes_midpoint, - "yes_spread": yes_spread, - "no_buy": no_buy, - "no_sell": no_sell, - "no_midpoint": no_midpoint, - "no_spread": no_spread, - "last_trade_price": last_trade_price, - "liquidity": liquidity, - "volume": volume, - "quote_source": yes_prices.get("quote_source"), - "quote_age_ms": _safe_int(yes_prices.get("quote_age_ms"), 0), - "price_analysis": price_analysis, - "sparkline": sparkline_values, - "top_buckets": top_buckets, - "all_buckets": all_buckets, - "distribution_bias": distribution_scan.get("distribution_bias"), - "window_phase": distribution_scan.get("window_phase"), - "window_score": distribution_scan.get("window_score"), - "primary_signal": primary_signal, - "signal_status": distribution_scan.get("signal_status"), - "candidate_count": distribution_scan.get("candidate_count"), - "scan_rows": distribution_scan.get("rows") or [], - "resolved_market_type": distribution_scan.get("resolved_market_type") or "maxtemp", - "websocket": { - "enabled": False, - "status": "disabled_rest_only", - "market_url": market_url, - "asset_ids": [ - token - for token in [ - yes_payload.get("token_id"), - no_payload.get("token_id"), - ] - if token - ], - "condition_ids": [condition_id] if condition_id else [], - }, - } - ) - self._debug_market_scan( - "scan_ready", - city=city_key, - market_city=market_city_key, - date=date_str, - selected_slug=market_slug, - market_price=scan.get("market_price"), - yes_buy=scan.get("yes_buy"), - yes_sell=scan.get("yes_sell"), - no_buy=scan.get("no_buy"), - no_sell=scan.get("no_sell"), - price_analysis_available=bool(price_analysis and price_analysis.get("available")), - all_buckets_count=len(all_buckets), - top_buckets=[ - { - "temp": row.get("temp"), - "yes_buy": row.get("yes_buy"), - "market_price": row.get("market_price"), - "quote_source": row.get("quote_source"), - "slug": row.get("slug"), - } - for row in all_buckets[:6] - if isinstance(row, dict) - ], - websocket=scan.get("websocket"), - ) - return scan - - def _hydrate_bucket_prices(self, buckets: List[Dict[str, Any]]) -> None: - for bucket in buckets: - if not isinstance(bucket, dict): - continue - yes_token_id = str(bucket.get("yes_token_id") or "").strip() - no_token_id = str(bucket.get("no_token_id") or "").strip() - if not yes_token_id: - continue - - yes_prices = self._get_token_market_data(yes_token_id) - no_prices = self._get_token_market_data(no_token_id) if no_token_id else {} - yes_buy = _extract_price(yes_prices.get("buy")) - yes_sell = _extract_price(yes_prices.get("sell")) - no_buy = _extract_price(no_prices.get("buy")) - no_sell = _extract_price(no_prices.get("sell")) - yes_midpoint = _extract_price(yes_prices.get("midpoint")) - - if yes_buy is not None: - bucket["yes_buy"] = yes_buy - if yes_sell is not None: - bucket["yes_sell"] = yes_sell - if no_buy is not None: - bucket["no_buy"] = no_buy - if no_sell is not None: - bucket["no_sell"] = no_sell - reference_price = yes_midpoint - if reference_price is None and yes_buy is not None and yes_sell is not None: - reference_price = (yes_buy + yes_sell) / 2.0 - if reference_price is None: - reference_price = yes_buy if yes_buy is not None else yes_sell - if reference_price is not None: - reference_price = max(0.0, min(1.0, float(reference_price))) - bucket["market_price"] = reference_price - if bucket.get("probability") is None: - bucket["probability"] = reference_price - # Keep model probability separate from market-implied price. - # Older code overwrote ``probability`` with the quote, which made - # downstream UI compare a market price against itself or display - # stale bucket probabilities as weather probabilities. - bucket.setdefault("model_probability", bucket.get("probability")) - if yes_prices.get("quote_source"): - bucket["quote_source"] = yes_prices.get("quote_source") - if yes_prices.get("quote_age_ms") is not None: - bucket["quote_age_ms"] = _safe_int(yes_prices.get("quote_age_ms"), 0) - - def _build_price_analysis( - self, - *, - model_probability: Optional[float], - yes_buy: Optional[float], - yes_sell: Optional[float], - no_buy: Optional[float], - no_sell: Optional[float], - ) -> Dict[str, Any]: - """Build read-only market price diagnostics. - - Polymarket CLOB naming is from the user's perspective: - BUY is the executable ask to buy that outcome, SELL is the executable bid. - Kelly here is a sizing reference only; no order execution is performed. - """ - p_yes = _clamp_probability(_safe_float(model_probability)) - p_no = _clamp_probability(1.0 - p_yes if p_yes is not None else None) - yes_ask = _clamp_probability(_safe_float(yes_buy)) - no_ask = _clamp_probability(_safe_float(no_buy)) - yes_bid = _clamp_probability(_safe_float(yes_sell)) - no_bid = _clamp_probability(_safe_float(no_sell)) - - yes = self._build_side_price_analysis("yes", p_yes, yes_ask, yes_bid) - no = self._build_side_price_analysis("no", p_no, no_ask, no_bid) - - ask_sum = None - lock_edge = None - lock_available = False - if yes_ask is not None and no_ask is not None: - ask_sum = yes_ask + no_ask - lock_edge = 1.0 - ask_sum - lock_available = lock_edge > 0 - - bid_sum = None - sell_side_edge = None - if yes_bid is not None and no_bid is not None: - bid_sum = yes_bid + no_bid - sell_side_edge = bid_sum - 1.0 - - best_side = None - side_rows = [ - row - for row in [yes, no] - if isinstance(row.get("edge"), (int, float)) - and isinstance(row.get("kelly_fraction"), (int, float)) - and row.get("kelly_fraction") > 0 - ] - if side_rows: - best_side = max( - side_rows, - key=lambda row: ( - float(row.get("edge") or 0.0), - float(row.get("kelly_fraction") or 0.0), - ), - ).get("side") - - return { - "available": any( - value is not None - for value in (yes_ask, no_ask, yes_bid, no_bid, p_yes) - ), - "source": "polymarket_clob_orderbook", - "model_probability": p_yes, - "yes": yes, - "no": no, - "best_side": best_side, - "lock": { - "available": lock_available, - "ask_sum": ask_sum, - "edge": lock_edge, - }, - "sell_side": { - "bid_sum": bid_sum, - "edge": sell_side_edge, - }, - } - - def _build_side_price_analysis( - self, - side: str, - probability: Optional[float], - ask: Optional[float], - bid: Optional[float], - ) -> Dict[str, Any]: - edge = None - kelly_fraction = None - if probability is not None and ask is not None: - edge = probability - ask - if 0.0 < ask < 1.0: - kelly_fraction = edge / (1.0 - ask) - - return { - "side": side, - "model_probability": probability, - "ask": ask, - "bid": bid, - "edge": edge, - "edge_percent": edge * 100.0 if edge is not None else None, - "kelly_fraction": kelly_fraction, - "quarter_kelly": ( - max(0.0, kelly_fraction) / 4.0 - if kelly_fraction is not None - else None - ), - } - - def _market_trade_state(self, market: Dict[str, Any]) -> Dict[str, Any]: - active = _safe_bool(market.get("active")) - closed_raw = _safe_bool(market.get("closed")) - closed = bool(closed_raw) if closed_raw is not None else False - accepting_orders = _safe_bool( - market.get("acceptingOrders", market.get("accepting_orders")) - ) - - ended_at = None - for key in ("endDate", "resolutionDate", "closedTime", "gameStartTime"): - parsed = _parse_iso_datetime_utc(market.get(key)) - if parsed is not None: - ended_at = parsed - break - - tradable = True - reason = None - if closed: - tradable = False - reason = "closed" - elif active is False: - tradable = False - reason = "inactive" - elif accepting_orders is False: - tradable = False - reason = "not_accepting_orders" - - return { - "active": active, - "closed": closed, - "accepting_orders": accepting_orders, - "ended_at_utc": ended_at.isoformat() if ended_at is not None else None, - "tradable": tradable, - "reason": reason, - } - - def _derive_signal( - self, - edge_percent: Optional[float], - liquidity: Optional[float], - ) -> Tuple[str, str]: - if edge_percent is None: - return "MONITOR", "low" - if liquidity is not None and liquidity < self.min_liquidity_for_signal: - return "MONITOR", "low" - - absolute_edge = abs(edge_percent) - if absolute_edge >= 8: - confidence = "high" - elif absolute_edge >= 4: - confidence = "medium" - else: - confidence = "low" - - if edge_percent >= self.edge_threshold: - return "BUY YES", confidence - if edge_percent <= -self.edge_threshold: - return "BUY NO", confidence - return "MONITOR", confidence - - def _find_primary_market( - self, - city_key: str, - target_date: str, - forced_market_slug: Optional[str] = None, - preferred_temp: Optional[float] = None, - ) -> Tuple[Optional[Dict[str, Any]], Optional[str]]: - if forced_market_slug: - return self._find_market_by_slug( - forced_market_slug, - preferred_temp=preferred_temp, - ) - - preferred_temp_key = ( - f"{float(preferred_temp):.2f}" - if preferred_temp is not None - else "none" - ) - cache_key = f"{city_key}|{target_date}|{preferred_temp_key}" - now = time.time() - - with self._lock: - cached = self._markets_cache.get(cache_key) - if cached and now - cached.get("t", 0) < self.market_cache_ttl: - return cached.get("market"), cached.get("reason") - - markets = self._load_markets(active_only=True) - if not markets: - return None, "No active markets returned by Gamma API." - - scored: List[Tuple[float, Dict[str, Any]]] = [] - for market in markets: - score = self._score_market( - city_key, - target_date, - market, - preferred_temp=preferred_temp, - ) - if score <= 0: - continue - scored.append((score, market)) - - # Fallback to broader active universe when strict filters miss. - if not scored: - broader = self._load_markets(active_only=False) - for market in broader: - score = self._score_market( - city_key, - target_date, - market, - preferred_temp=preferred_temp, - ) - if score <= 0: - continue - scored.append((score, market)) - - # Deterministic weather event fallback: - # If Gamma /markets discovery misses, resolve by canonical weather event slug. - if not scored: - event_slug = self._build_weather_event_slug(city_key, target_date) - if event_slug: - fallback_market, _ = self._find_market_by_slug( - event_slug, - preferred_temp=preferred_temp, - ) - if fallback_market: - with self._lock: - self._markets_cache[cache_key] = { - "market": fallback_market, - "reason": None, - "t": now, - } - return fallback_market, None - - scored.sort( - key=lambda item: ( - item[0], - _extract_price( - item[1].get("volumeNum") - or item[1].get("volume") - or item[1].get("volume24hr") - ) - or 0.0, - ), - reverse=True, - ) - - market = scored[0][1] if scored else None - reason = None if market else "No market matched city/date with weather filters." - - with self._lock: - self._markets_cache[cache_key] = {"market": market, "reason": reason, "t": now} - - return market, reason - - def _find_market_by_slug( - self, - market_slug: str, - preferred_temp: Optional[float] = None, - ) -> Tuple[Optional[Dict[str, Any]], Optional[str]]: - normalized_slug = str(market_slug or "").strip().lower() - if not normalized_slug: - return None, "market_slug is empty." - - # 0) Event slug path (Polymarket weather pages are often event slugs). - try: - resp = self._session.get( - f"{self.gamma_url}/events", - params={"slug": normalized_slug, "limit": 5}, - timeout=self.http_timeout, - ) - resp.raise_for_status() - payload = resp.json() - events = payload if isinstance(payload, list) else [] - for event in events: - if not isinstance(event, dict): - continue - event_slug = str(event.get("slug") or "").strip().lower() - markets = event.get("markets") if isinstance(event.get("markets"), list) else [] - market_candidates = [m for m in markets if isinstance(m, dict)] - # Try exact market slug match first. - for market in market_candidates: - item_slug = str(market.get("slug") or "").strip().lower() - if item_slug == normalized_slug: - market["eventSlug"] = market.get("eventSlug") or event_slug - market["eventTitle"] = market.get("eventTitle") or event.get("title") - return market, None - # If input is event slug, pick the most liquid active/ready market. - if event_slug == normalized_slug and market_candidates: - def _event_market_rank(m: Dict[str, Any]) -> Tuple[float, bool, bool, float]: - market_temp = self._extract_market_bucket_temp(m) - temp_score = 0.0 - if preferred_temp is not None and market_temp is not None: - temp_score = max(0.0, 100.0 - abs(market_temp - preferred_temp) * 10.0) - liquidity_score = ( - _extract_price( - m.get("volumeNum") - or m.get("volume") - or m.get("liquidityNum") - or m.get("liquidity") - ) - or 0.0 - ) - return ( - temp_score, - bool(m.get("active", False)), - not bool(m.get("closed", False)), - liquidity_score, - ) - - market_candidates.sort( - key=_event_market_rank, - reverse=True, - ) - best = market_candidates[0] - best["eventSlug"] = best.get("eventSlug") or event_slug - best["eventTitle"] = best.get("eventTitle") or event.get("title") - return best, None - except Exception: - pass - - # 1) Direct Gamma query by slug (fast-path for debug and deterministic checks). - query_params = [ - {"slug": normalized_slug, "limit": 20, "offset": 0, "archived": "false"}, - {"search": normalized_slug, "limit": 50, "offset": 0, "archived": "false"}, - ] - for params in query_params: - try: - resp = self._session.get( - f"{self.gamma_url}/markets", - params=params, - timeout=self.http_timeout, - ) - resp.raise_for_status() - payload = resp.json() - if isinstance(payload, dict): - candidates = payload.get("markets") - if not isinstance(candidates, list): - candidates = [] - elif isinstance(payload, list): - candidates = payload - else: - candidates = [] - for item in candidates: - if not isinstance(item, dict): - continue - item_slug = str(item.get("slug") or "").strip().lower() - if item_slug == normalized_slug: - return item, None - except Exception: - continue - - # 2) Fallback to cached discovery lists. - for active_only in (True, False): - for item in self._load_markets(active_only=active_only): - item_slug = str(item.get("slug") or "").strip().lower() - if item_slug == normalized_slug: - return item, None - - return None, f"Specified market_slug not found: {normalized_slug}" - - def _score_market( - self, - city_key: str, - target_date: str, - market: Dict[str, Any], - preferred_temp: Optional[float] = None, - ) -> float: - city_tokens = CITY_TOKEN_INDEX.get(city_key, [city_key]) - text_parts = [ - market.get("question"), - market.get("title"), - market.get("slug"), - market.get("eventSlug"), - market.get("description"), - ] - haystack = _normalize_text(" ".join(str(part or "") for part in text_parts)) - if not haystack: - return 0.0 - - city_hit = any(_contains_token(haystack, token) for token in city_tokens) - if not city_hit: - return 0.0 - - if not self._is_temperature_market(market): - return 0.0 - - score = 40.0 - score += 18.0 - - d_target = _parse_target_date(target_date) - text_dates = _extract_dates_from_text(haystack, d_target.year if d_target else None) - if d_target and text_dates: - diffs: List[int] = [] - for date_str in text_dates: - try: - diffs.append(abs((datetime.fromisoformat(date_str).date() - d_target.date()).days)) - except Exception: - continue - if diffs: - best = min(diffs) - if best == 0: - score += 45.0 - elif best == 1: - score += 20.0 - elif best == 2: - score += 10.0 - else: - score -= 6.0 - else: - market_date = self._extract_market_date(market) - if market_date and d_target: - try: - d_market = datetime.fromisoformat(market_date).date() - diff = abs((d_market - d_target.date()).days) - if diff == 0: - score += 18.0 - elif diff == 1: - score += 8.0 - elif diff == 2: - score += 3.0 - else: - score -= 2.0 - except Exception: - pass - - if bool(market.get("active", False)): - score += 5.0 - if not bool(market.get("closed", False)): - score += 5.0 - if bool(market.get("enableOrderBook", market.get("enable_order_book", False))): - score += 4.0 - - volume = ( - _extract_price( - market.get("volumeNum") - or market.get("volume") - or market.get("volume24hr") - ) - or 0.0 - ) - score += min(volume / 50000.0, 8.0) - - if preferred_temp is not None: - market_temp = self._extract_market_bucket_temp(market) - bucket_range = self._extract_market_bucket_range(market) - direction = self._extract_market_bucket_direction(market) - if market_temp is not None: - diff = abs(float(market_temp) - float(preferred_temp)) - score += max(-40.0, 60.0 - diff * 15.0) - - if bucket_range is not None: - lower, upper, _unit = bucket_range - contains_preferred = False - if upper is not None: - contains_preferred = lower <= float(preferred_temp) <= upper - elif direction == "above": - contains_preferred = float(preferred_temp) >= lower - elif direction == "below": - contains_preferred = float(preferred_temp) <= lower - else: - contains_preferred = abs(float(preferred_temp) - lower) <= 0.51 - - if contains_preferred: - if upper is not None or direction == "exact": - score += 28.0 - else: - score += 18.0 - return score - - def _is_temperature_market(self, market: Dict[str, Any]) -> bool: - text_parts = [ - market.get("question"), - market.get("title"), - market.get("slug"), - market.get("eventSlug"), - market.get("description"), - ] - raw_text = " ".join(str(part or "") for part in text_parts) - if not raw_text: - return False - - # Hard signal: contains explicit Celsius bucket text like "10C" / "10°C" - if re.search(r"(-?\d+(?:\.\d+)?)\s*[°º]?\s*c\b", raw_text, re.IGNORECASE): - return True - - text = _normalize_text(raw_text) - if not text: - return False - - # Weather temperature event patterns. - if "highest temperature" in text: - return True - if "temperature in" in text: - return True - if "high temperature" in text: - return True - - # Conservative fallback: must explicitly mention temperature and boundary wording. - if "temperature" in text and any( - key in text for key in ("or higher", "or above", "or lower", "or below", "and above", "and below") - ): - return True - - return False - - def _extract_market_date(self, market: Dict[str, Any]) -> Optional[str]: - for key in ( - "endDate", - "endDateIso", - "endDateISO", - "resolutionDate", - "gameStartTime", - "closedTime", - ): - date_str = _extract_iso_date(market.get(key)) - if date_str: - return date_str - return None - - def _extract_market_bucket_temp(self, market: Dict[str, Any]) -> Optional[float]: - parsed = self._extract_market_bucket_range(market) - if parsed: - lower, upper, _unit = parsed - if upper is not None: - return (lower + upper) / 2.0 - return lower - return None - - def _extract_market_bucket_range( - self, - market: Dict[str, Any], - ) -> Optional[Tuple[float, Optional[float], str]]: - slug = str(market.get("slug") or "").strip().lower() - slug_range = re.search( - r"-(\d+(?:\.\d+)?)-(\d+(?:\.\d+)?)([cf])(?:$|-or-higher|-or-lower|orhigher|orlower)", - slug, - re.IGNORECASE, - ) - if slug_range: - lower = _safe_float(slug_range.group(1)) - upper = _safe_float(slug_range.group(2)) - unit = slug_range.group(3).upper() - if lower is not None and upper is not None and abs(upper - lower) <= 20: - return min(lower, upper), max(lower, upper), unit - - text = " ".join( - str(part or "") - for part in ( - market.get("question"), - market.get("title"), - ) - ) - # Match range buckets such as "80-81°F" and "80 to 81F". - range_match = re.search( - r"(-?\d+(?:\.\d+)?)\s*(?:-|–|—|\bto\b)\s*(-?\d+(?:\.\d+)?)\s*°?\s*([cf])\b", - text, - re.IGNORECASE, - ) - if range_match: - lower = _safe_float(range_match.group(1)) - upper = _safe_float(range_match.group(2)) - unit = range_match.group(3).upper() - if lower is not None and upper is not None and abs(upper - lower) <= 20: - return min(lower, upper), max(lower, upper), unit - - slug_exact = re.search( - r"-(\d+(?:\.\d+)?)([cf])(?:$|-or-higher|-or-lower|orhigher|orlower)", - slug, - re.IGNORECASE, - ) - if slug_exact: - value = _safe_float(slug_exact.group(1)) - unit = slug_exact.group(2).upper() - if value is not None: - return value, None, unit - - # Match "... 9°C ..." / "... 9F ..." / "... -2 C ..." - match = re.search(r"(-?\d+(?:\.\d+)?)\s*°?\s*([cf])\b", text, re.IGNORECASE) - if match: - value = _safe_float(match.group(1)) - unit = match.group(2).upper() - if value is not None: - return value, None, unit - return None - - def _build_weather_event_slug(self, city_key: str, target_date: str) -> Optional[str]: - try: - dt = datetime.fromisoformat(str(target_date)) - except Exception: - return None - market_slug_key = MARKET_CITY_SLUG_ALIASES.get(city_key, city_key) - city_slug = str(market_slug_key or "").strip().lower().replace(" ", "-") - if not city_slug: - return None - month_name = dt.strftime("%B").lower() - return f"highest-temperature-in-{city_slug}-on-{month_name}-{dt.day}-{dt.year}" - - def _is_fahrenheit_symbol(self, symbol: Optional[str]) -> bool: - return "F" in str(symbol or "").upper() - - def _convert_temp_to_market_unit( - self, - value: Optional[float], - source_symbol: Optional[str], - market_unit: Optional[str], - ) -> Optional[float]: - numeric = _safe_float(value) - if numeric is None: - return None - normalized_unit = str(market_unit or "").upper() - source_is_f = self._is_fahrenheit_symbol(source_symbol) - if normalized_unit == "F": - return numeric if source_is_f else (numeric * 9.0 / 5.0) + 32.0 - return ((numeric - 32.0) * 5.0 / 9.0) if source_is_f else numeric - - def _market_bucket_contains_distribution_temp( - self, - market: Dict[str, Any], - distribution_temp: Optional[float], - temp_symbol: Optional[str], - ) -> bool: - compare_temp = self._convert_temp_to_market_unit( - distribution_temp, - source_symbol=temp_symbol, - market_unit=(self._extract_market_bucket_range(market) or (None, None, "C"))[2], - ) - if compare_temp is None: - return False - - bucket_range = self._extract_market_bucket_range(market) - lower = bucket_range[0] if bucket_range else None - upper = bucket_range[1] if bucket_range else None - unit = bucket_range[2] if bucket_range else "C" - direction = self._extract_market_bucket_direction(market) - - if lower is not None and upper is not None: - return compare_temp >= lower - 0.01 and compare_temp <= upper + 0.01 - if lower is not None and direction == "above": - return compare_temp >= lower - 0.01 - if lower is not None and direction == "below": - return compare_temp <= lower + 0.01 - - reference = self._extract_market_bucket_temp(market) - if reference is None: - return False - tolerance = 0.56 if str(unit or "").upper() == "F" else 0.26 - return abs(compare_temp - reference) <= tolerance - - def _aggregate_distribution_probability_for_market( - self, - market: Dict[str, Any], - probability_distribution: Optional[List[Dict[str, Any]]], - temp_symbol: Optional[str], - ) -> Optional[float]: - if not isinstance(probability_distribution, list) or not probability_distribution: - return None - - total = 0.0 - matched = 0 - for row in probability_distribution: - if not isinstance(row, dict): - continue - distribution_temp = _safe_float(row.get("value")) - if distribution_temp is None: - continue - if not self._market_bucket_contains_distribution_temp( - market, - distribution_temp, - temp_symbol, - ): - continue - raw_probability = _safe_float(row.get("probability")) - if raw_probability is None: - continue - probability = raw_probability / 100.0 if raw_probability > 1.0 else raw_probability - probability = max(0.0, min(1.0, probability)) - total += probability - matched += 1 - if matched > 0: - return max(0.0, min(1.0, total)) - - # Fallback: use Gaussian CDF when no distribution bucket matches the - # market bucket exactly. Compute mu/sigma from the distribution, then - # integrate the Gaussian tail or band that corresponds to the market. - values = [] - weights = [] - for row in probability_distribution: - v = _safe_float(row.get("value")) - p = _safe_float(row.get("probability")) - if v is not None and p is not None: - prob = p / 100.0 if p > 1.0 else p - values.append(v) - weights.append(max(0.0, prob)) - if len(values) < 2: - return None - - total_weight = sum(weights) - if total_weight <= 0: - return None - mu = sum(v * w for v, w in zip(values, weights)) / total_weight - variance = sum(w * (v - mu) ** 2 for v, w in zip(values, weights)) / total_weight - sigma = math.sqrt(max(variance, 0.01)) - - unit = str(temp_symbol or "C").upper() - bucket_range = self._extract_market_bucket_range(market) - lower = bucket_range[0] if bucket_range else None - upper = bucket_range[1] if bucket_range else None - direction = self._extract_market_bucket_direction(market) - if lower is not None: - lower = self._convert_temp_to_market_unit( - lower, source_symbol=None, market_unit=(bucket_range[2] if bucket_range else unit), - ) or lower - if upper is not None: - upper = self._convert_temp_to_market_unit( - upper, source_symbol=None, market_unit=(bucket_range[2] if bucket_range else unit), - ) or upper - - def _norm_cdf(x: float) -> float: - return 0.5 * (1.0 + math.erf((x - mu) / (sigma * math.sqrt(2.0)))) - - if lower is not None and upper is not None: - prob = _norm_cdf(upper + 0.5) - _norm_cdf(lower - 0.5) - elif lower is not None and direction == "above": - prob = 1.0 - _norm_cdf(lower - 0.5) - elif lower is not None and direction == "below": - prob = _norm_cdf(lower + 0.5) - elif lower is not None: - prob = _norm_cdf(lower + 1.5) - _norm_cdf(lower - 0.5) - else: - return None - - return max(0.0, min(1.0, prob)) - - def _load_markets(self, active_only: bool = True) -> List[Dict[str, Any]]: - now = time.time() - with self._lock: - cached = self._active_markets_cache if active_only else self._broad_markets_cache - if now - float(cached.get("t", 0)) < self.market_cache_ttl: - data = cached.get("data") - if isinstance(data, list): - return data - - all_markets: List[Dict[str, Any]] = [] - offset = 0 - for _ in range(max(self.discovery_pages, 1)): - params = {"archived": "false", "limit": self.discovery_limit, "offset": offset} - if active_only: - params.update({"active": "true", "closed": "false"}) - else: - params.update({"active": "true"}) - url = f"{self.gamma_url}/markets" - try: - resp = self._session.get(url, params=params, timeout=self.http_timeout) - resp.raise_for_status() - payload = resp.json() - except Exception as exc: - logger.warning(f"Gamma markets fetch failed (offset={offset}): {exc}") - break - - if isinstance(payload, dict): - batch = payload.get("markets") - if not isinstance(batch, list): - # Gamma can also return object arrays directly. - batch = [] - elif isinstance(payload, list): - batch = payload - else: - batch = [] - - if not batch: - break - - all_markets.extend(item for item in batch if isinstance(item, dict)) - if len(batch) < self.discovery_limit: - break - offset += self.discovery_limit - - with self._lock: - if active_only: - self._active_markets_cache = {"data": all_markets, "t": now} - else: - self._broad_markets_cache = {"data": all_markets, "t": now} - - return all_markets - - def _extract_market_tokens(self, market: Dict[str, Any]) -> List[Dict[str, Any]]: - result: List[Dict[str, Any]] = [] - - direct_tokens = market.get("tokens") - if isinstance(direct_tokens, list): - for token in direct_tokens: - token_obj = _to_plain_dict(token) - if not token_obj: - continue - token_id = str( - token_obj.get("token_id") - or token_obj.get("tokenId") - or token_obj.get("id") - or token_obj.get("clobTokenId") - or "" - ).strip() - if not token_id: - continue - result.append( - { - "outcome": token_obj.get("outcome") or token_obj.get("name"), - "token_id": token_id, - "implied_probability": _extract_price( - token_obj.get("price") - or token_obj.get("probability") - or token_obj.get("lastPrice") - ), - } - ) - if result: - return result - - outcomes = _json_or_list(market.get("outcomes")) - prices = _json_or_list(market.get("outcomePrices")) - token_ids = _json_or_list(market.get("clobTokenIds")) - if not token_ids: - token_ids = _json_or_list(market.get("tokenIds")) - - for index, outcome in enumerate(outcomes): - token_id = str(token_ids[index]).strip() if index < len(token_ids) else "" - if not token_id: - continue - implied_probability = ( - _extract_price(prices[index]) if index < len(prices) else None - ) - result.append( - { - "outcome": str(outcome), - "token_id": token_id, - "implied_probability": implied_probability, - } - ) - return result - - def _resolve_yes_no_tokens( - self, - tokens: List[Dict[str, Any]], - ) -> Tuple[Optional[Dict[str, Any]], Optional[Dict[str, Any]]]: - if not tokens: - return None, None - - yes_token = None - no_token = None - for token in tokens: - label = _normalize_text(token.get("outcome")) - if label in {"yes", "true", "above", "over"}: - yes_token = token - elif label in {"no", "false", "below", "under"}: - no_token = token - - if yes_token and no_token: - return yes_token, no_token - - if len(tokens) == 2: - # Fallback for markets with unnamed binary outcomes. - return tokens[0], tokens[1] - - return None, None - - def _get_token_market_data(self, token_id: str) -> Dict[str, Any]: - token_id = str(token_id or "").strip() - if not token_id: - return {} - - now = time.time() - with self._lock: - cached = self._price_cache.get(token_id) - if cached and now - cached.get("t", 0) < self.price_cache_ttl: - return cached.get("data", {}) - - ws_data = self._ws_cache.get_market_data(token_id) - if ws_data: - with self._lock: - self._price_cache[token_id] = {"data": ws_data, "t": now} - return ws_data - - self._ws_cache.subscribe([token_id]) - return {} - - def _fetch_token_market_data(self, token_id: str) -> Dict[str, Any]: - # REST-only path: CLOB public endpoints. - # Polymarket CLOB semantics: - # - side=BUY returns the executable ask, i.e. the price paid to buy. - # - side=SELL returns the executable bid, i.e. the price received to sell. - buy_price = _extract_price(self._clob_get("/price", {"token_id": token_id, "side": "BUY"})) - sell_price = _extract_price( - self._clob_get("/price", {"token_id": token_id, "side": "SELL"}) - ) - if self.fast_price_only: - buy, sell = self._resolve_trade_prices( - buy=buy_price, - sell=sell_price, - book=None, - ) - midpoint = (buy + sell) / 2.0 if buy is not None and sell is not None else (buy or sell) - spread = max(0.0, float(buy) - float(sell)) if buy is not None and sell is not None else None - return { - "buy": buy, - "sell": sell, - "midpoint": _clamp_probability(midpoint), - "spread": spread, - "last_trade_price": None, - "quote_source": "polymarket_clob_fast_price", - "quote_age_ms": 0, - "book": None, - "book_liquidity": None, - } - - midpoint = _extract_price(self._clob_get("/midpoint", {"token_id": token_id})) - last_trade = _extract_price( - self._clob_get("/last-trade-price", {"token_id": token_id}) - ) - orderbook_raw = self._clob_get("/book", {"token_id": token_id}) - book, book_liquidity = self._normalize_orderbook(orderbook_raw) - buy, sell = self._resolve_trade_prices( - buy=buy_price, - sell=sell_price, - book=book, - ) - if midpoint is None and buy is not None and sell is not None: - midpoint = (buy + sell) / 2.0 - spread = max(0.0, float(buy) - float(sell)) if buy is not None and sell is not None else None - return { - "buy": buy, - "sell": sell, - "midpoint": midpoint, - "spread": spread, - "last_trade_price": last_trade, - "quote_source": "polymarket_clob_rest", - "quote_age_ms": 0, - "book": book, - "book_liquidity": book_liquidity, - } - - def _has_quote_prices(self, quote: Optional[Dict[str, Any]]) -> bool: - if not isinstance(quote, dict) or not quote: - return False - return any( - _extract_price(quote.get(key)) is not None - for key in ("buy", "sell", "midpoint", "last_trade_price") - ) - - def _build_market_quote_fallback( - self, - market: Dict[str, Any], - outcome_side: str, - ) -> Dict[str, Any]: - """Build a price fallback from Gamma market-level quote fields. - - CLOB `/price` and `/book` remain the preferred source. Gamma's market - payload still carries public `bestBid` / `bestAsk` / `outcomePrices`; - using it prevents a total "price unavailable" state when the CLOB - endpoint, batch payload, or token lookup is temporarily unavailable. - """ - - if not isinstance(market, dict) or not market: - return {} - - side = str(outcome_side or "").strip().lower() - outcome_prices = _json_or_list(market.get("outcomePrices")) - yes_probability = _extract_price(outcome_prices[0]) if len(outcome_prices) >= 1 else None - no_probability = _extract_price(outcome_prices[1]) if len(outcome_prices) >= 2 else None - best_bid = _extract_price( - market.get("bestBid") - or market.get("best_bid") - or market.get("bid") - ) - best_ask = _extract_price( - market.get("bestAsk") - or market.get("best_ask") - or market.get("ask") - ) - spread = _extract_price(market.get("spread")) - if spread is None and best_bid is not None and best_ask is not None: - spread = max(0.0, float(best_ask) - float(best_bid)) - midpoint = ( - (best_bid + best_ask) / 2.0 - if best_bid is not None and best_ask is not None - else yes_probability - ) - last_trade = _extract_price(market.get("lastTradePrice") or market.get("last_trade_price")) - - if side == "no": - buy = _clamp_probability(1.0 - best_bid) if best_bid is not None else None - sell = _clamp_probability(1.0 - best_ask) if best_ask is not None else None - resolved_midpoint = ( - _clamp_probability(1.0 - midpoint) - if midpoint is not None - else _clamp_probability(no_probability) - ) - resolved_last_trade = ( - _clamp_probability(1.0 - last_trade) - if last_trade is not None - else None - ) - else: - buy = _clamp_probability(best_ask) - sell = _clamp_probability(best_bid) - resolved_midpoint = _clamp_probability(midpoint) - resolved_last_trade = _clamp_probability(last_trade) - - if not any(value is not None for value in (buy, sell, resolved_midpoint, resolved_last_trade)): - return {} - - return { - "buy": buy, - "sell": sell, - "midpoint": resolved_midpoint, - "spread": spread, - "last_trade_price": resolved_last_trade, - "quote_source": "polymarket_gamma_market_fallback", - "quote_age_ms": 0, - "book": None, - "book_liquidity": _extract_price( - market.get("liquidityClob") - or market.get("liquidityNum") - or market.get("liquidity") - ), - } - - def _merge_market_quote_fallback( - self, - quote: Optional[Dict[str, Any]], - market: Dict[str, Any], - outcome_side: str, - ) -> Dict[str, Any]: - fallback = self._build_market_quote_fallback(market, outcome_side) - if not fallback: - return dict(quote or {}) - if not isinstance(quote, dict) or not quote: - return fallback - - merged = dict(fallback) - for key, value in quote.items(): - if value is None: - continue - if isinstance(value, str) and not value.strip(): - continue - merged[key] = value - if self._has_quote_prices(quote): - merged["quote_source"] = quote.get("quote_source") or merged.get("quote_source") - return merged - - def _clob_get(self, path: str, params: Dict[str, Any]) -> Any: - url = f"{self.clob_url}{path}" - try: - resp = self._session.get(url, params=params, timeout=self.http_timeout) - resp.raise_for_status() - return resp.json() - except Exception: - return None - - def _resolve_trade_prices( - self, - buy: Optional[float], - sell: Optional[float], - book: Optional[Dict[str, Any]], - ) -> Tuple[Optional[float], Optional[float]]: - payload = book if isinstance(book, dict) else {} - best_bid = _extract_price(payload.get("best_bid")) - best_ask = _extract_price(payload.get("best_ask")) - resolved_buy = best_ask if best_ask is not None else buy - resolved_sell = best_bid if best_bid is not None else sell - if ( - best_ask is None - and best_bid is None - and buy is not None - and sell is not None - and buy < sell - ): - # When no order book is available, normalize raw CLOB /price - # snapshots into executable semantics used by the rest of this - # module: buy = ask-to-buy, sell = bid-to-sell. - resolved_buy, resolved_sell = sell, buy - return resolved_buy, resolved_sell - - def _normalize_orderbook(self, orderbook_raw: Any) -> Tuple[Optional[Dict[str, Any]], Optional[float]]: - payload = _to_plain_dict(orderbook_raw) - if not payload and isinstance(orderbook_raw, dict): - payload = orderbook_raw - if not payload: - return None, None - - bids_raw = payload.get("bids") or [] - asks_raw = payload.get("asks") or [] - - bid_levels: List[List[float]] = [] - ask_levels: List[List[float]] = [] - book_liquidity = 0.0 - - def _parse_side(items: Any, sink: List[List[float]]) -> None: - nonlocal book_liquidity - if not isinstance(items, list): - return - for item in items: - item_dict = _to_plain_dict(item) - if item_dict: - price = _extract_price(item_dict.get("price")) - size = _extract_price(item_dict.get("size") or item_dict.get("quantity")) - elif isinstance(item, (list, tuple)) and len(item) >= 2: - price = _extract_price(item[0]) - size = _extract_price(item[1]) - else: - continue - if price is None or size is None: - continue - sink.append([price, size]) - book_liquidity += max(0.0, price * size) - - _parse_side(bids_raw, bid_levels) - _parse_side(asks_raw, ask_levels) - - bid_levels.sort(key=lambda level: level[0], reverse=True) - ask_levels.sort(key=lambda level: level[0]) - best_bid = bid_levels[0][0] if bid_levels else None - best_ask = ask_levels[0][0] if ask_levels else None - normalized = { - "best_bid": best_bid, - "best_ask": best_ask, - "bid_levels": bid_levels[:10], - "ask_levels": ask_levels[:10], - } - return normalized, (book_liquidity if book_liquidity > 0 else None) - - def _build_market_url(self, market: Dict[str, Any]) -> Optional[str]: - slug = str(market.get("slug") or "").strip() - event_slug = str(market.get("eventSlug") or "").strip() - if event_slug: - return f"https://polymarket.com/event/{event_slug}" - if slug: - return f"https://polymarket.com/market/{slug}" - return None - - def _build_top_temperature_buckets( - self, - city_key: str, - target_date: str, - primary_market: Dict[str, Any], - probability_distribution: Optional[List[Dict[str, Any]]] = None, - temp_symbol: Optional[str] = None, - limit: int = 4, - ) -> List[Dict[str, Any]]: - candidate_markets = self._collect_related_temperature_markets( - city_key=city_key, - target_date=target_date, - primary_market=primary_market, - ) - if not candidate_markets: - return [] - - ranked: List[ - Tuple[ - float, - float, - float, - float, - Dict[str, Any], - Dict[str, Any], - Dict[str, Any], - Dict[str, Any], - Dict[str, Any], - Optional[Tuple[float, Optional[float], str]], - ] - ] = [] - for market in candidate_markets: - if not self._market_trade_state(market).get("tradable"): - continue - bucket_temp = self._extract_market_bucket_temp(market) - bucket_range = self._extract_market_bucket_range(market) - if bucket_temp is None: - continue - - tokens = self._extract_market_tokens(market) - yes_token, no_token = self._resolve_yes_no_tokens(tokens) - if not yes_token or not no_token: - continue - - yes_token_id = str(yes_token.get("token_id") or "").strip() - no_token_id = str(no_token.get("token_id") or "").strip() - yes_prices = ( - self._merge_market_quote_fallback( - self._get_token_market_data(yes_token_id), - market, - "yes", - ) - if yes_token_id - else {} - ) - no_prices = ( - self._merge_market_quote_fallback( - self._get_token_market_data(no_token_id), - market, - "no", - ) - if no_token_id - else {} - ) - - yes_midpoint = _extract_price(yes_prices.get("midpoint")) - yes_implied = _extract_price(yes_token.get("implied_probability")) - no_implied = _extract_price(no_token.get("implied_probability")) - market_prob = ( - yes_midpoint - if yes_midpoint is not None - else ( - yes_implied - if yes_implied is not None - else (1.0 - no_implied if no_implied is not None else None) - ) - ) - if market_prob is None: - continue - - market_prob = max(0.0, min(1.0, float(market_prob))) - model_prob = self._aggregate_distribution_probability_for_market( - market=market, - probability_distribution=probability_distribution, - temp_symbol=temp_symbol, - ) - volume = ( - _extract_price( - market.get("volumeNum") - or market.get("volume") - or market.get("volume24hr") - ) - or 0.0 - ) - ranked.append( - ( - model_prob if model_prob is not None else market_prob, - volume, - bucket_temp, - market_prob, - market, - yes_token, - no_token, - yes_prices, - no_prices, - bucket_range, - ) - ) - - if not ranked: - return [] - - ranked.sort(key=lambda item: (item[0], item[1]), reverse=True) - top_rows: List[Dict[str, Any]] = [] - max_items = max(1, int(limit or 4)) - primary_slug = str(primary_market.get("slug") or "").strip().lower() - primary_direction = self._extract_market_bucket_direction(primary_market) - seen_temp_keys: set = set() - - def _append_rows(enforce_primary_direction: bool) -> None: - for ( - model_prob, - _volume, - bucket_temp, - market_prob, - market, - yes_token, - no_token, - yes_prices, - no_prices, - bucket_range, - ) in ranked: - row_direction = self._extract_market_bucket_direction(market) - if ( - enforce_primary_direction - and primary_direction in {"above", "below"} - and row_direction != primary_direction - ): - continue - - temp_key = f"{round(float(bucket_temp), 2):.2f}" - if temp_key in seen_temp_keys: - continue - - yes_buy = _extract_price(yes_prices.get("buy")) - yes_sell = _extract_price(yes_prices.get("sell")) - yes_midpoint = _extract_price(yes_prices.get("midpoint")) or market_prob - no_buy = _extract_price(no_prices.get("buy")) - no_sell = _extract_price(no_prices.get("sell")) - - if no_buy is None and yes_buy is not None: - no_buy = max(0.0, min(1.0, 1.0 - yes_buy)) - if no_sell is None and yes_sell is not None: - no_sell = max(0.0, min(1.0, 1.0 - yes_sell)) - - market_slug = str(market.get("slug") or "").strip() - row_yes_token_id = str(yes_token.get("token_id") or "").strip() - row_no_token_id = str(no_token.get("token_id") or "").strip() - top_rows.append( - { - "label": self._extract_market_bucket_label(market, bucket_temp), - "value": bucket_temp, - "temp": bucket_temp, - "lower": bucket_range[0] if bucket_range else None, - "upper": bucket_range[1] if bucket_range else None, - "unit": bucket_range[2] if bucket_range else None, - "probability": model_prob, - "model_probability": model_prob, - "market_price": yes_midpoint, - "edge_percent": ( - (model_prob - yes_midpoint) * 100.0 - if model_prob is not None and yes_midpoint is not None - else None - ), - "yes_buy": yes_buy, - "yes_sell": yes_sell, - "no_buy": no_buy, - "no_sell": no_sell, - "yes_token_id": row_yes_token_id or None, - "no_token_id": row_no_token_id or None, - "quote_source": yes_prices.get("quote_source"), - "quote_age_ms": _safe_int(yes_prices.get("quote_age_ms"), 0), - "slug": market_slug or None, - "question": market.get("question") or market.get("title"), - "is_primary": bool( - primary_slug - and market_slug - and primary_slug == market_slug.strip().lower() - ), - } - ) - seen_temp_keys.add(temp_key) - if len(top_rows) >= max_items: - break - - if primary_direction in {"above", "below"}: - _append_rows(enforce_primary_direction=True) - if len(top_rows) < max_items: - _append_rows(enforce_primary_direction=False) - - return top_rows - - def _collect_related_temperature_markets( - self, - city_key: str, - target_date: str, - primary_market: Dict[str, Any], - ) -> List[Dict[str, Any]]: - related: List[Dict[str, Any]] = [] - canonical_event_slug = self._build_weather_event_slug(city_key, target_date) - if canonical_event_slug: - related.extend(self._load_event_markets(canonical_event_slug)) - - event_slug = self._extract_event_slug(primary_market) - if event_slug and event_slug != canonical_event_slug: - related.extend(self._load_event_markets(event_slug)) - - if not related: - for market in self._load_markets(active_only=True): - if self._score_market(city_key, target_date, market) <= 0: - continue - if self._extract_market_bucket_temp(market) is None: - continue - related.append(market) - - related.append(primary_market) - - unique: List[Dict[str, Any]] = [] - seen = set() - for market in related: - if not isinstance(market, dict): - continue - dedupe_key = str( - market.get("id") - or market.get("slug") - or market.get("conditionId") - or "" - ).strip() - if not dedupe_key: - continue - if dedupe_key in seen: - continue - seen.add(dedupe_key) - unique.append(market) - return unique - - def _extract_event_slug(self, market: Dict[str, Any]) -> Optional[str]: - event_slug = str(market.get("eventSlug") or "").strip().lower() - if event_slug: - return event_slug - - slug = str(market.get("slug") or "").strip().lower() - if not slug: - return None - - trimmed = re.sub( - r"-(?:m)?\d+(?:-\d+)?c(?:-or-(?:higher|lower|above|below))?$", - "", - slug, - ) - trimmed = trimmed.strip("-") - return trimmed or None - - def _load_event_markets(self, event_slug: str) -> List[Dict[str, Any]]: - normalized_slug = str(event_slug or "").strip().lower() - if not normalized_slug: - return [] - - try: - resp = self._session.get( - f"{self.gamma_url}/events", - params={"slug": normalized_slug, "limit": 5}, - timeout=self.http_timeout, - ) - resp.raise_for_status() - payload = resp.json() - except Exception: - return [] - - events = payload if isinstance(payload, list) else [] - out: List[Dict[str, Any]] = [] - for event in events: - if not isinstance(event, dict): - continue - event_item_slug = str(event.get("slug") or "").strip().lower() - if event_item_slug and event_item_slug != normalized_slug: - continue - for market in event.get("markets") or []: - if not isinstance(market, dict): - continue - market["eventSlug"] = market.get("eventSlug") or event_item_slug - market["eventTitle"] = market.get("eventTitle") or event.get("title") - out.append(market) - return out - - def get_market_holders( - self, condition_id: str, limit: int = 10 - ) -> List[Dict[str, Any]]: - """Fetch top token holders for a market from the Polymarket Data API. - - Endpoint: GET /holders?market={conditionId}&limit={limit} - Returns a list of holder objects with proxyWallet, amount, outcomeIndex, - pseudonym, name, profileImage, etc. - """ - cid = str(condition_id or "").strip() - if not cid: - return [] - try: - resp = self._session.get( - f"{self.data_url}/holders", - params={"market": cid, "limit": limit}, - timeout=self.http_timeout, - ) - resp.raise_for_status() - payload = resp.json() - except Exception as exc: - logger.warning(f"Polymarket holders fetch failed (condition={cid[:20]}): {exc}") - return [] - if isinstance(payload, list): - return payload - if isinstance(payload, dict): - return payload.get("holders") or payload.get("data") or [] - return [] - - def resolve_city_clob_tokens(self, city_key: str) -> List[Dict[str, Any]]: - """Resolve CLOB token IDs for a city using its local date.""" - local_date = _city_local_date(city_key) - market_slug = self._build_weather_event_slug(city_key, local_date) - if not market_slug: - return [] - markets = self._load_event_markets(market_slug) - tokens: List[Dict[str, Any]] = [] - for m in markets: - clob_ids = _json_or_list(m.get("clobTokenIds")) - question = str(m.get("question") or "").strip() - prices = _json_or_list(m.get("outcomePrices")) - if len(clob_ids) < 2: - continue - tokens.append({ - "city": city_key, - "local_date": local_date, - "question": question, - "slug": str(m.get("slug") or "").strip(), - "yes_token": clob_ids[0], - "no_token": clob_ids[1], - "yes_price": _safe_float(prices[0]) if len(prices) > 0 else None, - "no_price": _safe_float(prices[1]) if len(prices) > 1 else None, - }) - return tokens - - def resolve_all_cities_clob_tokens( - self, - cities: Optional[List[str]] = None, - ) -> Dict[str, List[Dict[str, Any]]]: - """Resolve CLOB tokens for all configured cities using local dates. - - Returns dict keyed by city_key, each value is a list of bucket token dicts. - """ - if cities is None: - cities = list(CITY_REGISTRY.keys()) - result: Dict[str, List[Dict[str, Any]]] = {} - for city_key in cities: - try: - buckets = self.resolve_city_clob_tokens(city_key) - if buckets: - result[city_key] = buckets - logger.info( - "polymarket market discovery city={} buckets={} date={}", - city_key, - len(buckets), - buckets[0]["local_date"] if buckets else "N/A", - ) - except Exception as exc: - logger.warning( - "polymarket market discovery failed city={} error={}", - city_key, - exc, - ) - return result - - def collect_all_clob_token_ids( - self, - cities: Optional[List[str]] = None, - ) -> List[str]: - """Collect all unique YES/NO CLOB token IDs for the given cities.""" - all_tokens = self.resolve_all_cities_clob_tokens(cities) - seen: set = set() - token_ids: List[str] = [] - for city_buckets in all_tokens.values(): - for bucket in city_buckets: - for key in ("yes_token", "no_token"): - tid = str(bucket.get(key) or "").strip() - if tid and tid not in seen: - seen.add(tid) - token_ids.append(tid) - return token_ids - - def _extract_market_bucket_label( - self, - market: Dict[str, Any], - bucket_temp: Optional[float], - ) -> str: - question = str(market.get("question") or market.get("title") or "").strip() - direction = self._extract_market_bucket_direction(market) - bucket_range = self._extract_market_bucket_range(market) - raw_unit = bucket_range[2] if bucket_range else "C" - unit = "F" if str(raw_unit).upper().endswith("F") else "°C" - if bucket_range and bucket_range[1] is not None: - return f"{bucket_range[0]:g}-{bucket_range[1]:g}{unit}" - if bucket_temp is not None: - if direction == "above": - return f"{bucket_temp:g}{unit}+" - if direction == "below": - return f"<={bucket_temp:g}{unit}" - return f"{bucket_temp:g}{unit}" - return question or str(market.get("slug") or "") - - def _extract_market_bucket_direction(self, market: Dict[str, Any]) -> str: - text = " ".join( - str(part or "") - for part in ( - market.get("question"), - market.get("title"), - market.get("slug"), - ) - ).lower() - if not text: - return "exact" - - if any( - token in text - for token in ( - "or higher", - "or above", - "and above", - "forhigher", - "forabove", - "or-higher", - "or-above", - ) - ): - return "above" - if any( - token in text - for token in ( - "or lower", - "or below", - "and below", - "forlower", - "forbelow", - "or-lower", - "or-below", - ) - ): - return "below" - return "exact" - - def _clob_post(self, path: str, payload: Any) -> Any: - url = f"{self.clob_url}{path}" - try: - resp = self._session.post(url, json=payload, timeout=self.http_timeout) - resp.raise_for_status() - return resp.json() - except Exception: - return None - - def _batch_chunks(self, values: List[str], size: int = 200) -> List[List[str]]: - if not values: - return [] - chunk_size = max(1, min(int(size or 200), 500)) - return [values[index : index + chunk_size] for index in range(0, len(values), chunk_size)] - - def _extract_payload_token_id(self, payload: Any) -> Optional[str]: - item = _to_plain_dict(payload) - if not item and isinstance(payload, dict): - item = payload - if not item: - return None - token_id = str( - item.get("asset_id") - or item.get("assetId") - or item.get("token_id") - or item.get("tokenId") - or item.get("id") - or "" - ).strip() - return token_id or None - - def _extract_batch_scalar_map(self, payload: Any) -> Dict[str, float]: - if not payload: - return {} - data = payload - if isinstance(data, dict): - for key in ("data", "midpoints", "spreads", "items", "results"): - nested = data.get(key) - if isinstance(nested, (dict, list)): - data = nested - break - result: Dict[str, float] = {} - if isinstance(data, dict): - for key, value in data.items(): - numeric = _extract_price(value) - if numeric is None: - continue - result[str(key).strip()] = numeric - return result - if isinstance(data, list): - for item in data: - token_id = self._extract_payload_token_id(item) - if not token_id: - continue - item_dict = _to_plain_dict(item) - numeric = _extract_price( - item_dict.get("midpoint") - or item_dict.get("mid_price") - or item_dict.get("spread") - or item_dict.get("price") - or item_dict.get("last_trade_price") - or item_dict.get("value") - ) - if numeric is None: - continue - result[token_id] = numeric - return result - - def _extract_batch_price_map(self, payload: Any, side: str) -> Dict[str, float]: - if not payload: - return {} - data = payload.get("data") if isinstance(payload, dict) and isinstance(payload.get("data"), dict) else payload - result: Dict[str, float] = {} - if not isinstance(data, dict): - return result - for token_id, side_map in data.items(): - token_key = str(token_id).strip() - if not token_key: - continue - item = _to_plain_dict(side_map) - if not item and isinstance(side_map, dict): - item = side_map - numeric = _extract_price(item.get(side) if item else side_map) - if numeric is None: - continue - result[token_key] = numeric - return result - - def _extract_batch_book_map(self, payload: Any) -> Dict[str, Dict[str, Any]]: - if not payload: - return {} - data = payload - if isinstance(data, dict): - for key in ("data", "books", "items", "results"): - nested = data.get(key) - if isinstance(nested, (dict, list)): - data = nested - break - result: Dict[str, Dict[str, Any]] = {} - if isinstance(data, dict): - for token_id, book in data.items(): - token_key = str(token_id).strip() - book_dict = _to_plain_dict(book) - if not token_key or not book_dict: - continue - result[token_key] = book_dict - return result - if isinstance(data, list): - for item in data: - token_id = self._extract_payload_token_id(item) - book_dict = _to_plain_dict(item) - if not token_id or not book_dict: - continue - result[token_id] = book_dict - return result - - def _batch_get_token_market_data( - self, - token_ids: List[str], - *, - include_books: bool = False, - ) -> Dict[str, Dict[str, Any]]: - unique_tokens = [] - seen = set() - for token_id in token_ids: - normalized = str(token_id or "").strip() - if not normalized or normalized in seen: - continue - seen.add(normalized) - unique_tokens.append(normalized) - - if not unique_tokens: - return {} - - now = time.time() - results: Dict[str, Dict[str, Any]] = {} - missing: List[str] = [] - with self._lock: - for token_id in unique_tokens: - cached = self._price_cache.get(token_id) - if not cached or now - cached.get("t", 0) >= self.price_cache_ttl: - missing.append(token_id) - continue - cached_data = cached.get("data", {}) or {} - if ( - include_books - and not self.fast_price_only - and not cached_data.get("book") - and cached_data.get("book_liquidity") is None - ): - missing.append(token_id) - continue - results[token_id] = dict(cached_data) - - if not missing: - return results - - # Pre-warm WS cache: subscribe all missing tokens at once, then - # wait briefly for the first quotes to arrive. Tokens that get - # WS data skip the REST fallback entirely. - self._ws_cache.subscribe(missing) - if self._ws_cache.enabled: - time.sleep(0.6) - for token_id in list(missing): - ws_data = self._ws_cache.get_market_data(token_id) - if ws_data: - with self._lock: - self._price_cache[token_id] = {"data": ws_data, "t": now} - results[token_id] = ws_data - missing.remove(token_id) - - # No REST fallback — prices exclusively from WebSocket. - return results - - def _normalize_scan_filters(self, scan_filters: Optional[Dict[str, Any]] = None) -> Dict[str, Any]: - raw = scan_filters if isinstance(scan_filters, dict) else {} - min_price = _clamp_float(_safe_float(raw.get("min_price")), 0.0, 1.0) - max_price = _clamp_float(_safe_float(raw.get("max_price")), 0.0, 1.0) - if min_price is None: - min_price = 0.001 - if max_price is None: - max_price = 0.999 - if min_price > max_price: - min_price, max_price = max_price, min_price - - high_liquidity_only = bool(_safe_bool(raw.get("high_liquidity_only"))) - min_liquidity = _safe_float(raw.get("min_liquidity")) - if min_liquidity is None: - min_liquidity = 5000.0 if high_liquidity_only else float(self.min_liquidity_for_signal or 500.0) - if high_liquidity_only: - min_liquidity = max(min_liquidity, 5000.0) - - return { - "scan_mode": str(raw.get("scan_mode") or "tradable").strip().lower() or "tradable", - "min_price": float(min_price), - "max_price": float(max_price), - "min_edge_pct": max(0.0, _safe_float(raw.get("min_edge_pct")) or float(self.edge_threshold or 2.0)), - "min_liquidity": max(0.0, float(min_liquidity)), - "high_liquidity_only": high_liquidity_only, - "market_type": str(raw.get("market_type") or "maxtemp").strip().lower() or "maxtemp", - "time_range": str(raw.get("time_range") or "today").strip().lower() or "today", - "limit": max(1, _safe_int(raw.get("limit"), 60)), - "max_spread": max(0.0, _safe_float(raw.get("max_spread")) or 0.2), - } - - def _build_window_meta( - self, - target_date: str, - scan_context: Optional[Dict[str, Any]] = None, - ) -> Dict[str, Any]: - context = scan_context if isinstance(scan_context, dict) else {} - local_date = _extract_iso_date(context.get("local_date")) or _extract_iso_date(target_date) - local_time = context.get("local_time") - peak = context.get("peak") if isinstance(context.get("peak"), dict) else {} - first_h = int(_safe_float(peak.get("first_h")) or 13) - last_h = int(_safe_float(peak.get("last_h")) or 15) - first_minutes = max(0, first_h * 60) - last_minutes = min(23 * 60 + 59, last_h * 60) - display_last_minutes = min(23 * 60 + 59, last_h * 60 + 59) - peak_fields: Dict[str, Any] = { - "peak_window_start": f"{first_h:02d}:00", - "peak_window_end": f"{last_h:02d}:59", - "peak_window_label": f"{first_h:02d}:00-{last_h:02d}:59", - "minutes_until_peak_start": None, - "minutes_until_peak_end": None, - "peak_start_minutes": first_minutes, - "peak_end_minutes": display_last_minutes, - } - target_iso = _extract_iso_date(target_date) - if not local_date or not target_iso: - return { - "phase": "today_default", - "score": 0.65, - "remaining_minutes": None, - "same_day": True, - **peak_fields, - } - - try: - diff_days = ( - datetime.fromisoformat(target_iso).date() - - datetime.fromisoformat(local_date).date() - ).days - except Exception: - diff_days = 0 - - now_minutes = _parse_hhmm_to_minutes(local_time) - if now_minutes is not None: - peak_fields["minutes_until_peak_start"] = diff_days * 1440 + first_minutes - now_minutes - peak_fields["minutes_until_peak_end"] = diff_days * 1440 + display_last_minutes - now_minutes - - if diff_days >= 2: - return { - "phase": "week_ahead", - "score": 0.45, - "remaining_minutes": peak_fields["minutes_until_peak_start"], - "same_day": False, - **peak_fields, - } - if diff_days == 1: - return { - "phase": "tomorrow", - "score": 0.60, - "remaining_minutes": peak_fields["minutes_until_peak_start"], - "same_day": False, - **peak_fields, - } - if diff_days < 0: - return { - "phase": "past", - "score": 0.0, - "remaining_minutes": None, - "same_day": False, - **peak_fields, - } - - if now_minutes is None: - return { - "phase": "today_default", - "score": 0.65, - "remaining_minutes": None, - "same_day": True, - **peak_fields, - } - - if now_minutes > last_minutes + 120: - return { - "phase": "post_peak", - "score": 0.50, - "remaining_minutes": 0, - "same_day": True, - **peak_fields, - } - if first_minutes <= now_minutes <= last_minutes + 120: - return { - "phase": "active_peak", - "score": 1.00, - "remaining_minutes": max(0, last_minutes + 120 - now_minutes), - "same_day": True, - **peak_fields, - } - if first_minutes - 180 <= now_minutes < first_minutes: - return { - "phase": "setup_today", - "score": 0.85, - "remaining_minutes": max(0, last_minutes + 120 - now_minutes), - "same_day": True, - **peak_fields, - } - return { - "phase": "early_today", - "score": 0.70, - "remaining_minutes": max(0, first_minutes - now_minutes), - "same_day": True, - **peak_fields, - } - - def _resolve_market_target_threshold( - self, - market_direction: str, - bucket_range: Optional[Tuple[float, Optional[float], str]], - bucket_temp: Optional[float], - ) -> Optional[float]: - if not bucket_range: - return bucket_temp - lower, upper, _unit = bucket_range - if market_direction in {"above", "below"}: - return lower - if upper is not None: - return (lower + upper) / 2.0 - return lower - - def _resolve_temperature_direction( - self, - *, - side: str, - market_direction: str, - target_threshold: Optional[float], - current_reference: Optional[float], - ) -> str: - if market_direction == "above": - return "hotter" if side == "yes" else "colder" - if market_direction == "below": - return "colder" if side == "yes" else "hotter" - hotter_bias = True - if target_threshold is not None and current_reference is not None: - hotter_bias = target_threshold >= current_reference - if side == "yes": - return "hotter" if hotter_bias else "colder" - return "colder" if hotter_bias else "hotter" - - def _is_trend_aligned( - self, - *, - temperature_direction: str, - trend_info: Optional[Dict[str, Any]], - network_lead_signal: Optional[Dict[str, Any]], - ) -> bool: - trend = trend_info if isinstance(trend_info, dict) else {} - network = network_lead_signal if isinstance(network_lead_signal, dict) else {} - trend_direction = _normalize_text(trend.get("direction")) - if temperature_direction == "hotter" and trend_direction == "rising": - return True - if temperature_direction == "colder" and trend_direction in {"falling", "stagnant"}: - return True - lead_delta = _safe_float(network.get("delta")) - if lead_delta is None: - return False - if temperature_direction == "hotter": - return lead_delta > 0 - return lead_delta < 0 - - def _build_distribution_scan_pack( - self, - *, - city_key: str, - target_date: str, - primary_market: Dict[str, Any], - probability_distribution: Optional[List[Dict[str, Any]]] = None, - temp_symbol: Optional[str] = None, - scan_context: Optional[Dict[str, Any]] = None, - scan_filters: Optional[Dict[str, Any]] = None, - ) -> Dict[str, Any]: - filters = self._normalize_scan_filters(scan_filters) - window_meta = self._build_window_meta(target_date, scan_context) - related_markets = self._collect_related_temperature_markets( - city_key=city_key, - target_date=target_date, - primary_market=primary_market, - ) - if not related_markets: - return { - "rows": [], - "distribution_bias": { - "available": False, - "value": None, - "direction": "balanced", - "score": 0.0, - "valid_markets": 0, - }, - "primary_signal": None, - "signal_status": "no_market", - "candidate_count": 0, - "window_phase": window_meta.get("phase"), - "window_score": window_meta.get("score"), - "resolved_market_type": "maxtemp", - } - - market_entries: List[Dict[str, Any]] = [] - token_ids: List[str] = [] - for market in related_markets: - tokens = self._extract_market_tokens(market) - yes_token, no_token = self._resolve_yes_no_tokens(tokens) - if not yes_token or not no_token: - continue - yes_token_id = str(yes_token.get("token_id") or "").strip() - no_token_id = str(no_token.get("token_id") or "").strip() - if not yes_token_id or not no_token_id: - continue - bucket_range = self._extract_market_bucket_range(market) - bucket_temp = self._extract_market_bucket_temp(market) - raw_direction = self._extract_market_bucket_direction(market) - market_direction = "range" if bucket_range and bucket_range[1] is not None else raw_direction - model_event_probability = self._aggregate_distribution_probability_for_market( - market=market, - probability_distribution=probability_distribution, - temp_symbol=temp_symbol, - ) - token_ids.extend([yes_token_id, no_token_id]) - market_entries.append( - { - "market": market, - "yes_token": yes_token, - "no_token": no_token, - "yes_token_id": yes_token_id, - "no_token_id": no_token_id, - "bucket_range": bucket_range, - "bucket_temp": bucket_temp, - "market_direction": market_direction, - "target_threshold": self._resolve_market_target_threshold( - market_direction, - bucket_range, - bucket_temp, - ), - "target_label": self._extract_market_bucket_label(market, bucket_temp), - "model_event_probability": model_event_probability, - "market_liquidity": _extract_price( - market.get("liquidityNum") - or market.get("liquidity") - or market.get("liquidityClob") - ), - "volume": _extract_price( - market.get("volumeNum") - or market.get("volume") - or market.get("volume24hr") - ), - "trade_state": self._market_trade_state(market), - "enable_order_book": bool( - market.get("enableOrderBook", market.get("enable_order_book", False)) - ), - } - ) - - broad_quotes = self._batch_get_token_market_data(token_ids, include_books=False) - bias_inputs: List[Tuple[float, float]] = [] - for entry in market_entries: - yes_quote = self._merge_market_quote_fallback( - broad_quotes.get(entry["yes_token_id"], {}), - entry["market"], - "yes", - ) - no_quote = self._merge_market_quote_fallback( - broad_quotes.get(entry["no_token_id"], {}), - entry["market"], - "no", - ) - market_event_probability = ( - _extract_price(yes_quote.get("midpoint")) - or _extract_price(yes_quote.get("buy")) - or _extract_price(yes_quote.get("sell")) - or _extract_price(entry["yes_token"].get("implied_probability")) - ) - if market_event_probability is not None: - market_event_probability = _clamp_probability(market_event_probability) - yes_ask = _extract_price(yes_quote.get("buy")) - yes_bid = _extract_price(yes_quote.get("sell")) - no_ask = _extract_price(no_quote.get("buy")) - no_bid = _extract_price(no_quote.get("sell")) - if no_ask is None and yes_ask is not None: - no_ask = _clamp_probability(1.0 - yes_bid) if yes_bid is not None else None - if no_bid is None and yes_bid is not None: - no_bid = _clamp_probability(1.0 - yes_ask) if yes_ask is not None else None - spread = _extract_price(yes_quote.get("spread")) - if spread is None and yes_ask is not None and yes_bid is not None: - spread = max(0.0, yes_ask - yes_bid) - - entry["market_event_probability"] = market_event_probability - entry["yes_ask"] = yes_ask - entry["yes_bid"] = yes_bid - entry["no_ask"] = no_ask - entry["no_bid"] = no_bid - entry["midpoint"] = _extract_price(yes_quote.get("midpoint")) or market_event_probability - entry["spread"] = spread - entry["yes_book_liquidity"] = _extract_price(yes_quote.get("book_liquidity")) - entry["no_book_liquidity"] = _extract_price(no_quote.get("book_liquidity")) - entry["quote_source"] = yes_quote.get("quote_source") or no_quote.get("quote_source") - entry["quote_age_ms"] = _safe_int( - yes_quote.get("quote_age_ms") if yes_quote.get("quote_age_ms") is not None else no_quote.get("quote_age_ms"), - 0, - ) - - model_event_probability = _clamp_probability(_safe_float(entry.get("model_event_probability"))) - if ( - model_event_probability is not None - and market_event_probability is not None - and entry["market_direction"] in {"above", "below"} - ): - gap = model_event_probability - market_event_probability - signed_gap = -gap if entry["market_direction"] == "below" else gap - bias_inputs.append((max(model_event_probability, 0.08), signed_gap)) - - distribution_bias_value = None - distribution_bias_score = 0.0 - distribution_bias_direction = "balanced" - if len(bias_inputs) >= 3: - total_weight = sum(weight for weight, _signed_gap in bias_inputs) - if total_weight > 0: - distribution_bias_value = sum(weight * signed_gap for weight, signed_gap in bias_inputs) / total_weight - distribution_bias_score = max(0.0, min(abs(distribution_bias_value) / 0.08, 1.0)) * 100.0 - if distribution_bias_value >= 0.015: - distribution_bias_direction = "hotter" - elif distribution_bias_value <= -0.015: - distribution_bias_direction = "colder" - - distribution_bias = { - "available": len(bias_inputs) >= 3 and distribution_bias_value is not None, - "value": distribution_bias_value, - "direction": distribution_bias_direction, - "score": distribution_bias_score, - "valid_markets": len(bias_inputs), - } - distribution_preview: List[Dict[str, Any]] = [] - for entry in market_entries: - label = str(entry.get("target_label") or "").strip() - if not label: - continue - preview_item = { - "label": label, - "value": _safe_float(entry.get("bucket_temp")), - "unit": ( - entry.get("bucket_range")[2] - if isinstance(entry.get("bucket_range"), tuple) - and len(entry.get("bucket_range")) >= 3 - else ("F" if self._is_fahrenheit_symbol(temp_symbol) else "C") - ), - "model_probability": _clamp_probability( - _safe_float(entry.get("model_event_probability")) - ), - "market_probability": _clamp_probability( - _safe_float(entry.get("market_event_probability")) - ), - "highlighted": False, - } - distribution_preview.append(preview_item) - - distribution_preview.sort( - key=lambda item: ( - _safe_float(item.get("value")) - if _safe_float(item.get("value")) is not None - else float("inf"), - str(item.get("label") or ""), - ) - ) - if distribution_preview: - highlighted_index = max( - range(len(distribution_preview)), - key=lambda index: _safe_float(distribution_preview[index].get("model_probability")) or 0.0, - ) - distribution_preview[highlighted_index]["highlighted"] = True - - peak_probability = None - peak_value = None - if distribution_preview: - highlighted_preview = next( - (item for item in distribution_preview if item.get("highlighted")), - None, - ) - if isinstance(highlighted_preview, dict): - peak_probability = _safe_float(highlighted_preview.get("model_probability")) - peak_value = _safe_float(highlighted_preview.get("value")) - - ordered_entry_indices = sorted( - range(len(market_entries)), - key=lambda index: ( - _safe_float(market_entries[index].get("bucket_temp")) - if _safe_float(market_entries[index].get("bucket_temp")) is not None - else float("inf"), - str(market_entries[index].get("target_label") or ""), - ), - ) - entry_order_map = { - ordered_entry_indices[position]: position - for position in range(len(ordered_entry_indices)) - } - peak_entry_order = None - if peak_value is not None and ordered_entry_indices: - peak_entry_order = min( - range(len(ordered_entry_indices)), - key=lambda position: abs( - ( - _safe_float( - market_entries[ordered_entry_indices[position]].get("bucket_temp") - ) - if _safe_float( - market_entries[ordered_entry_indices[position]].get("bucket_temp") - ) - is not None - else peak_value - ) - - peak_value - ), - ) - - raw_model_values: List[float] = [] - scan_models = (scan_context or {}).get("models") - if isinstance(scan_models, dict): - for raw_value in scan_models.values(): - value = _safe_float(raw_value) - if value is not None: - raw_model_values.append(value) - raw_deb_prediction = _safe_float((scan_context or {}).get("deb_prediction")) - - current_reference_raw = _safe_float( - (scan_context or {}).get("current_max_so_far") - or (scan_context or {}).get("current_temp") - ) - - def _median(values: List[float]) -> Optional[float]: - if not values: - return None - sorted_values = sorted(values) - middle = len(sorted_values) // 2 - if len(sorted_values) % 2: - return sorted_values[middle] - return (sorted_values[middle - 1] + sorted_values[middle]) / 2.0 - - def _build_cluster_meta(market_unit: str) -> Dict[str, Any]: - converted_values = [ - self._convert_temp_to_market_unit( - value, - source_symbol=temp_symbol, - market_unit=market_unit, - ) - for value in raw_model_values - ] - model_values = [value for value in converted_values if value is not None] - deb_reference = self._convert_temp_to_market_unit( - raw_deb_prediction, - source_symbol=temp_symbol, - market_unit=market_unit, - ) - median_value = _median(model_values) - if deb_reference is not None and median_value is not None: - center = (deb_reference + median_value) / 2.0 - elif deb_reference is not None: - center = deb_reference - elif median_value is not None: - center = median_value - elif peak_value is not None: - center = peak_value - else: - center = None - - unit_step = 1.8 if str(market_unit or "").upper() == "F" else 1.0 - return { - "available": center is not None and bool(model_values), - "center": center, - "core_low": center - 0.75 * unit_step if center is not None else None, - "core_high": center + 1.25 * unit_step if center is not None else None, - "low_tail": center - 0.75 * unit_step if center is not None else None, - "high_tail": center + 1.75 * unit_step if center is not None else None, - "model_count": len(model_values), - "deb_reference": deb_reference, - "median": median_value, - } - - def _cluster_role_for_target( - *, - target_value: Optional[float], - cluster_meta: Dict[str, Any], - ) -> str: - if not cluster_meta.get("available") or target_value is None: - return "unknown" - low_tail = _safe_float(cluster_meta.get("low_tail")) - high_tail = _safe_float(cluster_meta.get("high_tail")) - core_low = _safe_float(cluster_meta.get("core_low")) - core_high = _safe_float(cluster_meta.get("core_high")) - if low_tail is not None and target_value <= low_tail: - return "low_tail" - if high_tail is not None and target_value >= high_tail: - return "high_tail" - if ( - core_low is not None - and core_high is not None - and core_low < target_value <= core_high - ): - return "core" - return "shoulder" - - def _row_from_entry( - entry: Dict[str, Any], - side: str, - *, - entry_index: int, - ) -> Optional[Dict[str, Any]]: - raw_model_event_probability = _clamp_probability(_safe_float(entry.get("model_event_probability"))) - model_event_probability = raw_model_event_probability - market_event_probability = _clamp_probability(_safe_float(entry.get("market_event_probability"))) - ask = _clamp_probability(_safe_float(entry.get("yes_ask") if side == "yes" else entry.get("no_ask"))) - bid = _clamp_probability(_safe_float(entry.get("yes_bid") if side == "yes" else entry.get("no_bid"))) - if model_event_probability is None or ask is None: - return None - - market = entry["market"] - target_threshold = _safe_float(entry.get("target_threshold")) - bucket_range = entry.get("bucket_range") - market_unit = bucket_range[2] if bucket_range else ("F" if self._is_fahrenheit_symbol(temp_symbol) else "C") - cluster_meta = _build_cluster_meta(market_unit) - cluster_target = _safe_float(entry.get("bucket_temp")) or target_threshold - cluster_role = _cluster_role_for_target( - target_value=cluster_target, - cluster_meta=cluster_meta, - ) - cluster_adjusted = False - if ( - raw_model_event_probability is not None - and str(entry.get("market_direction") or "exact") in {"exact", "range"} - and cluster_role in {"low_tail", "high_tail"} - ): - model_event_probability = _clamp_probability(raw_model_event_probability * 0.45) - cluster_adjusted = True - - model_probability = ( - model_event_probability - if side == "yes" - else _clamp_probability(1.0 - model_event_probability) - ) - market_probability = ( - market_event_probability - if side == "yes" - else _clamp_probability(1.0 - market_event_probability) - ) - if model_probability is None: - return None - - current_reference = self._convert_temp_to_market_unit( - current_reference_raw, - source_symbol=temp_symbol, - market_unit=market_unit, - ) - gap_to_target = ( - target_threshold - current_reference - if target_threshold is not None and current_reference is not None - else None - ) - entry_order = entry_order_map.get(entry_index) - peak_distance = None - is_peak_candidate = False - if entry_order is not None and peak_entry_order is not None: - peak_distance = abs(entry_order - peak_entry_order) - is_peak_candidate = peak_distance <= 1 - market_structure = str(entry.get("market_direction") or "exact") - is_consensus_tail_no = ( - side == "no" - and market_structure in {"exact", "range"} - and cluster_role in {"low_tail", "high_tail"} - ) - is_consensus_core_yes = ( - side == "yes" - and market_structure in {"exact", "range"} - and cluster_role in {"core", "shoulder", "unknown"} - and (is_peak_candidate or cluster_role == "core") - ) - is_directional_candidate = ( - is_consensus_tail_no - or is_consensus_core_yes - or (market_structure not in {"exact", "range"} and is_peak_candidate) - ) - peak_alignment_score = 0.0 - if peak_distance is None: - peak_alignment_score = 0.35 - elif peak_distance == 0: - peak_alignment_score = 1.0 - elif peak_distance == 1: - peak_alignment_score = 0.8 - else: - peak_alignment_score = max(0.0, 0.55 - 0.15 * float(peak_distance - 2)) - temperature_direction = self._resolve_temperature_direction( - side=side, - market_direction=str(entry.get("market_direction") or "exact"), - target_threshold=target_threshold, - current_reference=current_reference, - ) - trend_alignment = self._is_trend_aligned( - temperature_direction=temperature_direction, - trend_info=(scan_context or {}).get("trend"), - network_lead_signal=(scan_context or {}).get("network_lead_signal"), - ) - edge = model_probability - ask - edge_percent = edge * 100.0 - kelly_fraction = edge / (1.0 - ask) if 0.0 < ask < 1.0 else None - liquidity_reference = max( - _safe_float( - entry.get("yes_book_liquidity") if side == "yes" else entry.get("no_book_liquidity") - ) or 0.0, - _safe_float(entry.get("market_liquidity")) or 0.0, - ) - if liquidity_reference >= 10000: - liquidity_score = 1.0 - elif liquidity_reference >= 5000: - liquidity_score = 0.8 - elif liquidity_reference >= 1000: - liquidity_score = 0.6 - else: - liquidity_score = 0.4 - if 0.10 <= ask <= 0.90: - price_usefulness_score = 1.0 - elif 0.05 <= ask < 0.10 or 0.90 < ask <= 0.95: - price_usefulness_score = 0.7 - else: - price_usefulness_score = 0.0 - bias_score = 0.0 - if distribution_bias["available"]: - if distribution_bias_direction == "balanced" or distribution_bias_direction == temperature_direction: - bias_score = distribution_bias_score / 100.0 - spread = _safe_float(entry.get("spread")) - spread_penalty = max( - 0.0, - min(((spread or 0.0) - 0.01) / 0.02, 1.0), - ) * 15.0 - edge_score = max(0.0, min(edge_percent / 12.0, 1.0)) - consensus_score = 1.0 if is_directional_candidate else 0.0 - final_score = 100.0 * ( - 0.32 * edge_score - + 0.25 * bias_score - + 0.20 * float(window_meta.get("score") or 0.0) - + 0.10 * liquidity_score - + 0.10 * price_usefulness_score - + 0.08 * peak_alignment_score - + 0.12 * consensus_score - ) - spread_penalty - market_slug = str(market.get("slug") or "").strip() - target_label = str(entry.get("target_label") or "").strip() - action = f"BUY {'YES' if side == 'yes' else 'NO'}" - if target_label: - action = f"{action} {target_label}" - return { - "id": f"{city_key}|{target_date}|{market_slug}|{side}", - "city": city_key, - "selected_date": target_date, - "market_slug": market_slug or None, - "market_question": market.get("question") or market.get("title"), - "market_url": self._build_market_url(market), - "side": side, - "action": action, - "market_direction": entry.get("market_direction"), - "temperature_direction": temperature_direction, - "target_label": entry.get("target_label"), - "target_value": entry.get("bucket_temp"), - "target_threshold": target_threshold, - "target_lower": bucket_range[0] if bucket_range else None, - "target_upper": bucket_range[1] if bucket_range else None, - "target_unit": market_unit, - "model_probability": model_probability, - "market_probability": market_probability, - "model_event_probability": model_event_probability, - "raw_model_event_probability": raw_model_event_probability, - "market_event_probability": market_event_probability, - "gap": ( - model_event_probability - market_event_probability - if model_event_probability is not None and market_event_probability is not None - else None - ), - "signed_gap": ( - -1.0 * (model_event_probability - market_event_probability) - if model_event_probability is not None - and market_event_probability is not None - and entry.get("market_direction") == "below" - else ( - model_event_probability - market_event_probability - if model_event_probability is not None and market_event_probability is not None - else None - ) - ), - "yes_token_id": entry.get("yes_token_id"), - "no_token_id": entry.get("no_token_id"), - "yes_ask": entry.get("yes_ask"), - "yes_bid": entry.get("yes_bid"), - "no_ask": entry.get("no_ask"), - "no_bid": entry.get("no_bid"), - "ask": ask, - "bid": bid, - "midpoint": entry.get("midpoint"), - "spread": spread, - "book_liquidity": _safe_float( - entry.get("yes_book_liquidity") if side == "yes" else entry.get("no_book_liquidity") - ), - "market_liquidity": entry.get("market_liquidity"), - "volume": entry.get("volume"), - "quote_source": entry.get("quote_source"), - "quote_age_ms": entry.get("quote_age_ms"), - "edge": edge, - "edge_percent": edge_percent, - "kelly_fraction": kelly_fraction, - "quarter_kelly": ( - max(0.0, kelly_fraction) / 4.0 - if kelly_fraction is not None - else None - ), - "edge_score": edge_score, - "bias_score": bias_score, - "consensus_score": consensus_score, - "window_phase": window_meta.get("phase"), - "window_score": window_meta.get("score"), - "remaining_window_minutes": window_meta.get("remaining_minutes"), - "peak_window_start": window_meta.get("peak_window_start"), - "peak_window_end": window_meta.get("peak_window_end"), - "peak_window_label": window_meta.get("peak_window_label"), - "minutes_until_peak_start": window_meta.get("minutes_until_peak_start"), - "minutes_until_peak_end": window_meta.get("minutes_until_peak_end"), - "liquidity_score": liquidity_score, - "price_usefulness_score": price_usefulness_score, - "spread_penalty": spread_penalty, - "final_score": final_score, - "distribution_bias_direction": distribution_bias_direction, - "distribution_bias_score": distribution_bias_score, - "distribution_bias_available": distribution_bias["available"], - "distribution_preview": distribution_preview[:6], - "peak_probability": peak_probability, - "peak_value": peak_value, - "peak_distance": peak_distance, - "peak_alignment_score": peak_alignment_score, - "is_peak_candidate": is_peak_candidate, - "is_directional_candidate": is_directional_candidate, - "cluster_adjusted": cluster_adjusted, - "cluster_role": cluster_role, - "cluster_center": cluster_meta.get("center"), - "cluster_core_low": cluster_meta.get("core_low"), - "cluster_core_high": cluster_meta.get("core_high"), - "cluster_model_count": cluster_meta.get("model_count"), - "cluster_deb_reference": cluster_meta.get("deb_reference"), - "cluster_median": cluster_meta.get("median"), - "current_reference": current_reference, - "gap_to_target": gap_to_target, - "touch_distance": abs(gap_to_target) if gap_to_target is not None else None, - "trend_alignment": trend_alignment, - "tradable": bool(entry["trade_state"].get("tradable")), - "active": entry["trade_state"].get("active"), - "closed": entry["trade_state"].get("closed"), - "accepting_orders": entry["trade_state"].get("accepting_orders"), - "enable_order_book": entry.get("enable_order_book"), - "is_primary_market": bool( - str(primary_market.get("slug") or "").strip().lower() - and market_slug - and str(primary_market.get("slug") or "").strip().lower() == market_slug.lower() - ), - } - - preliminary_rows: List[Dict[str, Any]] = [] - for entry_index, entry in enumerate(market_entries): - row_yes = _row_from_entry(entry, "yes", entry_index=entry_index) - row_no = _row_from_entry(entry, "no", entry_index=entry_index) - if row_yes: - preliminary_rows.append(row_yes) - if row_no: - preliminary_rows.append(row_no) - - preliminary_rows.sort(key=lambda row: float(row.get("final_score") or 0.0), reverse=True) - shortlist_market_slugs = [] - seen_slugs = set() - for row in preliminary_rows: - market_slug = str(row.get("market_slug") or "").strip() - if not market_slug or market_slug in seen_slugs: - continue - seen_slugs.add(market_slug) - shortlist_market_slugs.append(market_slug) - if len(shortlist_market_slugs) >= 10: - break - - shortlisted_tokens: List[str] = [] - for entry in market_entries: - market_slug = str(entry["market"].get("slug") or "").strip() - if market_slug not in seen_slugs: - continue - shortlisted_tokens.extend([entry["yes_token_id"], entry["no_token_id"]]) - - precise_quotes = self._batch_get_token_market_data( - shortlisted_tokens, - include_books=not self.fast_price_only, - ) - for entry in market_entries: - market_slug = str(entry["market"].get("slug") or "").strip() - if market_slug not in seen_slugs: - continue - yes_quote = self._merge_market_quote_fallback( - precise_quotes.get(entry["yes_token_id"], {}), - entry["market"], - "yes", - ) - no_quote = self._merge_market_quote_fallback( - precise_quotes.get(entry["no_token_id"], {}), - entry["market"], - "no", - ) - if yes_quote: - entry["yes_ask"] = _extract_price(yes_quote.get("buy")) or entry.get("yes_ask") - entry["yes_bid"] = _extract_price(yes_quote.get("sell")) or entry.get("yes_bid") - entry["midpoint"] = _extract_price(yes_quote.get("midpoint")) or entry.get("midpoint") - entry["spread"] = _extract_price(yes_quote.get("spread")) or entry.get("spread") - entry["yes_book_liquidity"] = _extract_price(yes_quote.get("book_liquidity")) or entry.get("yes_book_liquidity") - entry["quote_source"] = yes_quote.get("quote_source") or entry.get("quote_source") - if no_quote: - entry["no_ask"] = _extract_price(no_quote.get("buy")) or entry.get("no_ask") - entry["no_bid"] = _extract_price(no_quote.get("sell")) or entry.get("no_bid") - entry["no_book_liquidity"] = _extract_price(no_quote.get("book_liquidity")) or entry.get("no_book_liquidity") - entry["quote_source"] = no_quote.get("quote_source") or entry.get("quote_source") - if entry.get("spread") is None and entry.get("yes_ask") is not None and entry.get("yes_bid") is not None: - entry["spread"] = max(0.0, float(entry["yes_ask"]) - float(entry["yes_bid"])) - - final_rows: List[Dict[str, Any]] = [] - for entry_index, entry in enumerate(market_entries): - for side in ("yes", "no"): - row = _row_from_entry(entry, side, entry_index=entry_index) - if row: - final_rows.append(row) - - def _passes_hard_filters(row: Dict[str, Any]) -> bool: - ask = _safe_float(row.get("ask")) - edge_percent = _safe_float(row.get("edge_percent")) - spread = _safe_float(row.get("spread")) - liquidity = max( - _safe_float(row.get("book_liquidity")) or 0.0, - _safe_float(row.get("market_liquidity")) or 0.0, - ) - if ask is None or edge_percent is None: - return False - if not row.get("tradable") or row.get("accepting_orders") is False: - return False - if row.get("enable_order_book") is False: - return False - if ask < filters["min_price"] or ask > filters["max_price"]: - return False - if abs(edge_percent) < filters["min_edge_pct"]: - return False - if spread is not None and spread > filters["max_spread"]: - return False - if liquidity < filters["min_liquidity"]: - return False - - side = str(row.get("side") or "").lower() - market_direction = str(row.get("market_direction") or "").lower() - if ( - side == "no" - and market_direction in {"exact", "range"} - and ask >= 0.80 - and edge_percent < 10.0 - and not (row.get("cluster_adjusted") and row.get("is_directional_candidate")) - ): - return False - if spread is not None and spread > filters["max_spread"]: - return False - if liquidity < filters["min_liquidity"]: - return False - return True - - def _passes_mode_filters(row: Dict[str, Any]) -> bool: - scan_mode = filters["scan_mode"] - if scan_mode == "tradable": - return ( - float(row.get("window_score") or 0.0) >= 0.65 - and bool(row.get("is_directional_candidate")) - ) - if scan_mode == "early": - return str(row.get("window_phase") or "") in {"tomorrow", "week_ahead", "early_today"} - if scan_mode == "touch": - return ( - bool(window_meta.get("same_day")) - and str(row.get("window_phase") or "") in {"setup_today", "active_peak"} - and (_safe_float(row.get("touch_distance")) is not None) - and float(row.get("touch_distance")) <= 2.0 - ) - if scan_mode == "trend": - return bool(row.get("trend_alignment")) - return True - - filtered_rows = [ - row - for row in final_rows - if _passes_hard_filters(row) and _passes_mode_filters(row) - ] - filtered_rows.sort( - key=lambda row: ( - 1.0 if bool(row.get("is_directional_candidate")) else 0.0, - 1.0 if bool(row.get("is_peak_candidate")) else 0.0, - float(row.get("final_score") or 0.0), - float(row.get("edge_percent") or 0.0), - ), - reverse=True, - ) - - primary_signal = filtered_rows[0] if filtered_rows else None - signal_status = "ready" if primary_signal else "no_signal" - return { - "rows": filtered_rows[: filters["limit"]], - "distribution_bias": distribution_bias, - "primary_signal": primary_signal, - "signal_status": signal_status, - "candidate_count": len(filtered_rows), - "window_phase": window_meta.get("phase"), - "window_score": window_meta.get("score"), - "distribution_preview": distribution_preview[:6], - "distribution_full": distribution_preview, - "resolved_market_type": "maxtemp", - } \ No newline at end of file diff --git a/src/data_collection/polymarket_ws_cache.py b/src/data_collection/polymarket_ws_cache.py deleted file mode 100644 index a9cd7375..00000000 --- a/src/data_collection/polymarket_ws_cache.py +++ /dev/null @@ -1,427 +0,0 @@ -""" -Read-only Polymarket market WebSocket quote cache. - -The cache subscribes to public market-channel asset ids and stores executable -best bid / ask updates. It is deliberately optional: callers should keep REST -or CLOB polling as a fallback when the WebSocket client is unavailable. -""" - -from __future__ import annotations - -import asyncio -import json -import math -import os -import threading -import time -from typing import Any, Dict, Iterable, Optional, Set - -from loguru import logger - - -def _safe_float(value: Any) -> Optional[float]: - if value is None: - return None - try: - if isinstance(value, str): - value = value.strip() - if not value: - return None - numeric = float(value) - if math.isnan(numeric) or math.isinf(numeric): - return None - return numeric - except Exception: - return None - - -def _first_float(*values: Any) -> Optional[float]: - for value in values: - parsed = _safe_float(value) - if parsed is not None: - return parsed - return None - - -def _env_bool(name: str, default: bool = False) -> bool: - raw = os.getenv(name) - if raw is None: - return default - return raw.strip().lower() in {"1", "true", "yes", "on"} - - -class PolymarketWsQuoteCache: - def __init__( - self, - *, - enabled: bool = False, - endpoint: Optional[str] = None, - quote_ttl_sec: int = 8, - max_assets: int = 256, - reconnect_delay_sec: float = 3.0, - ) -> None: - self.enabled = enabled - self.endpoint = ( - endpoint - or os.getenv( - "POLYMARKET_WS_MARKET_URL", - "wss://ws-subscriptions-clob.polymarket.com/ws/market", - ) - or "" - ).strip() - self.quote_ttl_sec = max(1, int(quote_ttl_sec or 8)) - self.max_assets = max(1, int(max_assets or 256)) - self.reconnect_delay_sec = max(0.5, float(reconnect_delay_sec or 3.0)) - - self._desired_assets: Set[str] = set() - self._quotes: Dict[str, Dict[str, Any]] = {} - self._lock = threading.Lock() - self._thread: Optional[threading.Thread] = None - self._stop_event = threading.Event() - self._started = False - self._last_error: Optional[str] = None - self._last_connected_at: Optional[float] = None - self._last_message_at: Optional[float] = None - - @classmethod - def from_env(cls) -> "PolymarketWsQuoteCache": - return cls( - enabled=_env_bool("POLYMARKET_WS_PRICE_ENABLED", True), - endpoint=os.getenv("POLYMARKET_WS_MARKET_URL"), - quote_ttl_sec=int(os.getenv("POLYMARKET_WS_QUOTE_TTL_SEC", "8")), - max_assets=int(os.getenv("POLYMARKET_WS_MAX_ASSETS", "256")), - reconnect_delay_sec=float( - os.getenv("POLYMARKET_WS_RECONNECT_DELAY_SEC", "3") - ), - ) - - def start(self) -> None: - if not self.enabled or not self.endpoint: - return - with self._lock: - if self._started: - return - self._started = True - self._thread = threading.Thread( - target=self._thread_main, - name="polymarket-ws-quotes", - daemon=True, - ) - self._thread.start() - - def stop(self) -> None: - self._stop_event.set() - - def subscribe(self, asset_ids: Iterable[Any]) -> None: - if not self.enabled: - return - normalized = [] - for asset_id in asset_ids: - text = str(asset_id or "").strip() - if text: - normalized.append(text) - if not normalized: - return - - with self._lock: - remaining = self.max_assets - len(self._desired_assets) - for asset_id in normalized: - if asset_id in self._desired_assets: - continue - if remaining <= 0: - break - self._desired_assets.add(asset_id) - remaining -= 1 - self.start() - - def get_market_data(self, asset_id: Any) -> Optional[Dict[str, Any]]: - quote = self.get_quote(asset_id) - if not quote: - return None - - best_bid = _safe_float(quote.get("best_bid")) - best_ask = _safe_float(quote.get("best_ask")) - if best_bid is None and best_ask is None: - return None - - midpoint = None - if best_bid is not None and best_ask is not None: - midpoint = (best_bid + best_ask) / 2.0 - - age_ms = int((time.time() - float(quote.get("t") or time.time())) * 1000) - return { - "buy": best_ask, - "sell": best_bid, - "midpoint": midpoint, - "last_trade_price": _safe_float(quote.get("last_trade_price")), - "book": { - "best_bid": best_bid, - "best_ask": best_ask, - "bid_levels": [[best_bid, 0.0]] if best_bid is not None else [], - "ask_levels": [[best_ask, 0.0]] if best_ask is not None else [], - }, - "book_liquidity": None, - "quote_source": "polymarket_ws", - "quote_age_ms": age_ms, - } - - def get_quote(self, asset_id: Any) -> Optional[Dict[str, Any]]: - text = str(asset_id or "").strip() - if not text: - return None - now = time.time() - with self._lock: - quote = self._quotes.get(text) - if not quote: - return None - if now - float(quote.get("t") or 0.0) > self.quote_ttl_sec: - return None - return dict(quote) - - def status(self) -> Dict[str, Any]: - with self._lock: - return { - "enabled": self.enabled, - "started": self._started, - "endpoint": self.endpoint, - "asset_count": len(self._desired_assets), - "quote_count": len(self._quotes), - "last_error": self._last_error, - "last_connected_at": self._last_connected_at, - "last_message_at": self._last_message_at, - } - - def _thread_main(self) -> None: - try: - asyncio.run(self._run_forever()) - except Exception as exc: # pragma: no cover - defensive thread guard - with self._lock: - self._last_error = str(exc) - logger.warning(f"Polymarket WS quote cache stopped: {exc}") - - async def _run_forever(self) -> None: - try: - import websockets # type: ignore - except Exception as exc: - with self._lock: - self._last_error = f"websockets import failed: {exc}" - logger.warning(self._last_error) - return - - while not self._stop_event.is_set(): - try: - async with websockets.connect( - self.endpoint, - ping_interval=None, - close_timeout=2, - ) as ws: - with self._lock: - self._last_connected_at = time.time() - self._last_error = None - subscribed: Set[str] = set() - last_ping = 0.0 - - while not self._stop_event.is_set(): - desired = self._snapshot_assets() - missing = desired - subscribed - if missing: - await self._send_subscription( - ws, - missing, - initial=not subscribed, - ) - subscribed.update(missing) - - now = time.time() - if now - last_ping >= 10: - await ws.send(json.dumps({})) - last_ping = now - - try: - raw = await asyncio.wait_for(ws.recv(), timeout=1.0) - except asyncio.TimeoutError: - continue - self._handle_message(raw) - except Exception as exc: - with self._lock: - self._last_error = str(exc) - logger.warning(f"Polymarket WS reconnecting after error: {exc}") - await asyncio.sleep(self.reconnect_delay_sec) - - def _snapshot_assets(self) -> Set[str]: - with self._lock: - return set(self._desired_assets) - - async def _send_subscription( - self, - ws: Any, - asset_ids: Iterable[str], - *, - initial: bool, - ) -> None: - batch = [asset_id for asset_id in asset_ids if asset_id] - if not batch: - return - payload: Dict[str, Any] = { - "type": "subscribe", - "channel": "market", - "assets_ids": batch, - } - await ws.send(json.dumps(payload)) - - def _handle_message(self, raw: Any) -> None: - if raw in (None, "", "PONG"): - return - try: - payload = json.loads(raw) if isinstance(raw, str) else raw - except Exception: - return - - if isinstance(payload, list): - for item in payload: - self._handle_event(item) - return - self._handle_event(payload) - - def _handle_event(self, event: Any) -> None: - if not isinstance(event, dict): - return - if "event_type" in event: - event_type = str(event.get("event_type") or "").strip().lower() - else: - event_type = str(event.get("type") or "").strip().lower() - - # Polymarket market-channel messages may arrive without a type - # envelope — the payload contains price_changes / book / etc. - # directly at the top level. - has_price_data = any( - key in event - for key in ( - "price_changes", - "changes", - "assets", - "best_bid", - "best_ask", - "bid", - "ask", - "price", - ) - ) - - if event_type in { - "best_bid_ask", - "best_bid_ask_price_change", - "price_change", - "book", - "last_trade_price", - } or has_price_data: - self._handle_quote_event(event_type, event) - - def _handle_quote_event(self, event_type: str, event: Dict[str, Any]) -> None: - candidates = ( - event.get("price_changes") - or event.get("changes") - or event.get("assets") - or event.get("data") - ) - if isinstance(candidates, list): - for item in candidates: - if isinstance(item, dict): - self._upsert_quote(event_type, item, parent=event) - return - self._upsert_quote(event_type, event, parent=event) - - def _upsert_quote( - self, - event_type: str, - item: Dict[str, Any], - *, - parent: Dict[str, Any], - ) -> None: - asset_id = str( - item.get("asset_id") - or item.get("assetId") - or item.get("token_id") - or item.get("tokenId") - or parent.get("asset_id") - or parent.get("assetId") - or "" - ).strip() - if not asset_id: - return - - best_bid = _first_float( - item.get("best_bid"), - item.get("bid"), - item.get("bestBid"), - ) - best_ask = _first_float( - item.get("best_ask"), - item.get("ask"), - item.get("bestAsk"), - ) - if event_type == "book": - parsed_bid, parsed_ask = self._extract_book_top(item) - best_bid = best_bid if best_bid is not None else parsed_bid - best_ask = best_ask if best_ask is not None else parsed_ask - price = _safe_float(item.get("price")) - side = str(item.get("side") or "").strip().upper() - if event_type == "price_change" and price is not None: - if side == "BUY": - best_bid = price - elif side == "SELL": - best_ask = price - - last_trade = ( - _safe_float(item.get("last_trade_price")) - or _safe_float(item.get("lastTradePrice")) - or (price if event_type == "last_trade_price" else None) - ) - - now = time.time() - with self._lock: - previous = dict(self._quotes.get(asset_id) or {}) - if best_bid is not None: - previous["best_bid"] = best_bid - if best_ask is not None: - previous["best_ask"] = best_ask - if last_trade is not None: - previous["last_trade_price"] = last_trade - previous["asset_id"] = asset_id - previous["event_type"] = event_type - previous["t"] = now - self._quotes[asset_id] = previous - self._last_message_at = now - - def _extract_book_top( - self, - payload: Dict[str, Any], - ) -> tuple[Optional[float], Optional[float]]: - best_bid = None - best_ask = None - - bids = payload.get("bids") - if isinstance(bids, list): - for item in bids: - price = self._extract_level_price(item) - if price is None: - continue - best_bid = price if best_bid is None else max(best_bid, price) - - asks = payload.get("asks") - if isinstance(asks, list): - for item in asks: - price = self._extract_level_price(item) - if price is None: - continue - best_ask = price if best_ask is None else min(best_ask, price) - - return best_bid, best_ask - - @staticmethod - def _extract_level_price(level: Any) -> Optional[float]: - if isinstance(level, dict): - return _safe_float(level.get("price")) - if isinstance(level, (list, tuple)) and level: - return _safe_float(level[0]) - return None diff --git a/tests/test_polymarket_readonly.py b/tests/test_polymarket_readonly.py deleted file mode 100644 index 67b57b39..00000000 --- a/tests/test_polymarket_readonly.py +++ /dev/null @@ -1,952 +0,0 @@ -from src.data_collection.polymarket_readonly import PolymarketReadOnlyLayer - - -def test_normalize_orderbook_uses_sorted_best_prices(): - layer = PolymarketReadOnlyLayer() - raw = { - "bids": [ - {"price": "0.24", "size": "10"}, - {"price": "0.31", "size": "5"}, - {"price": "0.27", "size": "8"}, - ], - "asks": [ - {"price": "0.44", "size": "9"}, - {"price": "0.39", "size": "6"}, - {"price": "0.42", "size": "4"}, - ], - } - - book, _liquidity = layer._normalize_orderbook(raw) - - assert book is not None - assert book["best_bid"] == 0.31 - assert book["best_ask"] == 0.39 - assert book["bid_levels"][0][0] == 0.31 - assert book["ask_levels"][0][0] == 0.39 - - -def test_extract_market_bucket_range_supports_fahrenheit_ranges(): - layer = PolymarketReadOnlyLayer() - market = { - "question": "Will the highest temperature in Miami be between 80-81°F on April 21?", - "slug": "highest-temperature-in-miami-on-april-21-2026-80-81f", - } - - assert layer._extract_market_bucket_range(market) == (80.0, 81.0, "F") - assert layer._extract_market_bucket_temp(market) == 80.5 - assert layer._extract_market_bucket_label(market, 80.5) == "80-81F" - - -def test_fetch_token_market_data_uses_rest_orderbook_executable_prices(): - layer = PolymarketReadOnlyLayer() - layer.fast_price_only = False - payloads = { - ("/price", "BUY"): {"price": "0.27"}, - ("/price", "SELL"): {"price": "0.23"}, - ("/midpoint", None): {"midpoint": "0.50"}, - ("/last-trade-price", None): {"price": "0.49"}, - ("/book", None): { - "bids": [{"price": "0.24", "size": "10"}], - "asks": [{"price": "0.26", "size": "12"}], - }, - } - - def _fake_clob_get(path, params): - if path == "/price": - return payloads[(path, params.get("side"))] - return payloads[(path, None)] - - layer._clob_get = _fake_clob_get - - data = layer._fetch_token_market_data("token-1") - - # Executable BUY should match best ask from the book. - assert data["buy"] == 0.26 - # Executable SELL should match best bid from the book. - assert data["sell"] == 0.24 - assert data["midpoint"] == 0.5 - assert data["last_trade_price"] == 0.49 - assert data["quote_source"] == "polymarket_clob_rest" - - -def test_fetch_token_market_data_fast_price_only_skips_heavy_endpoints(): - layer = PolymarketReadOnlyLayer() - layer.fast_price_only = True - calls = [] - payloads = { - ("/price", "BUY"): {"price": "0.23"}, - ("/price", "SELL"): {"price": "0.27"}, - } - - def _fake_clob_get(path, params): - calls.append((path, params.get("side"))) - if path == "/price": - return payloads[(path, params.get("side"))] - return None - - layer._clob_get = _fake_clob_get - - data = layer._fetch_token_market_data("token-1") - - assert calls == [("/price", "BUY"), ("/price", "SELL")] - assert data["buy"] == 0.27 - assert data["sell"] == 0.23 - assert data["midpoint"] == 0.25 - assert round(data["spread"], 6) == 0.04 - assert data["last_trade_price"] is None - assert data["book"] is None - assert data["quote_source"] == "polymarket_clob_fast_price" - - -def test_fetch_token_market_data_keeps_buy_sell_semantics_without_orderbook(): - layer = PolymarketReadOnlyLayer() - layer.fast_price_only = False - payloads = { - ("/price", "BUY"): {"price": "0.23"}, - ("/price", "SELL"): {"price": "0.27"}, - ("/midpoint", None): {"midpoint": "0.25"}, - ("/last-trade-price", None): {"price": "0.24"}, - ("/book", None): None, - } - - def _fake_clob_get(path, params): - if path == "/price": - return payloads[(path, params.get("side"))] - return payloads[(path, None)] - - layer._clob_get = _fake_clob_get - - data = layer._fetch_token_market_data("token-1") - - assert data["buy"] == 0.27 - assert data["sell"] == 0.23 - assert data["midpoint"] == 0.25 - - -def test_weather_event_slug_uses_polymarket_city_aliases(): - layer = PolymarketReadOnlyLayer() - - assert ( - layer._build_weather_event_slug("new york", "2026-04-30") - == "highest-temperature-in-nyc-on-april-30-2026" - ) - assert ( - layer._build_weather_event_slug("aurora", "2026-04-30") - == "highest-temperature-in-denver-on-april-30-2026" - ) - - -def test_market_quote_fallback_uses_gamma_best_bid_ask_when_clob_missing(): - layer = PolymarketReadOnlyLayer() - market = { - "bestBid": "0.53", - "bestAsk": "0.54", - "lastTradePrice": "0.54", - "spread": "0.01", - "outcomePrices": '["0.535", "0.465"]', - "liquidityClob": "57040.5", - } - - yes = layer._merge_market_quote_fallback({}, market, "yes") - no = layer._merge_market_quote_fallback({}, market, "no") - - assert yes["buy"] == 0.54 - assert yes["sell"] == 0.53 - assert yes["midpoint"] == 0.535 - assert yes["quote_source"] == "polymarket_gamma_market_fallback" - assert no["buy"] == 0.47 - assert round(no["sell"], 6) == 0.46 - assert round(no["midpoint"], 6) == 0.465 - assert no["book_liquidity"] == 57040.5 - - -def test_market_quote_fallback_preserves_clob_prices_when_available(): - layer = PolymarketReadOnlyLayer() - market = { - "bestBid": "0.53", - "bestAsk": "0.54", - "outcomePrices": '["0.535", "0.465"]', - } - - merged = layer._merge_market_quote_fallback( - {"buy": 0.55, "sell": 0.52, "midpoint": 0.535, "quote_source": "polymarket_clob_rest"}, - market, - "yes", - ) - - assert merged["buy"] == 0.55 - assert merged["sell"] == 0.52 - assert merged["quote_source"] == "polymarket_clob_rest" - - -def test_get_token_market_data_uses_price_cache_within_ttl(): - layer = PolymarketReadOnlyLayer() - calls = [] - - def _fake_ws(_token_id): - calls.append(_token_id) - return {"buy": 0.33, "sell": 0.31, "midpoint": 0.32, "quote_source": "polymarket_ws"} - - layer._ws_cache.get_market_data = _fake_ws - - first = layer._get_token_market_data("token-1") - second = layer._get_token_market_data("token-1") - - assert first["buy"] == 0.33 - assert second["midpoint"] == 0.32 - assert calls == ["token-1"] - - -def test_price_analysis_computes_edge_kelly_and_lock(): - layer = PolymarketReadOnlyLayer() - - analysis = layer._build_price_analysis( - model_probability=0.62, - yes_buy=0.52, - yes_sell=0.50, - no_buy=0.45, - no_sell=0.43, - ) - - assert analysis["available"] is True - assert abs(analysis["yes"]["edge"] - 0.10) < 0.000001 - assert round(analysis["yes"]["kelly_fraction"], 6) == round( - (0.62 - 0.52) / (1.0 - 0.52), - 6, - ) - assert round(analysis["yes"]["quarter_kelly"], 6) == round( - ((0.62 - 0.52) / (1.0 - 0.52)) / 4.0, - 6, - ) - assert abs(analysis["no"]["edge"] - -0.07) < 0.000001 - assert analysis["lock"]["available"] is True - assert round(analysis["lock"]["edge"], 6) == 0.03 - assert analysis["best_side"] == "yes" - - -def test_trade_state_keeps_open_markets_tradable_after_gamma_end_date(): - layer = PolymarketReadOnlyLayer() - - state = layer._market_trade_state( - { - "active": True, - "closed": False, - "acceptingOrders": True, - "endDate": "2020-01-01T00:00:00Z", - } - ) - - assert state["tradable"] is True - assert state["reason"] is None - assert state["ended_at_utc"] == "2020-01-01T00:00:00+00:00" - - -def test_lau_fau_shan_uses_shenzhen_market_city(): - layer = PolymarketReadOnlyLayer() - captured = {} - - def _fake_find_primary_market(city_key, target_date, **_kwargs): - captured["primary_city_key"] = city_key - captured["target_date"] = target_date - return ( - { - "id": "market-1", - "question": "Will the highest temperature in Shenzhen be 30C or higher on April 23?", - "slug": "highest-temperature-in-shenzhen-on-april-23-2026-30c-or-higher", - "conditionId": "condition-1", - "active": True, - "closed": False, - "acceptingOrders": True, - "volumeNum": 1000, - "liquidityNum": 500, - }, - None, - ) - - layer._find_primary_market = _fake_find_primary_market - layer._extract_market_tokens = lambda _market: [ - {"outcome": "Yes", "token_id": "yes-token"}, - {"outcome": "No", "token_id": "no-token"}, - ] - layer._get_token_market_data = lambda token_id: ( - {"buy": 0.42, "sell": 0.40, "midpoint": 0.41} - if token_id == "yes-token" - else {"buy": 0.61, "sell": 0.59, "midpoint": 0.60} - ) - - def _fake_build_top_temperature_buckets(city_key, **_kwargs): - captured["bucket_city_key"] = city_key - return [] - - layer._build_top_temperature_buckets = _fake_build_top_temperature_buckets - - scan = layer.build_market_scan( - city="shenzhen", - target_date="2026-04-23", - temperature_bucket={"temp": 30, "probability": 0.58}, - model_probability=0.58, - ) - - assert captured["primary_city_key"] == "shenzhen" - assert captured["bucket_city_key"] == "shenzhen" - assert scan["city_key"] == "shenzhen" - assert scan["market_city_key"] == "shenzhen" - assert scan["selected_slug"] == "highest-temperature-in-shenzhen-on-april-23-2026-30c-or-higher" - - -def test_lau_fau_shan_alias_resolves_to_shenzhen_market_city(): - layer = PolymarketReadOnlyLayer() - captured = {} - - def _fake_find_primary_market(city_key, target_date, **_kwargs): - captured["primary_city_key"] = city_key - captured["target_date"] = target_date - return ( - { - "id": "market-1", - "question": "Will the highest temperature in Shenzhen be 30C or higher on April 23?", - "slug": "highest-temperature-in-shenzhen-on-april-23-2026-30c-or-higher", - "conditionId": "condition-1", - "active": True, - "closed": False, - "acceptingOrders": True, - }, - None, - ) - - layer._find_primary_market = _fake_find_primary_market - layer._extract_market_tokens = lambda _market: [ - {"outcome": "Yes", "token_id": "yes-token"}, - {"outcome": "No", "token_id": "no-token"}, - ] - layer._get_token_market_data = lambda _token_id: {"buy": 0.42, "sell": 0.40, "midpoint": 0.41} - layer._build_top_temperature_buckets = lambda *_args, **_kwargs: [] - - scan = layer.build_market_scan( - city="lau fau shan", - target_date="2026-04-23", - temperature_bucket={"temp": 30, "probability": 0.58}, - model_probability=0.58, - ) - - assert captured["primary_city_key"] == "shenzhen" - assert scan["city_key"] == "shenzhen" - assert scan["market_city_key"] == "shenzhen" - assert scan["selected_slug"] == "highest-temperature-in-shenzhen-on-april-23-2026-30c-or-higher" - - -def test_build_market_scan_lite_skips_related_buckets(): - layer = PolymarketReadOnlyLayer() - - layer._find_primary_market = lambda *_args, **_kwargs: ( - { - "id": "market-1", - "question": "Will the highest temperature in Shenzhen be 30C or higher on April 23?", - "slug": "highest-temperature-in-shenzhen-on-april-23-2026-30c-or-higher", - "conditionId": "condition-1", - "active": True, - "closed": False, - "acceptingOrders": True, - }, - None, - ) - layer._extract_market_tokens = lambda _market: [ - {"outcome": "Yes", "token_id": "yes-token"}, - {"outcome": "No", "token_id": "no-token"}, - ] - layer._get_token_market_data = lambda token_id: ( - {"buy": 0.42, "sell": 0.40, "midpoint": 0.41} - if token_id == "yes-token" - else {"buy": 0.61, "sell": 0.59, "midpoint": 0.60} - ) - - called = {"bucket": 0} - - def _fake_build_top_temperature_buckets(**_kwargs): - called["bucket"] += 1 - return [{"value": 30.0, "market_price": 0.41}] - - layer._build_top_temperature_buckets = _fake_build_top_temperature_buckets - - scan = layer.build_market_scan( - city="Shenzhen", - target_date="2026-04-23", - temperature_bucket={"temp": 30, "probability": 0.58}, - model_probability=0.58, - include_related_buckets=False, - ) - - assert scan["scan_scope"] == "lite" - assert scan["midpoint"] == 0.41 - assert round(scan["spread"], 6) == 0.02 - assert scan["top_buckets"] == [] - assert scan["all_buckets"] == [] - assert called["bucket"] == 0 - - -def test_build_market_scan_aggregates_emos_probability_for_threshold_market(): - layer = PolymarketReadOnlyLayer() - - layer._find_primary_market = lambda *_args, **_kwargs: ( - { - "id": "market-1", - "question": "Will the highest temperature in Shenzhen be 30C or higher on April 23?", - "slug": "highest-temperature-in-shenzhen-on-april-23-2026-30c-or-higher", - "conditionId": "condition-1", - "active": True, - "closed": False, - "acceptingOrders": True, - }, - None, - ) - layer._extract_market_tokens = lambda _market: [ - {"outcome": "Yes", "token_id": "yes-token"}, - {"outcome": "No", "token_id": "no-token"}, - ] - layer._get_token_market_data = lambda token_id: ( - {"buy": 0.42, "sell": 0.40, "midpoint": 0.41} - if token_id == "yes-token" - else {"buy": 0.61, "sell": 0.59, "midpoint": 0.60} - ) - layer._build_top_temperature_buckets = lambda **_kwargs: [] - - scan = layer.build_market_scan( - city="Shenzhen", - target_date="2026-04-23", - temperature_bucket={"temp": 30, "probability": 0.30}, - model_probability=0.30, - probability_distribution=[ - {"value": 29, "probability": 0.20}, - {"value": 30, "probability": 0.30}, - {"value": 31, "probability": 0.50}, - ], - temp_symbol="°C", - ) - - assert round(scan["model_probability"], 6) == 0.8 - assert round(scan["edge_percent"], 6) == 39.0 - - -def test_build_top_temperature_buckets_use_aggregated_emos_probability(): - layer = PolymarketReadOnlyLayer() - - primary_market = { - "slug": "highest-temperature-in-ankara-on-march-12-2026-14c-or-higher", - "question": "Will the highest temperature in Ankara be 14C or higher on March 12?", - "volumeNum": 1000, - } - markets = [ - primary_market, - { - "slug": "highest-temperature-in-ankara-on-march-12-2026-15c-or-higher", - "question": "Will the highest temperature in Ankara be 15C or higher on March 12?", - "volumeNum": 900, - }, - ] - layer._collect_related_temperature_markets = ( - lambda city_key, target_date, primary_market: markets - ) - layer._extract_market_tokens = lambda market: [ - {"outcome": "Yes", "token_id": f"{market['slug']}|yes"}, - {"outcome": "No", "token_id": f"{market['slug']}|no"}, - ] - layer._get_token_market_data = lambda token_id: ( - {"midpoint": 0.41, "buy": 0.42, "sell": 0.40} - if token_id.endswith("|yes") - else {"midpoint": 0.59, "buy": 0.60, "sell": 0.58} - ) - - rows = layer._build_top_temperature_buckets( - city_key="ankara", - target_date="2026-03-12", - primary_market=primary_market, - probability_distribution=[ - {"value": 13, "probability": 0.10}, - {"value": 14, "probability": 0.25}, - {"value": 15, "probability": 0.35}, - {"value": 16, "probability": 0.30}, - ], - temp_symbol="°C", - limit=4, - ) - - assert round(rows[0]["probability"], 6) == 0.9 - assert round(rows[0]["edge_percent"], 6) == 49.0 - assert round(rows[1]["probability"], 6) == 0.65 - - -def test_hydrate_bucket_prices_uses_executable_quotes_without_midpoint(): - layer = PolymarketReadOnlyLayer() - buckets = [ - { - "temp": 14.0, - "yes_token_id": "yes-token", - "no_token_id": "no-token", - } - ] - - def _fake_get_token_market_data(token_id): - if token_id == "yes-token": - return { - "buy": 0.66, - "sell": 0.70, - "quote_source": "polymarket_clob_rest", - "quote_age_ms": 0, - } - return {"buy": 0.30, "sell": 0.36} - - layer._get_token_market_data = _fake_get_token_market_data - - layer._hydrate_bucket_prices(buckets) - - assert buckets[0]["yes_buy"] == 0.66 - assert buckets[0]["yes_sell"] == 0.70 - assert buckets[0]["no_buy"] == 0.30 - assert buckets[0]["no_sell"] == 0.36 - assert round(buckets[0]["market_price"], 6) == 0.68 - assert round(buckets[0]["probability"], 6) == 0.68 - assert buckets[0]["quote_source"] == "polymarket_clob_rest" - - -def test_build_top_temperature_buckets_dedupes_same_temperature(): - layer = PolymarketReadOnlyLayer() - - primary_market = { - "slug": "highest-temperature-in-ankara-on-march-12-2026-14c-or-higher", - "question": "Will the highest temperature in Ankara be 14C or higher on March 12?", - "volumeNum": 1000, - } - markets = [ - primary_market, - { - "slug": "highest-temperature-in-ankara-on-march-12-2026-14c-or-higher-v2", - "question": "Will the highest temperature in Ankara be 14C or higher on March 12? (v2)", - "volumeNum": 900, - }, - { - "slug": "highest-temperature-in-ankara-on-march-12-2026-13c-or-higher", - "question": "Will the highest temperature in Ankara be 13C or higher on March 12?", - "volumeNum": 1100, - }, - { - "slug": "highest-temperature-in-ankara-on-march-12-2026-12c-or-higher", - "question": "Will the highest temperature in Ankara be 12C or higher on March 12?", - "volumeNum": 1200, - }, - { - "slug": "highest-temperature-in-ankara-on-march-12-2026-14c-or-lower", - "question": "Will the highest temperature in Ankara be 14C or lower on March 12?", - "volumeNum": 1300, - }, - ] - layer._collect_related_temperature_markets = ( - lambda city_key, target_date, primary_market: markets - ) - - def _fake_extract_market_tokens(market): - slug = str(market.get("slug") or "") - return [ - {"outcome": "Yes", "token_id": f"{slug}|yes"}, - {"outcome": "No", "token_id": f"{slug}|no"}, - ] - - layer._extract_market_tokens = _fake_extract_market_tokens - - midpoint_map = { - "highest-temperature-in-ankara-on-march-12-2026-14c-or-higher": 0.79, - "highest-temperature-in-ankara-on-march-12-2026-14c-or-higher-v2": 0.16, - "highest-temperature-in-ankara-on-march-12-2026-13c-or-higher": 0.06, - "highest-temperature-in-ankara-on-march-12-2026-12c-or-higher": 0.01, - "highest-temperature-in-ankara-on-march-12-2026-14c-or-lower": 0.92, - } - - def _fake_get_token_market_data(token_id): - slug, side = str(token_id).split("|", 1) - if side == "yes": - midpoint = midpoint_map.get(slug, 0.5) - return { - "midpoint": midpoint, - "buy": max(0.0, min(1.0, midpoint + 0.01)), - "sell": max(0.0, min(1.0, midpoint - 0.01)), - } - midpoint = 1.0 - midpoint_map.get(slug, 0.5) - return { - "midpoint": midpoint, - "buy": max(0.0, min(1.0, midpoint + 0.01)), - "sell": max(0.0, min(1.0, midpoint - 0.01)), - } - - layer._get_token_market_data = _fake_get_token_market_data - - rows = layer._build_top_temperature_buckets( - city_key="ankara", - target_date="2026-03-12", - primary_market=primary_market, - limit=4, - ) - - values = [row.get("value") for row in rows] - token_ids = [row.get("yes_token_id") for row in rows] - assert len(values) == len(set(values)) - assert len(token_ids) == len(set(token_ids)) - assert rows[0]["value"] == 14.0 - assert rows[0]["yes_token_id"] == ( - "highest-temperature-in-ankara-on-march-12-2026-14c-or-higher|yes" - ) - assert all(not str(row.get("label") or "").startswith("<=") for row in rows) - - -def test_find_primary_market_prefers_preferred_temperature_and_cache_key(): - layer = PolymarketReadOnlyLayer() - markets = [ - { - "slug": "highest-temperature-in-madrid-on-april-23-2026-22corbelow", - "question": "Will the highest temperature in Madrid be 22C or below on April 23?", - "volumeNum": 900000, - "active": True, - "closed": False, - "acceptingOrders": True, - "enableOrderBook": True, - }, - { - "slug": "highest-temperature-in-madrid-on-april-23-2026-27c", - "question": "Will the highest temperature in Madrid be 27C on April 23?", - "volumeNum": 1000, - "active": True, - "closed": False, - "acceptingOrders": True, - "enableOrderBook": True, - }, - ] - - layer._load_markets = lambda active_only=True: markets - - selected_27, reason_27 = layer._find_primary_market( - "madrid", - "2026-04-23", - preferred_temp=27.0, - ) - selected_22, reason_22 = layer._find_primary_market( - "madrid", - "2026-04-23", - preferred_temp=22.0, - ) - - assert reason_27 is None - assert reason_22 is None - assert selected_27["slug"] == "highest-temperature-in-madrid-on-april-23-2026-27c" - assert selected_22["slug"] == "highest-temperature-in-madrid-on-april-23-2026-22corbelow" - - -def _build_scan_test_layer(): - layer = PolymarketReadOnlyLayer() - markets = [ - { - "id": "m-above-14", - "slug": "highest-temperature-in-wellington-on-april-24-2026-14c-or-higher", - "question": "Will the highest temperature in Wellington be 14C or higher on April 24?", - "active": True, - "closed": False, - "acceptingOrders": True, - "enableOrderBook": True, - "liquidityNum": 12000, - "volumeNum": 4000, - "_model_prob": 0.60, - }, - { - "id": "m-below-16", - "slug": "highest-temperature-in-wellington-on-april-24-2026-16c-or-lower", - "question": "Will the highest temperature in Wellington be 16C or lower on April 24?", - "active": True, - "closed": False, - "acceptingOrders": True, - "enableOrderBook": True, - "liquidityNum": 9000, - "volumeNum": 3500, - "_model_prob": 0.30, - }, - { - "id": "m-above-17", - "slug": "highest-temperature-in-wellington-on-april-24-2026-17c-or-higher", - "question": "Will the highest temperature in Wellington be 17C or higher on April 24?", - "active": True, - "closed": False, - "acceptingOrders": True, - "enableOrderBook": True, - "liquidityNum": 7000, - "volumeNum": 2800, - "_model_prob": 0.20, - }, - ] - token_map = { - "m-above-14": {"yes": "yes-14", "no": "no-14"}, - "m-below-16": {"yes": "yes-16", "no": "no-16"}, - "m-above-17": {"yes": "yes-17", "no": "no-17"}, - } - quote_map = { - "yes-14": {"buy": 0.48, "sell": 0.46, "midpoint": 0.40, "spread": 0.02, "book_liquidity": 14000}, - "no-14": {"buy": 0.54, "sell": 0.52, "midpoint": 0.60, "spread": 0.02, "book_liquidity": 14000}, - "yes-16": {"buy": 0.42, "sell": 0.40, "midpoint": 0.50, "spread": 0.02, "book_liquidity": 9000}, - "no-16": {"buy": 0.56, "sell": 0.54, "midpoint": 0.50, "spread": 0.02, "book_liquidity": 9000}, - "yes-17": {"buy": 0.08, "sell": 0.07, "midpoint": 0.10, "spread": 0.01, "book_liquidity": 7500}, - "no-17": {"buy": 0.92, "sell": 0.91, "midpoint": 0.90, "spread": 0.01, "book_liquidity": 7500}, - } - - layer._collect_related_temperature_markets = lambda **_kwargs: markets - layer._aggregate_distribution_probability_for_market = ( - lambda market, **_kwargs: market.get("_model_prob") - ) - layer._extract_market_tokens = lambda market: [ - {"outcome": "Yes", "token_id": token_map[market["id"]]["yes"]}, - {"outcome": "No", "token_id": token_map[market["id"]]["no"]}, - ] - layer._batch_get_token_market_data = ( - lambda token_ids, include_books=False: { - token_id: dict(quote_map[token_id]) - for token_id in token_ids - if token_id in quote_map - } - ) - return layer, markets - - -def test_distribution_scan_bias_flips_below_markets_into_hotter_signal(): - layer, markets = _build_scan_test_layer() - - scan = layer._build_distribution_scan_pack( - city_key="wellington", - target_date="2026-04-24", - primary_market=markets[0], - probability_distribution=[], - temp_symbol="°C", - scan_context={ - "local_date": "2026-04-24", - "local_time": "13:10", - "peak": {"first_h": 14, "last_h": 16}, - "current_max_so_far": 13.4, - "current_temp": 13.0, - "trend": {"recent": []}, - "network_lead_signal": {}, - }, - scan_filters={"limit": 10}, - ) - - bias = scan["distribution_bias"] - assert bias["available"] is True - assert bias["direction"] == "hotter" - assert bias["score"] > 0 - - -def test_distribution_scan_returns_single_primary_signal_from_yes_no_mix(): - layer, markets = _build_scan_test_layer() - - scan = layer._build_distribution_scan_pack( - city_key="wellington", - target_date="2026-04-24", - primary_market=markets[0], - probability_distribution=[], - temp_symbol="°C", - scan_context={ - "local_date": "2026-04-24", - "local_time": "13:10", - "peak": {"first_h": 14, "last_h": 16}, - "current_max_so_far": 13.6, - "current_temp": 13.2, - "trend": {"recent": []}, - "network_lead_signal": {}, - }, - scan_filters={"limit": 10, "min_edge_pct": 2}, - ) - - assert scan["candidate_count"] >= 2 - assert isinstance(scan["primary_signal"], dict) - assert scan["primary_signal"]["side"] == "yes" - assert scan["primary_signal"]["id"] == scan["rows"][0]["id"] - assert scan["signal_status"] == "ready" - - -def test_distribution_scan_hard_filters_block_unusable_extreme_quotes(): - layer, markets = _build_scan_test_layer() - layer._batch_get_token_market_data = ( - lambda token_ids, include_books=False: { - token_id: { - "buy": 0.99 if token_id.startswith("yes") else 0.01, - "sell": 0.95 if token_id.startswith("yes") else 0.0, - "midpoint": 0.97 if token_id.startswith("yes") else 0.03, - "spread": 0.3, - "book_liquidity": 100, - } - for token_id in token_ids - } - ) - - scan = layer._build_distribution_scan_pack( - city_key="wellington", - target_date="2026-04-24", - primary_market=markets[0], - probability_distribution=[], - temp_symbol="°C", - scan_context={ - "local_date": "2026-04-24", - "local_time": "13:10", - "peak": {"first_h": 14, "last_h": 16}, - "current_max_so_far": 13.6, - "current_temp": 13.2, - "trend": {"recent": []}, - "network_lead_signal": {}, - }, - scan_filters={"limit": 10}, - ) - - assert scan["signal_status"] == "no_signal" - assert scan["candidate_count"] == 0 - assert scan["rows"] == [] - - -def test_distribution_scan_tradable_prefers_peak_bucket_and_adjacent_only(): - layer, markets = _build_scan_test_layer() - - scan = layer._build_distribution_scan_pack( - city_key="wellington", - target_date="2026-04-24", - primary_market=markets[0], - probability_distribution=[ - {"value": 14, "probability": 20}, - {"value": 15, "probability": 48}, - {"value": 16, "probability": 24}, - {"value": 17, "probability": 8}, - ], - temp_symbol="°C", - scan_context={ - "local_date": "2026-04-24", - "local_time": "13:10", - "peak": {"first_h": 14, "last_h": 16}, - "current_max_so_far": 13.6, - "current_temp": 13.2, - "trend": {"recent": []}, - "network_lead_signal": {}, - }, - scan_filters={"limit": 10, "scan_mode": "tradable", "min_edge_pct": 2}, - ) - - assert scan["signal_status"] == "ready" - assert scan["primary_signal"]["is_peak_candidate"] is True - assert scan["primary_signal"]["peak_distance"] in {0, 1} - assert all(bool(row.get("is_peak_candidate")) for row in scan["rows"]) - assert all((row.get("peak_distance") or 0) <= 1 for row in scan["rows"]) - - -def test_distribution_scan_uses_model_cluster_to_prefer_tail_no_over_yes(): - layer = PolymarketReadOnlyLayer() - markets = [ - { - "id": "m-21", - "slug": "highest-temperature-in-paris-on-april-24-2026-21c", - "question": "Will the highest temperature in Paris be 21C on April 24?", - "active": True, - "closed": False, - "acceptingOrders": True, - "enableOrderBook": True, - "liquidityNum": 6000, - "volumeNum": 5000, - "_model_prob": 0.205, - }, - { - "id": "m-22", - "slug": "highest-temperature-in-paris-on-april-24-2026-22c", - "question": "Will the highest temperature in Paris be 22C on April 24?", - "active": True, - "closed": False, - "acceptingOrders": True, - "enableOrderBook": True, - "liquidityNum": 6000, - "volumeNum": 5000, - "_model_prob": 0.34, - }, - { - "id": "m-24", - "slug": "highest-temperature-in-paris-on-april-24-2026-24c", - "question": "Will the highest temperature in Paris be 24C on April 24?", - "active": True, - "closed": False, - "acceptingOrders": True, - "enableOrderBook": True, - "liquidityNum": 6000, - "volumeNum": 5000, - "_model_prob": 0.06, - }, - ] - token_map = { - "m-21": {"yes": "yes-21", "no": "no-21"}, - "m-22": {"yes": "yes-22", "no": "no-22"}, - "m-24": {"yes": "yes-24", "no": "no-24"}, - } - quote_map = { - "yes-21": {"buy": 0.16, "sell": 0.14, "midpoint": 0.15, "spread": 0.02, "book_liquidity": 6000}, - "no-21": {"buy": 0.85, "sell": 0.83, "midpoint": 0.84, "spread": 0.02, "book_liquidity": 6000}, - "yes-22": {"buy": 0.34, "sell": 0.32, "midpoint": 0.33, "spread": 0.02, "book_liquidity": 6000}, - "no-22": {"buy": 0.67, "sell": 0.65, "midpoint": 0.66, "spread": 0.02, "book_liquidity": 6000}, - "yes-24": {"buy": 0.06, "sell": 0.05, "midpoint": 0.055, "spread": 0.01, "book_liquidity": 6000}, - "no-24": {"buy": 0.948, "sell": 0.93, "midpoint": 0.94, "spread": 0.018, "book_liquidity": 6000}, - } - - layer._collect_related_temperature_markets = lambda **_kwargs: markets - layer._aggregate_distribution_probability_for_market = ( - lambda market, **_kwargs: market.get("_model_prob") - ) - layer._extract_market_tokens = lambda market: [ - {"outcome": "Yes", "token_id": token_map[market["id"]]["yes"]}, - {"outcome": "No", "token_id": token_map[market["id"]]["no"]}, - ] - layer._batch_get_token_market_data = ( - lambda token_ids, include_books=False: { - token_id: dict(quote_map[token_id]) - for token_id in token_ids - if token_id in quote_map - } - ) - - scan = layer._build_distribution_scan_pack( - city_key="paris", - target_date="2026-04-24", - primary_market=markets[1], - probability_distribution=[], - temp_symbol="°C", - scan_context={ - "local_date": "2026-04-24", - "local_time": "08:54", - "peak": {"first_h": 14, "last_h": 16}, - "current_max_so_far": 20.0, - "current_temp": 20.0, - "trend": {"recent": []}, - "network_lead_signal": {}, - "deb_prediction": 22.0, - "models": { - "Open-Meteo": 22.4, - "ICON": 22.4, - "GEM": 22.2, - "GDPS": 22.2, - "ECMWF": 21.2, - "JMA": 20.9, - "GFS": 20.6, - "AIFS": 22.9, - }, - }, - scan_filters={"limit": 10, "scan_mode": "tradable", "min_edge_pct": 2}, - ) - - recommendations = {(row["target_value"], row["side"]) for row in scan["rows"]} - assert (21.0, "no") in recommendations - assert (24.0, "no") in recommendations - assert (21.0, "yes") not in recommendations - assert all(row.get("is_directional_candidate") for row in scan["rows"]) - assert all(row.get("cluster_adjusted") for row in scan["rows"]) - - -def test_normalize_scan_filters_raises_liquidity_floor_when_high_liquidity_only(): - layer = PolymarketReadOnlyLayer() - - filters = layer._normalize_scan_filters({"high_liquidity_only": True}) - - assert filters["high_liquidity_only"] is True - assert filters["min_liquidity"] >= 5000 diff --git a/tests/test_polymarket_ws_cache.py b/tests/test_polymarket_ws_cache.py deleted file mode 100644 index a08aadb3..00000000 --- a/tests/test_polymarket_ws_cache.py +++ /dev/null @@ -1,124 +0,0 @@ -import asyncio -import json -import time - -from src.data_collection.polymarket_ws_cache import PolymarketWsQuoteCache - - -def test_ws_cache_parses_best_bid_ask_event(): - cache = PolymarketWsQuoteCache(enabled=True, quote_ttl_sec=30) - - cache.subscribe(["asset-1"]) - cache._handle_message( - { - "event_type": "best_bid_ask", - "asset_id": "asset-1", - "best_bid": "0.41", - "best_ask": "0.44", - } - ) - - data = cache.get_market_data("asset-1") - - assert data is not None - assert data["sell"] == 0.41 - assert data["buy"] == 0.44 - assert data["midpoint"] == 0.425 - assert data["quote_source"] == "polymarket_ws" - - -def test_ws_cache_ignores_stale_quotes(): - cache = PolymarketWsQuoteCache(enabled=True, quote_ttl_sec=1) - cache._quotes["asset-1"] = { - "asset_id": "asset-1", - "best_bid": 0.41, - "best_ask": 0.44, - "t": time.time() - 10, - } - - assert cache.get_market_data("asset-1") is None - - -def test_ws_cache_parses_price_change_side_updates(): - cache = PolymarketWsQuoteCache(enabled=True, quote_ttl_sec=30) - - cache._handle_message( - { - "event_type": "price_change", - "changes": [ - { - "asset_id": "asset-1", - "side": "BUY", - "price": "0.48", - }, - { - "asset_id": "asset-1", - "side": "SELL", - "price": "0.45", - }, - ], - } - ) - - data = cache.get_market_data("asset-1") - - assert data is not None - assert data["buy"] == 0.45 - assert data["sell"] == 0.48 - - -def test_ws_cache_parses_price_changes_key_and_book_event(): - cache = PolymarketWsQuoteCache(enabled=True, quote_ttl_sec=30) - - cache._handle_message( - { - "event_type": "book", - "asset_id": "asset-1", - "bids": [{"price": "0.42"}, {"price": "0.44"}], - "asks": [{"price": "0.51"}, {"price": "0.49"}], - } - ) - cache._handle_message( - { - "event_type": "price_change", - "price_changes": [ - { - "asset_id": "asset-1", - "side": "BUY", - "price": "0.48", - } - ], - } - ) - - data = cache.get_market_data("asset-1") - - assert data is not None - assert data["sell"] == 0.48 - assert data["buy"] == 0.49 - - -def test_ws_cache_subscription_payloads_match_market_channel_shape(): - class FakeWs: - def __init__(self): - self.messages = [] - - async def send(self, payload): - self.messages.append(json.loads(payload)) - - cache = PolymarketWsQuoteCache(enabled=True) - ws = FakeWs() - - asyncio.run(cache._send_subscription(ws, ["asset-1"], initial=True)) - asyncio.run(cache._send_subscription(ws, ["asset-2"], initial=False)) - - assert ws.messages[0] == { - "type": "subscribe", - "channel": "market", - "assets_ids": ["asset-1"], - } - assert ws.messages[1] == { - "type": "subscribe", - "channel": "market", - "assets_ids": ["asset-2"], - }