first commit

2026-07-22 18:53:44 +08:00
commit 9259325d8a
60 changed files with 11081 additions and 0 deletions
@@ -0,0 +1,317 @@
+"""Polymarket Gamma REST client.
+
+Paginates /events, filters to active negRisk categoricals, and normalizes each
+event into our pydantic `Event` model. The shape of a Gamma event is documented
+in docs/api/negrisk.md; we read only the fields we need and ignore the rest.
+
+Refresh cadence and persistence live here too — the whole subsystem is kept in
+one file since the only consumer is the opportunity engine.
+"""
+from __future__ import annotations
+
+import asyncio
+import json
+from collections.abc import AsyncIterator, Iterable
+from dataclasses import dataclass
+from datetime import UTC, datetime
+from typing import Any
+
+import httpx
+import orjson
+from loguru import logger
+from tenacity import (
+    AsyncRetrying,
+    retry_if_exception_type,
+    stop_after_attempt,
+    wait_exponential,
+)
+
+from ..config import settings
+from ..db import db_conn
+from ..models import Event, Outcome
+
+GAMMA_EVENTS_PATH = "/events"
+DEFAULT_PAGE_SIZE = 100
+DEFAULT_TIMEOUT_S = 20.0
+
+
+@dataclass(slots=True)
+class DiscoveryStats:
+    pages_fetched: int = 0
+    events_seen: int = 0
+    neg_risk_events: int = 0
+    upserted: int = 0
+    skipped_inactive: int = 0
+    skipped_not_neg_risk: int = 0
+    skipped_malformed: int = 0
+
+
+class GammaClient:
+    """Thin async wrapper around Polymarket's Gamma REST API."""
+
+    def __init__(
+        self,
+        *,
+        host: str | None = None,
+        client: httpx.AsyncClient | None = None,
+        page_size: int = DEFAULT_PAGE_SIZE,
+        timeout_s: float = DEFAULT_TIMEOUT_S,
+    ) -> None:
+        self._host = (host or settings.gamma_host).rstrip("/")
+        self._page_size = page_size
+        self._timeout_s = timeout_s
+        self._owns_client = client is None
+        self._client = client or httpx.AsyncClient(
+            base_url=self._host,
+            timeout=timeout_s,
+            headers={"accept": "application/json"},
+            proxy=settings.proxy,
+        )
+
+    async def __aenter__(self) -> GammaClient:
+        return self
+
+    async def __aexit__(self, *_: object) -> None:
+        await self.aclose()
+
+    async def aclose(self) -> None:
+        if self._owns_client:
+            await self._client.aclose()
+
+    async def iter_active_event_pages(
+        self,
+        *,
+        order: str = "volume24hr",
+        ascending: bool = False,
+        max_pages: int = 50,
+    ) -> AsyncIterator[list[dict[str, Any]]]:
+        """Yield each page of raw event dicts until a short page or max_pages."""
+        offset = 0
+        for _ in range(max_pages):
+            params = {
+                "closed": "false",
+                "archived": "false",
+                "active": "true",
+                "limit": self._page_size,
+                "offset": offset,
+                "order": order,
+                "ascending": "true" if ascending else "false",
+            }
+            page = await self._get_json(GAMMA_EVENTS_PATH, params)
+            if not isinstance(page, list):
+                logger.warning("Gamma /events returned non-list: {!r}", type(page))
+                return
+            if not page:
+                return
+            yield page
+            if len(page) < self._page_size:
+                return
+            offset += self._page_size
+
+    async def _get_json(self, path: str, params: dict[str, Any]) -> Any:
+        async for attempt in AsyncRetrying(
+            stop=stop_after_attempt(4),
+            wait=wait_exponential(multiplier=0.5, max=5.0),
+            retry=retry_if_exception_type((httpx.HTTPError, httpx.TimeoutException)),
+            reraise=True,
+        ):
+            with attempt:
+                resp = await self._client.get(path, params=params)
+                resp.raise_for_status()
+                return orjson.loads(resp.content)
+        raise RuntimeError("unreachable")
+
+
+def normalize_event(raw: dict[str, Any]) -> Event | None:
+    """Translate a raw Gamma event dict into our `Event` model.
+
+    Returns None if the event is not an active negRisk categorical, or if its
+    required fields are missing/malformed.
+    """
+    if not raw.get("negRisk"):
+        return None
+    markets = raw.get("markets") or []
+    if len(markets) < 2:
+        return None
+
+    event_id = raw.get("negRiskMarketID") or str(raw.get("id") or "")
+    if not event_id:
+        return None
+
+    outcomes: list[Outcome] = []
+    for idx, m in enumerate(markets):
+        if m.get("closed") or m.get("archived"):
+            return None
+        token_ids_raw = m.get("clobTokenIds")
+        if isinstance(token_ids_raw, str):
+            try:
+                token_ids = orjson.loads(token_ids_raw)
+            except orjson.JSONDecodeError:
+                return None
+        else:
+            token_ids = token_ids_raw
+        if not isinstance(token_ids, list) or len(token_ids) < 1:
+            return None
+        yes_token_id = str(token_ids[0])
+        if not yes_token_id:
+            return None
+        name = (
+            m.get("groupItemTitle")
+            or m.get("outcome")
+            or m.get("question")
+            or f"Outcome {idx + 1}"
+        )
+        outcomes.append(Outcome(token_id=yes_token_id, name=str(name), outcome_index=idx))
+
+    if len(outcomes) < 2:
+        return None
+
+    seen: set[str] = set()
+    for o in outcomes:
+        if o.token_id in seen:
+            return None
+        seen.add(o.token_id)
+
+    end_date = _parse_end_date(raw.get("endDate"))
+
+    return Event(
+        id=event_id,
+        slug=str(raw.get("slug") or event_id),
+        title=str(raw.get("title") or raw.get("slug") or event_id),
+        is_neg_risk=True,
+        end_date=end_date,
+        outcomes=tuple(outcomes),
+    )
+
+
+def _parse_end_date(value: Any) -> datetime | None:
+    if not value or not isinstance(value, str):
+        return None
+    try:
+        dt = datetime.fromisoformat(value.replace("Z", "+00:00"))
+    except ValueError:
+        return None
+    return dt if dt.tzinfo else dt.replace(tzinfo=UTC)
+
+
+async def upsert_events(events: Iterable[Event], *, raw_by_id: dict[str, Any] | None = None) -> int:
+    """Persist events + outcomes to SQLite. Returns count of events upserted."""
+    now = datetime.now(UTC).isoformat()
+    raw_by_id = raw_by_id or {}
+    count = 0
+    async with db_conn() as conn:
+        for ev in events:
+            raw_json = json.dumps(raw_by_id.get(ev.id)) if ev.id in raw_by_id else None
+            await conn.execute(
+                """
+                INSERT INTO events (id, slug, title, is_neg_risk, end_date, active, raw_json, updated_at)
+                VALUES (?, ?, ?, ?, ?, 1, ?, ?)
+                ON CONFLICT(id) DO UPDATE SET
+                  slug=excluded.slug,
+                  title=excluded.title,
+                  is_neg_risk=excluded.is_neg_risk,
+                  end_date=excluded.end_date,
+                  active=1,
+                  raw_json=COALESCE(excluded.raw_json, events.raw_json),
+                  updated_at=excluded.updated_at
+                """,
+                (
+                    ev.id,
+                    ev.slug,
+                    ev.title,
+                    1 if ev.is_neg_risk else 0,
+                    ev.end_date.isoformat() if ev.end_date else None,
+                    raw_json,
+                    now,
+                ),
+            )
+            for o in ev.outcomes:
+                await conn.execute(
+                    """
+                    INSERT INTO outcomes (token_id, event_id, name, outcome_index, updated_at)
+                    VALUES (?, ?, ?, ?, ?)
+                    ON CONFLICT(token_id) DO UPDATE SET
+                      event_id=excluded.event_id,
+                      name=excluded.name,
+                      outcome_index=excluded.outcome_index,
+                      updated_at=excluded.updated_at
+                    """,
+                    (o.token_id, ev.id, o.name, o.outcome_index, now),
+                )
+            count += 1
+        await conn.commit()
+    return count
+
+
+async def mark_inactive(kept_event_ids: set[str]) -> int:
+    """Mark any previously-active event not in `kept_event_ids` as inactive."""
+    if not kept_event_ids:
+        return 0
+    now = datetime.now(UTC).isoformat()
+    placeholders = ",".join("?" for _ in kept_event_ids)
+    async with db_conn() as conn:
+        cursor = await conn.execute(
+            f"UPDATE events SET active=0, updated_at=? "
+            f"WHERE active=1 AND id NOT IN ({placeholders})",
+            (now, *kept_event_ids),
+        )
+        await conn.commit()
+        return cursor.rowcount or 0
+
+
+async def discover_once(
+    *,
+    client: GammaClient | None = None,
+    max_pages: int = 20,
+) -> DiscoveryStats:
+    """Single pass: walk Gamma, filter to negRisk, persist, mark drops inactive."""
+    stats = DiscoveryStats()
+    owned: GammaClient | None = None
+    if client is None:
+        owned = GammaClient()
+        client = owned
+
+    events: list[Event] = []
+    raw_by_id: dict[str, Any] = {}
+    try:
+        async for page in client.iter_active_event_pages(max_pages=max_pages):
+            stats.pages_fetched += 1
+            for raw in page:
+                stats.events_seen += 1
+                if not raw.get("negRisk"):
+                    stats.skipped_not_neg_risk += 1
+                    continue
+                stats.neg_risk_events += 1
+                if raw.get("closed") or raw.get("archived"):
+                    stats.skipped_inactive += 1
+                    continue
+                ev = normalize_event(raw)
+                if ev is None:
+                    stats.skipped_malformed += 1
+                    continue
+                events.append(ev)
+                raw_by_id[ev.id] = raw
+    finally:
+        if owned is not None:
+            await owned.aclose()
+
+    stats.upserted = await upsert_events(events, raw_by_id=raw_by_id)
+    await mark_inactive({ev.id for ev in events})
+    logger.info(
+        "discovery: seen={}, negRisk={}, upserted={}, malformed={}",
+        stats.events_seen,
+        stats.neg_risk_events,
+        stats.upserted,
+        stats.skipped_malformed,
+    )
+    return stats
+
+
+async def discovery_loop(interval_seconds: int = 120) -> None:
+    """Run discover_once forever, sleeping between passes."""
+    while True:
+        try:
+            await discover_once()
+        except Exception as exc:
+            logger.exception("discovery pass failed: {}", exc)
+        await asyncio.sleep(interval_seconds)
@@ -0,0 +1,282 @@
+"""Polymarket CLOB market-channel WebSocket subscriber.
+
+Wire format reference (April 2026):
+  - endpoint: wss://ws-subscriptions-clob.polymarket.com/ws/market
+  - subscribe:   {"type":"market","assets_ids":[...],"custom_feature_enabled":true}
+  - snapshot:    event_type="book", asks/bids are lists of {price,size} strings
+  - delta:       event_type="price_change", wrapper="price_changes",
+                 side="BUY"|"SELL", size="0" means remove
+  - keepalive:   send the literal text frame "PING" every ~10s
+  - desync:      no sequence numbers; on hash mismatch or missed heartbeat,
+                 drop state and re-subscribe (server re-pushes snapshot)
+  - sharding:    cap tokens/socket around 100-200; open more sockets as needed
+"""
+from __future__ import annotations
+
+import asyncio
+import itertools
+from datetime import UTC, datetime
+from decimal import Decimal, InvalidOperation
+from typing import Any
+
+import orjson
+import websockets
+from loguru import logger
+from websockets.asyncio.client import ClientConnection
+from websockets.exceptions import ConnectionClosed
+
+from ..book.l2 import BookRegistry, LevelChange, Side
+from ..config import settings
+import socket
+
+DEFAULT_SHARD_SIZE = 100
+PING_INTERVAL_S = 10.0
+RECONNECT_BACKOFF_S = (1.0, 2.0, 4.0, 8.0, 15.0)
+
+
+async def _tunnel_through_proxy(uri: str) -> socket.socket | None:
+    """Open an HTTP CONNECT tunnel to the WS host via the configured proxy.
+
+    Returns a connected socket ready for the WS handshake, or None if no proxy.
+    """
+    if not settings.proxy:
+        return None
+    from urllib.parse import urlparse
+
+    parsed = urlparse(uri)
+    proxy_parsed = urlparse(settings.proxy)
+    host, port = parsed.hostname, parsed.port or (443 if parsed.scheme == "wss" else 80)
+    sock = socket.create_connection((proxy_parsed.hostname, proxy_parsed.port), timeout=10)
+    connect_req = (
+        f"CONNECT {host}:{port} HTTP/1.1\r\n"
+        f"Host: {host}:{port}\r\n"
+        f"Proxy-Connection: keep-alive\r\n\r\n"
+    )
+    sock.sendall(connect_req.encode())
+    resp = b""
+    while b"\r\n\r\n" not in resp:
+        chunk = sock.recv(4096)
+        if not chunk:
+            raise OSError("proxy closed before CONNECT response")
+        resp += chunk
+    status = resp.split(b"\r\n", 1)[0]
+    if b" 200 " not in status:
+        sock.close()
+        raise OSError(f"proxy CONNECT failed: {status!r}")
+    sock.settimeout(None)
+    return sock
+
+
+def _to_decimal(v: object, default: Decimal = Decimal(0)) -> Decimal:
+    try:
+        return Decimal(str(v))
+    except (InvalidOperation, TypeError, ValueError):
+        return default
+
+
+def _parse_ts_ms(v: Any) -> datetime:
+    try:
+        ms = int(v)
+    except (TypeError, ValueError):
+        return datetime.now(UTC)
+    return datetime.fromtimestamp(ms / 1000.0, tz=UTC)
+
+
+def _parse_level_list(raw: Any) -> list[tuple[Decimal, Decimal]]:
+    out: list[tuple[Decimal, Decimal]] = []
+    if not isinstance(raw, list):
+        return out
+    for lvl in raw:
+        if not isinstance(lvl, dict):
+            continue
+        price = _to_decimal(lvl.get("price"))
+        size = _to_decimal(lvl.get("size"))
+        if price <= 0:
+            continue
+        out.append((price, size))
+    return out
+
+
+def _side_from_buy_sell(raw: Any) -> Side | None:
+    if not isinstance(raw, str):
+        return None
+    s = raw.strip().upper()
+    if s == "BUY":
+        return Side.BID
+    if s == "SELL":
+        return Side.ASK
+    return None
+
+
+class MarketChannel:
+    """Maintains a single WS connection for one shard of token ids."""
+
+    def __init__(
+        self,
+        token_ids: list[str],
+        *,
+        registry: BookRegistry,
+        url: str | None = None,
+    ) -> None:
+        if not token_ids:
+            raise ValueError("MarketChannel needs at least one token id")
+        self._token_ids = list(token_ids)
+        self._registry = registry
+        self._url = url or settings.ws_url
+        self._stop = asyncio.Event()
+
+    async def run(self) -> None:
+        backoff_cycle = itertools.cycle(RECONNECT_BACKOFF_S)
+        while not self._stop.is_set():
+            try:
+                await self._connect_and_consume()
+                # Clean exit (e.g. stop requested); break loop.
+                if self._stop.is_set():
+                    return
+                delay = 1.0
+            except (TimeoutError, ConnectionClosed, OSError) as exc:
+                delay = next(backoff_cycle)
+                logger.warning(
+                    "ws disconnect ({}); reconnecting in {}s", exc.__class__.__name__, delay
+                )
+            except Exception as exc:
+                delay = next(backoff_cycle)
+                logger.exception("ws fatal ({}); reconnecting in {}s", exc, delay)
+            await asyncio.sleep(delay)
+
+    def stop(self) -> None:
+        self._stop.set()
+
+    async def _connect_and_consume(self) -> None:
+        sock = await _tunnel_through_proxy(self._url)
+        connect_kwargs = {"max_size": 2**22}
+        if sock is not None:
+            connect_kwargs["sock"] = sock
+        async with websockets.connect(self._url, **connect_kwargs) as ws:
+            await self._subscribe(ws)
+            heartbeat = asyncio.create_task(self._heartbeat(ws))
+            try:
+                async for raw in ws:
+                    if self._stop.is_set():
+                        break
+                    self._handle_message(raw)
+            finally:
+                heartbeat.cancel()
+                try:
+                    await heartbeat
+                except asyncio.CancelledError:
+                    pass
+
+    async def _subscribe(self, ws: ClientConnection) -> None:
+        msg = orjson.dumps(
+            {
+                "type": "market",
+                "assets_ids": self._token_ids,
+                "custom_feature_enabled": True,
+            }
+        )
+        await ws.send(msg)
+        logger.info("ws subscribed to {} tokens", len(self._token_ids))
+
+    async def _heartbeat(self, ws: ClientConnection) -> None:
+        while not self._stop.is_set():
+            await asyncio.sleep(PING_INTERVAL_S)
+            try:
+                await ws.send("PING")
+            except ConnectionClosed:
+                return
+
+    def _handle_message(self, raw: str | bytes) -> None:
+        if isinstance(raw, str):
+            if raw.strip() in ("PONG", "PING"):
+                return
+            payload = orjson.loads(raw)
+        else:
+            # Binary frames shouldn't normally arrive from this channel.
+            try:
+                payload = orjson.loads(raw)
+            except orjson.JSONDecodeError:
+                return
+        if isinstance(payload, list):
+            for item in payload:
+                self._dispatch(item)
+        elif isinstance(payload, dict):
+            self._dispatch(payload)
+
+    def _dispatch(self, msg: dict[str, Any]) -> None:
+        event_type = msg.get("event_type")
+        if event_type == "book":
+            self._apply_book(msg)
+        elif event_type == "price_change":
+            self._apply_price_change(msg)
+        elif event_type in ("tick_size_change", "last_trade_price", "best_bid_ask"):
+            # Not used by the arb math today; logged at debug.
+            logger.debug("ws {}: {}", event_type, msg)
+        elif event_type in ("new_market", "market_resolved"):
+            logger.info("ws {}: {}", event_type, msg)
+        else:
+            logger.debug("ws unknown event_type={}: {}", event_type, msg)
+
+    def _apply_book(self, msg: dict[str, Any]) -> None:
+        asset_id = msg.get("asset_id")
+        if not isinstance(asset_id, str):
+            return
+        bids = _parse_level_list(msg.get("bids"))
+        asks = _parse_level_list(msg.get("asks"))
+        ts = _parse_ts_ms(msg.get("timestamp"))
+        self._registry.apply_snapshot(
+            asset_id, bids=bids, asks=asks, timestamp=ts, book_hash=msg.get("hash")
+        )
+
+    def _apply_price_change(self, msg: dict[str, Any]) -> None:
+        ts = _parse_ts_ms(msg.get("timestamp"))
+        changes_raw = msg.get("price_changes")
+        if not isinstance(changes_raw, list):
+            return
+        by_asset: dict[str, list[LevelChange]] = {}
+        latest_hash: dict[str, str] = {}
+        for c in changes_raw:
+            if not isinstance(c, dict):
+                continue
+            asset_id = c.get("asset_id")
+            side = _side_from_buy_sell(c.get("side"))
+            if not isinstance(asset_id, str) or side is None:
+                continue
+            price = _to_decimal(c.get("price"))
+            size = _to_decimal(c.get("size"))
+            if price <= 0:
+                continue
+            by_asset.setdefault(asset_id, []).append(
+                LevelChange(price=price, size=size, side=side)
+            )
+            h = c.get("hash")
+            if isinstance(h, str):
+                latest_hash[asset_id] = h
+        for asset_id, changes in by_asset.items():
+            self._registry.apply_delta(
+                asset_id,
+                changes,
+                timestamp=ts,
+                book_hash=latest_hash.get(asset_id),
+            )
+
+
+def shard_tokens(token_ids: list[str], shard_size: int = DEFAULT_SHARD_SIZE) -> list[list[str]]:
+    return [token_ids[i : i + shard_size] for i in range(0, len(token_ids), shard_size)]
+
+
+async def run_market_channels(
+    token_ids: list[str],
+    *,
+    registry: BookRegistry,
+    shard_size: int = DEFAULT_SHARD_SIZE,
+) -> None:
+    """Run one MarketChannel per shard concurrently. Returns when all exit."""
+    if not token_ids:
+        return
+    channels = [MarketChannel(shard, registry=registry) for shard in shard_tokens(token_ids, shard_size)]
+    try:
+        await asyncio.gather(*(ch.run() for ch in channels))
+    finally:
+        for ch in channels:
+            ch.stop()