"""Polymarket Gamma REST client. Paginates /events, filters to active negRisk categoricals, and normalizes each event into our pydantic `Event` model. The shape of a Gamma event is documented in docs/api/negrisk.md; we read only the fields we need and ignore the rest. Refresh cadence and persistence live here too — the whole subsystem is kept in one file since the only consumer is the opportunity engine. """ from __future__ import annotations import asyncio import json from collections.abc import AsyncIterator, Iterable from dataclasses import dataclass from datetime import UTC, datetime from typing import Any import httpx import orjson from loguru import logger from tenacity import ( AsyncRetrying, retry_if_exception_type, stop_after_attempt, wait_exponential, ) from ..config import settings from ..db import db_conn from ..models import Event, Outcome GAMMA_EVENTS_PATH = "/events" DEFAULT_PAGE_SIZE = 100 DEFAULT_TIMEOUT_S = 20.0 @dataclass(slots=True) class DiscoveryStats: pages_fetched: int = 0 events_seen: int = 0 neg_risk_events: int = 0 upserted: int = 0 skipped_inactive: int = 0 skipped_not_neg_risk: int = 0 skipped_malformed: int = 0 class GammaClient: """Thin async wrapper around Polymarket's Gamma REST API.""" def __init__( self, *, host: str | None = None, client: httpx.AsyncClient | None = None, page_size: int = DEFAULT_PAGE_SIZE, timeout_s: float = DEFAULT_TIMEOUT_S, ) -> None: self._host = (host or settings.gamma_host).rstrip("/") self._page_size = page_size self._timeout_s = timeout_s self._owns_client = client is None self._client = client or httpx.AsyncClient( base_url=self._host, timeout=timeout_s, headers={"accept": "application/json"}, proxy=settings.proxy, ) async def __aenter__(self) -> GammaClient: return self async def __aexit__(self, *_: object) -> None: await self.aclose() async def aclose(self) -> None: if self._owns_client: await self._client.aclose() async def iter_active_event_pages( self, *, order: str = "volume24hr", ascending: bool = False, max_pages: int = 50, ) -> AsyncIterator[list[dict[str, Any]]]: """Yield each page of raw event dicts until a short page or max_pages.""" offset = 0 for _ in range(max_pages): params = { "closed": "false", "archived": "false", "active": "true", "limit": self._page_size, "offset": offset, "order": order, "ascending": "true" if ascending else "false", } page = await self._get_json(GAMMA_EVENTS_PATH, params) if not isinstance(page, list): logger.warning("Gamma /events returned non-list: {!r}", type(page)) return if not page: return yield page if len(page) < self._page_size: return offset += self._page_size async def _get_json(self, path: str, params: dict[str, Any]) -> Any: async for attempt in AsyncRetrying( stop=stop_after_attempt(4), wait=wait_exponential(multiplier=0.5, max=5.0), retry=retry_if_exception_type((httpx.HTTPError, httpx.TimeoutException)), reraise=True, ): with attempt: resp = await self._client.get(path, params=params) resp.raise_for_status() return orjson.loads(resp.content) raise RuntimeError("unreachable") def normalize_event(raw: dict[str, Any]) -> Event | None: """Translate a raw Gamma event dict into our `Event` model. Returns None if the event is not an active negRisk categorical, or if its required fields are missing/malformed. """ if not raw.get("negRisk"): return None markets = raw.get("markets") or [] if len(markets) < 2: return None event_id = raw.get("negRiskMarketID") or str(raw.get("id") or "") if not event_id: return None outcomes: list[Outcome] = [] for idx, m in enumerate(markets): if m.get("closed") or m.get("archived"): return None token_ids_raw = m.get("clobTokenIds") if isinstance(token_ids_raw, str): try: token_ids = orjson.loads(token_ids_raw) except orjson.JSONDecodeError: return None else: token_ids = token_ids_raw if not isinstance(token_ids, list) or len(token_ids) < 1: return None yes_token_id = str(token_ids[0]) if not yes_token_id: return None name = ( m.get("groupItemTitle") or m.get("outcome") or m.get("question") or f"Outcome {idx + 1}" ) outcomes.append(Outcome(token_id=yes_token_id, name=str(name), outcome_index=idx)) if len(outcomes) < 2: return None seen: set[str] = set() for o in outcomes: if o.token_id in seen: return None seen.add(o.token_id) end_date = _parse_end_date(raw.get("endDate")) return Event( id=event_id, slug=str(raw.get("slug") or event_id), title=str(raw.get("title") or raw.get("slug") or event_id), is_neg_risk=True, end_date=end_date, outcomes=tuple(outcomes), ) def _parse_end_date(value: Any) -> datetime | None: if not value or not isinstance(value, str): return None try: dt = datetime.fromisoformat(value.replace("Z", "+00:00")) except ValueError: return None return dt if dt.tzinfo else dt.replace(tzinfo=UTC) async def upsert_events(events: Iterable[Event], *, raw_by_id: dict[str, Any] | None = None) -> int: """Persist events + outcomes to SQLite. Returns count of events upserted.""" now = datetime.now(UTC).isoformat() raw_by_id = raw_by_id or {} count = 0 async with db_conn() as conn: for ev in events: raw_json = json.dumps(raw_by_id.get(ev.id)) if ev.id in raw_by_id else None await conn.execute( """ INSERT INTO events (id, slug, title, is_neg_risk, end_date, active, raw_json, updated_at) VALUES (?, ?, ?, ?, ?, 1, ?, ?) ON CONFLICT(id) DO UPDATE SET slug=excluded.slug, title=excluded.title, is_neg_risk=excluded.is_neg_risk, end_date=excluded.end_date, active=1, raw_json=COALESCE(excluded.raw_json, events.raw_json), updated_at=excluded.updated_at """, ( ev.id, ev.slug, ev.title, 1 if ev.is_neg_risk else 0, ev.end_date.isoformat() if ev.end_date else None, raw_json, now, ), ) for o in ev.outcomes: await conn.execute( """ INSERT INTO outcomes (token_id, event_id, name, outcome_index, updated_at) VALUES (?, ?, ?, ?, ?) ON CONFLICT(token_id) DO UPDATE SET event_id=excluded.event_id, name=excluded.name, outcome_index=excluded.outcome_index, updated_at=excluded.updated_at """, (o.token_id, ev.id, o.name, o.outcome_index, now), ) count += 1 await conn.commit() return count async def mark_inactive(kept_event_ids: set[str]) -> int: """Mark any previously-active event not in `kept_event_ids` as inactive.""" if not kept_event_ids: return 0 now = datetime.now(UTC).isoformat() placeholders = ",".join("?" for _ in kept_event_ids) async with db_conn() as conn: cursor = await conn.execute( f"UPDATE events SET active=0, updated_at=? " f"WHERE active=1 AND id NOT IN ({placeholders})", (now, *kept_event_ids), ) await conn.commit() return cursor.rowcount or 0 async def discover_once( *, client: GammaClient | None = None, max_pages: int = 20, ) -> DiscoveryStats: """Single pass: walk Gamma, filter to negRisk, persist, mark drops inactive.""" stats = DiscoveryStats() owned: GammaClient | None = None if client is None: owned = GammaClient() client = owned events: list[Event] = [] raw_by_id: dict[str, Any] = {} try: async for page in client.iter_active_event_pages(max_pages=max_pages): stats.pages_fetched += 1 for raw in page: stats.events_seen += 1 if not raw.get("negRisk"): stats.skipped_not_neg_risk += 1 continue stats.neg_risk_events += 1 if raw.get("closed") or raw.get("archived"): stats.skipped_inactive += 1 continue ev = normalize_event(raw) if ev is None: stats.skipped_malformed += 1 continue events.append(ev) raw_by_id[ev.id] = raw finally: if owned is not None: await owned.aclose() stats.upserted = await upsert_events(events, raw_by_id=raw_by_id) await mark_inactive({ev.id for ev in events}) logger.info( "discovery: seen={}, negRisk={}, upserted={}, malformed={}", stats.events_seen, stats.neg_risk_events, stats.upserted, stats.skipped_malformed, ) return stats async def discovery_loop(interval_seconds: int = 120) -> None: """Run discover_once forever, sleeping between passes.""" while True: try: await discover_once() except Exception as exc: logger.exception("discovery pass failed: {}", exc) await asyncio.sleep(interval_seconds)