From cb4438655c92cd11f396fcdded7d9af1adbaff90 Mon Sep 17 00:00:00 2001 From: Daniel Sapkota Date: Sun, 5 Jul 2026 18:47:11 -0400 Subject: [PATCH] Harden it up --- README.md | 7 +- config/config.toml | 4 +- config/markets.toml | 10 + src/polymaker/cli.py | 12 ++ src/polymaker/config.py | 6 +- src/polymaker/engine.py | 176 +++++++++++++++--- src/polymaker/execution/gateway.py | 163 +++++++++++++++- src/polymaker/marketdata/orderbook.py | 8 +- src/polymaker/marketdata/service.py | 19 +- src/polymaker/moneydoctor.py | 244 ++++++++++++++++++++++++ src/polymaker/state/store.py | 59 ++++-- src/polymaker/state/tracker.py | 16 +- src/polymaker/userstream/client.py | 27 ++- tests/test_execution.py | 3 +- tests/test_hardening.py | 257 ++++++++++++++++++++++++++ 15 files changed, 943 insertions(+), 68 deletions(-) create mode 100644 src/polymaker/moneydoctor.py create mode 100644 tests/test_hardening.py diff --git a/README.md b/README.md index d970505..8c49adb 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ # poly-maker A maker-only market-making bot for **Polymarket CLOB V2**, focused on political -markets. Single async process, local-file config (no Google Sheets), typed and +markets. Single async process, local-file config, typed and tested. > [!WARNING] @@ -82,8 +82,9 @@ uv run polymaker run --paper # 4. preflight the wallet before going live uv run polymaker doctor -# 5. one safe live round-trip (~$5 post-only order, placed deep and cancelled) -uv run polymaker livetest +# 5. self-tests: a deep post-only order (free), then a real fill round-trip (~cents) +uv run polymaker livetest # place a deep post-only order + cancel (no fill) +uv run polymaker moneydoctor # limit rest + market buy + market sell, auto-flattens # 6. go live uv run polymaker run diff --git a/config/config.toml b/config/config.toml index 431bb9b..038bd69 100644 --- a/config/config.toml +++ b/config/config.toml @@ -18,7 +18,7 @@ signature_type = 3 clob_host = "https://clob.polymarket.com" gamma_host = "https://gamma-api.polymarket.com" data_api_host = "https://data-api.polymarket.com" -polygon_rpc = "https://polygon-rpc.com" +polygon_rpc = "https://polygon-bor-rpc.publicnode.com" [engine] debounce_ms = 200 # min gap between quote recomputes per market @@ -35,6 +35,8 @@ max_event_group_loss_usdc = 1000.0 # neg-risk group worst-case loss cap max_market_notional_usdc = 800.0 # per-market position+orders notional cap daily_loss_kill_usdc = 250.0 # realized daily loss -> halt new quotes ws_stale_halt_s = 10.0 # no book updates for this long -> halt market +user_ws_blind_halt_s = 15.0 # user WS down this long -> pull all quotes (can't see fills) +heartbeat_halt_failures = 3 # consecutive heartbeat misses -> halt + resync max_order_error_rate = 0.25 # rolling order-post error fraction -> halt [execution] diff --git a/config/markets.toml b/config/markets.toml index 5500664..04fa417 100644 --- a/config/markets.toml +++ b/config/markets.toml @@ -9,3 +9,13 @@ # profile = "political-longdated" # enabled = false # q_max_usdc = 800 # optional per-market override of the profile value + +[[markets]] +slug = "will-gavin-newsom-win-the-2028-democratic-presidential-nomination-568" +profile = "political-longdated" +enabled = true + +[[markets]] +slug = "will-jd-vance-win-the-2028-republican-presidential-nomination" +profile = "political-longdated" +enabled = true diff --git a/src/polymaker/cli.py b/src/polymaker/cli.py index 0652b86..286fa3b 100644 --- a/src/polymaker/cli.py +++ b/src/polymaker/cli.py @@ -193,6 +193,18 @@ def livetest( raise typer.Exit(0 if ok else 1) +@app.command() +def moneydoctor( + config_dir: str = typer.Option("config", help="config directory"), +) -> None: + """LIVE trading self-test: rest a limit, then market buy + sell (spends a little).""" + from polymaker.moneydoctor import run_moneydoctor + + cfg = Config.load(config_dir) + ok = asyncio.run(run_moneydoctor(cfg, console)) + raise typer.Exit(0 if ok else 1) + + @app.command(name="cancel-all") def cancel_all(config_dir: str = typer.Option("config", help="config directory")) -> None: """Cancel all open orders for the wallet (panic button).""" diff --git a/src/polymaker/config.py b/src/polymaker/config.py index 0c51136..58b5d51 100644 --- a/src/polymaker/config.py +++ b/src/polymaker/config.py @@ -26,7 +26,7 @@ class WalletConfig(BaseModel): clob_host: str = "https://clob.polymarket.com" gamma_host: str = "https://gamma-api.polymarket.com" data_api_host: str = "https://data-api.polymarket.com" - polygon_rpc: str = "https://polygon-rpc.com" + polygon_rpc: str = "https://polygon-bor-rpc.publicnode.com" class EngineConfig(BaseModel): @@ -45,6 +45,10 @@ class RiskConfig(BaseModel): max_market_notional_usdc: float = 800.0 daily_loss_kill_usdc: float = 250.0 ws_stale_halt_s: float = 10.0 + # user WS down this long -> we can't see our fills -> pull all quotes + user_ws_blind_halt_s: float = 15.0 + # consecutive heartbeat failures -> exchange is auto-cancelling us -> halt + heartbeat_halt_failures: int = 3 max_order_error_rate: float = 0.25 diff --git a/src/polymaker/engine.py b/src/polymaker/engine.py index 6b42be7..550f426 100644 --- a/src/polymaker/engine.py +++ b/src/polymaker/engine.py @@ -72,7 +72,14 @@ class Engine: self._sweep: dict[str, bool] = {} self._merging: set[str] = set() self._token_cid: dict[str, str] = {} - self._tasks: list[asyncio.Task[Any]] = [] + # supervised tasks: name -> (factory, task) so a dead task restarts + self._task_specs: dict[str, Any] = {} + self._tasks: dict[str, asyncio.Task[Any]] = {} + self._aux_tasks: list[asyncio.Task[Any]] = [] # fire-and-forget (merges) + # health / recovery signals + self._reconcile_now = asyncio.Event() + self._user_started = False # user WS task launched (live mode) + self._hb_was_down = False # ── lifecycle ─────────────────────────────────────────────────────── async def start(self) -> None: @@ -89,24 +96,49 @@ class Engine: self.gateway.creds, self.gateway.address, self.user_proc, other_token=self._other_token, condition_of_token=self._cid_of_token, journal=self.journal, proxy=self.cfg.proxy, + on_reconnect=self._on_user_reconnect, ) self.user.set_markets(list(self.metas)) - # launch tasks - self._tasks.append(asyncio.create_task(self.md.run(), name="market_ws")) + # launch supervised tasks (a dead task is restarted, never silently gone) + self._spawn("market_ws", self.md.run) if not self.paper: - self._tasks.append(asyncio.create_task(self.user.run(), name="user_ws")) - self._tasks.append(asyncio.create_task(self._heartbeat_loop(), name="heartbeat")) - self._tasks.append(asyncio.create_task(self._reconcile_loop(), name="reconcile")) + assert self.user is not None + self._spawn("user_ws", self.user.run) + self._spawn("heartbeat", self._heartbeat_loop) + self._user_started = True + self._spawn("reconcile", self._reconcile_loop) for cid in self.metas: - self._tasks.append(asyncio.create_task(self._quoter(cid), name=f"quote:{cid[:8]}")) + self._spawn(f"quote:{cid[:8]}", lambda c=cid: self._quoter(c)) + self._spawn("supervisor", self._supervise) self.risk.reset_day() log.info("engine_started", markets=len(self.metas), paper=self.paper) + def _spawn(self, name: str, factory: Any) -> None: + self._task_specs[name] = factory + self._tasks[name] = asyncio.create_task(factory(), name=name) + + _supervise_interval_s: float = 5.0 + + async def _supervise(self) -> None: + """Restart any engine task that exits while we're running. Never down.""" + while self._running: + await asyncio.sleep(self._supervise_interval_s) + for name, task in list(self._tasks.items()): + if name == "supervisor" or not task.done(): + continue + if not self._running: + return + exc = None + with contextlib.suppress(asyncio.CancelledError, asyncio.InvalidStateError): + exc = task.exception() + log.critical("task_died_restarting", task=name, err=str(exc) if exc else "exited") + self._tasks[name] = asyncio.create_task(self._task_specs[name](), name=name) + async def run_forever(self) -> None: await self.start() with contextlib.suppress(asyncio.CancelledError): - await asyncio.gather(*self._tasks) + await asyncio.gather(*self._tasks.values(), *self._aux_tasks) async def shutdown(self) -> None: self._running = False @@ -114,7 +146,7 @@ class Engine: self.md.stop() if self.user: self.user.stop() - for t in self._tasks: + for t in [*self._tasks.values(), *self._aux_tasks]: t.cancel() with contextlib.suppress(Exception): await self.gateway.cancel_all() @@ -150,6 +182,10 @@ class Engine: reward_rates: dict[str, float], ) -> MarketMeta | None: tag_id = self.catalog.cached_tag("politics") + if tag_id is None: # cold start: resolve + cache so the sweep is scoped + tag_id = await gamma.resolve_tag_id("politics") + if tag_id: + self.catalog.cache_tag("politics", tag_id) async for raw in gamma.iter_markets(tag_id=tag_id, max_pages=25): if (slug and raw.get("slug") == slug) or (condition_id and raw.get("conditionId") == condition_id): m = parse_market(raw, reward_rates) @@ -185,6 +221,16 @@ class Engine: if ev is not None: ev.set() + def _wake_all(self) -> None: + for ev in self._dirty.values(): + ev.set() + + def _on_user_reconnect(self) -> None: + """User WS reconnected: events during the gap were lost — force an + immediate REST reconcile before trusting our state again.""" + log.warning("user_ws_reconnected_forcing_reconcile") + self._reconcile_now.set() + def _on_trade(self, tp: TradePrint) -> None: cid = self._token_cid.get(tp.asset_id) if cid is None: @@ -247,10 +293,30 @@ class Engine: q_max = p.q_max_usdc inv_util = abs(pos_yes.size - pos_no.size) * fv / q_max if q_max > 0 else 0.0 hours_to_end = _hours_to_end(meta.end_date_iso, now) - ws_stale = (now - self.md.last_update_ts(meta.yes.token_id)) > self.cfg.risk.ws_stale_halt_s - rd = self.risk.evaluate(meta, ws_stale=ws_stale, + # ── blind/stale conditions: all use LOCAL receive time (skew-proof) ── + market_stale = ( + (now - self.md.last_local_ts(meta.yes.token_id)) > self.cfg.risk.ws_stale_halt_s + ) + user_blind = ( + self._user_started + and self.user is not None + and not self.user.connected + and (now - self.user.disconnected_since) > self.cfg.risk.user_ws_blind_halt_s + ) + hb_blind = ( + not self.paper + and self.cfg.engine.heartbeat + and self.gateway.heartbeat_failures >= self.cfg.risk.heartbeat_halt_failures + ) + blind = market_stale or user_blind or hb_blind + if blind: + log.warning("market_blind", cid=cid[:8], market_stale=market_stale, + user_blind=user_blind, hb_blind=hb_blind) + + rd = self.risk.evaluate(meta, ws_stale=blind, event_group_cost=self._event_group_cost(meta)) + ws_stale = blind regime = self.regime_m[cid].decide( RegimeInputs( now=now, tick=meta.tick_size, fv=fv, prev_fv=prev_fv, @@ -277,26 +343,55 @@ class Engine: return if plan.to_cancel: - await self.gateway.cancel(plan.to_cancel) - for oid in plan.to_cancel: - self.state.remove_order(oid) + ok = await self.gateway.cancel(plan.to_cancel) + if ok: + for oid in plan.to_cancel: + self.state.remove_order(oid) + else: + # cancel MAY have partially applied server-side — keep our view, + # resync from REST, and skip placing this cycle (avoid doubles) + await self._refresh_token_orders(meta, grace_s=10.0) + self._dirty[cid].set() + return if plan.to_place: placed = await self.gateway.place(plan.to_place, meta) - self.risk.note_order_result(bool(placed) or not plan.to_place) + self.risk.note_order_result(len(placed) == len(plan.to_place)) for o in placed: self.state.upsert_order(o) + if len(placed) < len(plan.to_place): + # QUARANTINE: a failed/partial batch may still have posted orders + # we don't have ids for. Cancel everything on these tokens + # (idempotent) and resync — never risk an untracked live order. + await self._quarantine(meta, reason="place_incomplete") log.info("requote", cid=cid[:8], regime=regime.value, fv=round(fv, 4), place=len(plan.to_place), cancel=len(plan.to_cancel), pos_yes=round(pos_yes.size, 1), pos_no=round(pos_no.size, 1)) self._maybe_merge(cid, meta, p, pos_yes.size, pos_no.size) + async def _quarantine(self, meta: MarketMeta, reason: str) -> None: + """Cancel all orders on a market's tokens and resync state from REST.""" + log.warning("quarantine", cid=meta.condition_id[:8], reason=reason) + for tok in (meta.yes.token_id, meta.no.token_id): + await self.gateway.cancel_asset(tok) + for o in self.state.orders_for(tok): + self.state.remove_order(o.order_id) + await self._refresh_token_orders(meta) + + async def _refresh_token_orders(self, meta: MarketMeta, grace_s: float = 0.0) -> None: + """Open-orders resync for one market's tokens (grace_s=0 = authoritative).""" + live = await self.gateway.open_orders() + for tok in (meta.yes.token_id, meta.no.token_id): + self.state.replace_open_orders( + tok, [o for o in live if o.token_id == tok], grace_s=grace_s + ) + def _maybe_merge(self, cid: str, meta: MarketMeta, p: StrategyProfile, yes_size: float, no_size: float) -> None: amount = min(yes_size, no_size) if amount < p.merge_min_size or cid in self._merging or self.paper: return self._merging.add(cid) - self._tasks.append(asyncio.create_task(self._merge_task(cid, meta, amount))) + self._aux_tasks.append(asyncio.create_task(self._merge_task(cid, meta, amount))) async def _merge_task(self, cid: str, meta: MarketMeta, amount: float) -> None: try: @@ -309,25 +404,56 @@ class Engine: async def _heartbeat_loop(self) -> None: if not self.cfg.engine.heartbeat: return + halt_after = self.cfg.risk.heartbeat_halt_failures while self._running: - await self.gateway.heartbeat() + ok = await self.gateway.heartbeat() + if not ok and self.gateway.heartbeat_failures >= halt_after and not self._hb_was_down: + # exchange is (or soon will be) auto-cancelling everything we + # have live; recompute will see hb_blind and pull quotes + self._hb_was_down = True + log.critical("heartbeat_down_halting", failures=self.gateway.heartbeat_failures) + self._wake_all() + elif ok and self._hb_was_down: + # recovered: our server-side orders were wiped — drop local + # order state, resync authoritatively, then resume quoting + self._hb_was_down = False + log.warning("heartbeat_recovered_resyncing") + self.state.clear_orders() + for meta in self.metas.values(): + with contextlib.suppress(Exception): + await self._refresh_token_orders(meta, grace_s=0.0) + self._wake_all() await asyncio.sleep(self.cfg.engine.heartbeat_interval_s) async def _reconcile_loop(self) -> None: while self._running: - await asyncio.sleep(self.cfg.engine.reconcile_interval_s) + # periodic cadence, but wake immediately when a reconnect/recovery + # demands an urgent resync + with contextlib.suppress(asyncio.TimeoutError): + await asyncio.wait_for( + self._reconcile_now.wait(), + timeout=self.cfg.engine.reconcile_interval_s, + ) + forced = self._reconcile_now.is_set() + self._reconcile_now.clear() try: positions = await self.gateway.positions() if positions: self.state.reconcile_positions(positions) live = await self.gateway.open_orders() - if live or not self.paper: - by_token: dict[str, list[Any]] = {} - for o in live: - by_token.setdefault(o.token_id, []).append(o) - for tok, orders in by_token.items(): - if self.state.inflight(tok) == 0: - self.state.replace_open_orders(tok, orders) + by_token: dict[str, list[Any]] = {} + for o in live: + by_token.setdefault(o.token_id, []).append(o) + # iterate ALL our tokens, not just those present in the REST + # response — a token whose orders all vanished server-side must + # be cleaned up too (grace window protects fresh placements) + for tok in self._token_cid: + if self.state.inflight(tok) == 0: + self.state.replace_open_orders(tok, by_token.get(tok, [])) + if forced: + log.info("forced_reconcile_done", positions=len(positions), + open_orders=len(live)) + self._wake_all() except Exception as exc: # noqa: BLE001 log.warning("reconcile_error", err=str(exc)) diff --git a/src/polymaker/execution/gateway.py b/src/polymaker/execution/gateway.py index debdb47..711dae3 100644 --- a/src/polymaker/execution/gateway.py +++ b/src/polymaker/execution/gateway.py @@ -53,6 +53,8 @@ class ExecutionGateway: self._order_bucket = TokenBucket(rate_per_s=200.0 * f, burst=500.0 * f) self._cancel_bucket = TokenBucket(rate_per_s=200.0 * f, burst=500.0 * f) self._paper_ids = itertools.count(1) + self._hb_id: str = "" # heartbeat chain + self._hb_failures: int = 0 @property def paper(self) -> bool: @@ -159,9 +161,11 @@ class ExecutionGateway: return out # ── cancellation ──────────────────────────────────────────────────── - async def cancel(self, order_ids: list[str]) -> None: + async def cancel(self, order_ids: list[str]) -> bool: + """Cancel by id. Returns True on success — callers must NOT drop the + orders from local state on failure (they may still be live).""" if not order_ids or self._paper: - return + return True await self._cancel_bucket.acquire(1) def _cancel() -> None: @@ -169,19 +173,27 @@ class ExecutionGateway: try: await asyncio.to_thread(_cancel) + return True except Exception as exc: # noqa: BLE001 log.error("cancel_failed", err=str(exc), n=len(order_ids)) + return False - async def cancel_asset(self, asset_id: str) -> None: + async def cancel_asset(self, asset_id: str) -> bool: + """Cancel every order on one token (idempotent quarantine primitive).""" if self._paper: - return + return True def _cancel() -> None: from py_clob_client_v2.clob_types import OrderMarketCancelParams self._client.cancel_market_orders(OrderMarketCancelParams(asset_id=asset_id)) - await asyncio.to_thread(_cancel) + try: + await asyncio.to_thread(_cancel) + return True + except Exception as exc: # noqa: BLE001 + log.error("cancel_asset_failed", err=str(exc), token=asset_id[:12]) + return False async def cancel_all(self) -> None: if self._paper or self._client is None: @@ -189,14 +201,145 @@ class ExecutionGateway: await asyncio.to_thread(self._client.cancel_all) log.info("cancel_all_sent") - # ── heartbeat (dead-man switch) ───────────────────────────────────── - async def heartbeat(self, hb_id: str = "") -> None: + # ── market (taker) orders — used by moneydoctor, NOT the maker strategy ── + async def market_order( + self, token_id: str, side: Side, amount: float, meta: MarketMeta, + *, fak: bool = True, + ) -> dict[str, Any]: + """Place a marketable order. amount = USD for BUY, shares for SELL. + + This is a TAKER order (crosses the spread) — only the moneydoctor live + self-test uses it; the maker strategy never does. + """ if self._paper or self._client is None: - return + return {"paper": True} + + def _do() -> dict[str, Any]: + from py_clob_client_v2.clob_types import ( + MarketOrderArgsV2, + OrderType, + PartialCreateOrderOptions, + ) + + ot = OrderType.FAK if fak else OrderType.FOK + args = MarketOrderArgsV2(token_id=token_id, amount=amount, + side=side.value, order_type=ot) + opts = PartialCreateOrderOptions(tick_size=_tick_str(meta.tick_size), + neg_risk=meta.neg_risk) + try: + resp = self._client.create_and_post_market_order(args, opts, order_type=ot) + return resp if isinstance(resp, dict) else {"resp": resp} + except Exception as exc: # noqa: BLE001 - surface as data, never crash the caller + return {"status": "failed", "error": str(exc)} + + return await asyncio.to_thread(_do) + + async def get_book(self, token_id: str) -> dict[str, float]: + """Live best bid/ask + touch depth for one token (public REST).""" try: - await asyncio.to_thread(self._client.post_heartbeat, hb_id) + async with httpx.AsyncClient(timeout=15.0) as c: + r = await c.get(f"{self._cfg.wallet.clob_host}/book", + params={"token_id": token_id}) + r.raise_for_status() + b = r.json() + bids = [(float(x["price"]), float(x["size"])) for x in b.get("bids", [])] + asks = [(float(x["price"]), float(x["size"])) for x in b.get("asks", [])] + best_bid = max(bids)[0] if bids else 0.0 + best_ask = min(asks)[0] if asks else 1.0 + ask_depth = sum(s for p, s in asks if p <= best_ask + 1e-9) + bid_depth = sum(s for p, s in bids if p >= best_bid - 1e-9) + return {"best_bid": best_bid, "best_ask": best_ask, + "ask_depth": ask_depth, "bid_depth": bid_depth} + except (httpx.HTTPError, KeyError, ValueError) as exc: + log.warning("get_book_failed", err=str(exc)) + return {} + + async def token_balance(self, token_id: str) -> float: + """Exact on-chain conditional-token balance (shares) held by the funder. + + Returns None on total RPC failure so callers can distinguish "0 shares" + from "couldn't read". + """ + bal = await self._token_balance_opt(token_id) + return bal if bal is not None else 0.0 + + async def _token_balance_opt(self, token_id: str) -> float | None: + def _read() -> float | None: + from web3 import Web3 + from web3.middleware import ExtraDataToPOAMiddleware + + configured = self._cfg.secrets.polygon_rpc or self._cfg.wallet.polygon_rpc + rpcs = [configured, "https://polygon-bor-rpc.publicnode.com", + "https://polygon.llamarpc.com", "https://rpc.ankr.com/polygon"] + abi = [{"name": "balanceOf", "type": "function", "stateMutability": "view", + "inputs": [{"name": "a", "type": "address"}, {"name": "id", "type": "uint256"}], + "outputs": [{"name": "", "type": "uint256"}]}] + for rpc in dict.fromkeys(rpcs): # dedupe, keep order + try: + w3 = Web3(Web3.HTTPProvider(rpc, request_kwargs={"timeout": 15})) + w3.middleware_onion.inject(ExtraDataToPOAMiddleware, layer=0) + ctf = w3.eth.contract( + address=Web3.to_checksum_address("0x4D97DCd97eC945f40cF65F87097ACe5EA0476045"), + abi=abi, + ) + raw = ctf.functions.balanceOf( + Web3.to_checksum_address(self.funder), int(token_id) + ).call() + return float(raw) / 1e6 + except Exception: # noqa: BLE001, PERF203 - try next RPC + continue + return None + + try: + return await asyncio.to_thread(_read) except Exception as exc: # noqa: BLE001 - log.warning("heartbeat_failed", err=str(exc)) + log.warning("token_balance_failed", err=str(exc)) + return None + + async def collateral_balance(self) -> float: + """pUSD balance (float) on the funder.""" + ba = await self.balance_allowance() + for k in ("balance", "collateral", "amount"): + if isinstance(ba, dict) and k in ba: + try: + v = float(ba[k]) + return v / 1e6 if v > 1e6 else v + except (ValueError, TypeError): + return 0.0 + return 0.0 + + # ── heartbeat (dead-man switch) ───────────────────────────────────── + async def heartbeat(self) -> bool: + """Send one chained heartbeat. Returns True on success. + + The exchange expects each heartbeat to carry the previous heartbeat_id. + Consecutive failures are tracked in `heartbeat_failures`: after enough + misses the exchange auto-cancels ALL our orders, so the engine must + stop quoting and resync once the heartbeat recovers. + """ + if self._paper or self._client is None: + return True + + def _beat() -> Any: + return self._client.post_heartbeat(self._hb_id) + + try: + resp = await asyncio.to_thread(_beat) + new_id = _first(resp, "heartbeat_id", "heartbeatId", "id") + self._hb_id = str(new_id) if new_id else "" + if self._hb_failures: + log.info("heartbeat_recovered", after_failures=self._hb_failures) + self._hb_failures = 0 + return True + except Exception as exc: # noqa: BLE001 + self._hb_failures += 1 + self._hb_id = "" # broken chain — restart it + log.warning("heartbeat_failed", err=str(exc), consecutive=self._hb_failures) + return False + + @property + def heartbeat_failures(self) -> int: + return self._hb_failures # ── reads ─────────────────────────────────────────────────────────── async def open_orders(self) -> list[OpenOrder]: diff --git a/src/polymaker/marketdata/orderbook.py b/src/polymaker/marketdata/orderbook.py index 474dbde..9f2933d 100644 --- a/src/polymaker/marketdata/orderbook.py +++ b/src/polymaker/marketdata/orderbook.py @@ -10,6 +10,7 @@ apply_* mutators. Nothing here does I/O; the WS layer drives it. from __future__ import annotations +import time from dataclasses import dataclass from sortedcontainers import SortedDict @@ -58,14 +59,15 @@ class BookView: class OrderBook: """YES-canonical L2 book for one market.""" - __slots__ = ("bids", "asks", "tick_size", "last_update_ts", "book_hash") + __slots__ = ("bids", "asks", "tick_size", "last_update_ts", "local_ts", "book_hash") def __init__(self, tick_size: float = 0.001) -> None: # price -> size. bids and asks both ascending in price. self.bids: SortedDict[float, float] = SortedDict() self.asks: SortedDict[float, float] = SortedDict() self.tick_size = tick_size - self.last_update_ts: float = 0.0 + self.last_update_ts: float = 0.0 # exchange timestamp (informational) + self.local_ts: float = 0.0 # local receive time — used for staleness (skew-proof) self.book_hash: str | None = None # ── mutation ──────────────────────────────────────────────────────── @@ -79,6 +81,7 @@ class OrderBook: self.bids = SortedDict({p: s for p, s in bids if s > 0}) self.asks = SortedDict({p: s for p, s in asks if s > 0}) self.last_update_ts = ts + self.local_ts = time.time() self.book_hash = book_hash def apply_delta(self, side: Side, price: float, size: float, ts: float) -> None: @@ -88,6 +91,7 @@ class OrderBook: else: book[price] = size self.last_update_ts = ts + self.local_ts = time.time() def set_tick_size(self, tick_size: float) -> None: self.tick_size = tick_size diff --git a/src/polymaker/marketdata/service.py b/src/polymaker/marketdata/service.py index 74da881..f69b086 100644 --- a/src/polymaker/marketdata/service.py +++ b/src/polymaker/marketdata/service.py @@ -55,6 +55,7 @@ class MarketDataService: self._subs: list[str] = [] self._ws: Any = None self._stop = asyncio.Event() + self.connected: bool = False # ── subscription management ───────────────────────────────────────── def set_markets(self, markets: list[tuple[str, list[str]]]) -> None: @@ -78,6 +79,11 @@ class MarketDataService: b = self.books.get(token_id) return b.last_update_ts if b else 0.0 + def last_local_ts(self, token_id: str) -> float: + """Local receive time of the last book mutation (skew-proof staleness).""" + b = self.books.get(token_id) + return b.local_ts if b else 0.0 + # ── run loop ──────────────────────────────────────────────────────── async def run(self) -> None: backoff = 1.0 @@ -98,15 +104,22 @@ class MarketDataService: if not self._subs: await asyncio.sleep(1.0) return - kwargs: dict[str, Any] = {"ping_interval": 5, "ping_timeout": None} + # ping_timeout matters: with None a half-dead TCP connection hangs + # forever. The server answers protocol pings (verified live), so a + # missing pong within 10s means the link is dead -> reconnect. + kwargs: dict[str, Any] = {"ping_interval": 5, "ping_timeout": 10, "open_timeout": 10} if self._proxy: kwargs["proxy"] = self._proxy async with websockets.connect(self._url, **kwargs) as ws: self._ws = ws await ws.send(json.dumps({"assets_ids": self._subs, "type": "market"})) + self.connected = True log.info("market_ws_subscribed", n=len(self._subs)) - async for raw in ws: - self._handle(raw) + try: + async for raw in ws: + self._handle(raw) + finally: + self.connected = False def stop(self) -> None: self._stop.set() diff --git a/src/polymaker/moneydoctor.py b/src/polymaker/moneydoctor.py new file mode 100644 index 0000000..52bbf11 --- /dev/null +++ b/src/polymaker/moneydoctor.py @@ -0,0 +1,244 @@ +"""`polymaker moneydoctor` — a LIVE trading self-test that actually moves money. + +Unlike `doctor` (read-only preflight) and `livetest` (a deep post-only order +that can't fill), this exercises the full order machinery for real: + + 1. LIMIT — place a post-only limit that rests on the book, confirm it appears, + then cancel it. (Free.) + 2. BUY — a market (taker) order that fills immediately; confirm shares land + on-chain. + 3. SELL — market-sell those shares back; confirm the position goes flat. + +It crosses the spread twice and pays taker fees, so it costs a small amount — +reported at the end as the round-trip cost. Sized near the minimum order size. +Taker orders are used ONLY here; the maker strategy never crosses the spread. +""" + +from __future__ import annotations + +import asyncio +import contextlib +import json +import math +from collections.abc import Callable +from typing import TYPE_CHECKING + +import websockets +from rich.console import Console + +from polymaker.config import Config +from polymaker.domain import MarketMeta, Quote, Side +from polymaker.strategy.quoting import round_to_tick + +if TYPE_CHECKING: + from polymaker.execution.gateway import ExecutionGateway + +USER_WS = "wss://ws-subscriptions-clob.polymarket.com/ws/user" + + +async def run_moneydoctor(cfg: Config, console: Console, notional_usdc: float | None = None) -> bool: + from polymaker.execution.gateway import ExecutionGateway + + ok = True + + def check(label: str, passed: bool, detail: str = "") -> None: + nonlocal ok + mark = "[green]✓[/green]" if passed else "[red]✗[/red]" + console.print(f" {mark} {label}" + (f" [dim]{detail}[/dim]" if detail else "")) + ok = ok and passed + + if not cfg.secrets.has_wallet: + console.print("[red]No wallet in .env.[/red]") + return False + + console.print("[bold]polymaker moneydoctor[/bold] [dim](spends a little real money)[/dim]") + gw = ExecutionGateway(cfg) + await gw.connect() + if cfg.proxy: + console.print(f" [dim]· routing via proxy {cfg.proxy.split('@')[-1]}[/dim]") + + meta, book = await _pick_market(cfg, gw) + if meta is None: + console.print("[yellow]No suitable liquid market found — run `polymaker scan` first.[/yellow]") + return False + token = meta.yes.token_id + tick, dec = meta.tick_size, meta.price_decimals + best_bid, best_ask = book["best_bid"], book["best_ask"] + console.print(f" market: [bold]{meta.question[:56]}[/bold]") + console.print(f" [dim]YES token · bid {best_bid} / ask {best_ask} · " + f"spread {round(best_ask - best_bid, 4)} · tick {tick:g}[/dim]") + + bal0 = await gw.collateral_balance() + console.print(f" [dim]starting balance: {bal0:.4f} pUSD[/dim]\n") + + # ── 1. LIMIT: rest + cancel ───────────────────────────────────────── + limit_price = round_to_tick(best_bid - 2 * tick, tick, dec, up=False) + limit_size = max(meta.min_order_size, 5.0) + placed = await gw.place([Quote(token, Side.BUY, limit_price, limit_size)], meta) + if placed: + await asyncio.sleep(1.5) + live = await gw.open_orders() + found = any(o.order_id == placed[0].order_id for o in live) + check("limit order rests on book", found, f"{limit_size:g} @ {limit_price}, {len(live)} live") + await gw.cancel([placed[0].order_id]) + await asyncio.sleep(1.0) + gone = not any(o.order_id == placed[0].order_id for o in await gw.open_orders()) + check("limit order cancels", gone) + else: + check("limit order placed", False, "post failed — see logs") + + # ── 2. MARKET BUY ─────────────────────────────────────────────────── + shares_target = meta.min_order_size + 3.0 + buy_usd = round(shares_target * best_ask * 1.06, 2) + before = await gw._token_balance_opt(token) or 0.0 # baseline shares + console.print(f"\n [dim]market BUY ~${buy_usd} of YES (targeting ~{shares_target:g} shares)…[/dim]") + resp_buy = await gw.market_order(token, Side.BUY, buy_usd, meta, fak=True) + bought, spent, status = _fill(resp_buy, Side.BUY) + check("market BUY matched", status == "matched" and bought > 0, + f"got {bought:.2f} shares for ${spent:.2f} [{status}]") + + # ── settle: user WS (fast) with on-chain as source of truth ───────── + if bought > 0: + console.print(" [dim]waiting for settlement (user WS + chain)…[/dim]") + settled = await _wait_settled(cfg, gw, token, before, timeout=60) + got = settled - before + check("buy settled on-chain", got > 0.5, f"{got:.2f} shares now available") + + # ── 3. MARKET SELL — retry until the exchange accepts it ──────── + sell_amt = math.floor(max(got, bought) * 100) / 100 + sold = await _sell_with_retry(gw, token, sell_amt, meta, before, console, check) + + await asyncio.sleep(3.0) + remaining = await gw._token_balance_opt(token) + if remaining is not None: + check("position flat after round-trip", remaining <= before + 0.5, + f"{remaining - before:.2f} net shares vs. start ({sold:.2f} sold)") + else: + console.print(" [yellow]! buy did not fill — nothing to sell.[/yellow]") + + # ── cost ──────────────────────────────────────────────────────────── + await asyncio.sleep(2.0) + bal1 = await gw.collateral_balance() + cost = bal0 - bal1 + console.print(f"\n [bold]round-trip cost: {cost:.4f} pUSD[/bold] " + f"[dim](spread + taker fees; balance {bal0:.2f} → {bal1:.2f})[/dim]") + console.print(f"\n[bold]{'ALL GOOD' if ok else 'CHECK LOGS'}[/bold]") + return ok + + +async def _pick_market(cfg: Config, gw: ExecutionGateway) -> tuple[MarketMeta | None, dict[str, float]]: + """Pick a liquid, mid-priced market with enough touch depth for a tiny order.""" + from polymaker.catalog.store import CatalogStore + + store = CatalogStore(cfg.paths.db) + rows = store.top(40) + store.close() + for meta, _sc in rows: + mid = (meta.best_bid + meta.best_ask) / 2 if (meta.best_bid and meta.best_ask) else 0.0 + if not (0.2 < mid < 0.6): + continue + book = await gw.get_book(meta.yes.token_id) + if not book or book["best_bid"] <= 0 or book["best_ask"] >= 1: + continue + need_shares = meta.min_order_size + 4 + if book["ask_depth"] >= need_shares and book["bid_depth"] >= need_shares: + spread = book["best_ask"] - book["best_bid"] + if spread <= 0.02: # keep the round-trip cost small + return meta, book + return None, {} + + +async def _wait_settled(cfg: Config, gw: ExecutionGateway, token: str, baseline: float, + *, timeout: float = 60.0) -> float: + """Return the settled on-chain share balance once the buy lands. + + Races two signals: the user WS `trade` status ladder (fast, push-based) and + an on-chain balance poll (slower, but the source of truth the exchange checks + when validating a sell). Returns the latest on-chain balance. + """ + done = asyncio.Event() + + async def chain_poll() -> None: + while not done.is_set(): + await asyncio.sleep(3.0) + bal = await gw._token_balance_opt(token) + if bal is not None and bal > baseline + 0.01: + done.set() + return + + async def ws_watch() -> None: + kw: dict[str, object] = {"ping_interval": 5, "ping_timeout": None, "open_timeout": 10} + if cfg.proxy: + kw["proxy"] = cfg.proxy + creds = gw.creds + with contextlib.suppress(Exception): + async with websockets.connect(USER_WS, **kw) as ws: # type: ignore[arg-type] + await ws.send(json.dumps({ + "type": "user", + "auth": {"apiKey": creds.api_key, "secret": creds.api_secret, + "passphrase": creds.api_passphrase}, + "markets": [], + })) + while not done.is_set(): + raw = await asyncio.wait_for(ws.recv(), timeout=5) + data = json.loads(raw) + for m in data if isinstance(data, list) else [data]: + if (isinstance(m, dict) and m.get("event_type") == "trade" + and str(m.get("asset_id")) == token + and str(m.get("status", "")).upper() in ("MINED", "CONFIRMED")): + done.set() + return + + tasks = [asyncio.create_task(chain_poll()), asyncio.create_task(ws_watch())] + with contextlib.suppress(asyncio.TimeoutError): + await asyncio.wait_for(done.wait(), timeout=timeout) + done.set() + for t in tasks: + t.cancel() + await asyncio.gather(*tasks, return_exceptions=True) + return await gw._token_balance_opt(token) or baseline + + +async def _sell_with_retry( + gw: ExecutionGateway, token: str, amount: float, meta: MarketMeta, baseline: float, + console: Console, check: Callable[..., None], attempts: int = 5, +) -> float: + """Market-sell `amount`, retrying until the exchange accepts (balance settles).""" + for i in range(attempts): + console.print(f" [dim]market SELL {amount:g} shares (attempt {i + 1})…[/dim]") + resp = await gw.market_order(token, Side.SELL, amount, meta, fak=True) + sold, recv, status = _fill(resp, Side.SELL) + if status == "matched" and sold > 0: + check("market SELL filled", True, f"sold {sold:.2f} shares for ${recv:.2f}") + return sold + err = resp.get("error", "") if isinstance(resp, dict) else "" + console.print(f" [dim] not filled yet ({status} {str(err)[:48]}); waiting to retry…[/dim]") + await asyncio.sleep(6.0) + bal = await gw._token_balance_opt(token) + if bal is not None: + amount = math.floor(max(0.0, bal - 0.0) * 100) / 100 # sell what's actually available + if amount < meta.min_order_size: + break + check("market SELL filled", False, "could not fill — flatten manually with cancel-all/limit") + return 0.0 + + +def _fill(resp: object, side: Side) -> tuple[float, float, str]: + """Parse a market-order response -> (shares_filled, usd, status). + + makingAmount = what we give, takingAmount = what we get. So for a BUY, + shares = takingAmount and usd = makingAmount; for a SELL it's the reverse. + """ + if not isinstance(resp, dict): + return 0.0, 0.0, "?" + status = str(resp.get("status", "")) + making = _f(resp.get("makingAmount")) + taking = _f(resp.get("takingAmount")) + return (taking, making, status) if side is Side.BUY else (making, taking, status) + + +def _f(x: object) -> float: + try: + return float(x) # type: ignore[arg-type] + except (ValueError, TypeError): + return 0.0 diff --git a/src/polymaker/state/store.py b/src/polymaker/state/store.py index f8de001..c8fafb4 100644 --- a/src/polymaker/state/store.py +++ b/src/polymaker/state/store.py @@ -65,8 +65,26 @@ class StateStore: def position(self, token_id: str) -> Position: return self.positions.get(token_id, Position(token_id)) - def apply_fill(self, fill: Fill) -> None: - """Apply a fill optimistically to inventory + avg price.""" + def apply_fill(self, fill: Fill) -> bool: + """Apply a fill optimistically to inventory + avg price. + + IDEMPOTENT: the SQLite fills table is the dedupe gate (trade_id is the + primary key). A replayed fill — WS redelivery after reconnect, a MATCHED + arriving again after CONFIRMED, or a replay across process restarts — + is detected by INSERT OR IGNORE and NOT applied twice. Returns False + for duplicates so callers can skip their side effects too. + """ + cur = self._conn.execute( + "INSERT OR IGNORE INTO fills(trade_id,token_id,side,price,size,is_maker,ts) VALUES(?,?,?,?,?,?,?)", + (fill.trade_id, fill.token_id, fill.side.value, fill.price, fill.size, + int(fill.is_maker), fill.ts), + ) + self._conn.commit() + if cur.rowcount == 0: + log.warning("duplicate_fill_ignored", trade_id=fill.trade_id, + token=fill.token_id[:12], side=fill.side.value, size=fill.size) + return False + pos = self.positions.setdefault(fill.token_id, Position(fill.token_id)) signed = fill.size if fill.side is Side.BUY else -fill.size new_size = pos.size + signed @@ -83,9 +101,9 @@ class StateStore: pos.avg_price = 0.0 self._last_fill_ts[fill.token_id] = fill.ts self._persist_position(pos) - self._record_fill(fill) log.info("fill", token=fill.token_id[:12], side=fill.side.value, price=fill.price, size=fill.size, pos=round(pos.size, 2)) + return True def set_position(self, token_id: str, size: float, avg_price: float) -> None: pos = Position(token_id, max(0.0, size), avg_price if size > 0 else 0.0) @@ -128,13 +146,33 @@ class StateStore: def remove_order(self, order_id: str) -> None: self.orders.pop(order_id, None) - def replace_open_orders(self, token_id: str, live: list[OpenOrder]) -> None: - """Replace our view of a token's open orders from a REST snapshot.""" - for oid in [o.order_id for o in self.orders.values() if o.token_id == token_id]: - self.orders.pop(oid, None) + def replace_open_orders( + self, token_id: str, live: list[OpenOrder], *, grace_s: float = 10.0 + ) -> None: + """Replace our view of a token's open orders from a REST snapshot. + + DOUBLE-ORDER GUARD: a REST snapshot can lag a placement by seconds. If we + dropped a just-placed order because the snapshot didn't include it yet, + the reconciler would immediately re-place it -> duplicate live orders. + So local orders younger than `grace_s` survive even when absent from the + snapshot (pass grace_s=0 to force an authoritative wipe, e.g. after the + exchange auto-cancelled everything on a heartbeat gap). + """ + now = time.time() + live_ids = {o.order_id for o in live} + for o in [o for o in self.orders.values() if o.token_id == token_id]: + if o.order_id in live_ids: + continue + if now - o.created_ts < grace_s: + continue # too young to trust its absence from the snapshot + self.orders.pop(o.order_id, None) for o in live: self.orders[o.order_id] = o + def clear_orders(self) -> None: + """Forget all local open orders (e.g. after a confirmed server-side wipe).""" + self.orders.clear() + # ── persistence ───────────────────────────────────────────────────── def _persist_position(self, pos: Position) -> None: self._conn.execute( @@ -143,13 +181,6 @@ class StateStore: ) self._conn.commit() - def _record_fill(self, f: Fill) -> None: - self._conn.execute( - "INSERT OR IGNORE INTO fills(trade_id,token_id,side,price,size,is_maker,ts) VALUES(?,?,?,?,?,?,?)", - (f.trade_id, f.token_id, f.side.value, f.price, f.size, int(f.is_maker), f.ts), - ) - self._conn.commit() - def _persist_order(self, o: OpenOrder) -> None: self._conn.execute( "INSERT OR REPLACE INTO order_log(order_id,token_id,side,price,size,state,ts) VALUES(?,?,?,?,?,?,?)", diff --git a/src/polymaker/state/tracker.py b/src/polymaker/state/tracker.py index 6694bed..3b181bd 100644 --- a/src/polymaker/state/tracker.py +++ b/src/polymaker/state/tracker.py @@ -62,9 +62,12 @@ class UserEventProcessor: def on_trade(self, ev: TradeEvent, condition_id: str) -> None: if ev.status is TradeState.MATCHED: if ev.trade_id in self._applied: - return # idempotent: already counted this match + return # idempotent: already counted this match (in-memory fast path) fill = Fill(ev.token_id, ev.our_side, ev.price, ev.size, ev.trade_id, ev.ts, is_maker=True) - self._store.apply_fill(fill) + if not self._store.apply_fill(fill): + # duplicate at the persistent layer (replay after CONFIRMED or + # across restarts) — apply NO side effects + return self._store.mark_inflight(ev.token_id) self._applied[ev.trade_id] = fill self._on_fill(fill) @@ -77,10 +80,15 @@ class UserEventProcessor: self._applied.pop(ev.trade_id, None) self._on_change(condition_id) - elif ev.status in (TradeState.FAILED, TradeState.RETRYING): + elif ev.status is TradeState.RETRYING: + # tx being retried on-chain — it may still succeed. Keep the + # optimistic fill and the inflight guard; only FAILED is terminal. + log.warning("trade_retrying", trade_id=ev.trade_id, token=ev.token_id[:12]) + + elif ev.status is TradeState.FAILED: prior = self._applied.pop(ev.trade_id, None) if prior is not None: - # reverse the optimistic fill + # reverse the optimistic fill (idempotent via the :reverse id) self._store.apply_fill( Fill(prior.token_id, prior.side.opposite, prior.price, prior.size, f"{prior.trade_id}:reverse", prior.ts, is_maker=True) diff --git a/src/polymaker/userstream/client.py b/src/polymaker/userstream/client.py index adbd49a..3ed83d2 100644 --- a/src/polymaker/userstream/client.py +++ b/src/polymaker/userstream/client.py @@ -8,6 +8,7 @@ from __future__ import annotations import asyncio import json +import time from collections.abc import Callable from typing import Any @@ -33,6 +34,7 @@ class UserStream: url: str = "wss://ws-subscriptions-clob.polymarket.com/ws/user", journal: Journal | None = None, proxy: str | None = None, + on_reconnect: Callable[[], None] | None = None, ) -> None: self._creds = creds self._address = our_address @@ -42,8 +44,14 @@ class UserStream: self._url = url self._journal = journal self._proxy = proxy + self._on_reconnect = on_reconnect or (lambda: None) self._markets: list[str] = [] self._stop = asyncio.Event() + # Connection health. ping_timeout guarantees a dead link flips + # `connected` to False within ~15s, so the engine can go blind-safe. + self.connected: bool = False + self.disconnected_since: float = time.time() + self._ever_connected = False def set_markets(self, condition_ids: list[str]) -> None: self._markets = condition_ids @@ -73,14 +81,25 @@ class UserStream: }, "markets": self._markets, } - kwargs: dict[str, Any] = {"ping_interval": 5, "ping_timeout": None} + kwargs: dict[str, Any] = {"ping_interval": 5, "ping_timeout": 10, "open_timeout": 10} if self._proxy: kwargs["proxy"] = self._proxy async with websockets.connect(self._url, **kwargs) as ws: await ws.send(json.dumps(sub)) - log.info("user_ws_subscribed", markets=len(self._markets)) - async for raw in ws: - self._handle(raw) + self.connected = True + is_reconnect = self._ever_connected + self._ever_connected = True + log.info("user_ws_subscribed", markets=len(self._markets), reconnect=is_reconnect) + if is_reconnect: + # events during the gap are LOST (no replay) — the engine must + # force a REST reconcile to recover any missed fills/cancels + self._on_reconnect() + try: + async for raw in ws: + self._handle(raw) + finally: + self.connected = False + self.disconnected_since = time.time() def stop(self) -> None: self._stop.set() diff --git a/tests/test_execution.py b/tests/test_execution.py index 7ef707a..7aaf245 100644 --- a/tests/test_execution.py +++ b/tests/test_execution.py @@ -55,7 +55,8 @@ async def test_paper_gateway_places_and_cancels_without_wallet(meta): async def test_paper_gateway_heartbeat_and_cancel_all_noop(): gw = ExecutionGateway(Config(), paper=True) - await gw.heartbeat("hb1") + assert await gw.heartbeat() is True # paper: healthy no-op + assert gw.heartbeat_failures == 0 await gw.cancel_all() # no client, must not raise diff --git a/tests/test_hardening.py b/tests/test_hardening.py new file mode 100644 index 0000000..ee034a2 --- /dev/null +++ b/tests/test_hardening.py @@ -0,0 +1,257 @@ +"""Hardening tests: the nasty failure modes that cause double-buys/sells, +phantom orders, and flying-blind quoting. Every scenario here maps to a real +Polymarket API failure mode (WS replay, snapshot lag, heartbeat gaps, timeouts). +""" + +from __future__ import annotations + +import asyncio +import time + +from polymaker.domain import ( + Fill, + OpenOrder, + OrderState, + Side, + TradeState, +) +from polymaker.state.store import StateStore +from polymaker.state.tracker import TradeEvent, UserEventProcessor + +# ══════════════════════ double-fill protection ══════════════════════════ + + +def test_replayed_matched_after_confirmed_not_double_applied(tmp_path): + """WS reconnects can replay events. MATCHED -> CONFIRMED -> MATCHED(replay) + must not double the position, even though the in-memory dedupe was cleared + at CONFIRMED.""" + s = StateStore(tmp_path / "s.db") + p = UserEventProcessor(s) + ev = TradeEvent("tok", Side.BUY, 0.5, 100, "t1", TradeState.MATCHED, 1.0) + p.on_trade(ev, "cid") + p.on_trade(TradeEvent("tok", Side.BUY, 0.5, 100, "t1", TradeState.CONFIRMED, 2.0), "cid") + assert s.position("tok").size == 100 + p.on_trade(ev, "cid") # replayed MATCHED after confirm + assert s.position("tok").size == 100 # NOT 200 + assert s.inflight("tok") == 0 # replay must not re-mark inflight + s.close() + + +def test_duplicate_fill_across_restart(tmp_path): + """Process restarts + WS replays the same trade: the SQLite fills table is + the dedupe gate, so the position is not double-applied.""" + db = tmp_path / "s.db" + s1 = StateStore(db) + assert s1.apply_fill(Fill("tok", Side.BUY, 0.5, 100, "t1")) is True + s1.close() + s2 = StateStore(db) + assert s2.apply_fill(Fill("tok", Side.BUY, 0.5, 100, "t1")) is False # duplicate + assert s2.position("tok").size == 100 + s2.close() + + +def test_duplicate_fill_side_effects_skipped(tmp_path): + """A duplicate MATCHED must not fire on_fill/on_change callbacks.""" + s = StateStore(tmp_path / "s.db") + fills, changes = [], [] + p = UserEventProcessor(s, on_change=changes.append, on_fill=fills.append) + ev = TradeEvent("tok", Side.BUY, 0.5, 50, "t1", TradeState.MATCHED, 1.0) + p.on_trade(ev, "cid") + p.on_trade(TradeEvent("tok", Side.BUY, 0.5, 50, "t1", TradeState.CONFIRMED, 2.0), "cid") + n_fills, n_changes = len(fills), len(changes) + p.on_trade(ev, "cid") # replay + assert len(fills) == n_fills # no new fill callback + assert len(changes) == n_changes + s.close() + + +def test_retrying_keeps_fill_failed_reverses_once(tmp_path): + """RETRYING is not terminal (tx may still land) -> keep the fill. + FAILED reverses exactly once, even if FAILED is replayed.""" + s = StateStore(tmp_path / "s.db") + p = UserEventProcessor(s) + p.on_trade(TradeEvent("tok", Side.BUY, 0.5, 100, "t1", TradeState.MATCHED, 1.0), "cid") + p.on_trade(TradeEvent("tok", Side.BUY, 0.5, 100, "t1", TradeState.RETRYING, 2.0), "cid") + assert s.position("tok").size == 100 # retrying: unchanged + assert s.inflight("tok") == 1 + p.on_trade(TradeEvent("tok", Side.BUY, 0.5, 100, "t1", TradeState.FAILED, 3.0), "cid") + assert s.position("tok").size == 0 # reversed + p.on_trade(TradeEvent("tok", Side.BUY, 0.5, 100, "t1", TradeState.FAILED, 4.0), "cid") + assert s.position("tok").size == 0 # replayed FAILED: no double reverse + s.close() + + +# ══════════════════════ double-order protection ═════════════════════════ + + +def _order(oid: str, tok: str = "tok", age_s: float = 60.0) -> OpenOrder: + o = OpenOrder(oid, tok, Side.BUY, 0.49, 100, OrderState.LIVE) + o.created_ts = time.time() - age_s + return o + + +def test_grace_window_protects_fresh_orders(tmp_path): + """A REST snapshot that lags a just-placed order must NOT evict it from + state (that eviction is what caused re-placement -> double orders).""" + s = StateStore(tmp_path / "s.db") + fresh = _order("young", age_s=2.0) + stale = _order("old", age_s=60.0) + s.upsert_order(fresh) + s.upsert_order(stale) + # snapshot doesn't include either (lag for young; old was really cancelled) + s.replace_open_orders("tok", [], grace_s=10.0) + ids = {o.order_id for o in s.orders_for("tok")} + assert "young" in ids # protected by grace + assert "old" not in ids # correctly dropped + s.close() + + +def test_grace_zero_is_authoritative_wipe(tmp_path): + """grace_s=0 (post-quarantine / heartbeat recovery) drops everything the + snapshot doesn't confirm — even fresh orders.""" + s = StateStore(tmp_path / "s.db") + s.upsert_order(_order("young", age_s=1.0)) + s.replace_open_orders("tok", [], grace_s=0.0) + assert s.orders_for("tok") == [] + s.close() + + +def test_replace_adopts_unknown_live_orders(tmp_path): + """Orders live on the exchange but missing from state (e.g. a timed-out + place that actually posted) are adopted so the reconciler can manage them.""" + s = StateStore(tmp_path / "s.db") + ghost = _order("ghost", age_s=30.0) + s.replace_open_orders("tok", [ghost]) + assert s.orders_for("tok")[0].order_id == "ghost" + s.close() + + +# ══════════════════════ engine failure handling ═════════════════════════ + + +def _mk_engine(tmp_path, meta): + from tests.test_engine import _engine_with_market, _feed_book + + eng = _engine_with_market(tmp_path, meta) + _feed_book(eng, meta) + return eng + + +async def test_place_failure_triggers_quarantine(tmp_path, meta): + """If a placement batch fails/returns incomplete, the engine must cancel + the tokens' orders (idempotent) and resync — never leave the possibility + of an untracked live order.""" + eng = _mk_engine(tmp_path, meta) + cancelled_assets: list[str] = [] + + async def failing_place(quotes, m): # posts may or may not have landed + return [] + + async def spy_cancel_asset(asset_id): + cancelled_assets.append(asset_id) + return True + + eng.gateway.place = failing_place # type: ignore[method-assign] + eng.gateway.cancel_asset = spy_cancel_asset # type: ignore[method-assign] + await eng._recompute(meta.condition_id) + + assert set(cancelled_assets) == {meta.yes.token_id, meta.no.token_id} + assert eng.state.orders == {} # nothing phantom left in state + eng.state.close() + eng.catalog.close() + + +async def test_cancel_failure_keeps_orders_and_skips_placement(tmp_path, meta): + """A failed cancel must NOT drop orders from state (they may be live), and + the engine must not place on top of them that cycle.""" + eng = _mk_engine(tmp_path, meta) + # seed a live order the strategy will want to reprice away (far off + stale) + stale = OpenOrder("stuck", meta.yes.token_id, Side.BUY, 0.10, 100, OrderState.LIVE) + stale.created_ts = time.time() - 120 + eng.state.upsert_order(stale) + + placed_calls: list[int] = [] + + async def failing_cancel(order_ids): + return False + + async def spy_place(quotes, m): + placed_calls.append(len(quotes)) + return [] + + async def rest_still_live(): # REST confirms the order is still on the book + return [stale] + + eng.gateway.cancel = failing_cancel # type: ignore[method-assign] + eng.gateway.place = spy_place # type: ignore[method-assign] + eng.gateway.open_orders = rest_still_live # type: ignore[method-assign] + await eng._recompute(meta.condition_id) + + assert "stuck" in {o.order_id for o in eng.state.orders_for(meta.yes.token_id)} + assert placed_calls == [] # skipped placement entirely this cycle + eng.state.close() + eng.catalog.close() + + +async def test_user_ws_blind_halts_market(tmp_path, meta): + """User WS down > threshold = we can't see fills -> pull all quotes.""" + from polymaker.userstream.client import UserStream + + eng = _mk_engine(tmp_path, meta) + # simulate a live-mode engine whose user stream has been down for a while + eng._user_started = True + eng.user = UserStream.__new__(UserStream) # bare instance, no connection + eng.user.connected = False + eng.user.disconnected_since = time.time() - 60.0 + + await eng._recompute(meta.condition_id) + assert eng.state.orders == {} # nothing quoted while blind + eng.state.close() + eng.catalog.close() + + +async def test_heartbeat_failures_halt_market(tmp_path, meta): + """Heartbeats failing = exchange is auto-cancelling us -> stop quoting.""" + eng = _mk_engine(tmp_path, meta) + eng.paper = False # hb_blind only applies in live mode + eng.gateway._hb_failures = 5 + await eng._recompute(meta.condition_id) + assert eng.state.orders == {} + eng.state.close() + eng.catalog.close() + + +async def test_healthy_engine_still_quotes(tmp_path, meta): + """Sanity: none of the blind checks fire on a healthy paper engine.""" + eng = _mk_engine(tmp_path, meta) + await eng._recompute(meta.condition_id) + assert len(eng.state.orders) > 0 + eng.state.close() + eng.catalog.close() + + +async def test_supervisor_restarts_dead_task(tmp_path, meta): + """A task that dies unexpectedly is restarted by the supervisor.""" + from tests.test_engine import _engine_with_market + + eng = _engine_with_market(tmp_path, meta) + runs: list[int] = [] + + async def flaky() -> None: + runs.append(1) + if len(runs) == 1: + raise RuntimeError("boom") # first run dies + await asyncio.sleep(30) # second run stays alive + + eng._supervise_interval_s = 0.05 # fast polling for the test + eng._spawn("flaky", flaky) + sup = asyncio.create_task(eng._supervise()) + for _ in range(40): + await asyncio.sleep(0.05) + if len(runs) >= 2: + break + sup.cancel() + eng._tasks["flaky"].cancel() + assert len(runs) >= 2, "dead task was not restarted" + eng.state.close() + eng.catalog.close()