Harden it up

README.md

@@ -1,7 +1,7 @@
 # poly-maker
 
 A maker-only market-making bot for **Polymarket CLOB V2**, focused on political
-markets. Single async process, local-file config (no Google Sheets), typed and
+markets. Single async process, local-file config, typed and
 tested.
 
 > [!WARNING]
@@ -82,8 +82,9 @@ uv run polymaker run --paper
 # 4. preflight the wallet before going live
 uv run polymaker doctor
 
-# 5. one safe live round-trip (~$5 post-only order, placed deep and cancelled)
-uv run polymaker livetest
+# 5. self-tests: a deep post-only order (free), then a real fill round-trip (~cents)
+uv run polymaker livetest      # place a deep post-only order + cancel (no fill)
+uv run polymaker moneydoctor   # limit rest + market buy + market sell, auto-flattens
 
 # 6. go live
 uv run polymaker run

config/config.toml

@@ -18,7 +18,7 @@ signature_type = 3
 clob_host = "https://clob.polymarket.com"
 gamma_host = "https://gamma-api.polymarket.com"
 data_api_host = "https://data-api.polymarket.com"
-polygon_rpc = "https://polygon-rpc.com"
+polygon_rpc = "https://polygon-bor-rpc.publicnode.com"
 
 [engine]
 debounce_ms = 200               # min gap between quote recomputes per market
@@ -35,6 +35,8 @@ max_event_group_loss_usdc = 1000.0   # neg-risk group worst-case loss cap
 max_market_notional_usdc = 800.0     # per-market position+orders notional cap
 daily_loss_kill_usdc = 250.0         # realized daily loss -> halt new quotes
 ws_stale_halt_s = 10.0               # no book updates for this long -> halt market
+user_ws_blind_halt_s = 15.0          # user WS down this long -> pull all quotes (can't see fills)
+heartbeat_halt_failures = 3          # consecutive heartbeat misses -> halt + resync
 max_order_error_rate = 0.25          # rolling order-post error fraction -> halt
 
 [execution]

config/markets.toml

@@ -9,3 +9,13 @@
 # profile     = "political-longdated"
 # enabled     = false
 # q_max_usdc  = 800      # optional per-market override of the profile value
+
+[[markets]]
+slug    = "will-gavin-newsom-win-the-2028-democratic-presidential-nomination-568"
+profile = "political-longdated"
+enabled = true
+
+[[markets]]
+slug    = "will-jd-vance-win-the-2028-republican-presidential-nomination"
+profile = "political-longdated"
+enabled = true

src/polymaker/cli.py

@@ -193,6 +193,18 @@ def livetest(
     raise typer.Exit(0 if ok else 1)
 
 
+@app.command()
+def moneydoctor(
+    config_dir: str = typer.Option("config", help="config directory"),
+) -> None:
+    """LIVE trading self-test: rest a limit, then market buy + sell (spends a little)."""
+    from polymaker.moneydoctor import run_moneydoctor
+
+    cfg = Config.load(config_dir)
+    ok = asyncio.run(run_moneydoctor(cfg, console))
+    raise typer.Exit(0 if ok else 1)
+
+
 @app.command(name="cancel-all")
 def cancel_all(config_dir: str = typer.Option("config", help="config directory")) -> None:
     """Cancel all open orders for the wallet (panic button)."""

src/polymaker/config.py

@@ -26,7 +26,7 @@ class WalletConfig(BaseModel):
     clob_host: str = "https://clob.polymarket.com"
     gamma_host: str = "https://gamma-api.polymarket.com"
     data_api_host: str = "https://data-api.polymarket.com"
-    polygon_rpc: str = "https://polygon-rpc.com"
+    polygon_rpc: str = "https://polygon-bor-rpc.publicnode.com"
 
 
 class EngineConfig(BaseModel):
@@ -45,6 +45,10 @@ class RiskConfig(BaseModel):
     max_market_notional_usdc: float = 800.0
     daily_loss_kill_usdc: float = 250.0
     ws_stale_halt_s: float = 10.0
+    # user WS down this long -> we can't see our fills -> pull all quotes
+    user_ws_blind_halt_s: float = 15.0
+    # consecutive heartbeat failures -> exchange is auto-cancelling us -> halt
+    heartbeat_halt_failures: int = 3
     max_order_error_rate: float = 0.25
 
 

src/polymaker/engine.py

@@ -72,7 +72,14 @@ class Engine:
         self._sweep: dict[str, bool] = {}
         self._merging: set[str] = set()
         self._token_cid: dict[str, str] = {}
-        self._tasks: list[asyncio.Task[Any]] = []
+        # supervised tasks: name -> (factory, task) so a dead task restarts
+        self._task_specs: dict[str, Any] = {}
+        self._tasks: dict[str, asyncio.Task[Any]] = {}
+        self._aux_tasks: list[asyncio.Task[Any]] = []  # fire-and-forget (merges)
+        # health / recovery signals
+        self._reconcile_now = asyncio.Event()
+        self._user_started = False  # user WS task launched (live mode)
+        self._hb_was_down = False
 
     # ── lifecycle ───────────────────────────────────────────────────────
     async def start(self) -> None:
@@ -89,24 +96,49 @@ class Engine:
             self.gateway.creds, self.gateway.address, self.user_proc,
             other_token=self._other_token, condition_of_token=self._cid_of_token,
             journal=self.journal, proxy=self.cfg.proxy,
+            on_reconnect=self._on_user_reconnect,
         )
         self.user.set_markets(list(self.metas))
 
-        # launch tasks
-        self._tasks.append(asyncio.create_task(self.md.run(), name="market_ws"))
+        # launch supervised tasks (a dead task is restarted, never silently gone)
+        self._spawn("market_ws", self.md.run)
         if not self.paper:
-            self._tasks.append(asyncio.create_task(self.user.run(), name="user_ws"))
-            self._tasks.append(asyncio.create_task(self._heartbeat_loop(), name="heartbeat"))
-        self._tasks.append(asyncio.create_task(self._reconcile_loop(), name="reconcile"))
+            assert self.user is not None
+            self._spawn("user_ws", self.user.run)
+            self._spawn("heartbeat", self._heartbeat_loop)
+            self._user_started = True
+        self._spawn("reconcile", self._reconcile_loop)
         for cid in self.metas:
-            self._tasks.append(asyncio.create_task(self._quoter(cid), name=f"quote:{cid[:8]}"))
+            self._spawn(f"quote:{cid[:8]}", lambda c=cid: self._quoter(c))
+        self._spawn("supervisor", self._supervise)
         self.risk.reset_day()
         log.info("engine_started", markets=len(self.metas), paper=self.paper)
 
+    def _spawn(self, name: str, factory: Any) -> None:
+        self._task_specs[name] = factory
+        self._tasks[name] = asyncio.create_task(factory(), name=name)
+
+    _supervise_interval_s: float = 5.0
+
+    async def _supervise(self) -> None:
+        """Restart any engine task that exits while we're running. Never down."""
+        while self._running:
+            await asyncio.sleep(self._supervise_interval_s)
+            for name, task in list(self._tasks.items()):
+                if name == "supervisor" or not task.done():
+                    continue
+                if not self._running:
+                    return
+                exc = None
+                with contextlib.suppress(asyncio.CancelledError, asyncio.InvalidStateError):
+                    exc = task.exception()
+                log.critical("task_died_restarting", task=name, err=str(exc) if exc else "exited")
+                self._tasks[name] = asyncio.create_task(self._task_specs[name](), name=name)
+
     async def run_forever(self) -> None:
         await self.start()
         with contextlib.suppress(asyncio.CancelledError):
-            await asyncio.gather(*self._tasks)
+            await asyncio.gather(*self._tasks.values(), *self._aux_tasks)
 
     async def shutdown(self) -> None:
         self._running = False
@@ -114,7 +146,7 @@ class Engine:
         self.md.stop()
         if self.user:
             self.user.stop()
-        for t in self._tasks:
+        for t in [*self._tasks.values(), *self._aux_tasks]:
             t.cancel()
         with contextlib.suppress(Exception):
             await self.gateway.cancel_all()
@@ -150,6 +182,10 @@ class Engine:
         reward_rates: dict[str, float],
     ) -> MarketMeta | None:
         tag_id = self.catalog.cached_tag("politics")
+        if tag_id is None:  # cold start: resolve + cache so the sweep is scoped
+            tag_id = await gamma.resolve_tag_id("politics")
+            if tag_id:
+                self.catalog.cache_tag("politics", tag_id)
         async for raw in gamma.iter_markets(tag_id=tag_id, max_pages=25):
             if (slug and raw.get("slug") == slug) or (condition_id and raw.get("conditionId") == condition_id):
                 m = parse_market(raw, reward_rates)
@@ -185,6 +221,16 @@ class Engine:
         if ev is not None:
             ev.set()
 
+    def _wake_all(self) -> None:
+        for ev in self._dirty.values():
+            ev.set()
+
+    def _on_user_reconnect(self) -> None:
+        """User WS reconnected: events during the gap were lost — force an
+        immediate REST reconcile before trusting our state again."""
+        log.warning("user_ws_reconnected_forcing_reconcile")
+        self._reconcile_now.set()
+
     def _on_trade(self, tp: TradePrint) -> None:
         cid = self._token_cid.get(tp.asset_id)
         if cid is None:
@@ -247,10 +293,30 @@ class Engine:
         q_max = p.q_max_usdc
         inv_util = abs(pos_yes.size - pos_no.size) * fv / q_max if q_max > 0 else 0.0
         hours_to_end = _hours_to_end(meta.end_date_iso, now)
-        ws_stale = (now - self.md.last_update_ts(meta.yes.token_id)) > self.cfg.risk.ws_stale_halt_s
 
-        rd = self.risk.evaluate(meta, ws_stale=ws_stale,
+        # ── blind/stale conditions: all use LOCAL receive time (skew-proof) ──
+        market_stale = (
+            (now - self.md.last_local_ts(meta.yes.token_id)) > self.cfg.risk.ws_stale_halt_s
+        )
+        user_blind = (
+            self._user_started
+            and self.user is not None
+            and not self.user.connected
+            and (now - self.user.disconnected_since) > self.cfg.risk.user_ws_blind_halt_s
+        )
+        hb_blind = (
+            not self.paper
+            and self.cfg.engine.heartbeat
+            and self.gateway.heartbeat_failures >= self.cfg.risk.heartbeat_halt_failures
+        )
+        blind = market_stale or user_blind or hb_blind
+        if blind:
+            log.warning("market_blind", cid=cid[:8], market_stale=market_stale,
+                        user_blind=user_blind, hb_blind=hb_blind)
+
+        rd = self.risk.evaluate(meta, ws_stale=blind,
                                 event_group_cost=self._event_group_cost(meta))
+        ws_stale = blind
         regime = self.regime_m[cid].decide(
             RegimeInputs(
                 now=now, tick=meta.tick_size, fv=fv, prev_fv=prev_fv,
@@ -277,26 +343,55 @@ class Engine:
             return
 
         if plan.to_cancel:
-            await self.gateway.cancel(plan.to_cancel)
-            for oid in plan.to_cancel:
-                self.state.remove_order(oid)
+            ok = await self.gateway.cancel(plan.to_cancel)
+            if ok:
+                for oid in plan.to_cancel:
+                    self.state.remove_order(oid)
+            else:
+                # cancel MAY have partially applied server-side — keep our view,
+                # resync from REST, and skip placing this cycle (avoid doubles)
+                await self._refresh_token_orders(meta, grace_s=10.0)
+                self._dirty[cid].set()
+                return
         if plan.to_place:
             placed = await self.gateway.place(plan.to_place, meta)
-            self.risk.note_order_result(bool(placed) or not plan.to_place)
+            self.risk.note_order_result(len(placed) == len(plan.to_place))
             for o in placed:
                 self.state.upsert_order(o)
+            if len(placed) < len(plan.to_place):
+                # QUARANTINE: a failed/partial batch may still have posted orders
+                # we don't have ids for. Cancel everything on these tokens
+                # (idempotent) and resync — never risk an untracked live order.
+                await self._quarantine(meta, reason="place_incomplete")
         log.info("requote", cid=cid[:8], regime=regime.value, fv=round(fv, 4),
                  place=len(plan.to_place), cancel=len(plan.to_cancel),
                  pos_yes=round(pos_yes.size, 1), pos_no=round(pos_no.size, 1))
         self._maybe_merge(cid, meta, p, pos_yes.size, pos_no.size)
 
+    async def _quarantine(self, meta: MarketMeta, reason: str) -> None:
+        """Cancel all orders on a market's tokens and resync state from REST."""
+        log.warning("quarantine", cid=meta.condition_id[:8], reason=reason)
+        for tok in (meta.yes.token_id, meta.no.token_id):
+            await self.gateway.cancel_asset(tok)
+            for o in self.state.orders_for(tok):
+                self.state.remove_order(o.order_id)
+        await self._refresh_token_orders(meta)
+
+    async def _refresh_token_orders(self, meta: MarketMeta, grace_s: float = 0.0) -> None:
+        """Open-orders resync for one market's tokens (grace_s=0 = authoritative)."""
+        live = await self.gateway.open_orders()
+        for tok in (meta.yes.token_id, meta.no.token_id):
+            self.state.replace_open_orders(
+                tok, [o for o in live if o.token_id == tok], grace_s=grace_s
+            )
+
     def _maybe_merge(self, cid: str, meta: MarketMeta, p: StrategyProfile,
                      yes_size: float, no_size: float) -> None:
         amount = min(yes_size, no_size)
         if amount < p.merge_min_size or cid in self._merging or self.paper:
             return
         self._merging.add(cid)
-        self._tasks.append(asyncio.create_task(self._merge_task(cid, meta, amount)))
+        self._aux_tasks.append(asyncio.create_task(self._merge_task(cid, meta, amount)))
 
     async def _merge_task(self, cid: str, meta: MarketMeta, amount: float) -> None:
         try:
@@ -309,25 +404,56 @@ class Engine:
     async def _heartbeat_loop(self) -> None:
         if not self.cfg.engine.heartbeat:
             return
+        halt_after = self.cfg.risk.heartbeat_halt_failures
         while self._running:
-            await self.gateway.heartbeat()
+            ok = await self.gateway.heartbeat()
+            if not ok and self.gateway.heartbeat_failures >= halt_after and not self._hb_was_down:
+                # exchange is (or soon will be) auto-cancelling everything we
+                # have live; recompute will see hb_blind and pull quotes
+                self._hb_was_down = True
+                log.critical("heartbeat_down_halting", failures=self.gateway.heartbeat_failures)
+                self._wake_all()
+            elif ok and self._hb_was_down:
+                # recovered: our server-side orders were wiped — drop local
+                # order state, resync authoritatively, then resume quoting
+                self._hb_was_down = False
+                log.warning("heartbeat_recovered_resyncing")
+                self.state.clear_orders()
+                for meta in self.metas.values():
+                    with contextlib.suppress(Exception):
+                        await self._refresh_token_orders(meta, grace_s=0.0)
+                self._wake_all()
             await asyncio.sleep(self.cfg.engine.heartbeat_interval_s)
 
     async def _reconcile_loop(self) -> None:
         while self._running:
-            await asyncio.sleep(self.cfg.engine.reconcile_interval_s)
+            # periodic cadence, but wake immediately when a reconnect/recovery
+            # demands an urgent resync
+            with contextlib.suppress(asyncio.TimeoutError):
+                await asyncio.wait_for(
+                    self._reconcile_now.wait(),
+                    timeout=self.cfg.engine.reconcile_interval_s,
+                )
+            forced = self._reconcile_now.is_set()
+            self._reconcile_now.clear()
             try:
                 positions = await self.gateway.positions()
                 if positions:
                     self.state.reconcile_positions(positions)
                 live = await self.gateway.open_orders()
-                if live or not self.paper:
-                    by_token: dict[str, list[Any]] = {}
-                    for o in live:
-                        by_token.setdefault(o.token_id, []).append(o)
-                    for tok, orders in by_token.items():
-                        if self.state.inflight(tok) == 0:
-                            self.state.replace_open_orders(tok, orders)
+                by_token: dict[str, list[Any]] = {}
+                for o in live:
+                    by_token.setdefault(o.token_id, []).append(o)
+                # iterate ALL our tokens, not just those present in the REST
+                # response — a token whose orders all vanished server-side must
+                # be cleaned up too (grace window protects fresh placements)
+                for tok in self._token_cid:
+                    if self.state.inflight(tok) == 0:
+                        self.state.replace_open_orders(tok, by_token.get(tok, []))
+                if forced:
+                    log.info("forced_reconcile_done", positions=len(positions),
+                             open_orders=len(live))
+                    self._wake_all()
             except Exception as exc:  # noqa: BLE001
                 log.warning("reconcile_error", err=str(exc))
 

src/polymaker/execution/gateway.py

@@ -53,6 +53,8 @@ class ExecutionGateway:
         self._order_bucket = TokenBucket(rate_per_s=200.0 * f, burst=500.0 * f)
         self._cancel_bucket = TokenBucket(rate_per_s=200.0 * f, burst=500.0 * f)
         self._paper_ids = itertools.count(1)
+        self._hb_id: str = ""  # heartbeat chain
+        self._hb_failures: int = 0
 
     @property
     def paper(self) -> bool:
@@ -159,9 +161,11 @@ class ExecutionGateway:
         return out
 
     # ── cancellation ────────────────────────────────────────────────────
-    async def cancel(self, order_ids: list[str]) -> None:
+    async def cancel(self, order_ids: list[str]) -> bool:
+        """Cancel by id. Returns True on success — callers must NOT drop the
+        orders from local state on failure (they may still be live)."""
         if not order_ids or self._paper:
-            return
+            return True
         await self._cancel_bucket.acquire(1)
 
         def _cancel() -> None:
@@ -169,19 +173,27 @@ class ExecutionGateway:
 
         try:
             await asyncio.to_thread(_cancel)
+            return True
         except Exception as exc:  # noqa: BLE001
             log.error("cancel_failed", err=str(exc), n=len(order_ids))
+            return False
 
-    async def cancel_asset(self, asset_id: str) -> None:
+    async def cancel_asset(self, asset_id: str) -> bool:
+        """Cancel every order on one token (idempotent quarantine primitive)."""
         if self._paper:
-            return
+            return True
 
         def _cancel() -> None:
             from py_clob_client_v2.clob_types import OrderMarketCancelParams
 
             self._client.cancel_market_orders(OrderMarketCancelParams(asset_id=asset_id))
 
-        await asyncio.to_thread(_cancel)
+        try:
+            await asyncio.to_thread(_cancel)
+            return True
+        except Exception as exc:  # noqa: BLE001
+            log.error("cancel_asset_failed", err=str(exc), token=asset_id[:12])
+            return False
 
     async def cancel_all(self) -> None:
         if self._paper or self._client is None:
@@ -189,14 +201,145 @@ class ExecutionGateway:
         await asyncio.to_thread(self._client.cancel_all)
         log.info("cancel_all_sent")
 
-    # ── heartbeat (dead-man switch) ─────────────────────────────────────
-    async def heartbeat(self, hb_id: str = "") -> None:
+    # ── market (taker) orders — used by moneydoctor, NOT the maker strategy ──
+    async def market_order(
+        self, token_id: str, side: Side, amount: float, meta: MarketMeta,
+        *, fak: bool = True,
+    ) -> dict[str, Any]:
+        """Place a marketable order. amount = USD for BUY, shares for SELL.
+
+        This is a TAKER order (crosses the spread) — only the moneydoctor live
+        self-test uses it; the maker strategy never does.
+        """
         if self._paper or self._client is None:
-            return
+            return {"paper": True}
+
+        def _do() -> dict[str, Any]:
+            from py_clob_client_v2.clob_types import (
+                MarketOrderArgsV2,
+                OrderType,
+                PartialCreateOrderOptions,
+            )
+
+            ot = OrderType.FAK if fak else OrderType.FOK
+            args = MarketOrderArgsV2(token_id=token_id, amount=amount,
+                                     side=side.value, order_type=ot)
+            opts = PartialCreateOrderOptions(tick_size=_tick_str(meta.tick_size),
+                                             neg_risk=meta.neg_risk)
+            try:
+                resp = self._client.create_and_post_market_order(args, opts, order_type=ot)
+                return resp if isinstance(resp, dict) else {"resp": resp}
+            except Exception as exc:  # noqa: BLE001 - surface as data, never crash the caller
+                return {"status": "failed", "error": str(exc)}
+
+        return await asyncio.to_thread(_do)
+
+    async def get_book(self, token_id: str) -> dict[str, float]:
+        """Live best bid/ask + touch depth for one token (public REST)."""
         try:
-            await asyncio.to_thread(self._client.post_heartbeat, hb_id)
+            async with httpx.AsyncClient(timeout=15.0) as c:
+                r = await c.get(f"{self._cfg.wallet.clob_host}/book",
+                                params={"token_id": token_id})
+                r.raise_for_status()
+                b = r.json()
+                bids = [(float(x["price"]), float(x["size"])) for x in b.get("bids", [])]
+                asks = [(float(x["price"]), float(x["size"])) for x in b.get("asks", [])]
+                best_bid = max(bids)[0] if bids else 0.0
+                best_ask = min(asks)[0] if asks else 1.0
+                ask_depth = sum(s for p, s in asks if p <= best_ask + 1e-9)
+                bid_depth = sum(s for p, s in bids if p >= best_bid - 1e-9)
+                return {"best_bid": best_bid, "best_ask": best_ask,
+                        "ask_depth": ask_depth, "bid_depth": bid_depth}
+        except (httpx.HTTPError, KeyError, ValueError) as exc:
+            log.warning("get_book_failed", err=str(exc))
+            return {}
+
+    async def token_balance(self, token_id: str) -> float:
+        """Exact on-chain conditional-token balance (shares) held by the funder.
+
+        Returns None on total RPC failure so callers can distinguish "0 shares"
+        from "couldn't read".
+        """
+        bal = await self._token_balance_opt(token_id)
+        return bal if bal is not None else 0.0
+
+    async def _token_balance_opt(self, token_id: str) -> float | None:
+        def _read() -> float | None:
+            from web3 import Web3
+            from web3.middleware import ExtraDataToPOAMiddleware
+
+            configured = self._cfg.secrets.polygon_rpc or self._cfg.wallet.polygon_rpc
+            rpcs = [configured, "https://polygon-bor-rpc.publicnode.com",
+                    "https://polygon.llamarpc.com", "https://rpc.ankr.com/polygon"]
+            abi = [{"name": "balanceOf", "type": "function", "stateMutability": "view",
+                    "inputs": [{"name": "a", "type": "address"}, {"name": "id", "type": "uint256"}],
+                    "outputs": [{"name": "", "type": "uint256"}]}]
+            for rpc in dict.fromkeys(rpcs):  # dedupe, keep order
+                try:
+                    w3 = Web3(Web3.HTTPProvider(rpc, request_kwargs={"timeout": 15}))
+                    w3.middleware_onion.inject(ExtraDataToPOAMiddleware, layer=0)
+                    ctf = w3.eth.contract(
+                        address=Web3.to_checksum_address("0x4D97DCd97eC945f40cF65F87097ACe5EA0476045"),
+                        abi=abi,
+                    )
+                    raw = ctf.functions.balanceOf(
+                        Web3.to_checksum_address(self.funder), int(token_id)
+                    ).call()
+                    return float(raw) / 1e6
+                except Exception:  # noqa: BLE001, PERF203 - try next RPC
+                    continue
+            return None
+
+        try:
+            return await asyncio.to_thread(_read)
         except Exception as exc:  # noqa: BLE001
-            log.warning("heartbeat_failed", err=str(exc))
+            log.warning("token_balance_failed", err=str(exc))
+            return None
+
+    async def collateral_balance(self) -> float:
+        """pUSD balance (float) on the funder."""
+        ba = await self.balance_allowance()
+        for k in ("balance", "collateral", "amount"):
+            if isinstance(ba, dict) and k in ba:
+                try:
+                    v = float(ba[k])
+                    return v / 1e6 if v > 1e6 else v
+                except (ValueError, TypeError):
+                    return 0.0
+        return 0.0
+
+    # ── heartbeat (dead-man switch) ─────────────────────────────────────
+    async def heartbeat(self) -> bool:
+        """Send one chained heartbeat. Returns True on success.
+
+        The exchange expects each heartbeat to carry the previous heartbeat_id.
+        Consecutive failures are tracked in `heartbeat_failures`: after enough
+        misses the exchange auto-cancels ALL our orders, so the engine must
+        stop quoting and resync once the heartbeat recovers.
+        """
+        if self._paper or self._client is None:
+            return True
+
+        def _beat() -> Any:
+            return self._client.post_heartbeat(self._hb_id)
+
+        try:
+            resp = await asyncio.to_thread(_beat)
+            new_id = _first(resp, "heartbeat_id", "heartbeatId", "id")
+            self._hb_id = str(new_id) if new_id else ""
+            if self._hb_failures:
+                log.info("heartbeat_recovered", after_failures=self._hb_failures)
+            self._hb_failures = 0
+            return True
+        except Exception as exc:  # noqa: BLE001
+            self._hb_failures += 1
+            self._hb_id = ""  # broken chain — restart it
+            log.warning("heartbeat_failed", err=str(exc), consecutive=self._hb_failures)
+            return False
+
+    @property
+    def heartbeat_failures(self) -> int:
+        return self._hb_failures
 
     # ── reads ───────────────────────────────────────────────────────────
     async def open_orders(self) -> list[OpenOrder]:

src/polymaker/marketdata/orderbook.py

@@ -10,6 +10,7 @@ apply_* mutators. Nothing here does I/O; the WS layer drives it.
 
 from __future__ import annotations
 
+import time
 from dataclasses import dataclass
 
 from sortedcontainers import SortedDict
@@ -58,14 +59,15 @@ class BookView:
 class OrderBook:
     """YES-canonical L2 book for one market."""
 
-    __slots__ = ("bids", "asks", "tick_size", "last_update_ts", "book_hash")
+    __slots__ = ("bids", "asks", "tick_size", "last_update_ts", "local_ts", "book_hash")
 
     def __init__(self, tick_size: float = 0.001) -> None:
         # price -> size. bids and asks both ascending in price.
         self.bids: SortedDict[float, float] = SortedDict()
         self.asks: SortedDict[float, float] = SortedDict()
         self.tick_size = tick_size
-        self.last_update_ts: float = 0.0
+        self.last_update_ts: float = 0.0  # exchange timestamp (informational)
+        self.local_ts: float = 0.0  # local receive time — used for staleness (skew-proof)
         self.book_hash: str | None = None
 
     # ── mutation ────────────────────────────────────────────────────────
@@ -79,6 +81,7 @@ class OrderBook:
         self.bids = SortedDict({p: s for p, s in bids if s > 0})
         self.asks = SortedDict({p: s for p, s in asks if s > 0})
         self.last_update_ts = ts
+        self.local_ts = time.time()
         self.book_hash = book_hash
 
     def apply_delta(self, side: Side, price: float, size: float, ts: float) -> None:
@@ -88,6 +91,7 @@ class OrderBook:
         else:
             book[price] = size
         self.last_update_ts = ts
+        self.local_ts = time.time()
 
     def set_tick_size(self, tick_size: float) -> None:
         self.tick_size = tick_size

src/polymaker/marketdata/service.py

@@ -55,6 +55,7 @@ class MarketDataService:
         self._subs: list[str] = []
         self._ws: Any = None
         self._stop = asyncio.Event()
+        self.connected: bool = False
 
     # ── subscription management ─────────────────────────────────────────
     def set_markets(self, markets: list[tuple[str, list[str]]]) -> None:
@@ -78,6 +79,11 @@ class MarketDataService:
         b = self.books.get(token_id)
         return b.last_update_ts if b else 0.0
 
+    def last_local_ts(self, token_id: str) -> float:
+        """Local receive time of the last book mutation (skew-proof staleness)."""
+        b = self.books.get(token_id)
+        return b.local_ts if b else 0.0
+
     # ── run loop ────────────────────────────────────────────────────────
     async def run(self) -> None:
         backoff = 1.0
@@ -98,15 +104,22 @@ class MarketDataService:
         if not self._subs:
             await asyncio.sleep(1.0)
             return
-        kwargs: dict[str, Any] = {"ping_interval": 5, "ping_timeout": None}
+        # ping_timeout matters: with None a half-dead TCP connection hangs
+        # forever. The server answers protocol pings (verified live), so a
+        # missing pong within 10s means the link is dead -> reconnect.
+        kwargs: dict[str, Any] = {"ping_interval": 5, "ping_timeout": 10, "open_timeout": 10}
         if self._proxy:
             kwargs["proxy"] = self._proxy
         async with websockets.connect(self._url, **kwargs) as ws:
             self._ws = ws
             await ws.send(json.dumps({"assets_ids": self._subs, "type": "market"}))
+            self.connected = True
             log.info("market_ws_subscribed", n=len(self._subs))
-            async for raw in ws:
-                self._handle(raw)
+            try:
+                async for raw in ws:
+                    self._handle(raw)
+            finally:
+                self.connected = False
 
     def stop(self) -> None:
         self._stop.set()

src/polymaker/moneydoctor.py

@@ -0,0 +1,244 @@
+"""`polymaker moneydoctor` — a LIVE trading self-test that actually moves money.
+
+Unlike `doctor` (read-only preflight) and `livetest` (a deep post-only order
+that can't fill), this exercises the full order machinery for real:
+
+  1. LIMIT  — place a post-only limit that rests on the book, confirm it appears,
+              then cancel it. (Free.)
+  2. BUY    — a market (taker) order that fills immediately; confirm shares land
+              on-chain.
+  3. SELL   — market-sell those shares back; confirm the position goes flat.
+
+It crosses the spread twice and pays taker fees, so it costs a small amount —
+reported at the end as the round-trip cost. Sized near the minimum order size.
+Taker orders are used ONLY here; the maker strategy never crosses the spread.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import contextlib
+import json
+import math
+from collections.abc import Callable
+from typing import TYPE_CHECKING
+
+import websockets
+from rich.console import Console
+
+from polymaker.config import Config
+from polymaker.domain import MarketMeta, Quote, Side
+from polymaker.strategy.quoting import round_to_tick
+
+if TYPE_CHECKING:
+    from polymaker.execution.gateway import ExecutionGateway
+
+USER_WS = "wss://ws-subscriptions-clob.polymarket.com/ws/user"
+
+
+async def run_moneydoctor(cfg: Config, console: Console, notional_usdc: float | None = None) -> bool:
+    from polymaker.execution.gateway import ExecutionGateway
+
+    ok = True
+
+    def check(label: str, passed: bool, detail: str = "") -> None:
+        nonlocal ok
+        mark = "[green]✓[/green]" if passed else "[red]✗[/red]"
+        console.print(f"  {mark} {label}" + (f"  [dim]{detail}[/dim]" if detail else ""))
+        ok = ok and passed
+
+    if not cfg.secrets.has_wallet:
+        console.print("[red]No wallet in .env.[/red]")
+        return False
+
+    console.print("[bold]polymaker moneydoctor[/bold]  [dim](spends a little real money)[/dim]")
+    gw = ExecutionGateway(cfg)
+    await gw.connect()
+    if cfg.proxy:
+        console.print(f"  [dim]· routing via proxy {cfg.proxy.split('@')[-1]}[/dim]")
+
+    meta, book = await _pick_market(cfg, gw)
+    if meta is None:
+        console.print("[yellow]No suitable liquid market found — run `polymaker scan` first.[/yellow]")
+        return False
+    token = meta.yes.token_id
+    tick, dec = meta.tick_size, meta.price_decimals
+    best_bid, best_ask = book["best_bid"], book["best_ask"]
+    console.print(f"  market: [bold]{meta.question[:56]}[/bold]")
+    console.print(f"  [dim]YES token · bid {best_bid} / ask {best_ask} · "
+                  f"spread {round(best_ask - best_bid, 4)} · tick {tick:g}[/dim]")
+
+    bal0 = await gw.collateral_balance()
+    console.print(f"  [dim]starting balance: {bal0:.4f} pUSD[/dim]\n")
+
+    # ── 1. LIMIT: rest + cancel ─────────────────────────────────────────
+    limit_price = round_to_tick(best_bid - 2 * tick, tick, dec, up=False)
+    limit_size = max(meta.min_order_size, 5.0)
+    placed = await gw.place([Quote(token, Side.BUY, limit_price, limit_size)], meta)
+    if placed:
+        await asyncio.sleep(1.5)
+        live = await gw.open_orders()
+        found = any(o.order_id == placed[0].order_id for o in live)
+        check("limit order rests on book", found, f"{limit_size:g} @ {limit_price}, {len(live)} live")
+        await gw.cancel([placed[0].order_id])
+        await asyncio.sleep(1.0)
+        gone = not any(o.order_id == placed[0].order_id for o in await gw.open_orders())
+        check("limit order cancels", gone)
+    else:
+        check("limit order placed", False, "post failed — see logs")
+
+    # ── 2. MARKET BUY ───────────────────────────────────────────────────
+    shares_target = meta.min_order_size + 3.0
+    buy_usd = round(shares_target * best_ask * 1.06, 2)
+    before = await gw._token_balance_opt(token) or 0.0  # baseline shares
+    console.print(f"\n  [dim]market BUY ~${buy_usd} of YES (targeting ~{shares_target:g} shares)…[/dim]")
+    resp_buy = await gw.market_order(token, Side.BUY, buy_usd, meta, fak=True)
+    bought, spent, status = _fill(resp_buy, Side.BUY)
+    check("market BUY matched", status == "matched" and bought > 0,
+          f"got {bought:.2f} shares for ${spent:.2f} [{status}]")
+
+    # ── settle: user WS (fast) with on-chain as source of truth ─────────
+    if bought > 0:
+        console.print("  [dim]waiting for settlement (user WS + chain)…[/dim]")
+        settled = await _wait_settled(cfg, gw, token, before, timeout=60)
+        got = settled - before
+        check("buy settled on-chain", got > 0.5, f"{got:.2f} shares now available")
+
+        # ── 3. MARKET SELL — retry until the exchange accepts it ────────
+        sell_amt = math.floor(max(got, bought) * 100) / 100
+        sold = await _sell_with_retry(gw, token, sell_amt, meta, before, console, check)
+
+        await asyncio.sleep(3.0)
+        remaining = await gw._token_balance_opt(token)
+        if remaining is not None:
+            check("position flat after round-trip", remaining <= before + 0.5,
+                  f"{remaining - before:.2f} net shares vs. start ({sold:.2f} sold)")
+    else:
+        console.print("  [yellow]! buy did not fill — nothing to sell.[/yellow]")
+
+    # ── cost ────────────────────────────────────────────────────────────
+    await asyncio.sleep(2.0)
+    bal1 = await gw.collateral_balance()
+    cost = bal0 - bal1
+    console.print(f"\n  [bold]round-trip cost: {cost:.4f} pUSD[/bold] "
+                  f"[dim](spread + taker fees; balance {bal0:.2f} → {bal1:.2f})[/dim]")
+    console.print(f"\n[bold]{'ALL GOOD' if ok else 'CHECK LOGS'}[/bold]")
+    return ok
+
+
+async def _pick_market(cfg: Config, gw: ExecutionGateway) -> tuple[MarketMeta | None, dict[str, float]]:
+    """Pick a liquid, mid-priced market with enough touch depth for a tiny order."""
+    from polymaker.catalog.store import CatalogStore
+
+    store = CatalogStore(cfg.paths.db)
+    rows = store.top(40)
+    store.close()
+    for meta, _sc in rows:
+        mid = (meta.best_bid + meta.best_ask) / 2 if (meta.best_bid and meta.best_ask) else 0.0
+        if not (0.2 < mid < 0.6):
+            continue
+        book = await gw.get_book(meta.yes.token_id)
+        if not book or book["best_bid"] <= 0 or book["best_ask"] >= 1:
+            continue
+        need_shares = meta.min_order_size + 4
+        if book["ask_depth"] >= need_shares and book["bid_depth"] >= need_shares:
+            spread = book["best_ask"] - book["best_bid"]
+            if spread <= 0.02:  # keep the round-trip cost small
+                return meta, book
+    return None, {}
+
+
+async def _wait_settled(cfg: Config, gw: ExecutionGateway, token: str, baseline: float,
+                        *, timeout: float = 60.0) -> float:
+    """Return the settled on-chain share balance once the buy lands.
+
+    Races two signals: the user WS `trade` status ladder (fast, push-based) and
+    an on-chain balance poll (slower, but the source of truth the exchange checks
+    when validating a sell). Returns the latest on-chain balance.
+    """
+    done = asyncio.Event()
+
+    async def chain_poll() -> None:
+        while not done.is_set():
+            await asyncio.sleep(3.0)
+            bal = await gw._token_balance_opt(token)
+            if bal is not None and bal > baseline + 0.01:
+                done.set()
+                return
+
+    async def ws_watch() -> None:
+        kw: dict[str, object] = {"ping_interval": 5, "ping_timeout": None, "open_timeout": 10}
+        if cfg.proxy:
+            kw["proxy"] = cfg.proxy
+        creds = gw.creds
+        with contextlib.suppress(Exception):
+            async with websockets.connect(USER_WS, **kw) as ws:  # type: ignore[arg-type]
+                await ws.send(json.dumps({
+                    "type": "user",
+                    "auth": {"apiKey": creds.api_key, "secret": creds.api_secret,
+                             "passphrase": creds.api_passphrase},
+                    "markets": [],
+                }))
+                while not done.is_set():
+                    raw = await asyncio.wait_for(ws.recv(), timeout=5)
+                    data = json.loads(raw)
+                    for m in data if isinstance(data, list) else [data]:
+                        if (isinstance(m, dict) and m.get("event_type") == "trade"
+                                and str(m.get("asset_id")) == token
+                                and str(m.get("status", "")).upper() in ("MINED", "CONFIRMED")):
+                            done.set()
+                            return
+
+    tasks = [asyncio.create_task(chain_poll()), asyncio.create_task(ws_watch())]
+    with contextlib.suppress(asyncio.TimeoutError):
+        await asyncio.wait_for(done.wait(), timeout=timeout)
+    done.set()
+    for t in tasks:
+        t.cancel()
+    await asyncio.gather(*tasks, return_exceptions=True)
+    return await gw._token_balance_opt(token) or baseline
+
+
+async def _sell_with_retry(
+    gw: ExecutionGateway, token: str, amount: float, meta: MarketMeta, baseline: float,
+    console: Console, check: Callable[..., None], attempts: int = 5,
+) -> float:
+    """Market-sell `amount`, retrying until the exchange accepts (balance settles)."""
+    for i in range(attempts):
+        console.print(f"  [dim]market SELL {amount:g} shares (attempt {i + 1})…[/dim]")
+        resp = await gw.market_order(token, Side.SELL, amount, meta, fak=True)
+        sold, recv, status = _fill(resp, Side.SELL)
+        if status == "matched" and sold > 0:
+            check("market SELL filled", True, f"sold {sold:.2f} shares for ${recv:.2f}")
+            return sold
+        err = resp.get("error", "") if isinstance(resp, dict) else ""
+        console.print(f"  [dim]  not filled yet ({status} {str(err)[:48]}); waiting to retry…[/dim]")
+        await asyncio.sleep(6.0)
+        bal = await gw._token_balance_opt(token)
+        if bal is not None:
+            amount = math.floor(max(0.0, bal - 0.0) * 100) / 100  # sell what's actually available
+            if amount < meta.min_order_size:
+                break
+    check("market SELL filled", False, "could not fill — flatten manually with cancel-all/limit")
+    return 0.0
+
+
+def _fill(resp: object, side: Side) -> tuple[float, float, str]:
+    """Parse a market-order response -> (shares_filled, usd, status).
+
+    makingAmount = what we give, takingAmount = what we get. So for a BUY,
+    shares = takingAmount and usd = makingAmount; for a SELL it's the reverse.
+    """
+    if not isinstance(resp, dict):
+        return 0.0, 0.0, "?"
+    status = str(resp.get("status", ""))
+    making = _f(resp.get("makingAmount"))
+    taking = _f(resp.get("takingAmount"))
+    return (taking, making, status) if side is Side.BUY else (making, taking, status)
+
+
+def _f(x: object) -> float:
+    try:
+        return float(x)  # type: ignore[arg-type]
+    except (ValueError, TypeError):
+        return 0.0

src/polymaker/state/store.py

@@ -65,8 +65,26 @@ class StateStore:
     def position(self, token_id: str) -> Position:
         return self.positions.get(token_id, Position(token_id))
 
-    def apply_fill(self, fill: Fill) -> None:
-        """Apply a fill optimistically to inventory + avg price."""
+    def apply_fill(self, fill: Fill) -> bool:
+        """Apply a fill optimistically to inventory + avg price.
+
+        IDEMPOTENT: the SQLite fills table is the dedupe gate (trade_id is the
+        primary key). A replayed fill — WS redelivery after reconnect, a MATCHED
+        arriving again after CONFIRMED, or a replay across process restarts —
+        is detected by INSERT OR IGNORE and NOT applied twice. Returns False
+        for duplicates so callers can skip their side effects too.
+        """
+        cur = self._conn.execute(
+            "INSERT OR IGNORE INTO fills(trade_id,token_id,side,price,size,is_maker,ts) VALUES(?,?,?,?,?,?,?)",
+            (fill.trade_id, fill.token_id, fill.side.value, fill.price, fill.size,
+             int(fill.is_maker), fill.ts),
+        )
+        self._conn.commit()
+        if cur.rowcount == 0:
+            log.warning("duplicate_fill_ignored", trade_id=fill.trade_id,
+                        token=fill.token_id[:12], side=fill.side.value, size=fill.size)
+            return False
+
         pos = self.positions.setdefault(fill.token_id, Position(fill.token_id))
         signed = fill.size if fill.side is Side.BUY else -fill.size
         new_size = pos.size + signed
@@ -83,9 +101,9 @@ class StateStore:
             pos.avg_price = 0.0
         self._last_fill_ts[fill.token_id] = fill.ts
         self._persist_position(pos)
-        self._record_fill(fill)
         log.info("fill", token=fill.token_id[:12], side=fill.side.value,
                  price=fill.price, size=fill.size, pos=round(pos.size, 2))
+        return True
 
     def set_position(self, token_id: str, size: float, avg_price: float) -> None:
         pos = Position(token_id, max(0.0, size), avg_price if size > 0 else 0.0)
@@ -128,13 +146,33 @@ class StateStore:
     def remove_order(self, order_id: str) -> None:
         self.orders.pop(order_id, None)
 
-    def replace_open_orders(self, token_id: str, live: list[OpenOrder]) -> None:
-        """Replace our view of a token's open orders from a REST snapshot."""
-        for oid in [o.order_id for o in self.orders.values() if o.token_id == token_id]:
-            self.orders.pop(oid, None)
+    def replace_open_orders(
+        self, token_id: str, live: list[OpenOrder], *, grace_s: float = 10.0
+    ) -> None:
+        """Replace our view of a token's open orders from a REST snapshot.
+
+        DOUBLE-ORDER GUARD: a REST snapshot can lag a placement by seconds. If we
+        dropped a just-placed order because the snapshot didn't include it yet,
+        the reconciler would immediately re-place it -> duplicate live orders.
+        So local orders younger than `grace_s` survive even when absent from the
+        snapshot (pass grace_s=0 to force an authoritative wipe, e.g. after the
+        exchange auto-cancelled everything on a heartbeat gap).
+        """
+        now = time.time()
+        live_ids = {o.order_id for o in live}
+        for o in [o for o in self.orders.values() if o.token_id == token_id]:
+            if o.order_id in live_ids:
+                continue
+            if now - o.created_ts < grace_s:
+                continue  # too young to trust its absence from the snapshot
+            self.orders.pop(o.order_id, None)
         for o in live:
             self.orders[o.order_id] = o
 
+    def clear_orders(self) -> None:
+        """Forget all local open orders (e.g. after a confirmed server-side wipe)."""
+        self.orders.clear()
+
     # ── persistence ─────────────────────────────────────────────────────
     def _persist_position(self, pos: Position) -> None:
         self._conn.execute(
@@ -143,13 +181,6 @@ class StateStore:
         )
         self._conn.commit()
 
-    def _record_fill(self, f: Fill) -> None:
-        self._conn.execute(
-            "INSERT OR IGNORE INTO fills(trade_id,token_id,side,price,size,is_maker,ts) VALUES(?,?,?,?,?,?,?)",
-            (f.trade_id, f.token_id, f.side.value, f.price, f.size, int(f.is_maker), f.ts),
-        )
-        self._conn.commit()
-
     def _persist_order(self, o: OpenOrder) -> None:
         self._conn.execute(
             "INSERT OR REPLACE INTO order_log(order_id,token_id,side,price,size,state,ts) VALUES(?,?,?,?,?,?,?)",

src/polymaker/state/tracker.py

@@ -62,9 +62,12 @@ class UserEventProcessor:
     def on_trade(self, ev: TradeEvent, condition_id: str) -> None:
         if ev.status is TradeState.MATCHED:
             if ev.trade_id in self._applied:
-                return  # idempotent: already counted this match
+                return  # idempotent: already counted this match (in-memory fast path)
             fill = Fill(ev.token_id, ev.our_side, ev.price, ev.size, ev.trade_id, ev.ts, is_maker=True)
-            self._store.apply_fill(fill)
+            if not self._store.apply_fill(fill):
+                # duplicate at the persistent layer (replay after CONFIRMED or
+                # across restarts) — apply NO side effects
+                return
             self._store.mark_inflight(ev.token_id)
             self._applied[ev.trade_id] = fill
             self._on_fill(fill)
@@ -77,10 +80,15 @@ class UserEventProcessor:
                 self._applied.pop(ev.trade_id, None)
                 self._on_change(condition_id)
 
-        elif ev.status in (TradeState.FAILED, TradeState.RETRYING):
+        elif ev.status is TradeState.RETRYING:
+            # tx being retried on-chain — it may still succeed. Keep the
+            # optimistic fill and the inflight guard; only FAILED is terminal.
+            log.warning("trade_retrying", trade_id=ev.trade_id, token=ev.token_id[:12])
+
+        elif ev.status is TradeState.FAILED:
             prior = self._applied.pop(ev.trade_id, None)
             if prior is not None:
-                # reverse the optimistic fill
+                # reverse the optimistic fill (idempotent via the :reverse id)
                 self._store.apply_fill(
                     Fill(prior.token_id, prior.side.opposite, prior.price, prior.size,
                          f"{prior.trade_id}:reverse", prior.ts, is_maker=True)

src/polymaker/userstream/client.py

@@ -8,6 +8,7 @@ from __future__ import annotations
 
 import asyncio
 import json
+import time
 from collections.abc import Callable
 from typing import Any
 
@@ -33,6 +34,7 @@ class UserStream:
         url: str = "wss://ws-subscriptions-clob.polymarket.com/ws/user",
         journal: Journal | None = None,
         proxy: str | None = None,
+        on_reconnect: Callable[[], None] | None = None,
     ) -> None:
         self._creds = creds
         self._address = our_address
@@ -42,8 +44,14 @@ class UserStream:
         self._url = url
         self._journal = journal
         self._proxy = proxy
+        self._on_reconnect = on_reconnect or (lambda: None)
         self._markets: list[str] = []
         self._stop = asyncio.Event()
+        # Connection health. ping_timeout guarantees a dead link flips
+        # `connected` to False within ~15s, so the engine can go blind-safe.
+        self.connected: bool = False
+        self.disconnected_since: float = time.time()
+        self._ever_connected = False
 
     def set_markets(self, condition_ids: list[str]) -> None:
         self._markets = condition_ids
@@ -73,14 +81,25 @@ class UserStream:
             },
             "markets": self._markets,
         }
-        kwargs: dict[str, Any] = {"ping_interval": 5, "ping_timeout": None}
+        kwargs: dict[str, Any] = {"ping_interval": 5, "ping_timeout": 10, "open_timeout": 10}
         if self._proxy:
             kwargs["proxy"] = self._proxy
         async with websockets.connect(self._url, **kwargs) as ws:
             await ws.send(json.dumps(sub))
-            log.info("user_ws_subscribed", markets=len(self._markets))
-            async for raw in ws:
-                self._handle(raw)
+            self.connected = True
+            is_reconnect = self._ever_connected
+            self._ever_connected = True
+            log.info("user_ws_subscribed", markets=len(self._markets), reconnect=is_reconnect)
+            if is_reconnect:
+                # events during the gap are LOST (no replay) — the engine must
+                # force a REST reconcile to recover any missed fills/cancels
+                self._on_reconnect()
+            try:
+                async for raw in ws:
+                    self._handle(raw)
+            finally:
+                self.connected = False
+                self.disconnected_since = time.time()
 
     def stop(self) -> None:
         self._stop.set()

tests/test_execution.py

@@ -55,7 +55,8 @@ async def test_paper_gateway_places_and_cancels_without_wallet(meta):
 
 async def test_paper_gateway_heartbeat_and_cancel_all_noop():
     gw = ExecutionGateway(Config(), paper=True)
-    await gw.heartbeat("hb1")
+    assert await gw.heartbeat() is True  # paper: healthy no-op
+    assert gw.heartbeat_failures == 0
     await gw.cancel_all()  # no client, must not raise
 
 

tests/test_hardening.py

@@ -0,0 +1,257 @@
+"""Hardening tests: the nasty failure modes that cause double-buys/sells,
+phantom orders, and flying-blind quoting. Every scenario here maps to a real
+Polymarket API failure mode (WS replay, snapshot lag, heartbeat gaps, timeouts).
+"""
+
+from __future__ import annotations
+
+import asyncio
+import time
+
+from polymaker.domain import (
+    Fill,
+    OpenOrder,
+    OrderState,
+    Side,
+    TradeState,
+)
+from polymaker.state.store import StateStore
+from polymaker.state.tracker import TradeEvent, UserEventProcessor
+
+# ══════════════════════ double-fill protection ══════════════════════════
+
+
+def test_replayed_matched_after_confirmed_not_double_applied(tmp_path):
+    """WS reconnects can replay events. MATCHED -> CONFIRMED -> MATCHED(replay)
+    must not double the position, even though the in-memory dedupe was cleared
+    at CONFIRMED."""
+    s = StateStore(tmp_path / "s.db")
+    p = UserEventProcessor(s)
+    ev = TradeEvent("tok", Side.BUY, 0.5, 100, "t1", TradeState.MATCHED, 1.0)
+    p.on_trade(ev, "cid")
+    p.on_trade(TradeEvent("tok", Side.BUY, 0.5, 100, "t1", TradeState.CONFIRMED, 2.0), "cid")
+    assert s.position("tok").size == 100
+    p.on_trade(ev, "cid")  # replayed MATCHED after confirm
+    assert s.position("tok").size == 100  # NOT 200
+    assert s.inflight("tok") == 0  # replay must not re-mark inflight
+    s.close()
+
+
+def test_duplicate_fill_across_restart(tmp_path):
+    """Process restarts + WS replays the same trade: the SQLite fills table is
+    the dedupe gate, so the position is not double-applied."""
+    db = tmp_path / "s.db"
+    s1 = StateStore(db)
+    assert s1.apply_fill(Fill("tok", Side.BUY, 0.5, 100, "t1")) is True
+    s1.close()
+    s2 = StateStore(db)
+    assert s2.apply_fill(Fill("tok", Side.BUY, 0.5, 100, "t1")) is False  # duplicate
+    assert s2.position("tok").size == 100
+    s2.close()
+
+
+def test_duplicate_fill_side_effects_skipped(tmp_path):
+    """A duplicate MATCHED must not fire on_fill/on_change callbacks."""
+    s = StateStore(tmp_path / "s.db")
+    fills, changes = [], []
+    p = UserEventProcessor(s, on_change=changes.append, on_fill=fills.append)
+    ev = TradeEvent("tok", Side.BUY, 0.5, 50, "t1", TradeState.MATCHED, 1.0)
+    p.on_trade(ev, "cid")
+    p.on_trade(TradeEvent("tok", Side.BUY, 0.5, 50, "t1", TradeState.CONFIRMED, 2.0), "cid")
+    n_fills, n_changes = len(fills), len(changes)
+    p.on_trade(ev, "cid")  # replay
+    assert len(fills) == n_fills  # no new fill callback
+    assert len(changes) == n_changes
+    s.close()
+
+
+def test_retrying_keeps_fill_failed_reverses_once(tmp_path):
+    """RETRYING is not terminal (tx may still land) -> keep the fill.
+    FAILED reverses exactly once, even if FAILED is replayed."""
+    s = StateStore(tmp_path / "s.db")
+    p = UserEventProcessor(s)
+    p.on_trade(TradeEvent("tok", Side.BUY, 0.5, 100, "t1", TradeState.MATCHED, 1.0), "cid")
+    p.on_trade(TradeEvent("tok", Side.BUY, 0.5, 100, "t1", TradeState.RETRYING, 2.0), "cid")
+    assert s.position("tok").size == 100  # retrying: unchanged
+    assert s.inflight("tok") == 1
+    p.on_trade(TradeEvent("tok", Side.BUY, 0.5, 100, "t1", TradeState.FAILED, 3.0), "cid")
+    assert s.position("tok").size == 0  # reversed
+    p.on_trade(TradeEvent("tok", Side.BUY, 0.5, 100, "t1", TradeState.FAILED, 4.0), "cid")
+    assert s.position("tok").size == 0  # replayed FAILED: no double reverse
+    s.close()
+
+
+# ══════════════════════ double-order protection ═════════════════════════
+
+
+def _order(oid: str, tok: str = "tok", age_s: float = 60.0) -> OpenOrder:
+    o = OpenOrder(oid, tok, Side.BUY, 0.49, 100, OrderState.LIVE)
+    o.created_ts = time.time() - age_s
+    return o
+
+
+def test_grace_window_protects_fresh_orders(tmp_path):
+    """A REST snapshot that lags a just-placed order must NOT evict it from
+    state (that eviction is what caused re-placement -> double orders)."""
+    s = StateStore(tmp_path / "s.db")
+    fresh = _order("young", age_s=2.0)
+    stale = _order("old", age_s=60.0)
+    s.upsert_order(fresh)
+    s.upsert_order(stale)
+    # snapshot doesn't include either (lag for young; old was really cancelled)
+    s.replace_open_orders("tok", [], grace_s=10.0)
+    ids = {o.order_id for o in s.orders_for("tok")}
+    assert "young" in ids  # protected by grace
+    assert "old" not in ids  # correctly dropped
+    s.close()
+
+
+def test_grace_zero_is_authoritative_wipe(tmp_path):
+    """grace_s=0 (post-quarantine / heartbeat recovery) drops everything the
+    snapshot doesn't confirm — even fresh orders."""
+    s = StateStore(tmp_path / "s.db")
+    s.upsert_order(_order("young", age_s=1.0))
+    s.replace_open_orders("tok", [], grace_s=0.0)
+    assert s.orders_for("tok") == []
+    s.close()
+
+
+def test_replace_adopts_unknown_live_orders(tmp_path):
+    """Orders live on the exchange but missing from state (e.g. a timed-out
+    place that actually posted) are adopted so the reconciler can manage them."""
+    s = StateStore(tmp_path / "s.db")
+    ghost = _order("ghost", age_s=30.0)
+    s.replace_open_orders("tok", [ghost])
+    assert s.orders_for("tok")[0].order_id == "ghost"
+    s.close()
+
+
+# ══════════════════════ engine failure handling ═════════════════════════
+
+
+def _mk_engine(tmp_path, meta):
+    from tests.test_engine import _engine_with_market, _feed_book
+
+    eng = _engine_with_market(tmp_path, meta)
+    _feed_book(eng, meta)
+    return eng
+
+
+async def test_place_failure_triggers_quarantine(tmp_path, meta):
+    """If a placement batch fails/returns incomplete, the engine must cancel
+    the tokens' orders (idempotent) and resync — never leave the possibility
+    of an untracked live order."""
+    eng = _mk_engine(tmp_path, meta)
+    cancelled_assets: list[str] = []
+
+    async def failing_place(quotes, m):  # posts may or may not have landed
+        return []
+
+    async def spy_cancel_asset(asset_id):
+        cancelled_assets.append(asset_id)
+        return True
+
+    eng.gateway.place = failing_place  # type: ignore[method-assign]
+    eng.gateway.cancel_asset = spy_cancel_asset  # type: ignore[method-assign]
+    await eng._recompute(meta.condition_id)
+
+    assert set(cancelled_assets) == {meta.yes.token_id, meta.no.token_id}
+    assert eng.state.orders == {}  # nothing phantom left in state
+    eng.state.close()
+    eng.catalog.close()
+
+
+async def test_cancel_failure_keeps_orders_and_skips_placement(tmp_path, meta):
+    """A failed cancel must NOT drop orders from state (they may be live), and
+    the engine must not place on top of them that cycle."""
+    eng = _mk_engine(tmp_path, meta)
+    # seed a live order the strategy will want to reprice away (far off + stale)
+    stale = OpenOrder("stuck", meta.yes.token_id, Side.BUY, 0.10, 100, OrderState.LIVE)
+    stale.created_ts = time.time() - 120
+    eng.state.upsert_order(stale)
+
+    placed_calls: list[int] = []
+
+    async def failing_cancel(order_ids):
+        return False
+
+    async def spy_place(quotes, m):
+        placed_calls.append(len(quotes))
+        return []
+
+    async def rest_still_live():  # REST confirms the order is still on the book
+        return [stale]
+
+    eng.gateway.cancel = failing_cancel  # type: ignore[method-assign]
+    eng.gateway.place = spy_place  # type: ignore[method-assign]
+    eng.gateway.open_orders = rest_still_live  # type: ignore[method-assign]
+    await eng._recompute(meta.condition_id)
+
+    assert "stuck" in {o.order_id for o in eng.state.orders_for(meta.yes.token_id)}
+    assert placed_calls == []  # skipped placement entirely this cycle
+    eng.state.close()
+    eng.catalog.close()
+
+
+async def test_user_ws_blind_halts_market(tmp_path, meta):
+    """User WS down > threshold = we can't see fills -> pull all quotes."""
+    from polymaker.userstream.client import UserStream
+
+    eng = _mk_engine(tmp_path, meta)
+    # simulate a live-mode engine whose user stream has been down for a while
+    eng._user_started = True
+    eng.user = UserStream.__new__(UserStream)  # bare instance, no connection
+    eng.user.connected = False
+    eng.user.disconnected_since = time.time() - 60.0
+
+    await eng._recompute(meta.condition_id)
+    assert eng.state.orders == {}  # nothing quoted while blind
+    eng.state.close()
+    eng.catalog.close()
+
+
+async def test_heartbeat_failures_halt_market(tmp_path, meta):
+    """Heartbeats failing = exchange is auto-cancelling us -> stop quoting."""
+    eng = _mk_engine(tmp_path, meta)
+    eng.paper = False  # hb_blind only applies in live mode
+    eng.gateway._hb_failures = 5
+    await eng._recompute(meta.condition_id)
+    assert eng.state.orders == {}
+    eng.state.close()
+    eng.catalog.close()
+
+
+async def test_healthy_engine_still_quotes(tmp_path, meta):
+    """Sanity: none of the blind checks fire on a healthy paper engine."""
+    eng = _mk_engine(tmp_path, meta)
+    await eng._recompute(meta.condition_id)
+    assert len(eng.state.orders) > 0
+    eng.state.close()
+    eng.catalog.close()
+
+
+async def test_supervisor_restarts_dead_task(tmp_path, meta):
+    """A task that dies unexpectedly is restarted by the supervisor."""
+    from tests.test_engine import _engine_with_market
+
+    eng = _engine_with_market(tmp_path, meta)
+    runs: list[int] = []
+
+    async def flaky() -> None:
+        runs.append(1)
+        if len(runs) == 1:
+            raise RuntimeError("boom")  # first run dies
+        await asyncio.sleep(30)  # second run stays alive
+
+    eng._supervise_interval_s = 0.05  # fast polling for the test
+    eng._spawn("flaky", flaky)
+    sup = asyncio.create_task(eng._supervise())
+    for _ in range(40):
+        await asyncio.sleep(0.05)
+        if len(runs) >= 2:
+            break
+    sup.cancel()
+    eng._tasks["flaky"].cancel()
+    assert len(runs) >= 2, "dead task was not restarted"
+    eng.state.close()
+    eng.catalog.close()