exits: fix the 50-row page cap + incremental exits cache in duckdb

/closed-positions serves 50-row pages regardless of limit; stepping by the
requested size truncated every wallet's exit history to its most recent ~50
closes (1% coverage for hyperactive wallets) — sold-mirroring was silently
falling back to the hold-to-res ceiling almost everywhere. Now: page by
returned size, cache exits incrementally in duckdb (immutable events —
deep backfill once, a page or two per refresh; title/outcome columns for
display). Deep-exit 30d replay: $18,270 (+1727%), 259W/79L/29R/284S,
misses 253 -> 1 (mirrored exits recycle capital as fast as the signal does).

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

live/cache.py

@@ -183,6 +183,52 @@ def get_bets(wallet):
     return [_bet_row(*row) for row in rows]
 
 
+_con.execute("""CREATE TABLE IF NOT EXISTS exits(
+    wallet TEXT, asset TEXT, ts BIGINT, exit_p DOUBLE, p DOUBLE,
+    iv DOUBLE, cond TEXT, PRIMARY KEY (wallet, asset))""")
+for _col in ("title", "outcome"):
+    try:
+        _con.execute(f"ALTER TABLE exits ADD COLUMN {_col} TEXT")
+    except Exception:
+        pass  # already there
+_con.execute("CREATE TABLE IF NOT EXISTS pulled_exits(wallet TEXT PRIMARY KEY, newest_ts BIGINT, pulled_at BIGINT)")
+
+
+def closed_exits(wallet, max_age_s=6 * 3600):
+    """{asset: {ts, exit_p, p, iv, cond}} of the wallet's fully-closed
+    positions — INCREMENTAL: close events are immutable, so each refresh only
+    pages the data-api down to the newest cached close (the first backfill is
+    deep; after that it's a page or two). Shared exit model for the backtest
+    (portfolio.py) and the sharps stats (validate_timing.py).
+    NB the endpoint serves 50-row pages regardless of `limit` — see
+    smart_money.closed_exits for the paging gotcha this caused."""
+    import smart_money as _sm            # local import: smart_money has no local deps
+    now = int(time.time())
+    with _lock:
+        r = _con.execute("SELECT newest_ts, pulled_at FROM pulled_exits WHERE wallet=?",
+                         [wallet]).fetchone()
+    newest, fresh = (r[0], now - r[1] < max_age_s) if r else (0, False)
+    if not fresh:
+        new = _sm.closed_exits(wallet, newest_bound=newest)
+        with _lock:
+            if new:
+                _con.executemany(
+                    "INSERT OR REPLACE INTO exits VALUES (?,?,?,?,?,?,?,?,?)",
+                    [(wallet, a, c["ts"], c["exit_p"], c["p"], c["iv"], c["cond"],
+                      c.get("title") or "", c.get("outcome") or "")
+                     for a, c in new.items()])
+                newest = max(newest, max(c["ts"] for c in new.values()))
+            _con.execute("INSERT OR REPLACE INTO pulled_exits VALUES (?,?,?)",
+                         [wallet, newest, now])
+    with _lock:
+        rows = _con.execute(
+            "SELECT asset, ts, exit_p, p, iv, cond, title, outcome FROM exits WHERE wallet=?",
+            [wallet]).fetchall()
+    return {a: {"ts": ts, "exit_p": xp, "p": p, "iv": iv, "cond": cond,
+                "title": t or "", "outcome": o or ""}
+            for a, ts, xp, p, iv, cond, t, o in rows}
+
+
 def invalidate(wallets):
     """Force a re-pull of these wallets on next get_bets (for daily watchlist
     forward-refresh)."""

live/portfolio.py

@@ -234,7 +234,7 @@ def window_bets():
                         "their": cx["iv"], "entry_t": et, "p": cx["p"],
                         "won": None, "res_t": 0,
                         "exit_t": cx["ts"], "exit_p": cx["exit_p"],
-                        "title": cx["title"]})
+                        "title": cx.get("title") or ""})
     # chain-truth payouts for the replayed markets: refunds pay 0.5/share, and
     # a cache `won` mark can be wrong on operator-resolved markets — the
     # replay must settle at what a redeem actually pays (see payouts.py)
@@ -245,11 +245,10 @@ def window_bets():
 
 
 def closed_positions(wallet):
-    """The wallet's fully-closed positions with in-window close times —
-    shared implementation in smart_money.closed_exits (validate_timing uses
-    the same one, so the backtest and the sharps stats mirror exits
-    identically)."""
-    return sm.closed_exits(wallet, since_ts=START)
+    """The wallet's fully-closed positions — cache.closed_exits, the
+    incremental cached layer (validate_timing uses the same one, so the
+    backtest and the sharps stats mirror exits identically)."""
+    return cache.closed_exits(wallet)
 
 
 def open_bets():

live/validate_timing.py

@@ -146,7 +146,7 @@ def display_stats(w):
     # the wallet SOLD pre-resolution counts at its exit price (status SOLD) —
     # the same exit-mirroring the backtest and live bot use. Exits beyond the
     # closed-positions data horizon (~4000 rows) fall back to hold-to-res. ----
-    exits = sm.closed_exits(w)
+    exits = cache.closed_exits(w)
     tbest = {}
     for cond, asset, won, p, res_t, size in trows:
         if cond not in tbest or size > tbest[cond][3]:

smart_money.py

@@ -107,21 +107,29 @@ def leaderboard_candidates(pool):
     return ranked[:pool]
 
 
-def closed_exits(wallet, since_ts=0, max_rows=4000):
+def closed_exits(wallet, since_ts=0, max_rows=25000, newest_bound=0):
     """{asset: {ts, exit_p, p, iv, cond, title, outcome}} for the wallet's
     FULLY-CLOSED positions, newest first. `ts` is the close (sell/redeem)
     timestamp; the exit price is reconstructed from realized P&L over shares
     bought (exit_p = avgPrice + realizedPnl/totalBought — exact for a full
-    single-price exit, share-weighted otherwise). Shared by the backtest
-    (portfolio.py) and the sharps stats (validate_timing.py) so both books
-    mirror the signal's exits identically. Beyond max_rows (or before
-    since_ts) history falls back to hold-to-resolution — a data-horizon
-    ceiling, honest by construction."""
+    single-price exit, share-weighted otherwise).
+
+    PAGING GOTCHA: /closed-positions serves at most 50 rows per page no
+    matter what `limit` says — step by the RETURNED page size, never by the
+    requested one (assuming limit-sized pages silently truncated every
+    wallet's exit history to its most recent 50 closes, which put a 16x
+    hold-to-res ceiling back into a scalper's stats). Stops at since_ts,
+    newest_bound (for incremental refresh: rows older than what's already
+    cached), max_rows, or an empty page. Prefer cache.closed_exits — the
+    incremental cached layer over this raw fetcher."""
     out = {}
-    for off in range(0, max_rows, 500):
+    off = 0
+    while off < max_rows:
         page = get_json("/closed-positions",
                         {"user": wallet, "limit": 500, "offset": off,
-                         "sortBy": "TIMESTAMP", "sortDirection": "DESC"}) or []
+                         "sortBy": "TIMESTAMP", "sortDirection": "DESC"})
+        if not page:
+            break
         for r in page:
             ts = r.get("timestamp") or 0
             tb = r.get("totalBought") or 0
@@ -133,8 +141,10 @@ def closed_exits(wallet, since_ts=0, max_rows=4000):
                 "ts": ts, "exit_p": exit_p, "p": max(0.001, min(0.999, avg)),
                 "iv": r.get("initialValue") or avg * tb, "cond": r.get("conditionId"),
                 "title": r.get("title") or "", "outcome": r.get("outcome") or ""})
-        if len(page) < 500 or (page and (page[-1].get("timestamp") or 0) < since_ts):
+        oldest = page[-1].get("timestamp") or 0
+        if oldest < since_ts or oldest < newest_bound:
             break
+        off += len(page)                 # actual page size — the server caps at 50
     return out