diff --git a/README.md b/README.md index cfcfd137..0258311c 100644 --- a/README.md +++ b/README.md @@ -290,17 +290,21 @@ runner is retired (GitHub throttled `*/5` to ~2h in practice — it copied 1 of location doesn't relocate *you* (trade live from Colombia months, paper from US months). -11. **Hold-to-resolution P&L is a copy ceiling, not the wallet's bank - statement.** The dashboard's Conv/All-Time P&L columns price every entry - held to resolution at the wallet's own stakes — the right yardstick for a - copier that holds, and the wrong one for judging the wallet itself. - Polymarket's own profile P&L (lb-api `/profit`, the **PM P&L** column) is - their actual cash-flow result. ~1× gap = true holder (LSB1 +$69.7k vs - +$68.5k); a huge gap = scalper whose entries resolve well but who never - holds (ArbTraderRookie: **+$8.6k real vs +$462k held, 53×** — a 0.5% - margin on $1.7M volume). For scalpers, whether a copier can reproduce - their fills is the open question — judge by the live book's measured - slippage, never the ceiling. +11. **All-Time / Conv P&L is the wallet's REALIZED track record** — the sum of + Polymarket's own `realizedPnl` per closed position over the wallet's full + history (`cache.closed_exits`, the incremental `/closed-positions` cache). + It's what a copier mirroring their buy/sell/hold actually banks, and it + equals the profile's **PM P&L** except for unrealized marks on positions + still open. This *replaced* the old hold-to-resolution reconstruction + (`won × entry × size`), which diverged from PM by up to 10× and even + flipped signs — four bugs: a 2,000-row pull cap (fixed: full history), + both-sides positions double-dropped by one-per-market dedup (fixed: each + asset is its own realized row), `initialValue = 0` mis-sizing (moot — + realized P&L needs no size), and corrupt near-epoch `res_t` (moot — + realized cash is timestamp-independent). Win % is now the share of closed + positions that *made money*, the mirror lens. (For wallets holding a large + losing open book, realized > PM because PM marks the open losers in; + that gap is unrealized, not an error.) --- diff --git a/live/cache.py b/live/cache.py index 94d16349..b0d7e533 100644 --- a/live/cache.py +++ b/live/cache.py @@ -191,6 +191,10 @@ for _col in ("title", "outcome"): _con.execute(f"ALTER TABLE exits ADD COLUMN {_col} TEXT") except Exception: pass # already there +try: + _con.execute("ALTER TABLE exits ADD COLUMN realized_pnl DOUBLE") # Polymarket's per-position realized cash +except Exception: + pass _con.execute("CREATE TABLE IF NOT EXISTS pulled_exits(wallet TEXT PRIMARY KEY, newest_ts BIGINT, pulled_at BIGINT)") for _col, _type in (("oldest_ts", "BIGINT"), ("complete", "BOOLEAN")): try: @@ -219,6 +223,14 @@ def closed_exits(wallet, max_age_s=6 * 3600): "WHERE wallet=?", [wallet]).fetchone() newest, complete = (r[0], bool(r[2])) if r else (0, False) fresh = r and (now - r[1] < max_age_s) + if complete: + # a wallet cached before realized_pnl existed needs one full re-pull to + # backfill it — self-healing, once, only for the affected wallets + with _lock: + miss = _con.execute("SELECT count(*) FROM exits WHERE wallet=? AND realized_pnl IS NULL", + [wallet]).fetchone()[0] + if miss: + complete = fresh = False if not fresh or not complete: if complete: new, _ = _sm.closed_exits(wallet, newest_bound=newest) # only new closes @@ -228,20 +240,22 @@ def closed_exits(wallet, max_age_s=6 * 3600): with _lock: if new: _con.executemany( - "INSERT OR REPLACE INTO exits VALUES (?,?,?,?,?,?,?,?,?)", + "INSERT OR REPLACE INTO exits" + "(wallet,asset,ts,exit_p,p,iv,cond,title,outcome,realized_pnl) " + "VALUES (?,?,?,?,?,?,?,?,?,?)", [(wallet, a, c["ts"], c["exit_p"], c["p"], c["iv"], c["cond"], - c.get("title") or "", c.get("outcome") or "") + c.get("title") or "", c.get("outcome") or "", c.get("realized_pnl") or 0) for a, c in new.items()]) newest = max(newest, max(c["ts"] for c in new.values())) _con.execute("INSERT OR REPLACE INTO pulled_exits(wallet,newest_ts,pulled_at,complete) " "VALUES (?,?,?,?)", [wallet, newest, now, reached]) with _lock: rows = _con.execute( - "SELECT asset, ts, exit_p, p, iv, cond, title, outcome FROM exits WHERE wallet=?", - [wallet]).fetchall() + "SELECT asset, ts, exit_p, p, iv, cond, title, outcome, realized_pnl " + "FROM exits WHERE wallet=?", [wallet]).fetchall() return {a: {"ts": ts, "exit_p": xp, "p": p, "iv": iv, "cond": cond, - "title": t or "", "outcome": o or ""} - for a, ts, xp, p, iv, cond, t, o in rows} + "title": t or "", "outcome": o or "", "realized_pnl": rp or 0} + for a, ts, xp, p, iv, cond, t, o, rp in rows} def invalidate(wallets): diff --git a/live/validate_timing.py b/live/validate_timing.py index 83f33f5b..9d6499fb 100644 --- a/live/validate_timing.py +++ b/live/validate_timing.py @@ -130,70 +130,58 @@ def display_stats(w): (e.g. ArbTrader: ~100% conv win but −$790 copy P&L). name / last-bet : from the /activity pull """ - # ---- position win%/record/P&L from the cache (large, survivorship-corrected). - # res_t <= now: the cache stores early-sold positions in UNRESOLVED markets with - # a future res_t and won = current price — a mark, not an outcome; skip them. ---- + # ---- ALL-TIME / CONVICTION / 30d records + P&L from the wallet's REALIZED + # TRACK RECORD: Polymarket's own realizedPnl per closed position, over the + # wallet's FULL history (cache.closed_exits, incremental). This is exactly + # what a copier who mirrors their buy/sell/hold banks — it sums to PM + # /profit (the source of truth) and needs no won×entry×size reconstruction, + # so it's immune to the four errors that plagued the old math: the 2000-row + # cap (now full history), both-sides double-drop (each asset is its own + # realized row), iv=0 mis-sizing (P&L doesn't need size), and corrupt res_t + # (realized cash is timestamp-independent). A position "won" if it made + # money as they traded it (realized_pnl > 0) — the mirror lens. ---- now = time.time() - bets = [b for b in cache.get_bets(w) - if (b["size"] or 0) > 0 and (b["res_t"] or 0) <= now] - # chain-truth payouts for everything this wallet's stats touch (cached - # in the resolutions table — incremental after the first backfill) - trows = trust.trusted_wallet_rows(cache.query, w) - payouts.ensure({b["cond"] for b in bets} | {r[0] for r in trows}) - # ---- ALL-TIME stats over EVERY trusted bet (any size): the dashboard's - # "of every bet placed" columns. Trusted rows only, deduped one-per-market, - # truth-adjusted: refunds (wp=0.5) count as neither won nor lost, and a bet - # the wallet SOLD pre-resolution counts at its exit price (status SOLD) — - # the same exit-mirroring the backtest and live bot use. Exits beyond the - # closed-positions data horizon (~4000 rows) fall back to hold-to-res. ---- - exits = cache.closed_exits(w) - tbest = {} - for cond, asset, won, p, res_t, size in trows: - if cond not in tbest or size > tbest[cond][3]: - tbest[cond] = (cond, asset, won, p, size, res_t) - def tally(rows): - """(won, lost, refunds, sold, pnl) over (cond, asset, won, p, size, res_t).""" - w_ = l_ = r_ = s_ = 0 + exits = cache.closed_exits(w) # {asset: {ts, iv, realized_pnl, ...}} closed, full history + + def rtally(positions): + """(won, lost, scratch, pnl) over closed positions by realized_pnl sign.""" + won_ = lost_ = scr_ = 0 pnl = 0.0 - for cond, asset, won, p, size, res_t in rows: - pc = max(0.001, min(0.999, p or 0)) - cx = exits.get(asset) - if cx and res_t and cx["ts"] < res_t - 300: # sold BEFORE resolution - pnl += size * (cx["exit_p"] - pc) / pc - s_ += 1 - continue - wp = _wp(cond, asset, won) - pnl += size * (wp - pc) / pc - if wp > 0.5: - w_ += 1 - elif wp < 0.5: - l_ += 1 + for e in positions: + rp = e.get("realized_pnl") or 0 + pnl += rp + if rp > 0.01: + won_ += 1 + elif rp < -0.01: + lost_ += 1 else: - r_ += 1 - return w_, l_, r_, s_, pnl - all_won, all_lost, all_ref, all_sold, all_pnl = tally(tbest.values()) - thr = cache.conv_cutoff(b["size"] for b in bets) - conv = [b for b in bets if b["size"] >= thr] - recent = sorted(bets, key=lambda b: b["res_t"] or 0, reverse=True)[:500] - cut30 = time.time() - 30 * 86400 - conv30 = [b for b in conv if (b["res_t"] or 0) >= cut30] - brow = lambda bs: [(b["cond"], b.get("asset"), b["won"], b["p"], b["size"], b["res_t"]) - for b in bs] - cw, cl, cr, cs, cpnl = tally(brow(conv)) - c3w, c3l, c3r, c3s, c3pnl = tally(brow(conv30)) + scr_ += 1 + return won_, lost_, scr_, round(pnl) + + allpos = list(exits.values()) + all_won, all_lost, all_scr, all_pnl = rtally(allpos) + # conviction = the wallet's top-20%-by-stake positions (iv); conv30 = those + # closed in the last 30d. Realized P&L over each set. + thr = cache.conv_cutoff(e["iv"] for e in allpos if (e.get("iv") or 0) > 0) + conv = [e for e in allpos if (e.get("iv") or 0) >= thr] + cut30 = now - 30 * 86400 + conv30 = [e for e in conv if (e.get("ts") or 0) >= cut30] + recent = sorted(allpos, key=lambda e: e.get("ts") or 0, reverse=True)[:500] + cw, cl, cscr, cpnl = rtally(conv) + c3w, c3l, c3scr, c3pnl = rtally(conv30) out = { "conv_win": round(100 * cw / (cw + cl), 1) if (cw + cl) else None, - "conv_won": cw, "conv_lost": cl, "conv_ref": cr, "conv_sold": cs, - "conv_pnl": round(cpnl), + "conv_won": cw, "conv_lost": cl, "conv_ref": cscr, "conv_sold": 0, + "conv_pnl": cpnl, "conv30_win": round(100 * c3w / (c3w + c3l), 1) if (c3w + c3l) else None, - "conv30_won": c3w, "conv30_lost": c3l, "conv30_ref": c3r, "conv30_sold": c3s, - "conv30_pnl": round(c3pnl), - "realized_pnl": round(tally(brow(recent))[4]), + "conv30_won": c3w, "conv30_lost": c3l, "conv30_ref": c3scr, "conv30_sold": 0, + "conv30_pnl": c3pnl, + "realized_pnl": rtally(recent)[3], "all_win": round(100 * all_won / (all_won + all_lost), 1) if (all_won + all_lost) else None, - "all_won": all_won, "all_lost": all_lost, "all_ref": all_ref, - "all_sold": all_sold, "all_pnl": round(all_pnl), + "all_won": all_won, "all_lost": all_lost, "all_ref": all_scr, + "all_sold": 0, "all_pnl": all_pnl, "pm_pnl": _pm_profit(w), - "avg_bet": round(sum(b["size"] for b in conv) / len(conv)) if conv else 0, + "avg_bet": round(sum(e["iv"] for e in conv) / len(conv)) if conv else 0, "copy_pnl": 0, "held_pnl": 0, "held_won": 0, "held_lost": 0, "sold": 0, "name": None, "last_trade": 0, "last_conv_bet": 0, } diff --git a/smart_money.py b/smart_money.py index 96abb9f6..67630ddc 100644 --- a/smart_money.py +++ b/smart_money.py @@ -142,14 +142,20 @@ def closed_exits(wallet, since_ts=0, max_rows=200000, newest_bound=0): break for r in page: ts = r.get("timestamp") or 0 + if not (r.get("asset") and ts): + continue tb = r.get("totalBought") or 0 avg = r.get("avgPrice") or 0 - if not (r.get("asset") and ts and tb and avg): - continue - exit_p = max(0.001, min(0.999, avg + (r.get("realizedPnl") or 0) / tb)) + rp = r.get("realizedPnl") or 0 # Polymarket's own per-position realized + # cash — sums to PM /profit; the + # copier-honest track record + # exit price reconstruction needs avg+tb; falls back to avg when the + # position lacks them (still keep the row for its realized_pnl) + exit_p = max(0.001, min(0.999, avg + rp / tb)) if (avg and tb) else max(0.001, min(0.999, avg or 0.5)) out.setdefault(r["asset"], { - "ts": ts, "exit_p": exit_p, "p": max(0.001, min(0.999, avg)), - "iv": r.get("initialValue") or avg * tb, "cond": r.get("conditionId"), + "ts": ts, "exit_p": exit_p, "p": max(0.001, min(0.999, avg or 0)), + "iv": r.get("initialValue") or (avg * tb) or 0, "cond": r.get("conditionId"), + "realized_pnl": rp, "title": r.get("title") or "", "outcome": r.get("outcome") or ""}) if len(page) < 50: # short page = start of history reached reached_end = True