diff --git a/live/conviction_scan.py b/live/conviction_scan.py index a58c28fe..4a13c31f 100644 --- a/live/conviction_scan.py +++ b/live/conviction_scan.py @@ -22,6 +22,14 @@ Profile gates (on TRAIN conviction bets): * median conviction stake >= MIN_MED_STAKE (dust wallets betting $2-$6 clips aren't followable and their fills aren't reproducible) Then validate forward and count how many keep the profile. + +NB (2026-07-06): this scan reads the cache's `won` marks, which count 50/50 +REFUNDS as wins for both sides (28% of resolved markets in the in-play niche) +— so it OVER-generates candidates. That's acceptable: this is the candidate +layer; final selection (validate_timing.py) re-judges every candidate against +chain-truth payouts (payouts.py) and rejects refund-inflated records. Chain- +checking all ~19M rows here would take days of RPC; the funnel does it only +for the few dozen wallets that survive to the copy replay. """ import math, os, time diff --git a/live/payouts.py b/live/payouts.py new file mode 100644 index 00000000..c2e989c0 --- /dev/null +++ b/live/payouts.py @@ -0,0 +1,206 @@ +#!/usr/bin/env python3 +"""Chain-truth resolution overlay for cache.duckdb — the 2026-07-06 audit fix. + +The cache's `won` is `curPrice >= 0.5` at pull time, and that lies two ways +(see the audit + [[polymarket-resolution-truth]] in project memory): + + * 50/50 REFUNDS (walkovers/abandonments — 28% of the follow set's resolved + markets in the 30d sample!) leave curPrice at 0.5, so EVERY holder of + EITHER side gets won=True and (1-p)/p phantom profit. The whales' 92-100% + displayed win rates were largely this. + * operator-resolved in-play markets can leave a stale mark on the losing + side (one confirmed both-sides-won market), which resolved=TRUE certifies. + +The fix keeps the cache AS PULLED and overlays the on-chain truth: the CTF +contract's payout vector per condition, plus the market's token order so a +row's `asset` maps to its payout. Resolved payouts are immutable — fetched +once, cached in a `resolutions` table forever. Unresolved conditions recheck +after RECHECK_S. + +Usage: + import payouts + payouts.ensure(conds) # batch-backfill (chain + CLOB, cached) + wp = payouts.truth(cond, asset) # 1.0 / 0.0 / 0.5 / None (unknown) + +Scoring rule for consumers: truth 1/0 -> real win/loss; 0.5 -> REFUND, count +as neither and P&L = size*(0.5-p)/p; None -> fall back to the cache's `won` +(legacy NULL-asset rows, unresolved markets, RPC gaps). + +RPC: config.json `alchemy_key` (or ALCHEMY_RPC_URL env). Batched JSON-RPC +(BATCH per POST); the CLOB market fetch supplies token order. Selection runs +on the Mac, so this module is never needed by the Fly worker. +""" +import json +import os +import ssl +import threading +import time +import urllib.request +from concurrent.futures import ThreadPoolExecutor + +import cache + +_SSL = ssl._create_unverified_context() +HERE = os.path.dirname(__file__) +CTF = "0x4D97DCd97eC945f40cF65F87097ACe5EA0476045" +SEL_DEN = "0xdd34de67" # payoutDenominator(bytes32) +SEL_NUM = "0x0504c814" # payoutNumerators(bytes32,uint256) +BATCH = 60 # JSON-RPC calls per POST +PACE_S = float(os.environ.get("PAYOUTS_PACE_S", 5.0)) # sleep between batches: + # free-tier Alchemy sustains ~12 eth_calls/s — an + # unpaced backfill just trades 429 retries for gaps +RECHECK_S = 6 * 3600 # re-ask the chain about unresolved conds after this + +cache.query("""CREATE TABLE IF NOT EXISTS resolutions( + cond TEXT PRIMARY KEY, p0 DOUBLE, p1 DOUBLE, + token0 TEXT, token1 TEXT, checked_at BIGINT)""") + +_mem = {} # cond -> row tuple, warm in-process copy +_lock = threading.Lock() + + +def _rpc_url(): + url = os.environ.get("ALCHEMY_RPC_URL") + if url: + return url + try: + key = json.load(open(os.path.join(HERE, "..", "config.json")))["alchemy_key"] + return f"https://polygon-mainnet.g.alchemy.com/v2/{key}" + except Exception: + return None + + +def _rpc_batch(calls): + """calls: [hex calldata] -> [int results or None], one batched POST.""" + url = _rpc_url() + if not url: + return [None] * len(calls) + body = json.dumps([{"jsonrpc": "2.0", "id": i, "method": "eth_call", + "params": [{"to": CTF, "data": d}, "latest"]} + for i, d in enumerate(calls)]).encode() + # retry with backoff — Alchemy 429s a fast backfill after ~2 batches, and a + # failed batch must surface as None (skip-this-run), never "unresolved" + for attempt in range(4): + try: + req = urllib.request.Request(url, data=body, + headers={"Content-Type": "application/json"}) + rs = json.loads(urllib.request.urlopen(req, timeout=30, context=_SSL).read()) + if isinstance(rs, dict): # whole-batch error object + raise OSError(str(rs.get("error"))[:80]) + by_id = {r.get("id"): r for r in rs} + out = [] + for i in range(len(calls)): + v = by_id.get(i, {}).get("result") + try: + # "0x" (revert/empty) must not kill the whole batch — one + # bad item used to blank every cond in the chunk + out.append(int(v, 16) if v and v != "0x" else None) + except (TypeError, ValueError): + out.append(None) + return out + except Exception: + if attempt == 3: + return [None] * len(calls) + time.sleep(2 ** attempt) + + +def _clob_tokens(cond): + """(token0, token1) in the CLOB market's outcome order — payout index order.""" + try: + req = urllib.request.Request("https://clob.polymarket.com/markets/" + cond, + headers={"User-Agent": "Mozilla/5.0"}) + m = json.loads(urllib.request.urlopen(req, timeout=15, context=_SSL).read()) + toks = [str(t.get("token_id")) for t in (m.get("tokens") or [])[:2]] + return (toks + [None, None])[:2] + except Exception: + return None, None + + +def _load(conds): + with _lock: + missing = [c for c in conds if c and c not in _mem] + if missing: + rows = cache.query( + "SELECT cond, p0, p1, token0, token1, checked_at FROM resolutions " + "WHERE cond IN (SELECT UNNEST(?::VARCHAR[]))", [missing]) + with _lock: + for r in rows: + _mem[r[0]] = r + + +def ensure(conds, workers=8): + """Make sure every cond has a resolutions row (fetching chain + CLOB for the + unknown ones). Resolved rows are permanent; unresolved recheck after + RECHECK_S. Safe to call with thousands of conds — everything is cached.""" + conds = [c for c in {c for c in conds if c}] + _load(conds) + now = int(time.time()) + with _lock: + todo = [c for c in conds + if c not in _mem + or (_mem[c][1] is None and now - (_mem[c][5] or 0) > RECHECK_S)] + if not todo: + return + # chain payouts, batched: den + num0 per cond (binary: num1 = den - num0) + payout = {} + for i in range(0, len(todo), BATCH // 2): + chunk = todo[i:i + BATCH // 2] + calls = [] + for c in chunk: + h = c[2:].rjust(64, "0") + calls += [SEL_DEN + h, + SEL_NUM + h + "0".rjust(64, "0")] + res = _rpc_batch(calls) + for j, c in enumerate(chunk): + den, n0 = res[2 * j], res[2 * j + 1] + if den is None: + payout[c] = "rpc-fail" # do NOT cache — retry next run + elif den and n0 is not None: + payout[c] = (n0 / den, (den - n0) / den) + else: + payout[c] = (None, None) # chain says: not resolved yet + if i + BATCH // 2 < len(todo): + time.sleep(PACE_S) + # token order for the RESOLVED ones (index -> asset mapping); cached forever + resolved = [c for c in todo + if isinstance(payout.get(c), tuple) and payout[c][0] is not None] + tokens = {} + with ThreadPoolExecutor(max_workers=workers) as ex: + for c, tk in zip(resolved, ex.map(_clob_tokens, resolved)): + tokens[c] = tk + with _lock: + for c in todo: + p = payout.get(c) + if p == "rpc-fail" or p is None: + continue # transient — leave uncached, retry next run + t0, t1 = tokens.get(c, (None, None)) + row = (c, p[0], p[1], t0, t1, now) + _mem[c] = row + cache.query("INSERT OR REPLACE INTO resolutions VALUES (?,?,?,?,?,?)", list(row)) + + +def truth(cond, asset=None): + """Chain-truth payout of this position: 1.0 / 0.0 / 0.5 / None (unknown). + Refunds ([0.5,0.5]) need no asset; decided markets map asset -> index.""" + _load([cond]) + r = _mem.get(cond) + if not r or r[1] is None: + return None + p0, p1, t0, t1 = r[1], r[2], r[3], r[4] + if p0 == p1: # refund (or exotic even split) + return p0 + if asset is not None: + if str(asset) == str(t0): + return p0 + if str(asset) == str(t1): + return p1 + return None # decided, but we can't map the side + + +def stats(): + n = cache.query("SELECT count(*), count(p0) FROM resolutions")[0] + return {"rows": n[0], "resolved": n[1]} + + +if __name__ == "__main__": + print(stats()) diff --git a/live/portfolio.py b/live/portfolio.py index de926b76..b95bb81f 100644 --- a/live/portfolio.py +++ b/live/portfolio.py @@ -31,6 +31,7 @@ import urllib.request from concurrent.futures import ThreadPoolExecutor import cache +import payouts import smart_money as sm import trust @@ -179,24 +180,31 @@ def window_bets(): rows = trust.trusted_wallet_rows(cache.query, w["wallet"], now) best = {} # one bet per market: largest-stake token for cond, asset, won, p, res_t, size in rows: - if cond not in best or size > best[cond][3]: - best[cond] = (won, p, res_t, size) + if cond not in best or size > best[cond][4]: + best[cond] = (asset, won, p, res_t, size) if w.get("class") == "whale": thr = 0.0 else: - pre = [size for won, p, res_t, size in best.values() if res_t < START] + pre = [size for asset, won, p, res_t, size in best.values() if res_t < START] thr = cache.conv_cutoff(pre if pre else [size for *_, size in best.values()]) _WALLET_THR[w["wallet"]] = thr - for cond, (won, p, res_t, size) in best.items(): + for cond, (asset, won, p, res_t, size) in best.items(): if size < thr or p > MAX_ENTRY: continue et = ent.get(cond) if not et or et < START: # only in-window entries continue out.append({"wallet": w["wallet"], "name": w["name"], "cond": cond, - "cls": w.get("class", "volume"), "their": size, - "entry_t": et, "p": p, "won": won, "res_t": res_t or 0}) + "asset": asset, "cls": w.get("class", "volume"), + "their": size, "entry_t": et, "p": p, "won": won, + "res_t": res_t or 0}) + # chain-truth payouts for the replayed markets: refunds pay 0.5/share, and + # a cache `won` mark can be wrong on operator-resolved markets — the + # replay must settle at what a redeem actually pays (see payouts.py) + payouts.ensure({b["cond"] for b in out}) + for b in out: + b["wp"] = payouts.truth(b["cond"], b.get("asset")) return out @@ -301,7 +309,12 @@ def main(): for ft, cost, payoff, rec in held: if ft and ft <= upto and rec["kind"] == "res": cash += payoff; realized += payoff - cost; perW[rec["wallet"]]["realized"] += payoff - cost - perW[rec["wallet"]]["won" if rec["won"] else "lost"] += 1 + wp = rec.get("wp") + won = rec["won"] if wp is None else wp > 0.5 + perW[rec["wallet"]]["won" if won else "lost"] += 1 + rec["won"] = won # truth-adjusted for the feed + if wp == 0.5: + rec["refund"] = True rec["pnl"] = payoff - cost resolved.append(rec) else: @@ -325,7 +338,11 @@ def main(): cash -= cost; fees_paid += fee; perW[b["wallet"]]["bets"] += 1 shares = stake / p_eff # lag-adjusted entry price if b["kind"] == "res": - payoff = shares * (1.0 if b["won"] else 0.0) # redeem is fee-free + # chain-truth payout (1/0/0.5) when known, else the cache mark + wp = b.get("wp") + if wp is None: + wp = 1.0 if b["won"] else 0.0 + payoff = shares * wp # redeem is fee-free held.append((b["res_t"] or now, cost, payoff, b)) else: # currently open -> mark to market, no free yet held.append((None, cost, 0.0, b)) @@ -356,7 +373,10 @@ def main(): stake = m.get("stake") or STAKE_MIN p_eff, fee, cost = entry_model(m["p"], stake) if "won" in m: - return (stake / p_eff) - cost if m["won"] else -cost + wp = m.get("wp") + if wp is None: + wp = 1.0 if m["won"] else 0.0 + return (stake / p_eff) * wp - cost return stake * (m.get("cur", p_eff) / p_eff) - cost missed.sort(key=lambda m: m.get("res_t") or 0, reverse=True) diff --git a/live/trust.py b/live/trust.py index a6958a6b..ae2da425 100644 --- a/live/trust.py +++ b/live/trust.py @@ -103,17 +103,23 @@ def trusted_wallet_rows(runq, wallet, now=None): [wallet, now, now]) -def conviction_record(runq, wallet, days=90, pctile=0.80, now=None): +def conviction_record(runq, wallet, days=90, pctile=0.80, now=None, truthfn=None): """Trailing trusted CONVICTION record for the held-edge gate: {n, wr, roi} over the wallet's top-(1-pctile) stake bets resolved in the last `days`. The conviction cutoff is that wallet's stake p80 over its FULL trusted history (matching cache.conv_cutoff semantics). roi is the flat- stake hold-to-resolution copy ROI per bet, fee/slip-free (gates compare it - to 0, and fees are already charged in copy_pnl, the other selection leg).""" + to 0, and fees are already charged in copy_pnl, the other selection leg). + + truthfn(cond, asset) -> 1/0/0.5/None (payouts.truth): chain-truth payouts. + 50/50 REFUNDS (28% of the follow-set niche's resolved markets!) then count + as NEITHER won nor lost, with roi credited at (0.5-p)/p — the cache alone + marks every refund won=True for both sides, which is exactly the inflation + these gates must not select on. None -> the cache's won (old behavior).""" now = int(now or time.time()) rows = trusted_wallet_rows(runq, wallet, now) if not rows: - return dict(n=0, wr=0.0, roi=0.0) + return dict(n=0, wr=0.0, roi=0.0, refunds=0) sizes = sorted(r[5] for r in rows) k = (len(sizes) - 1) * pctile f = int(k) @@ -123,11 +129,24 @@ def conviction_record(runq, wallet, days=90, pctile=0.80, now=None): best = {} for cond, asset, won, p, res_t, size in rows: if size >= thr and res_t >= cut: - if cond not in best or size > best[cond][2]: - best[cond] = (won, p, size) + if cond not in best or size > best[cond][3]: + best[cond] = (cond, asset, won, p, size) conv = list(best.values()) if not conv: - return dict(n=0, wr=0.0, roi=0.0) - wins = sum(1 for won, _, _ in conv if won) - roi = sum(((1 - p) / p if won else -1.0) for won, p, _ in conv) / len(conv) - return dict(n=len(conv), wr=wins / len(conv), roi=roi) + return dict(n=0, wr=0.0, roi=0.0, refunds=0) + wins = losses = refunds = 0 + roi_sum = 0.0 + for cond, asset, won, p, size in conv: + wp = truthfn(cond, asset) if truthfn else None + if wp is None: + wp = 1.0 if won else 0.0 + roi_sum += (wp - p) / p + if wp > 0.5: + wins += 1 + elif wp < 0.5: + losses += 1 + else: + refunds += 1 + decided = wins + losses + return dict(n=decided, wr=(wins / decided) if decided else 0.0, + roi=roi_sum / len(conv), refunds=refunds) diff --git a/live/validate_timing.py b/live/validate_timing.py index fa7bd4cb..2f9d69ca 100644 --- a/live/validate_timing.py +++ b/live/validate_timing.py @@ -37,6 +37,7 @@ import urllib.request from concurrent.futures import ThreadPoolExecutor import cache +import payouts import smart_money as sm import trust @@ -93,11 +94,25 @@ def _pm_profit(w): return None -def _bet_pnl(b): - """Resolved (outcome) P&L of one cache bet: a $size stake at avg price p pays - size/p if won, else $0 — so P&L = size·(1−p)/p if won else −size.""" +def _wp(cond, asset, won): + """Chain-truth payout for a bet (1/0/0.5), falling back to the cache's + `won` mark when the chain can't say (unresolved, legacy NULL-asset rows on + decided markets, RPC gaps). The fallback keeps old behavior; the truth + path kills the two cache lies: 50/50 refunds counted as wins for BOTH + sides (28% of the follow set's resolved markets!) and stale both-sides-won + marks on operator-resolved markets.""" + wp = payouts.truth(cond, asset) + return (1.0 if won else 0.0) if wp is None else wp + + +def _bet_pnl(b, wp=None): + """Resolved P&L of one cache bet at payout wp: a $size stake at avg price p + returns size·(wp−p)/p — wp=1 win, 0 loss, 0.5 refund ($0.50/share, NOT + money-back: flat near coin-flip entries, ruinous for favorites).""" p = max(0.001, min(0.999, b["p"] or 0)) - return b["size"] * ((1 - p) / p if b["won"] else -1) + if wp is None: + wp = _wp(b.get("cond"), b.get("asset"), b["won"]) + return b["size"] * (wp - p) / p def display_stats(w): @@ -121,35 +136,51 @@ def display_stats(w): now = time.time() bets = [b for b in cache.get_bets(w) if (b["size"] or 0) > 0 and (b["res_t"] or 0) <= now] + # chain-truth payouts for everything this wallet's stats touch (cached + # in the resolutions table — incremental after the first backfill) + trows = trust.trusted_wallet_rows(cache.query, w) + payouts.ensure({b["cond"] for b in bets} | {r[0] for r in trows}) # ---- ALL-TIME stats over EVERY trusted bet (any size): the dashboard's # "of every bet placed" columns. Trusted rows only, deduped one-per-market, - # so scalper-poisoned cache marks can't inflate them. ---- - trows = trust.trusted_wallet_rows(cache.query, w) + # truth-adjusted: refunds (wp=0.5) count as neither won nor lost. ---- tbest = {} for cond, asset, won, p, res_t, size in trows: - if cond not in tbest or size > tbest[cond][2]: - tbest[cond] = (won, p, size) - all_won = sum(1 for won, _, _ in tbest.values() if won) - all_lost = len(tbest) - all_won - all_pnl = sum(size * ((1 - p) / p if won else -1.0) - for won, p, size in tbest.values()) + if cond not in tbest or size > tbest[cond][3]: + tbest[cond] = (cond, asset, won, p, size) + def tally(rows): + """(won, lost, refunds, pnl) over (cond, asset, won, p, size) rows.""" + w_ = l_ = r_ = 0 + pnl = 0.0 + for cond, asset, won, p, size in rows: + wp = _wp(cond, asset, won) + pc = max(0.001, min(0.999, p or 0)) + pnl += size * (wp - pc) / pc + if wp > 0.5: + w_ += 1 + elif wp < 0.5: + l_ += 1 + else: + r_ += 1 + return w_, l_, r_, pnl + all_won, all_lost, all_ref, all_pnl = tally(tbest.values()) thr = cache.conv_cutoff(b["size"] for b in bets) conv = [b for b in bets if b["size"] >= thr] - won = sum(1 for b in conv if b["won"]) recent = sorted(bets, key=lambda b: b["res_t"] or 0, reverse=True)[:500] cut30 = time.time() - 30 * 86400 conv30 = [b for b in conv if (b["res_t"] or 0) >= cut30] - won30 = sum(1 for b in conv30 if b["won"]) + brow = lambda bs: [(b["cond"], b.get("asset"), b["won"], b["p"], b["size"]) for b in bs] + cw, cl, cr, cpnl = tally(brow(conv)) + c3w, c3l, c3r, c3pnl = tally(brow(conv30)) out = { - "conv_win": round(100 * won / len(conv), 1) if conv else None, - "conv_won": won, "conv_lost": len(conv) - won, - "conv_pnl": round(sum(_bet_pnl(b) for b in conv)), - "conv30_win": round(100 * won30 / len(conv30), 1) if conv30 else None, - "conv30_won": won30, "conv30_lost": len(conv30) - won30, - "conv30_pnl": round(sum(_bet_pnl(b) for b in conv30)), - "realized_pnl": round(sum(_bet_pnl(b) for b in recent)), + "conv_win": round(100 * cw / (cw + cl), 1) if (cw + cl) else None, + "conv_won": cw, "conv_lost": cl, "conv_ref": cr, + "conv_pnl": round(cpnl), + "conv30_win": round(100 * c3w / (c3w + c3l), 1) if (c3w + c3l) else None, + "conv30_won": c3w, "conv30_lost": c3l, "conv30_ref": c3r, + "conv30_pnl": round(c3pnl), + "realized_pnl": round(tally(brow(recent))[3]), "all_win": round(100 * all_won / (all_won + all_lost), 1) if (all_won + all_lost) else None, - "all_won": all_won, "all_lost": all_lost, "all_pnl": round(all_pnl), + "all_won": all_won, "all_lost": all_lost, "all_ref": all_ref, "all_pnl": round(all_pnl), "pm_pnl": _pm_profit(w), "avg_bet": round(sum(b["size"] for b in conv) / len(conv)) if conv else 0, "copy_pnl": 0, "held_pnl": 0, "held_won": 0, "held_lost": 0, "sold": 0, @@ -206,13 +237,18 @@ def display_stats(w): scalp += sh * pr - STAKE - openp[c]["fee"] - fee_out sold += 1; del openp[c] for c, p in openp.items(): # settle held bets at resolution - wv = resmap.get(p["a"]) + wv = payouts.truth(c, p["a"]) # chain first: refunds pay 0.5 + if wv is None: + wv = resmap.get(p["a"]) if wv is None: wv = _clob_winner(c, p["a"]) # clob fallback for out-of-pull markets if wv is None: continue # not resolved yet -> exclude - held += (p["sh"] if wv else 0) - STAKE - p["fee"] # redeem itself is fee-free - hw += wv; hl += 1 - wv + held += p["sh"] * wv - STAKE - p["fee"] # redeem itself is fee-free + if wv > 0.5: + hw += 1 + elif wv < 0.5: + hl += 1 out.update(copy_pnl=round(scalp + held), held_pnl=round(held), held_won=hw, held_lost=hl, sold=sold) return out @@ -281,8 +317,9 @@ def main(): # holders are judged on their real resolved sample (the replay's own held leg # is mostly "unresolved" for them and only reported for display). tr = trust.conviction_record(cache.query, c["wallet"], days=TRUST_DAYS, - pctile=cache.CONV_PCTILE) + pctile=cache.CONV_PCTILE, truthfn=payouts.truth) c["trust_n"], c["trust_wr"], c["trust_roi"] = tr["n"], round(tr["wr"], 3), round(tr["roi"], 3) + c["trust_refunds"] = tr.get("refunds", 0) # SELECT a copyable sharp: active, copy-positive (fee-aware replay), and a # genuine hold-to-resolution edge — trailing trusted conviction record wins a # clear majority with positive flat-stake ROI on a real sample, so the edge