diff --git a/.github/workflows/copybot.yml b/.github/workflows/copybot.yml index f9fe7723..73dd422a 100644 --- a/.github/workflows/copybot.yml +++ b/.github/workflows/copybot.yml @@ -10,9 +10,16 @@ name: copybot-paper # not Actions.) on: - schedule: - - cron: "*/5 * * * *" # GitHub may delay/skip under load; fine for ≥3h leads - workflow_dispatch: {} # lets you run it manually from the Actions tab + # cron DISABLED 2026-07-01: GitHub throttled "*/5" to one run every ~1.5-2.5h in + # practice, and copybot skips any trade older than RECENT_TRADE_WINDOW_S (10 min) + # — measured result: 1 of ~104 qualifying conviction buys copied June 25-Jul 1. + # The launchd poller (com.jaxperro.copybot.plist, --poll 60) is now the sole + # runner; it commits state + feed + fills itself. Keep this workflow for manual + # runs only — re-enabling the cron alongside the local poller would fork the + # book's state. + # schedule: + # - cron: "*/5 * * * *" + workflow_dispatch: {} # manual backstop from the Actions tab permissions: contents: write # GITHUB_TOKEN pushes the feed + state @@ -44,6 +51,7 @@ jobs: git config user.name "copybot[bot]" git config user.email "copybot@users.noreply.github.com" git add -f live/copybot_live.json copybot_state.json + git add -f copybot_fills.jsonl 2>/dev/null || true # per-fill lag/slippage ledger if git diff --cached --quiet; then echo "no change this run" exit 0 diff --git a/copybot.py b/copybot.py index 601613cd..187d0986 100644 --- a/copybot.py +++ b/copybot.py @@ -75,6 +75,19 @@ FILL_LOG = "copybot_fills.jsonl" # append-only ledger of every copy fil FEED = os.path.join("live", "copybot_live.json") # published feed the trading dashboard reads FEED_PUSH_MIN_S = 120 # min seconds between feed git-pushes (commit-on-change) +# Polymarket taker fee (Fee Structure V2, live since 2026-03-30): +# fee = shares × rate × p × (1−p) +# charged on marketable BUYs and SELLs (we always take — FOK/market copies); +# redeeming a resolved position on-chain is fee-free. Rate is per market +# category — sports 0.03, finance/politics/tech 0.04, econ/culture/weather 0.05, +# crypto 0.07, geopolitics 0. The follow set is currently all-sports; override +# with "taker_fee_rate" in config.json if that changes. +TAKER_FEE_RATE = 0.03 + + +def taker_fee(shares, price, rate): + return shares * rate * price * (1.0 - price) + def log(m): print(f"{time.strftime('%H:%M:%S')} {m}", flush=True) @@ -278,16 +291,24 @@ class Copybot: self.conds = engine.state.setdefault("conds", {}) # token_id -> conditionId (open positions) engine.state.setdefault("cash", cfg["bankroll_usd"]) # free cash (recycles on sell/resolution) engine.state.setdefault("lag", {"n": 0, "sum_s": 0.0, "sum_slip_pct": 0.0}) + engine.state.setdefault("fees_paid", 0.0) + self.fee_rate = float(cfg.get("taker_fee_rate", TAKER_FEE_RATE)) def _drain_fills(self): """Apply cash flows from any fills the engine just made; return the BUY - fills so the caller can log lag/slippage against the source trade.""" + fills so the caller can log lag/slippage against the source trade. + Every marketable fill (buy or sell) pays the taker fee — in live mode the + protocol charges it at match time, so the paper book must charge it too or + it overstates the edge.""" ex = self.engine.ex buys = [] if hasattr(ex, "fills") and ex.fills: for f in ex.fills: sign = -1 if f["side"] == "BUY" else 1 - self.engine.state["cash"] += sign * f["shares"] * f["price"] + fee = taker_fee(f["shares"], f["price"], self.fee_rate) + f["fee"] = round(fee, 4) + self.engine.state["cash"] += sign * f["shares"] * f["price"] - fee + self.engine.state["fees_paid"] = self.engine.state.get("fees_paid", 0.0) + fee if f["side"] == "BUY": buys.append(f) ex.fills.clear() @@ -311,6 +332,7 @@ class Copybot: "their_price": round(their_p, 4), "my_price": round(my_p, 4), "slippage_pct": round(slip_pct, 4), "shares": round(fill["shares"], 2), "cost": round(fill["shares"] * my_p, 2), + "fee": fill.get("fee", 0), "mode": "live" if self.engine.ex.live else "paper", } try: @@ -331,6 +353,7 @@ class Copybot: "their_price": round(their_p, 4), "my_price": round(my_p, 4), "slippage_pct": round(slip_pct, 4), "shares": round(fill["shares"], 2), "cost": round(fill["shares"] * my_p, 2), + "fee": fill.get("fee", 0), "opened": int(their_ts or now), "status": "open", "exit_price": None, "pnl": None, "settled": None, } @@ -357,6 +380,8 @@ class Copybot: "bankroll": bank, "stake": round(bank * self.cfg["bankroll_pct"], 2), "cash": round(cash, 2), "deployed": round(exp, 2), "realized": round(cash + exp - bank, 2), "open_count": len(mp), + "fees_paid": round(st.get("fees_paid", 0.0), 2), + "fee_rate": self.fee_rate, "lag": {"n": lag.get("n", 0), "avg_s": round(lag["sum_s"] / lag["n"], 1) if lag.get("n") else None, "avg_slip_pct": round(lag["sum_slip_pct"] / lag["n"], 4) if lag.get("n") else None}, @@ -380,9 +405,13 @@ class Copybot: os.replace(tmp, path) def publish_feed(self): - """Commit + push the feed to GitHub so the public dashboard can read it. - Throttled and commit-on-change, so pushes track betting activity, not the - poll rate. Best-effort — never crashes the run.""" + """Commit + push the feed, the state file, and the fills ledger so (a) the + public dashboard reads the current book, (b) the book survives machine + loss, and (c) the per-fill lag/slippage evidence is preserved — the Actions + runner used to discard copybot_fills.jsonl on every run. Committing state + here (not just the feed) also keeps `git pull --rebase` from wedging on a + dirty tracked file now that this local poller is the sole runner. + Throttled and commit-on-change. Best-effort — never crashes the run.""" st = self.engine.state now = time.time() if now - st.get("feed_pushed_at", 0) < FEED_PUSH_MIN_S: @@ -391,13 +420,25 @@ class Copybot: try: import subprocess repo = self.here - c = subprocess.run(["git", "-C", repo, "commit", FEED, "-m", + # publish only when the FEED itself changed (a bet placed/settled) — + # the state file churns bookkeeping every cycle and would otherwise + # commit every FEED_PUSH_MIN_S forever. + if subprocess.run(["git", "-C", repo, "diff", "--quiet", "--", FEED], + capture_output=True).returncode == 0: + return + paths = [p for p in (FEED, self.engine.state_path, FILL_LOG) + if os.path.exists(os.path.join(repo, p))] + subprocess.run(["git", "-C", repo, "add", "-f"] + paths, capture_output=True) + if subprocess.run(["git", "-C", repo, "diff", "--cached", "--quiet"], + capture_output=True).returncode == 0: + return # nothing changed + c = subprocess.run(["git", "-C", repo, "commit", "-q", "-m", "copybot: live paper feed [skip ci]"], capture_output=True, text=True) if c.returncode != 0: - return # nothing changed - subprocess.run(["git", "-C", repo, "pull", "--rebase", "-q", "origin", "main"], - capture_output=True) + return + subprocess.run(["git", "-C", repo, "pull", "--rebase", "--autostash", "-q", + "origin", "main"], capture_output=True) p = subprocess.run(["git", "-C", repo, "push", "-q", "origin", "main"], capture_output=True, text=True) log("published live feed → dashboard" if p.returncode == 0 @@ -504,10 +545,11 @@ class Copybot: log(f" ⚠ redeem failed ({info}) — keeping position, will retry") continue log(f" ↳ redeemed on-chain: {info}") - proceeds = pos["shares"] * wp - pnl = proceeds - pos["cost"] - self.engine.state["cash"] += proceeds # recycle freed capital + proceeds = pos["shares"] * wp # redeem is fee-free on-chain b = self.engine.state.get("bets", {}).get(token) + fee_in = (b or {}).get("fee") or 0 # entry taker fee, already off cash + pnl = proceeds - pos["cost"] - fee_in + self.engine.state["cash"] += proceeds # recycle freed capital if b: b.update(status=("won" if wp >= 0.5 else "lost"), exit_price=wp, pnl=round(pnl, 2), settled=int(time.time())) diff --git a/live/cache.py b/live/cache.py index 2fe16f7b..00b53221 100644 --- a/live/cache.py +++ b/live/cache.py @@ -113,6 +113,13 @@ def invalidate(wallets): _con.execute("DELETE FROM pulled WHERE wallet=?", [w]) +def pulled_ages(): + """{wallet: pulled_at} for every wallet ever pulled — lets collect.py bound + how many stale re-pulls one run takes on.""" + with _lock: + return dict(_con.execute("SELECT wallet, pulled_at FROM pulled").fetchall()) + + def stats(): with _lock: w = _con.execute("SELECT count(*) FROM pulled").fetchone()[0] diff --git a/live/collect.py b/live/collect.py index b44a46bb..f83a39c3 100644 --- a/live/collect.py +++ b/live/collect.py @@ -19,13 +19,27 @@ import cache HERE = os.path.dirname(__file__) WORKERS = 16 +# Bound each run: every never-pulled wallet is collected, but at most STALE_CAP of +# the already-cached ones are refreshed (stalest first). Without the cap, the day +# the bulk-ingested pool crosses MAX_AGE_DAYS together, a "daily" run balloons into +# a ~40h re-pull that blocks every scoring step behind it in daily.sh and holds the +# DuckDB write lock (even read_only connections fail) all day. At 2,500/day the +# whole pool still turns over well inside the 14-day freshness window. +STALE_CAP = int(os.environ.get("STALE_CAP", 2500)) def main(): cands = json.load(open(os.path.join(HERE, "candidates.json"))) cands.sort(key=lambda c: c.get("markets_seen", 0), reverse=True) - wallets = [c["wallet"] for c in cands] - print(f"collecting {len(wallets):,} wallets up to present · {WORKERS} workers", flush=True) + ages = cache.pulled_ages() + fresh_cut = time.time() - cache.MAX_AGE_DAYS * 86400 + new = [c["wallet"] for c in cands if c["wallet"] not in ages] + stale = sorted((c["wallet"] for c in cands + if 0 < ages.get(c["wallet"], 0) < fresh_cut), key=ages.get) + wallets = new + stale[:STALE_CAP] + print(f"collecting {len(wallets):,} wallets ({len(new):,} new + " + f"{len(stale[:STALE_CAP]):,} of {len(stale):,} stale, cap {STALE_CAP}) · " + f"{WORKERS} workers", flush=True) done, t0 = 0, time.time() with ThreadPoolExecutor(max_workers=WORKERS) as ex: futs = [ex.submit(cache.get_bets, w) for w in wallets] diff --git a/live/conviction_scan.py b/live/conviction_scan.py index 215fc712..8db721b0 100644 --- a/live/conviction_scan.py +++ b/live/conviction_scan.py @@ -46,12 +46,17 @@ def main(): con = duckdb.connect(os.path.join(HERE, "cache.duckdb"), read_only=True) # per-wallet conviction cutoff = p80 of that wallet's own positive stakes, then # keep only its bets at/above that cutoff (its top ~20% by size). + # res_t <= now: the cache stores early-sold positions in UNRESOLVED markets with + # a future res_t and won = curPrice at pull time — a mark, not an outcome. They + # were ~5% of the June test window with a 72% pseudo-"win" rate, inflating the + # forward validation; only actually-resolved bets may score. rows = con.execute( "WITH thr AS (SELECT wallet, quantile_cont(size, ?) AS t " " FROM bets WHERE size > 0 GROUP BY wallet) " "SELECT b.wallet, b.p, b.won, b.res_t " "FROM bets b JOIN thr ON b.wallet = thr.wallet " - "WHERE b.size > 0 AND b.size >= thr.t", [CONV_PCTILE]).fetchall() + "WHERE b.size > 0 AND b.size >= thr.t AND b.res_t <= ?", + [CONV_PCTILE, int(time.time())]).fetchall() byw = {} for w, p, won, rt in rows: byw.setdefault(w, []).append((max(0.001, min(0.999, p or 0)), won, rt or 0)) diff --git a/live/portfolio.py b/live/portfolio.py index 733f487f..179f8049 100644 --- a/live/portfolio.py +++ b/live/portfolio.py @@ -60,6 +60,7 @@ def market_meta(cond): def conviction_bets(): """Every followed wallet's resolved conviction bets from the cache, with entry time.""" out = [] + now = time.time() for w in WALLETS: ent = cache.get_entries(w["wallet"]) # cond -> first buy ts bets = [b for b in cache.get_bets(w["wallet"]) if (b["size"] or 0) > 0] @@ -67,8 +68,15 @@ def conviction_bets(): for b in bets: if b["size"] < thr: continue + if (b["res_t"] or 0) > now: + # unresolved market (early-sold position): won is a curPrice mark, + # not an outcome — and a future res_t would never free its stake + # (cash out at entry, freed at res_t > now, absent from `invested` + # = equity silently loses $STAKE). The live /positions pull is the + # source for genuinely-open bets. + continue et = ent.get(b["cond"]) - if not et or et < START: # only June 1+ entries + if not et or et < START: # only post-START entries continue out.append({"wallet": w["wallet"], "name": w["name"], "cond": b["cond"], "entry_t": et, "p": max(0.001, min(0.999, b["p"] or 0)), @@ -159,11 +167,20 @@ def main(): resolved.sort(key=lambda r: r.get("res_t") or 0, reverse=True) for r in resolved[:60]: m = market_meta(r["cond"]); r["title"] = m["title"] + # hypothetical P&L had we been able to afford it: resolved bets at their + # outcome, still-open bets marked to the current price. Missed bets can be + # kind=="open" (no "won"/"res_t" keys) — indexing m["won"] here used to + # KeyError and kill the whole portfolio step the first time capital ran out + # while a followed wallet had a live position. + def hypo_pnl(m): + if "won" in m: + return STAKE * ((1.0 / m["p"]) - 1) if m["won"] else -STAKE + return STAKE * (m.get("cur", m["p"]) / m["p"] - 1) + missed.sort(key=lambda m: m.get("res_t") or 0, reverse=True) for m in missed[:60]: m["title"] = market_meta(m["cond"])["title"] - # hypothetical P&L had we been able to afford it (held to resolution) - m["pnl"] = STAKE * ((1.0 / m["p"]) - 1) if m["won"] else -STAKE + m["pnl"] = hypo_pnl(m) wins = sum(1 for r in resolved if r.get("won")) # per-wallet conviction threshold (cache p80) so the dashboard can filter LIVE open # positions the same way; 1e12 = "no sized bets" (nothing qualifies) @@ -190,10 +207,10 @@ def main(): "resolved": [{"title": r.get("title", ""), "name": r["name"], "won": r["won"], "stake": STAKE, "pnl": round(r["pnl"], 2), "date": r.get("res_t")} for r in resolved[:60]], - "missed": [{"title": m.get("title", ""), "name": m["name"], "won": m["won"], + "missed": [{"title": m.get("title", ""), "name": m["name"], "won": m.get("won"), "stake": STAKE, "pnl": round(m["pnl"], 2), "date": m.get("res_t")} for m in missed[:60]], - "missed_pnl": round(sum(STAKE * ((1.0 / m["p"]) - 1) if m["won"] else -STAKE for m in missed), 2), + "missed_pnl": round(sum(hypo_pnl(m) for m in missed), 2), } json.dump(out, open(os.path.join(HERE, "portfolio.json"), "w"), separators=(",", ":")) print(f"portfolio: equity ${equity:,.0f} ({(equity-BANK)/BANK*100:+.0f}%) | realized ${realized:+,.0f} " diff --git a/live/skill.py b/live/skill.py index ab9814f6..78abe43b 100644 --- a/live/skill.py +++ b/live/skill.py @@ -59,6 +59,11 @@ def zstats(bets): def score_wallet(c): bets = cache.get_bets(c["wallet"]) # cached — pulls the data-api only once per wallet + # resolved only: the cache stores early-sold positions in unresolved markets + # with res_t in the future and won = curPrice at pull time (a mark, not an + # outcome) — they must not count toward z. + now_t = time.time() + bets = [b for b in bets if (b.get("res_t") or 0) <= now_t] if BEFORE: # clean OOS: only bets resolved before cutoff bets = [b for b in bets if (b.get("res_t") or 0) < BEFORE] n = len(bets) diff --git a/live/validate_timing.py b/live/validate_timing.py index 95089958..42125a53 100644 --- a/live/validate_timing.py +++ b/live/validate_timing.py @@ -30,6 +30,12 @@ HERE = os.path.dirname(__file__) COPYABLE_MED_LEAD = 24.0 # median lead (h) on winning conviction bets to count as copyable JUN1 = time.mktime(time.strptime("2026-06-01", "%Y-%m-%d")) # portfolio copy-start STAKE = 50.0 # flat $/trade the copy portfolio uses +# Polymarket taker fee (since 2026-03-30): fee = shares·rate·p·(1−p), paid on +# marketable entries AND mirror exits; redeeming at resolution is free. 0.03 is +# the sports rate (the follow set's category). Making copy_pnl fee-aware makes +# the SELECTION fee-aware — a wallet only counts as a copyable sharp if copying +# it clears the fees a real copier pays. +FEE_RATE = 0.03 _SSL = ssl._create_unverified_context() _CLOB = {} # conditionId -> {token_id: winner-price 1/0/None} @@ -72,8 +78,12 @@ def display_stats(w): (e.g. ArbTrader: ~100% conv win but −$790 copy P&L). name / last-bet : from the /activity pull """ - # ---- position win%/record/P&L from the cache (large, survivorship-corrected) ---- - bets = [b for b in cache.get_bets(w) if (b["size"] or 0) > 0] + # ---- position win%/record/P&L from the cache (large, survivorship-corrected). + # res_t <= now: the cache stores early-sold positions in UNRESOLVED markets with + # a future res_t and won = current price — a mark, not an outcome; skip them. ---- + now = time.time() + bets = [b for b in cache.get_bets(w) + if (b["size"] or 0) > 0 and (b["res_t"] or 0) <= now] thr = cache.conv_cutoff(b["size"] for b in bets) conv = [b for b in bets if b["size"] >= thr] won = sum(1 for b in conv if b["won"]) @@ -136,16 +146,20 @@ def display_stats(w): if t.get("side") == "BUY": if mkt.get(c, 0) < cthr or c in entered or c in openp: continue - entered.add(c); openp[c] = {"sh": STAKE / pr, "a": asset} + fee_in = STAKE * FEE_RATE * (1 - pr) # taker fee on the entry + entered.add(c); openp[c] = {"sh": STAKE / pr, "a": asset, "fee": fee_in} elif c in openp: # mirror their exit (scalp) - scalp += openp[c]["sh"] * pr - STAKE; sold += 1; del openp[c] + sh = openp[c]["sh"] + fee_out = sh * FEE_RATE * pr * (1 - pr) # taker fee on the exit too + scalp += sh * pr - STAKE - openp[c]["fee"] - fee_out + sold += 1; del openp[c] for c, p in openp.items(): # settle held bets at resolution wv = resmap.get(p["a"]) if wv is None: wv = _clob_winner(c, p["a"]) # clob fallback for out-of-pull markets if wv is None: continue # not resolved yet -> exclude - held += (p["sh"] if wv else 0) - STAKE + held += (p["sh"] if wv else 0) - STAKE - p["fee"] # redeem itself is fee-free hw += wv; hl += 1 - wv out.update(copy_pnl=round(scalp + held), held_pnl=round(held), held_won=hw, held_lost=hl, sold=sold) @@ -154,7 +168,8 @@ def display_stats(w): def lead_profile(w): ent = cache.get_entries(w) - bets = cache.get_bets(w) + now = time.time() + bets = [b for b in cache.get_bets(w) if (b["res_t"] or 0) <= now] # resolved only cut = cache.conv_cutoff(b["size"] for b in bets) # this wallet's top-20% stake cutoff leads = [(b["res_t"] - ent[b["cond"]]) / 3600.0 for b in bets if b["won"] and (b["size"] or 0) >= cut and b["cond"] in ent