paper-run realism: taker fees, resolved-only scoring, bounded collect, local runner

- copybot: model Polymarket taker fees (V2, since 2026-03-30: shares*rate*p*(1-p),
  sports 0.03) on every paper/live fill; track fees_paid in state + feed; settle
  P&L nets the entry fee. publish_feed now commits state + fills ledger with the
  feed (autostash rebase) so the local poller is the sole state writer.
- validate_timing: copy_pnl/held_pnl are fee-aware -> sharp selection now requires
  clearing real copy costs; conv stats + lead profile use resolved bets only.
- conviction_scan/skill: exclude res_t>now rows (early-sold positions in
  unresolved markets scored at curPrice - a mark, not an outcome; was ~5% of the
  June test window at a 72% pseudo-win rate).
- portfolio: skip unresolved rows (stake used to vanish from equity); missed bets
  of kind=open no longer KeyError - mark-to-market hypothetical P&L.
- collect: cap stale refreshes at STALE_CAP=2500/run (bulk-aged pool turned the
  daily refresh into a ~40h pull holding the DuckDB lock).
- Actions cron disabled: GitHub ran */5 every ~1.5-2.5h and the 10-min stale
  window skipped the rest -> 1 of ~104 qualifying buys copied. launchd --poll 60
  is now the runner; workflow stays as manual backstop.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

.github/workflows/copybot.yml

@@ -10,9 +10,16 @@ name: copybot-paper
 # not Actions.)
 
 on:
-  schedule:
-    - cron: "*/5 * * * *"      # GitHub may delay/skip under load; fine for ≥3h leads
-  workflow_dispatch: {}         # lets you run it manually from the Actions tab
+  # cron DISABLED 2026-07-01: GitHub throttled "*/5" to one run every ~1.5-2.5h in
+  # practice, and copybot skips any trade older than RECENT_TRADE_WINDOW_S (10 min)
+  # — measured result: 1 of ~104 qualifying conviction buys copied June 25-Jul 1.
+  # The launchd poller (com.jaxperro.copybot.plist, --poll 60) is now the sole
+  # runner; it commits state + feed + fills itself. Keep this workflow for manual
+  # runs only — re-enabling the cron alongside the local poller would fork the
+  # book's state.
+  # schedule:
+  #   - cron: "*/5 * * * *"
+  workflow_dispatch: {}         # manual backstop from the Actions tab
 
 permissions:
   contents: write               # GITHUB_TOKEN pushes the feed + state
@@ -44,6 +51,7 @@ jobs:
           git config user.name  "copybot[bot]"
           git config user.email "copybot@users.noreply.github.com"
           git add -f live/copybot_live.json copybot_state.json
+          git add -f copybot_fills.jsonl 2>/dev/null || true   # per-fill lag/slippage ledger
           if git diff --cached --quiet; then
             echo "no change this run"
             exit 0

copybot.py

@@ -75,6 +75,19 @@ FILL_LOG = "copybot_fills.jsonl"          # append-only ledger of every copy fil
 FEED = os.path.join("live", "copybot_live.json")   # published feed the trading dashboard reads
 FEED_PUSH_MIN_S = 120                     # min seconds between feed git-pushes (commit-on-change)
 
+# Polymarket taker fee (Fee Structure V2, live since 2026-03-30):
+#   fee = shares × rate × p × (1−p)
+# charged on marketable BUYs and SELLs (we always take — FOK/market copies);
+# redeeming a resolved position on-chain is fee-free. Rate is per market
+# category — sports 0.03, finance/politics/tech 0.04, econ/culture/weather 0.05,
+# crypto 0.07, geopolitics 0. The follow set is currently all-sports; override
+# with "taker_fee_rate" in config.json if that changes.
+TAKER_FEE_RATE = 0.03
+
+
+def taker_fee(shares, price, rate):
+    return shares * rate * price * (1.0 - price)
+
 
 def log(m):
     print(f"{time.strftime('%H:%M:%S')}  {m}", flush=True)
@@ -278,16 +291,24 @@ class Copybot:
         self.conds = engine.state.setdefault("conds", {})   # token_id -> conditionId (open positions)
         engine.state.setdefault("cash", cfg["bankroll_usd"])  # free cash (recycles on sell/resolution)
         engine.state.setdefault("lag", {"n": 0, "sum_s": 0.0, "sum_slip_pct": 0.0})
+        engine.state.setdefault("fees_paid", 0.0)
+        self.fee_rate = float(cfg.get("taker_fee_rate", TAKER_FEE_RATE))
 
     def _drain_fills(self):
         """Apply cash flows from any fills the engine just made; return the BUY
-        fills so the caller can log lag/slippage against the source trade."""
+        fills so the caller can log lag/slippage against the source trade.
+        Every marketable fill (buy or sell) pays the taker fee — in live mode the
+        protocol charges it at match time, so the paper book must charge it too or
+        it overstates the edge."""
         ex = self.engine.ex
         buys = []
         if hasattr(ex, "fills") and ex.fills:
             for f in ex.fills:
                 sign = -1 if f["side"] == "BUY" else 1
-                self.engine.state["cash"] += sign * f["shares"] * f["price"]
+                fee = taker_fee(f["shares"], f["price"], self.fee_rate)
+                f["fee"] = round(fee, 4)
+                self.engine.state["cash"] += sign * f["shares"] * f["price"] - fee
+                self.engine.state["fees_paid"] = self.engine.state.get("fees_paid", 0.0) + fee
                 if f["side"] == "BUY":
                     buys.append(f)
             ex.fills.clear()
@@ -311,6 +332,7 @@ class Copybot:
             "their_price": round(their_p, 4), "my_price": round(my_p, 4),
             "slippage_pct": round(slip_pct, 4),
             "shares": round(fill["shares"], 2), "cost": round(fill["shares"] * my_p, 2),
+            "fee": fill.get("fee", 0),
             "mode": "live" if self.engine.ex.live else "paper",
         }
         try:
@@ -331,6 +353,7 @@ class Copybot:
             "their_price": round(their_p, 4), "my_price": round(my_p, 4),
             "slippage_pct": round(slip_pct, 4),
             "shares": round(fill["shares"], 2), "cost": round(fill["shares"] * my_p, 2),
+            "fee": fill.get("fee", 0),
             "opened": int(their_ts or now), "status": "open",
             "exit_price": None, "pnl": None, "settled": None,
         }
@@ -357,6 +380,8 @@ class Copybot:
             "bankroll": bank, "stake": round(bank * self.cfg["bankroll_pct"], 2),
             "cash": round(cash, 2), "deployed": round(exp, 2),
             "realized": round(cash + exp - bank, 2), "open_count": len(mp),
+            "fees_paid": round(st.get("fees_paid", 0.0), 2),
+            "fee_rate": self.fee_rate,
             "lag": {"n": lag.get("n", 0),
                     "avg_s": round(lag["sum_s"] / lag["n"], 1) if lag.get("n") else None,
                     "avg_slip_pct": round(lag["sum_slip_pct"] / lag["n"], 4) if lag.get("n") else None},
@@ -380,9 +405,13 @@ class Copybot:
         os.replace(tmp, path)
 
     def publish_feed(self):
-        """Commit + push the feed to GitHub so the public dashboard can read it.
-        Throttled and commit-on-change, so pushes track betting activity, not the
-        poll rate. Best-effort — never crashes the run."""
+        """Commit + push the feed, the state file, and the fills ledger so (a) the
+        public dashboard reads the current book, (b) the book survives machine
+        loss, and (c) the per-fill lag/slippage evidence is preserved — the Actions
+        runner used to discard copybot_fills.jsonl on every run. Committing state
+        here (not just the feed) also keeps `git pull --rebase` from wedging on a
+        dirty tracked file now that this local poller is the sole runner.
+        Throttled and commit-on-change. Best-effort — never crashes the run."""
         st = self.engine.state
         now = time.time()
         if now - st.get("feed_pushed_at", 0) < FEED_PUSH_MIN_S:
@@ -391,13 +420,25 @@ class Copybot:
         try:
             import subprocess
             repo = self.here
-            c = subprocess.run(["git", "-C", repo, "commit", FEED, "-m",
+            # publish only when the FEED itself changed (a bet placed/settled) —
+            # the state file churns bookkeeping every cycle and would otherwise
+            # commit every FEED_PUSH_MIN_S forever.
+            if subprocess.run(["git", "-C", repo, "diff", "--quiet", "--", FEED],
+                              capture_output=True).returncode == 0:
+                return
+            paths = [p for p in (FEED, self.engine.state_path, FILL_LOG)
+                     if os.path.exists(os.path.join(repo, p))]
+            subprocess.run(["git", "-C", repo, "add", "-f"] + paths, capture_output=True)
+            if subprocess.run(["git", "-C", repo, "diff", "--cached", "--quiet"],
+                              capture_output=True).returncode == 0:
+                return                                  # nothing changed
+            c = subprocess.run(["git", "-C", repo, "commit", "-q", "-m",
                                 "copybot: live paper feed [skip ci]"],
                                capture_output=True, text=True)
             if c.returncode != 0:
-                return                                  # nothing changed
-            subprocess.run(["git", "-C", repo, "pull", "--rebase", "-q", "origin", "main"],
-                           capture_output=True)
+                return
+            subprocess.run(["git", "-C", repo, "pull", "--rebase", "--autostash", "-q",
+                            "origin", "main"], capture_output=True)
             p = subprocess.run(["git", "-C", repo, "push", "-q", "origin", "main"],
                                capture_output=True, text=True)
             log("published live feed → dashboard" if p.returncode == 0
@@ -504,10 +545,11 @@ class Copybot:
                             log(f"  ⚠ redeem failed ({info}) — keeping position, will retry")
                             continue
                         log(f"  ↳ redeemed on-chain: {info}")
-                proceeds = pos["shares"] * wp
-                pnl = proceeds - pos["cost"]
-                self.engine.state["cash"] += proceeds   # recycle freed capital
+                proceeds = pos["shares"] * wp           # redeem is fee-free on-chain
                 b = self.engine.state.get("bets", {}).get(token)
+                fee_in = (b or {}).get("fee") or 0      # entry taker fee, already off cash
+                pnl = proceeds - pos["cost"] - fee_in
+                self.engine.state["cash"] += proceeds   # recycle freed capital
                 if b:
                     b.update(status=("won" if wp >= 0.5 else "lost"),
                              exit_price=wp, pnl=round(pnl, 2), settled=int(time.time()))

live/cache.py

@@ -113,6 +113,13 @@ def invalidate(wallets):
             _con.execute("DELETE FROM pulled WHERE wallet=?", [w])
 
 
+def pulled_ages():
+    """{wallet: pulled_at} for every wallet ever pulled — lets collect.py bound
+    how many stale re-pulls one run takes on."""
+    with _lock:
+        return dict(_con.execute("SELECT wallet, pulled_at FROM pulled").fetchall())
+
+
 def stats():
     with _lock:
         w = _con.execute("SELECT count(*) FROM pulled").fetchone()[0]

live/collect.py

@@ -19,13 +19,27 @@ import cache
 
 HERE = os.path.dirname(__file__)
 WORKERS = 16
+# Bound each run: every never-pulled wallet is collected, but at most STALE_CAP of
+# the already-cached ones are refreshed (stalest first). Without the cap, the day
+# the bulk-ingested pool crosses MAX_AGE_DAYS together, a "daily" run balloons into
+# a ~40h re-pull that blocks every scoring step behind it in daily.sh and holds the
+# DuckDB write lock (even read_only connections fail) all day. At 2,500/day the
+# whole pool still turns over well inside the 14-day freshness window.
+STALE_CAP = int(os.environ.get("STALE_CAP", 2500))
 
 
 def main():
     cands = json.load(open(os.path.join(HERE, "candidates.json")))
     cands.sort(key=lambda c: c.get("markets_seen", 0), reverse=True)
-    wallets = [c["wallet"] for c in cands]
-    print(f"collecting {len(wallets):,} wallets up to present · {WORKERS} workers", flush=True)
+    ages = cache.pulled_ages()
+    fresh_cut = time.time() - cache.MAX_AGE_DAYS * 86400
+    new = [c["wallet"] for c in cands if c["wallet"] not in ages]
+    stale = sorted((c["wallet"] for c in cands
+                    if 0 < ages.get(c["wallet"], 0) < fresh_cut), key=ages.get)
+    wallets = new + stale[:STALE_CAP]
+    print(f"collecting {len(wallets):,} wallets ({len(new):,} new + "
+          f"{len(stale[:STALE_CAP]):,} of {len(stale):,} stale, cap {STALE_CAP}) · "
+          f"{WORKERS} workers", flush=True)
     done, t0 = 0, time.time()
     with ThreadPoolExecutor(max_workers=WORKERS) as ex:
         futs = [ex.submit(cache.get_bets, w) for w in wallets]

live/conviction_scan.py

@@ -46,12 +46,17 @@ def main():
     con = duckdb.connect(os.path.join(HERE, "cache.duckdb"), read_only=True)
     # per-wallet conviction cutoff = p80 of that wallet's own positive stakes, then
     # keep only its bets at/above that cutoff (its top ~20% by size).
+    # res_t <= now: the cache stores early-sold positions in UNRESOLVED markets with
+    # a future res_t and won = curPrice at pull time — a mark, not an outcome. They
+    # were ~5% of the June test window with a 72% pseudo-"win" rate, inflating the
+    # forward validation; only actually-resolved bets may score.
     rows = con.execute(
         "WITH thr AS (SELECT wallet, quantile_cont(size, ?) AS t "
         "             FROM bets WHERE size > 0 GROUP BY wallet) "
         "SELECT b.wallet, b.p, b.won, b.res_t "
         "FROM bets b JOIN thr ON b.wallet = thr.wallet "
-        "WHERE b.size > 0 AND b.size >= thr.t", [CONV_PCTILE]).fetchall()
+        "WHERE b.size > 0 AND b.size >= thr.t AND b.res_t <= ?",
+        [CONV_PCTILE, int(time.time())]).fetchall()
     byw = {}
     for w, p, won, rt in rows:
         byw.setdefault(w, []).append((max(0.001, min(0.999, p or 0)), won, rt or 0))

live/portfolio.py

@@ -60,6 +60,7 @@ def market_meta(cond):
 def conviction_bets():
     """Every followed wallet's resolved conviction bets from the cache, with entry time."""
     out = []
+    now = time.time()
     for w in WALLETS:
         ent = cache.get_entries(w["wallet"])               # cond -> first buy ts
         bets = [b for b in cache.get_bets(w["wallet"]) if (b["size"] or 0) > 0]
@@ -67,8 +68,15 @@ def conviction_bets():
         for b in bets:
             if b["size"] < thr:
                 continue
+            if (b["res_t"] or 0) > now:
+                # unresolved market (early-sold position): won is a curPrice mark,
+                # not an outcome — and a future res_t would never free its stake
+                # (cash out at entry, freed at res_t > now, absent from `invested`
+                # = equity silently loses $STAKE). The live /positions pull is the
+                # source for genuinely-open bets.
+                continue
             et = ent.get(b["cond"])
-            if not et or et < START:                       # only June 1+ entries
+            if not et or et < START:                       # only post-START entries
                 continue
             out.append({"wallet": w["wallet"], "name": w["name"], "cond": b["cond"],
                         "entry_t": et, "p": max(0.001, min(0.999, b["p"] or 0)),
@@ -159,11 +167,20 @@ def main():
     resolved.sort(key=lambda r: r.get("res_t") or 0, reverse=True)
     for r in resolved[:60]:
         m = market_meta(r["cond"]); r["title"] = m["title"]
+    # hypothetical P&L had we been able to afford it: resolved bets at their
+    # outcome, still-open bets marked to the current price. Missed bets can be
+    # kind=="open" (no "won"/"res_t" keys) — indexing m["won"] here used to
+    # KeyError and kill the whole portfolio step the first time capital ran out
+    # while a followed wallet had a live position.
+    def hypo_pnl(m):
+        if "won" in m:
+            return STAKE * ((1.0 / m["p"]) - 1) if m["won"] else -STAKE
+        return STAKE * (m.get("cur", m["p"]) / m["p"] - 1)
+
     missed.sort(key=lambda m: m.get("res_t") or 0, reverse=True)
     for m in missed[:60]:
         m["title"] = market_meta(m["cond"])["title"]
-        # hypothetical P&L had we been able to afford it (held to resolution)
-        m["pnl"] = STAKE * ((1.0 / m["p"]) - 1) if m["won"] else -STAKE
+        m["pnl"] = hypo_pnl(m)
     wins = sum(1 for r in resolved if r.get("won"))
     # per-wallet conviction threshold (cache p80) so the dashboard can filter LIVE open
     # positions the same way; 1e12 = "no sized bets" (nothing qualifies)
@@ -190,10 +207,10 @@ def main():
         "resolved": [{"title": r.get("title", ""), "name": r["name"], "won": r["won"],
                       "stake": STAKE, "pnl": round(r["pnl"], 2), "date": r.get("res_t")}
                      for r in resolved[:60]],
-        "missed": [{"title": m.get("title", ""), "name": m["name"], "won": m["won"],
+        "missed": [{"title": m.get("title", ""), "name": m["name"], "won": m.get("won"),
                     "stake": STAKE, "pnl": round(m["pnl"], 2), "date": m.get("res_t")}
                    for m in missed[:60]],
-        "missed_pnl": round(sum(STAKE * ((1.0 / m["p"]) - 1) if m["won"] else -STAKE for m in missed), 2),
+        "missed_pnl": round(sum(hypo_pnl(m) for m in missed), 2),
     }
     json.dump(out, open(os.path.join(HERE, "portfolio.json"), "w"), separators=(",", ":"))
     print(f"portfolio: equity ${equity:,.0f} ({(equity-BANK)/BANK*100:+.0f}%) | realized ${realized:+,.0f} "

live/skill.py

@@ -59,6 +59,11 @@ def zstats(bets):
 
 def score_wallet(c):
     bets = cache.get_bets(c["wallet"])   # cached — pulls the data-api only once per wallet
+    # resolved only: the cache stores early-sold positions in unresolved markets
+    # with res_t in the future and won = curPrice at pull time (a mark, not an
+    # outcome) — they must not count toward z.
+    now_t = time.time()
+    bets = [b for b in bets if (b.get("res_t") or 0) <= now_t]
     if BEFORE:                       # clean OOS: only bets resolved before cutoff
         bets = [b for b in bets if (b.get("res_t") or 0) < BEFORE]
     n = len(bets)

live/validate_timing.py

@@ -30,6 +30,12 @@ HERE = os.path.dirname(__file__)
 COPYABLE_MED_LEAD = 24.0     # median lead (h) on winning conviction bets to count as copyable
 JUN1 = time.mktime(time.strptime("2026-06-01", "%Y-%m-%d"))   # portfolio copy-start
 STAKE = 50.0                 # flat $/trade the copy portfolio uses
+# Polymarket taker fee (since 2026-03-30): fee = shares·rate·p·(1−p), paid on
+# marketable entries AND mirror exits; redeeming at resolution is free. 0.03 is
+# the sports rate (the follow set's category). Making copy_pnl fee-aware makes
+# the SELECTION fee-aware — a wallet only counts as a copyable sharp if copying
+# it clears the fees a real copier pays.
+FEE_RATE = 0.03
 _SSL = ssl._create_unverified_context()
 _CLOB = {}                   # conditionId -> {token_id: winner-price 1/0/None}
 
@@ -72,8 +78,12 @@ def display_stats(w):
                              (e.g. ArbTrader: ~100% conv win but −$790 copy P&L).
       name / last-bet      : from the /activity pull
     """
-    # ---- position win%/record/P&L from the cache (large, survivorship-corrected) ----
-    bets = [b for b in cache.get_bets(w) if (b["size"] or 0) > 0]
+    # ---- position win%/record/P&L from the cache (large, survivorship-corrected).
+    # res_t <= now: the cache stores early-sold positions in UNRESOLVED markets with
+    # a future res_t and won = current price — a mark, not an outcome; skip them. ----
+    now = time.time()
+    bets = [b for b in cache.get_bets(w)
+            if (b["size"] or 0) > 0 and (b["res_t"] or 0) <= now]
     thr = cache.conv_cutoff(b["size"] for b in bets)
     conv = [b for b in bets if b["size"] >= thr]
     won = sum(1 for b in conv if b["won"])
@@ -136,16 +146,20 @@ def display_stats(w):
             if t.get("side") == "BUY":
                 if mkt.get(c, 0) < cthr or c in entered or c in openp:
                     continue
-                entered.add(c); openp[c] = {"sh": STAKE / pr, "a": asset}
+                fee_in = STAKE * FEE_RATE * (1 - pr)            # taker fee on the entry
+                entered.add(c); openp[c] = {"sh": STAKE / pr, "a": asset, "fee": fee_in}
             elif c in openp:                                    # mirror their exit (scalp)
-                scalp += openp[c]["sh"] * pr - STAKE; sold += 1; del openp[c]
+                sh = openp[c]["sh"]
+                fee_out = sh * FEE_RATE * pr * (1 - pr)         # taker fee on the exit too
+                scalp += sh * pr - STAKE - openp[c]["fee"] - fee_out
+                sold += 1; del openp[c]
         for c, p in openp.items():                              # settle held bets at resolution
             wv = resmap.get(p["a"])
             if wv is None:
                 wv = _clob_winner(c, p["a"])                    # clob fallback for out-of-pull markets
             if wv is None:
                 continue                                        # not resolved yet -> exclude
-            held += (p["sh"] if wv else 0) - STAKE
+            held += (p["sh"] if wv else 0) - STAKE - p["fee"]   # redeem itself is fee-free
             hw += wv; hl += 1 - wv
         out.update(copy_pnl=round(scalp + held), held_pnl=round(held),
                    held_won=hw, held_lost=hl, sold=sold)
@@ -154,7 +168,8 @@ def display_stats(w):
 
 def lead_profile(w):
     ent = cache.get_entries(w)
-    bets = cache.get_bets(w)
+    now = time.time()
+    bets = [b for b in cache.get_bets(w) if (b["res_t"] or 0) <= now]  # resolved only
     cut = cache.conv_cutoff(b["size"] for b in bets)   # this wallet's top-20% stake cutoff
     leads = [(b["res_t"] - ent[b["cond"]]) / 3600.0 for b in bets
              if b["won"] and (b["size"] or 0) >= cut and b["cond"] in ent