From baa41823435d3bde5397d873a901d8b5ae32f562 Mon Sep 17 00:00:00 2001 From: jaxperro Date: Sun, 19 Jul 2026 14:07:41 -0400 Subject: [PATCH] audit hardening: book fully serialized, chain-gated sweep, boot-id single-writer guard, TLS on user-ws, atomic valuebot state, hot-path throttle, newest-kept seen_tx, daily lock, ingest txn, AIcAIc pin Co-Authored-By: Claude Fable 5 --- HANDOFF.md | 22 +++++++++++++++ config.live.example.json | 3 +- copybot.py | 61 ++++++++++++++++++++++++++++++++++++---- copytrade.py | 26 +++++++++++++++-- live/daily.sh | 6 +++- recorder/ingest.py | 4 ++- tests/test_dust_sweep.py | 4 +++ value/valuebot.py | 13 +++++++-- 8 files changed, 126 insertions(+), 13 deletions(-) diff --git a/HANDOFF.md b/HANDOFF.md index 591aa11c..e8d957f9 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -1,5 +1,27 @@ # Session handoff — 2026-07-19 (rev 15: tape+bench+gate; rev 14: +$44 bankroll, exact paper-parity config; rev 13: FAK parity; rev 12: chain-seed) +## 2026-07-19 (rev 16): audit hardening shipped (top-10 items 1-9) +Deep-audit fixes, all deployed same day: (1) the book is now fully +serialized — resolve_pendings/retry_stuck_exits/write_feed take bot.lock, +_drain_fills snapshot-swaps the shared fills list (kills the last July-12 +phantom-cash race class); (2) sweep_dust gates every action on CHAIN share +balance and credits proportionally (data-api ghosts book ~$0 forever); +(4) BOOT-ID single-writer guard — a stale process that survived machine-stop +sees a newer boot in origin's state at publish time and EXITS instead of +overwriting surgery (gotcha 15c is now an invariant, not a procedure); +(5) user-ws verifies TLS (it carries L2 creds; RTDS stays CERT_NONE — public +data); (6) valuebot state is tmp+rename atomic with a loud ⚠ STATE RESET +alarm; (7) settle_resolved throttled 20s off the webhook hot path (faster +copies in bursts); (8) daily.sh mkdir-lock (macOS has no flock) + ingest's +two inserts are one transaction; (9) seen_tx keeps the NEWEST 5000 via a +recency dict (was an arbitrary set slice); (10) AIcAIc live floor pinned to +paper's 341.23 (was boot-nondeterministic, violating mirror-exactly). +STILL QUEUED from the audit: redeem.py pUSD/deposit-wallet fix before any +auto_redeem re-enable (audit 3.3 — it would book phantom credits today); +listener-start-after-seed reorder (3.9); dead-code sweep (copytrade +LiveExecutor + broken CLI, _book_snapshot dedupe); reconcile_entries +positions-pull diet; sftp tape transport; rpc-down heartbeat flag. + ## 2026-07-19 (rev 15): the rest of the day - **RTDS TAPE live end-to-end**: wwf-recorder (own silo, no repo clone — code baked in image) tapes every platform trade to hour-rotated gz diff --git a/config.live.example.json b/config.live.example.json index fd8970bc..9e263223 100644 --- a/config.live.example.json +++ b/config.live.example.json @@ -69,7 +69,8 @@ { "wallet": "0x921433c93558b9a4ba807ec824d02aad7ea2ddbf", "name": "AIcAIc", - "class": "volume" + "class": "volume", + "floor": 341.23 } ], "feed_path": "live/copybot_live_real.json", diff --git a/copybot.py b/copybot.py index 257b828d..acc8a992 100644 --- a/copybot.py +++ b/copybot.py @@ -48,6 +48,7 @@ Usage: """ import argparse +import uuid import hashlib import hmac import json @@ -846,7 +847,9 @@ class UserFillsListener: try: app = websocket.WebSocketApp(self.URL, on_open=on_open, on_message=on_message) - app.run_forever(sslopt={"cert_reqs": ssl.CERT_NONE}) + app.run_forever() # audit 3.6: this socket carries live L2 + # API creds — VERIFY certs (CERT_NONE was cargo-culted from the + # read-only RTDS pattern, where it stays) except Exception as e: log(f"user-ws: listener error {str(e)[:80]}") self.state = "down" @@ -914,8 +917,11 @@ class Copybot: it overstates the edge.""" ex = self.engine.ex buys = [] - if hasattr(ex, "fills") and ex.fills: - for f in ex.fills: + # atomic snapshot-and-swap (audit 3.1): two threads draining the same + # shared list each applied every fill's cash delta before one cleared + fills, ex.fills = (ex.fills if hasattr(ex, "fills") else []), [] + if fills: + for f in fills: sign = -1 if f["side"] == "BUY" else 1 fee = taker_fee(f["shares"], f["price"], self.fee_rate) f["fee"] = round(fee, 4) @@ -945,7 +951,6 @@ class Copybot: }) + "\n") except Exception: pass - ex.fills.clear() return buys def ledger_drift(self): @@ -1239,6 +1244,10 @@ class Copybot: return drift def write_feed(self): + with self.lock: # audit 3.1: reconcile mutates bet records + return self._write_feed_locked() + + def _write_feed_locked(self): """Publish the bot's live book to live/copybot_live.json — the feed the top of jaxperro.com/trading reads. Reconciles any open bet no longer held (mirror-sold) to 'closed'.""" @@ -1402,6 +1411,20 @@ class Copybot: subprocess.run(["git", "-C", repo, "commit", "-q", "-m", "copybot: live paper feed (resynced after " "conflicted rebase) [skip ci]"], capture_output=True) + try: # audit 3.4: stale-writer suicide guard + up = subprocess.run(["git", "-C", repo, "show", + "@{u}:" + self.engine.state_path], + capture_output=True, text=True) + if up.returncode == 0: + ub = (json.loads(up.stdout).get("boot") or {}) + ours = st.get("boot") or {} + if (ub.get("id") and ours.get("id") and ub["id"] != ours["id"] + and ub.get("ts", 0) > ours.get("ts", 0)): + log("⚠ a NEWER boot owns the book on origin — this " + "stale writer is exiting instead of overwriting it") + os._exit(0) + except Exception: + pass p = subprocess.run(["git", "-C", repo, "push", "-q", "origin", "main"], capture_output=True, text=True) log("published live feed → dashboard" if p.returncode == 0 @@ -1459,7 +1482,8 @@ class Copybot: if not rl.acquire(blocking=False): return try: - self._resolve_pendings_locked(st) + with self.lock: # audit 3.1: adoption mutates cash/my_pos/spend + self._resolve_pendings_locked(st) finally: rl.release() @@ -1584,6 +1608,10 @@ class Copybot: retries = st.get("exit_retries") or [] if not retries: return + with self.lock: # audit 3.1: sells + fill drains mutate the book + self._retry_stuck_exits_locked(st, retries) + + def _retry_stuck_exits_locked(self, st, retries): keep = [] for r in retries: tok = r["token"] @@ -1671,6 +1699,19 @@ class Copybot: continue if tried >= 10: break + # audit 3.2: the data-api LAGS the chain — a residual the chain + # already zeroed (platform auto-redeem) keeps reappearing here. + # Gate every action on exchange-view truth; act only on confirmed + # shares; credit proportionally so a stale row books ~$0. + try: + chain_sh = self.engine.ex._shares_held(tok) + except Exception: + continue # no chain truth -> no action + if chain_sh < max(0.01, shares * 0.5): + continue # data-api ghost — skip forever + if chain_sh < shares: + val *= chain_sh / shares + shares = chain_sh if p.get("redeemable") or cur >= 0.99: # resolved WINNER residual (the 2026-07-17 $1.90 case): the # tracked-position redeemer never sees untracked tokens, so @@ -2176,6 +2217,12 @@ class Copybot: self.engine.persist() def settle_resolved(self): + # audit 4: this ran on EVERY webhook POST ahead of the copy itself — + # dozens of redundant CLOB fetches during clip bursts. 20s throttle; + # the 60s heartbeat still guarantees timely settles. + if time.time() - getattr(self, "_settle_last", 0) < 20: + return + self._settle_last = time.time() """Free capital like the dashboard: when an open position's market has resolved, settle it at the winner price (1/0), recycle the cash, and tally realized P&L. This is the resolution path the engine's sell-only mirror @@ -2486,6 +2533,10 @@ def main(): if want_live else "") filt = FollowFilter(cfg) bot = Copybot(cfg, engine, filt, redeemer=redeemer) + # single-writer invariant (audit 3.4): every boot stamps the state; a + # stale process that survived `machine stop` (gotcha 15c) sees a newer + # boot on origin at publish time and yields instead of overwriting it. + engine.state["boot"] = {"id": uuid.uuid4().hex[:12], "ts": int(time.time())} # T0 detection — the RTDS trade stream (~1s, wallet-attributed). Paper # runs it by default (the 24h shadow run, 2026-07-10); the REAL-MONEY diff --git a/copytrade.py b/copytrade.py index 0b630dfb..ed659469 100644 --- a/copytrade.py +++ b/copytrade.py @@ -129,6 +129,26 @@ def save_json(path, data): os.replace(tmp, path) +class _SeenTx(dict): + """Set-compatible seen-tx registry that remembers insertion recency so + persist() can keep the NEWEST N (membership: `tx in seen`; value = a + monotonically increasing sequence number).""" + _n = 0 + + def __init__(self, items=()): + super().__init__(items) + if self: # resume the sequence ABOVE loaded entries, or fresh + _SeenTx._n = max(_SeenTx._n, max(self.values()) + 1) + + def add(self, tx): + _SeenTx._n += 1 + self[tx] = _SeenTx._n + + def update(self, txs): + for t in txs: + self.add(t) + + def new_state(): return { "started_at": time.time(), @@ -310,7 +330,7 @@ class CopyTrader: self.ex = executor self.state_path = state_path self.risk = cfg["risk"] - self.seen = set(state["seen_tx"]) + self.seen = _SeenTx((tx, i) for i, tx in enumerate(state["seen_tx"])) self.webhook = cfg.get("discord_webhook", "") self._discord_warned = False @@ -397,7 +417,9 @@ class CopyTrader: del missed[:-200] # keep the recent 200 def persist(self): - self.state["seen_tx"] = list(self.seen)[-5000:] + # newest-5000 by recency (audit 3.8: a set slice kept an ARBITRARY + # 5000 — a crash+fast-restart could re-copy a fresh trade) + self.state["seen_tx"] = sorted(self.seen, key=self.seen.get)[-5000:] save_json(self.state_path, self.state) # -- risk gate: returns (allowed_usd, reason_if_blocked) -- diff --git a/live/daily.sh b/live/daily.sh index 972bd54a..8ad8915c 100755 --- a/live/daily.sh +++ b/live/daily.sh @@ -30,7 +30,11 @@ if [ -z "${DAILY_COPY:-}" ]; then export DAILY_SRC DAILY_COPY exec /bin/bash "$DAILY_COPY" "$@" fi -trap 'rm -f "$DAILY_COPY"' EXIT +LOCKDIR="${TMPDIR:-/tmp}/wwf-daily.lock.d" +if ! mkdir "$LOCKDIR" 2>/dev/null; then + echo "[daily] another run holds the lock ($LOCKDIR) — exiting"; exit 0 +fi +trap 'rmdir "$LOCKDIR" 2>/dev/null; rm -f "$DAILY_COPY"' EXIT cd "$DAILY_SRC" # heads-up ping so the run's start is visible in Discord (digest comes at the end) python3 discord_daily.py --ping "🔄 Daily pipeline started $(date '+%H:%M') — refreshing the bet cache (takes a while); sharp digest lands when it finishes." || true diff --git a/recorder/ingest.py b/recorder/ingest.py index 536e6b95..9adf93b1 100644 --- a/recorder/ingest.py +++ b/recorder/ingest.py @@ -57,10 +57,12 @@ def main(): d.get("size"), d.get("tx"), d.get("title"))) except Exception: pass - if rows: + con.execute("BEGIN") # audit 3.10: atomic pair — a crash + if rows: # between the two inserts re-ingested con.executemany("INSERT INTO trades VALUES (?,?,?,?,?,?,?,?,?)", rows) con.execute("INSERT INTO ingested VALUES (?,?,?)", [s, len(rows), int(time.time())]) + con.execute("COMMIT") box(f"rm {SEG}/{s}") total += len(rows) print(f"[ingest] {s}: {len(rows)} rows") diff --git a/tests/test_dust_sweep.py b/tests/test_dust_sweep.py index 17a51104..0b005200 100644 --- a/tests/test_dust_sweep.py +++ b/tests/test_dust_sweep.py @@ -15,6 +15,10 @@ class Ex: def __init__(self): self.fills, self.sold = [], [] + def _shares_held(self, tok): + # chain truth for the audit-3.2 gate: DUST really exists on chain + return {"DUST": 1.0}.get(tok, 0.0) + def sell(self, tok, shares, price, meta): self.sold.append(tok) self.fills.append({"side": "SELL", "token": tok, "shares": shares, diff --git a/value/valuebot.py b/value/valuebot.py index ca5b8f90..a3d3d906 100644 --- a/value/valuebot.py +++ b/value/valuebot.py @@ -227,8 +227,13 @@ def onchain_payouts(cond, rpc): def load_state(): try: return json.load(open(STATE)) - except Exception: - return {"cash": BANK, "my_pos": {}, "resolved": [], "missed": [], + except FileNotFoundError: + pass + except Exception as e: + # audit 3.5: a corrupt state must NEVER silently become a fresh book + log(f"⚠⚠ STATE RESET — {STATE} unreadable ({e}); starting a fresh " + "$1k book. History is in git if this was a torn write.") + return {"cash": BANK, "my_pos": {}, "resolved": [], "missed": [], "attempted": {}, "stats": {"attempts": 0, "fills": 0, "misses": 0, "resolved": 0, "wins": 0, "refunds": 0, "losses": 0, "staked": 0.0, "returned": 0.0, "fees": 0.0}, @@ -236,7 +241,9 @@ def load_state(): def save_state(st): - json.dump(st, open(STATE, "w")) + tmp = STATE + ".tmp" # audit 3.5: tmp+rename — a crash mid-write + json.dump(st, open(tmp, "w")) # can't torch the book + os.replace(tmp, STATE) def open_positions(st, cands, budget):