diff --git a/.gitignore b/.gitignore index b536ed32..e31b6f16 100644 --- a/.gitignore +++ b/.gitignore @@ -43,3 +43,4 @@ live/history/ live/slug_cache.json archive/local/ live/edge_verdict.txt +live/rtds.duckdb diff --git a/live/daily.sh b/live/daily.sh index f78b53b4..972bd54a 100755 --- a/live/daily.sh +++ b/live/daily.sh @@ -99,6 +99,8 @@ echo "[daily] $(date '+%F %T') edge: parity-era per-signal edge vs fee hurdle -> # The bankroll-decision number (HANDOFF rev 13 / 2026-07-16): one row/day; # the verdict line rides the Discord digest footer below. python3 edge.py || echo "[daily] edge skipped" +echo "[daily] $(date '+%F %T') tape: ingest RTDS segments -> live/rtds.duckdb" +python3 ../recorder/ingest.py || echo "[daily] tape ingest skipped" echo "[daily] $(date '+%F %T') 6/7 dashboard" python3 dashboard.py mkdir -p history && cp watch_skilled.json "history/watch_$(date '+%Y%m%d').json" 2>/dev/null diff --git a/recorder/ingest.py b/recorder/ingest.py new file mode 100644 index 00000000..d8680a5c --- /dev/null +++ b/recorder/ingest.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python3 +"""Nightly tape ingest (daily.sh): pull closed RTDS segments off the +wwf-recorder volume into live/rtds.duckdb, delete on the box ONLY after the +inserted row count matches. Own DB file on purpose — must never contend with +cache.duckdb's single-writer lock (2026-07-17 collision). Transport is +`flyctl ssh console` + base64 (no ingress on the recorder by design).""" +import base64 +import gzip +import json +import os +import subprocess +import time + +HERE = os.path.dirname(os.path.abspath(__file__)) +DB = os.path.join(HERE, "..", "live", "rtds.duckdb") +APP, SEG = "wwf-recorder", "/data/segments" + + +def box(cmd): + r = subprocess.run(["flyctl", "ssh", "console", "-a", APP, "-C", + f"bash -c '{cmd}'"], capture_output=True, text=True, + timeout=300) + return r.stdout + + +def main(): + import duckdb + con = duckdb.connect(DB) + con.execute("""CREATE TABLE IF NOT EXISTS trades( + ts DOUBLE, wallet VARCHAR, asset VARCHAR, cond VARCHAR, side VARCHAR, + price DOUBLE, size DOUBLE, tx VARCHAR, title VARCHAR)""") + con.execute("""CREATE TABLE IF NOT EXISTS ingested( + segment VARCHAR PRIMARY KEY, rows BIGINT, ingested_at BIGINT)""") + have = {r[0] for r in con.execute("SELECT segment FROM ingested").fetchall()} + segs = [s for s in box(f"ls {SEG}").split() + if s.endswith(".gz") and s not in have] + total = 0 + for s in sorted(segs): + raw = box(f"base64 {SEG}/{s}") + try: + lines = gzip.decompress(base64.b64decode(raw)).decode().splitlines() + except Exception as e: + print(f"[ingest] {s}: fetch/decode failed ({e}) — left on box") + continue + rows = [] + for ln in lines: + try: + d = json.loads(ln) + rows.append((d.get("ts"), d.get("wallet"), d.get("asset"), + d.get("cond"), d.get("side"), d.get("price"), + d.get("size"), d.get("tx"), d.get("title"))) + except Exception: + pass + if rows: + con.executemany("INSERT INTO trades VALUES (?,?,?,?,?,?,?,?,?)", rows) + con.execute("INSERT INTO ingested VALUES (?,?,?)", + [s, len(rows), int(time.time())]) + box(f"rm {SEG}/{s}") + total += len(rows) + print(f"[ingest] {s}: {len(rows)} rows") + n = con.execute("SELECT count(*) FROM trades").fetchone()[0] + print(f"[ingest] +{total} rows · rtds.duckdb now {n:,} trades") + + +if __name__ == "__main__": + main()