From 48b51896ae2a6b23b4c66a0a10adb1e3abd9c178 Mon Sep 17 00:00:00 2001
From: jaxperro <jaxtorres@gmail.com>
Date: Mon, 27 Jul 2026 11:28:01 -0400
Subject: [PATCH] nightly: stale-lock takeover via recorded PID; reconcile:
 wait out the tape-sync write lock
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

A killed nightly (pause/crash/reboot) left .nightly.lock.d behind — the
EXIT trap never fires on SIGKILL — and EVERY later run then skipped
silently (caught 2026-07-27: today's run exited in <1s). Lock now
records its PID and a run whose owner is gone takes it over.
Reconcile hit the 15-min tape-sync agent's single-writer lock and died;
now waits it out (20x30s).

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
---
 research/copy_reconcile.py | 15 ++++++++++++++-
 research/nightly.sh        | 13 +++++++++++--
 2 files changed, 25 insertions(+), 3 deletions(-)

diff --git a/research/copy_reconcile.py b/research/copy_reconcile.py
index 67aeaaeb..0cac8c21 100644
--- a/research/copy_reconcile.py
+++ b/research/copy_reconcile.py
@@ -106,7 +106,20 @@ def main():
             and b.get("entry_t", 0) >= fs.get(nm2w.get(b["name"], ""), 0)]
     print(f"backtest bets in the parity era with join keys "
           f"(follow-since applied): {len(bets)}", flush=True)
-    db = tape.connect()
+    # rtds.duckdb is single-writer and the 15-min tape-sync agent holds it in
+    # bursts; the daily's reconcile step used to die outright on the
+    # collision (2026-07-27). Wait it out — the sync finishes in seconds.
+    for _try in range(20):
+        try:
+            db = tape.connect()
+            break
+        except Exception as e:
+            if "lock" not in str(e).lower() or _try == 19:
+                raise
+            if _try == 0:
+                print("[reconcile] tape locked by the sync agent — waiting…",
+                      flush=True)
+            time.sleep(30)
     tape.build_resolved(db)
     pays = fwd.payouts_for(db, [b["asset"] for b in bets])
     v = bot_views()
diff --git a/research/nightly.sh b/research/nightly.sh
index a50d367e..49aa7d6a 100755
--- a/research/nightly.sh
+++ b/research/nightly.sh
@@ -22,9 +22,18 @@ POLL_S=900
 # run: exit 1 + a stale lock that would have skipped every future night).
 LOCK="$(pwd)/.nightly.lock.d"
 if ! mkdir "$LOCK" 2>/dev/null; then
-  echo "$(date -u +%FT%TZ) already running — skip" >> forward.log
-  exit 0
+  # STALE-LOCK GUARD (2026-07-27): a killed run (pause, crash, reboot) can
+  # leave the dir behind — the EXIT trap never fires on SIGKILL — and every
+  # later night then skips SILENTLY. Trust the recorded PID: if it is gone,
+  # the lock is debris, so take it over.
+  OWNER=$(cat "$LOCK/pid" 2>/dev/null)
+  if [ -n "$OWNER" ] && kill -0 "$OWNER" 2>/dev/null; then
+    echo "$(date -u +%FT%TZ) already running (pid $OWNER) — skip" >> forward.log
+    exit 0
+  fi
+  echo "$(date -u +%FT%TZ) STALE lock (pid ${OWNER:-none} gone) — taking over" >> forward.log
 fi
+echo $$ > "$LOCK/pid"
 trap 'rmdir "$LOCK"' EXIT
 
 tape_age() {