diff --git a/research/copy_reconcile.py b/research/copy_reconcile.py index 67aeaaeb..0cac8c21 100644 --- a/research/copy_reconcile.py +++ b/research/copy_reconcile.py @@ -106,7 +106,20 @@ def main(): and b.get("entry_t", 0) >= fs.get(nm2w.get(b["name"], ""), 0)] print(f"backtest bets in the parity era with join keys " f"(follow-since applied): {len(bets)}", flush=True) - db = tape.connect() + # rtds.duckdb is single-writer and the 15-min tape-sync agent holds it in + # bursts; the daily's reconcile step used to die outright on the + # collision (2026-07-27). Wait it out — the sync finishes in seconds. + for _try in range(20): + try: + db = tape.connect() + break + except Exception as e: + if "lock" not in str(e).lower() or _try == 19: + raise + if _try == 0: + print("[reconcile] tape locked by the sync agent — waiting…", + flush=True) + time.sleep(30) tape.build_resolved(db) pays = fwd.payouts_for(db, [b["asset"] for b in bets]) v = bot_views() diff --git a/research/nightly.sh b/research/nightly.sh index a50d367e..49aa7d6a 100755 --- a/research/nightly.sh +++ b/research/nightly.sh @@ -22,9 +22,18 @@ POLL_S=900 # run: exit 1 + a stale lock that would have skipped every future night). LOCK="$(pwd)/.nightly.lock.d" if ! mkdir "$LOCK" 2>/dev/null; then - echo "$(date -u +%FT%TZ) already running — skip" >> forward.log - exit 0 + # STALE-LOCK GUARD (2026-07-27): a killed run (pause, crash, reboot) can + # leave the dir behind — the EXIT trap never fires on SIGKILL — and every + # later night then skips SILENTLY. Trust the recorded PID: if it is gone, + # the lock is debris, so take it over. + OWNER=$(cat "$LOCK/pid" 2>/dev/null) + if [ -n "$OWNER" ] && kill -0 "$OWNER" 2>/dev/null; then + echo "$(date -u +%FT%TZ) already running (pid $OWNER) — skip" >> forward.log + exit 0 + fi + echo "$(date -u +%FT%TZ) STALE lock (pid ${OWNER:-none} gone) — taking over" >> forward.log fi +echo $$ > "$LOCK/pid" trap 'rmdir "$LOCK"' EXIT tape_age() {