From 3b4475b0e2aece8e406a6255e6dda12decdeaf1d Mon Sep 17 00:00:00 2001
From: jaxperro <jaxtorres@gmail.com>
Date: Wed, 22 Jul 2026 11:52:01 -0400
Subject: [PATCH] sync_tape: base64-over-console fallback + sane sftp timeout
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

flyctl sftp flaked again (900s timeouts, magic-byte-less partial parquets)
and 14h of tape queued behind it — the research nightly waits on tape
freshness by design, so everything downstream stalled too. Port ingest.py's
proven console fallback: sftp gets 240s, then base64 over ssh console
(600s); the manifest row-count verify remains the integrity gate and
partials are cleaned on both paths.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
---
 recorder/sync_tape.py | 43 ++++++++++++++++++++++++++++++++++++++-----
 1 file changed, 38 insertions(+), 5 deletions(-)

diff --git a/recorder/sync_tape.py b/recorder/sync_tape.py
index b058d2b1..d060bd32 100644
--- a/recorder/sync_tape.py
+++ b/recorder/sync_tape.py
@@ -41,11 +41,44 @@ def box(cmd, timeout=120):
     return r.stdout
 
 
-def sftp_get(remote, local, timeout=900):
-    subprocess.run([FLYCTL, "ssh", "sftp", "get", remote, local, "-a", APP],
-                   capture_output=True, timeout=timeout,
-                   stdin=subprocess.DEVNULL)
-    return os.path.exists(local) and os.path.getsize(local) > 0
+def sftp_get(remote, local, timeout=240):
+    """sftp first (a healthy multi-MB fetch takes <60s — the old 900s window
+    just burned the 15-min cadence when it hung), then the base64-over-
+    console fallback that carried ingest.py through the same flakiness
+    (2026-07-22: sftp timeouts left magic-byte-less partial parquets while
+    console commands stayed fast). Integrity is the caller's manifest
+    row-count verify either way; partial files are removed here."""
+    try:
+        subprocess.run([FLYCTL, "ssh", "sftp", "get", remote, local, "-a", APP],
+                       capture_output=True, timeout=timeout,
+                       stdin=subprocess.DEVNULL)
+    except subprocess.TimeoutExpired:
+        pass
+    if os.path.exists(local) and os.path.getsize(local) > 0:
+        return True
+    try:
+        os.remove(local)
+    except OSError:
+        pass
+    try:
+        import base64
+        raw = box(f"base64 {remote}", timeout=600)
+        blob = base64.b64decode(raw)
+        if not blob:
+            return False
+        with open(local, "wb") as fh:
+            fh.write(blob)
+        print(f"[sync] {os.path.basename(remote)}: sftp failed — "
+              f"base64 fallback carried {len(blob)//1024}KB")
+        return True
+    except Exception as e:
+        print(f"[sync] {os.path.basename(remote)}: both transports failed "
+              f"({type(e).__name__})")
+        try:
+            os.remove(local)
+        except OSError:
+            pass
+        return False
 
 
 def load_manifest(path):