From 70865690244a096ba047551e9b65c1e4c047f826 Mon Sep 17 00:00:00 2001 From: jaxperro Date: Sun, 19 Jul 2026 17:12:45 -0400 Subject: [PATCH] ingest: sftp timeout 900s, fallback timeout contained, both-transports-failed skips instead of crashing the backlog (#11 hardening) Co-Authored-By: Claude Fable 5 --- recorder/ingest.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/recorder/ingest.py b/recorder/ingest.py index 3d161c8e..cdb94900 100644 --- a/recorder/ingest.py +++ b/recorder/ingest.py @@ -71,7 +71,7 @@ def main(): local = os.path.join("/tmp", s) try: subprocess.run([FLYCTL, "ssh", "sftp", "get", f"{SEG}/{s}", local, - "-a", APP], capture_output=True, timeout=300) + "-a", APP], capture_output=True, timeout=900) with gzip.open(local, "rb") as fh: lines = fh.read().decode().splitlines() except Exception: @@ -82,11 +82,15 @@ def main(): except OSError: pass if lines is None: - raw = box(f"base64 {SEG}/{s}") try: + raw = box(f"base64 {SEG}/{s}") lines = gzip.decompress(base64.b64decode(raw)).decode().splitlines() except Exception as e: - print(f"[ingest] {s}: fetch/decode failed ({e}) — left on box") + # BOTH transports failed — skip, never crash: a single monster + # segment (rtds_20260718_21, 2026-07-19) blocked the whole + # sorted backlog behind it when this timeout escaped the try + print(f"[ingest] {s}: both transports failed ({type(e).__name__}) " + "— left on box, continuing") continue rows, aux = [], [] for ln in lines: