From 6f6bd81cb1c09a7edee9922c1796afa9001c6176 Mon Sep 17 00:00:00 2001 From: jaxperro Date: Mon, 20 Jul 2026 13:19:57 -0400 Subject: [PATCH] fix tape ingest data loss + daily digest send MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ingest: sftp needs stdin=DEVNULL under launchd (no tty) — without it, empty/partial files decoded to 0 lines and the missing integrity guard MARKED+DELETED them (36 segments lost). Guard: a >200B gz that yields 0 lines is a failed fetch, left on box to retry. digest: retry+backoff moved INTO _post so the digest send (not just the start ping) survives the post-wake DNS gap; ingest ping gets the same. Co-Authored-By: Claude Fable 5 --- live/discord_daily.py | 31 +++++++++++++++++++------------ recorder/ingest.py | 39 ++++++++++++++++++++++++++++++--------- 2 files changed, 49 insertions(+), 21 deletions(-) diff --git a/live/discord_daily.py b/live/discord_daily.py index d860b431..ed8ada60 100644 --- a/live/discord_daily.py +++ b/live/discord_daily.py @@ -24,12 +24,24 @@ HERE = os.path.dirname(os.path.abspath(__file__)) MAX_ROWS = 30 # embed description caps at 4096 chars; 30 rows fits -def _post(hook, payload): +def _post(hook, payload, tries=6): + """POST with retry+backoff on ANY error (2026-07-20: the DIGEST send — not + just the start ping — was dying on the post-wake DNS gap / transient network + blips with no retry, so the whole daily digest silently never arrived). + Backoff 5,10,20,40,80s ≈ 2.5min of coverage; raises only if all tries fail.""" req = urllib.request.Request( hook, data=json.dumps(payload).encode(), headers={"Content-Type": "application/json", "User-Agent": "Mozilla/5.0"}) # Discord 403s the default UA - urllib.request.urlopen(req, timeout=15, context=ssl._create_unverified_context()) + for attempt in range(tries): + try: + urllib.request.urlopen(req, timeout=15, + context=ssl._create_unverified_context()) + return + except Exception: + if attempt == tries - 1: + raise + time.sleep(5 * (2 ** attempt)) def main(): @@ -48,16 +60,11 @@ def main(): # after wake, so a single try fails instantly with getaddrinfo "nodename # nor servname" (harmless: the end-of-run digest lands fine). Retry a few # times with backoff to ride out the post-wake network gap. - for attempt in range(4): - try: - _post(hook, {"content": sys.argv[2]}) - print("[discord] ping sent" + (f" (attempt {attempt + 1})" if attempt else "")) - return - except Exception as e: - if attempt == 3: - print("[discord] ping failed after retries:", e) - else: - time.sleep(5 * (attempt + 1)) # 5s, 10s, 15s — DNS usually up by 15s + try: + _post(hook, {"content": sys.argv[2]}) # _post now retries internally + print("[discord] ping sent") + except Exception as e: + print("[discord] ping failed after retries:", e) return try: sharps = json.load(open(os.path.join(HERE, "watch_sharps.json"))) diff --git a/recorder/ingest.py b/recorder/ingest.py index cdb94900..d24a49de 100644 --- a/recorder/ingest.py +++ b/recorder/ingest.py @@ -29,8 +29,14 @@ def ping(msg): req = urllib.request.Request(hook, data=json.dumps({"content": msg}).encode(), headers={"Content-Type": "application/json", "User-Agent": "Mozilla/5.0"}) - urllib.request.urlopen(req, timeout=10, - context=ssl._create_unverified_context()).read() + for attempt in range(5): # ride out the post-wake DNS gap (2026-07-20) + try: + urllib.request.urlopen(req, timeout=10, + context=ssl._create_unverified_context()).read() + return + except Exception: + if attempt < 4: + time.sleep(5 * (2 ** attempt)) except Exception: pass @@ -41,9 +47,11 @@ FLYCTL = shutil.which("flyctl") or "/opt/homebrew/bin/flyctl" def box(cmd): + # stdin=DEVNULL: under launchd there is no tty, and any flyctl subcommand + # that reads stdin will otherwise hang or read garbage (2026-07-20). r = subprocess.run([FLYCTL, "ssh", "console", "-a", APP, "-C", f"bash -c '{cmd}'"], capture_output=True, text=True, - timeout=900) # busy-hour segments (~15MB gz) outgrow 300s over ssh + timeout=900, stdin=subprocess.DEVNULL) return r.stdout @@ -67,11 +75,17 @@ def main(): # console was +33% bytes and needed 900s timeouts at busy-hour sizes); # the old path stays as the fallback because sftp exits 0 even on # some failures — the gunzip is the integrity check either way. - lines = None + lines, fsize = None, 0 local = os.path.join("/tmp", s) try: + # stdin=DEVNULL is LOAD-BEARING (2026-07-20): without it, sftp + # under launchd (no tty) produced empty/partial files with rc 0, + # which decoded to [] and — via the missing guard below — got + # marked ingested and DELETED. 36 segments were lost this way. subprocess.run([FLYCTL, "ssh", "sftp", "get", f"{SEG}/{s}", local, - "-a", APP], capture_output=True, timeout=900) + "-a", APP], capture_output=True, timeout=900, + stdin=subprocess.DEVNULL) + fsize = os.path.getsize(local) with gzip.open(local, "rb") as fh: lines = fh.read().decode().splitlines() except Exception: @@ -84,14 +98,21 @@ def main(): if lines is None: try: raw = box(f"base64 {SEG}/{s}") - lines = gzip.decompress(base64.b64decode(raw)).decode().splitlines() + blob = base64.b64decode(raw) + fsize = len(blob) + lines = gzip.decompress(blob).decode().splitlines() except Exception as e: - # BOTH transports failed — skip, never crash: a single monster - # segment (rtds_20260718_21, 2026-07-19) blocked the whole - # sorted backlog behind it when this timeout escaped the try print(f"[ingest] {s}: both transports failed ({type(e).__name__}) " "— left on box, continuing") continue + # INTEGRITY GUARD (2026-07-20): a non-trivial gz that yields ZERO lines + # is a truncated/empty FETCH, not an empty hour (a genuinely empty hour + # gzips to ~20-30 bytes). NEVER mark+delete it — leave it on the box to + # retry. This is the guard whose absence cost 36 segments. + if not lines and fsize > 200: + print(f"[ingest] {s}: {fsize}B gz decoded to 0 lines — FETCH FAILED, " + "left on box for retry") + continue rows, aux = [], [] for ln in lines: try: