From 44eb6327bf65445c870bc8dafc46d15bbfba6bc2 Mon Sep 17 00:00:00 2001
From: Casey Judice <cjudice@umich.edu>
Date: Sat, 30 May 2026 12:32:11 -0400
Subject: [PATCH] Harden month-rollover tool against rate-limit truncation

gj() now retries 5x with exponential backoff (0.5->8s) instead of 3x
flat, so a throttled burst mid-discovery no longer silently truncates a
ladder. build() reports any asset whose Kalshi ladder came back empty,
and main() aborts (exit 2) rather than overwriting pairs.json when that
happens -- a failed fetch must not be mistaken for "no markets listed."

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
---
 scripts/roll_month.py | 29 ++++++++++++++++++++++-------
 1 file changed, 22 insertions(+), 7 deletions(-)

diff --git a/scripts/roll_month.py b/scripts/roll_month.py
index 3d6b68b92..bd5a92a49 100644
--- a/scripts/roll_month.py
+++ b/scripts/roll_month.py
@@ -68,14 +68,20 @@ ASSETS = {
 }
 
 
-def gj(url):
+def gj(url, tries=5):
+    """GET JSON with exponential backoff. Kalshi/Poly rate-limit bursts, and a
+    silent failure mid-discovery would truncate the ladder, so retry hard."""
     rq = urllib.request.Request(url, headers={"User-Agent": "Mozilla/5.0"})
-    for _ in range(3):
+    delay = 0.5
+    for i in range(tries):
         try:
-            with urllib.request.urlopen(rq, timeout=20) as r:
+            with urllib.request.urlopen(rq, timeout=25) as r:
                 return json.loads(r.read().decode())
         except Exception:
-            time.sleep(0.6)
+            if i == tries - 1:
+                return None
+            time.sleep(delay)
+            delay = min(delay * 2, 8.0)
     return None
 
 
@@ -167,11 +173,14 @@ def nearest(poly_for_dir, strike):
 
 # ------------------------------------------------------------------- main
 def build(month_name, year):
-    rows = []
+    rows, zero_kalshi = [], []
     expcode = expiry_code(year, _MONTHNUM[month_name])
     print("Target expiry code: %s   Poly month: %s-%s" % (expcode, month_name, year))
     for asset, cfg in ASSETS.items():
         kl = kalshi_ladder(asset, expcode)
+        if not kl:
+            zero_kalshi.append(asset)        # every tracked asset has a ladder;
+                                             # empty == a failed/throttled fetch
         pl = poly_ladder(asset, cfg, month_name, year)
         by_dir = {"above": [], "below": []}
         for (d, ps), slug in pl.items():
@@ -191,7 +200,7 @@ def build(month_name, year):
             n_pair += slug is not None
         print("  %-5s kalshi=%-3d poly=%-3d  -> %d rows (%d with Poly)"
               % (asset, len(kl), len(pl), len(kl), n_pair))
-    return rows
+    return rows, zero_kalshi
 
 
 _MONTHNUM = {calendar.month_name[i].lower(): i for i in range(1, 13)}
@@ -214,10 +223,16 @@ def main():
         y, m = (t.year + 1, 1) if t.month == 12 else (t.year, t.month + 1)
     month_name = calendar.month_name[m].lower()
 
-    rows = build(month_name, y)
+    rows, zero_kalshi = build(month_name, y)
     with_poly = sum(1 for r in rows if r["poly_slug"])
     print("\nDiscovered %d rows (%d with a Polymarket leg)." % (len(rows), with_poly))
 
+    if zero_kalshi and not args.force:
+        print("ABORT: Kalshi discovery returned nothing for %s — almost "
+              "certainly a throttled/failed fetch, not reality. Re-run in a "
+              "minute (or --force to write a partial file)."
+              % ", ".join(zero_kalshi), file=sys.stderr)
+        return 2
     if rows and with_poly < 0.3 * len(rows):
         print("WARNING: very few rows got a Polymarket leg (%d/%d). Polymarket "
               "may have changed its %s event-slug pattern — check the per-asset "