live: fix enumerate.py infinite loop past gamma's offset cap

recent_markets() only stopped on end-of-data (page <100) or MAX_SCAN.
When gamma 422s past its max offset, every page in a wave returns "ERR"
and is skipped — so `scanned` never grows and `ended` never trips,
spinning forever. This wedged every daily.sh run at "scanned 2,100…"
(step 1/6), so the cron never actually completed. Now bail after a
second consecutive all-error wave (one is tolerated; _page already
retries transient errors).

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
jaxperro
2026-06-22 15:25:44 -06:00
parent 3190dee803
commit 6cde76768e
+13 -1
View File
@@ -73,15 +73,17 @@ def recent_markets():
cutoff = time.time() - WINDOW_DAYS * 86400
end_min = time.strftime("%Y-%m-%dT00:00:00Z", time.gmtime(cutoff))
out, scanned, base, WAVE = [], 0, 0, 4 # low concurrency — gamma throttles bursts
fail_streak = 0
with ThreadPoolExecutor(max_workers=WAVE) as ex:
while len(out) < MAX_MARKETS and scanned < MAX_SCAN:
pages = list(ex.map(lambda o: _page(o, end_min),
[base + i * 100 for i in range(WAVE)]))
base += WAVE * 100
ended = False
ended = got_any = False
for page in pages:
if page == "ERR" or page is None:
continue # skip transient gaps, keep going
got_any = True
if len(page) < 100:
ended = True # genuine end of data
scanned += len(page)
@@ -95,6 +97,16 @@ def recent_markets():
print(f" scanned {scanned:,}… kept {len(out)}", flush=True)
if ended:
break
# a whole wave erroring out means we've hit gamma's offset ceiling (it
# 422s past a max offset) — not a transient blip, since _page already
# retries. Allow one such wave, bail on the second so we never spin.
if not got_any:
fail_streak += 1
if fail_streak >= 2:
print(" (gamma returned no more pages — stopping scan)", flush=True)
break
else:
fail_streak = 0
return out[:MAX_MARKETS]