mirror of
https://github.com/jaxperro/winning-wallet-finder.git
synced 2026-07-27 15:57:47 +00:00
jaxperro
e6730ef579
- surgebot/oraclebot: settles now append-only to /data/*_settles.jsonl (SETTLED_TRIM rotation can never lose a settle; graders pull them) - forward.py: backfills any tape-covered day missing from the ledger — a Mac offline gap > RESCORE_DAYS no longer leaves verdict-evidence holes - meta_snap.py: nightly gzipped snapshot of all active markets (12k, 2MB/d, local-only) — end dates make tau knowable at trigger for every tape trigger; token->outcome maps kill the label-gap scorer artifact class - .gitignore: pulled raw streams + meta stay local (re-fetchable, lean repo) Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
89 lines
3.3 KiB
Python
89 lines
3.3 KiB
Python
#!/usr/bin/env python3
|
|
"""Nightly market-metadata snapshot — the tape records prints and ticks but
|
|
not what markets ARE. This closes that gap additively: one gzipped JSONL
|
|
per UTC day under research/meta/ (LOCAL-only, gitignored — re-fetchable,
|
|
so repo stays lean) with every active market's cond, question, slug,
|
|
end date, tags, tokens/outcomes and neg-risk flag.
|
|
|
|
Why (2026-07-22 data-moat pass): end dates make τ knowable AT TRIGGER for
|
|
every tape trigger (not just harness fills); tags replace title-heuristic
|
|
niches for future studies; token->outcome maps kill the label-gap class of
|
|
scorer artifact (FINDINGS round 3 disclosure 2). Append-only by
|
|
construction: a day file is written once and never rewritten (rerun same
|
|
day = idempotent overwrite of today only)."""
|
|
import gzip
|
|
import json
|
|
import os
|
|
import ssl
|
|
import time
|
|
import urllib.request
|
|
|
|
HERE = os.path.dirname(os.path.abspath(__file__))
|
|
OUT_DIR = os.path.join(HERE, "meta")
|
|
GAMMA = "https://gamma-api.polymarket.com/markets"
|
|
PAGE = 100 # gamma hard-caps limit at 100
|
|
SSL_CTX = ssl._create_unverified_context()
|
|
KEEP = ("id", "conditionId", "question", "slug", "endDate", "endDateIso",
|
|
"category", "tags", "clobTokenIds", "outcomes", "negRisk",
|
|
"closed", "active", "volumeNum", "liquidityNum", "startDate")
|
|
|
|
|
|
def fetch(url):
|
|
req = urllib.request.Request(url, headers={"User-Agent": "Mozilla/5.0"})
|
|
with urllib.request.urlopen(req, timeout=30, context=SSL_CTX) as r:
|
|
return json.loads(r.read().decode())
|
|
|
|
|
|
def iso(t):
|
|
return time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(t))
|
|
|
|
|
|
def main():
|
|
os.makedirs(OUT_DIR, exist_ok=True)
|
|
day = time.strftime("%Y%m%d", time.gmtime())
|
|
out = os.path.join(OUT_DIR, f"meta_{day}.jsonl.gz")
|
|
# gamma 422s past offset ~2100, so page inside end-date windows instead
|
|
now = time.time()
|
|
D = 86400
|
|
edges = [now - 30 * D, now, now + 2 * D, now + 7 * D, now + 30 * D,
|
|
now + 120 * D, now + 730 * D, now + 3650 * D]
|
|
seen, rows = set(), []
|
|
for dmin, dmax in zip(edges, edges[1:]):
|
|
offset = 0
|
|
while True:
|
|
try:
|
|
page = fetch(f"{GAMMA}?active=true&closed=false&limit={PAGE}"
|
|
f"&offset={offset}&end_date_min={iso(dmin)}"
|
|
f"&end_date_max={iso(dmax)}")
|
|
except Exception as e:
|
|
print(f"[meta_snap] window {iso(dmin)[:10]}: fetch failed at "
|
|
f"offset {offset}: {e} — moving on")
|
|
break
|
|
if not page:
|
|
break
|
|
for m in page:
|
|
if m.get("id") in seen:
|
|
continue
|
|
seen.add(m.get("id"))
|
|
rows.append({k: m.get(k) for k in KEEP})
|
|
offset += len(page)
|
|
if len(page) < PAGE or offset >= 2000: # stay under the 422 wall
|
|
break
|
|
if not rows:
|
|
print("[meta_snap] nothing fetched — keeping yesterday's file")
|
|
return 0
|
|
fetched_at = int(time.time())
|
|
tmp = out + ".tmp"
|
|
with gzip.open(tmp, "wt") as fh:
|
|
for r in rows:
|
|
r["fetched_at"] = fetched_at
|
|
fh.write(json.dumps(r) + "\n")
|
|
os.replace(tmp, out)
|
|
print(f"[meta_snap] {len(rows)} active markets -> {os.path.basename(out)} "
|
|
f"({os.path.getsize(out) // 1024}KB)")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|