mirror of
https://github.com/jaxperro/winning-wallet-finder.git
synced 2026-07-27 15:57:47 +00:00
cache schema v2: token-keyed upsert archive, provenance, raw prices, honest failures
- bets table gains asset (token id = position identity), src/ts (endpoint provenance + close time), resolved (False = early-sold in an unended market; won is a curPrice mark). Auto-migrates v1 in place (~8s, 3,697 exact dupes merged); legacy rows carry NULLs until their wallet refreshes. - refresh is now an upsert by token instead of a wallet wipe: rows sliding out of the rolling pull window survive, so per-wallet history accumulates into a permanent archive. Same-asset rows from both endpoints (partially-closed positions) dedupe to the larger-stake row - kills the two-endpoint double-count class (~35k suspect pairs found in the audit). - p stored raw (0 = avgPrice missing), clamped on read by get_bets, so missing prices stay distinguishable from real 0.1c longshots; insider CLI + oos clamp their own direct use. - resolved_bets(strict=True): a failed page raises instead of returning a silently truncated history; get_bets no longer caches or marks failed pulls (pre-v2 an API error cached the wallet as empty-and-fresh for 14 days - Kruto2027 was a live victim of this last night). Verified: migration 18,289,320 -> 18,285,623 rows; forced refreshes of two sharps show 0 same-asset dups, 0 legacy/new mixing, clamped reads, conviction stats intact (Kruto conv win 73%). Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
jaxperro
+95
-17
@@ -1,11 +1,28 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Local cache of per-wallet resolved bets, so we stop re-pulling the data-api.
|
||||
|
||||
Each wallet's resolved bets (won, entry price p, conditionId, resolution time,
|
||||
size) are stored once in cache.duckdb. Because we keep res_t per bet, ANY date
|
||||
cutoff — pre-June-1, full window, future experiments — reads the same cached
|
||||
rows and filters locally. A pull only happens for wallets not seen, or older
|
||||
than MAX_AGE_DAYS.
|
||||
Each wallet's resolved bets are stored once in cache.duckdb. Because we keep
|
||||
res_t per bet, ANY date cutoff — pre-June-1, full window, future experiments —
|
||||
reads the same cached rows and filters locally. A pull only happens for wallets
|
||||
not seen, or older than MAX_AGE_DAYS.
|
||||
|
||||
Schema v2 (migrated automatically on first open; legacy rows keep NULLs in the
|
||||
new columns until their wallet refreshes):
|
||||
* asset — token id, the position identity. Dedupes the two-endpoint union
|
||||
(the same asset from /closed-positions AND /positions is one
|
||||
position seen twice) and disambiguates YES/NO both-sides rows.
|
||||
* src/ts — endpoint provenance ('closed'/'open') + close timestamp.
|
||||
* resolved — False for early-sold positions in markets that hadn't ended at
|
||||
pull time (their `won` is a curPrice mark, not an outcome).
|
||||
* p — stored RAW (0 = avgPrice missing); get_bets clamps to
|
||||
[0.001, 0.999] on read, so consumers see the same values as
|
||||
before while the DB keeps missing-vs-real-longshot separable.
|
||||
* upsert — refresh replaces only the re-pulled tokens instead of wiping
|
||||
the wallet, so history beyond the rolling WINDOW_DAYS pull
|
||||
accumulates (permanent archive instead of overwrite-on-refresh).
|
||||
* failures — a failed pull is returned empty but NOT cached and NOT marked
|
||||
pulled, so it retries next call instead of masquerading as
|
||||
"wallet has no bets" for MAX_AGE_DAYS.
|
||||
|
||||
Thread-safe: API pulls (the slow part) run outside the lock; only the small
|
||||
DuckDB reads/writes are serialized, so skill.py's worker pool still parallelizes
|
||||
@@ -49,7 +66,35 @@ def conv_cutoff(sizes, q=CONV_PCTILE):
|
||||
_lock = threading.Lock()
|
||||
_con = duckdb.connect(DB)
|
||||
_con.execute("""CREATE TABLE IF NOT EXISTS bets(
|
||||
wallet TEXT, cond TEXT, won BOOLEAN, p DOUBLE, res_t BIGINT, size DOUBLE)""")
|
||||
wallet TEXT, cond TEXT, asset TEXT, won BOOLEAN, p DOUBLE, res_t BIGINT,
|
||||
size DOUBLE, src TEXT, ts BIGINT, resolved BOOLEAN)""")
|
||||
|
||||
|
||||
def _migrate_v2():
|
||||
"""One-shot in-place migration of a v1 `bets` table (no asset/src/ts/resolved
|
||||
columns). Rebuilds via SELECT DISTINCT — v1 had no position identity, so its
|
||||
few thousand byte-identical duplicate rows are unrecoverable noise and are
|
||||
merged. Legacy rows keep NULLs in the new columns until their wallet is
|
||||
re-pulled; `p` stays clamped for them (raw-p is forward-only)."""
|
||||
cols = {r[0] for r in _con.execute("DESCRIBE bets").fetchall()}
|
||||
if "asset" in cols:
|
||||
return
|
||||
n0 = _con.execute("SELECT count(*) FROM bets").fetchone()[0]
|
||||
_con.execute("BEGIN")
|
||||
_con.execute("""CREATE TABLE bets_v2(
|
||||
wallet TEXT, cond TEXT, asset TEXT, won BOOLEAN, p DOUBLE, res_t BIGINT,
|
||||
size DOUBLE, src TEXT, ts BIGINT, resolved BOOLEAN)""")
|
||||
_con.execute("""INSERT INTO bets_v2(wallet, cond, won, p, res_t, size)
|
||||
SELECT DISTINCT wallet, cond, won, p, res_t, size FROM bets""")
|
||||
_con.execute("DROP TABLE bets")
|
||||
_con.execute("ALTER TABLE bets_v2 RENAME TO bets")
|
||||
_con.execute("COMMIT")
|
||||
n1 = _con.execute("SELECT count(*) FROM bets").fetchone()[0]
|
||||
print(f"[cache] migrated bets to schema v2: {n0:,} -> {n1:,} rows "
|
||||
f"({n0 - n1:,} exact duplicates merged)", flush=True)
|
||||
|
||||
|
||||
_migrate_v2()
|
||||
_con.execute("CREATE INDEX IF NOT EXISTS bets_w ON bets(wallet)")
|
||||
_con.execute("CREATE TABLE IF NOT EXISTS pulled(wallet TEXT PRIMARY KEY, pulled_at BIGINT)")
|
||||
_con.execute("CREATE TABLE IF NOT EXISTS entries(wallet TEXT, cond TEXT, first_buy BIGINT)")
|
||||
@@ -79,30 +124,63 @@ def get_entries(wallet):
|
||||
return first_buy
|
||||
|
||||
|
||||
def _bet_row(won, p, cond, res_t, size, asset, src, ts, resolved):
|
||||
"""The dict shape get_bets returns — p clamped on read so consumer math is
|
||||
unchanged while the DB stores it raw."""
|
||||
return {"won": won, "p": max(0.001, min(0.999, p or 0)), "cond": cond,
|
||||
"res_t": res_t, "size": size, "asset": asset, "src": src,
|
||||
"ts": ts, "resolved": resolved}
|
||||
|
||||
|
||||
def get_bets(wallet):
|
||||
"""Resolved bets for a wallet — from cache if fresh, else pull and store."""
|
||||
"""Resolved bets for a wallet — from cache if fresh, else pull and upsert."""
|
||||
now = time.time()
|
||||
with _lock:
|
||||
r = _con.execute("SELECT pulled_at FROM pulled WHERE wallet=?", [wallet]).fetchone()
|
||||
if r and now - r[0] < MAX_AGE_DAYS * 86400:
|
||||
rows = _con.execute(
|
||||
"SELECT won,p,cond,res_t,size FROM bets WHERE wallet=?", [wallet]).fetchall()
|
||||
return [{"won": w, "p": p, "cond": c, "res_t": rt, "size": s}
|
||||
for w, p, c, rt, s in rows]
|
||||
"SELECT won,p,cond,res_t,size,asset,src,ts,resolved "
|
||||
"FROM bets WHERE wallet=?", [wallet]).fetchall()
|
||||
return [_bet_row(*row) for row in rows]
|
||||
# cache miss / stale -> pull (slow, outside the lock so workers stay parallel)
|
||||
try:
|
||||
bets = insider.resolved_bets(wallet, now - WINDOW_DAYS * 86400)
|
||||
bets = insider.resolved_bets(wallet, now - WINDOW_DAYS * 86400, strict=True)
|
||||
except Exception:
|
||||
bets = []
|
||||
return [] # transient API failure — do NOT cache or mark pulled;
|
||||
# the next call retries instead of trusting a bad pull
|
||||
# one row per token: the endpoint union returns the same asset twice for a
|
||||
# partially-closed position (closed portion + open remainder) — keep the
|
||||
# larger-stake row rather than double-counting one position as two bets.
|
||||
best = {}
|
||||
for b in bets:
|
||||
k = (b["cond"], b.get("asset"))
|
||||
if k not in best or (b.get("size") or 0) > (best[k].get("size") or 0):
|
||||
best[k] = b
|
||||
bets = list(best.values())
|
||||
with _lock:
|
||||
_con.execute("DELETE FROM bets WHERE wallet=?", [wallet])
|
||||
# upsert: replace only what this pull re-observed — re-pulled tokens, plus
|
||||
# any legacy (pre-v2, NULL-asset) rows of the re-pulled markets they
|
||||
# supersede. Rows older than the rolling pull window survive, so per-wallet
|
||||
# history now accumulates instead of being overwritten each refresh.
|
||||
assets = [b["asset"] for b in bets if b.get("asset")]
|
||||
conds = list({b["cond"] for b in bets if b.get("cond")})
|
||||
_con.execute(
|
||||
"""DELETE FROM bets WHERE wallet = ?
|
||||
AND (asset IN (SELECT UNNEST(?::VARCHAR[]))
|
||||
OR (asset IS NULL AND cond IN (SELECT UNNEST(?::VARCHAR[]))))""",
|
||||
[wallet, assets, conds])
|
||||
if bets:
|
||||
_con.executemany(
|
||||
"INSERT INTO bets(wallet,cond,won,p,res_t,size) VALUES (?,?,?,?,?,?)",
|
||||
[(wallet, b["cond"], b["won"], b["p"], b.get("res_t"), b.get("size"))
|
||||
for b in bets])
|
||||
"INSERT INTO bets(wallet,cond,asset,won,p,res_t,size,src,ts,resolved) "
|
||||
"VALUES (?,?,?,?,?,?,?,?,?,?)",
|
||||
[(wallet, b["cond"], b.get("asset"), b["won"], b.get("p"),
|
||||
b.get("res_t"), b.get("size"), b.get("src"), b.get("ts"),
|
||||
b.get("resolved")) for b in bets])
|
||||
_con.execute("INSERT OR REPLACE INTO pulled VALUES (?,?)", [wallet, int(now)])
|
||||
return bets
|
||||
rows = _con.execute(
|
||||
"SELECT won,p,cond,res_t,size,asset,src,ts,resolved "
|
||||
"FROM bets WHERE wallet=?", [wallet]).fetchall()
|
||||
return [_bet_row(*row) for row in rows]
|
||||
|
||||
|
||||
def invalidate(wallets):
|
||||
|
||||
Reference in New Issue
Block a user