diff --git a/live/cache.py b/live/cache.py index c61619e5..94d16349 100644 --- a/live/cache.py +++ b/live/cache.py @@ -192,43 +192,39 @@ for _col in ("title", "outcome"): except Exception: pass # already there _con.execute("CREATE TABLE IF NOT EXISTS pulled_exits(wallet TEXT PRIMARY KEY, newest_ts BIGINT, pulled_at BIGINT)") -try: - _con.execute("ALTER TABLE pulled_exits ADD COLUMN oldest_ts BIGINT") # backfill-completeness marker -except Exception: - pass +for _col, _type in (("oldest_ts", "BIGINT"), ("complete", "BOOLEAN")): + try: + _con.execute(f"ALTER TABLE pulled_exits ADD COLUMN {_col} {_type}") + except Exception: + pass # already there -def closed_exits(wallet, max_age_s=6 * 3600, window_days=WINDOW_DAYS): - """{asset: {ts, exit_p, p, iv, cond}} of the wallet's fully-closed - positions over the last `window_days` (default 180 — the same window the - bets cache scores, so the exit overlay COMPLETELY covers every bet the - sharps/backtest stats touch; no arbitrary row cap that could truncate a - hyperactive wallet's window). Shared exit model for the backtest +def closed_exits(wallet, max_age_s=6 * 3600): + """{asset: {ts, exit_p, p, iv, cond}} of the wallet's FULLY-CLOSED + positions — the COMPLETE history, no window and no cap (the exit overlay + covers every bet any stat could touch). Shared exit model for the backtest (portfolio.py) and the sharps stats (validate_timing.py). - Correct + incremental: `oldest_ts` records how far back the cache is known - complete. If the window isn't fully covered yet (first run, or a backfill - that got interrupted) it re-pulls the whole window down to the target; once - complete, later refreshes only page the NEW closes since `newest_ts` (a - page or two). Close events are immutable, so nothing already cached is - re-fetched. NB the endpoint serves 50-row pages regardless of `limit` — - see smart_money.closed_exits for the paging gotcha this caused.""" + Correct + incremental: `complete` marks that the backfill reached the true + start of the wallet's history. Until then (first run, or an interrupted + deep pull) it re-pulls the full history from newest; once complete, later + refreshes only page the NEW closes since `newest_ts` (a page or two). Close + events are immutable, so nothing already cached is re-fetched. The deep + first pull is a one-time cost. NB the endpoint serves 50-row pages + regardless of `limit` — see smart_money.closed_exits for the paging gotcha.""" import smart_money as _sm # local import: smart_money has no local deps now = int(time.time()) - target_oldest = now - window_days * 86400 with _lock: - r = _con.execute("SELECT newest_ts, pulled_at, oldest_ts FROM pulled_exits " + r = _con.execute("SELECT newest_ts, pulled_at, complete FROM pulled_exits " "WHERE wallet=?", [wallet]).fetchone() - newest, oldest = (r[0], r[2]) if r else (0, None) + newest, complete = (r[0], bool(r[2])) if r else (0, False) fresh = r and (now - r[1] < max_age_s) - complete = oldest is not None and oldest <= target_oldest if not fresh or not complete: if complete: - new = _sm.closed_exits(wallet, newest_bound=newest) # only new closes - new_oldest = oldest + new, _ = _sm.closed_exits(wallet, newest_bound=newest) # only new closes + reached = True # already complete else: - new = _sm.closed_exits(wallet, since_ts=target_oldest) # full window (re)pull - new_oldest = target_oldest + new, reached = _sm.closed_exits(wallet) # FULL history (re)pull with _lock: if new: _con.executemany( @@ -237,8 +233,8 @@ def closed_exits(wallet, max_age_s=6 * 3600, window_days=WINDOW_DAYS): c.get("title") or "", c.get("outcome") or "") for a, c in new.items()]) newest = max(newest, max(c["ts"] for c in new.values())) - _con.execute("INSERT OR REPLACE INTO pulled_exits(wallet,newest_ts,pulled_at,oldest_ts) " - "VALUES (?,?,?,?)", [wallet, newest, now, new_oldest]) + _con.execute("INSERT OR REPLACE INTO pulled_exits(wallet,newest_ts,pulled_at,complete) " + "VALUES (?,?,?,?)", [wallet, newest, now, reached]) with _lock: rows = _con.execute( "SELECT asset, ts, exit_p, p, iv, cond, title, outcome FROM exits WHERE wallet=?", diff --git a/smart_money.py b/smart_money.py index f8333760..96abb9f6 100644 --- a/smart_money.py +++ b/smart_money.py @@ -107,35 +107,38 @@ def leaderboard_candidates(pool): return ranked[:pool] -def closed_exits(wallet, since_ts=0, max_rows=20000, newest_bound=0): - """{asset: {ts, exit_p, p, iv, cond, title, outcome}} for the wallet's - FULLY-CLOSED positions, newest first. `ts` is the close (sell/redeem) - timestamp; the exit price is reconstructed from realized P&L over shares - bought (exit_p = avgPrice + realizedPnl/totalBought — exact for a full - single-price exit, share-weighted otherwise). +def closed_exits(wallet, since_ts=0, max_rows=200000, newest_bound=0): + """(exits, reached_end) — the wallet's FULLY-CLOSED positions. + exits = {asset: {ts, exit_p, p, iv, cond, title, outcome}}, newest first. + `ts` is the close (sell/redeem) timestamp; exit_p is reconstructed from + realized P&L over shares bought (avgPrice + realizedPnl/totalBought — exact + for a full single-price exit, share-weighted otherwise). - PAGING GOTCHA: /closed-positions serves at most 50 rows per page no - matter what `limit` says — step by the RETURNED page size, never by the - requested one (assuming limit-sized pages silently truncated every - wallet's exit history to its most recent 50 closes, which put a 16x - hold-to-res ceiling back into a scalper's stats). Stops at since_ts, - newest_bound (for incremental refresh: rows older than what's already - cached), max_rows, or an empty page. Prefer cache.closed_exits — the - incremental cached layer over this raw fetcher. + FULL HISTORY by default (since_ts=0): pages the wallet's ENTIRE closed + history — no window, no meaningful cap. The exit overlay covers every bet + the stats could ever touch; max_rows=200000 is only a runaway guard. The + cost is a one-time deep pull, amortized by the incremental exits cache + (cache.closed_exits) which then only pages new closes. - The real bound is `since_ts` (callers pass the window they score — 180d for - the sharps overlay, 30d for the backtest), so the pull covers exactly the - scored window and stops. max_rows=20000 is only a safety ceiling for a - pathological wallet; the date bound stops well before it for anyone real. - The original since_ts=0 (unbounded, all-history) pull is what stalled the - daily pipeline — bounding by date fixes completeness AND runtime.""" + `reached_end` is True when the pull hit the TRUE start of the wallet's + history (an empty or short <50 page), False when it stopped early on a + bound (newest_bound for incremental refresh, or the max_rows guard). The + cache uses it to tell a COMPLETE backfill from an interrupted one, so a + killed deep pull re-completes instead of falsely reporting done. + + PAGING GOTCHA: /closed-positions serves at most 50 rows per page no matter + what `limit` says — step by the RETURNED page size, never the requested + one (that silently truncated every wallet's history to its most recent 50 + closes, reviving a 16x hold-to-res ceiling in scalper stats).""" out = {} off = 0 + reached_end = False while off < max_rows: page = get_json("/closed-positions", {"user": wallet, "limit": 500, "offset": off, "sortBy": "TIMESTAMP", "sortDirection": "DESC"}) if not page: + reached_end = True # ran out of history — complete break for r in page: ts = r.get("timestamp") or 0 @@ -148,11 +151,16 @@ def closed_exits(wallet, since_ts=0, max_rows=20000, newest_bound=0): "ts": ts, "exit_p": exit_p, "p": max(0.001, min(0.999, avg)), "iv": r.get("initialValue") or avg * tb, "cond": r.get("conditionId"), "title": r.get("title") or "", "outcome": r.get("outcome") or ""}) - oldest = page[-1].get("timestamp") or 0 - if oldest < since_ts or oldest < newest_bound: + if len(page) < 50: # short page = start of history reached + reached_end = True break + oldest = page[-1].get("timestamp") or 0 + if since_ts and oldest < since_ts: + break # bounded stop — NOT the true end + if newest_bound and oldest < newest_bound: + break # incremental: reached already-cached rows off += len(page) # actual page size — the server caps at 50 - return out + return out, reached_end WIN_WINDOW_DAYS = 90 # measure win rate over resolved bets in this window