research/ is a hard silo (README rules): read-only tape, no bot imports, own launchd (com.jaxperro.research-nightly 09:15, after daily ingest). - tape.py: proxy-resolution (the 742/742-validated method), niche + crypto strike/expiry/sprint parsers, tick loaders - sim.py: FAK execution replayer; hold_s=3 fitted on 29 real labeled live attempts (79% fill/miss classification), price noise 2-4c, measured OPTIMISM BIAS -2c/fill carried into every verdict threshold - requote.py: crater refill timing per niche (crypto 94% <4s, esports 83% <10s, sports needs ~25s, geo/politics minutes) -> params/requote_timing.json - study_flow.py + robustness: in-play surge momentum. Identity NULL result: 10 pooled controls +23.85/fill == informed +23.68 -> hypothesis revised at freeze, surge-EV primary, identity secondary (#16) - study_oracle.py: oracle digital fair value. 86% craters, winner's-curse inversion at big edges, nothing frozen (no cell at 30 fills) (#17) - forward.py + nightly.sh: re-scores frozen studies on last 3 tape days, appends forward_ledger.jsonl; verdicts ONLY from post-freeze rows Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
jaxperro
@@ -44,3 +44,7 @@ live/slug_cache.json
|
||||
archive/local/
|
||||
live/edge_verdict.txt
|
||||
live/rtds.duckdb
|
||||
research/forward.log
|
||||
research/launchd.log
|
||||
research/.nightly.lock.d
|
||||
research/__pycache__/
|
||||
|
||||
@@ -0,0 +1,31 @@
|
||||
# research/ — the edge factory (SILO)
|
||||
|
||||
Standing rules (user directive 2026-07-20: "built in a silo to not affect
|
||||
anything on the live bot"):
|
||||
|
||||
- **Nothing here is imported by, or imports, the bot** (`copybot.py`,
|
||||
`copytrade.py`, their configs). The Fly workers run pinned entrypoints;
|
||||
this directory is inert to them.
|
||||
- **Tape access is read-only** (`duckdb.connect(..., read_only=True)`).
|
||||
- The ONE shared write: `live/cache.duckdb::resolutions` via `live/payouts.py`
|
||||
— append-only immutable chain facts, the same store the daily pipeline
|
||||
already feeds. Nothing else in `live/` is touched.
|
||||
- The existing paper bot is the live test's CONTROL — graduated edges get
|
||||
their own paper harness here, never that one.
|
||||
- Studies are pre-registered (GitHub issue per study: hypothesis, params,
|
||||
verdict + kill criteria) BEFORE their forward window opens. Exploration
|
||||
happens on already-collected tape; **verdicts only come from
|
||||
`forward_ledger.jsonl` rows dated after the params freeze commit.**
|
||||
|
||||
Layout:
|
||||
tape.py read-only loaders · tape proxy-resolution (terminal-VWAP +
|
||||
sibling veto, the 742/742 chain-validated method) · title
|
||||
parsers (niche, crypto strike/expiry/sprint)
|
||||
sim.py execution replayer calibrated on OUR live fills ledger
|
||||
(lag, FAK no-match, protected band, 3% taker fee)
|
||||
study_flow.py Study A — informed-flow state signal
|
||||
study_oracle.py Study B — crypto oracle fair value vs the book
|
||||
requote.py crater→requote timing measurement (retry tuning)
|
||||
forward.py scores frozen studies on new tape days → forward_ledger.jsonl
|
||||
params/ frozen study parameters (committed = frozen)
|
||||
nightly.sh manual/launchd runner (separate from daily.sh)
|
||||
@@ -0,0 +1,168 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Forward verdict ledger — the only place belief comes from.
|
||||
|
||||
Each run re-scores the FROZEN studies on the last RESCORE_DAYS UTC days of
|
||||
tape and appends one row per (study, day) to forward_ledger.jsonl. Days are
|
||||
recomputed on later runs so pending (unresolved-at-the-time) triggers
|
||||
resolve into their day's row; readers keep the newest computed_at per key.
|
||||
|
||||
Studies:
|
||||
flow frozen params from params/study_flow.json — informed set as-of
|
||||
each day's 00:00 UTC, scored at p50 lag, first- AND worst-print
|
||||
fills; plus 3 FIXED control seeds (identity-lift tracking).
|
||||
oracle params/study_oracle.json grid — ALL edge levels tracked until
|
||||
one accumulates >= 30 forward fills (then the selection rule in
|
||||
the pre-registration applies). Skips days without tick coverage.
|
||||
|
||||
Verdicts are pre-registered in the study issues; this script only reports.
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import time
|
||||
|
||||
import tape
|
||||
import sim as simmod
|
||||
import study_flow as sf
|
||||
import study_oracle as so
|
||||
|
||||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||
LEDGER = os.path.join(HERE, "forward_ledger.jsonl")
|
||||
RESCORE_DAYS = 3
|
||||
CONTROL_SEEDS = (1, 2, 3)
|
||||
|
||||
|
||||
def day_bounds(d):
|
||||
lo = time.mktime(time.strptime(d, "%Y-%m-%d")) - time.timezone
|
||||
return lo, lo + 86400
|
||||
|
||||
|
||||
def score_flow(db, fz, d, hold_s):
|
||||
lo, hi = day_bounds(d)
|
||||
t_max = db.execute("SELECT max(ts) FROM trades").fetchone()[0]
|
||||
hi = min(hi, t_max)
|
||||
S = sf.informed_set(db, lo, fz["top_n"])
|
||||
tape.build_resolved(db)
|
||||
trig = sf.signals(db, S, lo, hi, fz["window_s"], fz["flow_usd"])
|
||||
row = {"triggers": len(trig), "set_size": len(S)}
|
||||
for mode in ("first", "worst"):
|
||||
s = simmod.Sim(db, lag_s=simmod.LAG_P50, hold_s=hold_s, fill=mode)
|
||||
agg = dict(fills=0, misses=0, pending=0, pnl=0.0, wins=0)
|
||||
for t in trig:
|
||||
pay = db.execute("SELECT payout::DOUBLE FROM res_tok WHERE asset=?",
|
||||
[t["asset"]]).fetchone()
|
||||
r = s.try_buy(t["asset"], t["ts"], t["p_ref"], stake_usd=sf.STAKE)
|
||||
if not r["filled"]:
|
||||
agg["misses"] += 1
|
||||
elif pay is None:
|
||||
agg["pending"] += 1
|
||||
else:
|
||||
agg["fills"] += 1
|
||||
agg["pnl"] += r["shares"] * (pay[0] - r["price"]) - r["fee"]
|
||||
agg["wins"] += pay[0] == 1.0
|
||||
agg["pnl"] = round(agg["pnl"], 2)
|
||||
if agg["fills"]:
|
||||
agg["ev_per_fill"] = round(agg["pnl"] / agg["fills"], 2)
|
||||
agg["hit"] = round(agg["wins"] / agg["fills"], 3)
|
||||
row[mode] = agg
|
||||
ctl = []
|
||||
for seed in CONTROL_SEEDS:
|
||||
C = sf.matched_random_set(db, lo, fz["top_n"], seed)
|
||||
ctrig = sf.signals(db, C, lo, hi, fz["window_s"], fz["flow_usd"])
|
||||
s = simmod.Sim(db, lag_s=simmod.LAG_P50, hold_s=hold_s, fill="worst")
|
||||
fills = 0
|
||||
pnl = 0.0
|
||||
for t in ctrig:
|
||||
pay = db.execute("SELECT payout::DOUBLE FROM res_tok WHERE asset=?",
|
||||
[t["asset"]]).fetchone()
|
||||
r = s.try_buy(t["asset"], t["ts"], t["p_ref"], stake_usd=sf.STAKE)
|
||||
if r["filled"] and pay is not None:
|
||||
fills += 1
|
||||
pnl += r["shares"] * (pay[0] - r["price"]) - r["fee"]
|
||||
ctl.append({"seed": seed, "fills": fills, "pnl": round(pnl, 2)})
|
||||
row["controls_worst"] = ctl
|
||||
return row
|
||||
|
||||
|
||||
def score_oracle(db, P, d, hold_s):
|
||||
lo, hi = day_bounds(d)
|
||||
series = {s: so.TickSeries(tape.load_ticks(db, s))
|
||||
for s in ("btcusdt", "ethusdt", "solusdt", "xrpusdt",
|
||||
"bnbusdt", "dogeusdt")}
|
||||
have = [s for s in series.values() if s.ts and s.ts[0] < hi and s.ts[-1] > lo]
|
||||
if not have:
|
||||
return {"skipped": "no tick coverage"}
|
||||
outcomes = so.outcome_map(db)
|
||||
tape.build_resolved(db)
|
||||
uni = so.crypto_universe(db, outcomes, series)
|
||||
payout = {a: p for a, p in db.execute(
|
||||
"SELECT asset, payout::DOUBLE FROM res_tok").fetchall()}
|
||||
sim = simmod.Sim(db, hold_s=hold_s)
|
||||
row = {}
|
||||
for u in uni:
|
||||
prints = db.execute("""SELECT ts, price FROM trades WHERE asset=?
|
||||
AND ts > ? AND ts <= ? ORDER BY ts""", [u["asset"], lo, hi]).fetchall()
|
||||
s = series[u["mkt"]["sym"]]
|
||||
last_ev = 0.0
|
||||
for ts, px in prints:
|
||||
if ts - last_ev < so.COOLDOWN_S:
|
||||
continue
|
||||
f = so.fair_value(u["mkt"], u["up"], s.at(ts), s.vol_1s(ts), ts)
|
||||
if f is None:
|
||||
continue
|
||||
edge = f - float(px)
|
||||
if edge < min(so.EDGE_GRID):
|
||||
continue
|
||||
last_ev = ts
|
||||
r = sim.try_buy(u["asset"], ts, float(px), stake_usd=so.STAKE)
|
||||
for E in so.EDGE_GRID:
|
||||
if edge < E:
|
||||
continue
|
||||
g = row.setdefault(str(E), {"events": 0, "fills": 0,
|
||||
"pending": 0, "pnl": 0.0, "wins": 0})
|
||||
g["events"] += 1
|
||||
if not r["filled"]:
|
||||
continue
|
||||
pay = payout.get(u["asset"])
|
||||
if pay is None:
|
||||
g["pending"] += 1
|
||||
continue
|
||||
g["fills"] += 1
|
||||
g["pnl"] += r["shares"] * (pay - r["price"]) - r["fee"]
|
||||
g["wins"] += pay == 1.0
|
||||
for g in row.values():
|
||||
g["pnl"] = round(g["pnl"], 2)
|
||||
if g["fills"]:
|
||||
g["ev_per_fill"] = round(g["pnl"] / g["fills"], 2)
|
||||
g["hit"] = round(g["wins"] / g["fills"], 3)
|
||||
return row
|
||||
|
||||
|
||||
def main():
|
||||
db = tape.connect()
|
||||
cal = json.load(open(os.path.join(HERE, "params", "sim_calibration.json")))
|
||||
flow_p = json.load(open(os.path.join(HERE, "params", "study_flow.json")))
|
||||
fz = flow_p["frozen"]
|
||||
frozen_at = flow_p["frozen_at"]
|
||||
t_max = db.execute("SELECT max(ts) FROM trades").fetchone()[0]
|
||||
days = [time.strftime("%Y-%m-%d", time.gmtime(t_max - i * 86400))
|
||||
for i in range(RESCORE_DAYS)]
|
||||
now = time.strftime("%Y-%m-%d %H:%M UTC", time.gmtime())
|
||||
with open(LEDGER, "a") as fh:
|
||||
for d in days:
|
||||
r1 = score_flow(db, fz, d, cal["hold_s"])
|
||||
fh.write(json.dumps({"study": "flow", "day": d, "computed_at": now,
|
||||
"frozen_at": frozen_at, **r1},
|
||||
default=float) + "\n")
|
||||
print(f"flow {d}: trig {r1['triggers']} "
|
||||
f"worst {r1['worst'].get('ev_per_fill')} "
|
||||
f"({r1['worst']['fills']} fills, {r1['worst']['pending']} pend)")
|
||||
r2 = score_oracle(db, None, d, cal["hold_s"])
|
||||
fh.write(json.dumps({"study": "oracle", "day": d, "computed_at": now,
|
||||
**r2}, default=float) + "\n")
|
||||
print(f"oracle {d}: " + (r2.get("skipped") or
|
||||
" ".join(f"E{E}:{g.get('ev_per_fill')}({g['fills']}f)"
|
||||
for E, g in sorted(r2.items()))))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,6 @@
|
||||
{"study": "flow", "day": "2026-07-20", "computed_at": "2026-07-20 21:23 UTC", "frozen_at": "2026-07-20 21:14 UTC", "triggers": 1083, "set_size": 150, "first": {"fills": 122, "misses": 776, "pending": 185, "pnl": 4876.4, "wins": 95, "ev_per_fill": 39.97, "hit": 0.779}, "worst": {"fills": 122, "misses": 776, "pending": 185, "pnl": 4820.99, "wins": 95, "ev_per_fill": 39.52, "hit": 0.779}, "controls_worst": [{"seed": 1, "fills": 0, "pnl": 0.0}, {"seed": 2, "fills": 6, "pnl": 13.45}, {"seed": 3, "fills": 5, "pnl": 331.95}]}
|
||||
{"study": "oracle", "day": "2026-07-20", "computed_at": "2026-07-20 21:23 UTC", "0.04": {"events": 6739, "fills": 23, "pending": 1123, "pnl": 167.89, "wins": 12, "ev_per_fill": 7.3, "hit": 0.522}, "0.07": {"events": 3991, "fills": 15, "pending": 397, "pnl": -57.86, "wins": 7, "ev_per_fill": -3.86, "hit": 0.467}, "0.1": {"events": 2828, "fills": 10, "pending": 213, "pnl": -434.48, "wins": 4, "ev_per_fill": -43.45, "hit": 0.4}}
|
||||
{"study": "flow", "day": "2026-07-19", "computed_at": "2026-07-20 21:23 UTC", "frozen_at": "2026-07-20 21:14 UTC", "triggers": 2588, "set_size": 150, "first": {"fills": 482, "misses": 1879, "pending": 227, "pnl": 11963.33, "wins": 348, "ev_per_fill": 24.82, "hit": 0.722}, "worst": {"fills": 482, "misses": 1879, "pending": 227, "pnl": 11738.61, "wins": 348, "ev_per_fill": 24.35, "hit": 0.722}, "controls_worst": [{"seed": 1, "fills": 29, "pnl": 1142.87}, {"seed": 2, "fills": 20, "pnl": -191.23}, {"seed": 3, "fills": 38, "pnl": 978.03}]}
|
||||
{"study": "oracle", "day": "2026-07-19", "computed_at": "2026-07-20 21:23 UTC", "0.04": {"events": 1316, "fills": 9, "pending": 232, "pnl": -27.62, "wins": 7, "ev_per_fill": -3.07, "hit": 0.778}, "0.07": {"events": 767, "fills": 2, "pending": 105, "pnl": -8.42, "wins": 1, "ev_per_fill": -4.21, "hit": 0.5}, "0.1": {"events": 524, "fills": 1, "pending": 64, "pnl": 93.2, "wins": 1, "ev_per_fill": 93.2, "hit": 1.0}}
|
||||
{"study": "flow", "day": "2026-07-18", "computed_at": "2026-07-20 21:23 UTC", "frozen_at": "2026-07-20 21:14 UTC", "triggers": 1924, "set_size": 30, "first": {"fills": 292, "misses": 1374, "pending": 258, "pnl": 13471.87, "wins": 229, "ev_per_fill": 46.14, "hit": 0.784}, "worst": {"fills": 292, "misses": 1374, "pending": 258, "pnl": 13061.87, "wins": 229, "ev_per_fill": 44.73, "hit": 0.784}, "controls_worst": [{"seed": 1, "fills": 6, "pnl": 842.01}, {"seed": 2, "fills": 37, "pnl": 2348.85}, {"seed": 3, "fills": 25, "pnl": 1725.52}]}
|
||||
{"study": "oracle", "day": "2026-07-18", "computed_at": "2026-07-20 21:23 UTC", "skipped": "no tick coverage"}
|
||||
Executable
+24
@@ -0,0 +1,24 @@
|
||||
#!/bin/bash
|
||||
# research nightly — scores the frozen studies on fresh tape and versions
|
||||
# the forward ledger. SILO: touches only research/ (+ the append-only
|
||||
# resolutions cache via payouts.py). Runs at 09:15 local via
|
||||
# com.jaxperro.research-nightly (after the 08:00 daily pipeline's ingest);
|
||||
# safe to run by hand any time: python3 research/forward.py
|
||||
set -e
|
||||
cd "$(dirname "$0")"
|
||||
|
||||
if ! mkdir .nightly.lock.d 2>/dev/null; then
|
||||
echo "$(date -u +%FT%TZ) already running — skip" >> forward.log
|
||||
exit 0
|
||||
fi
|
||||
trap 'rmdir .nightly.lock.d' EXIT
|
||||
|
||||
echo "== $(date -u +%FT%TZ) nightly ==" >> forward.log
|
||||
python3 forward.py >> forward.log 2>&1
|
||||
|
||||
cd ..
|
||||
git add research/forward_ledger.jsonl
|
||||
if ! git diff --cached --quiet; then
|
||||
git commit -q -m "research: forward ledger $(date -u +%F) [skip ci]"
|
||||
git pull --rebase --autostash -q && git push -q
|
||||
fi
|
||||
@@ -0,0 +1,60 @@
|
||||
{
|
||||
"crypto": {
|
||||
"n": 625819,
|
||||
"p50": 0.0,
|
||||
"p75": 0.0,
|
||||
"p90": 2.0,
|
||||
"within_4s": 0.941,
|
||||
"within_10s": 0.966,
|
||||
"within_25s": 0.983
|
||||
},
|
||||
"other": {
|
||||
"n": 69962,
|
||||
"p50": 1.0,
|
||||
"p75": 102.0,
|
||||
"p90": 1085.0,
|
||||
"within_4s": 0.565,
|
||||
"within_10s": 0.607,
|
||||
"within_25s": 0.659
|
||||
},
|
||||
"sports": {
|
||||
"n": 52515,
|
||||
"p50": 1.0,
|
||||
"p75": 21.0,
|
||||
"p90": 225.0,
|
||||
"within_4s": 0.639,
|
||||
"within_10s": 0.697,
|
||||
"within_25s": 0.763
|
||||
},
|
||||
"esports": {
|
||||
"n": 24370,
|
||||
"p50": 0.0,
|
||||
"p75": 3.0,
|
||||
"p90": 32.0,
|
||||
"within_4s": 0.772,
|
||||
"within_10s": 0.829,
|
||||
"within_25s": 0.887
|
||||
},
|
||||
"geo": {
|
||||
"n": 2129,
|
||||
"p50": 0.0,
|
||||
"p75": 98.0,
|
||||
"p90": 2110.0,
|
||||
"within_4s": 0.643,
|
||||
"within_10s": 0.669,
|
||||
"within_25s": 0.7
|
||||
},
|
||||
"politics": {
|
||||
"n": 571,
|
||||
"p50": 1.0,
|
||||
"p75": 879.0,
|
||||
"p90": 13761.0,
|
||||
"within_4s": 0.562,
|
||||
"within_10s": 0.578,
|
||||
"within_25s": 0.606
|
||||
},
|
||||
"_meta": {
|
||||
"jump": 0.03,
|
||||
"generated": "2026-07-20 17:11"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,41 @@
|
||||
{
|
||||
"n_fills": 16,
|
||||
"n_misses": 13,
|
||||
"grid": {
|
||||
"3": {
|
||||
"fill_recall": 0.75,
|
||||
"miss_recall": 0.8461538461538461,
|
||||
"acc": 0.793
|
||||
},
|
||||
"5": {
|
||||
"fill_recall": 0.75,
|
||||
"miss_recall": 0.7692307692307693,
|
||||
"acc": 0.759
|
||||
},
|
||||
"10": {
|
||||
"fill_recall": 0.8125,
|
||||
"miss_recall": 0.7692307692307693,
|
||||
"acc": 0.793
|
||||
},
|
||||
"20": {
|
||||
"fill_recall": 0.8125,
|
||||
"miss_recall": 0.6153846153846154,
|
||||
"acc": 0.724
|
||||
},
|
||||
"45": {
|
||||
"fill_recall": 0.875,
|
||||
"miss_recall": 0.46153846153846156,
|
||||
"acc": 0.69
|
||||
},
|
||||
"90": {
|
||||
"fill_recall": 0.875,
|
||||
"miss_recall": 0.3076923076923077,
|
||||
"acc": 0.621
|
||||
}
|
||||
},
|
||||
"hold_s": 3,
|
||||
"px_err_p50": 0.02,
|
||||
"px_err_p90": 0.04,
|
||||
"px_within_1c": 0.083,
|
||||
"px_bias_mean": -0.0197
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,77 @@
|
||||
{
|
||||
"informed_first": {
|
||||
"fills": 479,
|
||||
"ev": 24.14,
|
||||
"hit": 0.72,
|
||||
"avg_px": 0.573
|
||||
},
|
||||
"informed_worst": {
|
||||
"fills": 479,
|
||||
"ev": 23.68,
|
||||
"hit": 0.72,
|
||||
"avg_px": 0.574
|
||||
},
|
||||
"controls_worst": [
|
||||
{
|
||||
"fills": 29,
|
||||
"ev": 39.41,
|
||||
"hit": 0.759,
|
||||
"avg_px": 0.57
|
||||
},
|
||||
{
|
||||
"fills": 20,
|
||||
"ev": -9.6,
|
||||
"hit": 0.6,
|
||||
"avg_px": 0.573
|
||||
},
|
||||
{
|
||||
"fills": 37,
|
||||
"ev": 25.73,
|
||||
"hit": 0.784,
|
||||
"avg_px": 0.582
|
||||
},
|
||||
{
|
||||
"fills": 74,
|
||||
"ev": 17.5,
|
||||
"hit": 0.622,
|
||||
"avg_px": 0.552
|
||||
},
|
||||
{
|
||||
"fills": 59,
|
||||
"ev": 53.09,
|
||||
"hit": 0.78,
|
||||
"avg_px": 0.541
|
||||
},
|
||||
{
|
||||
"fills": 23,
|
||||
"ev": -9.21,
|
||||
"hit": 0.522,
|
||||
"avg_px": 0.53
|
||||
},
|
||||
{
|
||||
"fills": 33,
|
||||
"ev": 7.26,
|
||||
"hit": 0.606,
|
||||
"avg_px": 0.503
|
||||
},
|
||||
{
|
||||
"fills": 24,
|
||||
"ev": 14.74,
|
||||
"hit": 0.667,
|
||||
"avg_px": 0.598
|
||||
},
|
||||
{
|
||||
"fills": 56,
|
||||
"ev": 21.94,
|
||||
"hit": 0.732,
|
||||
"avg_px": 0.607
|
||||
},
|
||||
{
|
||||
"fills": 26,
|
||||
"ev": 44.16,
|
||||
"hit": 0.808,
|
||||
"avg_px": 0.602
|
||||
}
|
||||
],
|
||||
"controls_pooled_ev": 23.85
|
||||
}
|
||||
@@ -0,0 +1,37 @@
|
||||
{
|
||||
"grid": {
|
||||
"0.04": {
|
||||
"events": 6169,
|
||||
"fills": 26,
|
||||
"misses": 5292,
|
||||
"pending": 851,
|
||||
"ev_per_fill": 12.39,
|
||||
"hit": 0.577,
|
||||
"pnl": 322.14
|
||||
},
|
||||
"0.07": {
|
||||
"events": 4225,
|
||||
"fills": 16,
|
||||
"misses": 3781,
|
||||
"pending": 428,
|
||||
"ev_per_fill": 2.29,
|
||||
"hit": 0.5,
|
||||
"pnl": 36.72
|
||||
},
|
||||
"0.1": {
|
||||
"events": 3151,
|
||||
"fills": 11,
|
||||
"misses": 2885,
|
||||
"pending": 255,
|
||||
"ev_per_fill": -31.03,
|
||||
"hit": 0.455,
|
||||
"pnl": -341.28
|
||||
}
|
||||
},
|
||||
"frozen_edge": null,
|
||||
"n_universe": 8264,
|
||||
"tick_lo": 1784490952.0,
|
||||
"our_fills_graded": [],
|
||||
"frozen_at": "2026-07-20 21:21 UTC",
|
||||
"note": "NO holdout exists (21h ticks) \u2014 belief deferred entirely to forward_ledger"
|
||||
}
|
||||
@@ -0,0 +1,57 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Crater -> requote timing: after an aggressive up-move print (>= 3c above
|
||||
the previous print, the shape our FAK misses die in), how long until the
|
||||
SAME token prints again — i.e. how long does the crater stay empty?
|
||||
|
||||
Directly tunes copybot's fak_retry_s (currently a flat 10s): the retry
|
||||
should arrive when liquidity is back, per niche. Pure measurement, no bot
|
||||
changes here.
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import time
|
||||
|
||||
import tape
|
||||
|
||||
JUMP = 0.03
|
||||
|
||||
|
||||
def run():
|
||||
db = tape.connect()
|
||||
rows = db.execute(f"""
|
||||
WITH p AS (
|
||||
SELECT asset, ts, price, title,
|
||||
lag(price) OVER (PARTITION BY asset ORDER BY ts, tx) prev_p,
|
||||
lead(ts) OVER (PARTITION BY asset ORDER BY ts, tx) next_ts
|
||||
FROM trades
|
||||
)
|
||||
SELECT title, ts, next_ts - ts AS gap
|
||||
FROM p
|
||||
WHERE prev_p IS NOT NULL AND price - prev_p >= {JUMP}
|
||||
AND next_ts IS NOT NULL
|
||||
""").fetchall()
|
||||
by = {}
|
||||
for title, ts, gap in rows:
|
||||
by.setdefault(tape.niche(title), []).append(gap)
|
||||
out = {}
|
||||
print(f"{len(rows):,} crater prints (>= {JUMP:.02f} up-moves)\n")
|
||||
print(f"{'niche':<10} {'n':>8} {'p50':>7} {'p75':>7} {'p90':>7} "
|
||||
f"{'<=4s':>6} {'<=10s':>6} {'<=25s':>6}")
|
||||
for niche, gaps in sorted(by.items(), key=lambda kv: -len(kv[1])):
|
||||
gaps.sort()
|
||||
n = len(gaps)
|
||||
q = lambda f: gaps[min(int(n * f), n - 1)]
|
||||
frac = lambda s: sum(g <= s for g in gaps) / n
|
||||
out[niche] = {"n": n, "p50": q(.5), "p75": q(.75), "p90": q(.9),
|
||||
"within_4s": round(frac(4), 3),
|
||||
"within_10s": round(frac(10), 3),
|
||||
"within_25s": round(frac(25), 3)}
|
||||
print(f"{niche:<10} {n:>8,} {q(.5):>7.1f} {q(.75):>7.1f} {q(.9):>7.1f} "
|
||||
f"{frac(4):>6.0%} {frac(10):>6.0%} {frac(25):>6.0%}")
|
||||
out["_meta"] = {"jump": JUMP, "generated": time.strftime("%Y-%m-%d %H:%M")}
|
||||
json.dump(out, open(os.path.join(tape.HERE, "params", "requote_timing.json"),
|
||||
"w"), indent=1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
run()
|
||||
@@ -0,0 +1,92 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Study A robustness: (1) pessimistic fill — pay the WORST in-band print in
|
||||
the hold window (burst tops), not the first; (2) identity lift vs 10
|
||||
activity-matched control sets. Decides how the pre-registration is framed."""
|
||||
import json
|
||||
import os
|
||||
import statistics as st
|
||||
import time
|
||||
|
||||
import tape
|
||||
import sim as simmod
|
||||
import study_flow as sf
|
||||
|
||||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||
|
||||
|
||||
class PessimisticSim(simmod.Sim):
|
||||
def try_buy(self, asset, t_sig, p_ref, stake_usd=100.0, lag_s=None):
|
||||
lag = self.lag_s if lag_s is None else lag_s
|
||||
arrive = t_sig + lag
|
||||
cap = min(p_ref * (1 + self.slip_cap), 0.99)
|
||||
r = self.db.execute("""SELECT max(price) FROM trades
|
||||
WHERE asset = ? AND ts > ? AND ts <= ? AND price <= ?""",
|
||||
[asset, arrive, arrive + self.hold_s, cap]).fetchone()
|
||||
if not r or r[0] is None:
|
||||
return {"filled": False, "reason": "no print inside band"}
|
||||
px = float(r[0])
|
||||
shares = stake_usd / px
|
||||
return {"filled": True, "price": px, "shares": shares,
|
||||
"cost": shares * px, "fee": simmod.fee(shares, px, self.fee_rate),
|
||||
"fill_ts": arrive}
|
||||
|
||||
|
||||
def score_with(simcls, db, triggers, lag_s, hold_s):
|
||||
s = simcls(db, lag_s=lag_s, hold_s=hold_s)
|
||||
fills = wins = 0
|
||||
pnl = 0.0
|
||||
prices = []
|
||||
for t in triggers:
|
||||
pay = db.execute("SELECT payout::DOUBLE FROM res_tok WHERE asset = ?",
|
||||
[t["asset"]]).fetchone()
|
||||
if pay is None:
|
||||
continue
|
||||
r = s.try_buy(t["asset"], t["ts"], t["p_ref"])
|
||||
if not r["filled"]:
|
||||
continue
|
||||
fills += 1
|
||||
prices.append(r["price"])
|
||||
pnl += r["shares"] * (pay[0] - r["price"]) - r["fee"]
|
||||
wins += pay[0] == 1.0
|
||||
return {"fills": fills, "ev": round(pnl / fills, 2) if fills else None,
|
||||
"hit": round(wins / fills, 3) if fills else None,
|
||||
"avg_px": round(st.mean(prices), 3) if prices else None}
|
||||
|
||||
|
||||
def main():
|
||||
db = tape.connect()
|
||||
P = json.load(open(os.path.join(HERE, "params", "study_flow.json")))
|
||||
fz = P["frozen"]
|
||||
day = lambda d: time.mktime(time.strptime(f"2026-07-{d:02d}", "%Y-%m-%d")) \
|
||||
- time.timezone
|
||||
fit_lo, fit_hi = day(19), day(20)
|
||||
|
||||
S = sf.informed_set(db, fit_lo, fz["top_n"])
|
||||
tape.build_resolved(db)
|
||||
trig = sf.signals(db, S, fit_lo, fit_hi, fz["window_s"], fz["flow_usd"])
|
||||
opt = score_with(simmod.Sim, db, trig, simmod.LAG_P50, fz["hold_s"])
|
||||
pes = score_with(PessimisticSim, db, trig, simmod.LAG_P50, fz["hold_s"])
|
||||
print(f"informed first-print: {opt} \n worst-print: {pes}")
|
||||
|
||||
evs = []
|
||||
for seed in range(1, 11):
|
||||
C = sf.matched_random_set(db, fit_lo, fz["top_n"], seed)
|
||||
ctrig = sf.signals(db, C, fit_lo, fit_hi, fz["window_s"], fz["flow_usd"])
|
||||
c = score_with(PessimisticSim, db, ctrig, simmod.LAG_P50, fz["hold_s"])
|
||||
evs.append(c)
|
||||
print(f"control {seed:>2} worst-print: {c}")
|
||||
with_ev = [c["ev"] for c in evs if c["ev"] is not None]
|
||||
tot_fills = sum(c["fills"] for c in evs)
|
||||
wt = sum(c["ev"] * c["fills"] for c in evs if c["ev"] is not None) \
|
||||
/ max(tot_fills, 1)
|
||||
print(f"\ncontrols: {len(with_ev)} scored · pooled fills {tot_fills} · "
|
||||
f"fill-weighted EV {wt:+.2f} · mean {st.mean(with_ev):+.2f} · "
|
||||
f"informed pessimistic EV {pes['ev']:+.2f}")
|
||||
json.dump({"informed_first": opt, "informed_worst": pes,
|
||||
"controls_worst": evs, "controls_pooled_ev": round(wt, 2)},
|
||||
open(os.path.join(HERE, "params", "study_flow_robustness.json"),
|
||||
"w"), indent=1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
+167
@@ -0,0 +1,167 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Execution replayer calibrated on the live bot's OWN ledger.
|
||||
|
||||
Model: a signal at t with reference price p_ref becomes a marketable FAK
|
||||
arriving at t+lag with protected cap p_ref*(1+slip_cap). The tape has no
|
||||
book stream, so standing liquidity at arrival is proxied by PRINTS: the
|
||||
order fills at the first trade print on the token inside
|
||||
(arrive, arrive+hold_s] whose price is inside the cap — else it dies
|
||||
no-match (the crater). hold_s is NOT a free choice: `calibrate()` fits it
|
||||
so the model best separates the bot's real live fills (should fill) from
|
||||
its real FAK-rejected misses (should miss), and reports fill-price error
|
||||
with the bot's own prints EXCLUDED (else the validation is circular — our
|
||||
fill is itself a tape print).
|
||||
|
||||
Fees mirror the venue: fee = rate * shares * min(p, 1-p) (verified against
|
||||
the live ledger: 7.81sh @ .64 -> $0.0844, 5.26sh @ .95 -> $0.0075).
|
||||
|
||||
Everything is deterministic — scenarios (lag percentiles) not RNG.
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
|
||||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||
ROOT = os.path.dirname(HERE)
|
||||
BOT_WALLET = "0x455e252e45ee46d6c4cc1c8fadd3899d68f245a1"
|
||||
|
||||
FEE_RATE = 0.03
|
||||
LAG_P50, LAG_P90 = 6.7, 66.4 # live ledger 2026-07-20 (102 BUY fills)
|
||||
|
||||
|
||||
def fee(shares, price, rate=FEE_RATE):
|
||||
return rate * shares * min(price, 1.0 - price)
|
||||
|
||||
|
||||
class Sim:
|
||||
def __init__(self, db, lag_s=LAG_P50, slip_cap=0.05, hold_s=10,
|
||||
fee_rate=FEE_RATE, exclude_wallet=None, fill="first"):
|
||||
self.db = db
|
||||
self.lag_s = lag_s
|
||||
self.slip_cap = slip_cap
|
||||
self.hold_s = hold_s
|
||||
self.fee_rate = fee_rate
|
||||
self.excl = (exclude_wallet or "").lower()
|
||||
self.fill = fill # "first" print, or "worst" (pessimistic)
|
||||
|
||||
def first_print(self, asset, t0, t1, cap=None):
|
||||
"""First trade print on asset in (t0, t1], optionally inside cap."""
|
||||
q = """SELECT ts, price FROM trades
|
||||
WHERE asset = ? AND ts > ? AND ts <= ?"""
|
||||
args = [asset, t0, t1]
|
||||
if self.excl:
|
||||
q += " AND lower(wallet) != ?"
|
||||
args.append(self.excl)
|
||||
if cap is not None:
|
||||
q += " AND price <= ?"
|
||||
args.append(cap)
|
||||
q += " ORDER BY ts LIMIT 1"
|
||||
r = self.db.execute(q, args).fetchone()
|
||||
return r # (ts, price) or None
|
||||
|
||||
def try_buy(self, asset, t_sig, p_ref, stake_usd=100.0, lag_s=None):
|
||||
"""-> dict(filled, price, shares, cost, fee) — FAK with protected cap."""
|
||||
lag = self.lag_s if lag_s is None else lag_s
|
||||
arrive = t_sig + lag
|
||||
cap = min(p_ref * (1 + self.slip_cap), 0.99)
|
||||
if self.fill == "worst": # pay the top of the burst
|
||||
# (ORDER BY form: max(ts),max(price) trips a duckdb-internal
|
||||
# statistics-propagation assertion on this temp-table layout)
|
||||
pr = self.db.execute("""SELECT ts, price FROM trades
|
||||
WHERE asset = ? AND ts > ? AND ts <= ? AND price <= ?
|
||||
ORDER BY price DESC LIMIT 1""",
|
||||
[asset, arrive, arrive + self.hold_s, cap]).fetchone()
|
||||
else:
|
||||
pr = self.first_print(asset, arrive, arrive + self.hold_s, cap)
|
||||
if not pr:
|
||||
return {"filled": False, "reason": "no print inside band (crater)"}
|
||||
px = float(pr[1])
|
||||
shares = stake_usd / px
|
||||
return {"filled": True, "price": px, "shares": shares,
|
||||
"cost": shares * px, "fee": fee(shares, px, self.fee_rate),
|
||||
"fill_ts": pr[0]}
|
||||
|
||||
def markout(self, asset, t_fill, horizon_s):
|
||||
"""Last print at/before t_fill+horizon (None if nothing printed)."""
|
||||
r = self.db.execute("""SELECT price FROM trades WHERE asset = ?
|
||||
AND ts > ? AND ts <= ? ORDER BY ts DESC LIMIT 1""",
|
||||
[asset, t_fill, t_fill + horizon_s]).fetchone()
|
||||
return r[0] if r else None
|
||||
|
||||
|
||||
# ── calibration against the live ledger ─────────────────────────────────────
|
||||
|
||||
def _live_attempts(tape_lo, tape_hi):
|
||||
"""Real BUY attempts inside the tape window:
|
||||
fills from copybot_fills.live.jsonl (label filled=True) and FAK
|
||||
no-match misses from copybot_state.live.json (label filled=False)."""
|
||||
fills = []
|
||||
for ln in open(os.path.join(ROOT, "copybot_fills.live.jsonl")):
|
||||
r = json.loads(ln)
|
||||
if r.get("untracked") or r.get("side") == "SELL":
|
||||
continue
|
||||
if r.get("detect_lag_s") is None or not r.get("their_price"):
|
||||
continue
|
||||
t_sig = r["ts"] - r["detect_lag_s"]
|
||||
if not (tape_lo <= t_sig <= tape_hi - 120):
|
||||
continue
|
||||
fills.append({"filled": True, "asset": str(r["token"]),
|
||||
"t_sig": t_sig, "p_ref": r["their_price"],
|
||||
"lag": r["detect_lag_s"], "actual_px": r["my_price"]})
|
||||
st = json.load(open(os.path.join(ROOT, "copybot_state.live.json")))
|
||||
misses = []
|
||||
for m in st.get("missed", []):
|
||||
if "no orders found to match" not in str(m.get("reason", "")):
|
||||
continue
|
||||
if not (tape_lo <= m["ts"] <= tape_hi - 120):
|
||||
continue
|
||||
misses.append({"filled": False, "asset": str(m["token"]),
|
||||
"t_sig": m["ts"], "p_ref": m["price"], "lag": LAG_P50})
|
||||
return fills, misses
|
||||
|
||||
|
||||
def calibrate(db, tape_lo, tape_hi, holds=(3, 5, 10, 20, 45, 90)):
|
||||
"""Fit hold_s on real outcomes; report the confusion + price error."""
|
||||
fills, misses = _live_attempts(tape_lo, tape_hi)
|
||||
out = {"n_fills": len(fills), "n_misses": len(misses), "grid": {}}
|
||||
best = None
|
||||
for h in holds:
|
||||
sim = Sim(db, hold_s=h, exclude_wallet=BOT_WALLET)
|
||||
tp = sum(1 for a in fills
|
||||
if sim.try_buy(a["asset"], a["t_sig"], a["p_ref"],
|
||||
lag_s=a["lag"])["filled"])
|
||||
tn = sum(1 for a in misses
|
||||
if not sim.try_buy(a["asset"], a["t_sig"], a["p_ref"],
|
||||
lag_s=a["lag"])["filled"])
|
||||
acc = (tp + tn) / max(len(fills) + len(misses), 1)
|
||||
out["grid"][h] = {"fill_recall": tp / max(len(fills), 1),
|
||||
"miss_recall": tn / max(len(misses), 1),
|
||||
"acc": round(acc, 3)}
|
||||
if best is None or acc > best[1]:
|
||||
best = (h, acc)
|
||||
out["hold_s"] = best[0]
|
||||
sim = Sim(db, hold_s=best[0], exclude_wallet=BOT_WALLET)
|
||||
errs, signed = [], []
|
||||
for a in fills:
|
||||
r = sim.try_buy(a["asset"], a["t_sig"], a["p_ref"], lag_s=a["lag"])
|
||||
if r["filled"]:
|
||||
errs.append(abs(r["price"] - a["actual_px"]))
|
||||
signed.append(r["price"] - a["actual_px"])
|
||||
errs.sort()
|
||||
if errs:
|
||||
out["px_err_p50"] = round(errs[len(errs) // 2], 4)
|
||||
out["px_err_p90"] = round(errs[int(len(errs) * 0.9)], 4)
|
||||
out["px_within_1c"] = round(sum(e <= 0.01 for e in errs) / len(errs), 3)
|
||||
# signed bias: negative = sim fills cheaper than reality = OPTIMISTIC
|
||||
# (study EVs must clear |bias| + noise before they mean anything)
|
||||
out["px_bias_mean"] = round(sum(signed) / len(signed), 4)
|
||||
return out
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import tape
|
||||
db = tape.connect()
|
||||
lo, hi = db.execute("SELECT min(ts), max(ts) FROM trades").fetchone()
|
||||
cal = calibrate(db, lo, hi)
|
||||
print(json.dumps(cal, indent=2))
|
||||
json.dump(cal, open(os.path.join(HERE, "params", "sim_calibration.json"),
|
||||
"w"), indent=1)
|
||||
@@ -0,0 +1,218 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Study A — flow-state signal: trade the informed herd's lean, not any one
|
||||
wallet's print.
|
||||
|
||||
Hypothesis (pre-registered, issue TBD): when the trailing net $-flow of the
|
||||
tape-scored informed set crosses a threshold in one in-play sports/esports
|
||||
market, the market's resolution probability exceeds its price by enough to
|
||||
clear real execution (calibrated sim: lag, FAK crater, fees, ~2c optimism
|
||||
bias) — because the herd's aggregate lean IS the event detector, arriving
|
||||
before makers reprice.
|
||||
|
||||
Discipline:
|
||||
* informed set as-of T uses ONLY tape < T (scoring + resolutions as-of T).
|
||||
* FIT day and grid are fixed below; selection rule: highest after-fee EV
|
||||
per trigger at p50 lag with >= MIN_FILLS fills. Params freeze into
|
||||
params/study_flow.json; the holdout day is scored ONCE with frozen
|
||||
params; forward days accrue via forward.py.
|
||||
* Controls: 3 activity-matched shuffled sets through the identical
|
||||
pipeline — the edge must vanish when the wallets are random.
|
||||
|
||||
Signal: per token (sports/esports niche only), rolling W-second sum of
|
||||
signed informed flow (+buy$ / -sell$). Trigger when sum >= F with the
|
||||
triggering print inside PRICE_BAND; COOLDOWN_S per token. Entry at the
|
||||
triggering print's price through sim; hold to resolution; stake $100 flat.
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import time
|
||||
|
||||
import tape
|
||||
import sim as simmod
|
||||
|
||||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||
PARAMS_F = os.path.join(HERE, "params", "study_flow.json")
|
||||
|
||||
# pre-registered exploration grid + universe (do not widen after the fact)
|
||||
GRID = {"top_n": [50, 150], "window_s": [60, 300], "flow_usd": [300, 1000]}
|
||||
PRICE_BAND = (0.10, 0.90)
|
||||
NICHES = {"sports", "esports"}
|
||||
COOLDOWN_S = 900
|
||||
STAKE = 100.0
|
||||
MIN_FILLS = 30
|
||||
SET_MIN_Z, SET_MIN_BETS = 2.5, 6
|
||||
|
||||
|
||||
def informed_set(db, before_ts, top_n):
|
||||
"""Top-N wallets by improbability z using ONLY tape before before_ts."""
|
||||
tape.build_resolved(db, t_end=before_ts)
|
||||
rows = db.execute(f"""
|
||||
WITH bets AS (
|
||||
SELECT tr.wallet,
|
||||
any_value(tk.payout) payout,
|
||||
sum(CASE WHEN tr.side='BUY' THEN tr.size ELSE -tr.size END) net,
|
||||
sum(CASE WHEN tr.side='BUY' THEN tr.size*tr.price END)
|
||||
/ nullif(sum(CASE WHEN tr.side='BUY' THEN tr.size END),0) vwap
|
||||
FROM trades tr JOIN res_tok tk ON tr.asset = tk.asset
|
||||
WHERE tr.ts <= {before_ts}
|
||||
GROUP BY tr.wallet, tr.asset
|
||||
HAVING net >= 5 AND vwap BETWEEN 0.05 AND 0.95
|
||||
)
|
||||
SELECT wallet,
|
||||
count(*) n,
|
||||
sum(CASE WHEN payout=1.0 THEN 1 ELSE 0 END) wins,
|
||||
sum(vwap) exp_w, sum(vwap*(1-vwap)) var_s,
|
||||
sum(net*(payout - vwap)) pnl
|
||||
FROM bets GROUP BY wallet
|
||||
HAVING n >= {SET_MIN_BETS} AND var_s > 0 AND pnl > 0
|
||||
""").fetchall()
|
||||
scored = []
|
||||
for w, n, wins, exp_w, var_s, pnl in rows:
|
||||
z = (wins - exp_w) / (var_s ** 0.5)
|
||||
if z >= SET_MIN_Z:
|
||||
scored.append((z, w))
|
||||
scored.sort(reverse=True)
|
||||
return [w for _, w in scored[:top_n]]
|
||||
|
||||
|
||||
def matched_random_set(db, before_ts, size, seed):
|
||||
"""Activity-matched control: wallets with >= 20 trades before before_ts,
|
||||
deterministic pseudo-shuffle by md5(wallet||seed) — no RNG state."""
|
||||
rows = db.execute(f"""
|
||||
SELECT wallet FROM trades WHERE ts <= {before_ts}
|
||||
GROUP BY wallet HAVING count(*) >= 20
|
||||
ORDER BY md5(wallet || '{seed}') LIMIT {size}""").fetchall()
|
||||
return [w for (w,) in rows]
|
||||
|
||||
|
||||
def signals(db, wallets, t_lo, t_hi, window_s, flow_usd):
|
||||
"""Rolling-window triggers over the informed set's prints."""
|
||||
if not wallets:
|
||||
return []
|
||||
rows = db.execute("""
|
||||
SELECT tr.asset, tr.ts, tr.side, tr.price, tr.size, any_value(tr.title)
|
||||
FROM trades tr
|
||||
WHERE tr.ts > ? AND tr.ts <= ?
|
||||
AND tr.wallet IN (SELECT unnest(?::varchar[]))
|
||||
GROUP BY tr.asset, tr.ts, tr.side, tr.price, tr.size, tr.tx
|
||||
ORDER BY tr.asset, tr.ts""", [t_lo, t_hi, wallets]).fetchall()
|
||||
trig, cur, buf, last_trig = [], None, [], {}
|
||||
for asset, ts, side, price, size, title in rows:
|
||||
if asset != cur:
|
||||
cur, buf = asset, []
|
||||
if tape.niche(title) not in NICHES:
|
||||
continue
|
||||
usd = price * size * (1 if side == "BUY" else -1)
|
||||
buf.append((ts, usd))
|
||||
while buf and buf[0][0] < ts - window_s:
|
||||
buf.pop(0)
|
||||
flow = sum(u for _, u in buf)
|
||||
if flow >= flow_usd and PRICE_BAND[0] <= price <= PRICE_BAND[1] \
|
||||
and ts - last_trig.get(asset, 0) >= COOLDOWN_S:
|
||||
last_trig[asset] = ts
|
||||
trig.append({"asset": asset, "ts": ts, "p_ref": price,
|
||||
"flow": round(flow), "title": title})
|
||||
return trig
|
||||
|
||||
|
||||
def score(db, triggers, lag_s, hold_s):
|
||||
"""Sim each trigger; outcome from res_tok (resolved-by-tape-end only)."""
|
||||
s = simmod.Sim(db, lag_s=lag_s, hold_s=hold_s)
|
||||
res = dict(fills=0, misses=0, pending=0, pnl=0.0, wins=0, staked=0.0)
|
||||
for t in triggers:
|
||||
pay = db.execute("SELECT payout::DOUBLE FROM res_tok WHERE asset = ?",
|
||||
[t["asset"]]).fetchone()
|
||||
r = s.try_buy(t["asset"], t["ts"], t["p_ref"], stake_usd=STAKE)
|
||||
if not r["filled"]:
|
||||
res["misses"] += 1
|
||||
continue
|
||||
if pay is None: # fired, filled, not yet resolved
|
||||
res["pending"] += 1
|
||||
continue
|
||||
res["fills"] += 1
|
||||
res["staked"] += r["cost"]
|
||||
pnl = r["shares"] * (pay[0] - r["price"]) - r["fee"]
|
||||
res["pnl"] += pnl
|
||||
res["wins"] += pay[0] == 1.0
|
||||
if res["fills"]:
|
||||
res["ev_per_fill"] = round(res["pnl"] / res["fills"], 2)
|
||||
res["hit"] = round(res["wins"] / res["fills"], 3)
|
||||
res["pnl"] = round(res["pnl"], 2)
|
||||
return res
|
||||
|
||||
|
||||
def run_cell(db, as_of, t_lo, t_hi, top_n, window_s, flow_usd, hold_s, wallets=None):
|
||||
S = wallets if wallets is not None else informed_set(db, as_of, top_n)
|
||||
tape.build_resolved(db) # scoring truth = full tape
|
||||
trig = signals(db, S, t_lo, t_hi, window_s, flow_usd)
|
||||
out = {"triggers": len(trig), "set_size": len(S)}
|
||||
for lag, tag in ((simmod.LAG_P50, "p50"), (simmod.LAG_P90, "p90")):
|
||||
out[tag] = score(db, trig, lag, hold_s)
|
||||
return out, trig
|
||||
|
||||
|
||||
def main():
|
||||
db = tape.connect()
|
||||
cal = json.load(open(os.path.join(HERE, "params", "sim_calibration.json")))
|
||||
hold_s = cal["hold_s"]
|
||||
lo, hi = db.execute("SELECT min(ts), max(ts) FROM trades").fetchone()
|
||||
day = lambda d, h=0: time.mktime(time.strptime(f"2026-07-{d:02d}", "%Y-%m-%d")) \
|
||||
- time.timezone + h * 3600
|
||||
fit_lo, fit_hi = day(19), day(20) # fit day: Jul 19 UTC
|
||||
hold_lo, hold_hi = day(20), hi # holdout: Jul 20 (partial)
|
||||
|
||||
print("== FIT (Jul 19, set as-of Jul 19 00:00 UTC) ==")
|
||||
results = []
|
||||
for tn in GRID["top_n"]:
|
||||
for w in GRID["window_s"]:
|
||||
for f in GRID["flow_usd"]:
|
||||
r, _ = run_cell(db, fit_lo, fit_lo, fit_hi, tn, w, f, hold_s)
|
||||
ev = r["p50"].get("ev_per_fill")
|
||||
results.append(((tn, w, f), r))
|
||||
print(f"top{tn:<4} W={w:<4} F=${f:<5} -> trig {r['triggers']:>4} "
|
||||
f"fills {r['p50']['fills']:>3} miss {r['p50']['misses']:>3} "
|
||||
f"EV/fill {ev if ev is not None else '—'} "
|
||||
f"hit {r['p50'].get('hit', '—')}")
|
||||
eligible = [(p, r) for p, r in results
|
||||
if r["p50"]["fills"] >= MIN_FILLS and "ev_per_fill" in r["p50"]]
|
||||
if not eligible:
|
||||
print("\nNO cell reached MIN_FILLS — study inconclusive at this tape size.")
|
||||
return
|
||||
best_p, best_r = max(eligible, key=lambda pr: pr[1]["p50"]["ev_per_fill"])
|
||||
tn, w, f = best_p
|
||||
print(f"\nFROZEN: top_n={tn} window={w}s flow=${f} "
|
||||
f"(fit EV/fill {best_r['p50']['ev_per_fill']}, hit {best_r['p50']['hit']})")
|
||||
|
||||
print("\n== CONTROLS (fit day, matched random sets) ==")
|
||||
controls = []
|
||||
for seed in (1, 2, 3):
|
||||
S = matched_random_set(db, fit_lo, tn, seed)
|
||||
r, _ = run_cell(db, fit_lo, fit_lo, fit_hi, tn, w, f, hold_s, wallets=S)
|
||||
controls.append(r)
|
||||
print(f"seed {seed}: trig {r['triggers']} fills {r['p50']['fills']} "
|
||||
f"EV/fill {r['p50'].get('ev_per_fill', '—')} "
|
||||
f"hit {r['p50'].get('hit', '—')}")
|
||||
|
||||
print("\n== HOLDOUT (Jul 20 partial, set as-of Jul 20 00:00 UTC) ==")
|
||||
hr, htrig = run_cell(db, hold_lo, hold_lo, hold_hi, tn, w, f, hold_s)
|
||||
print(f"trig {hr['triggers']} fills {hr['p50']['fills']} "
|
||||
f"miss {hr['p50']['misses']} pending {hr['p50']['pending']} "
|
||||
f"EV/fill {hr['p50'].get('ev_per_fill', '—')} "
|
||||
f"hit {hr['p50'].get('hit', '—')} (p90 lag: EV "
|
||||
f"{hr['p90'].get('ev_per_fill', '—')})")
|
||||
|
||||
json.dump({"frozen": {"top_n": tn, "window_s": w, "flow_usd": f,
|
||||
"hold_s": hold_s, "price_band": PRICE_BAND,
|
||||
"niches": sorted(NICHES), "cooldown_s": COOLDOWN_S,
|
||||
"stake": STAKE,
|
||||
"set_min_z": SET_MIN_Z, "set_min_bets": SET_MIN_BETS},
|
||||
"fit": best_r, "fit_grid": [{"params": p, **r} for p, r in results],
|
||||
"controls": controls, "holdout": hr,
|
||||
"frozen_at": time.strftime("%Y-%m-%d %H:%M UTC", time.gmtime()),
|
||||
"pending_triggers": htrig},
|
||||
open(PARAMS_F, "w"), indent=1, default=float)
|
||||
print(f"\nfroze {PARAMS_F}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,250 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Study B — crypto oracle fair value vs the book.
|
||||
|
||||
The tape's `crypto_prices` aux stream IS the venue's settlement feed
|
||||
(Binance-sourced, ms-stamped, ~1/s per symbol since 2026-07-19 19:55). Every
|
||||
strike/sprint crypto market is a digital option on that feed, so fair value
|
||||
is computable tick-by-tick with no basis risk:
|
||||
|
||||
above K, expiry T: fair(Yes) = Phi( ln(S_t/K) / (sigma*sqrt(tau)) )
|
||||
between K1..K2: Phi(ln(K2/S)/sv) - Phi(ln(K1/S)/sv)
|
||||
sprint (window t0..t1): strike = S_{t0} read from the same feed
|
||||
Down/No tokens: 1 - fair(up-side). sigma = trailing 30min realized
|
||||
vol of 1s log-returns (drift negligible at these horizons).
|
||||
|
||||
Signal: at a market print, edge = fair - print >= E for that token ->
|
||||
simulated FAK entry (calibrated sim), hold to resolution (tape truth).
|
||||
|
||||
IMPORTANT scope honesty: tick coverage is ~21h, so there is no holdout —
|
||||
this run only CHOOSES E (grid below) and freezes it; ALL belief is deferred
|
||||
to the forward ledger. Also grades the live bot's own crypto fills against
|
||||
fair value at their fill times (objective score of the 0xbadaf319-class
|
||||
copies)."""
|
||||
import bisect
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import statistics as st
|
||||
import time
|
||||
|
||||
import tape
|
||||
import sim as simmod
|
||||
|
||||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||
PARAMS_F = os.path.join(HERE, "params", "study_oracle.json")
|
||||
|
||||
EDGE_GRID = [0.04, 0.07, 0.10]
|
||||
VOL_WIN_S = 1800
|
||||
TAU_MIN, TAU_MAX = 60, 12 * 3600
|
||||
COOLDOWN_S = 300
|
||||
STAKE = 100.0
|
||||
MIN_FILLS = 30
|
||||
UP_WORDS = {"up", "yes"}
|
||||
DOWN_WORDS = {"down", "no"}
|
||||
|
||||
|
||||
def phi(x):
|
||||
return 0.5 * (1.0 + math.erf(x / math.sqrt(2.0)))
|
||||
|
||||
|
||||
class TickSeries:
|
||||
def __init__(self, ticks):
|
||||
self.ts = [t for t, _ in ticks]
|
||||
self.px = [p for _, p in ticks]
|
||||
|
||||
def at(self, t):
|
||||
i = bisect.bisect_right(self.ts, t) - 1
|
||||
return self.px[i] if i >= 0 else None
|
||||
|
||||
def vol_1s(self, t, win=VOL_WIN_S):
|
||||
"""stdev of 1s log returns over the trailing window (per-sqrt-second)."""
|
||||
lo = bisect.bisect_left(self.ts, t - win)
|
||||
hi = bisect.bisect_right(self.ts, t)
|
||||
if hi - lo < 60:
|
||||
return None
|
||||
rets = []
|
||||
for i in range(lo + 1, hi):
|
||||
dt = self.ts[i] - self.ts[i - 1]
|
||||
if dt <= 0:
|
||||
continue
|
||||
r = math.log(self.px[i] / self.px[i - 1]) / math.sqrt(dt)
|
||||
rets.append(r)
|
||||
return st.pstdev(rets) if len(rets) >= 30 else None
|
||||
|
||||
|
||||
def fair_value(mkt, up_side, S, sigma, t):
|
||||
tau = mkt["t1"] - t
|
||||
if not (TAU_MIN <= tau <= TAU_MAX) or not S or not sigma:
|
||||
return None
|
||||
sv = sigma * math.sqrt(tau)
|
||||
if sv <= 0:
|
||||
return None
|
||||
k = mkt["kind"]
|
||||
if k == "sprint":
|
||||
if mkt.get("s0") is None:
|
||||
return None
|
||||
f_up = phi(math.log(S / mkt["s0"]) / sv)
|
||||
elif k == "above":
|
||||
f_up = phi(math.log(S / mkt["k1"]) / sv)
|
||||
elif k == "below":
|
||||
f_up = 1.0 - phi(math.log(S / mkt["k1"]) / sv)
|
||||
elif k == "between":
|
||||
f_up = phi(math.log(mkt["k2"] / S) / sv) - phi(math.log(mkt["k1"] / S) / sv)
|
||||
else:
|
||||
return None
|
||||
return f_up if up_side else 1.0 - f_up
|
||||
|
||||
|
||||
def outcome_map(db):
|
||||
"""asset -> lowercase outcome name, from the orders_matched aux stream."""
|
||||
rows = db.execute("""
|
||||
SELECT json_extract_string(payload,'$.asset'),
|
||||
lower(any_value(json_extract_string(payload,'$.outcome')))
|
||||
FROM aux WHERE type = 'orders_matched'
|
||||
AND json_extract_string(payload,'$.outcome') != ''
|
||||
GROUP BY 1""").fetchall()
|
||||
return {a: o for a, o in rows if a and o}
|
||||
|
||||
|
||||
def crypto_universe(db, outcomes, series):
|
||||
"""Parseable crypto tokens with a knowable side + tick coverage."""
|
||||
rows = db.execute("""
|
||||
SELECT asset, any_value(title), min(ts), max(ts)
|
||||
FROM trades GROUP BY asset""").fetchall()
|
||||
out = []
|
||||
for asset, title, lo, hi in rows:
|
||||
mkt = tape.crypto_parse(title or "")
|
||||
if not mkt or mkt["sym"] not in series:
|
||||
continue
|
||||
o = outcomes.get(asset, "")
|
||||
up = o in UP_WORDS or (o == "" and mkt["kind"] != "sprint")
|
||||
if o and o not in UP_WORDS | DOWN_WORDS:
|
||||
continue # unknown side label — skip honestly
|
||||
if mkt["kind"] == "sprint":
|
||||
if not o:
|
||||
continue # sprints NEED the Up/Down label
|
||||
mkt["s0"] = series[mkt["sym"]].at(mkt["t0"])
|
||||
if mkt["s0"] is None:
|
||||
continue
|
||||
out.append({"asset": asset, "mkt": mkt, "up": up, "title": title})
|
||||
return out
|
||||
|
||||
|
||||
def run_study(db, hold_s):
|
||||
series = {s: TickSeries(tape.load_ticks(db, s))
|
||||
for s in ("btcusdt", "ethusdt", "solusdt", "xrpusdt",
|
||||
"bnbusdt", "dogeusdt")}
|
||||
tick_lo = min(s.ts[0] for s in series.values() if s.ts)
|
||||
outcomes = outcome_map(db)
|
||||
tape.build_resolved(db)
|
||||
uni = crypto_universe(db, outcomes, series)
|
||||
payout = {a: p for a, p in db.execute(
|
||||
"SELECT asset, payout::DOUBLE FROM res_tok").fetchall()}
|
||||
print(f"crypto universe: {len(uni)} tokens with side + ticks "
|
||||
f"({sum(1 for u in uni if u['asset'] in payout)} resolved in-tape)")
|
||||
|
||||
events = [] # candidate mispricings at prints
|
||||
for u in uni:
|
||||
prints = db.execute("""SELECT ts, price FROM trades
|
||||
WHERE asset = ? AND ts >= ? ORDER BY ts""",
|
||||
[u["asset"], tick_lo]).fetchall()
|
||||
s = series[u["mkt"]["sym"]]
|
||||
last_ev = 0.0
|
||||
for ts, px in prints:
|
||||
if ts - last_ev < COOLDOWN_S:
|
||||
continue
|
||||
S = s.at(ts)
|
||||
sig = s.vol_1s(ts)
|
||||
f = fair_value(u["mkt"], u["up"], S, sig, ts)
|
||||
if f is None:
|
||||
continue
|
||||
edge = f - float(px)
|
||||
if edge > 0.02: # collect loosely; grid filters below
|
||||
last_ev = ts
|
||||
events.append({"asset": u["asset"], "ts": ts, "p_ref": float(px),
|
||||
"fair": round(f, 4), "edge": round(edge, 4),
|
||||
"kind": u["mkt"]["kind"], "title": u["title"]})
|
||||
print(f"candidate mispricing events (edge > 2c): {len(events)}")
|
||||
|
||||
sim = simmod.Sim(db, hold_s=hold_s)
|
||||
grid = {}
|
||||
for E in EDGE_GRID:
|
||||
sel = [e for e in events if e["edge"] >= E]
|
||||
fills = wins = 0
|
||||
pnl = staked = 0.0
|
||||
misses = pending = 0
|
||||
for e in sel:
|
||||
r = sim.try_buy(e["asset"], e["ts"], e["p_ref"], stake_usd=STAKE)
|
||||
if not r["filled"]:
|
||||
misses += 1
|
||||
continue
|
||||
pay = payout.get(e["asset"])
|
||||
if pay is None:
|
||||
pending += 1
|
||||
continue
|
||||
fills += 1
|
||||
staked += r["cost"]
|
||||
pnl += r["shares"] * (pay - r["price"]) - r["fee"]
|
||||
wins += pay == 1.0
|
||||
grid[E] = {"events": len(sel), "fills": fills, "misses": misses,
|
||||
"pending": pending,
|
||||
"ev_per_fill": round(pnl / fills, 2) if fills else None,
|
||||
"hit": round(wins / fills, 3) if fills else None,
|
||||
"pnl": round(pnl, 2)}
|
||||
print(f"E >= {E:.2f}: {grid[E]}")
|
||||
eligible = [(E, g) for E, g in grid.items()
|
||||
if g["fills"] >= MIN_FILLS and g["ev_per_fill"] is not None]
|
||||
frozen_E = max(eligible, key=lambda eg: eg[1]["ev_per_fill"])[0] \
|
||||
if eligible else None
|
||||
return {"grid": grid, "frozen_edge": frozen_E, "n_universe": len(uni),
|
||||
"tick_lo": tick_lo}, events
|
||||
|
||||
|
||||
def grade_our_fills(db):
|
||||
"""Fair-value edge of the live bot's own crypto fills at fill time."""
|
||||
series = {}
|
||||
graded = []
|
||||
for ln in open(os.path.join(tape.ROOT, "copybot_fills.live.jsonl")):
|
||||
r = json.loads(ln)
|
||||
if r.get("side") == "SELL" or r.get("untracked"):
|
||||
continue
|
||||
mkt = tape.crypto_parse(r.get("title") or "")
|
||||
if not mkt:
|
||||
continue
|
||||
sym = mkt["sym"]
|
||||
if sym not in series:
|
||||
series[sym] = TickSeries(tape.load_ticks(db, sym))
|
||||
s = series[sym]
|
||||
if not s.ts or r["ts"] < s.ts[0] or mkt["kind"] == "sprint":
|
||||
continue
|
||||
S, sig = s.at(r["ts"]), s.vol_1s(r["ts"])
|
||||
up = (r.get("outcome") or "").lower() in UP_WORDS
|
||||
f = fair_value(mkt, up, S, sig, r["ts"])
|
||||
if f is None:
|
||||
continue
|
||||
graded.append({"title": r["title"][:60], "outcome": r.get("outcome"),
|
||||
"px": r["my_price"], "fair": round(f, 3),
|
||||
"edge": round(f - r["my_price"], 3),
|
||||
"wallet": r.get("name")})
|
||||
return graded
|
||||
|
||||
|
||||
def main():
|
||||
db = tape.connect()
|
||||
cal = json.load(open(os.path.join(HERE, "params", "sim_calibration.json")))
|
||||
res, events = run_study(db, cal["hold_s"])
|
||||
graded = grade_our_fills(db)
|
||||
print(f"\nour crypto fills graded vs fair value: {len(graded)}")
|
||||
for g in graded:
|
||||
print(f" {g['edge']:+.3f} {g['wallet']:<12} {g['outcome']:<4} "
|
||||
f"@{g['px']:.3f} fair {g['fair']:.3f} {g['title']}")
|
||||
json.dump({**res, "our_fills_graded": graded,
|
||||
"frozen_at": time.strftime("%Y-%m-%d %H:%M UTC", time.gmtime()),
|
||||
"note": "NO holdout exists (21h ticks) — belief deferred "
|
||||
"entirely to forward_ledger"},
|
||||
open(PARAMS_F, "w"), indent=1, default=float)
|
||||
print(f"\nfroze {PARAMS_F}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,193 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Read-only tape access + ground truth for research/ (see README silo rules).
|
||||
|
||||
Resolution method is the one chain-validated 742/742 on 2026-07-20
|
||||
(live/tape_sharps.py first run): a token is proxy-resolved when its final-
|
||||
30min VWAP converged (>= 0.97 / <= 0.03) AND it went quiet >= QUIET_H before
|
||||
the tape end AND no sibling of the same condition disagrees (still trading,
|
||||
or a second proxy-winner). `chain_overlay()` upgrades any subset to CTF
|
||||
payout-vector truth via live/payouts.py (append-only shared cache).
|
||||
|
||||
Timezones: tape ts are epoch UTC. Polymarket crypto titles quote ET; the
|
||||
tape era is July 2026 = EDT = UTC-4 (ET_OFF). Sprints/hourlies embed their
|
||||
window in the title; "on <date>" dailies resolve at 12:00 ET by venue
|
||||
convention.
|
||||
"""
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
|
||||
import duckdb
|
||||
|
||||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||
ROOT = os.path.dirname(HERE)
|
||||
RTDS = os.path.join(ROOT, "live", "rtds.duckdb")
|
||||
|
||||
WIN_T, LOSE_T = 0.97, 0.03
|
||||
QUIET_H = 2
|
||||
ET_OFF = 4 * 3600 # EDT (July) = UTC-4
|
||||
YEAR = 2026 # tape era; revisit at year roll
|
||||
|
||||
MONTHS = {m: i + 1 for i, m in enumerate(
|
||||
["january", "february", "march", "april", "may", "june", "july",
|
||||
"august", "september", "october", "november", "december"])}
|
||||
|
||||
SYMBOLS = {"bitcoin": "btcusdt", "btc": "btcusdt",
|
||||
"ethereum": "ethusdt", "eth": "ethusdt",
|
||||
"solana": "solusdt", "sol": "solusdt",
|
||||
"xrp": "xrpusdt", "bnb": "bnbusdt", "doge": "dogeusdt",
|
||||
"dogecoin": "dogeusdt"}
|
||||
|
||||
|
||||
def connect():
|
||||
return duckdb.connect(RTDS, read_only=True)
|
||||
|
||||
|
||||
# ── tape proxy-resolution ───────────────────────────────────────────────────
|
||||
|
||||
def build_resolved(db, t_end=None):
|
||||
"""TEMP tables `res_tok` (asset, cond, last_ts, term_vwap, payout) and
|
||||
`res_bad` on this connection. Idempotent per connection."""
|
||||
if t_end is None:
|
||||
t_end = db.execute("SELECT max(ts) FROM trades").fetchone()[0]
|
||||
quiet = t_end - QUIET_H * 3600
|
||||
db.execute(f"""
|
||||
CREATE OR REPLACE TEMP TABLE _alltok AS
|
||||
WITH last AS (
|
||||
SELECT asset, any_value(cond) cond, max(ts) last_ts
|
||||
FROM trades WHERE cond IS NOT NULL AND cond != ''
|
||||
AND ts <= {t_end} GROUP BY asset
|
||||
), term AS (
|
||||
SELECT t.asset, sum(t.price * t.size) / nullif(sum(t.size), 0) term_vwap
|
||||
FROM trades t JOIN last l ON t.asset = l.asset
|
||||
WHERE t.ts >= l.last_ts - 1800 AND t.ts <= {t_end} GROUP BY t.asset
|
||||
)
|
||||
SELECT l.asset, l.cond, l.last_ts, tm.term_vwap,
|
||||
l.last_ts > {quiet} AS alive,
|
||||
CASE WHEN tm.term_vwap >= {WIN_T} THEN 1.0
|
||||
WHEN tm.term_vwap <= {LOSE_T} THEN 0.0 END AS payout
|
||||
FROM last l JOIN term tm ON l.asset = tm.asset""")
|
||||
db.execute("""
|
||||
CREATE OR REPLACE TEMP TABLE res_bad AS
|
||||
SELECT cond FROM _alltok GROUP BY cond
|
||||
HAVING bool_or(alive)
|
||||
OR sum(CASE WHEN payout = 1.0 THEN 1 ELSE 0 END) > 1""")
|
||||
db.execute("""
|
||||
CREATE OR REPLACE TEMP TABLE res_tok AS
|
||||
SELECT asset, cond, last_ts, term_vwap, payout FROM _alltok
|
||||
WHERE NOT alive AND payout IS NOT NULL
|
||||
AND cond NOT IN (SELECT cond FROM res_bad)""")
|
||||
return t_end
|
||||
|
||||
|
||||
def chain_overlay(pairs):
|
||||
"""[(cond, asset)] -> {(cond, asset): 1.0/0.0/0.5/None} via payouts.py.
|
||||
The only shared write in research/ (append-only resolutions cache)."""
|
||||
sys.path.insert(0, os.path.join(ROOT, "live"))
|
||||
import payouts
|
||||
payouts.ensure(sorted({c for c, _ in pairs}))
|
||||
return {(c, a): payouts.truth(c, a) for c, a in pairs}
|
||||
|
||||
|
||||
# ── title parsers ───────────────────────────────────────────────────────────
|
||||
|
||||
NICHE_PATTERNS = [
|
||||
("esports", ["lol:", "dota", "cs2", "csgo", "valorant", "esports",
|
||||
"bilibili", "map ", "game 1", "game 2", "game 3"]),
|
||||
("tennis", ["tennis", "atp", "wta", "wimbledon", "set winner"]),
|
||||
("sports", [" vs. ", " vs ", " @ ", "mlb", "nba", "nhl", "ufc",
|
||||
"world cup", "f1", "grand prix", "fifa"]),
|
||||
("crypto", ["bitcoin", "btc", "ethereum", "solana", "xrp", "doge",
|
||||
"price of", "up or down"]),
|
||||
("politics", ["election", "president", "senate", "governor", "mayor",
|
||||
"nominee", "impeach", "tariff", "fed ", "rate cut"]),
|
||||
("geo", ["iran", "israel", "russia", "ukraine", "china", "taiwan",
|
||||
"ceasefire", "strike", "nato"]),
|
||||
]
|
||||
|
||||
|
||||
def niche(title):
|
||||
t = (title or "").lower()
|
||||
for label, pats in NICHE_PATTERNS:
|
||||
if any(p in t for p in pats):
|
||||
return label
|
||||
return "other"
|
||||
|
||||
|
||||
def _et(mon, day, hh, mm):
|
||||
return time.mktime(time.struct_time(
|
||||
(YEAR, mon, day, 0, 0, 0, 0, 0, 0))) - time.timezone + hh * 3600 \
|
||||
+ mm * 60 + ET_OFF
|
||||
|
||||
|
||||
def _clock(h, m, ap):
|
||||
h = int(h) % 12 + (12 if ap.lower() == "pm" else 0)
|
||||
return h, int(m or 0)
|
||||
|
||||
|
||||
RE_SPRINT = re.compile(
|
||||
r"(?i)^(\w+)\s+up or down\s*-\s*(\w+)\s+(\d+),\s*"
|
||||
r"(\d+)(?::(\d+))?(am|pm)-(\d+)(?::(\d+))?(pm|am)\s*et")
|
||||
RE_HOURLY = re.compile(
|
||||
r"(?i)^(\w+)\s+(above|below)\s+([\d,\.]+)\s+on\s+(\w+)\s+(\d+),\s*"
|
||||
r"(\d+)(?::(\d+))?\s*(am|pm)\s*et")
|
||||
RE_DAILY = re.compile(
|
||||
r"(?i)price of (\w+) be (above|below|between)\s+\$?([\d,\.]+)"
|
||||
r"(?:\s+and\s+\$?([\d,\.]+))?\s+on\s+(\w+)\s+(\d+)")
|
||||
# NOT parsed on purpose: "dip to / reach $K" one-touch claims are
|
||||
# path-dependent (barrier, not terminal digital) — the Φ fair value below
|
||||
# would misprice them. v2 if the terminal edge proves out.
|
||||
|
||||
|
||||
def _num(s):
|
||||
return float(s.replace(",", "")) if s else None
|
||||
|
||||
|
||||
def crypto_parse(title):
|
||||
"""-> dict(sym, kind, k1, k2, t0, t1) or None.
|
||||
kind: sprint (S_t1 > S_t0), above/below/between (vs strike at t1).
|
||||
t0 only for sprints (window open)."""
|
||||
t = title or ""
|
||||
m = RE_SPRINT.match(t)
|
||||
if m:
|
||||
sym = SYMBOLS.get(m.group(1).lower())
|
||||
mon = MONTHS.get(m.group(2).lower())
|
||||
if not sym or not mon:
|
||||
return None
|
||||
day = int(m.group(3))
|
||||
h0, m0 = _clock(m.group(4), m.group(5), m.group(6))
|
||||
h1, m1 = _clock(m.group(7), m.group(8), m.group(9))
|
||||
return {"sym": sym, "kind": "sprint", "k1": None, "k2": None,
|
||||
"t0": _et(mon, day, h0, m0), "t1": _et(mon, day, h1, m1)}
|
||||
m = RE_HOURLY.match(t)
|
||||
if m:
|
||||
sym, mon = SYMBOLS.get(m.group(1).lower()), MONTHS.get(m.group(4).lower())
|
||||
if not sym or not mon:
|
||||
return None
|
||||
h, mi = _clock(m.group(6), m.group(7), m.group(8))
|
||||
return {"sym": sym, "kind": m.group(2).lower(), "k1": _num(m.group(3)),
|
||||
"k2": None, "t0": None,
|
||||
"t1": _et(mon, int(m.group(5)), h, mi)}
|
||||
m = RE_DAILY.search(t)
|
||||
if m:
|
||||
sym, mon = SYMBOLS.get(m.group(1).lower()), MONTHS.get(m.group(5).lower())
|
||||
if not sym or not mon:
|
||||
return None
|
||||
return {"sym": sym, "kind": m.group(2).lower(), "k1": _num(m.group(3)),
|
||||
"k2": _num(m.group(4)), "t0": None,
|
||||
"t1": _et(mon, int(m.group(6)), 12, 0)} # dailies: 12PM ET
|
||||
return None
|
||||
|
||||
|
||||
# ── tick series ─────────────────────────────────────────────────────────────
|
||||
|
||||
def load_ticks(db, sym):
|
||||
"""[(ts, price)] sorted — ms feed timestamps preferred over ingest ts."""
|
||||
rows = db.execute("""
|
||||
SELECT coalesce(cast(json_extract(payload,'$.timestamp') AS DOUBLE)/1000, ts) t,
|
||||
cast(json_extract(payload,'$.value') AS DOUBLE) v
|
||||
FROM aux WHERE topic = 'crypto_prices'
|
||||
AND json_extract_string(payload,'$.symbol') = ?
|
||||
ORDER BY 1""", [sym]).fetchall()
|
||||
return [(t, v) for t, v in rows if v]
|
||||
Reference in New Issue
Block a user