fix: remove leaked models + add honest data/backtest tooling
- Delete all old XGBoost/HMM models trained with the order-block look-ahead leak (models/backups/* + root models). They reproduced a fake 63.9% WR / 2.64 PF that collapses to ~35% WR / 0.95 PF once the leak is fixed. - scripts/collect_data.py: dedicated raw M1+M15 collector (paginated) - scripts/fast_backtest.py: vectorized GPU backtest for honest validation - backtest_live_sync.py: read SYMBOL from env (XM uses GOLD, not XAUUSD) - stop tracking generated data/training_data.parquet See upstream report: GifariKemal/xaubot-ai#4
This commit is contained in:
Vanszs
@@ -113,3 +113,4 @@ backtests/.claude/
|
||||
|
||||
# Generated multi-TF dataset cache
|
||||
data/multitf_dataset.parquet
|
||||
data/training_data.parquet
|
||||
|
||||
@@ -884,7 +884,9 @@ def main():
|
||||
|
||||
# Fetch maximum historical data
|
||||
print("Fetching historical data...")
|
||||
df = mt5.get_market_data(symbol="XAUUSD", timeframe="M15", count=50000)
|
||||
import os
|
||||
_symbol = os.getenv("SYMBOL", "XAUUSD")
|
||||
df = mt5.get_market_data(symbol=_symbol, timeframe="M15", count=50000)
|
||||
|
||||
if len(df) == 0:
|
||||
print("ERROR: No data received")
|
||||
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -0,0 +1,110 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Raw Data Collector (M1 + M15 GOLD)
|
||||
==================================
|
||||
Collects RAW OHLCV bars from MT5 (via the Linux Wine bridge) and saves them
|
||||
UNPROCESSED to data/raw/. Keeping raw data separate from features means we can
|
||||
re-run preprocessing/labeling experiments without re-downloading.
|
||||
|
||||
Pulls the maximum the broker provides (paginated where possible).
|
||||
|
||||
Prereq: bridge up -> scripts/mt5_bridge.sh up
|
||||
|
||||
Usage:
|
||||
python scripts/collect_data.py [--symbol GOLD] [--m1 99999] [--m15 99999]
|
||||
"""
|
||||
import argparse, os, sys
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
|
||||
try:
|
||||
from dotenv import load_dotenv; load_dotenv()
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
import polars as pl
|
||||
from loguru import logger
|
||||
|
||||
|
||||
TF_MAP = {"M1": 1, "M5": 5, "M15": 15, "M30": 30, "H1": 16385}
|
||||
|
||||
|
||||
def _rates_to_df(r):
|
||||
return pl.DataFrame({
|
||||
"time": [datetime.utcfromtimestamp(int(x[0])) for x in r],
|
||||
"open": [float(x[1]) for x in r],
|
||||
"high": [float(x[2]) for x in r],
|
||||
"low": [float(x[3]) for x in r],
|
||||
"close": [float(x[4]) for x in r],
|
||||
"volume":[float(x[5]) for x in r],
|
||||
})
|
||||
|
||||
|
||||
def collect(m, symbol, tf_name, want):
|
||||
"""Fetch up to `want` bars, paginating backwards past the per-call cap."""
|
||||
tf = getattr(m, f"TIMEFRAME_{tf_name}")
|
||||
frames = []
|
||||
r = m.copy_rates_from_pos(symbol, tf, 0, min(want, 99999))
|
||||
if r is None or len(r) == 0:
|
||||
logger.error(f"{tf_name}: no data ({m.last_error()})")
|
||||
return None
|
||||
df = _rates_to_df(r)
|
||||
frames.append(df)
|
||||
got = df.height
|
||||
oldest = df["time"].min()
|
||||
|
||||
# paginate backwards
|
||||
while got < want:
|
||||
import datetime as dt
|
||||
r = m.copy_rates_from(symbol, tf, oldest - dt.timedelta(minutes=TF_MAP[tf_name]),
|
||||
min(want - got, 99999))
|
||||
if r is None or len(r) <= 1:
|
||||
break
|
||||
prev = _rates_to_df(r).filter(pl.col("time") < oldest)
|
||||
if prev.height == 0:
|
||||
break
|
||||
frames.append(prev)
|
||||
got += prev.height
|
||||
new_oldest = prev["time"].min()
|
||||
if new_oldest >= oldest:
|
||||
break
|
||||
oldest = new_oldest
|
||||
|
||||
out = pl.concat(frames).unique(subset=["time"]).sort("time")
|
||||
logger.info(f"{tf_name}: {out.height} bars | {out['time'].min()} -> {out['time'].max()}")
|
||||
return out
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--symbol", default=os.getenv("SYMBOL", "GOLD"))
|
||||
ap.add_argument("--m1", type=int, default=99999)
|
||||
ap.add_argument("--m15", type=int, default=99999)
|
||||
ap.add_argument("--host", default=os.getenv("MT5_BRIDGE_HOST", "127.0.0.1"))
|
||||
ap.add_argument("--port", type=int, default=int(os.getenv("MT5_BRIDGE_PORT", "18812")))
|
||||
ap.add_argument("--outdir", default="data/raw")
|
||||
args = ap.parse_args()
|
||||
|
||||
from mt5linux import MetaTrader5
|
||||
m = MetaTrader5(host=args.host, port=args.port, timeout=240)
|
||||
if not m.initialize():
|
||||
m.initialize(login=int(os.getenv("MT5_LOGIN", "0")),
|
||||
password=os.getenv("MT5_PASSWORD", ""),
|
||||
server=os.getenv("MT5_SERVER", ""),
|
||||
path=os.getenv("MT5_WIN_PATH", ""))
|
||||
m.symbol_select(args.symbol, True)
|
||||
|
||||
Path(args.outdir).mkdir(parents=True, exist_ok=True)
|
||||
for tf, want in (("M1", args.m1), ("M15", args.m15)):
|
||||
df = collect(m, args.symbol, tf, want)
|
||||
if df is not None:
|
||||
p = f"{args.outdir}/{args.symbol}_{tf}.parquet"
|
||||
df.write_parquet(p)
|
||||
logger.info(f"saved -> {p}")
|
||||
m.shutdown()
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,170 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Fast Vectorized Backtest (validation)
|
||||
=====================================
|
||||
Answers ONE question quickly: does the model's signal produce a positive
|
||||
expectancy / win-rate under realistic TP/SL — or was the headline win-rate an
|
||||
artifact of look-ahead leakage?
|
||||
|
||||
Speed: features are already in data/training_data.parquet, the model predicts
|
||||
the WHOLE set in one batched GPU call, and trade outcomes are evaluated with a
|
||||
single vectorized forward scan (no O(n^2) per-bar recompute).
|
||||
|
||||
Trade model:
|
||||
- Enter when model prob crosses the confidence threshold (long if p>=thr,
|
||||
short if p<=1-thr).
|
||||
- Exit via triple barrier: TP = tp_atr*ATR, SL = sl_atr*ATR, else time limit.
|
||||
- Apply spread cost (points) per round trip.
|
||||
|
||||
Usage:
|
||||
python scripts/fast_backtest.py [--model models/xgboost_model.pkl]
|
||||
[--data data/training_data.parquet] [--tp-atr 2 --sl-atr 1 --max-hold 24]
|
||||
[--thr 0.6] [--spread 20] [--device cuda]
|
||||
"""
|
||||
import argparse, pickle, warnings, sys
|
||||
from pathlib import Path
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
|
||||
warnings.filterwarnings("ignore")
|
||||
|
||||
import numpy as np
|
||||
import polars as pl
|
||||
import xgboost as xgb
|
||||
from loguru import logger
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--model", default="models/xgboost_model.pkl")
|
||||
ap.add_argument("--data", default="data/training_data.parquet")
|
||||
ap.add_argument("--tp-atr", type=float, default=2.0)
|
||||
ap.add_argument("--sl-atr", type=float, default=1.0)
|
||||
ap.add_argument("--max-hold", type=int, default=24)
|
||||
ap.add_argument("--thr", type=float, default=None)
|
||||
ap.add_argument("--spread", type=float, default=20.0, help="spread cost in points")
|
||||
ap.add_argument("--device", default="cuda")
|
||||
args = ap.parse_args()
|
||||
|
||||
m = pickle.load(open(args.model, "rb"))
|
||||
feats = m["feature_names"]
|
||||
thr = args.thr if args.thr is not None else m.get("confidence_threshold", 0.6)
|
||||
booster = m["model"]
|
||||
|
||||
df = pl.read_parquet(args.data)
|
||||
|
||||
d = df
|
||||
# Required price cols must exist; feature cols may be missing (e.g. regime
|
||||
# added later by HMM) -> fill with 0 to match live fallback.
|
||||
price_need = {"high", "low", "close", "atr"}
|
||||
pmiss = price_need - set(df.columns)
|
||||
if pmiss:
|
||||
logger.error(f"data missing price cols: {pmiss}")
|
||||
return 1
|
||||
missing_feats = [f for f in feats if f not in d.columns]
|
||||
if missing_feats:
|
||||
logger.warning(f"filling {len(missing_feats)} missing feature(s) with 0: {missing_feats}")
|
||||
d = d.with_columns([pl.lit(0.0).alias(f) for f in missing_feats])
|
||||
|
||||
d = d.drop_nulls(subset=list(price_need))
|
||||
n = d.height
|
||||
X = np.nan_to_num(d.select(feats).to_numpy().astype(np.float32))
|
||||
close = d["close"].to_numpy().astype(np.float64)
|
||||
high = d["high"].to_numpy().astype(np.float64)
|
||||
low = d["low"].to_numpy().astype(np.float64)
|
||||
atr = d["atr"].to_numpy().astype(np.float64)
|
||||
|
||||
# --- batched GPU prediction (whole dataset at once) ---
|
||||
dm = xgb.DMatrix(X, feature_names=feats)
|
||||
try:
|
||||
booster.set_param({"device": args.device})
|
||||
except Exception:
|
||||
pass
|
||||
prob = booster.predict(dm)
|
||||
logger.info(f"predicted {n} bars | prob mean={prob.mean():.3f}")
|
||||
|
||||
# --- signal: long p>=thr, short p<=1-thr ---
|
||||
sig = np.zeros(n, dtype=np.int8)
|
||||
sig[prob >= thr] = 1
|
||||
sig[prob <= (1 - thr)] = -1
|
||||
|
||||
# --- vectorized triple-barrier outcome per entry bar ---
|
||||
H = args.max_hold
|
||||
wins = losses = flat = 0
|
||||
pnl_points = 0.0
|
||||
rets = []
|
||||
n_trades = 0
|
||||
last_exit = -1 # simple non-overlap: no new entry until prior trade exits
|
||||
|
||||
for i in range(n - H):
|
||||
if sig[i] == 0 or i <= last_exit:
|
||||
continue
|
||||
a = atr[i]
|
||||
if a <= 0 or np.isnan(a):
|
||||
continue
|
||||
entry = close[i]
|
||||
direction = sig[i]
|
||||
if direction == 1:
|
||||
tp = entry + args.tp_atr * a
|
||||
sl = entry - args.sl_atr * a
|
||||
else:
|
||||
tp = entry - args.tp_atr * a
|
||||
sl = entry + args.sl_atr * a
|
||||
|
||||
outcome = None
|
||||
for j in range(1, H + 1):
|
||||
hi, lo = high[i + j], low[i + j]
|
||||
if direction == 1:
|
||||
if lo <= sl: # SL first (conservative)
|
||||
outcome = ("loss", sl); break
|
||||
if hi >= tp:
|
||||
outcome = ("win", tp); break
|
||||
else:
|
||||
if hi >= sl:
|
||||
outcome = ("loss", sl); break
|
||||
if lo <= tp:
|
||||
outcome = ("win", tp); break
|
||||
if outcome is None:
|
||||
exit_px = close[i + H]
|
||||
r = (exit_px - entry) * direction
|
||||
outcome = ("win" if r > 0 else "loss", exit_px)
|
||||
last_exit = i + H
|
||||
else:
|
||||
last_exit = i + j
|
||||
|
||||
label, exit_px = outcome
|
||||
gross = (exit_px - entry) * direction
|
||||
net = gross - args.spread # spread cost per round trip (points)
|
||||
pnl_points += net
|
||||
rets.append(net)
|
||||
n_trades += 1
|
||||
if net > 0:
|
||||
wins += 1
|
||||
else:
|
||||
losses += 1
|
||||
|
||||
rets = np.array(rets) if rets else np.array([0.0])
|
||||
win_rate = wins / n_trades * 100 if n_trades else 0
|
||||
gross_win = rets[rets > 0].sum()
|
||||
gross_loss = -rets[rets < 0].sum()
|
||||
pf = gross_win / gross_loss if gross_loss > 0 else float("inf")
|
||||
expectancy = rets.mean()
|
||||
sharpe = rets.mean() / rets.std() * np.sqrt(len(rets)) if rets.std() > 0 else 0
|
||||
|
||||
print("\n" + "=" * 50)
|
||||
print("FAST VECTORIZED BACKTEST (validation)")
|
||||
print("=" * 50)
|
||||
print(f"Bars : {n}")
|
||||
print(f"Threshold : {thr}")
|
||||
print(f"TP/SL/hold : {args.tp_atr}/{args.sl_atr} ATR, {H} bars")
|
||||
print(f"Spread cost : {args.spread} pts/trade")
|
||||
print(f"Total trades : {n_trades}")
|
||||
print(f"Win rate : {win_rate:.1f}%")
|
||||
print(f"Profit factor : {pf:.2f}")
|
||||
print(f"Expectancy : {expectancy:.2f} pts/trade")
|
||||
print(f"Net P/L (points): {pnl_points:.0f}")
|
||||
print(f"Sharpe (approx) : {sharpe:.2f}")
|
||||
print("=" * 50)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in New Issue
Block a user