fix: remove leaked models + add honest data/backtest tooling

- Delete all old XGBoost/HMM models trained with the order-block look-ahead
  leak (models/backups/* + root models). They reproduced a fake 63.9% WR /
  2.64 PF that collapses to ~35% WR / 0.95 PF once the leak is fixed.
- scripts/collect_data.py: dedicated raw M1+M15 collector (paginated)
- scripts/fast_backtest.py: vectorized GPU backtest for honest validation
- backtest_live_sync.py: read SYMBOL from env (XM uses GOLD, not XAUUSD)
- stop tracking generated data/training_data.parquet

See upstream report: GifariKemal/xaubot-ai#4

.gitignore

@@ -113,3 +113,4 @@ backtests/.claude/
 
 # Generated multi-TF dataset cache
 data/multitf_dataset.parquet
+data/training_data.parquet

backtests/backtest_live_sync.py

@@ -884,7 +884,9 @@ def main():
 
     # Fetch maximum historical data
     print("Fetching historical data...")
-    df = mt5.get_market_data(symbol="XAUUSD", timeframe="M15", count=50000)
+    import os
+    _symbol = os.getenv("SYMBOL", "XAUUSD")
+    df = mt5.get_market_data(symbol=_symbol, timeframe="M15", count=50000)
 
     if len(df) == 0:
         print("ERROR: No data received")

scripts/collect_data.py

@@ -0,0 +1,110 @@
+#!/usr/bin/env python3
+"""
+Raw Data Collector (M1 + M15 GOLD)
+==================================
+Collects RAW OHLCV bars from MT5 (via the Linux Wine bridge) and saves them
+UNPROCESSED to data/raw/. Keeping raw data separate from features means we can
+re-run preprocessing/labeling experiments without re-downloading.
+
+Pulls the maximum the broker provides (paginated where possible).
+
+Prereq: bridge up -> scripts/mt5_bridge.sh up
+
+Usage:
+    python scripts/collect_data.py [--symbol GOLD] [--m1 99999] [--m15 99999]
+"""
+import argparse, os, sys
+from datetime import datetime
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
+try:
+    from dotenv import load_dotenv; load_dotenv()
+except ImportError:
+    pass
+
+import polars as pl
+from loguru import logger
+
+
+TF_MAP = {"M1": 1, "M5": 5, "M15": 15, "M30": 30, "H1": 16385}
+
+
+def _rates_to_df(r):
+    return pl.DataFrame({
+        "time":  [datetime.utcfromtimestamp(int(x[0])) for x in r],
+        "open":  [float(x[1]) for x in r],
+        "high":  [float(x[2]) for x in r],
+        "low":   [float(x[3]) for x in r],
+        "close": [float(x[4]) for x in r],
+        "volume":[float(x[5]) for x in r],
+    })
+
+
+def collect(m, symbol, tf_name, want):
+    """Fetch up to `want` bars, paginating backwards past the per-call cap."""
+    tf = getattr(m, f"TIMEFRAME_{tf_name}")
+    frames = []
+    r = m.copy_rates_from_pos(symbol, tf, 0, min(want, 99999))
+    if r is None or len(r) == 0:
+        logger.error(f"{tf_name}: no data ({m.last_error()})")
+        return None
+    df = _rates_to_df(r)
+    frames.append(df)
+    got = df.height
+    oldest = df["time"].min()
+
+    # paginate backwards
+    while got < want:
+        import datetime as dt
+        r = m.copy_rates_from(symbol, tf, oldest - dt.timedelta(minutes=TF_MAP[tf_name]),
+                              min(want - got, 99999))
+        if r is None or len(r) <= 1:
+            break
+        prev = _rates_to_df(r).filter(pl.col("time") < oldest)
+        if prev.height == 0:
+            break
+        frames.append(prev)
+        got += prev.height
+        new_oldest = prev["time"].min()
+        if new_oldest >= oldest:
+            break
+        oldest = new_oldest
+
+    out = pl.concat(frames).unique(subset=["time"]).sort("time")
+    logger.info(f"{tf_name}: {out.height} bars | {out['time'].min()} -> {out['time'].max()}")
+    return out
+
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--symbol", default=os.getenv("SYMBOL", "GOLD"))
+    ap.add_argument("--m1", type=int, default=99999)
+    ap.add_argument("--m15", type=int, default=99999)
+    ap.add_argument("--host", default=os.getenv("MT5_BRIDGE_HOST", "127.0.0.1"))
+    ap.add_argument("--port", type=int, default=int(os.getenv("MT5_BRIDGE_PORT", "18812")))
+    ap.add_argument("--outdir", default="data/raw")
+    args = ap.parse_args()
+
+    from mt5linux import MetaTrader5
+    m = MetaTrader5(host=args.host, port=args.port, timeout=240)
+    if not m.initialize():
+        m.initialize(login=int(os.getenv("MT5_LOGIN", "0")),
+                     password=os.getenv("MT5_PASSWORD", ""),
+                     server=os.getenv("MT5_SERVER", ""),
+                     path=os.getenv("MT5_WIN_PATH", ""))
+    m.symbol_select(args.symbol, True)
+
+    Path(args.outdir).mkdir(parents=True, exist_ok=True)
+    for tf, want in (("M1", args.m1), ("M15", args.m15)):
+        df = collect(m, args.symbol, tf, want)
+        if df is not None:
+            p = f"{args.outdir}/{args.symbol}_{tf}.parquet"
+            df.write_parquet(p)
+            logger.info(f"saved -> {p}")
+    m.shutdown()
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

scripts/fast_backtest.py

@@ -0,0 +1,170 @@
+#!/usr/bin/env python3
+"""
+Fast Vectorized Backtest (validation)
+=====================================
+Answers ONE question quickly: does the model's signal produce a positive
+expectancy / win-rate under realistic TP/SL — or was the headline win-rate an
+artifact of look-ahead leakage?
+
+Speed: features are already in data/training_data.parquet, the model predicts
+the WHOLE set in one batched GPU call, and trade outcomes are evaluated with a
+single vectorized forward scan (no O(n^2) per-bar recompute).
+
+Trade model:
+  - Enter when model prob crosses the confidence threshold (long if p>=thr,
+    short if p<=1-thr).
+  - Exit via triple barrier: TP = tp_atr*ATR, SL = sl_atr*ATR, else time limit.
+  - Apply spread cost (points) per round trip.
+
+Usage:
+  python scripts/fast_backtest.py [--model models/xgboost_model.pkl]
+      [--data data/training_data.parquet] [--tp-atr 2 --sl-atr 1 --max-hold 24]
+      [--thr 0.6] [--spread 20] [--device cuda]
+"""
+import argparse, pickle, warnings, sys
+from pathlib import Path
+sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
+warnings.filterwarnings("ignore")
+
+import numpy as np
+import polars as pl
+import xgboost as xgb
+from loguru import logger
+
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--model", default="models/xgboost_model.pkl")
+    ap.add_argument("--data", default="data/training_data.parquet")
+    ap.add_argument("--tp-atr", type=float, default=2.0)
+    ap.add_argument("--sl-atr", type=float, default=1.0)
+    ap.add_argument("--max-hold", type=int, default=24)
+    ap.add_argument("--thr", type=float, default=None)
+    ap.add_argument("--spread", type=float, default=20.0, help="spread cost in points")
+    ap.add_argument("--device", default="cuda")
+    args = ap.parse_args()
+
+    m = pickle.load(open(args.model, "rb"))
+    feats = m["feature_names"]
+    thr = args.thr if args.thr is not None else m.get("confidence_threshold", 0.6)
+    booster = m["model"]
+
+    df = pl.read_parquet(args.data)
+
+    d = df
+    # Required price cols must exist; feature cols may be missing (e.g. regime
+    # added later by HMM) -> fill with 0 to match live fallback.
+    price_need = {"high", "low", "close", "atr"}
+    pmiss = price_need - set(df.columns)
+    if pmiss:
+        logger.error(f"data missing price cols: {pmiss}")
+        return 1
+    missing_feats = [f for f in feats if f not in d.columns]
+    if missing_feats:
+        logger.warning(f"filling {len(missing_feats)} missing feature(s) with 0: {missing_feats}")
+        d = d.with_columns([pl.lit(0.0).alias(f) for f in missing_feats])
+
+    d = d.drop_nulls(subset=list(price_need))
+    n = d.height
+    X = np.nan_to_num(d.select(feats).to_numpy().astype(np.float32))
+    close = d["close"].to_numpy().astype(np.float64)
+    high = d["high"].to_numpy().astype(np.float64)
+    low = d["low"].to_numpy().astype(np.float64)
+    atr = d["atr"].to_numpy().astype(np.float64)
+
+    # --- batched GPU prediction (whole dataset at once) ---
+    dm = xgb.DMatrix(X, feature_names=feats)
+    try:
+        booster.set_param({"device": args.device})
+    except Exception:
+        pass
+    prob = booster.predict(dm)
+    logger.info(f"predicted {n} bars | prob mean={prob.mean():.3f}")
+
+    # --- signal: long p>=thr, short p<=1-thr ---
+    sig = np.zeros(n, dtype=np.int8)
+    sig[prob >= thr] = 1
+    sig[prob <= (1 - thr)] = -1
+
+    # --- vectorized triple-barrier outcome per entry bar ---
+    H = args.max_hold
+    wins = losses = flat = 0
+    pnl_points = 0.0
+    rets = []
+    n_trades = 0
+    last_exit = -1  # simple non-overlap: no new entry until prior trade exits
+
+    for i in range(n - H):
+        if sig[i] == 0 or i <= last_exit:
+            continue
+        a = atr[i]
+        if a <= 0 or np.isnan(a):
+            continue
+        entry = close[i]
+        direction = sig[i]
+        if direction == 1:
+            tp = entry + args.tp_atr * a
+            sl = entry - args.sl_atr * a
+        else:
+            tp = entry - args.tp_atr * a
+            sl = entry + args.sl_atr * a
+
+        outcome = None
+        for j in range(1, H + 1):
+            hi, lo = high[i + j], low[i + j]
+            if direction == 1:
+                if lo <= sl:  # SL first (conservative)
+                    outcome = ("loss", sl); break
+                if hi >= tp:
+                    outcome = ("win", tp); break
+            else:
+                if hi >= sl:
+                    outcome = ("loss", sl); break
+                if lo <= tp:
+                    outcome = ("win", tp); break
+        if outcome is None:
+            exit_px = close[i + H]
+            r = (exit_px - entry) * direction
+            outcome = ("win" if r > 0 else "loss", exit_px)
+            last_exit = i + H
+        else:
+            last_exit = i + j
+
+        label, exit_px = outcome
+        gross = (exit_px - entry) * direction
+        net = gross - args.spread  # spread cost per round trip (points)
+        pnl_points += net
+        rets.append(net)
+        n_trades += 1
+        if net > 0:
+            wins += 1
+        else:
+            losses += 1
+
+    rets = np.array(rets) if rets else np.array([0.0])
+    win_rate = wins / n_trades * 100 if n_trades else 0
+    gross_win = rets[rets > 0].sum()
+    gross_loss = -rets[rets < 0].sum()
+    pf = gross_win / gross_loss if gross_loss > 0 else float("inf")
+    expectancy = rets.mean()
+    sharpe = rets.mean() / rets.std() * np.sqrt(len(rets)) if rets.std() > 0 else 0
+
+    print("\n" + "=" * 50)
+    print("FAST VECTORIZED BACKTEST (validation)")
+    print("=" * 50)
+    print(f"Bars            : {n}")
+    print(f"Threshold       : {thr}")
+    print(f"TP/SL/hold      : {args.tp_atr}/{args.sl_atr} ATR, {H} bars")
+    print(f"Spread cost     : {args.spread} pts/trade")
+    print(f"Total trades    : {n_trades}")
+    print(f"Win rate        : {win_rate:.1f}%")
+    print(f"Profit factor   : {pf:.2f}")
+    print(f"Expectancy      : {expectancy:.2f} pts/trade")
+    print(f"Net P/L (points): {pnl_points:.0f}")
+    print(f"Sharpe (approx) : {sharpe:.2f}")
+    print("=" * 50)
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())