diff --git a/.gitignore b/.gitignore index d67b400..43fc46e 100644 --- a/.gitignore +++ b/.gitignore @@ -113,3 +113,4 @@ backtests/.claude/ # Generated multi-TF dataset cache data/multitf_dataset.parquet +data/training_data.parquet diff --git a/backtests/backtest_live_sync.py b/backtests/backtest_live_sync.py index 60c3026..3d47f06 100644 --- a/backtests/backtest_live_sync.py +++ b/backtests/backtest_live_sync.py @@ -884,7 +884,9 @@ def main(): # Fetch maximum historical data print("Fetching historical data...") - df = mt5.get_market_data(symbol="XAUUSD", timeframe="M15", count=50000) + import os + _symbol = os.getenv("SYMBOL", "XAUUSD") + df = mt5.get_market_data(symbol=_symbol, timeframe="M15", count=50000) if len(df) == 0: print("ERROR: No data received") diff --git a/data/training_data.parquet b/data/training_data.parquet deleted file mode 100644 index 6c6afe3..0000000 Binary files a/data/training_data.parquet and /dev/null differ diff --git a/models/backups/20260205_000415/hmm_regime.pkl b/models/backups/20260205_000415/hmm_regime.pkl deleted file mode 100644 index 0e33bd0..0000000 Binary files a/models/backups/20260205_000415/hmm_regime.pkl and /dev/null differ diff --git a/models/backups/20260205_000415/xgboost_model.pkl b/models/backups/20260205_000415/xgboost_model.pkl deleted file mode 100644 index 4fa206d..0000000 Binary files a/models/backups/20260205_000415/xgboost_model.pkl and /dev/null differ diff --git a/models/backups/20260206_050134/hmm_regime.pkl b/models/backups/20260206_050134/hmm_regime.pkl deleted file mode 100644 index ec28dfe..0000000 Binary files a/models/backups/20260206_050134/hmm_regime.pkl and /dev/null differ diff --git a/models/backups/20260206_050134/xgboost_model.pkl b/models/backups/20260206_050134/xgboost_model.pkl deleted file mode 100644 index 740e673..0000000 Binary files a/models/backups/20260206_050134/xgboost_model.pkl and /dev/null differ diff --git a/models/backups/20260210_020005/hmm_regime.pkl b/models/backups/20260210_020005/hmm_regime.pkl deleted file mode 100644 index 22954b4..0000000 Binary files a/models/backups/20260210_020005/hmm_regime.pkl and /dev/null differ diff --git a/models/backups/20260210_020005/xgboost_model.pkl b/models/backups/20260210_020005/xgboost_model.pkl deleted file mode 100644 index 32d9e45..0000000 Binary files a/models/backups/20260210_020005/xgboost_model.pkl and /dev/null differ diff --git a/scripts/collect_data.py b/scripts/collect_data.py new file mode 100644 index 0000000..4012006 --- /dev/null +++ b/scripts/collect_data.py @@ -0,0 +1,110 @@ +#!/usr/bin/env python3 +""" +Raw Data Collector (M1 + M15 GOLD) +================================== +Collects RAW OHLCV bars from MT5 (via the Linux Wine bridge) and saves them +UNPROCESSED to data/raw/. Keeping raw data separate from features means we can +re-run preprocessing/labeling experiments without re-downloading. + +Pulls the maximum the broker provides (paginated where possible). + +Prereq: bridge up -> scripts/mt5_bridge.sh up + +Usage: + python scripts/collect_data.py [--symbol GOLD] [--m1 99999] [--m15 99999] +""" +import argparse, os, sys +from datetime import datetime +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) +try: + from dotenv import load_dotenv; load_dotenv() +except ImportError: + pass + +import polars as pl +from loguru import logger + + +TF_MAP = {"M1": 1, "M5": 5, "M15": 15, "M30": 30, "H1": 16385} + + +def _rates_to_df(r): + return pl.DataFrame({ + "time": [datetime.utcfromtimestamp(int(x[0])) for x in r], + "open": [float(x[1]) for x in r], + "high": [float(x[2]) for x in r], + "low": [float(x[3]) for x in r], + "close": [float(x[4]) for x in r], + "volume":[float(x[5]) for x in r], + }) + + +def collect(m, symbol, tf_name, want): + """Fetch up to `want` bars, paginating backwards past the per-call cap.""" + tf = getattr(m, f"TIMEFRAME_{tf_name}") + frames = [] + r = m.copy_rates_from_pos(symbol, tf, 0, min(want, 99999)) + if r is None or len(r) == 0: + logger.error(f"{tf_name}: no data ({m.last_error()})") + return None + df = _rates_to_df(r) + frames.append(df) + got = df.height + oldest = df["time"].min() + + # paginate backwards + while got < want: + import datetime as dt + r = m.copy_rates_from(symbol, tf, oldest - dt.timedelta(minutes=TF_MAP[tf_name]), + min(want - got, 99999)) + if r is None or len(r) <= 1: + break + prev = _rates_to_df(r).filter(pl.col("time") < oldest) + if prev.height == 0: + break + frames.append(prev) + got += prev.height + new_oldest = prev["time"].min() + if new_oldest >= oldest: + break + oldest = new_oldest + + out = pl.concat(frames).unique(subset=["time"]).sort("time") + logger.info(f"{tf_name}: {out.height} bars | {out['time'].min()} -> {out['time'].max()}") + return out + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--symbol", default=os.getenv("SYMBOL", "GOLD")) + ap.add_argument("--m1", type=int, default=99999) + ap.add_argument("--m15", type=int, default=99999) + ap.add_argument("--host", default=os.getenv("MT5_BRIDGE_HOST", "127.0.0.1")) + ap.add_argument("--port", type=int, default=int(os.getenv("MT5_BRIDGE_PORT", "18812"))) + ap.add_argument("--outdir", default="data/raw") + args = ap.parse_args() + + from mt5linux import MetaTrader5 + m = MetaTrader5(host=args.host, port=args.port, timeout=240) + if not m.initialize(): + m.initialize(login=int(os.getenv("MT5_LOGIN", "0")), + password=os.getenv("MT5_PASSWORD", ""), + server=os.getenv("MT5_SERVER", ""), + path=os.getenv("MT5_WIN_PATH", "")) + m.symbol_select(args.symbol, True) + + Path(args.outdir).mkdir(parents=True, exist_ok=True) + for tf, want in (("M1", args.m1), ("M15", args.m15)): + df = collect(m, args.symbol, tf, want) + if df is not None: + p = f"{args.outdir}/{args.symbol}_{tf}.parquet" + df.write_parquet(p) + logger.info(f"saved -> {p}") + m.shutdown() + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/fast_backtest.py b/scripts/fast_backtest.py new file mode 100644 index 0000000..0da5f3a --- /dev/null +++ b/scripts/fast_backtest.py @@ -0,0 +1,170 @@ +#!/usr/bin/env python3 +""" +Fast Vectorized Backtest (validation) +===================================== +Answers ONE question quickly: does the model's signal produce a positive +expectancy / win-rate under realistic TP/SL — or was the headline win-rate an +artifact of look-ahead leakage? + +Speed: features are already in data/training_data.parquet, the model predicts +the WHOLE set in one batched GPU call, and trade outcomes are evaluated with a +single vectorized forward scan (no O(n^2) per-bar recompute). + +Trade model: + - Enter when model prob crosses the confidence threshold (long if p>=thr, + short if p<=1-thr). + - Exit via triple barrier: TP = tp_atr*ATR, SL = sl_atr*ATR, else time limit. + - Apply spread cost (points) per round trip. + +Usage: + python scripts/fast_backtest.py [--model models/xgboost_model.pkl] + [--data data/training_data.parquet] [--tp-atr 2 --sl-atr 1 --max-hold 24] + [--thr 0.6] [--spread 20] [--device cuda] +""" +import argparse, pickle, warnings, sys +from pathlib import Path +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) +warnings.filterwarnings("ignore") + +import numpy as np +import polars as pl +import xgboost as xgb +from loguru import logger + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--model", default="models/xgboost_model.pkl") + ap.add_argument("--data", default="data/training_data.parquet") + ap.add_argument("--tp-atr", type=float, default=2.0) + ap.add_argument("--sl-atr", type=float, default=1.0) + ap.add_argument("--max-hold", type=int, default=24) + ap.add_argument("--thr", type=float, default=None) + ap.add_argument("--spread", type=float, default=20.0, help="spread cost in points") + ap.add_argument("--device", default="cuda") + args = ap.parse_args() + + m = pickle.load(open(args.model, "rb")) + feats = m["feature_names"] + thr = args.thr if args.thr is not None else m.get("confidence_threshold", 0.6) + booster = m["model"] + + df = pl.read_parquet(args.data) + + d = df + # Required price cols must exist; feature cols may be missing (e.g. regime + # added later by HMM) -> fill with 0 to match live fallback. + price_need = {"high", "low", "close", "atr"} + pmiss = price_need - set(df.columns) + if pmiss: + logger.error(f"data missing price cols: {pmiss}") + return 1 + missing_feats = [f for f in feats if f not in d.columns] + if missing_feats: + logger.warning(f"filling {len(missing_feats)} missing feature(s) with 0: {missing_feats}") + d = d.with_columns([pl.lit(0.0).alias(f) for f in missing_feats]) + + d = d.drop_nulls(subset=list(price_need)) + n = d.height + X = np.nan_to_num(d.select(feats).to_numpy().astype(np.float32)) + close = d["close"].to_numpy().astype(np.float64) + high = d["high"].to_numpy().astype(np.float64) + low = d["low"].to_numpy().astype(np.float64) + atr = d["atr"].to_numpy().astype(np.float64) + + # --- batched GPU prediction (whole dataset at once) --- + dm = xgb.DMatrix(X, feature_names=feats) + try: + booster.set_param({"device": args.device}) + except Exception: + pass + prob = booster.predict(dm) + logger.info(f"predicted {n} bars | prob mean={prob.mean():.3f}") + + # --- signal: long p>=thr, short p<=1-thr --- + sig = np.zeros(n, dtype=np.int8) + sig[prob >= thr] = 1 + sig[prob <= (1 - thr)] = -1 + + # --- vectorized triple-barrier outcome per entry bar --- + H = args.max_hold + wins = losses = flat = 0 + pnl_points = 0.0 + rets = [] + n_trades = 0 + last_exit = -1 # simple non-overlap: no new entry until prior trade exits + + for i in range(n - H): + if sig[i] == 0 or i <= last_exit: + continue + a = atr[i] + if a <= 0 or np.isnan(a): + continue + entry = close[i] + direction = sig[i] + if direction == 1: + tp = entry + args.tp_atr * a + sl = entry - args.sl_atr * a + else: + tp = entry - args.tp_atr * a + sl = entry + args.sl_atr * a + + outcome = None + for j in range(1, H + 1): + hi, lo = high[i + j], low[i + j] + if direction == 1: + if lo <= sl: # SL first (conservative) + outcome = ("loss", sl); break + if hi >= tp: + outcome = ("win", tp); break + else: + if hi >= sl: + outcome = ("loss", sl); break + if lo <= tp: + outcome = ("win", tp); break + if outcome is None: + exit_px = close[i + H] + r = (exit_px - entry) * direction + outcome = ("win" if r > 0 else "loss", exit_px) + last_exit = i + H + else: + last_exit = i + j + + label, exit_px = outcome + gross = (exit_px - entry) * direction + net = gross - args.spread # spread cost per round trip (points) + pnl_points += net + rets.append(net) + n_trades += 1 + if net > 0: + wins += 1 + else: + losses += 1 + + rets = np.array(rets) if rets else np.array([0.0]) + win_rate = wins / n_trades * 100 if n_trades else 0 + gross_win = rets[rets > 0].sum() + gross_loss = -rets[rets < 0].sum() + pf = gross_win / gross_loss if gross_loss > 0 else float("inf") + expectancy = rets.mean() + sharpe = rets.mean() / rets.std() * np.sqrt(len(rets)) if rets.std() > 0 else 0 + + print("\n" + "=" * 50) + print("FAST VECTORIZED BACKTEST (validation)") + print("=" * 50) + print(f"Bars : {n}") + print(f"Threshold : {thr}") + print(f"TP/SL/hold : {args.tp_atr}/{args.sl_atr} ATR, {H} bars") + print(f"Spread cost : {args.spread} pts/trade") + print(f"Total trades : {n_trades}") + print(f"Win rate : {win_rate:.1f}%") + print(f"Profit factor : {pf:.2f}") + print(f"Expectancy : {expectancy:.2f} pts/trade") + print(f"Net P/L (points): {pnl_points:.0f}") + print(f"Sharpe (approx) : {sharpe:.2f}") + print("=" * 50) + return 0 + + +if __name__ == "__main__": + sys.exit(main())