Vanszs
a55148f232
Add M1+M15 multi-timeframe SMC scalping training pipeline (GPU XGBoost), then fix data-leakage and non-stationarity issues found in a skeptical audit. Pipeline: - src/triple_barrier.py: TP/SL/time labeling (ATR-scaled, asymmetric RR) - src/multi_tf_dataset.py: M1 base + M15 HTF context, point-in-time join_asof (only CLOSED M15 candles visible to each M1 bar - proven no leakage) - src/economic_calendar.py: point-in-time forecast/actual/surprise provider - src/smc_polars.py: add premium/discount + displacement SMC features - scripts/train_multitf_scalper.py: GPU (device=cuda) training + walk-forward - scripts/download_training_data.py: 1y data downloader Leakage / robustness fixes (audit): - CRITICAL: order block signal was written to the ORIGIN bar (future info); now assigned at the CONFIRMATION bar -> matches live conditions - replace non-stationary absolute features (ema_9/21, macd*) with scale-free forms (ema*_dist_atr, ema_spread_atr, macd_*_bps) -> valid at any price level - drop constant-zero calendar features from defaults (recurring provider has no real values); re-add when a real calendar CSV is configured - walk-forward + train/test now embargo the max_holding label horizon and drop warmup rows (NaN->0 artifacts) - news calendar features remain point-in-time (actual only at/after release) Honest result: after fixes the spurious +2.35% edge collapses to ~random (AUC 0.49). The prior edge was caused by the order-block look-ahead. Pipeline is now leakage-free; a real edge still needs more M1 history / better features. Also: test infra (pytest.ini asyncio, hmmlearn), TRAIN_BARS, cleanup of dead modules. 14 tests pass.
87 lines
2.9 KiB
Python
87 lines
2.9 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
Download ~1 year of training data from MT5 (via the Linux Wine bridge) and
|
|
build the full feature set (technical + SMC + news calendar). Saves to
|
|
data/training_data.parquet.
|
|
|
|
Prereq: bridge up -> scripts/mt5_bridge.sh up
|
|
|
|
Usage:
|
|
python scripts/download_training_data.py [--bars 35000] [--symbol GOLD]
|
|
"""
|
|
import argparse
|
|
import os
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
|
|
|
|
try:
|
|
from dotenv import load_dotenv
|
|
load_dotenv()
|
|
except ImportError:
|
|
pass
|
|
|
|
import polars as pl
|
|
from loguru import logger
|
|
|
|
from src.mt5_connector import MT5Connector
|
|
from src.feature_eng import FeatureEngineer
|
|
from src.smc_polars import SMCAnalyzer
|
|
|
|
|
|
def main() -> int:
|
|
ap = argparse.ArgumentParser()
|
|
ap.add_argument("--bars", type=int, default=int(os.getenv("TRAIN_BARS", "35000")),
|
|
help="Max M15 bars to request (~25k = 1 year). Broker returns up to its limit.")
|
|
ap.add_argument("--symbol", default=os.getenv("SYMBOL", "GOLD"))
|
|
ap.add_argument("--timeframe", default=os.getenv("EXECUTION_TIMEFRAME", "M15"))
|
|
ap.add_argument("--out", default="data/training_data.parquet")
|
|
args = ap.parse_args()
|
|
|
|
conn = MT5Connector(
|
|
login=int(os.getenv("MT5_LOGIN", "0")),
|
|
password=os.getenv("MT5_PASSWORD", ""),
|
|
server=os.getenv("MT5_SERVER", ""),
|
|
path=os.getenv("MT5_WIN_PATH") or os.getenv("MT5_PATH"),
|
|
)
|
|
if not conn.connect():
|
|
logger.error("Could not connect to MT5. Is the bridge up? (scripts/mt5_bridge.sh up)")
|
|
return 1
|
|
|
|
logger.info(f"Requesting {args.bars} bars of {args.symbol} {args.timeframe} ...")
|
|
df = conn.get_market_data(args.symbol, args.timeframe, args.bars)
|
|
conn.disconnect()
|
|
|
|
if df is None or len(df) == 0:
|
|
logger.error("No data returned. Check the symbol name (XM uses 'GOLD').")
|
|
return 1
|
|
|
|
n = len(df)
|
|
span = df["time"].max() - df["time"].min()
|
|
logger.info(f"Received {n} bars | {df['time'].min()} -> {df['time'].max()} ({span})")
|
|
if n < args.bars:
|
|
logger.warning(f"Broker returned fewer bars than requested ({n} < {args.bars}) — "
|
|
"this is the broker's max available history.")
|
|
|
|
# Build features (technical + SMC + time + NEWS calendar)
|
|
fe = FeatureEngineer()
|
|
df = fe.calculate_all(df, include_ml_features=True)
|
|
smc = SMCAnalyzer(swing_length=5)
|
|
df = smc.calculate_all(df)
|
|
df = fe.create_target(df, lookahead=1)
|
|
|
|
news_cols = [c for c in ("news_high_impact_today", "news_window",
|
|
"hours_to_news", "news_risk") if c in df.columns]
|
|
logger.info(f"News features present: {news_cols}")
|
|
logger.info(f"Total columns: {len(df.columns)}")
|
|
|
|
Path(args.out).parent.mkdir(parents=True, exist_ok=True)
|
|
df.write_parquet(args.out)
|
|
logger.info(f"Saved -> {args.out} ({n} rows, {len(df.columns)} cols)")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|