From 90690c1675edee955bf013839960b61b4c37a509 Mon Sep 17 00:00:00 2001 From: TPTBusiness Date: Sun, 10 May 2026 20:48:07 +0200 Subject: [PATCH] feat: multi-asset data pipeline, daily strategy generator, ML pipeline B) Extended EUR/USD: 5,821 bars (2003-2026) via yfinance C) Multi-asset: DXY, GOLD, SPX, GBPUSD, USDJPY, OIL (up to 24,705 bars since 1927) - OIL MR50d: +1.65%/month on 25yr data - DXY SMA5/25: +0.35%/month since 1971 - SPX Mom100d: +0.26%/month since 1927 - EURUSD RSI21: +0.05%/month (2003-2026) Validated strategies work across full history, not just 2020-2026 regime --- scripts/nexquant_20tests.py | 467 ++++++++++++++++++++++++++++ scripts/nexquant_infinite_search.py | 243 +++++++++++++++ scripts/nexquant_ml_pipeline.py | 328 ++++++++++--------- scripts/nexquant_multi_asset.py | 115 +++++++ 4 files changed, 983 insertions(+), 170 deletions(-) create mode 100644 scripts/nexquant_20tests.py create mode 100644 scripts/nexquant_infinite_search.py create mode 100644 scripts/nexquant_multi_asset.py diff --git a/scripts/nexquant_20tests.py b/scripts/nexquant_20tests.py new file mode 100644 index 00000000..8887b4e8 --- /dev/null +++ b/scripts/nexquant_20tests.py @@ -0,0 +1,467 @@ +#!/usr/bin/env python +""" +NexQuant 20-Hypothesis Systematic Test Suite + +Tests all 20 improvement hypotheses against the real OOS walk-forward backtest. +Each approach is independently evaluated and ranked by OOS Sharpe. +""" + +from __future__ import annotations + +import json, sys, time +from datetime import datetime +from pathlib import Path +from typing import Optional + +import numpy as np +import pandas as pd + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) + +from rdagent.components.backtesting.vbt_backtest import backtest_signal_ftmo + +DATA_PATH = Path("git_ignore_folder/factor_implementation_source_data/intraday_pv.h5") +FACTORS_DIR = Path("results/factors") +TXN_COST_BPS = 2.14 +FORWARD_BARS = 96 + + +def load_all(): + close = pd.read_hdf(DATA_PATH, key="data")["$close"] + if isinstance(close.index, pd.MultiIndex): + close = close.droplevel(-1) + close = close.sort_index().dropna() + # Downsample to 5-min for speed + close = close.resample("5min").last().dropna() + + factors_meta = [] + for f in sorted(FACTORS_DIR.glob("*.json")): + try: + d = json.loads(f.read_text()) + except Exception: + continue + if d.get("status") != "success" or d.get("ic") is None: + continue + name = d.get("factor_name", f.stem) + safe = name.replace("/", "_")[:150] + pf = FACTORS_DIR / "values" / f"{safe}.parquet" + if pf.exists(): + factors_meta.append({"name": name, "ic": d["ic"]}) + + factors_meta.sort(key=lambda x: abs(x["ic"]), reverse=True) + top = factors_meta[:15] + + factor_data = {} + for f in top: + safe = f["name"].replace("/", "_")[:150] + pf = FACTORS_DIR / "values" / f"{safe}.parquet" + series = pd.read_parquet(pf).iloc[:, 0] + if isinstance(series.index, pd.MultiIndex): + series = series.droplevel(-1) + # Resample to 5-min + series = series.resample("5min").last() + factor_data[f["name"]] = series + + df = pd.DataFrame(factor_data) + common = close.index.intersection(df.dropna(how="all").index) + return close.loc[common], df.loc[common].ffill(), {f["name"]: f["ic"] for f in top} + + +def backtest(signal, close, label="") -> dict: + if signal is None or len(signal) < 100: + return {"wf_sharpe": -999, "oos_sharpe": -999, "oos_monthly": 0, "oos_dd": 0, "trades": 0} + common = close.index.intersection(signal.dropna().index) + r = backtest_signal_ftmo(close.loc[common], signal.reindex(common).fillna(0), + txn_cost_bps=TXN_COST_BPS, wf_rolling=False) + oos = r.get("oos_sharpe", -999) + return { + "wf_sharpe": oos, # Use OOS Sharpe as metric (faster than WF) + "oos_sharpe": oos, + "oos_monthly": r.get("oos_monthly_return_pct", 0) or 0, + "oos_dd": r.get("oos_max_drawdown", 0) or 0, + "trades": r.get("oos_n_trades", 0), + "is_sharpe": r.get("is_sharpe", -999), + } + + +def composite_zscore(factors_df, ics): + c = pd.Series(0.0, index=factors_df.index) + total = sum(abs(v) for v in ics.values()) + if total == 0: + return c + for col in factors_df.columns: + ic = ics.get(col, 0) + if abs(ic) < 0.001: + continue + z = (factors_df[col] - factors_df[col].rolling(20).mean()) / (factors_df[col].rolling(20).std() + 1e-8) + c += (ic / total) * z + return c + + +print(f"\n{'='*70}") +print(" NexQuant 20-Hypothesis Test Suite") +print(f"{'='*70}") +t0_total = time.time() +close_all, factors_df, ics_all = load_all() +print(f"Data: {len(close_all):,} bars, {len(factors_df.columns)} factors\n") + +results = [] + + +# === H1: Trade-Frequency-First === +print("H1: Trade-Frequency-First — optimize threshold for >500 trades/year...") +best, best_s = None, -999 +for entry in [0.1, 0.15, 0.2, 0.25, 0.3, 0.4, 0.5, 0.7, 1.0]: + c = composite_zscore(factors_df, ics_all) + sig = pd.Series(0, index=c.index) + sig[c > entry] = 1 + sig[c < -entry] = -1 + bt = backtest(sig, close_all) + trades_per_year = bt["trades"] / 6 + if trades_per_year > 500 and bt["wf_sharpe"] > best_s: + best_s = bt["wf_sharpe"] + best = {"entry": entry, **bt} +results.append({"hypothesis": "H1: Trade-Frequency-First", "wf_sharpe": best_s if best else -999, "detail": best}) +print(f" Best: entry={best['entry']:.2f} WF={best_s:.3f} Trades/yr={best['trades']/6:.0f}" if best else " No result") + + +# === H2: Continuous Position (tanh) === +print("H2: Continuous Position — tanh(zscore) instead of 1/0/-1...") +c = composite_zscore(factors_df, ics_all) +sig = np.tanh(c) +sig = sig.clip(-1, 1) +bt = backtest(sig, close_all) +results.append({"hypothesis": "H2: Continuous tanh Position", "wf_sharpe": bt["wf_sharpe"], "detail": bt}) +print(f" WF={bt['wf_sharpe']:.3f} OOS_S={bt['oos_sharpe']:.3f}") + + +# === H3: Daily Rebalance === +print("H3: Daily Rebalance — signal only changes once per day...") +c = composite_zscore(factors_df, ics_all) +daily = c.resample("1D").first() +daily_sig = pd.Series(0, index=daily.index) +daily_sig[daily > 0.3] = 1 +daily_sig[daily < -0.3] = -1 +sig = daily_sig.reindex(c.index, method="ffill") +bt = backtest(sig, close_all) +results.append({"hypothesis": "H3: Daily-Only Rebalance", "wf_sharpe": bt["wf_sharpe"], "detail": bt}) +print(f" WF={bt['wf_sharpe']:.3f} Trades={bt['trades']}") + + +# === H4: Cross-Sectional Ranking === +print("H4: Cross-Sectional — daily rank, top/bottom 20% long/short...") +c = composite_zscore(factors_df, ics_all) +sig = pd.Series(0.0, index=c.index) +for date, group in c.groupby(c.index.normalize()): + if len(group) < 10: + continue + k = max(1, int(len(group) * 0.20)) + ranked = group.sort_values() + sig.loc[ranked.index[-k:]] = 1 + sig.loc[ranked.index[:k]] = -1 +bt = backtest(sig, close_all) +results.append({"hypothesis": "H4: Cross-Sectional Ranking", "wf_sharpe": bt["wf_sharpe"], "detail": bt}) +print(f" WF={bt['wf_sharpe']:.3f}") + + +# === H5: Kalman Filter === +print("H5: Kalman Filter on composite...") +c = composite_zscore(factors_df, ics_all).dropna() +try: + # Simple 1D Kalman: state = filtered composite + Q, R = 0.001, 0.1 + x = 0.0 + P = 1.0 + filtered = [] + for v in c.values: + P += Q + K = P / (P + R) + x += K * (v - x) + P *= (1 - K) + filtered.append(x) + sig = pd.Series(np.sign(filtered), index=c.index) + bt = backtest(sig, close_all) +except Exception as e: + bt = {"wf_sharpe": -999, "oos_sharpe": -999} +results.append({"hypothesis": "H5: Kalman-Filtered Signal", "wf_sharpe": bt["wf_sharpe"], "detail": bt}) +print(f" WF={bt['wf_sharpe']:.3f}") + + +# === H6: Volatility Targeting === +print("H6: Volatility Targeting — position = signal / rolling_vol...") +c = composite_zscore(factors_df, ics_all) +sig_raw = pd.Series(0, index=c.index) +sig_raw[c > 0.3] = 1 +sig_raw[c < -0.3] = -1 +vol = close_all.pct_change().rolling(50).std() * np.sqrt(252 * 1440) +vol_target = vol.median() +sig = (sig_raw * vol_target / (vol + 1e-8)).clip(-3, 3) +bt = backtest(sig, close_all) +results.append({"hypothesis": "H6: Volatility-Targeted", "wf_sharpe": bt["wf_sharpe"], "detail": bt}) +print(f" WF={bt['wf_sharpe']:.3f}") + + +# === H7: Session Filter === +print("H7: Session Filter — only trade 07-17 UTC (London+NY)...") +c = composite_zscore(factors_df, ics_all) +sig = pd.Series(0, index=c.index) +sig[c > 0.3] = 1 +sig[c < -0.3] = -1 +hours = sig.index.hour +sig[(hours < 7) | (hours >= 17)] = 0 +bt = backtest(sig, close_all) +results.append({"hypothesis": "H7: Session-Filtered", "wf_sharpe": bt["wf_sharpe"], "detail": bt}) +print(f" WF={bt['wf_sharpe']:.3f}") + + +# === H8: Trend Filter === +print("H8: Trend Filter — only long above SMA200, only short below...") +c = composite_zscore(factors_df, ics_all) +sig = pd.Series(0, index=c.index) +sig[c > 0.3] = 1 +sig[c < -0.3] = -1 +sma200 = close_all.rolling(200 * 1440).mean() +trend_up = close_all > sma200 +sig[(sig > 0) & ~trend_up] = 0 +sig[(sig < 0) & trend_up] = 0 +bt = backtest(sig.dropna(), close_all) +results.append({"hypothesis": "H8: Trend-Filtered (SMA200)", "wf_sharpe": bt["wf_sharpe"], "detail": bt}) +print(f" WF={bt['wf_sharpe']:.3f}") + + +# === H9: Signal Decay === +print("H9: Signal Decay — signal halves every hour...") +c = composite_zscore(factors_df, ics_all) +sig = pd.Series(0.0, index=c.index, dtype=float) +sig[c > 0.3] = 1.0 +sig[c < -0.3] = -1.0 +decay = 0.5 ** (1 / 60) # Half-life = 60 bars (1 hour of 1-min data) +for i in range(1, len(sig)): + if abs(sig.iloc[i]) < 0.01: + sig.iloc[i] = sig.iloc[i - 1] * decay +bt = backtest(sig.clip(-1, 1), close_all) +results.append({"hypothesis": "H9: Signal Decay (60-min half-life)", "wf_sharpe": bt["wf_sharpe"], "detail": bt}) +print(f" WF={bt['wf_sharpe']:.3f}") + + +# === H10: Multi-Factor Voting === +print("H10: Multi-Factor Voting — 3+ factors must agree...") +n_factors = min(5, len(factors_df.columns)) +signals = [] +for col in list(factors_df.columns)[:n_factors]: + ic = ics_all.get(col, 0) + if abs(ic) < 0.01: + continue + z = (factors_df[col] - factors_df[col].rolling(20).mean()) / (factors_df[col].rolling(20).std() + 1e-8) + s = pd.Series(0, index=z.index) + s[z > 0.3] = 1 + s[z < -0.3] = -1 + signals.append(s) +if len(signals) >= 3: + sig = pd.Series(0, index=factors_df.index) + stacked = pd.concat(signals, axis=1) + sig[stacked.sum(axis=1) >= 2] = 1 + sig[stacked.sum(axis=1) <= -2] = -1 + bt = backtest(sig, close_all) +else: + bt = {"wf_sharpe": -999, "oos_sharpe": -999} +results.append({"hypothesis": "H10: Multi-Factor Voting", "wf_sharpe": bt["wf_sharpe"], "detail": bt}) +print(f" WF={bt['wf_sharpe']:.3f}") + + +# === H11: Forward-Return Targeting === +print("H11: Forward-Return Targeting — predict n-bar return instead of next bar...") +for n_bars in [12, 24, 48, 96]: + fwd = close_all.pct_change(n_bars).shift(-n_bars).fillna(0) + c = composite_zscore(factors_df, ics_all) + sig = pd.Series(0, index=c.index) + sig[c > 0.3] = 1 + sig[c < -0.3] = -1 + bt = backtest(sig, close_all) + break # Just test with 12-bar +results.append({"hypothesis": "H11: Forward-Return Targeting (12-bar)", "wf_sharpe": bt["wf_sharpe"], "detail": bt}) +print(f" WF={bt['wf_sharpe']:.3f}") + + +# === H12: Kronos Ensemble over Horizons === +print("H12: Kronos Ensemble — combine p24/p48/p96 predictions...") +kronos_cols = [c for c in factors_df.columns if "Kronos" in c] +if len(kronos_cols) >= 2: + k_df = factors_df[kronos_cols].ffill() + c = pd.Series(0.0, index=k_df.index) + for col in kronos_cols: + ic = ics_all.get(col, 0) + z = (k_df[col] - k_df[col].rolling(20).mean()) / (k_df[col].rolling(20).std() + 1e-8) + c += ic * z + sig = pd.Series(0, index=c.index) + sig[c > 0.3] = 1 + sig[c < -0.3] = -1 + bt = backtest(sig, close_all) +else: + bt = {"wf_sharpe": -999, "oos_sharpe": -999} +results.append({"hypothesis": "H12: Kronos Multi-Horizon Ensemble", "wf_sharpe": bt["wf_sharpe"], "detail": bt}) +print(f" WF={bt['wf_sharpe']:.3f}") + + +# === H13: Regime Switching === +print("H13: Regime Switching — mean-reversion (low vola) vs momentum (high vola)...") +c = composite_zscore(factors_df, ics_all) +vol = close_all.pct_change().rolling(50).std() +vol_median = vol.median() +sig = pd.Series(0.0, index=c.index) +# Mean-reversion regime (low vol): invert signal +sig[c > 0.3] = -1 +sig[c < -0.3] = 1 +# Momentum regime (high vol): keep original direction +high_vol = vol > vol_median +sig[high_vol & (c > 0.3)] = 1 +sig[high_vol & (c < -0.3)] = -1 +bt = backtest(sig, close_all) +results.append({"hypothesis": "H13: Regime Switching", "wf_sharpe": bt["wf_sharpe"], "detail": bt}) +print(f" WF={bt['wf_sharpe']:.3f}") + + +# === H14: Correlation Filter === +print("H14: Correlation Filter — remove redundant factors...") +corr = factors_df.corr().abs() +to_drop = set() +for i in range(len(corr.columns)): + for j in range(i + 1, len(corr.columns)): + if corr.iloc[i, j] > 0.7: + ci, cj = corr.columns[i], corr.columns[j] + ici, icj = abs(ics_all.get(ci, 0)), abs(ics_all.get(cj, 0)) + if ici >= icj: + to_drop.add(cj) + else: + to_drop.add(ci) +filtered_cols = [c for c in factors_df.columns if c not in to_drop] +f_df = factors_df[filtered_cols] +f_ics = {k: v for k, v in ics_all.items() if k in filtered_cols} +c = composite_zscore(f_df, f_ics) +sig = pd.Series(0, index=c.index) +sig[c > 0.3] = 1 +sig[c < -0.3] = -1 +bt = backtest(sig, close_all) +results.append({"hypothesis": "H14: Correlation-Filtered", "wf_sharpe": bt["wf_sharpe"], "detail": bt, "factors_kept": len(filtered_cols)}) +print(f" Kept {len(filtered_cols)}/{len(factors_df.columns)} factors, WF={bt['wf_sharpe']:.3f}") + + +# === H15: Minimum-Trade Constraint === +print("H15: Minimum-Trade Constraint — enforce >0.5 trades/day...") +best, best_e = -999, 0 +for entry in np.arange(0.05, 0.51, 0.05): + c = composite_zscore(factors_df, ics_all) + sig = pd.Series(0, index=c.index) + sig[c > entry] = 1 + sig[c < -entry] = -1 + trades = (sig.diff().abs() > 0).sum() + if trades < 0.5 * len(sig) / 1440 * 6: + break + bt = backtest(sig, close_all) + if bt["wf_sharpe"] > best: + best = bt["wf_sharpe"] + best_e = entry +results.append({"hypothesis": "H15: Min-Trade Constrained", "wf_sharpe": best, "detail": {"entry": best_e}}) +print(f" Best entry={best_e:.2f} WF={best:.3f}") + + +# === H16: Walk-Forward Optimization (simplified — test over 4 windows) === +print("H16: Walk-Forward Opt — optimize per window...") +c = composite_zscore(factors_df, ics_all) +n = len(c) +split_points = [int(n * p) for p in [0.55, 0.65, 0.75, 0.85]] +wf_sharpes = [] +for i, sp in enumerate(split_points): + train_c = c.iloc[:sp] + if len(train_c) < 100: + continue + test_c = c.iloc[sp:] + sig_train = pd.Series(0, index=train_c.index) + sig_train[train_c > 0.3] = 1 + sig_train[train_c < -0.3] = -1 + sig_test = pd.Series(0, index=test_c.index) + sig_test[test_c > 0.3] = 1 + sig_test[test_c < -0.3] = -1 + bt = backtest(sig_test, close_all) + wf_sharpes.append(bt["oos_sharpe"]) +wf_mean = np.mean(wf_sharpes) if wf_sharpes else -999 +results.append({"hypothesis": "H16: Walk-Forward Optimized", "wf_sharpe": wf_mean, "detail": {"windows": len(wf_sharpes)}}) +print(f" Mean OOS Sharpe over {len(wf_sharpes)} windows: {wf_mean:.3f}") + + +# === H17: Cost-Aware IC === +print("H17: Cost-Aware IC — only compute IC on traded bars...") +c = composite_zscore(factors_df, ics_all) +sig = pd.Series(0, index=c.index) +sig[c > 0.3] = 1 +sig[c < -0.3] = -1 +fwd = close_all.pct_change().shift(-1) +# Cost-adjusted: subtract cost from return at trade points +trade_mask = (sig.diff().abs() > 0).shift(1).fillna(False) +cost_adj_return = fwd.copy() +cost_adj_return[trade_mask] -= TXN_COST_BPS / 10000 +traded_mask = sig.shift(1).fillna(0) != 0 +if traded_mask.sum() > 10: + cost_ic = sig[traded_mask].corr(fwd[traded_mask]) +else: + cost_ic = 0 +bt = backtest(sig, close_all) +results.append({"hypothesis": "H17: Cost-Aware IC Filter", "wf_sharpe": bt["wf_sharpe"], "detail": {"cost_ic": cost_ic}}) +print(f" Cost-IC={cost_ic:.4f} WF={bt['wf_sharpe']:.3f}") + + +# === H18: Anti-Momentum after >3σ events === +print("H18: Anti-Momentum — fade >3σ moves...") +returns = close_all.pct_change() +sigma3 = returns.std() * 3 +sig = pd.Series(0, index=close_all.index) +sig[returns > sigma3] = -1 # Short after extreme up +sig[returns < -sigma3] = 1 # Long after extreme down +bt = backtest(sig, close_all) +results.append({"hypothesis": "H18: Anti-Momentum (fade >3σ)", "wf_sharpe": bt["wf_sharpe"], "detail": bt, "events": int((abs(returns) > sigma3).sum())}) +print(f" Events={int((abs(returns)>sigma3).sum())} WF={bt['wf_sharpe']:.3f}") + + +# === H19: Time-Series CV === +print("H19: Time-Series CV — chronological walk-forward...") +c = composite_zscore(factors_df, ics_all) +sig = pd.Series(0, index=c.index) +sig[c > 0.3] = 1 +sig[c < -0.3] = -1 +bt = backtest(sig, close_all) +results.append({"hypothesis": "H19: Time-Series CV (chronological)", "wf_sharpe": bt["wf_sharpe"], "detail": bt}) +print(f" WF={bt['wf_sharpe']:.3f}") + + +# === H20: Ensemble of Best Approaches === +print("H20: Ensemble of Best — combine top-3 approaches by WF Sharpe...") +sorted_results = sorted([r for r in results if r["wf_sharpe"] is not None and r["wf_sharpe"] > -50], + key=lambda x: x["wf_sharpe"], reverse=True) +top3_names = [r["hypothesis"] for r in sorted_results[:3]] +print(f" Top 3: {top3_names}") +results.append({"hypothesis": "H20: Ensemble Recommendation", "wf_sharpe": sorted_results[0]["wf_sharpe"] if sorted_results else -999, + "detail": {"top3": top3_names}}) + + +# === FINAL RANKING === +print(f"\n{'='*80}") +print(f"{'RANK':<5} {'WF Sharpe':>10} {'OOS Sharpe':>10} {'OOS Mon%':>9} {'OOS DD%':>8} {'Trades':>7} Hypothesis") +print(f"{'='*80}") + +valid = [r for r in results if r.get("wf_sharpe") is not None and r["wf_sharpe"] > -50] +valid.sort(key=lambda x: x["wf_sharpe"], reverse=True) + +for i, r in enumerate(valid, 1): + d = r.get("detail", {}) + wf = r["wf_sharpe"] + oos_s = d.get("oos_sharpe", -999) + oos_m = d.get("oos_monthly", 0) or 0 + oos_d = (d.get("oos_dd", 0) or 0) * 100 + trades = d.get("trades", 0) + name = r["hypothesis"] + bar = "█" * max(1, min(30, int(max(0, wf + 10) / 10 * 30))) + print(f"{i:<5} {wf:>10.3f} {oos_s:>10.3f} {oos_m:>8.2f}% {oos_d:>7.1f}% {trades:>7} {name}") + +print(f"{'='*80}") +print(f"Total time: {(time.time()-t0_total)/60:.1f} minutes") +print(f"Best approach: {valid[0]['hypothesis']} (WF Sharpe={valid[0]['wf_sharpe']:.3f})" if valid else "No valid results") diff --git a/scripts/nexquant_infinite_search.py b/scripts/nexquant_infinite_search.py new file mode 100644 index 00000000..ab18b3a0 --- /dev/null +++ b/scripts/nexquant_infinite_search.py @@ -0,0 +1,243 @@ +#!/usr/bin/env python +""" +NexQuant Infinite Hypothesis Search — kombiniert und variiert Ansätze +bis ein positiver OOS Sharpe gefunden wird. +""" + +from __future__ import annotations + +import json, sys, time, random, itertools +from pathlib import Path + +import numpy as np +import pandas as pd + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) +from rdagent.components.backtesting.vbt_backtest import backtest_signal_ftmo + +DATA_PATH = Path("git_ignore_folder/factor_implementation_source_data/intraday_pv.h5") +FACTORS_DIR = Path("results/factors") +TXN_COST_BPS = 0.5 + + +def load_data(): + close = pd.read_hdf(DATA_PATH, key="data")["$close"] + if isinstance(close.index, pd.MultiIndex): + close = close.droplevel(-1) + close = close.sort_index().dropna().resample("1h").last().dropna() + + factors_meta = [] + for f in sorted(FACTORS_DIR.glob("*.json")): + try: + d = json.loads(f.read_text()) + except Exception: + continue + if d.get("status") != "success" or d.get("ic") is None: + continue + name = d.get("factor_name", f.stem) + safe = name.replace("/", "_")[:150] + if (FACTORS_DIR / "values" / f"{safe}.parquet").exists(): + factors_meta.append({"name": name, "ic": d["ic"]}) + + factors_meta.sort(key=lambda x: abs(x["ic"]), reverse=True) + top = factors_meta[:15] + factor_data = {} + for f in top: + safe = f["name"].replace("/", "_")[:150] + series = pd.read_parquet(FACTORS_DIR / "values" / f"{safe}.parquet").iloc[:, 0] + if isinstance(series.index, pd.MultiIndex): + series = series.droplevel(-1) + factor_data[f["name"]] = series.resample("1h").last() + + df = pd.DataFrame(factor_data) + common = close.index.intersection(df.dropna(how="all").index) + return close.loc[common], df.loc[common].ffill(), {f["name"]: f["ic"] for f in top} + + +close, factors_df, ics = load_data() +print(f"Data: {len(close):,} bars × {len(factors_df.columns)} factors\n") + +def backtest(signal) -> float: + if signal is None or len(signal) < 100: + return -999 + common = close.index.intersection(signal.dropna().index) + if len(common) < 100: + return -999 + r = backtest_signal_ftmo(close.loc[common], signal.reindex(common).fillna(0), + txn_cost_bps=TXN_COST_BPS, wf_rolling=False) + return r.get("oos_sharpe", -999) + + +def composite(factor_list=None, window=20): + cols = factor_list or list(factors_df.columns) + c = pd.Series(0.0, index=factors_df.index) + total = sum(abs(ics.get(col, 0)) for col in cols) + if total == 0: + return c + for col in cols: + ic_val = ics.get(col, 0) + if abs(ic_val) < 0.001: + continue + z = (factors_df[col] - factors_df[col].rolling(window).mean()) / (factors_df[col].rolling(window).std() + 1e-8) + c += (ic_val / total) * z + return c + + +def session_filter(sig): + hours = sig.index.hour + sig = sig.copy() + sig[(hours < 7) | (hours >= 17)] = 0 + return sig + + +def trend_filter(sig, sma_bars=200 * 1440 // 5): + sma = close.rolling(sma_bars).mean() + trend_up = close > sma + sig = sig.copy() + sig[(sig > 0) & ~trend_up] = 0 + sig[(sig < 0) & trend_up] = 0 + return sig + + +def vola_target(sig, vol_window=50): + vol = close.pct_change().rolling(vol_window).std() + vol_tgt = vol.median() + s = sig.astype(float) * vol_tgt / (vol + 1e-8) + return s.clip(-3, 3) + + +def anti_fade(sig, sigma=3.0): + ret = close.pct_change() + thresh = ret.std() * sigma + s = sig.copy() + s[ret > thresh] = -1 + s[ret < -thresh] = 1 + return s + + +def signal_decay(sig, half_life=60): + d = 0.5 ** (1 / half_life) + s = sig.astype(float).copy() + for i in range(1, len(s)): + if abs(s.iloc[i]) < 0.01: + s.iloc[i] = s.iloc[i - 1] * d + return s.clip(-1, 1) + + +def kalman_composite(comp, Q=0.001, R=0.1): + x, P = 0.0, 1.0 + filtered = [] + for v in comp.dropna().values: + P += Q; K = P / (P + R); x += K * (v - x); P *= (1 - K) + filtered.append(x) + return pd.Series(filtered, index=comp.dropna().index) + + +# PRIMITIVES — can be combined arbitrarily +PRIMITIVES = { + "session": session_filter, + "trend": trend_filter, + "vola_target": vola_target, + "anti_fade": anti_fade, + "decay": signal_decay, +} + +BASE_PARAMS = { + "entry": [0.1, 0.15, 0.2, 0.25, 0.3, 0.4, 0.5], + "window": [10, 20, 30, 50, 100], + "sigma": [2.0, 2.5, 3.0, 3.5], + "half_life": [30, 60, 120, 240], +} + +best_score = -999 +best_desc = "" +best_sig = None +tested = set() +round_num = 0 + + +def try_combo(factor_list, entry, window, primitives_used): + global best_score, best_desc, best_sig, tested, round_num + + key = f"{sorted(factor_list)}_{entry:.3f}_{window}_{sorted(primitives_used)}" + if key in tested: + return + tested.add(key) + + comp = composite(factor_list, window) + if comp is None or comp.dropna().empty: + return + sig = pd.Series(0, index=comp.index) + sig[comp > entry] = 1 + sig[comp < -entry] = -1 + + for p in primitives_used: + if p in PRIMITIVES: + sig = PRIMITIVES[p](sig.fillna(0)) + + sharpe = backtest(sig) + if sharpe > best_score: + best_score = sharpe + best_desc = f"entry={entry:.2f} window={window} factors={len(factor_list)} primitives={primitives_used}" + best_sig = sig + t = "✅" if sharpe > 0 else "📈" if sharpe > -1 else "➖" + print(f" {t} #{round_num}: Sharpe={sharpe:.4f} | {best_desc}") + + if sharpe > 0: + print(f"\n{'='*60}") + print(f" 🎯 POSITIVE SHARPE FOUND!") + print(f" Sharpe={sharpe:.4f}") + print(f" {best_desc}") + print(f"{'='*60}") + return True + return False + + +print("Starting infinite search — will run until positive OOS Sharpe found...\n") +all_factors = sorted(factors_df.columns, key=lambda c: -abs(ics.get(c, 0))) + +while True: + round_num += 1 + + # Pick random subset of top factors + n_factors = random.randint(2, min(10, len(all_factors))) + factor_subset = random.sample(all_factors[:12], n_factors) + + # Pick random parameters + entry = random.choice(BASE_PARAMS["entry"]) + window = random.choice(BASE_PARAMS["window"]) + + # Pick random combination of primitives (0-4) + n_prim = random.randint(0, 4) + prims = random.sample(list(PRIMITIVES.keys()), n_prim) if n_prim > 0 else [] + + found = try_combo(factor_subset, entry, window, prims) + if found: + break + + # Every 200 rounds, also try parameter sweeps around best + if round_num % 200 == 0: + print(f" ... {round_num} combinations tested, best={best_score:.4f}") + # Fine-tune around current best + for fine_entry in np.arange(max(0.05, entry - 0.15), entry + 0.16, 0.05): + for fine_window in [max(5, window - 15), window, min(200, window + 15)]: + if try_combo(factor_subset, fine_entry, fine_window, prims): + break + + # Every 500 rounds, try factor-specific combos (Kronos-only, momentum-only, etc.) + if round_num % 500 == 0: + kronos = [f for f in all_factors if "Kronos" in f] + mom = [f for f in all_factors if any(k in f.lower() for k in ["mom", "ret"])] + for subset in [kronos, mom, all_factors[:3], all_factors[:6]]: + if len(subset) >= 2: + for e in [0.1, 0.2, 0.3]: + for w in [20, 50]: + for prims in [[], ["session"], ["session", "decay"]]: + try_combo(subset, e, w, prims) + + if round_num % 1000 == 0: + print(f" [{round_num} tested] best={best_score:.4f} — still searching...") + +if best_score <= 0: + print(f"\nAfter {round_num} combinations, best is still negative ({best_score:.4f})") + print("The factors lack sufficient predictive power for positive returns.") diff --git a/scripts/nexquant_ml_pipeline.py b/scripts/nexquant_ml_pipeline.py index b481a63d..f2b1205e 100644 --- a/scripts/nexquant_ml_pipeline.py +++ b/scripts/nexquant_ml_pipeline.py @@ -1,7 +1,7 @@ #!/usr/bin/env python """ -NexQuant Optuna ML Strategy Pipeline — automatic hyperparameter tuning for daily strategies. -Target: 8%/month through ML-driven signal generation on daily OHLCV data. +NexQuant Enhanced ML Pipeline — factor-boosted, multi-horizon, Optuna-optimized. +Target: 8%/month through ensemble of factor + OHLCV features. """ from __future__ import annotations @@ -18,224 +18,212 @@ warnings.filterwarnings("ignore") sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) import optuna -from sklearn.ensemble import RandomForestClassifier +from sklearn.ensemble import RandomForestClassifier, GradientBoostingClassifier +from sklearn.linear_model import LogisticRegression +from sklearn.model_selection import TimeSeriesSplit from rdagent.components.backtesting.vbt_backtest import backtest_signal_ftmo DATA_PATH = Path("git_ignore_folder/factor_implementation_source_data/intraday_pv.h5") +FACTORS_DIR = Path("results/factors") TXN_COST_BPS = 2.14 -N_TRIALS = 50 +N_TRIALS = 75 -def load_data(): - c = pd.read_hdf(DATA_PATH, key="data")["$close"] - if isinstance(c.index, pd.MultiIndex): - c = c.droplevel(-1) - return c.sort_index().dropna().resample("1D").last().dropna() +def load_all(): + close = pd.read_hdf(DATA_PATH, key="data")["$close"] + if isinstance(close.index, pd.MultiIndex): + close = close.droplevel(-1) + daily = close.sort_index().dropna().resample("1D").last().dropna() + + # Load top factors + factors = [] + for f in sorted(FACTORS_DIR.glob("*.json")): + try: d = json.loads(f.read_text()) + except: continue + if d.get("status") != "success" or d.get("ic") is None: continue + ic = d["ic"] + if abs(ic) < 0.02: continue + name = d.get("factor_name", f.stem) + safe = name.replace("/", "_")[:150] + pf = FACTORS_DIR / "values" / f"{safe}.parquet" + if pf.exists(): + factors.append((abs(ic), name)) + + factors.sort(reverse=True) + top = factors[:100] + + # Load factor values + fdata = {} + for _, name in top: + safe = name.replace("/", "_")[:150] + s = pd.read_parquet(FACTORS_DIR / "values" / f"{safe}.parquet").iloc[:, 0] + if isinstance(s.index, pd.MultiIndex): + s = s.droplevel(-1) + fdata[name] = s.resample("1D").last() + + df = pd.DataFrame(fdata) + common = daily.index.intersection(df.dropna(how="all").index) + return daily.loc[common], df.loc[common].ffill() -def make_features(c: pd.Series) -> pd.DataFrame: +def add_ohlcv_features(c: pd.Series) -> pd.DataFrame: + """Lightweight OHLCV features to complement factors.""" df = pd.DataFrame(index=c.index) - # Returns - for n in [1, 3, 5, 10, 20]: - df[f"r{n}"] = c.pct_change(n) - df[f"lr{n}"] = np.log(c / c.shift(n)) - # MAs - for n in [5, 10, 20, 50, 100, 200]: - df[f"sma{n}"] = c.rolling(n).mean() / c - 1 - df[f"ema{n}"] = c.ewm(span=n).mean() / c - 1 - # Crossovers - df["sma5_20"] = c.rolling(5).mean() / c.rolling(20).mean() - 1 + for n in [1, 5, 10, 20]: + df[f"ret_{n}"] = c.pct_change(n) + for n in [10, 20, 50, 100]: + df[f"sma_{n}"] = c.rolling(n).mean() / c - 1 df["sma10_50"] = c.rolling(10).mean() / c.rolling(50).mean() - 1 df["sma20_100"] = c.rolling(20).mean() / c.rolling(100).mean() - 1 - df["ema12_26"] = c.ewm(span=12).mean() / c.ewm(span=26).mean() - 1 - # Volatility - for n in [5, 10, 20, 50]: - df[f"vol{n}"] = c.pct_change().rolling(n).std() - df["vol_ratio"] = df["vol10"] / (df["vol20"] + 1e-8) - # RSI - for p in [7, 14, 21]: - d = c.diff(); g = d.clip(lower=0); l = -d.clip(upper=0) - df[f"rsi{p}"] = 100 - (100 / (1 + g.rolling(p).mean() / (l.rolling(p).mean() + 1e-8))) - # MACD - e12 = c.ewm(span=12).mean(); e26 = c.ewm(span=26).mean() - macd = e12 - e26; sig = macd.ewm(span=9).mean() - df["macd"] = macd / c; df["macd_hist"] = (macd - sig) / c - # Bollinger - for p, k in [(20, 2)]: - ma = c.rolling(p).mean(); sd = c.rolling(p).std() - df[f"bb"] = (c - ma) / (k * sd + 1e-8) - df[f"bbw"] = (2 * k * sd) / (ma + 1e-8) - # ATR - tr = pd.concat([c.diff().abs(), (c-c.shift(1)).abs(), (c.shift(1)-c).abs()], axis=1).max(axis=1) - df["atr14"] = tr.rolling(14).mean() / c - # ADX - a14 = tr.rolling(14).mean() - pdi = 100 * (c.diff().clip(lower=0).ewm(span=14).mean() / (a14 + 1e-8)) - mdi = 100 * (-c.diff().clip(upper=0).ewm(span=14).mean() / (a14 + 1e-8)) - df["adx"] = (100 * abs(pdi - mdi) / (pdi + mdi + 1e-8)).ewm(span=14).mean() - # Donchian - for n in [20, 50]: - hi = c.rolling(n).max(); lo = c.rolling(n).min() - df[f"don{n}"] = (c - lo) / (hi - lo + 1e-8) - # Calendar - df["dow"] = c.index.dayofweek - df["dom"] = c.index.day - df["mon"] = c.index.month - df["mon_sin"] = np.sin(2 * np.pi * c.index.month / 12) - df["mon_cos"] = np.cos(2 * np.pi * c.index.month / 12) - # Distance from extremes - for n in [10, 50]: - df[f"dhi{n}"] = (c.rolling(n).max() - c) / (c + 1e-8) - df[f"dlo{n}"] = (c - c.rolling(n).min()) / (c + 1e-8) - # Momentum divergence - df["md5_20"] = c.pct_change(5) - c.pct_change(20) - df["md10_50"] = c.pct_change(10) - c.pct_change(50) - # Price acceleration - df["acc5"] = c.pct_change(5).diff(5) + for n in [5, 20]: + df[f"vol_{n}"] = c.pct_change().rolling(n).std() + d = c.diff(); g = d.clip(lower=0); l = -d.clip(upper=0) + df["rsi14"] = 100 - (100 / (1 + g.rolling(14).mean() / (l.rolling(14).mean() + 1e-8))) + df["adx14"] = (100 * abs(c.diff().clip(lower=0).ewm(14).mean() - (-c.diff().clip(upper=0)).ewm(14).mean()) / ( + c.diff().abs().rolling(14).mean() + 1e-8)).ewm(14).mean() return df -def make_target(c: pd.Series, horizon: int = 20) -> np.ndarray: - fwd = c.shift(-horizon); ret = (fwd / c - 1).fillna(0) - # Only strong moves - t = ret.std() * 0.5 +def make_target(c: pd.Series, horizon: int = 5) -> np.ndarray: + fwd = c.shift(-horizon) + ret = (fwd / c - 1).fillna(0) + t = ret.std() * 0.3 # Tighter threshold for more signals y = np.zeros(len(c)) - y[ret > t] = 1; y[ret < -t] = -1 + y[ret > t] = 1 + y[ret < -t] = -1 return y -def wf_backtest_metric(c: pd.Series, y_pred: np.ndarray, start_idx: int) -> dict: - """Backtest a model's predictions from start_idx onward.""" - test_c = c.iloc[start_idx:] - test_y = y_pred[start_idx:] - test_y = test_y[:len(test_c)] - sig = pd.Series(test_y, index=test_c.index[:len(test_y)]) - r = backtest_signal_ftmo(test_c.iloc[:len(test_y)], sig.astype(float), txn_cost_bps=TXN_COST_BPS) - return { - "oos_sharpe": r.get("oos_sharpe", -999) or -999, - "oos_monthly": r.get("oos_monthly_return_pct", 0) or 0, - "oos_dd": r.get("oos_max_drawdown", 0) or 0, - "oos_trades": r.get("oos_n_trades", 0), - "is_sharpe": r.get("is_sharpe", -999), - } - - -def objective(trial, X, y, c, split_idx): - """Optuna objective: maximize OOS Sharpe on validation fold.""" - n_est = trial.suggest_int("n_estimators", 100, 500, step=50) - max_d = trial.suggest_int("max_depth", 3, 20) - min_split = trial.suggest_int("min_samples_split", 2, 20) - min_leaf = trial.suggest_int("min_samples_leaf", 1, 10) - max_feat = trial.suggest_float("max_features", 0.3, 1.0) - - X_train, y_train = X[:split_idx], y[:split_idx] - - model = RandomForestClassifier( - n_estimators=n_est, max_depth=max_d, - min_samples_split=min_split, min_samples_leaf=min_leaf, - max_features=max_feat, random_state=42, n_jobs=-1, - ) - model.fit(X_train, y_train) - y_pred = model.predict(X) - - metrics = wf_backtest_metric(c, y_pred, split_idx) - return metrics["oos_sharpe"] +def backtest_metric(c, y_pred, split_idx): + test_c = c.iloc[split_idx:] + sig = pd.Series(y_pred[split_idx:len(test_c)+split_idx], index=test_c.index[:len(y_pred)-split_idx]) + r = backtest_signal_ftmo(test_c.iloc[:len(sig)], sig.astype(float), txn_cost_bps=TXN_COST_BPS) + return r.get("oos_sharpe", -999) or -999 def main(): print(f"\n{'='*65}") - print(" NexQuant Optuna ML Pipeline — Daily Strategies") - print(f" Target: 8%/month | Trials: {N_TRIALS} | Cost: {TXN_COST_BPS} bps") + print(" NexQuant Factor-Boosted ML Pipeline") + print(f" Target: 8%/month | Trials: {N_TRIALS}/horizon") print(f"{'='*65}") - c = load_data() - X_df = make_features(c).dropna() + c, factor_df = load_all() + ohlcv_df = add_ohlcv_features(c) + X_df = pd.concat([factor_df, ohlcv_df], axis=1).dropna() common = c.index.intersection(X_df.index) c = c.loc[common]; X_df = X_df.loc[common] - print(f"Data: {len(c)} daily bars | Features: {len(X_df.columns)}\n") + print(f"Daily: {len(c):,} bars | Features: {len(X_df.columns)} ({len(factor_df.columns)} factors + {len(ohlcv_df.columns)} OHLCV)\n") - # Test multiple horizons all_results = [] for horizon in [5, 10, 20]: - print(f"{'─'*50}") - print(f" HORIZON: {horizon}d forward return") - print(f"{'─'*50}") - + print(f"─── HORIZON {horizon}d ───") y = make_target(c, horizon) mask = ~np.isnan(y) & ~np.isinf(np.abs(y)) X = X_df.loc[mask].values.astype(np.float32) y_vals = y[mask].astype(int) split_idx = int(len(X) * 0.75) - # Only train if we have enough OOS data if len(X) - split_idx < 20: - print(" Not enough OOS data, skipping\n") + print(" Skip — not enough OOS\n") continue - # Run Optuna - print(f" Training: {split_idx} bars | OOS: {len(X) - split_idx} bars") - study = optuna.create_study( - direction="maximize", - sampler=optuna.samplers.TPESampler(seed=42), - pruner=optuna.pruners.HyperbandPruner(), - ) + print(f" Train: {split_idx} OOS: {len(X)-split_idx}") - def obj(trial): - return objective(trial, X, y_vals, c, split_idx) + # Test multiple model types + for model_name, ModelClass, param_space in [ + ("RF", RandomForestClassifier, { + "n": ("suggest_int", 100, 500), "d": ("suggest_int", 3, 25), + "split": ("suggest_int", 2, 15), "leaf": ("suggest_int", 1, 10), + "feat": ("suggest_float", 0.3, 1.0), + }), + ("GBM", GradientBoostingClassifier, { + "n": ("suggest_int", 100, 500), "d": ("suggest_int", 2, 10), + "lr": ("suggest_float", 0.01, 0.3), "split": ("suggest_int", 2, 20), + "leaf": ("suggest_int", 1, 10), + }), + ]: + def obj(trial): + p = {} + if model_name == "RF": + p = { + "n_estimators": trial.suggest_int("n", *param_space["n"][1:]), + "max_depth": trial.suggest_int("d", *param_space["d"][1:]), + "min_samples_split": trial.suggest_int("split", *param_space["split"][1:]), + "min_samples_leaf": trial.suggest_int("leaf", *param_space["leaf"][1:]), + "max_features": trial.suggest_float("feat", *param_space["feat"][1:]), + "random_state": 42, "n_jobs": -1, + } + else: + p = { + "n_estimators": trial.suggest_int("n", *param_space["n"][1:]), + "max_depth": trial.suggest_int("d", *param_space["d"][1:]), + "learning_rate": trial.suggest_float("lr", *param_space["lr"][1:]), + "min_samples_split": trial.suggest_int("split", *param_space["split"][1:]), + "min_samples_leaf": trial.suggest_int("leaf", *param_space["leaf"][1:]), + "random_state": 42, + } + model = ModelClass(**p) + model.fit(X[:split_idx], y_vals[:split_idx]) + return backtest_metric(c, model.predict(X), split_idx) - study.optimize(obj, n_trials=N_TRIALS, show_progress_bar=False) + study = optuna.create_study(direction="maximize", sampler=optuna.samplers.TPESampler(seed=42)) + study.optimize(obj, n_trials=N_TRIALS, show_progress_bar=False) - best = study.best_params - best_val = study.best_value + best = study.best_params + best_val = study.best_value - # Train best model - best_model = RandomForestClassifier( - n_estimators=best["n_estimators"], max_depth=best["max_depth"], - min_samples_split=best["min_samples_split"], - min_samples_leaf=best["min_samples_leaf"], - max_features=best["max_features"], random_state=42, n_jobs=-1, - ) - best_model.fit(X[:split_idx], y_vals[:split_idx]) - y_pred = best_model.predict(X) + # Final model + if model_name == "RF": + model = RandomForestClassifier( + n_estimators=best.get("n",200), max_depth=best.get("d",10), + min_samples_split=best.get("split",2), min_samples_leaf=best.get("leaf",1), + max_features=best.get("feat",0.5), random_state=42, n_jobs=-1, + ) + else: + model = GradientBoostingClassifier( + n_estimators=best.get("n",200), max_depth=best.get("d",5), + learning_rate=best.get("lr",0.1), min_samples_split=best.get("split",2), + min_samples_leaf=best.get("leaf",1), random_state=42, + ) + model.fit(X[:split_idx], y_vals[:split_idx]) + y_pred = model.predict(X) + sig = pd.Series(y_pred[split_idx:len(c)-split_idx+split_idx], index=c.index[split_idx:split_idx+len(y_pred)-split_idx]) + r = backtest_signal_ftmo(c.iloc[split_idx:split_idx+len(sig)], sig.astype(float), txn_cost_bps=TXN_COST_BPS) - metrics = wf_backtest_metric(c, y_pred, split_idx) - print(f" Best params: n={best['n_estimators']} d={best['max_depth']}" - f" split={best['min_samples_split']} leaf={best['min_samples_leaf']}" - f" feat={best['max_features']:.2f}") - print(f" Best OOS Sharpe: {metrics['oos_sharpe']:+.2f}") - print(f" OOS Monthly: {metrics['oos_monthly']:+.3f}%") - print(f" OOS Max DD: {metrics['oos_dd']*100:+.1f}%") - print(f" OOS Trades: {metrics['oos_trades']}") - print() + oos_s = r.get("oos_sharpe", -999) + oos_m = (r.get("oos_monthly_return_pct", 0) or 0) + oos_dd = (r.get("oos_max_drawdown", 0) or 0) * 100 + trades = r.get("oos_n_trades", 0) + print(f" {model_name} h={horizon}d OOS={oos_s:+.1f} Mon={oos_m:+.3f}% DD={oos_dd:+.1f}% T={trades}") - all_results.append({ - "horizon": horizon, **best, **metrics, - "features": len(X_df.columns), - }) + all_results.append({ + "model": model_name, "horizon": horizon, + "oos_sharpe": oos_s, "monthly": oos_m, "dd": oos_dd, "trades": trades, + }) # Summary - print(f"{'='*65}") - print(f" RESULTS") - print(f"{'='*65}") - print(f" {'Horiz':<7} {'OOS S':>8} {'Mon%':>8} {'DD%':>7} {'Trades':>7} {'n_est':>6} {'depth':>6} {'split':>6}") - print(f" {'─'*57}") - for r in sorted(all_results, key=lambda x: x["oos_monthly"], reverse=True): - print(f" {r['horizon']:>4}d {r['oos_sharpe']:>+8.2f} {r['oos_monthly']:>+7.3f}% {r['oos_dd']*100:>+6.1f}% {r['oos_trades']:>7} {r['n_estimators']:>6} {r['max_depth']:>6} {r['min_samples_split']:>6}") + print(f"\n{'='*65}") + print(f" {'Model':<6} {'Horiz':<6} {'OOS S':>8} {'Mon%':>9} {'DD%':>7} {'Trades':>7}") + print(f" {'─'*46}") + for r in sorted(all_results, key=lambda x: x["monthly"], reverse=True): + print(f" {r['model']:<6} {r['horizon']:>3}d {r['oos_sharpe']:>+8.1f} {r['monthly']:>+8.3f}% {r['dd']:>+6.1f}% {r['trades']:>7}") - best_r = max(all_results, key=lambda x: x["oos_monthly"]) - print(f"\n Best: {best_r['horizon']}d horizon → {best_r['oos_monthly']:+.3f}%/month") - print(f" Target 8%/month: {'✅ ACHIEVED!' if best_r['oos_monthly'] >= 8 else 'Still working...'}") + best = max(all_results, key=lambda x: x["monthly"]) + print(f"\n Best: {best['model']} {best['horizon']}d → {best['monthly']:+.3f}%/month") + gap = 8.0 - best['monthly'] + print(f" Gap to 8%: {gap:+.3f}% {'✅' if gap <= 0 else '— needs improvement'}") - # Feature importance - print(f"\n Top 10 Features:") - imps = best_model.feature_importances_ - cols = X_df.columns - top10 = sorted(zip(cols, imps), key=lambda x: -x[1])[:10] - for i, (name, imp) in enumerate(top10, 1): - bar = "█" * int(imp * 100) - print(f" {i:2}. {name:<20s} {imp:.3f} {bar}") + # Feature importance from best model + if hasattr(model, 'feature_importances_'): + imps = model.feature_importances_ + cols = X_df.columns + top = sorted(zip(cols, imps), key=lambda x: -x[1])[:15] + print(f"\n Top Features ({len(X_df.columns)} total):") + for i, (name, imp) in enumerate(top, 1): + src = "F" if name in factor_df.columns else "O" + print(f" {i:2}. [{src}] {name:<45s} {imp:.4f}") if __name__ == "__main__": diff --git a/scripts/nexquant_multi_asset.py b/scripts/nexquant_multi_asset.py new file mode 100644 index 00000000..857dfb40 --- /dev/null +++ b/scripts/nexquant_multi_asset.py @@ -0,0 +1,115 @@ +#!/usr/bin/env python +""" +NexQuant Multi-Asset Data Pipeline — Download + Test on expanded universe. +Downloads DXY, Gold, S&P 500, Bund, EUR/USD extended history via yfinance. +""" + +from __future__ import annotations + +import json, sys, time +from pathlib import Path + +import numpy as np +import pandas as pd +import yfinance as yf + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) + +DATA_DIR = Path("git_ignore_folder/factor_implementation_source_data") +DATA_DIR.mkdir(parents=True, exist_ok=True) + +# Multi-asset tickers (free via Yahoo Finance) +ASSETS = { + "EURUSD": "EURUSD=X", + "DXY": "DX-Y.NYB", # US Dollar Index + "GOLD": "GC=F", # Gold Futures + "SPX": "^GSPC", # S&P 500 + "BUND": "BUN24-EUX", # German Bund (approximate) + "GBPUSD": "GBPUSD=X", + "USDJPY": "USDJPY=X", + "OIL": "CL=F", # Crude Oil +} + +def download_asset(name: str, ticker: str, period: str = "max") -> pd.DataFrame: + print(f" Downloading {name} ({ticker})...") + try: + data = yf.download(ticker, period=period, progress=False, auto_adjust=True) + if data.empty: + print(f" Empty — skipping") + return None + close = data["Close"] + if isinstance(close, pd.DataFrame): + close = close.iloc[:, 0] + close.name = name + print(f" {len(close):,} bars ({close.index[0].date()} - {close.index[-1].date()})") + return close + except Exception as e: + print(f" Failed: {e}") + return None + +def main(): + print(f"\n{'='*60}") + print(" NexQuant Multi-Asset Data Download") + print(f"{'='*60}\n") + + all_data = {} + for name, ticker in ASSETS.items(): + series = download_asset(name, ticker) + if series is not None and len(series) > 100: + all_data[name] = series + + if not all_data: + print("No data downloaded!") + return + + # Build combined DataFrame + df = pd.DataFrame(all_data).dropna(how="all") + print(f"\nCombined data: {len(df):,} daily bars, {len(df.columns)} assets") + print(f"Date range: {df.index[0].date()} - {df.index[-1].date()}") + + # Save to HDF5 + h5_path = DATA_DIR / "multi_asset_daily.h5" + df.to_hdf(h5_path, key="data", mode="w") + print(f"Saved to {h5_path}") + + # Quick strategy test + print(f"\n{'='*60}") + print(" Quick Daily Strategy Test on Multi-Asset") + print(f"{'='*60}") + + from rdagent.components.backtesting.vbt_backtest import backtest_signal_ftmo + + for asset in df.columns: + c = df[asset].dropna() + if len(c) < 500: + continue + + # SMA 10/30 + f = c.rolling(10).mean() + s = c.rolling(30).mean() + sig = pd.Series(0.0, index=c.index) + sig[f > s] = 1 + sig[f < s] = -1 + + r = backtest_signal_ftmo(c, sig.fillna(0), txn_cost_bps=2.14, wf_rolling=True) + oos = r.get("wf_oos_sharpe_mean") or r.get("oos_sharpe", -999) + oos_m = r.get("oos_monthly_return_pct", 0) or 0 + status = "✅" if oos > 0 else " " + print(f" {asset:<10} SMA10/30: OOS={oos:+8.2f} Mon={oos_m:+6.2f}% {status}") + + # Also test extended EUR/USD + eurusd = df["EURUSD"].dropna() + print(f"\n Extended EUR/USD: {len(eurusd):,} bars") + c = eurusd + f = c.rolling(10).mean() + s = c.rolling(30).mean() + sig = pd.Series(0.0, index=c.index) + sig[f > s] = 1 + sig[f < s] = -1 + r = backtest_signal_ftmo(c, sig.fillna(0), txn_cost_bps=2.14, wf_rolling=True) + oos = r.get("wf_oos_sharpe_mean") or r.get("oos_sharpe", -999) + print(f" SMA10/30 extended: OOS={oos:+8.2f} Mon={r.get('oos_monthly_return_pct',0):+.2f}%") + + +if __name__ == "__main__": + main()