mirror of
https://github.com/lucasmos/GoldRegime_X.git
synced 2026-11-02 06:26:15 +00:00
feat: add strategy backtesting module for live model evaluation
- Introduced `strategy_backtest.py` as a single source of truth for scoring and backtesting. - Implemented functions to load live model bundles, build feature frames, and execute backtests. - Ensured consistency between optimized models and their validation/trading counterparts. - Added functionality for generating trading signals and retrieving the latest trading decision.
This commit is contained in:
Luke Mosoti
@@ -1,40 +0,0 @@
|
||||
{
|
||||
"tf": "H1",
|
||||
"n_states": 3,
|
||||
"taxonomy": "TREND/MEAN_REVERSION/VOLATILITY_SHOCK",
|
||||
"states": {
|
||||
"raw_state_0": {
|
||||
"label": "TREND",
|
||||
"raw_id": 0,
|
||||
"canonical_id": 0,
|
||||
"persistence": 0.962502,
|
||||
"entropy_bits": 0.250578,
|
||||
"vol_mean_scaled": -0.630554,
|
||||
"return_mean_scaled": -0.044516,
|
||||
"return_dispersion": 0.519793,
|
||||
"tradeable": true
|
||||
},
|
||||
"raw_state_1": {
|
||||
"label": "VOLATILITY_SHOCK",
|
||||
"raw_id": 1,
|
||||
"canonical_id": 2,
|
||||
"persistence": 0.95024,
|
||||
"entropy_bits": 0.302959,
|
||||
"vol_mean_scaled": 1.534357,
|
||||
"return_mean_scaled": -0.026177,
|
||||
"return_dispersion": 1.84906,
|
||||
"tradeable": true
|
||||
},
|
||||
"raw_state_2": {
|
||||
"label": "VOLATILITY_SHOCK",
|
||||
"raw_id": 2,
|
||||
"canonical_id": 2,
|
||||
"persistence": 0.938565,
|
||||
"entropy_bits": 0.386847,
|
||||
"vol_mean_scaled": 0.032875,
|
||||
"return_mean_scaled": 0.065863,
|
||||
"return_dispersion": 0.852745,
|
||||
"tradeable": true
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,40 +0,0 @@
|
||||
{
|
||||
"tf": "M15",
|
||||
"n_states": 3,
|
||||
"taxonomy": "TREND/MEAN_REVERSION/VOLATILITY_SHOCK",
|
||||
"states": {
|
||||
"raw_state_0": {
|
||||
"label": "TREND",
|
||||
"raw_id": 0,
|
||||
"canonical_id": 0,
|
||||
"persistence": 0.980975,
|
||||
"entropy_bits": 0.14093,
|
||||
"vol_mean_scaled": -0.573979,
|
||||
"return_mean_scaled": 0.002087,
|
||||
"return_dispersion": 0.39388,
|
||||
"tradeable": true
|
||||
},
|
||||
"raw_state_1": {
|
||||
"label": "VOLATILITY_SHOCK",
|
||||
"raw_id": 1,
|
||||
"canonical_id": 2,
|
||||
"persistence": 0.968925,
|
||||
"entropy_bits": 0.228487,
|
||||
"vol_mean_scaled": 0.025243,
|
||||
"return_mean_scaled": 0.037496,
|
||||
"return_dispersion": 0.770531,
|
||||
"tradeable": true
|
||||
},
|
||||
"raw_state_2": {
|
||||
"label": "VOLATILITY_SHOCK",
|
||||
"raw_id": 2,
|
||||
"canonical_id": 2,
|
||||
"persistence": 0.971048,
|
||||
"entropy_bits": 0.190025,
|
||||
"vol_mean_scaled": 1.499342,
|
||||
"return_mean_scaled": -0.100202,
|
||||
"return_dispersion": 2.076334,
|
||||
"tradeable": true
|
||||
}
|
||||
}
|
||||
}
|
||||
Binary file not shown.
@@ -1,5 +1,5 @@
|
||||
timeframe,scenario,xgb_threshold,score,win_rate,profit_factor,max_drawdown_pct,n_trades,ending_balance_cents,net_return_pct,net_profit_cents,profit_per_trade_cents,profit_per_trade_usd,relative_score_drop_is_to_oos
|
||||
M15,IS,0.15,53.61044281966572,0.9833333333333333,66.69065253287326,1.055540927138539,900,58587.15669185458,3805.8104461236417,57087.15669185458,63.43017410206065,0.6343017410206064,0.585221952810662
|
||||
M15,OOS,0.15,22.23643478111139,0.9655172413793104,92.46043244582772,1.2008414494742097,232,51525.01707423991,3335.0011382826615,50025.01707423991,215.6250735958617,2.1562507359586167,0.585221952810662
|
||||
M5,IS,0.15,106.65137860849234,0.9924630690382876,160.2712737353011,0.9895161452638874,3317,90617.94448091932,5941.196298727954,89117.94448091932,26.867031800096267,0.2686703180009627,0.5165154461611555
|
||||
M5,OOS,0.15,51.564294202308076,0.9955035971223022,482.8599031807081,0.3376801244677803,1112,102784.05925053441,6752.270616702292,101284.05925053441,91.08278709580432,0.9108278709580432,0.5165154461611555
|
||||
M5,IS,0.15,106.7391942701783,0.9924721469436917,160.57624353987356,0.9895161448766849,3321,90788.58593731541,5952.572395821025,89288.58593731541,26.88605418166679,0.2688605418166679,0.5150850730014184
|
||||
M5,OOS,0.15,51.759428596895845,0.9955197132616488,484.6923420371007,0.3376801244677803,1116,103169.22689880466,6777.948459920306,101669.22689880466,91.10145779462783,0.9110145779462784,0.5150850730014184
|
||||
|
||||
|
File diff suppressed because one or more lines are too long
Binary file not shown.
@@ -1,3 +1,3 @@
|
||||
timeframe,months,start,end,bars,trades,win_rate_%,profit_factor,net_profit_$,expectancy_$,return_%,max_dd_raw_%,max_dd_seasoned_%,max_dd_$,sharpe,sortino,calmar,recovery_factor,mar,final_balance_$,final_equity_$
|
||||
M15,6,2026-01-30,2026-07-30,11726,106,97.17,119.093,407.42,3.8436,2716.07,2.83,0.45,2.45,3.21,0.0,29733.101,34.099,29733.101,422.41,422.41
|
||||
M5,6,2026-01-30,2026-07-30,35073,543,99.63,619.148,877.77,1.6165,5851.37,94.83,3.97,14.22,3.562,0.0,4018.406,1.037,4018.406,892.71,892.71
|
||||
M15,6,2026-02-02,2026-07-31,11793,106,97.17,119.093,407.42,3.8436,2716.07,2.83,0.45,2.45,5.213,0.0,32059.942,34.099,32059.942,422.41,422.41
|
||||
M5,6,2026-02-02,2026-07-31,35273,545,99.63,613.613,869.91,1.5962,5799.02,7.2,2.29,8.87,6.863,0.0,57027.09,13.661,57027.09,884.85,884.85
|
||||
|
||||
|
+545
-543
File diff suppressed because it is too large
Load Diff
@@ -1,7 +1,7 @@
|
||||
regime,timeframe,trades,net_profit_$,final_balance_$,final_equity_$,return_%,max_drawdown_%,profit_factor
|
||||
baseline 0.02 (fixed),M15,106,291.47,306.42,306.42,1942.83,1.98,127.726
|
||||
baseline 0.02 (fixed),M5,543,602.72,617.72,617.72,4018.13,94.83,635.442
|
||||
baseline 0.02 (fixed),M5,545,598.11,613.1,613.1,3987.36,7.2,630.589
|
||||
escalate ->0.03 @ $50 profit,M15,106,407.42,422.41,422.41,2716.07,2.83,119.093
|
||||
escalate ->0.03 @ $50 profit,M5,543,877.77,892.71,892.71,5851.37,94.83,619.148
|
||||
escalate ->0.03 @ $50 profit,M5,545,869.91,884.85,884.85,5799.02,7.2,613.613
|
||||
escalate ->0.04 @ $50 profit,M15,106,523.39,538.4,538.4,3489.3,3.6,115.028
|
||||
escalate ->0.04 @ $50 profit,M5,543,1152.65,1167.69,1167.69,7684.61,94.83,607.658
|
||||
escalate ->0.04 @ $50 profit,M5,545,1141.55,1156.6,1156.6,7610.68,7.2,601.816
|
||||
|
||||
|
@@ -0,0 +1,5 @@
|
||||
timeframe,method,n_trades,obs_return_%,median_return_%,return_p5_%,return_p95_%,median_dd_%,dd_p95_%,prob_profit_%,prob_ruin_%
|
||||
M15,reshuffle,106,2716.13,2716.13,2716.13,2716.13,1.21,5.99,100.0,0.0
|
||||
M15,bootstrap,106,2716.13,2711.77,2478.52,2969.21,1.1,6.05,100.0,0.0
|
||||
M5,reshuffle,545,5799.4,5799.4,5799.4,5799.4,0.26,1.82,100.0,0.0
|
||||
M5,bootstrap,545,5799.4,5794.53,5581.32,6018.69,0.23,1.83,100.0,0.0
|
||||
|
@@ -0,0 +1,32 @@
|
||||
{
|
||||
"timeframe": "M5",
|
||||
"regimes": {
|
||||
"TREND": {
|
||||
"bars": 591226,
|
||||
"percentage": 81.905,
|
||||
"avg_return": 2.662627305839949e-06,
|
||||
"volatility": 0.0003755672215492284,
|
||||
"avg_duration": 174.97070139094407,
|
||||
"transition_in": 3378,
|
||||
"transition_out": 3379
|
||||
},
|
||||
"MEAN_REVERSION": {
|
||||
"bars": 0,
|
||||
"percentage": 0.0,
|
||||
"avg_return": 0.0,
|
||||
"volatility": 0.0,
|
||||
"avg_duration": 0.0,
|
||||
"transition_in": 0,
|
||||
"transition_out": 0
|
||||
},
|
||||
"VOLATILITY_SHOCK": {
|
||||
"bars": 130619,
|
||||
"percentage": 18.095,
|
||||
"avg_return": -3.545537126976018e-06,
|
||||
"volatility": 0.0011826338307082084,
|
||||
"avg_duration": 38.656111275525305,
|
||||
"transition_in": 3379,
|
||||
"transition_out": 3378
|
||||
}
|
||||
}
|
||||
}
|
||||
+8
-1
@@ -1,8 +1,9 @@
|
||||
# GoldRegimeX2 dependencies
|
||||
# Optuna removed as part of the grid-search-plateau migration (was: optuna>=3.5).
|
||||
numpy>=1.24
|
||||
pandas>=2.0
|
||||
hmmlearn>=0.3
|
||||
xgboost>=2.0
|
||||
optuna>=3.5
|
||||
scikit-learn>=1.3
|
||||
joblib>=1.3
|
||||
onnx>=1.15
|
||||
@@ -13,3 +14,9 @@ matplotlib>=3.8
|
||||
onnxconverter-common>=1.14
|
||||
metatrader5
|
||||
tensorflow>=2.13
|
||||
|
||||
# Optional (not required): speeds up the Numba backtest engine in the
|
||||
# grid-search-plateau pipeline. The engine detects Numba at import time and
|
||||
# falls back to pure Python when it is missing, so this is safe to omit —
|
||||
# recommended if your Python version has no compatible numba wheel.
|
||||
# numba>=0.59
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
+260
-1073
File diff suppressed because it is too large
Load Diff
+136
-2547
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,187 @@
|
||||
"""src/strategy_backtest.py -- single source of truth for scoring + backtesting.
|
||||
|
||||
This module is the *only* bridge between the trained notebook engine
|
||||
(``src.grid_search_plateau``) and the rest of the application (train / validate
|
||||
/ live). It loads the exported live bundle produced by ``--mode optimize``
|
||||
(``pipeline_verification_bundle/models/goldregimex_live_model.pkl``) and runs
|
||||
inference + backtests through the SAME code path the notebook uses
|
||||
(``grid_search_plateau.evaluate_ml_model``), so:
|
||||
|
||||
the model that is optimized == the model that is validated
|
||||
== the model that is traded.
|
||||
|
||||
The legacy ML stack (processor / engine_hmm / engine_xgb / backtester /
|
||||
signal_engine) has been retired -- see CONSOLIDATION.md.
|
||||
|
||||
Canonical engine contract (from grid_search_plateau):
|
||||
feat = build_features(panel, tf) # adds regime_code, atr14, Close, ...
|
||||
probs = model.predict_proba_raw(feat) # [:,0]=P(down) [:,2]=P(up)
|
||||
trades, metrics = run_ml_filtered_backtest(tf, feat, probs, base_params, threshold)
|
||||
Signals are TREND-gated (regime_code == 1) and confidence-filtered on the
|
||||
grid-selected xgb_threshold.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import pickle
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
|
||||
from src.logger import setup_logger
|
||||
|
||||
logger = setup_logger(__name__)
|
||||
|
||||
# Repo-root anchored path to the exported live bundle.
|
||||
_REPO_ROOT = Path(__file__).resolve().parents[1]
|
||||
LIVE_MODEL_PKL = _REPO_ROOT / "pipeline_verification_bundle" / "models" / "goldregimex_live_model.pkl"
|
||||
|
||||
|
||||
def load_live_bundle(path: Path | str = LIVE_MODEL_PKL) -> dict:
|
||||
"""Load the exported live bundle (models + thresholds + base_params).
|
||||
|
||||
Raises FileNotFoundError with actionable guidance if optimize hasn't run.
|
||||
"""
|
||||
path = Path(path)
|
||||
if not path.exists():
|
||||
raise FileNotFoundError(
|
||||
f"Live model bundle not found at {path}. "
|
||||
"Run python main.py --mode optimize first."
|
||||
)
|
||||
with open(path, "rb") as fh:
|
||||
bundle = pickle.load(fh)
|
||||
return bundle
|
||||
|
||||
|
||||
def bundle_params_for(tf: str, bundle: Optional[dict] = None):
|
||||
"""Return (model, base_params, threshold) for a timeframe from the bundle.
|
||||
|
||||
base_params already carries ``exit_model`` (matching the notebook export).
|
||||
``threshold`` is the plateau-selected xgb_threshold for the timeframe.
|
||||
"""
|
||||
tf = str(tf).upper()
|
||||
if bundle is None:
|
||||
bundle = load_live_bundle()
|
||||
models = bundle.get("models", {}) or {}
|
||||
thresholds = bundle.get("thresholds", {}) or {}
|
||||
base = bundle.get("base_params", {}) or {}
|
||||
if tf not in models:
|
||||
raise KeyError(f"Timeframe {tf} not in live bundle models: {list(models)}")
|
||||
if tf not in thresholds:
|
||||
raise KeyError(f"Timeframe {tf} not in live bundle thresholds: {list(thresholds)}")
|
||||
model = models[tf]
|
||||
base_params = dict(base.get(tf) or {})
|
||||
threshold = float(thresholds[tf])
|
||||
return model, base_params, threshold
|
||||
|
||||
|
||||
def _build_feat(tf: str, df: Optional[pd.DataFrame] = None) -> pd.DataFrame:
|
||||
"""Load the exec panel (or use ``df``) and build the engine feature frame.
|
||||
|
||||
Features are always computed on the full supplied history so rolling
|
||||
windows (EMA200, ATR, triple-barrier, ...) are correct; callers may slice
|
||||
the RESULT afterwards.
|
||||
"""
|
||||
from src import grid_search_plateau as engine
|
||||
panel = df if df is not None else engine.load_panel(tf)
|
||||
return engine.build_features(panel, tf)
|
||||
|
||||
|
||||
def build_scored_frame(tf: str, df: Optional[pd.DataFrame] = None,
|
||||
bundle: Optional[dict] = None) -> pd.DataFrame:
|
||||
"""Return the feature frame with model probabilities + filtered signal.
|
||||
|
||||
Adds columns: prob_down, prob_flat, prob_up, raw_signal, signal.
|
||||
Signal semantics mirror ``run_ml_filtered_backtest`` exactly:
|
||||
* raw_signal from TrendPullbackStrategy, zeroed outside TREND (regime_code==1)
|
||||
* signal keeps a long only when prob_up >= threshold, a short only when
|
||||
prob_down >= threshold, else 0.
|
||||
"""
|
||||
from src import grid_search_plateau as engine
|
||||
model, base_params, threshold = bundle_params_for(tf, bundle)
|
||||
feat = _build_feat(tf, df)
|
||||
|
||||
probs = np.asarray(model.predict_proba_raw(feat))
|
||||
feat = feat.copy()
|
||||
feat["prob_down"] = probs[:, 0]
|
||||
feat["prob_flat"] = probs[:, 1] if probs.shape[1] > 2 else 0.0
|
||||
feat["prob_up"] = probs[:, 2] if probs.shape[1] > 2 else probs[:, -1]
|
||||
|
||||
raw = engine.TrendPullbackStrategy().generate_signals(feat, base_params)
|
||||
trend_mask = feat["regime_code"].to_numpy() == 1
|
||||
raw = raw.where(pd.Series(trend_mask, index=feat.index), 0)
|
||||
raw_arr = raw.to_numpy(dtype=np.int8)
|
||||
|
||||
filtered = raw_arr.copy()
|
||||
filtered[(raw_arr == 1) & (feat["prob_up"].to_numpy() < threshold)] = 0
|
||||
filtered[(raw_arr == -1) & (feat["prob_down"].to_numpy() < threshold)] = 0
|
||||
|
||||
feat["raw_signal"] = raw_arr
|
||||
feat["signal"] = filtered
|
||||
return feat
|
||||
|
||||
|
||||
def backtest_tf(tf: str, df: Optional[pd.DataFrame] = None,
|
||||
bundle: Optional[dict] = None, tail: Optional[int] = None) -> dict:
|
||||
"""Backtest a timeframe through the notebook engine and return metrics.
|
||||
|
||||
Mirrors ``grid_search_plateau.evaluate_ml_model``. ``df`` may supply an
|
||||
alternative panel (e.g. freshly synced data); otherwise ``load_panel`` is
|
||||
used. ``tail`` slices the FEATURE frame to the most recent N rows AFTER
|
||||
feature construction (used by the validator for a recent-window Sharpe).
|
||||
|
||||
Returned dict keys come straight from ``compute_metrics``:
|
||||
profit_factor, sharpe, sortino, calmar, max_drawdown (PERCENT),
|
||||
expectancy, win_rate (0-1), trade_count, net_profit, profit_per_trade,
|
||||
net_return_pct.
|
||||
"""
|
||||
from src import grid_search_plateau as engine
|
||||
model, base_params, threshold = bundle_params_for(tf, bundle)
|
||||
feat = _build_feat(tf, df)
|
||||
if tail and len(feat) > int(tail):
|
||||
feat = feat.iloc[-int(tail):]
|
||||
|
||||
probs = np.asarray(model.predict_proba_raw(feat))
|
||||
_trades, metrics = engine.run_ml_filtered_backtest(
|
||||
tf.upper(), feat, probs, base_params, float(threshold)
|
||||
)
|
||||
return dict(metrics)
|
||||
|
||||
|
||||
def latest_signal(tf: str, df: Optional[pd.DataFrame] = None,
|
||||
bundle: Optional[dict] = None) -> dict:
|
||||
"""Return the most recent bar's trading decision for live execution.
|
||||
|
||||
The returned dict is engine-consistent with the backtest and carries
|
||||
everything the live executor needs to size an order identically to how the
|
||||
backtest fills it:
|
||||
|
||||
signal -1 / 0 / +1 (SELL / no-trade / BUY)
|
||||
prob_up float P(up) for the last bar
|
||||
prob_down float P(down) for the last bar
|
||||
threshold float grid-selected xgb_threshold
|
||||
regime_code int HMM regime of the last bar (1 == TREND)
|
||||
atr float atr14 of the last bar (SL/TP sizing base)
|
||||
close float last close price
|
||||
base_params dict strategy params incl. atr_stop/atr_target/
|
||||
leg_a_atr_target/exit_model
|
||||
timestamp Timestamp index of the last bar
|
||||
|
||||
SL/TP for live must be sized as base_params['atr_stop'] * atr (stop) and
|
||||
base_params['atr_target'] * atr (target) so live == backtest.
|
||||
"""
|
||||
_model, base_params, threshold = bundle_params_for(tf, bundle)
|
||||
feat = build_scored_frame(tf, df=df, bundle=bundle)
|
||||
last = feat.iloc[-1]
|
||||
return {
|
||||
"signal": int(last["signal"]),
|
||||
"prob_up": float(last["prob_up"]),
|
||||
"prob_down": float(last["prob_down"]),
|
||||
"threshold": float(threshold),
|
||||
"regime_code": int(last["regime_code"]),
|
||||
"atr": float(last["atr14"]) if "atr14" in feat.columns else float("nan"),
|
||||
"close": float(last["Close"]) if "Close" in feat.columns else float("nan"),
|
||||
"base_params": dict(base_params),
|
||||
"timestamp": feat.index[-1],
|
||||
}
|
||||
+107
-328
@@ -1,320 +1,143 @@
|
||||
"""Model Validation Gatekeeper.
|
||||
"""Model Validation Gatekeeper (consolidated onto the notebook engine).
|
||||
|
||||
Loads recently synced MT5 data, applies the full feature pipeline, runs live
|
||||
inference through the saved HMM + XGBoost models, and computes an annualised
|
||||
Sharpe Ratio over the recent window. The result is classified as:
|
||||
Validation now runs the SAME grid-search-plateau engine used by --mode optimize
|
||||
and by live trading (via ``src.strategy_backtest``) over a recent window of the
|
||||
exec-timeframe panel, then classifies the result. Because the model that is
|
||||
validated is the exact live bundle that is traded, this Sharpe is directly
|
||||
comparable to live behaviour.
|
||||
|
||||
pass (Sharpe >= 0.8) — model is stable; proceed to live.
|
||||
warn (Sharpe 0.5–0.8) — borderline; proceed with extreme caution.
|
||||
fail (Sharpe < 0.5) — model drift detected; retune before going live.
|
||||
|
||||
NOTE: This validation Sharpe is computed on the most recent N months of live
|
||||
market data. It is NOT equivalent to the OOS Sharpe from training, which uses
|
||||
an 80/20 chronological split of the full 10-year dataset. Both metrics matter,
|
||||
but they answer different questions.
|
||||
The legacy processor -> HMM -> XGB -> vectorized_backtest path was retired in
|
||||
the consolidation (see CONSOLIDATION.md). ``obs_cov`` / ``trans_cov`` and the
|
||||
MR-leak / regime-coverage / fold-instability gates are retained only for API
|
||||
compatibility; with the notebook engine they are no-ops.
|
||||
"""
|
||||
|
||||
import time as _time
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
from pathlib import Path
|
||||
|
||||
from src.logger import setup_logger
|
||||
from src.processor import (
|
||||
TF_CONFIG,
|
||||
USDCHF_MASTER_PATH,
|
||||
_USDCHF_PATH_BY_TF,
|
||||
_EXTERNAL_ASSET_PATHS,
|
||||
load_usdchf_data,
|
||||
map_usdchf_to_bars,
|
||||
load_asset_data,
|
||||
map_asset_to_bars,
|
||||
compute_log_returns,
|
||||
kalman_smooth,
|
||||
compute_volatility,
|
||||
compute_rsi,
|
||||
compute_atr,
|
||||
compute_gmm_vol_cluster,
|
||||
compute_synth_vix,
|
||||
load_gmm_model,
|
||||
load_feature_scaler,
|
||||
)
|
||||
from src.engine_hmm import load_model as load_hmm, predict_states, get_model_path as hmm_model_path, MODEL_PATH as HMM_GENERIC_PATH
|
||||
from src.engine_xgb import (
|
||||
load_xgb_ensemble, prepare_features, get_predictions_ensemble, get_ensemble_path,
|
||||
ENSEMBLE_PKL_PATH as XGB_GENERIC_PATH,
|
||||
load_regime_models, get_regime_predictions,
|
||||
)
|
||||
from src.backtester import vectorized_backtest
|
||||
from src.risk_manager import BROKER_CONFIGS
|
||||
from src.strategy_backtest import backtest_tf, load_live_bundle, LIVE_MODEL_PKL
|
||||
|
||||
logger = setup_logger(__name__)
|
||||
|
||||
SYNC_DATA_PATH = Path("data/processed/mt5_sync_data.csv")
|
||||
SHARPE_PASS_THRESHOLD = 0.8
|
||||
SHARPE_WARN_THRESHOLD = 0.5
|
||||
# H1 fires ~10 trades/year: 45 trades gives ±0.29 CI → relax threshold to avoid false fails
|
||||
# H1 fires rarely; relax per-TF thresholds to avoid false fails on small samples.
|
||||
TF_SHARPE_PASS: dict = {"H1": 0.25, "M15": 0.50, "M5": 0.70}
|
||||
TF_SHARPE_WARN: dict = {"H1": 0.05, "M15": 0.20, "M5": 0.40}
|
||||
# TF-aware minimum: H1 naturally fires ~10 trades/quarter, M15 ~15, M5 ~30+
|
||||
MIN_TRADES_WARNING_BY_TF: dict = {"M5": 30, "M15": 15, "H1": 10}
|
||||
|
||||
# Recent-window sizes (in bars) per timeframe when ``period`` is not supplied.
|
||||
_DEFAULT_WINDOW_BARS = {"M5": 20000, "M15": 12000, "H1": 6000}
|
||||
# Approximate bars-per-calendar-day per timeframe for parsing ``period``.
|
||||
_BARS_PER_DAY = {"M5": 288, "M15": 96, "H1": 24}
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# Internal helpers
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
def _apply_features(
|
||||
df: pd.DataFrame,
|
||||
tf: str,
|
||||
obs_cov: float,
|
||||
trans_cov: float,
|
||||
broker: str = "headway_cent",
|
||||
) -> pd.DataFrame:
|
||||
"""Apply the same feature steps as process_pipeline() to an arbitrary frame.
|
||||
def _window_bars(tf: str, period) -> int:
|
||||
"""Resolve a recent-window length in bars from a ``period`` string.
|
||||
|
||||
This function mirrors processor.process_pipeline() exactly. If any
|
||||
standalone function in processor.py changes its behaviour, this must be
|
||||
updated to match — both must produce identical column values for the same
|
||||
input data.
|
||||
Accepts '3m', '6m', '1y', '90d', '4w'. Falls back to the per-TF default.
|
||||
"""
|
||||
cfg = TF_CONFIG[tf.upper()]
|
||||
obs_cov = obs_cov if obs_cov is not None else cfg["obs_cov_default"]
|
||||
trans_cov = trans_cov if trans_cov is not None else cfg["trans_cov_default"]
|
||||
tf = tf.upper()
|
||||
default = _DEFAULT_WINDOW_BARS.get(tf, 6000)
|
||||
if not period:
|
||||
return default
|
||||
try:
|
||||
unit = str(period)[-1].lower()
|
||||
qty = float(str(period)[:-1])
|
||||
per_day = _BARS_PER_DAY.get(tf, 24)
|
||||
if unit == "d":
|
||||
days = qty
|
||||
elif unit == "w":
|
||||
days = qty * 7
|
||||
elif unit == "m":
|
||||
days = qty * 30
|
||||
elif unit == "y":
|
||||
days = qty * 365
|
||||
else:
|
||||
return default
|
||||
return max(500, int(days * per_day))
|
||||
except Exception:
|
||||
return default
|
||||
|
||||
df = df.copy()
|
||||
df["log_return"] = compute_log_returns(df["Close"])
|
||||
df["kalman_return"] = kalman_smooth(df["log_return"].values, obs_cov, trans_cov)
|
||||
df["volatility"] = compute_volatility(df["log_return"])
|
||||
df["rsi"] = compute_rsi(df["Close"])
|
||||
df["rsi_slope"] = df["rsi"].diff()
|
||||
df["atr_normalized"] = compute_atr(df)
|
||||
|
||||
# Load the training GMM + scaler — strictly no re-fitting on live/validation data.
|
||||
_gmm, _scaler = load_gmm_model(tf, broker)
|
||||
df["gmm_vol_cluster"] = compute_gmm_vol_cluster(
|
||||
df["volatility"].values, fitted_gmm=_gmm, fitted_scaler=_scaler
|
||||
)
|
||||
|
||||
# All external cross-asset features — mirrors process_pipeline exactly.
|
||||
for asset_key, path_by_tf in _EXTERNAL_ASSET_PATHS.items():
|
||||
col_name = f"{asset_key}_log_return"
|
||||
path = path_by_tf.get(tf.upper(), path_by_tf.get("H1"))
|
||||
asset_df = load_asset_data(path, col_name)
|
||||
if asset_df is not None:
|
||||
df[col_name] = map_asset_to_bars(df.index, asset_df, col_name)
|
||||
|
||||
# Backfill so short-history assets (e.g. XTIUSD from Feb 2017) don't wipe rows.
|
||||
_ext_cols = [c for c in df.columns if c.endswith("_log_return") and c != "log_return"]
|
||||
if _ext_cols:
|
||||
df[_ext_cols] = df[_ext_cols].bfill()
|
||||
|
||||
df.dropna(inplace=True)
|
||||
|
||||
# Synthetic VIX (Williams VIX Fix z-score) — must come after dropna on core cols.
|
||||
df["synth_vix_zscore"] = compute_synth_vix(df)
|
||||
|
||||
return df
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# Primary entry point
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
def run_validation(
|
||||
sync_data_path: Path = SYNC_DATA_PATH,
|
||||
tf: str = "H1",
|
||||
broker: str = "headway_cent",
|
||||
account_size: float = 15.0,
|
||||
obs_cov: float = None,
|
||||
trans_cov: float = None,
|
||||
period: str | None = None, # TODO: filter validation window to last N months
|
||||
# (e.g. '3m', '6m'). Currently accepted for guardian
|
||||
# API compatibility; the full synced CSV is always used.
|
||||
obs_cov: float = None, # retired (Kalman); accepted for API compatibility
|
||||
trans_cov: float = None, # retired (Kalman); accepted for API compatibility
|
||||
period=None,
|
||||
) -> dict:
|
||||
"""Validate the saved models against recent synced MT5 data.
|
||||
"""Validate the live bundle against a recent window of the exec panel.
|
||||
|
||||
Returns a dict:
|
||||
sharpe (float) — annualised Sharpe over the validation window
|
||||
n_trades (int) — number of trades fired by the model
|
||||
win_rate (float) — fraction of winning trades
|
||||
max_dd (float) — maximum drawdown (0–1)
|
||||
status (str) — 'pass', 'warn', or 'fail'
|
||||
message (str) — human-readable gate decision
|
||||
Returns a dict compatible with the historical validator surface (sharpe,
|
||||
n_trades, win_rate, max_dd, status, message, ...).
|
||||
|
||||
Raises:
|
||||
FileNotFoundError if the sync CSV or model files are missing.
|
||||
Raises FileNotFoundError if the live bundle is missing.
|
||||
"""
|
||||
sync_data_path = Path(sync_data_path)
|
||||
if not sync_data_path.exists():
|
||||
raise FileNotFoundError(
|
||||
f"Sync data not found at {sync_data_path}. "
|
||||
"Run python main.py --mode sync_validate first."
|
||||
)
|
||||
|
||||
# Attempt to load Optuna best params; fall back to TF defaults
|
||||
tf = tf.upper()
|
||||
try:
|
||||
from src.optimizer import get_best_params
|
||||
params = get_best_params(balance=account_size, broker=broker, tf=tf)
|
||||
obs_cov = params.get("obs_cov") or obs_cov
|
||||
trans_cov = params.get("trans_cov") or trans_cov
|
||||
except Exception:
|
||||
params = {}
|
||||
|
||||
# Load and featurise data
|
||||
df = pd.read_csv(sync_data_path, index_col="Date", parse_dates=True)
|
||||
logger.info("Loaded %d bars from %s for validation", len(df), sync_data_path)
|
||||
df = _apply_features(df, tf, obs_cov, trans_cov, broker=broker)
|
||||
|
||||
# Load models — prefer broker+TF specific file; fall back to generic
|
||||
hmm_path = hmm_model_path(tf, broker)
|
||||
if not hmm_path.exists():
|
||||
hmm_path = HMM_GENERIC_PATH
|
||||
if hmm_path.exists():
|
||||
logger.warning(
|
||||
"No TF-specific HMM model found for %s; falling back to %s. "
|
||||
"Run --mode train --tf %s to create a dedicated model.", tf, hmm_path, tf
|
||||
)
|
||||
try:
|
||||
model_hmm = load_hmm(hmm_path)
|
||||
bundle = load_live_bundle()
|
||||
except FileNotFoundError:
|
||||
raise FileNotFoundError(
|
||||
f"HMM model not found at {hmm_path}. "
|
||||
"Run python main.py --mode train first."
|
||||
f"Live model bundle not found at {LIVE_MODEL_PKL}. "
|
||||
"Run python main.py --mode optimize first."
|
||||
)
|
||||
|
||||
# Inference — load the same feature scaler used at training time
|
||||
states = predict_states(model_hmm, df)
|
||||
try:
|
||||
_feat_scaler = load_feature_scaler(tf=tf, broker=broker)
|
||||
except FileNotFoundError:
|
||||
logger.warning(
|
||||
"Feature scaler not found for [%s/%s] — validating without scaling. "
|
||||
"Re-run --mode train to generate it.",
|
||||
tf, broker,
|
||||
)
|
||||
_feat_scaler = None
|
||||
X, _, df_aligned, _ = prepare_features(df, states, feature_scaler=_feat_scaler, tf=tf)
|
||||
states_aligned = states[df.index.isin(df_aligned.index)]
|
||||
tail = _window_bars(tf, period)
|
||||
logger.info("Validating [%s] on the most recent ~%d bars (broker=%s).", tf, tail, broker)
|
||||
|
||||
# Regime-first inference: use regime models when available; ensemble is fallback only
|
||||
trend_model, shock_model = load_regime_models(tf=tf, broker=broker)
|
||||
use_regime_models = (trend_model is not None) or (shock_model is not None)
|
||||
if use_regime_models:
|
||||
logger.info("Validation using regime models [%s/%s].", tf, broker)
|
||||
probabilities = get_regime_predictions(
|
||||
X, states_aligned, trend_model, shock_model, fallback_prob=0.50
|
||||
)
|
||||
xgb_meta: dict = {}
|
||||
else:
|
||||
logger.info("Regime models not found for [%s/%s]; falling back to ensemble.", tf, broker)
|
||||
xgb_path = get_ensemble_path(tf, broker)
|
||||
if not xgb_path.exists():
|
||||
xgb_path = XGB_GENERIC_PATH
|
||||
if xgb_path.exists():
|
||||
logger.warning(
|
||||
"No broker+TF XGB ensemble found for %s/%s; falling back to %s. "
|
||||
"Run --mode train --tf %s --broker %s to create a dedicated model.",
|
||||
tf, broker, xgb_path, tf, broker,
|
||||
)
|
||||
try:
|
||||
models_xgb, thresholds_xgb, xgb_meta = load_xgb_ensemble(xgb_path)
|
||||
except FileNotFoundError:
|
||||
raise FileNotFoundError(
|
||||
"XGB ensemble not found at models/xgb_ensemble.pkl. "
|
||||
"Run python main.py --mode train first."
|
||||
)
|
||||
_, probabilities = get_predictions_ensemble(models_xgb, thresholds_xgb, X)
|
||||
# backtest_tf builds features on the full engine panel, then slices the
|
||||
# feature frame to the recent window (so rolling features stay correct).
|
||||
metrics = backtest_tf(tf, bundle=bundle, tail=tail)
|
||||
|
||||
# Backtest the full synced window — no IS/OOS split
|
||||
_z = params.get("z_cutoff_bull")
|
||||
_eval_cfg = {"Z_CUTOFF_BULL": _z, "Z_CUTOFF_BEAR": -_z} if _z else None
|
||||
result = vectorized_backtest(
|
||||
df_aligned, probabilities, states_aligned,
|
||||
split_idx=None,
|
||||
account_size=account_size,
|
||||
broker=broker,
|
||||
tf=tf,
|
||||
regime_stats=xgb_meta.get("regime_stats"),
|
||||
evaluator_config=_eval_cfg,
|
||||
)
|
||||
sharpe = float(metrics.get("sharpe", 0.0))
|
||||
n_trades = int(metrics.get("trade_count", 0))
|
||||
win_rate = float(metrics.get("win_rate", 0.0))
|
||||
max_dd_pct = float(metrics.get("max_drawdown", 0.0)) # PERCENT from engine
|
||||
max_dd_frac = max_dd_pct / 100.0 # fraction for the DD gate
|
||||
pf = float(metrics.get("profit_factor", 1.0))
|
||||
total_return = float(metrics.get("net_return_pct", 0.0)) # PERCENT
|
||||
score = float(metrics.get("stability_adjusted_sharpe", sharpe))
|
||||
recovery = float(metrics.get("calmar", 0.0))
|
||||
|
||||
sharpe = result["sharpe_ratio"]
|
||||
n_trades = result["n_trades"]
|
||||
win_rate = result["win_rate"]
|
||||
floating_dd = float(result.get("floating_max_drawdown", result.get("max_drawdown", 0.0)))
|
||||
max_drawdown = float(result.get("max_drawdown", floating_dd))
|
||||
max_dd = floating_dd # backward-compat alias
|
||||
pf = result.get("profit_factor", 1.0)
|
||||
payoff = result.get("expected_payoff", 0.0)
|
||||
rf = result.get("recovery_factor", 0.0)
|
||||
avg_eff = result.get("avg_efficiency", 0.0)
|
||||
cost_eff = result.get("cost_efficiency", 0.0)
|
||||
total_return = result.get("total_return", 0.0)
|
||||
mr_trades = int(result.get("mr_trades", 0))
|
||||
mr_leak_count = int(result.get("mr_leak_count", mr_trades))
|
||||
from src.engine_hmm import CANONICAL_REGIME_ID, REGIME_MR
|
||||
_mr_id = CANONICAL_REGIME_ID[REGIME_MR]
|
||||
regime_coverage = float(np.mean(states_aligned != _mr_id)) if len(states_aligned) else 0.0
|
||||
|
||||
# Complex Criterion score — same formula as optimizer._score_result
|
||||
_fdd = floating_dd if floating_dd > 0 else 0.0
|
||||
_net = result.get("total_return", 0.0)
|
||||
_rf_capped = min(_net / _fdd, 50.0) if _fdd > 0 else (50.0 if _net > 0 else 0.0)
|
||||
score = float((_rf_capped * 0.4) + (pf * 0.3) + (sharpe * 0.3))
|
||||
|
||||
min_trades_warn = MIN_TRADES_WARNING_BY_TF.get(tf.upper(), 15)
|
||||
_period_suggestions = {"M5": "3m", "M15": "6m", "H1": "1y"}
|
||||
_period_hint = _period_suggestions.get(tf.upper(), "1y")
|
||||
min_trades_warn = MIN_TRADES_WARNING_BY_TF.get(tf, 15)
|
||||
if n_trades < min_trades_warn:
|
||||
logger.warning(
|
||||
"Only %d trades in the validation window — Sharpe estimate may be "
|
||||
"unreliable. Consider using a longer --period (e.g. '%s').",
|
||||
n_trades, _period_hint,
|
||||
"Only %d trades in the validation window; Sharpe estimate may be "
|
||||
"unreliable. Consider a longer --period.", n_trades,
|
||||
)
|
||||
|
||||
# Spread-Payoff Ratio check — warn if broker costs eat >50% of the model's edge
|
||||
_bcfg = BROKER_CONFIGS.get(broker, BROKER_CONFIGS["standard"])
|
||||
_spread_cost = _bcfg["spread_frac"] + _bcfg.get("commission_frac", 0.0)
|
||||
_payoff_usd = payoff * account_size # log-return units → approximate USD
|
||||
_spread_usd = _spread_cost * account_size # cost per round-trip
|
||||
if _payoff_usd > 0 and _spread_usd > 0.5 * _payoff_usd:
|
||||
logger.warning(
|
||||
"High Spread Erosion: spread=$%.4f vs payoff=$%.4f (%.0f%% consumed). "
|
||||
"Edge is too thin to survive broker costs.",
|
||||
_spread_usd, _payoff_usd, (_spread_usd / _payoff_usd) * 100,
|
||||
)
|
||||
print(" [!] WARNING: High Spread Erosion. Spread is consuming > 50% of your edge.")
|
||||
|
||||
# Gate decision — TF-specific thresholds: H1 fires rarely, needs lower bar
|
||||
_pass_thr = TF_SHARPE_PASS.get(tf.upper(), SHARPE_PASS_THRESHOLD)
|
||||
_warn_thr = TF_SHARPE_WARN.get(tf.upper(), SHARPE_WARN_THRESHOLD)
|
||||
|
||||
_pass_thr = TF_SHARPE_PASS.get(tf, SHARPE_PASS_THRESHOLD)
|
||||
_warn_thr = TF_SHARPE_WARN.get(tf, SHARPE_WARN_THRESHOLD)
|
||||
if sharpe >= _pass_thr:
|
||||
status = "pass"
|
||||
message = (
|
||||
f"Recent-Period Sharpe {sharpe:.3f} >= {_pass_thr} [{tf} threshold]. "
|
||||
"Model is stable — safe to go live."
|
||||
f"Recent-window Sharpe {sharpe:.3f} >= {_pass_thr} [{tf}]. "
|
||||
"Model is stable; safe to go live."
|
||||
)
|
||||
elif sharpe >= _warn_thr:
|
||||
status = "warn"
|
||||
message = (
|
||||
f"Recent-Period Sharpe {sharpe:.3f} is borderline "
|
||||
f"({_warn_thr}–{_pass_thr}, {tf} thresholds). "
|
||||
"Proceed with reduced position size or wait for a clearer regime."
|
||||
f"Recent-window Sharpe {sharpe:.3f} is borderline "
|
||||
f"({_warn_thr}-{_pass_thr}, {tf}). Proceed with reduced size or wait."
|
||||
)
|
||||
else:
|
||||
status = "fail"
|
||||
message = (
|
||||
f"Recent-Period Sharpe {sharpe:.3f} < {_warn_thr} [{tf} threshold]. "
|
||||
"Market drift detected — DO NOT go live. "
|
||||
"Run --mode optimize then --mode train to retune the model."
|
||||
f"Recent-window Sharpe {sharpe:.3f} < {_warn_thr} [{tf}]. "
|
||||
"Drift detected; DO NOT go live. Re-run --mode optimize."
|
||||
)
|
||||
|
||||
logger.info(
|
||||
"Validation [%s]: status=%s score=%.2f sharpe=%.3f pf=%.2f eff=%.2fx trades=%d wr=%.1f%% dd=%.1f%%",
|
||||
tf, status, score, sharpe, pf, avg_eff, n_trades, win_rate * 100, max_dd * 100,
|
||||
"Validation [%s]: status=%s score=%.2f sharpe=%.3f pf=%.2f trades=%d wr=%.1f%% dd=%.1f%%",
|
||||
tf, status, score, sharpe, pf, n_trades, win_rate * 100, max_dd_pct,
|
||||
)
|
||||
if status in ("warn", "fail"):
|
||||
logger.warning("VALIDATION %s: %s", status.upper(), message)
|
||||
@@ -323,17 +146,18 @@ def run_validation(
|
||||
"sharpe": sharpe,
|
||||
"n_trades": n_trades,
|
||||
"win_rate": win_rate,
|
||||
"floating_max_drawdown": floating_dd,
|
||||
"max_drawdown": max_drawdown,
|
||||
"max_dd": floating_dd,
|
||||
"mr_trades": mr_trades,
|
||||
"mr_leak_count": mr_leak_count,
|
||||
"regime_coverage": regime_coverage,
|
||||
"floating_max_drawdown": max_dd_frac,
|
||||
"max_drawdown": max_dd_frac,
|
||||
"max_dd": max_dd_frac,
|
||||
"mr_trades": 0,
|
||||
"mr_leak_count": 0,
|
||||
"regime_coverage": 1.0,
|
||||
"fold_sharpe_std": 0.0,
|
||||
"profit_factor": pf,
|
||||
"expected_payoff": payoff,
|
||||
"recovery_factor": rf,
|
||||
"avg_efficiency": avg_eff,
|
||||
"cost_efficiency": cost_eff,
|
||||
"expected_payoff": 0.0,
|
||||
"recovery_factor": recovery,
|
||||
"avg_efficiency": 0.0,
|
||||
"cost_efficiency": 0.0,
|
||||
"total_return": total_return,
|
||||
"score": score,
|
||||
"status": status,
|
||||
@@ -341,35 +165,16 @@ def run_validation(
|
||||
}
|
||||
|
||||
|
||||
# ── Deployment Gate ───────────────────────────────────────────────────────────
|
||||
# Minimum per-TF trade counts for a strategy to be deployable.
|
||||
_MIN_TRADES_PER_TF = {"H1": 30, "M15": 60, "M5": 120}
|
||||
|
||||
def validate_strategy(
|
||||
result: dict,
|
||||
tf: str,
|
||||
metrics: dict | None = None,
|
||||
) -> dict:
|
||||
|
||||
def validate_strategy(result: dict, tf: str, metrics: dict = None) -> dict:
|
||||
"""Deployment gate: hard-fail or warn based on backtest result quality.
|
||||
|
||||
Checks are applied in priority order:
|
||||
FAIL dd_cap_violated — floating DD > 20%
|
||||
FAIL mr_leak — any trade executed in MR regime
|
||||
FAIL min_trades — too few trades for statistical significance
|
||||
WARN fold_instability — high cross-fold Sharpe variance
|
||||
WARN regime_instability — MR coverage < 20% (degenerate HMM)
|
||||
|
||||
Args:
|
||||
result: Dict from ``vectorized_backtest`` (or CPCV aggregated dict).
|
||||
tf: Timeframe string ("H1", "M15", "M5") — used for min_trades.
|
||||
metrics: Optional supplemental metrics dict (e.g. CPCV fold stats).
|
||||
Merged on top of ``result`` for look-up purposes.
|
||||
|
||||
Returns:
|
||||
dict with keys:
|
||||
status — "pass" | "warn" | "fail"
|
||||
reason — canonical reason code or "" for pass
|
||||
details — dict of the specific check values
|
||||
Priority: FAIL dd_cap_violated (DD>20%) -> FAIL mr_leak (now always 0) ->
|
||||
FAIL min_trades -> WARN fold_instability (now 0) -> WARN regime_instability
|
||||
(now 1.0, never fires).
|
||||
"""
|
||||
_m = dict(result)
|
||||
if metrics:
|
||||
@@ -379,75 +184,49 @@ def validate_strategy(
|
||||
min_trades = _MIN_TRADES_PER_TF.get(tf_up, 30)
|
||||
|
||||
details: dict = {}
|
||||
warnings: list[str] = []
|
||||
warnings: list = []
|
||||
|
||||
# ── HARD FAIL CHECKS ─────────────────────────────────────────────────────
|
||||
floating_dd = float(
|
||||
_m.get("floating_max_drawdown", _m.get("max_drawdown", _m.get("max_dd", 0.0)))
|
||||
)
|
||||
details["floating_dd"] = floating_dd
|
||||
if floating_dd > 0.20:
|
||||
return {
|
||||
"status": "fail",
|
||||
"reason": "dd_cap_violated",
|
||||
"details": {**details, "threshold": 0.20},
|
||||
}
|
||||
return {"status": "fail", "reason": "dd_cap_violated",
|
||||
"details": {**details, "threshold": 0.20}}
|
||||
|
||||
mr_leak = int(_m.get("mr_leak_count", _m.get("mr_trades", 0)))
|
||||
details["mr_leak_count"] = mr_leak
|
||||
if mr_leak > 0:
|
||||
return {
|
||||
"status": "fail",
|
||||
"reason": "mr_leak",
|
||||
"details": {**details, "mr_trades_found": mr_leak},
|
||||
}
|
||||
return {"status": "fail", "reason": "mr_leak",
|
||||
"details": {**details, "mr_trades_found": mr_leak}}
|
||||
|
||||
n_trades = int(_m.get("n_trades", 0))
|
||||
details["n_trades"] = n_trades
|
||||
if n_trades < min_trades:
|
||||
return {
|
||||
"status": "fail",
|
||||
"reason": "min_trades",
|
||||
"details": {**details, "required": min_trades},
|
||||
}
|
||||
return {"status": "fail", "reason": "min_trades",
|
||||
"details": {**details, "required": min_trades}}
|
||||
|
||||
# ── WARN CHECKS ──────────────────────────────────────────────────────────
|
||||
fold_sharpe_std = float(_m.get("fold_sharpe_std", _m.get("std_sharpe", 0.0)))
|
||||
details["fold_sharpe_std"] = fold_sharpe_std
|
||||
if fold_sharpe_std > 1.5:
|
||||
warnings.append("fold_instability")
|
||||
|
||||
# Regime coverage: fraction of bars that are TREND or SHOCK
|
||||
# (less than 20% means the HMM collapsed to near-all-MR — degenerate)
|
||||
regime_coverage = float(_m.get("regime_coverage", 1.0))
|
||||
details["regime_coverage"] = regime_coverage
|
||||
if regime_coverage < 0.20:
|
||||
warnings.append("regime_instability")
|
||||
|
||||
if warnings:
|
||||
return {
|
||||
"status": "warn",
|
||||
"reason": warnings[0], # primary warning code
|
||||
"details": {**details, "all_warnings": warnings},
|
||||
}
|
||||
return {"status": "warn", "reason": warnings[0],
|
||||
"details": {**details, "all_warnings": warnings}}
|
||||
|
||||
return {"status": "pass", "reason": "", "details": details}
|
||||
|
||||
|
||||
def check_model_age(tf: str = "H1", broker: str = "headway_cent") -> float:
|
||||
"""Return the age of the saved ensemble model in days, based on file mtime.
|
||||
|
||||
Checks the broker+TF specific ensemble pkl first; falls back to the generic
|
||||
path. Returns ``float('inf')`` if no model file exists anywhere — the caller
|
||||
should treat this as an infinitely stale model.
|
||||
"""
|
||||
from src.engine_xgb import get_ensemble_path, ENSEMBLE_PKL_PATH as _GENERIC
|
||||
|
||||
path = get_ensemble_path(tf, broker)
|
||||
if not path.exists():
|
||||
path = _GENERIC
|
||||
"""Return the age of the live model bundle in days (float('inf') if absent)."""
|
||||
path = Path(LIVE_MODEL_PKL)
|
||||
if not path.exists():
|
||||
return float("inf")
|
||||
|
||||
age_sec = _time.time() - path.stat().st_mtime
|
||||
return age_sec / 86_400 # convert seconds → days
|
||||
return age_sec / 86_400
|
||||
|
||||
+1
-1
@@ -414,7 +414,7 @@ def plot_summary_dashboard(result, params, tf="H1", broker="headway_cent",
|
||||
y -= 0.01
|
||||
_is_cpcv = "cpcv_score" in result
|
||||
_oos_label = (
|
||||
f"CPCV OOS ({result.get('cpcv_n_valid_paths', '?')}/{6} valid paths)"
|
||||
f"CPCV OOS ({result.get('cpcv_n_valid_paths', '?')}/{result.get('cpcv_n_paths') or len(result.get('cpcv_path_scores') or []) or 6} valid paths)"
|
||||
if _is_cpcv else "Out-of-Sample"
|
||||
)
|
||||
ax.text(0.05, y, _oos_label, fontsize=12, fontweight="bold",
|
||||
|
||||
Reference in New Issue
Block a user