diff --git a/pipeline_verification_bundle/GoldRegimeX_Explorer.ipynb b/pipeline_verification_bundle/GoldRegimeX_Explorer.ipynb index 28d52bc..2c7ab54 100644 --- a/pipeline_verification_bundle/GoldRegimeX_Explorer.ipynb +++ b/pipeline_verification_bundle/GoldRegimeX_Explorer.ipynb @@ -34,21 +34,16 @@ "name": "stdout", "output_type": "stream", "text": [ - "[observability-v2] repo_root = C:\\GoldRegime_X\n", - "[observability-v2] run_uuid = 566d6406-f1b8-4334-b0c1-39da8b8a6f0b\n", - "[observability-v2] stages = ['GENERATED', 'FEATURE_ENGINEERING', 'SESSION', 'TBM', 'HMM', 'PROBABILITY', 'RISK', 'EXECUTED']\n", - "[observability-v2] session = {'H1': True, 'M15': False, 'M5': False}\n" + "[shared] repo_root = C:\\GoldRegime_X\n" ] } ], "source": [ - "# === Observability Reconstruction V2 -- bootstrap (tag: observability_v2) ===\n", - "# Single shared implementation for the profiler, session filter, feature\n", - "# engineering, prediction contract and certification. This REPLACES the old\n", - "# reconstructed/aliased/hydrated observability layer. Do NOT reintroduce local\n", - "# PipelineProfiler / CandidateTrade / SessionFilter classes, stage alias tables,\n", - "# import_from_profiler hydration, or set-union count preservation.\n", - "import os, sys, uuid\n", + "# === Shared library imports (feature engineering + session filter) ===\n", + "# Feature engineering, the ML feature builder and the session filter live in\n", + "# pipeline_verification_bundle/shared so they stay identical across notebooks.\n", + "# This cell only locates the bundle and imports those helpers.\n", + "import os, sys\n", "from pathlib import Path\n", "\n", "_MARKER = \"pipeline_verification_bundle\"\n", @@ -56,7 +51,7 @@ "def _find_repo_root(start: Path) -> Path:\n", " start = start.resolve()\n", " for cand in (start, *start.parents):\n", - " if (cand / _MARKER / \"shared\" / \"pipeline_profiler.py\").exists():\n", + " if (cand / _MARKER / \"shared\" / \"features.py\").exists():\n", " return cand\n", " for cand in (start, *start.parents):\n", " if (cand / _MARKER).is_dir():\n", @@ -82,172 +77,15 @@ " if _p.exists() and str(_p) not in sys.path:\n", " sys.path.insert(0, str(_p))\n", "\n", - "from shared.pipeline_profiler import (\n", - " PipelineProfiler, PipelineEvent, PipelineIntegrityError, PIPELINE_STAGES,\n", - " write_oos_reports, write_is_debug_report, purge_reports_dir, build_run_meta,\n", - " CandidateTrade, make_candidate_id, parameter_set_id, RejectionReason,\n", - ")\n", - "from shared.session_filter import (\n", - " SessionFilter, ENABLE_SESSION_FILTER, session_col_from_value,\n", - ")\n", - "from shared.features import (\n", - " build_features, build_ml_features, feature_hash, ema, true_range, atr, rsi, adx,\n", - ")\n", - "from shared.prediction import PredictionResult, make_prediction, directional_probability\n", - "from shared.pipeline_certification import certify, render_certification, self_validate_notebooks\n", + "from shared.session_filter import SessionFilter, ENABLE_SESSION_FILTER, session_col_from_value\n", + "from shared.features import build_features, build_ml_features, feature_hash, ema, true_range, atr, rsi, adx\n", "\n", - "# Phase 6: diagnostics MUST use the production canonical inference, never\n", - "# predict_states_canonical(model, ...). predict_states applies canonical remap + smoothing.\n", - "try:\n", - " from src.engine_hmm import predict_states as _engine_predict_states\n", - "except Exception:\n", - " _engine_predict_states = None\n", - "\n", - "def predict_states_canonical(model, X_hmm):\n", - " \"\"\"Canonical HMM states for diagnostics (same mapping as live pipeline).\"\"\"\n", - " if _engine_predict_states is not None:\n", - " try:\n", - " return _engine_predict_states(model, X_hmm)\n", - " except Exception:\n", - " pass\n", - " _ps = getattr(model, \"predict_states\", None)\n", - " if callable(_ps):\n", - " return _ps(X_hmm)\n", - " raise PipelineIntegrityError(\n", - " \"No canonical predict_states available; refusing raw model.hmm.predict in diagnostics.\"\n", - " )\n", - "\n", - "OBS_OUTPUT_DIR = _repo_root / _MARKER / \"reports\" / \"observability\"\n", - "OBS_OUTPUT_DIR.mkdir(parents=True, exist_ok=True)\n", - "\n", - "RUN_UUID = str(uuid.uuid4())\n", - "# Phase 5: completely separate diagnostics. Only profiler_oos drives the\n", - "# canonical lifecycle reports; profiler_is is debug-only. `profiler` == OOS.\n", - "profiler_oos = PipelineProfiler(dataset=\"OOS\", run_uuid=RUN_UUID)\n", - "profiler_is = PipelineProfiler(dataset=\"IS\", run_uuid=RUN_UUID)\n", - "profiler = profiler_oos\n", - "\n", - "# Diagnostics accumulators consumed by the final Certification cell.\n", - "V2_OBSERVED_STATES = []\n", - "V2_MODEL_UUIDS = {\"train\": None, \"eval\": None, \"export\": None}\n", - "\n", - "class _ObsShim:\n", - " \"\"\"Back-compat shim so any leftover obs.* calls are harmless no-ops.\n", - " The real observability now flows through profiler_oos / profiler_is.\n", - " Uses __getattr__ only (no named methods) so no legacy observability method\n", - " names are re-declared in the notebook.\"\"\"\n", - " def __init__(self, prof):\n", - " object.__setattr__(self, \"_p\", prof)\n", - " def __getattr__(self, _name):\n", - " return (lambda *a, **k: None)\n", - "\n", - "obs = _ObsShim(profiler_oos)\n", - "\n", - "print(\"[observability-v2] repo_root =\", _repo_root)\n", - "print(\"[observability-v2] run_uuid =\", RUN_UUID)\n", - "print(\"[observability-v2] stages =\", PIPELINE_STAGES)\n", - "print(\"[observability-v2] session =\", ENABLE_SESSION_FILTER)\n" + "print(\"[shared] repo_root =\", _repo_root)\n" ] }, { "cell_type": "code", "execution_count": 2, - "id": "3ed90f12", - "metadata": { - "tags": [ - "pipeline_observability_v1" - ] - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "[observability] EXPORT_FORENSICS=False VERIFY_PIPELINE=True\n" - ] - } - ], - "source": [ - "# --- Forensic observability config flags (tag: pipeline_observability_v1) ---\n", - "# EXPORT_FORENSICS : when True (default) the finalize step writes the full\n", - "# 18-phase forensic artifact suite. Set to False for a\n", - "# lightweight research run that skips the heavy exports.\n", - "# VERIFY_PIPELINE : when True (default) the Phase-17 self-verification cell\n", - "# asserts every required artifact exists and contains data.\n", - "EXPORT_FORENSICS = False\n", - "VERIFY_PIPELINE = True\n", - "print(f\"[observability] EXPORT_FORENSICS={EXPORT_FORENSICS} VERIFY_PIPELINE={VERIFY_PIPELINE}\")\n" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "id": "obs_manifest_validate_v1", - "metadata": { - "tags": [ - "pipeline_observability_v1" - ] - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "[manifest] loaded: {'feature_hash': '623f341ad1b78b4e7c919f12106b8693da20a38a180f0e7cc6b70d33b6ad50ef', 'session_filter_hash': 'c023b0b650f67baa63d91e5f0e35040763bad77a561b19c55e323e7d99db84f3', 'strategy_hash': 'e83d755769aefe1c0f9533fce138127f326ba7da88ea6b3fca5e3848a0ea293c', 'candidate_hash': '12ae32cb1ec02d01eda3581b127c1fee3b0dc53572ed6baf239721a03d82e126', 'model_hash': 'c89a148be40e6752261e3038609a4b68de22fa3bfdaf32f884edffb8480b9bbe', 'pipeline_version': 'v1.0.0', 'written_at': '2026-07-14T07:23:44.615783+00:00'}\n", - "[manifest] validation: FAIL\n", - " MISMATCH: feature_hash: manifest='623f341ad1b78b4e7c919f12106b8693da20a38a180f0e7cc6b70d33b6ad50ef' notebook=\n", - "WARNING: Pipeline manifest mismatch detected; continuing Explorer run.\n" - ] - } - ], - "source": [ - "import json\n", - "\n", - "# --- Pipeline Manifest validation (tag: pipeline_observability_v1) -------\n", - "# Reads the manifest at the SAME absolute path Strategy Tester writes\n", - "# (`pipeline_verification_bundle/reports/pipeline_manifest.json`), so the\n", - "# two notebooks stay in sync no matter which cwd they were launched from.\n", - "# Aborts the Explorer run if declared hashes do not match the manifest.\n", - "_manifest_path = Path(\"pipeline_verification_bundle/reports/pipeline_manifest.json\")\n", - "if _manifest_path.exists():\n", - " with _manifest_path.open(\"r\", encoding=\"utf-8\") as fh:\n", - " _loaded = json.load(fh)\n", - "\n", - " print(\"[manifest] loaded:\", {k: _loaded[k] for k in _loaded if k != \"extras\"})\n", - "\n", - " _expected = {}\n", - " for _k in (\n", - " \"feature_hash\",\n", - " \"session_filter_hash\",\n", - " \"strategy_hash\",\n", - " \"candidate_hash\",\n", - " \"model_hash\",\n", - " \"pipeline_version\",\n", - " ):\n", - " if _k in globals():\n", - " _expected[_k] = globals()[_k]\n", - "\n", - " if _expected:\n", - " _mismatches = []\n", - " for _k, _v in _expected.items():\n", - " manifest_val = _loaded.get(_k)\n", - " if manifest_val != _v:\n", - " _mismatches.append((_k, manifest_val, _v))\n", - "\n", - " if _mismatches:\n", - " print(\"[manifest] validation: FAIL\")\n", - " for _k, _manifest_val, _expected_val in _mismatches:\n", - " print(f\" MISMATCH: {_k}: manifest={_manifest_val!r} notebook={_expected_val!r}\")\n", - " print(\"WARNING: Pipeline manifest mismatch detected; continuing Explorer run.\")\n", - " else:\n", - " print(\"[manifest] validation: PASS\")\n", - "else:\n", - " print(f\"[manifest] no manifest at {_manifest_path} -- run Strategy Tester first.\")\n" - ] - }, - { - "cell_type": "code", - "execution_count": 4, "id": "b61de1db", "metadata": {}, "outputs": [ @@ -425,7 +263,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 3, "id": "35e34f68", "metadata": {}, "outputs": [ @@ -589,7 +427,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 4, "id": "bridge001", "metadata": {}, "outputs": [ @@ -674,7 +512,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 5, "id": "7a4d4a1a", "metadata": {}, "outputs": [], @@ -819,7 +657,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 6, "id": "94907985", "metadata": {}, "outputs": [], @@ -919,7 +757,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 7, "id": "3f2da117", "metadata": {}, "outputs": [], @@ -976,7 +814,7 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 8, "id": "76c6d481", "metadata": {}, "outputs": [], @@ -1140,7 +978,7 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 9, "id": "4b17aa31", "metadata": {}, "outputs": [], @@ -1606,7 +1444,7 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 10, "id": "172adcb0", "metadata": {}, "outputs": [], @@ -1777,7 +1615,7 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 11, "id": "ab0168cc", "metadata": {}, "outputs": [], @@ -1877,7 +1715,7 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 12, "id": "a7635e18", "metadata": {}, "outputs": [ @@ -1964,7 +1802,7 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": 13, "id": "333cfe71", "metadata": {}, "outputs": [ @@ -1984,9 +1822,9 @@ " HMM converged (diag/0.01) LL=-1207905.8\n", " -> Training ML Fold 4/4...\n", " HMM converged (diag/0.01) LL=-1196579.1\n", - "Pre-computation for M15 complete in 1.5 minutes.\n", + "Pre-computation for M15 complete in 1.4 minutes.\n", "\n", - "Grid sweep for M15 complete in 96.66 seconds\n", + "Grid sweep for M15 complete in 84.99 seconds\n", "\n", "M15 Coarse Results Top 3:\n" ] @@ -2104,9 +1942,9 @@ " HMM converged (diag/0.01) LL=-3563862.1\n", " -> Training ML Fold 4/4...\n", " HMM converged (diag/0.01) LL=-3516858.4\n", - "Pre-computation for M5 complete in 4.4 minutes.\n", + "Pre-computation for M5 complete in 4.9 minutes.\n", "\n", - "Grid sweep for M5 complete in 265.41 seconds\n", + "Grid sweep for M5 complete in 296.87 seconds\n", "\n", "M5 Coarse Results Top 3:\n" ] @@ -2240,7 +2078,7 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": 14, "id": "f2c89972", "metadata": {}, "outputs": [ @@ -2250,7 +2088,7 @@ "text": [ "\n", "--- FINE PASS (M15) ---\n", - "Grid sweep for M15 complete in 0.47 seconds\n", + "Grid sweep for M15 complete in 0.56 seconds\n", "\n", "M15 Fine Results Top 3:\n" ] @@ -2358,7 +2196,7 @@ "text": [ "\n", "--- FINE PASS (M5) ---\n", - "Grid sweep for M5 complete in 0.64 seconds\n", + "Grid sweep for M5 complete in 0.70 seconds\n", "\n", "M5 Fine Results Top 3:\n" ] @@ -2494,7 +2332,7 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": 15, "id": "ca2f1399", "metadata": {}, "outputs": [ @@ -2689,7 +2527,7 @@ }, { "cell_type": "code", - "execution_count": 18, + "execution_count": 16, "id": "e5ee47e2", "metadata": {}, "outputs": [ @@ -3076,7 +2914,7 @@ }, { "cell_type": "code", - "execution_count": 19, + "execution_count": 17, "id": "8373f155", "metadata": {}, "outputs": [ @@ -3304,7 +3142,7 @@ }, { "cell_type": "code", - "execution_count": 20, + "execution_count": 18, "id": "diag-phase9", "metadata": {}, "outputs": [ @@ -3453,7 +3291,7 @@ }, { "cell_type": "code", - "execution_count": 21, + "execution_count": 19, "id": "6f93528e", "metadata": {}, "outputs": [], @@ -3510,4290 +3348,6 @@ "\n", " return \"GREEN\", f\"Healthy. live sharpe={live_sharpe:.2f}\"" ] - }, - { - "cell_type": "code", - "execution_count": 22, - "id": "diag-setup", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Diagnostic instrumentation layer loaded.\n" - ] - } - ], - "source": [ - "# ============================================================\n", - "# DIAGNOSTIC INSTRUMENTATION LAYER — Setup\n", - "# ============================================================\n", - "\n", - "from dataclasses import dataclass, field\n", - "from typing import Dict, List, Any, Tuple\n", - "import numpy as np\n", - "import pandas as pd\n", - "import warnings\n", - "import math\n", - "\n", - "warnings.filterwarnings(\"ignore\")\n", - "\n", - "# Additional sklearn imports for calibration diagnostics\n", - "try:\n", - " from sklearn.calibration import calibration_curve\n", - " from sklearn.metrics import brier_score_loss, log_loss, roc_auc_score, precision_recall_curve, average_precision_score\n", - " CALIBRATION_OK = True\n", - "except Exception as e:\n", - " CALIBRATION_OK = False\n", - " print(\"WARNING: sklearn calibration metrics not available:\", e)\n", - "\n", - "# --- Central diagnostic storage ---\n", - "diagnostics = {\n", - " \"M15\": {},\n", - " \"M5\": {},\n", - "}\n", - "candidate_reports = {\n", - " \"M15\": {},\n", - " \"M5\": {},\n", - "}\n", - "\n", - "@dataclass\n", - "class PipelineFunnel:\n", - " \"\"\"Tracks observation counts at every processing stage.\"\"\"\n", - " raw_bars: int = 0\n", - " feature_rows: int = 0\n", - " after_nan_removal: int = 0\n", - " after_tbm: int = 0\n", - " trend_pullback_signals: int = 0\n", - " after_session_filter: int = 0\n", - " after_regime_filter: int = 0\n", - " after_probability_filter: int = 0\n", - " executed_trades: int = 0\n", - " winning_trades: int = 0\n", - " losing_trades: int = 0\n", - "\n", - "@dataclass\n", - "class FeatureLossAudit:\n", - " \"\"\"Tracks how many rows each feature engineering step removes.\"\"\"\n", - " raw: int = 0\n", - " steps: List[Tuple[str, int]] = field(default_factory=list)\n", - " final: int = 0\n", - "\n", - "@dataclass\n", - "class RejectionBreakdown:\n", - " \"\"\"Tracks why candidate trades were rejected.\"\"\"\n", - " session_filter: int = 0\n", - " regime_filter: int = 0\n", - " probability_filter: int = 0\n", - " atr_filter: int = 0\n", - " spread_filter: int = 0\n", - " risk_filter: int = 0\n", - " duplicate_position: int = 0\n", - " max_exposure: int = 0\n", - " other: int = 0\n", - "\n", - "print(\"Diagnostic instrumentation layer loaded.\")\n" - ] - }, - { - "cell_type": "code", - "execution_count": 23, - "id": "diag-feature-trace", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "build_features_with_trace ready.\n" - ] - } - ], - "source": [ - "# ============================================================\n", - "# Phase 8: Feature Loss Audit + Phase 1 Pipeline Trace\n", - "# ============================================================\n", - "\n", - "def _count_after_dropna(df, cols):\n", - " \"\"\"Return len of df[cols] after dropna.\"\"\"\n", - " subset = [c for c in cols if c in df.columns]\n", - " return len(df[subset].dropna())\n", - "\n", - "def build_features_with_trace(exec_panel: pd.DataFrame, exec_tf: str) -> tuple:\n", - " \"\"\"Replica of build_features that returns (df, trace, audit).\"\"\"\n", - " exec_tf = exec_tf.upper()\n", - " exec_df = exec_panel.copy()\n", - " audit = FeatureLossAudit()\n", - " audit.raw = len(exec_df)\n", - "\n", - " trace = {\"raw\": len(exec_df)}\n", - " cols_so_far = [\"Open\", \"High\", \"Low\", \"Close\"]\n", - "\n", - " exec_df[\"log_return\"] = np.log(exec_df[\"Close\"] / exec_df[\"Close\"].shift(1))\n", - " cols_so_far.append(\"log_return\")\n", - " audit.steps.append((\"log_return\", _count_after_dropna(exec_df, cols_so_far)))\n", - "\n", - " exec_df[\"xag_log_return\"] = np.log(exec_df[\"XAG_Close\"] / exec_df[\"XAG_Close\"].shift(1))\n", - " cols_so_far.append(\"xag_log_return\")\n", - " audit.steps.append((\"xag_log_return\", _count_after_dropna(exec_df, cols_so_far)))\n", - "\n", - " exec_df[\"xti_log_return\"] = np.log(exec_df[\"XTI_Close\"] / exec_df[\"XTI_Close\"].shift(1))\n", - " cols_so_far.append(\"xti_log_return\")\n", - " audit.steps.append((\"xti_log_return\", _count_after_dropna(exec_df, cols_so_far)))\n", - "\n", - " exec_df[\"gold_silver_ratio\"] = exec_df[\"Close\"] / exec_df[\"XAG_Close\"]\n", - " exec_df[\"gold_oil_ratio\"] = exec_df[\"Close\"] / exec_df[\"XTI_Close\"]\n", - " cols_so_far.extend([\"gold_silver_ratio\", \"gold_oil_ratio\"])\n", - " audit.steps.append((\"gold_ratios\", _count_after_dropna(exec_df, cols_so_far)))\n", - "\n", - " rw = 64\n", - " gs_m = exec_df[\"gold_silver_ratio\"].rolling(rw, min_periods=rw).mean()\n", - " gs_s = exec_df[\"gold_silver_ratio\"].rolling(rw, min_periods=rw).std().replace(0, np.nan)\n", - " go_m = exec_df[\"gold_oil_ratio\"].rolling(rw, min_periods=rw).mean()\n", - " go_s = exec_df[\"gold_oil_ratio\"].rolling(rw, min_periods=rw).std().replace(0, np.nan)\n", - "\n", - " exec_df[\"gold_silver_ratio_z\"] = ((exec_df[\"gold_silver_ratio\"] - gs_m) / gs_s).replace([np.inf, -np.inf], np.nan).fillna(0.0)\n", - " exec_df[\"gold_oil_ratio_z\"] = ((exec_df[\"gold_oil_ratio\"] - go_m) / go_s).replace([np.inf, -np.inf], np.nan).fillna(0.0)\n", - " cols_so_far.extend([\"gold_silver_ratio_z\", \"gold_oil_ratio_z\"])\n", - " audit.steps.append((\"gold_ratio_z\", _count_after_dropna(exec_df, cols_so_far)))\n", - "\n", - " tr = true_range(exec_df[\"High\"], exec_df[\"Low\"], exec_df[\"Close\"])\n", - " exec_df[\"atr_20\"] = tr.rolling(20, min_periods=20).mean()\n", - " exec_df[\"atr_normalized\"] = exec_df[\"atr_20\"] / exec_df[\"Close\"]\n", - " exec_df[\"volatility_20\"] = exec_df[\"log_return\"].rolling(20, min_periods=20).std()\n", - " exec_df[\"synth_vix_zscore\"] = synth_vix_zscore(exec_df[\"High\"], exec_df[\"Low\"], exec_df[\"Close\"], period=22)\n", - " cols_so_far.extend([\"atr_20\", \"atr_normalized\", \"volatility_20\", \"synth_vix_zscore\"])\n", - " audit.steps.append((\"volatility_cluster\", _count_after_dropna(exec_df, cols_so_far)))\n", - "\n", - " exec_df[\"hour\"] = exec_df.index.hour\n", - " exec_df[\"hour_sin\"] = np.sin(2 * np.pi * exec_df[\"hour\"] / 24.0)\n", - " exec_df[\"hour_cos\"] = np.cos(2 * np.pi * exec_df[\"hour\"] / 24.0)\n", - " cols_so_far.extend([\"hour_sin\", \"hour_cos\"])\n", - " audit.steps.append((\"temporal\", _count_after_dropna(exec_df, cols_so_far)))\n", - "\n", - " exec_df[\"rsi5\"] = rsi(exec_df[\"Close\"], period=5)\n", - " exec_df[\"atr14\"] = atr(exec_df[\"High\"], exec_df[\"Low\"], exec_df[\"Close\"], period=14)\n", - " exec_df[\"atr100\"] = atr(exec_df[\"High\"], exec_df[\"Low\"], exec_df[\"Close\"], period=100)\n", - " exec_df[\"atr_expansion\"] = exec_df[\"atr14\"] / exec_df[\"atr100\"].replace(0.0, np.nan)\n", - " cols_so_far.extend([\"rsi5\", \"atr14\", \"atr100\", \"atr_expansion\"])\n", - " audit.steps.append((\"oscillators_atr\", _count_after_dropna(exec_df, cols_so_far)))\n", - "\n", - " exec_df[\"macro_ema50\"] = ema(exec_df[\"Close\"], period=50)\n", - " exec_df[\"macro_ema200\"] = ema(exec_df[\"Close\"], period=200)\n", - " exec_df[\"macro_adx14\"] = adx(exec_df[\"High\"], exec_df[\"Low\"], exec_df[\"Close\"], period=14)\n", - " cols_so_far.extend([\"macro_ema50\", \"macro_ema200\", \"macro_adx14\"])\n", - " audit.steps.append((\"macro_trend\", _count_after_dropna(exec_df, cols_so_far)))\n", - "\n", - " merged = add_session_features(exec_df)\n", - " merged[\"spread\"] = SPREAD_CAP_POINTS\n", - " cols_so_far.extend([\"session_mask_none\", \"session_mask_london\", \"session_mask_ny\", \"session_mask_london_ny\", \"spread\"])\n", - " audit.steps.append((\"session_features\", _count_after_dropna(merged, cols_so_far)))\n", - "\n", - " is_trend = (merged[\"macro_adx14\"] > 15.0) & (merged[\"atr_expansion\"] < 1.3)\n", - " is_shock = merged[\"atr_expansion\"] >= 1.3\n", - " merged[\"regime_str\"] = np.where(is_shock, \"SHOCK\", np.where(is_trend, \"TREND\", \"MR\"))\n", - " merged[\"regime_code\"] = merged[\"regime_str\"].map({\"TREND\": 1, \"SHOCK\": 2, \"MR\": 3})\n", - " cols_so_far.extend([\"regime_str\", \"regime_code\"])\n", - " audit.steps.append((\"regime_assignment\", _count_after_dropna(merged, cols_so_far)))\n", - "\n", - " keep = [\n", - " \"Open\", \"High\", \"Low\", \"Close\",\n", - " \"log_return\", \"xag_log_return\", \"xti_log_return\",\n", - " \"gold_silver_ratio_z\", \"gold_oil_ratio_z\",\n", - " \"atr_20\", \"atr_normalized\", \"volatility_20\",\n", - " \"synth_vix_zscore\", \"hour_sin\", \"hour_cos\",\n", - " \"rsi5\", \"atr14\", \"atr100\", \"atr_expansion\",\n", - " \"macro_ema50\", \"macro_ema200\", \"macro_adx14\",\n", - " \"spread\", \"regime_str\", \"regime_code\",\n", - " \"session_mask_none\", \"session_mask_london\", \"session_mask_ny\", \"session_mask_london_ny\",\n", - " ]\n", - "\n", - " before_dropna = len(merged)\n", - " merged = merged[keep].dropna().copy()\n", - " after_dropna = len(merged)\n", - " audit.final = after_dropna\n", - "\n", - " result = triple_barrier(merged, exec_tf)\n", - " trace[\"after_dropna\"] = after_dropna\n", - " trace[\"after_tbm\"] = len(result)\n", - "\n", - " return result, trace, audit\n", - "\n", - "print(\"build_features_with_trace ready.\")\n" - ] - }, - { - "cell_type": "code", - "execution_count": 24, - "id": "diag-signal-trace", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "generate_signals_diagnostic and diagnostic_signal_trace ready.\n" - ] - } - ], - "source": [ - "# ============================================================\n", - "# Phase 9: Signal & Rejection Tracking (Pre-Backtest)\n", - "# ============================================================\n", - "\n", - "def generate_signals_diagnostic(df: pd.DataFrame, params: dict) -> tuple:\n", - " \"\"\"Generate signals and return both pre-session and post-session arrays.\"\"\"\n", - " adx_threshold = float(params.get(\"adx_threshold\", 15.0))\n", - " pullback_rsi = float(params.get(\"pullback_rsi\", 40.0))\n", - " confirmation_bars = int(params.get(\"confirmation_bars\", 1))\n", - " session_filter = params.get(\"session_filter\", None)\n", - " session_col = session_col_from_value(session_filter)\n", - "\n", - " trend_up_raw = (df[\"macro_ema50\"] > df[\"macro_ema200\"]) & (df[\"macro_adx14\"] > adx_threshold)\n", - " trend_dn_raw = (df[\"macro_ema50\"] < df[\"macro_ema200\"]) & (df[\"macro_adx14\"] > adx_threshold)\n", - "\n", - " if confirmation_bars > 1:\n", - " trend_up = trend_up_raw.rolling(confirmation_bars, min_periods=confirmation_bars).sum().eq(confirmation_bars)\n", - " trend_dn = trend_dn_raw.rolling(confirmation_bars, min_periods=confirmation_bars).sum().eq(confirmation_bars)\n", - " else:\n", - " trend_up, trend_dn = trend_up_raw, trend_dn_raw\n", - "\n", - " # Pre-session: trend + pullback only\n", - " long_pre = trend_up & (df[\"rsi5\"] < pullback_rsi)\n", - " short_pre = trend_dn & (df[\"rsi5\"] > (100.0 - pullback_rsi))\n", - " sig_pre = pd.Series(0, index=df.index, dtype=np.int8)\n", - " sig_pre.loc[long_pre.fillna(False)] = 1\n", - " sig_pre.loc[short_pre.fillna(False)] = -1\n", - "\n", - " # Post-session: add session filter\n", - " long_post = long_pre & df[session_col].astype(bool)\n", - " short_post = short_pre & df[session_col].astype(bool)\n", - " sig_post = pd.Series(0, index=df.index, dtype=np.int8)\n", - " sig_post.loc[long_post.fillna(False)] = 1\n", - " sig_post.loc[short_post.fillna(False)] = -1\n", - "\n", - " return sig_pre, sig_post\n", - "\n", - "def diagnostic_signal_trace(df: pd.DataFrame, ml_probs: np.ndarray, base_params: dict, xgb_threshold: float) -> dict:\n", - " \"\"\"Trace signal counts through every filtering stage.\"\"\"\n", - " sig_pre, sig_post = generate_signals_diagnostic(df, base_params)\n", - "\n", - " # Regime filter (TREND only)\n", - " trend_mask = df[\"regime_code\"].to_numpy() == 1\n", - " regime_filtered = sig_post.where(trend_mask, 0)\n", - "\n", - " raw_arr = sig_pre.to_numpy(dtype=np.int8)\n", - " post_arr = sig_post.to_numpy(dtype=np.int8)\n", - " regime_arr = regime_filtered.to_numpy(dtype=np.int8)\n", - "\n", - " prob_down = ml_probs[:, 0]\n", - " prob_up = ml_probs[:, 2]\n", - " filtered_arr = regime_arr.copy()\n", - " filtered_arr[(regime_arr == 1) & (prob_up < xgb_threshold)] = 0\n", - " filtered_arr[(regime_arr == -1) & (prob_down < xgb_threshold)] = 0\n", - "\n", - " return {\n", - " \"raw_signals\": int(np.sum(raw_arr != 0)),\n", - " \"after_session_filter\": int(np.sum(post_arr != 0)),\n", - " \"after_regime_filter\": int(np.sum(regime_arr != 0)),\n", - " \"after_probability_filter\": int(np.sum(filtered_arr != 0)),\n", - " \"signal_array\": filtered_arr,\n", - " }\n", - "\n", - "print(\"generate_signals_diagnostic and diagnostic_signal_trace ready.\")\n" - ] - }, - { - "cell_type": "code", - "execution_count": 25, - "id": "diag-backtest-trace", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "diagnostic_backtest_trace ready.\n" - ] - } - ], - "source": [ - "# ============================================================\n", - "# Phase 9 (continued): Execution-Level Rejection Tracking\n", - "# ============================================================\n", - "\n", - "def diagnostic_backtest_trace(timeframe: str, df: pd.DataFrame, ml_probs: np.ndarray, \n", - " base_params: dict, xgb_threshold: float) -> dict:\n", - " \"\"\"Run full backtest and trace execution-level rejections.\"\"\"\n", - "\n", - " # 1. Signal trace (pre-backtest)\n", - " sig_trace = diagnostic_signal_trace(df, ml_probs, base_params, xgb_threshold)\n", - " filtered_arr = sig_trace[\"signal_array\"]\n", - "\n", - " # 2. Run actual backtest\n", - " trades_df, met = run_ml_filtered_backtest(timeframe, df, ml_probs, base_params, xgb_threshold)\n", - "\n", - " n_executed = len(trades_df) if not trades_df.empty else 0\n", - " n_win = (trades_df[\"pnl_cents\"] > 0).sum() if not trades_df.empty else 0\n", - " n_loss = (trades_df[\"pnl_cents\"] <= 0).sum() if not trades_df.empty else 0\n", - "\n", - " # 3. Execution-level rejection analysis\n", - " rej = RejectionBreakdown()\n", - " prob_passed = sig_trace[\"after_probability_filter\"]\n", - " execution_rejected = prob_passed - n_executed\n", - "\n", - " if execution_rejected > 0:\n", - " # Analyse each signal bar that did not result in a trade\n", - " sig = filtered_arr\n", - " high = df[\"High\"].to_numpy(dtype=np.float64)\n", - " low = df[\"Low\"].to_numpy(dtype=np.float64)\n", - " close = df[\"Close\"].to_numpy(dtype=np.float64)\n", - " spread = df[\"spread\"].fillna(40.0).to_numpy(dtype=np.float64)\n", - " atr14 = df[\"atr14\"].to_numpy(dtype=np.float64)\n", - " regime_code = df[\"regime_code\"].to_numpy(dtype=np.int32)\n", - "\n", - " # We need to know which bars had entries.\n", - " # Re-run the backtest internals to get entry indices.\n", - " idx = df.index\n", - " time_minutes = (idx.view(\"int64\") // 60000000000).astype(np.int64)\n", - " is_m5 = 1 if timeframe == \"M5\" else 0\n", - " exit_model_map = {\"fixed_tp\": 0, \"mr_exit\": 1, \"fixed_tp_plus_mr\": 2, \"partial_tp_plus_mr\": 3, \"partial_tp_mr_time_stop\": 4}\n", - " exit_mode_code = int(exit_model_map.get(base_params.get(\"exit_model\", \"fixed_tp\"), 0))\n", - "\n", - " sl_dist = float(base_params.get(\"atr_stop\", base_params.get(\"sl_atr\", base_params.get(\"atr_mult_sl\", base_params.get(\"stop_loss_multiplier\", 2.0)))))\n", - " tp_dist = float(base_params.get(\"atr_target\", base_params.get(\"tp_atr\", base_params.get(\"atr_mult_tp\", base_params.get(\"take_profit_multiplier\", 2.0)))))\n", - " leg_a_tp = float(base_params.get(\"leg_a_atr_target\", base_params.get(\"leg_a_tp\", 1.0)))\n", - "\n", - " (trade_entry_idx, trade_exit_idx, trade_leg, trade_side, trade_entry_px,\n", - " trade_exit_px, trade_move_pips, trade_pnl_cents, trade_reason,\n", - " trade_entry_regime, trade_count, eq, eq_count, ending_balance) = _run_backtest_numba(\n", - " sig, high, low, close, spread, atr14, regime_code, time_minutes,\n", - " sl_dist, tp_dist, leg_a_tp, exit_mode_code, \n", - " float(base_params.get(\"time_stop_minutes\", -1.0)), float(base_params.get(\"trail_mult\", 0.0)),\n", - " float(INITIAL_BALANCE_CENTS), 0.10, 100.0, 0.30, 40.0, 0.0, int(is_m5)\n", - " )\n", - "\n", - " entry_bars = set(trade_entry_idx[:trade_count]) if trade_count > 0 else set()\n", - "\n", - " # Build a simple position-active tracker\n", - " n = len(sig)\n", - " position_active = np.zeros(n, dtype=bool)\n", - " for t in range(trade_count):\n", - " e = int(trade_entry_idx[t])\n", - " x = int(trade_exit_idx[t])\n", - " if e < n and x < n:\n", - " position_active[e:x+1] = True\n", - "\n", - " # Balance tracker (interpolate from equity curve at trade boundaries)\n", - " balance_at = np.full(n, float(INITIAL_BALANCE_CENTS), dtype=float)\n", - " for t in range(trade_count):\n", - " x = int(trade_exit_idx[t])\n", - " if x < n:\n", - " balance_at[x:] = float(eq[min(t+1, eq_count-1)])\n", - "\n", - " for i in range(n):\n", - " if sig[i] == 0:\n", - " continue\n", - " if i in entry_bars:\n", - " continue\n", - "\n", - " # Signal passed probability filter but did not enter\n", - " if balance_at[i] <= 1000.0:\n", - " rej.risk_filter += 1\n", - " elif not np.isfinite(atr14[i]) or atr14[i] <= 0.0:\n", - " rej.atr_filter += 1\n", - " elif (spread[i] / 10.0 * 0.10) > (0.8 * atr14[i]):\n", - " rej.spread_filter += 1\n", - " elif position_active[i]:\n", - " # Already in a position (duplicate or max exposure)\n", - " rej.duplicate_position += 1\n", - " else:\n", - " rej.other += 1\n", - "\n", - " return {\n", - " \"raw_signals\": sig_trace[\"raw_signals\"],\n", - " \"after_session_filter\": sig_trace[\"after_session_filter\"],\n", - " \"after_regime_filter\": sig_trace[\"after_regime_filter\"],\n", - " \"after_probability_filter\": sig_trace[\"after_probability_filter\"],\n", - " \"executed_trades\": n_executed,\n", - " \"winning_trades\": int(n_win),\n", - " \"losing_trades\": int(n_loss),\n", - " \"rejection_breakdown\": rej,\n", - " }, trades_df, met\n", - "\n", - "print(\"diagnostic_backtest_trace ready.\")\n" - ] - }, - { - "cell_type": "code", - "execution_count": 26, - "id": "diag-hmm", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "hmm_diagnostics ready.\n" - ] - } - ], - "source": [ - "# ============================================================\n", - "# Phase 4: HMM Diagnostics\n", - "# ============================================================\n", - "\n", - "def hmm_diagnostics(model, df: pd.DataFrame) -> dict:\n", - " \"\"\"Extract HMM state diagnostics from a trained model and dataset.\"\"\"\n", - " if model is None or model.hmm is None:\n", - " return {}\n", - " \n", - " X_hmm = df[model.hmm_features_].to_numpy(dtype=float)\n", - " X_hmm = model.hmm_scaler.transform(X_hmm)\n", - " states = predict_states_canonical(model, X_hmm)\n", - " \n", - " n_samples = len(states)\n", - " occupancy = {}\n", - " state_stats = {}\n", - " \n", - " for s in range(model.n_components):\n", - " mask = states == s\n", - " count = int(mask.sum())\n", - " pct = count / n_samples * 100 if n_samples > 0 else 0.0\n", - " occupancy[s] = {\"count\": count, \"pct\": pct}\n", - " \n", - " avg_ret = float(df.loc[mask, \"log_return\"].mean()) if \"log_return\" in df.columns else 0.0\n", - " avg_atr = float(df.loc[mask, \"atr14\"].mean()) if \"atr14\" in df.columns else 0.0\n", - " avg_adx = float(df.loc[mask, \"macro_adx14\"].mean()) if \"macro_adx14\" in df.columns else 0.0\n", - " \n", - " state_stats[s] = {\n", - " \"samples\": count,\n", - " \"avg_return\": avg_ret,\n", - " \"avg_atr\": avg_atr,\n", - " \"avg_adx\": avg_adx,\n", - " }\n", - " \n", - " # Transition matrix\n", - " trans = model.hmm.transmat_ if hasattr(model.hmm, \"transmat_\") else None\n", - " \n", - " return {\n", - " \"occupancy\": occupancy,\n", - " \"state_stats\": state_stats,\n", - " \"transition_matrix\": trans.tolist() if trans is not None else None,\n", - " }\n", - "\n", - "print(\"hmm_diagnostics ready.\")\n" - ] - }, - { - "cell_type": "code", - "execution_count": 27, - "id": "diag-prob-thresh", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "probability_distribution and threshold_sensitivity ready.\n" - ] - } - ], - "source": [ - "# ============================================================\n", - "# Phase 5: XGBoost Probability Distribution\n", - "# Phase 6: Threshold Sensitivity\n", - "# ============================================================\n", - "\n", - "def probability_distribution(model, df: pd.DataFrame) -> dict:\n", - " \"\"\"Compute probability distribution statistics.\"\"\"\n", - " if model is None:\n", - " return {}\n", - " probs = model.predict_proba_raw(df)\n", - " p_down = probs[:, 0]\n", - " p_up = probs[:, 2]\n", - " p_extreme = np.maximum(p_down, p_up)\n", - " \n", - " def stats(arr):\n", - " a = np.asarray(arr)\n", - " return {\n", - " \"min\": float(np.min(a)),\n", - " \"max\": float(np.max(a)),\n", - " \"mean\": float(np.mean(a)),\n", - " \"median\": float(np.median(a)),\n", - " \"std\": float(np.std(a)),\n", - " \"p10\": float(np.percentile(a, 10)),\n", - " \"p25\": float(np.percentile(a, 25)),\n", - " \"p50\": float(np.percentile(a, 50)),\n", - " \"p75\": float(np.percentile(a, 75)),\n", - " \"p90\": float(np.percentile(a, 90)),\n", - " }\n", - " \n", - " return {\n", - " \"prob_down\": stats(p_down),\n", - " \"prob_up\": stats(p_up),\n", - " \"prob_extreme\": stats(p_extreme),\n", - " }\n", - "\n", - "def threshold_sensitivity(model, df: pd.DataFrame, tf: str, base_params: dict, \n", - " thresholds: list = None) -> pd.DataFrame:\n", - " \"\"\"Evaluate backtest at multiple thresholds (measurement, not optimisation).\"\"\"\n", - " if thresholds is None:\n", - " thresholds = [0.45, 0.50, 0.55, 0.60, 0.65, 0.70, 0.75]\n", - " \n", - " rows = []\n", - " probs = model.predict_proba_raw(df) if model else None\n", - " \n", - " for thr in thresholds:\n", - " if probs is None:\n", - " rows.append({\"threshold\": thr, \"trades\": 0, \"win_rate\": 0.0, \"profit_factor\": 0.0})\n", - " continue\n", - " trades, met = run_ml_filtered_backtest(tf, df, probs, base_params, float(thr))\n", - " n_trades = len(trades) if not trades.empty else 0\n", - " wr = float(met.get(\"win_rate\", 0.0)) if n_trades > 0 else 0.0\n", - " pf = float(met.get(\"profit_factor\", 0.0)) if n_trades > 0 else 0.0\n", - " rows.append({\n", - " \"threshold\": thr,\n", - " \"trades\": n_trades,\n", - " \"win_rate\": wr,\n", - " \"profit_factor\": pf,\n", - " })\n", - " \n", - " return pd.DataFrame(rows)\n", - "\n", - "print(\"probability_distribution and threshold_sensitivity ready.\")\n" - ] - }, - { - "cell_type": "code", - "execution_count": 28, - "id": "diag-calibration", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "calibration_metrics ready.\n" - ] - } - ], - "source": [ - "# ============================================================\n", - "# Phase 10: Classifier Calibration\n", - "# ============================================================\n", - "\n", - "def calibration_metrics(model, df: pd.DataFrame) -> dict:\n", - " \"\"\"Compute calibration and discrimination metrics.\"\"\"\n", - " if not CALIBRATION_OK or model is None:\n", - " return {}\n", - " \n", - " probs = model.predict_proba_raw(df)\n", - " y_true = df[\"tb_label\"].to_numpy(dtype=int)\n", - " \n", - " # Map labels to classes: -1->0, 0->1, 1->2\n", - " y_cls = np.where(y_true == -1, 0, np.where(y_true == 1, 2, 1))\n", - " \n", - " # Filter out neutral (class 1) for binary calibration on extreme classes\n", - " mask_extreme = y_true != 0\n", - " y_bin = np.where(y_true[mask_extreme] == 1, 1, 0) # 1 = up, 0 = down\n", - " p_up = probs[mask_extreme, 2]\n", - " p_down = probs[mask_extreme, 0]\n", - " p_pos = p_up / (p_up + p_down + 1e-12)\n", - " \n", - " # Brier score (on up-vs-down binary task)\n", - " brier = brier_score_loss(y_bin, p_pos) if len(y_bin) > 0 else np.nan\n", - " \n", - " # Log loss (multiclass)\n", - " ll = log_loss(y_cls, probs, labels=[0,1,2]) if len(set(y_cls)) > 1 else np.nan\n", - " \n", - " # ROC AUC (up-vs-down, one-vs-rest)\n", - " try:\n", - " roc_auc = roc_auc_score(y_bin, p_pos) if len(set(y_bin)) > 1 else np.nan\n", - " except Exception:\n", - " roc_auc = np.nan\n", - " \n", - " # Average precision\n", - " try:\n", - " avg_prec = average_precision_score(y_bin, p_pos) if len(set(y_bin)) > 1 else np.nan\n", - " except Exception:\n", - " avg_prec = np.nan\n", - " \n", - " # Calibration curve data\n", - " prob_true, prob_pred = calibration_curve(y_bin, p_pos, n_bins=10) if len(y_bin) > 0 else (np.array([]), np.array([]))\n", - " \n", - " return {\n", - " \"brier_score\": float(brier),\n", - " \"log_loss\": float(ll),\n", - " \"roc_auc\": float(roc_auc),\n", - " \"average_precision\": float(avg_prec),\n", - " \"calibration_prob_true\": prob_true.tolist(),\n", - " \"calibration_prob_pred\": prob_pred.tolist(),\n", - " }\n", - "\n", - "print(\"calibration_metrics ready.\")\n" - ] - }, - { - "cell_type": "code", - "execution_count": 29, - "id": "diag-strategy-tester", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "compare_strategy_tester_signals ready.\n" - ] - } - ], - "source": [ - "# ============================================================\n", - "# Phase 7: Strategy Tester vs Explorer Comparison\n", - "# ============================================================\n", - "\n", - "def compare_strategy_tester_signals(tf: str, explorer_signals: int) -> dict:\n", - " \"\"\"Compare signal counts between Strategy Tester and Explorer.\"\"\"\n", - " st_path = Path(\"reports/strategy_winners_for_explorer.csv\")\n", - " st_signals = None\n", - " st_trades = None\n", - " if st_path.exists():\n", - " try:\n", - " st_df = pd.read_csv(st_path)\n", - " row = st_df[st_df[\"timeframe\"] == tf]\n", - " if not row.empty:\n", - " st_trades = int(row.iloc[0][\"trade_count\"])\n", - " # Strategy tester trade_count is not signal count, but it's the closest proxy\n", - " # We use it as a reference for \"how many trades the ST produced\"\n", - " except Exception as e:\n", - " print(f\" Could not load ST data for {tf}: {e}\")\n", - " \n", - " return {\n", - " \"explorer_signals\": explorer_signals,\n", - " \"strategy_tester_trades\": st_trades,\n", - " \"difference\": (explorer_signals - st_trades) if st_trades is not None else None,\n", - " \"pct_lost\": ((explorer_signals - st_trades) / st_trades * 100) if st_trades and st_trades > 0 else None,\n", - " }\n", - "\n", - "print(\"compare_strategy_tester_signals ready.\")\n" - ] - }, - { - "cell_type": "code", - "execution_count": 31, - "id": "diag-runner", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "============================================================\n", - "RUNNING DIAGNOSTICS FOR M15\n", - "============================================================\n", - "Diagnostics for M15 complete.\n", - "\n", - "============================================================\n", - "RUNNING DIAGNOSTICS FOR M5\n", - "============================================================\n", - "Diagnostics for M5 complete.\n", - "\n", - "All diagnostics collected.\n" - ] - } - ], - "source": [ - "# ============================================================\n", - "# MAIN DIAGNOSTIC RUNNER — Phases 1-12\n", - "# ============================================================\n", - "\n", - "all_diagnostics = {}\n", - "\n", - "for tf in TIMEFRAMES:\n", - " print(f\"\\n{'='*60}\")\n", - " print(f\"RUNNING DIAGNOSTICS FOR {tf}\")\n", - " print(f\"{'='*60}\")\n", - " \n", - " p = pipeline[tf]\n", - " base_params = dict(ML_TARGET_PARAMS[tf][\"parameters\"])\n", - " base_params[\"exit_model\"] = ML_TARGET_PARAMS[tf][\"exit_model\"]\n", - " \n", - " train_df = p.train_df\n", - " oos_df = p.oos_df\n", - " model = p.model\n", - " thr = float(p.threshold) if p.threshold is not None else 0.15\n", - " \n", - " # --- Phase 1: Pipeline Funnel (IS) ---\n", - " funnel = PipelineFunnel()\n", - " funnel.raw_bars = len(p.raw_all)\n", - " \n", - " feat_is, trace_is, audit_is = build_features_with_trace(train_df, tf)\n", - " funnel.feature_rows = trace_is[\"raw\"]\n", - " funnel.after_nan_removal = audit_is.final\n", - " funnel.after_tbm = len(feat_is)\n", - " \n", - " if model is not None:\n", - " probs_is = model.predict_proba_raw(feat_is)\n", - " diag_is, trades_is, met_is = diagnostic_backtest_trace(tf, feat_is, probs_is, base_params, thr)\n", - " else:\n", - " diag_is = {\"raw_signals\":0, \"after_session_filter\":0, \"after_regime_filter\":0, \"after_probability_filter\":0, \"executed_trades\":0, \"winning_trades\":0, \"losing_trades\":0, \"rejection_breakdown\": RejectionBreakdown()}\n", - " trades_is = pd.DataFrame()\n", - " met_is = {}\n", - " \n", - " funnel.trend_pullback_signals = diag_is[\"raw_signals\"]\n", - " funnel.after_session_filter = diag_is[\"after_session_filter\"]\n", - " funnel.after_regime_filter = diag_is[\"after_regime_filter\"]\n", - " funnel.after_probability_filter = diag_is[\"after_probability_filter\"]\n", - " funnel.executed_trades = diag_is[\"executed_trades\"]\n", - " funnel.winning_trades = diag_is[\"winning_trades\"]\n", - " funnel.losing_trades = diag_is[\"losing_trades\"]\n", - " \n", - " # --- OOS trace ---\n", - " feat_oos, trace_oos, audit_oos = build_features_with_trace(oos_df, tf)\n", - " if model is not None:\n", - " probs_oos = model.predict_proba_raw(feat_oos)\n", - " diag_oos, trades_oos, met_oos = diagnostic_backtest_trace(tf, feat_oos, probs_oos, base_params, thr)\n", - " else:\n", - " diag_oos = {\"raw_signals\":0, \"after_session_filter\":0, \"after_regime_filter\":0, \"after_probability_filter\":0, \"executed_trades\":0, \"winning_trades\":0, \"losing_trades\":0, \"rejection_breakdown\": RejectionBreakdown()}\n", - " trades_oos = pd.DataFrame()\n", - " met_oos = {}\n", - " \n", - " # --- Phase 3: Label Distribution ---\n", - " labels_is = feat_is[\"tb_label\"].value_counts().to_dict()\n", - " labels_oos = feat_oos[\"tb_label\"].value_counts().to_dict()\n", - " \n", - " # --- Phase 4: HMM Diagnostics ---\n", - " if model:\n", - " try:\n", - " hmm_diag = hmm_diagnostics(model, feat_is)\n", - " except PipelineIntegrityError:\n", - " # model not ready for raw HMM prediction — skip diagnostics\n", - " hmm_diag = {}\n", - " except Exception:\n", - " # any unexpected error: fail safe to empty diagnostics\n", - " hmm_diag = {}\n", - " else:\n", - " hmm_diag = {}\n", - " \n", - " # --- Phase 5: Probability Distribution ---\n", - " prob_dist = probability_distribution(model, feat_is) if model else {}\n", - " \n", - " # --- Phase 6: Threshold Sensitivity (OOS) ---\n", - " thresh_sens = threshold_sensitivity(model, feat_oos, tf, base_params) if model else pd.DataFrame()\n", - " \n", - " # --- Phase 7: Strategy Tester Comparison ---\n", - " st_cmp = compare_strategy_tester_signals(tf, diag_is[\"raw_signals\"])\n", - " \n", - " # --- Phase 8: Feature Loss Audit ---\n", - " feature_audit = {\"IS\": audit_is, \"OOS\": audit_oos}\n", - " \n", - " # --- Phase 9: Rejection Breakdown ---\n", - " rej_is = diag_is[\"rejection_breakdown\"]\n", - " rej_oos = diag_oos[\"rejection_breakdown\"]\n", - " \n", - " # --- Phase 10: Calibration ---\n", - " cal = calibration_metrics(model, feat_is) if model else {}\n", - " \n", - " all_diagnostics[tf] = {\n", - " \"funnel\": funnel,\n", - " \"labels_is\": labels_is,\n", - " \"labels_oos\": labels_oos,\n", - " \"hmm\": hmm_diag,\n", - " \"prob_dist\": prob_dist,\n", - " \"thresh_sens\": thresh_sens,\n", - " \"st_cmp\": st_cmp,\n", - " \"feature_audit\": feature_audit,\n", - " \"rej_is\": rej_is,\n", - " \"rej_oos\": rej_oos,\n", - " \"calibration\": cal,\n", - " \"diag_is\": diag_is,\n", - " \"diag_oos\": diag_oos,\n", - " \"met_is\": met_is,\n", - " \"met_oos\": met_oos,\n", - " }\n", - " \n", - " \n", - " # Phase 2: Candidate Report\n", - " candidate_reports[tf] = {\n", - " \"raw_bars\": funnel.raw_bars,\n", - " \"trend_pullback_signals\": funnel.trend_pullback_signals,\n", - " \"after_feature_engineering\": funnel.after_nan_removal,\n", - " \"after_tbm\": funnel.after_tbm,\n", - " \"after_label_drop\": funnel.after_tbm,\n", - " \"after_probability_filter\": funnel.after_probability_filter,\n", - " \"executed_trades\": funnel.executed_trades,\n", - " }\n", - " print(f\"Diagnostics for {tf} complete.\")\n", - "\n", - "print(\"\\nAll diagnostics collected.\")\n" - ] - }, - { - "cell_type": "code", - "execution_count": 32, - "id": "diag-reports", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "============================================================\n", - "TIMEFRAME : M15 (IS)\n", - "============================================================\n", - "Raw Bars : 116101\n", - "Feature Rows : 92881\n", - "Rows after NaN Removal : 92682\n", - "Rows after TBM : 92682\n", - "Trend Pullback Signals : 8107\n", - "After Session Filter : 2995\n", - "After Regime Filter : 1963\n", - "After Probability Filter : 582\n", - "Executed Trades : 968\n", - "Winning Trades : 952\n", - "Losing Trades : 16\n", - "\n", - "============================================================\n", - "TBM LABELS M15 IS\n", - "============================================================\n", - "Positive : 20934\n", - "Negative : 20825\n", - "Neutral : 50923\n", - "Class Ratio : 20934:20825:50923\n", - "Positive % : 22.6%\n", - "Negative % : 22.5%\n", - "Neutral % : 54.9%\n", - "\n", - "============================================================\n", - "TBM LABELS M15 OOS\n", - "============================================================\n", - "Positive : 4974\n", - "Negative : 5373\n", - "Neutral : 12674\n", - "Class Ratio : 4974:5373:12674\n", - "Positive % : 21.6%\n", - "Negative % : 23.3%\n", - "Neutral % : 55.1%\n", - "\n", - "============================================================\n", - "XGBoost Probability Distribution M15\n", - "============================================================\n", - "\n", - "prob_down:\n", - " Min : 0.0000\n", - " Max : 0.9999\n", - " Mean : 0.2247\n", - " Median : 0.0001\n", - " Std : 0.4173\n", - " P10 : 0.0000\n", - " P25 : 0.0000\n", - " P50 : 0.0001\n", - " P75 : 0.0001\n", - " P90 : 0.9999\n", - "\n", - "prob_up:\n", - " Min : 0.0000\n", - " Max : 0.9999\n", - " Mean : 0.2259\n", - " Median : 0.0000\n", - " Std : 0.4181\n", - " P10 : 0.0000\n", - " P25 : 0.0000\n", - " P50 : 0.0000\n", - " P75 : 0.0001\n", - " P90 : 0.9999\n", - "\n", - "prob_extreme:\n", - " Min : 0.0000\n", - " Max : 0.9999\n", - " Mean : 0.4505\n", - " Median : 0.0001\n", - " Std : 0.4975\n", - " P10 : 0.0000\n", - " P25 : 0.0000\n", - " P50 : 0.0001\n", - " P75 : 0.9999\n", - " P90 : 0.9999\n", - "\n", - "============================================================\n", - "THRESHOLD SENSITIVITY M15\n", - "============================================================\n", - " threshold trades win_rate profit_factor\n", - " 0.45 243 0.971193 175.649324\n", - " 0.50 243 0.971193 175.649324\n", - " 0.55 243 0.971193 175.649324\n", - " 0.60 243 0.971193 175.649324\n", - " 0.65 243 0.971193 175.649324\n", - " 0.70 243 0.971193 175.649324\n", - " 0.75 243 0.971193 175.649324\n", - "\n", - "============================================================\n", - "FEATURE LOSS AUDIT M15\n", - "============================================================\n", - "Raw : 92881\n", - " log_return : 92880\n", - " xag_log_return : 92880\n", - " xti_log_return : 92880\n", - " gold_ratios : 92880\n", - " gold_ratio_z : 92880\n", - " volatility_cluster : 92861\n", - " temporal : 92861\n", - " oscillators_atr : 92782\n", - " macro_trend : 92682\n", - " session_features : 92682\n", - " regime_assignment : 92682\n", - "Final : 92682\n", - "Total Loss: 199 rows (0.21%)\n", - "\n", - "============================================================\n", - "TRADE REJECTION REASONS M15 IS\n", - "============================================================\n", - "Session Filter : 0\n", - "Regime Filter : 0\n", - "Probability Filter : 0\n", - "ATR Filter : 0\n", - "Spread Filter : 0\n", - "Risk Filter : 0\n", - "Duplicate Position : 0\n", - "Max Exposure : 0\n", - "Other : 0\n", - "\n", - "============================================================\n", - "TRADE REJECTION REASONS M15 OOS\n", - "============================================================\n", - "Session Filter : 0\n", - "Regime Filter : 0\n", - "Probability Filter : 0\n", - "ATR Filter : 0\n", - "Spread Filter : 0\n", - "Risk Filter : 0\n", - "Duplicate Position : 0\n", - "Max Exposure : 0\n", - "Other : 0\n", - "\n", - "============================================================\n", - "CLASSIFIER CALIBRATION M15\n", - "============================================================\n", - "Brier Score : 0.0000\n", - "Log Loss : 0.0001\n", - "ROC AUC : 1.0000\n", - "Average Precision : 1.0000\n", - "\n", - "============================================================\n", - "STRATEGY TESTER vs EXPLORER M15\n", - "============================================================\n", - "Strategy Tester Trades : 2778\n", - "Explorer Signals : 8107\n", - "Difference : 5329\n", - "Percentage Lost : 191.8%\n", - "WARNING: Signal Loss Detected (>20%)\n", - "\n", - "============================================================\n", - "TIMEFRAME : M5 (IS)\n", - "============================================================\n", - "Raw Bars : 347571\n", - "Feature Rows : 278056\n", - "Rows after NaN Removal : 277857\n", - "Rows after TBM : 277857\n", - "Trend Pullback Signals : 26773\n", - "After Session Filter : 9337\n", - "After Regime Filter : 5468\n", - "After Probability Filter : 2418\n", - "Executed Trades : 3203\n", - "Winning Trades : 3134\n", - "Losing Trades : 69\n", - "\n", - "============================================================\n", - "TBM LABELS M5 IS\n", - "============================================================\n", - "Positive : 115685\n", - "Negative : 114490\n", - "Neutral : 47682\n", - "Class Ratio : 115685:114490:47682\n", - "Positive % : 41.6%\n", - "Negative % : 41.2%\n", - "Neutral % : 17.2%\n", - "\n", - "============================================================\n", - "TBM LABELS M5 OOS\n", - "============================================================\n", - "Positive : 28375\n", - "Negative : 29843\n", - "Neutral : 11098\n", - "Class Ratio : 28375:29843:11098\n", - "Positive % : 40.9%\n", - "Negative % : 43.1%\n", - "Neutral % : 16.0%\n", - "\n", - "============================================================\n", - "XGBoost Probability Distribution M5\n", - "============================================================\n", - "\n", - "prob_down:\n", - " Min : 0.0001\n", - " Max : 0.9999\n", - " Mean : 0.4120\n", - " Median : 0.0001\n", - " Std : 0.4921\n", - " P10 : 0.0001\n", - " P25 : 0.0001\n", - " P50 : 0.0001\n", - " P75 : 0.9999\n", - " P90 : 0.9999\n", - "\n", - "prob_up:\n", - " Min : 0.0001\n", - " Max : 0.9999\n", - " Mean : 0.4163\n", - " Median : 0.0001\n", - " Std : 0.4929\n", - " P10 : 0.0001\n", - " P25 : 0.0001\n", - " P50 : 0.0001\n", - " P75 : 0.9999\n", - " P90 : 0.9999\n", - "\n", - "prob_extreme:\n", - " Min : 0.0001\n", - " Max : 0.9999\n", - " Mean : 0.8283\n", - " Median : 0.9999\n", - " Std : 0.3770\n", - " P10 : 0.0001\n", - " P25 : 0.9999\n", - " P50 : 0.9999\n", - " P75 : 0.9999\n", - " P90 : 0.9999\n", - "\n", - "============================================================\n", - "THRESHOLD SENSITIVITY M5\n", - "============================================================\n", - " threshold trades win_rate profit_factor\n", - " 0.45 1066 0.976548 58.555536\n", - " 0.50 1066 0.976548 58.555536\n", - " 0.55 1066 0.976548 58.555536\n", - " 0.60 1066 0.976548 58.555536\n", - " 0.65 1066 0.976548 58.555536\n", - " 0.70 1066 0.976548 58.555536\n", - " 0.75 1066 0.976548 58.555536\n", - "\n", - "============================================================\n", - "FEATURE LOSS AUDIT M5\n", - "============================================================\n", - "Raw : 278056\n", - " log_return : 278055\n", - " xag_log_return : 278055\n", - " xti_log_return : 278055\n", - " gold_ratios : 278055\n", - " gold_ratio_z : 278055\n", - " volatility_cluster : 278036\n", - " temporal : 278036\n", - " oscillators_atr : 277957\n", - " macro_trend : 277857\n", - " session_features : 277857\n", - " regime_assignment : 277857\n", - "Final : 277857\n", - "Total Loss: 199 rows (0.07%)\n", - "\n", - "============================================================\n", - "TRADE REJECTION REASONS M5 IS\n", - "============================================================\n", - "Session Filter : 0\n", - "Regime Filter : 0\n", - "Probability Filter : 0\n", - "ATR Filter : 0\n", - "Spread Filter : 0\n", - "Risk Filter : 0\n", - "Duplicate Position : 0\n", - "Max Exposure : 0\n", - "Other : 0\n", - "\n", - "============================================================\n", - "TRADE REJECTION REASONS M5 OOS\n", - "============================================================\n", - "Session Filter : 0\n", - "Regime Filter : 0\n", - "Probability Filter : 0\n", - "ATR Filter : 0\n", - "Spread Filter : 0\n", - "Risk Filter : 0\n", - "Duplicate Position : 0\n", - "Max Exposure : 0\n", - "Other : 0\n", - "\n", - "============================================================\n", - "CLASSIFIER CALIBRATION M5\n", - "============================================================\n", - "Brier Score : 0.0000\n", - "Log Loss : 0.0001\n", - "ROC AUC : 1.0000\n", - "Average Precision : 1.0000\n", - "\n", - "============================================================\n", - "STRATEGY TESTER vs EXPLORER M5\n", - "============================================================\n", - "Strategy Tester Trades : 6450\n", - "Explorer Signals : 26773\n", - "Difference : 20323\n", - "Percentage Lost : 315.1%\n", - "WARNING: Signal Loss Detected (>20%)\n" - ] - } - ], - "source": [ - "# ============================================================\n", - "# Print All Diagnostic Reports\n", - "# ============================================================\n", - "\n", - "def print_funnel(tf, funnel, scenario=\"IS\"):\n", - " print(f\"\\n{'='*60}\")\n", - " print(f\"TIMEFRAME : {tf} ({scenario})\")\n", - " print(f\"{'='*60}\")\n", - " print(f\"Raw Bars : {funnel.raw_bars}\")\n", - " print(f\"Feature Rows : {funnel.feature_rows}\")\n", - " print(f\"Rows after NaN Removal : {funnel.after_nan_removal}\")\n", - " print(f\"Rows after TBM : {funnel.after_tbm}\")\n", - " print(f\"Trend Pullback Signals : {funnel.trend_pullback_signals}\")\n", - " print(f\"After Session Filter : {funnel.after_session_filter}\")\n", - " print(f\"After Regime Filter : {funnel.after_regime_filter}\")\n", - " print(f\"After Probability Filter : {funnel.after_probability_filter}\")\n", - " print(f\"Executed Trades : {funnel.executed_trades}\")\n", - " print(f\"Winning Trades : {funnel.winning_trades}\")\n", - " print(f\"Losing Trades : {funnel.losing_trades}\")\n", - "\n", - "def print_labels(tf, labels, scenario=\"IS\"):\n", - " pos = int(labels.get(1, 0))\n", - " neg = int(labels.get(-1, 0))\n", - " neu = int(labels.get(0, 0))\n", - " total = pos + neg + neu\n", - " print(f\"\\n{'='*60}\")\n", - " print(f\"TBM LABELS {tf} {scenario}\")\n", - " print(f\"{'='*60}\")\n", - " print(f\"Positive : {pos}\")\n", - " print(f\"Negative : {neg}\")\n", - " print(f\"Neutral : {neu}\")\n", - " print(f\"Class Ratio : {pos}:{neg}:{neu}\")\n", - " if total > 0:\n", - " print(f\"Positive % : {pos/total*100:.1f}%\")\n", - " print(f\"Negative % : {neg/total*100:.1f}%\")\n", - " print(f\"Neutral % : {neu/total*100:.1f}%\")\n", - "\n", - "def print_hmm(tf, hmm_diag):\n", - " if not hmm_diag:\n", - " return\n", - " print(f\"\\n{'='*60}\")\n", - " print(f\"HMM DIAGNOSTICS {tf}\")\n", - " print(f\"{'='*60}\")\n", - " occ = hmm_diag.get(\"occupancy\", {})\n", - " stats = hmm_diag.get(\"state_stats\", {})\n", - " for s in sorted(occ.keys()):\n", - " pct = occ[s][\"pct\"]\n", - " print(f\"\\nHidden State {s}\")\n", - " print(f\" Samples : {stats[s]['samples']}\")\n", - " print(f\" Average Return : {stats[s]['avg_return']:.6f}\")\n", - " print(f\" Average ATR : {stats[s]['avg_atr']:.4f}\")\n", - " print(f\" Average ADX : {stats[s]['avg_adx']:.4f}\")\n", - " print(f\" State Occupancy %: {pct:.1f}%\")\n", - " if pct < 5:\n", - " print(\" WARNING: Sparse HMM State (<5%)\")\n", - " trans = hmm_diag.get(\"transition_matrix\")\n", - " if trans:\n", - " print(f\"\\nTransition Probabilities:\")\n", - " for row in trans:\n", - " print(\" \" + \" \".join([f\"{v:.3f}\" for v in row]))\n", - "\n", - "def print_prob_dist(tf, prob_dist):\n", - " if not prob_dist:\n", - " return\n", - " print(f\"\\n{'='*60}\")\n", - " print(f\"XGBoost Probability Distribution {tf}\")\n", - " print(f\"{'='*60}\")\n", - " for key in [\"prob_down\", \"prob_up\", \"prob_extreme\"]:\n", - " s = prob_dist.get(key, {})\n", - " if not s:\n", - " continue\n", - " print(f\"\\n{key}:\")\n", - " print(f\" Min : {s['min']:.4f}\")\n", - " print(f\" Max : {s['max']:.4f}\")\n", - " print(f\" Mean : {s['mean']:.4f}\")\n", - " print(f\" Median : {s['median']:.4f}\")\n", - " print(f\" Std : {s['std']:.4f}\")\n", - " print(f\" P10 : {s['p10']:.4f}\")\n", - " print(f\" P25 : {s['p25']:.4f}\")\n", - " print(f\" P50 : {s['p50']:.4f}\")\n", - " print(f\" P75 : {s['p75']:.4f}\")\n", - " print(f\" P90 : {s['p90']:.4f}\")\n", - "\n", - "def print_thresh_sens(tf, thresh_sens):\n", - " if thresh_sens.empty:\n", - " return\n", - " print(f\"\\n{'='*60}\")\n", - " print(f\"THRESHOLD SENSITIVITY {tf}\")\n", - " print(f\"{'='*60}\")\n", - " print(thresh_sens.to_string(index=False))\n", - "\n", - "def print_feature_audit(tf, audit):\n", - " print(f\"\\n{'='*60}\")\n", - " print(f\"FEATURE LOSS AUDIT {tf}\")\n", - " print(f\"{'='*60}\")\n", - " print(f\"Raw : {audit.raw}\")\n", - " prev = audit.raw\n", - " for step_name, count in audit.steps:\n", - " print(f\" {step_name:30s} : {count}\")\n", - " prev = count\n", - " print(f\"Final : {audit.final}\")\n", - " if audit.raw > 0:\n", - " print(f\"Total Loss: {audit.raw - audit.final} rows ({(audit.raw - audit.final)/audit.raw*100:.2f}%)\")\n", - "\n", - "def print_rejections(tf, rej, scenario=\"IS\"):\n", - " print(f\"\\n{'='*60}\")\n", - " print(f\"TRADE REJECTION REASONS {tf} {scenario}\")\n", - " print(f\"{'='*60}\")\n", - " print(f\"Session Filter : {rej.session_filter}\")\n", - " print(f\"Regime Filter : {rej.regime_filter}\")\n", - " print(f\"Probability Filter : {rej.probability_filter}\")\n", - " print(f\"ATR Filter : {rej.atr_filter}\")\n", - " print(f\"Spread Filter : {rej.spread_filter}\")\n", - " print(f\"Risk Filter : {rej.risk_filter}\")\n", - " print(f\"Duplicate Position : {rej.duplicate_position}\")\n", - " print(f\"Max Exposure : {rej.max_exposure}\")\n", - " print(f\"Other : {rej.other}\")\n", - "\n", - "def print_calibration(tf, cal):\n", - " if not cal:\n", - " return\n", - " print(f\"\\n{'='*60}\")\n", - " print(f\"CLASSIFIER CALIBRATION {tf}\")\n", - " print(f\"{'='*60}\")\n", - " print(f\"Brier Score : {cal.get('brier_score', np.nan):.4f}\")\n", - " print(f\"Log Loss : {cal.get('log_loss', np.nan):.4f}\")\n", - " print(f\"ROC AUC : {cal.get('roc_auc', np.nan):.4f}\")\n", - " print(f\"Average Precision : {cal.get('average_precision', np.nan):.4f}\")\n", - "\n", - "def print_st_comparison(tf, cmp):\n", - " print(f\"\\n{'='*60}\")\n", - " print(f\"STRATEGY TESTER vs EXPLORER {tf}\")\n", - " print(f\"{'='*60}\")\n", - " st_trades = cmp.get(\"strategy_tester_trades\")\n", - " exp_signals = cmp.get(\"explorer_signals\")\n", - " if st_trades is not None and exp_signals is not None:\n", - " diff = cmp.get(\"difference\", 0)\n", - " pct = cmp.get(\"pct_lost\", 0)\n", - " print(f\"Strategy Tester Trades : {st_trades}\")\n", - " print(f\"Explorer Signals : {exp_signals}\")\n", - " print(f\"Difference : {diff}\")\n", - " if pct is not None:\n", - " print(f\"Percentage Lost : {pct:.1f}%\")\n", - " if abs(pct) > 20:\n", - " print(\"WARNING: Signal Loss Detected (>20%)\")\n", - " else:\n", - " print(\"Strategy Tester comparison data not available.\")\n", - "\n", - "# --- Print everything ---\n", - "for tf in TIMEFRAMES:\n", - " d = all_diagnostics[tf]\n", - " print_funnel(tf, d[\"funnel\"], \"IS\")\n", - " print_labels(tf, d[\"labels_is\"], \"IS\")\n", - " print_labels(tf, d[\"labels_oos\"], \"OOS\")\n", - " print_hmm(tf, d[\"hmm\"])\n", - " print_prob_dist(tf, d[\"prob_dist\"])\n", - " print_thresh_sens(tf, d[\"thresh_sens\"])\n", - " print_feature_audit(tf, d[\"feature_audit\"][\"IS\"])\n", - " print_rejections(tf, d[\"rej_is\"], \"IS\")\n", - " print_rejections(tf, d[\"rej_oos\"], \"OOS\")\n", - " print_calibration(tf, d[\"calibration\"])\n", - " print_st_comparison(tf, d[\"st_cmp\"])\n" - ] - }, - { - "cell_type": "code", - "execution_count": 33, - "id": "diag-comparison", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "============================================================\n", - "TIMEFRAME COMPARISON TABLE\n", - "============================================================\n", - "Metric M15 M5\n", - "Raw Bars 116101.000000 347571.000000\n", - "Candidate Signals 8107.000000 26773.000000\n", - "Positive Labels 20934.000000 115685.000000\n", - "HMM Samples 92682.000000 277857.000000\n", - "XGB Samples 92682.000000 277857.000000\n", - "Mean Probability 0.450516 0.828328\n", - "Median Probability 0.000053 0.999903\n", - "Threshold 0.150000 0.150000\n", - "Accepted Trades 582.000000 2418.000000\n", - "OOS Trades 243.000000 1066.000000\n", - "Win Rate (IS) 0.983471 0.978458\n", - "PF (IS) 79.598347 59.574039\n", - "\n", - "============================================================\n", - "DIFFERENCES > 15%\n", - "============================================================\n", - " Raw Bars: 116101.00 vs 347571.00 (66.6% diff)\n", - " Candidate Signals: 8107.00 vs 26773.00 (69.7% diff)\n", - " Positive Labels: 20934.00 vs 115685.00 (81.9% diff)\n", - " HMM Samples: 92682.00 vs 277857.00 (66.6% diff)\n", - " XGB Samples: 92682.00 vs 277857.00 (66.6% diff)\n", - " Mean Probability: 0.45 vs 0.83 (45.6% diff)\n", - " Median Probability: 0.00 vs 1.00 (100.0% diff)\n", - " Accepted Trades: 582.00 vs 2418.00 (75.9% diff)\n", - " OOS Trades: 243.00 vs 1066.00 (77.2% diff)\n", - " PF (IS): 79.60 vs 59.57 (25.2% diff)\n", - "\n", - "============================================================\n", - "ROOT CAUSE DETECTION\n", - "============================================================\n", - "WARNING: Candidate generation collapse in M15 relative to M5\n", - "WARNING: Signal Loss Detected vs Strategy Tester (191.8%)\n", - "\n", - "Root cause detection complete.\n" - ] - } - ], - "source": [ - "# ============================================================\n", - "# Phase 11: Automatic Timeframe Comparison\n", - "# Phase 12: Root Cause Detection\n", - "# ============================================================\n", - "\n", - "def build_comparison_table(all_diagnostics: dict) -> pd.DataFrame:\n", - " rows = []\n", - " for tf in TIMEFRAMES:\n", - " d = all_diagnostics[tf]\n", - " funnel = d[\"funnel\"]\n", - " labels = d[\"labels_is\"]\n", - " prob = d.get(\"prob_dist\", {}).get(\"prob_extreme\", {})\n", - " diag = d[\"diag_is\"]\n", - " met = d.get(\"met_is\", {})\n", - " \n", - " pos = labels.get(1, 0)\n", - " neg = labels.get(-1, 0)\n", - " total_labels = pos + neg + labels.get(0, 0)\n", - " pos_ratio = pos / total_labels if total_labels > 0 else 0\n", - " \n", - " rows.append({\n", - " \"Metric\": tf,\n", - " \"Raw Bars\": funnel.raw_bars,\n", - " \"Candidate Signals\": funnel.trend_pullback_signals,\n", - " \"Positive Labels\": pos,\n", - " \"HMM Samples\": funnel.after_tbm,\n", - " \"XGB Samples\": funnel.after_tbm,\n", - " \"Mean Probability\": prob.get(\"mean\", np.nan),\n", - " \"Median Probability\": prob.get(\"median\", np.nan),\n", - " \"Threshold\": pipeline[tf].threshold if pipeline[tf].threshold else np.nan,\n", - " \"Accepted Trades\": diag[\"after_probability_filter\"],\n", - " \"OOS Trades\": d[\"diag_oos\"][\"executed_trades\"],\n", - " \"Win Rate (IS)\": met.get(\"win_rate\", 0),\n", - " \"PF (IS)\": met.get(\"profit_factor\", 0),\n", - " })\n", - " \n", - " df = pd.DataFrame(rows).set_index(\"Metric\").T\n", - " return df\n", - "\n", - "comp = build_comparison_table(all_diagnostics)\n", - "print(f\"\\n{'='*60}\")\n", - "print(\"TIMEFRAME COMPARISON TABLE\")\n", - "print(f\"{'='*60}\")\n", - "print(comp.to_string())\n", - "\n", - "# Highlight differences > 15%\n", - "print(f\"\\n{'='*60}\")\n", - "print(\"DIFFERENCES > 15%\")\n", - "print(f\"{'='*60}\")\n", - "if len(comp.columns) == 2:\n", - " m15 = comp[TIMEFRAMES[0]]\n", - " m5 = comp[TIMEFRAMES[1]]\n", - " for idx in comp.index:\n", - " v1 = m15[idx]\n", - " v2 = m5[idx]\n", - " if pd.isna(v1) or pd.isna(v2) or v1 == 0 or v2 == 0:\n", - " continue\n", - " diff = abs(v1 - v2) / max(abs(v1), abs(v2), 1e-12)\n", - " if diff > 0.15:\n", - " print(f\" {idx}: {v1:.2f} vs {v2:.2f} ({diff*100:.1f}% diff)\")\n", - "\n", - "# --- Phase 12: Root Cause Warnings ---\n", - "print(f\"\\n{'='*60}\")\n", - "print(\"ROOT CAUSE DETECTION\")\n", - "print(f\"{'='*60}\")\n", - "\n", - "reference_tf = \"M5\"\n", - "target_tf = \"M15\"\n", - "ref = all_diagnostics[reference_tf]\n", - "tgt = all_diagnostics[target_tf]\n", - "\n", - "# Check 1: Candidate generation collapse\n", - "if tgt[\"funnel\"].trend_pullback_signals < 0.5 * ref[\"funnel\"].trend_pullback_signals:\n", - " print(\"WARNING: Candidate generation collapse in M15 relative to M5\")\n", - "\n", - "# Check 2: Class imbalance\n", - "pos = tgt[\"labels_is\"].get(1, 0)\n", - "neg = tgt[\"labels_is\"].get(-1, 0)\n", - "neu = tgt[\"labels_is\"].get(0, 0)\n", - "total = pos + neg + neu\n", - "if total > 0 and pos / total < 0.15:\n", - " print(\"WARNING: Severe class imbalance in M15 (<15% positive)\")\n", - "\n", - "# Check 3: Probability calibration issue\n", - "prob_mean = tgt.get(\"prob_dist\", {}).get(\"prob_extreme\", {}).get(\"mean\", 1.0)\n", - "thr = pipeline[\"M15\"].threshold if pipeline[\"M15\"].threshold else 0.15\n", - "if prob_mean < thr:\n", - " print(\"WARNING: Probability calibration issue (mean prob < threshold)\")\n", - "\n", - "# Check 4: HMM state instability\n", - "hmm_occ = tgt.get(\"hmm\", {}).get(\"occupancy\", {})\n", - "for s, info in hmm_occ.items():\n", - " if info[\"pct\"] < 5:\n", - " print(f\"WARNING: HMM state instability (State {s} occupancy {info['pct']:.1f}%)\")\n", - "\n", - "# Check 5: Feature loss\n", - "m15_loss = (tgt[\"funnel\"].raw_bars - tgt[\"funnel\"].after_nan_removal) / max(tgt[\"funnel\"].raw_bars, 1)\n", - "m5_loss = (ref[\"funnel\"].raw_bars - ref[\"funnel\"].after_nan_removal) / max(ref[\"funnel\"].raw_bars, 1)\n", - "if m15_loss > m5_loss * 1.5 and m15_loss > 0.01:\n", - " print(f\"WARNING: M15 losing more rows to feature engineering ({m15_loss*100:.1f}% vs {m5_loss*100:.1f}%)\")\n", - "\n", - "# Check 6: Signal loss vs Strategy Tester\n", - "st_cmp = tgt.get(\"st_cmp\", {})\n", - "pct_lost = st_cmp.get(\"pct_lost\")\n", - "if pct_lost is not None and abs(pct_lost) > 20:\n", - " print(f\"WARNING: Signal Loss Detected vs Strategy Tester ({pct_lost:.1f}%)\")\n", - "\n", - "print(\"\\nRoot cause detection complete.\")\n" - ] - }, - { - "cell_type": "code", - "execution_count": 34, - "id": "diag-export", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Exported: reports/Explorer_Diagnostics_M15.csv\n", - "Exported: reports/Explorer_Diagnostics_M5.csv\n", - "Exported: reports/Explorer_Timeframe_Comparison.csv\n", - "\n", - "All diagnostic reports exported to reports/\n" - ] - } - ], - "source": [ - "# ============================================================\n", - "# Phase 13: Export Diagnostic Reports\n", - "# ============================================================\n", - "\n", - "import os\n", - "\n", - "os.makedirs(\"reports\", exist_ok=True)\n", - "\n", - "for tf in TIMEFRAMES:\n", - " d = all_diagnostics[tf]\n", - " funnel = d[\"funnel\"]\n", - " \n", - " report = {\n", - " \"timeframe\": tf,\n", - " \"raw_bars\": funnel.raw_bars,\n", - " \"feature_rows\": funnel.feature_rows,\n", - " \"after_nan_removal\": funnel.after_nan_removal,\n", - " \"after_tbm\": funnel.after_tbm,\n", - " \"trend_pullback_signals\": funnel.trend_pullback_signals,\n", - " \"after_session_filter\": funnel.after_session_filter,\n", - " \"after_regime_filter\": funnel.after_regime_filter,\n", - " \"after_probability_filter\": funnel.after_probability_filter,\n", - " \"executed_trades_is\": funnel.executed_trades,\n", - " \"winning_trades_is\": funnel.winning_trades,\n", - " \"losing_trades_is\": funnel.losing_trades,\n", - " \"executed_trades_oos\": d[\"diag_oos\"][\"executed_trades\"],\n", - " \"positive_labels_is\": d[\"labels_is\"].get(1, 0),\n", - " \"negative_labels_is\": d[\"labels_is\"].get(-1, 0),\n", - " \"neutral_labels_is\": d[\"labels_is\"].get(0, 0),\n", - " \"positive_labels_oos\": d[\"labels_oos\"].get(1, 0),\n", - " \"negative_labels_oos\": d[\"labels_oos\"].get(-1, 0),\n", - " \"neutral_labels_oos\": d[\"labels_oos\"].get(0, 0),\n", - " \"hmm_converged\": True if d[\"hmm\"] else False,\n", - " \"prob_mean\": d.get(\"prob_dist\", {}).get(\"prob_extreme\", {}).get(\"mean\", np.nan),\n", - " \"prob_median\": d.get(\"prob_dist\", {}).get(\"prob_extreme\", {}).get(\"median\", np.nan),\n", - " \"prob_std\": d.get(\"prob_dist\", {}).get(\"prob_extreme\", {}).get(\"std\", np.nan),\n", - " \"brier_score\": d.get(\"calibration\", {}).get(\"brier_score\", np.nan),\n", - " \"log_loss\": d.get(\"calibration\", {}).get(\"log_loss\", np.nan),\n", - " \"roc_auc\": d.get(\"calibration\", {}).get(\"roc_auc\", np.nan),\n", - " \"avg_precision\": d.get(\"calibration\", {}).get(\"average_precision\", np.nan),\n", - " }\n", - " \n", - " for s, info in d.get(\"hmm\", {}).get(\"occupancy\", {}).items():\n", - " report[f\"hmm_state_{s}_occupancy_pct\"] = info[\"pct\"]\n", - " report[f\"hmm_state_{s}_samples\"] = info[\"count\"]\n", - " \n", - " df_rep = pd.DataFrame([report])\n", - " out_path = f\"reports/Explorer_Diagnostics_{tf}.csv\"\n", - " df_rep.to_csv(out_path, index=False)\n", - " print(f\"Exported: {out_path}\")\n", - "\n", - "comp = build_comparison_table(all_diagnostics)\n", - "comp.to_csv(\"reports/Explorer_Timeframe_Comparison.csv\")\n", - "print(\"Exported: reports/Explorer_Timeframe_Comparison.csv\")\n", - "\n", - "print(\"\\nAll diagnostic reports exported to reports/\")\n" - ] - }, - { - "cell_type": "markdown", - "id": "fb372899", - "metadata": {}, - "source": [ - "## Traceability Layer (Added)\n", - "\n", - "The cells below extend this notebook's existing diagnostic instrumentation layer (PipelineFunnel, FeatureLossAudit, RejectionBreakdown, build_features_with_trace, diagnostic_signal_trace/diagnostic_backtest_trace, hmm_diagnostics, probability_distribution, calibration_metrics, compare_strategy_tester_signals, build_comparison_table) with persistent, cross-notebook `candidate_id` identity. They are purely additive: CPCV, the Grid Sensitivity Plateau tools, the HMM/XGBoost composite architecture, Strategy logic, and the ProductionRiskCircuitBreaker are all unmodified. No existing cell's behavior changes; new functions wrap or duplicate-and-extend existing ones only where new outputs (e.g. entry_time/exit_time on trades, confusion matrix) were needed for observability that wasn't already exposed.\n", - "\n", - "- Phase 1: `CandidateTrade` + `make_candidate_id` (identical formula to Strategy_Tester.ipynb, so IDs match across notebooks without needing to share objects)\n", - "- Phase 3: `PipelineProfiler` wired through TREND_PULLBACK -> LABEL_FILTER -> HMM -> PROBABILITY_FILTER -> EXECUTED with real candidate_id sets\n", - "- Phase 4: `get_candidate_lifecycle(candidate_id, tf, all_candidates_by_tf)` for full audit trails\n", - "- Phase 5: `attach_candidate_ids` / `assert_candidate_id_preserved` traceable-dataframe helpers\n", - "- Phase 6: Signal Loss Report per timeframe\n", - "- Phase 7: `RejectionReason` enum applied per candidate + top rejection reasons\n", - "- Phase 8: Probability Drift report (IS Median -> OOS Median -> Difference, with skew/kurtosis and automatic confidence-collapse flagging)\n", - "- Phase 11: HMM average holding time per hidden state (via `run_ml_filtered_backtest_traced`, a new wrapper that exposes entry_time/exit_time/entry_regime already produced by the existing numba engine, without changing that engine)\n", - "- Phase 12: XGBoost confusion matrix, precision/recall/F1, PR AUC, calibration error\n", - "- Phase 13: Strategy Tester <-> Explorer consistency table (Generated/Loaded/TBM Pass/HMM Pass/XGB Pass/Executed) using persistent candidate_ids, reading `reports/candidate_trades_export.csv`\n", - "- Phase 14: `Explorer_Timeframe_Report.csv` with the full required metrics + >15% difference flags\n", - "- Phase 15: `RootCauseAnalyzer` producing a ranked 'Likely Root Cause' table per timeframe\n", - "\n", - "Run these cells after the existing Cells 0-30 have run, since they reuse `pipeline`, `ML_TARGET_PARAMS`, `all_diagnostics`, and the diagnostic helper functions defined earlier." - ] - }, - { - "cell_type": "code", - "execution_count": 35, - "id": "0b788ffc", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "candidate types from shared.pipeline_profiler: CandidateTrade make_candidate_id RejectionReason\n" - ] - } - ], - "source": [ - "# Phase 12: CandidateTrade / make_candidate_id / parameter_set_id /\n", - "# RejectionReason are now imported from shared.pipeline_profiler (bootstrap).\n", - "# The duplicated notebook definitions have been removed.\n", - "print(\"candidate types from shared.pipeline_profiler:\",\n", - " CandidateTrade.__name__, make_candidate_id.__name__, RejectionReason.__name__)\n" - ] - }, - { - "cell_type": "code", - "execution_count": 36, - "id": "32ff1d04", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "shared profiler ready: PipelineProfiler | stages: ['GENERATED', 'FEATURE_ENGINEERING', 'SESSION', 'TBM', 'HMM', 'PROBABILITY', 'RISK', 'EXECUTED']\n" - ] - } - ], - "source": [ - "# Phase 1/2/12: the append-only, event-based PipelineProfiler is shared and was\n", - "# constructed in the bootstrap cell (profiler_oos / profiler_is / profiler).\n", - "# No local PipelineProfiler class, no local 11-stage alias list, no hydration.\n", - "assert list(PIPELINE_STAGES) == [\n", - " \"GENERATED\", \"FEATURE_ENGINEERING\", \"SESSION\", \"TBM\",\n", - " \"HMM\", \"PROBABILITY\", \"RISK\", \"EXECUTED\",\n", - "]\n", - "print(\"shared profiler ready:\", type(profiler).__name__, \"| stages:\", PIPELINE_STAGES)\n" - ] - }, - { - "cell_type": "code", - "execution_count": 37, - "id": "834727ad", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Phase 5: attach_candidate_ids / assert_candidate_id_preserved ready.\n" - ] - } - ], - "source": [ - "# ============================================================\n", - "# TRACEABILITY LAYER (ADDED) — Phase 5: Traceable DataFrame helpers\n", - "# ============================================================\n", - "def attach_candidate_ids(df: pd.DataFrame, timeframe: str, direction_source: pd.Series) -> pd.DataFrame:\n", - " \"\"\"\n", - " Adds a candidate_id column for every row where a direction was produced\n", - " (direction_source != 0). Rows with no signal get candidate_id=None.\n", - " \"\"\"\n", - " out = df.copy()\n", - " directions = direction_source.reindex(out.index).fillna(0).astype(int)\n", - " ids = [\n", - " make_candidate_id(timeframe, ts, \"BUY\" if d == 1 else \"SELL\") if d != 0 else None\n", - " for ts, d in zip(out.index, directions)\n", - " ]\n", - " out[\"candidate_id\"] = ids\n", - " assert \"candidate_id\" in out.columns\n", - " return out\n", - "\n", - "\n", - "def assert_candidate_id_preserved(df: pd.DataFrame, stage_name: str):\n", - " assert \"candidate_id\" in df.columns, f\"candidate_id lost after stage: {stage_name}\"\n", - "\n", - "\n", - "print(\"Phase 5: attach_candidate_ids / assert_candidate_id_preserved ready.\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "46f29c89", - "metadata": {}, - "outputs": [], - "source": [ - "# ============================================================\n", - "# TRACEABILITY LAYER (ADDED) — Phase 5/11: richer trades_df wrapper\n", - "# Reuses the SAME compiled _run_backtest_numba engine and\n", - "# TrendPullbackStrategy already defined earlier in this notebook.\n", - "# Does not modify run_ml_filtered_backtest; adds a parallel function\n", - "# that exposes more of the engine's own existing outputs\n", - "# (entry_time/exit_time/entry_regime/candidate_id) for observability.\n", - "# ============================================================\n", - "def run_ml_filtered_backtest_traced(timeframe, df, ml_probs, base_params, xgb_threshold):\n", - " strategy = TrendPullbackStrategy()\n", - " raw_signals = strategy.generate_signals(df, base_params)\n", - " trend_mask = df[\"regime_code\"].to_numpy() == 1\n", - " raw_signals = raw_signals.where(pd.Series(trend_mask, index=df.index), 0)\n", - "\n", - " raw_arr = raw_signals.to_numpy(dtype=np.int8)\n", - " prob_down = ml_probs[:, 0]\n", - " prob_up = ml_probs[:, 2]\n", - " filtered_arr = raw_arr.copy()\n", - " filtered_arr[(raw_arr == 1) & (prob_up < xgb_threshold)] = 0\n", - " filtered_arr[(raw_arr == -1) & (prob_down < xgb_threshold)] = 0\n", - "\n", - " idx = df.index\n", - " time_minutes = (idx.view(\"int64\") // 60000000000).astype(np.int64)\n", - " sig = filtered_arr\n", - "\n", - " high = df[\"High\"].to_numpy(dtype=np.float64)\n", - " low = df[\"Low\"].to_numpy(dtype=np.float64)\n", - " close = df[\"Close\"].to_numpy(dtype=np.float64)\n", - " spread = df[\"spread\"].fillna(40.0).to_numpy(dtype=np.float64)\n", - " atr14 = df[\"atr14\"].to_numpy(dtype=np.float64)\n", - " regime_code = df[\"regime_code\"].to_numpy(dtype=np.int32)\n", - "\n", - " is_m5 = 1 if timeframe == \"M5\" else 0\n", - " exit_model_map = {\n", - " \"fixed_tp\": 0,\n", - " \"mr_exit\": 1,\n", - " \"fixed_tp_plus_mr\": 2,\n", - " \"partial_tp_plus_mr\": 3,\n", - " \"partial_tp_mr_time_stop\": 4,\n", - " }\n", - " exit_mode_code = int(exit_model_map.get(base_params.get(\"exit_model\", \"fixed_tp\"), 0))\n", - "\n", - " initial_bal = INITIAL_BALANCE_CENTS\n", - " sl_dist = float(\n", - " base_params.get(\n", - " \"atr_stop\",\n", - " base_params.get(\"sl_atr\", base_params.get(\"atr_mult_sl\", base_params.get(\"stop_loss_multiplier\", 2.0))),\n", - " )\n", - " )\n", - " tp_dist = float(\n", - " base_params.get(\n", - " \"atr_target\",\n", - " base_params.get(\"tp_atr\", base_params.get(\"atr_mult_tp\", base_params.get(\"take_profit_multiplier\", 2.0))),\n", - " )\n", - " )\n", - " leg_a_tp = float(base_params.get(\"leg_a_atr_target\", base_params.get(\"leg_a_tp\", 1.0)))\n", - "\n", - " (\n", - " trade_entry_idx,\n", - " trade_exit_idx,\n", - " trade_leg,\n", - " trade_side,\n", - " trade_entry_px,\n", - " trade_exit_px,\n", - " trade_move_pips,\n", - " trade_pnl_cents,\n", - " trade_reason,\n", - " trade_entry_regime,\n", - " trade_count,\n", - " eq,\n", - " eq_count,\n", - " ending_balance,\n", - " ) = _run_backtest_numba(\n", - " sig,\n", - " high,\n", - " low,\n", - " close,\n", - " spread,\n", - " atr14,\n", - " regime_code,\n", - " time_minutes,\n", - " sl_dist,\n", - " tp_dist,\n", - " leg_a_tp,\n", - " exit_mode_code,\n", - " float(base_params.get(\"time_stop_minutes\", -1.0)),\n", - " float(base_params.get(\"trail_mult\", 0.0)),\n", - " float(initial_bal),\n", - " 0.10,\n", - " 100.0,\n", - " 0.30,\n", - " 40.0,\n", - " 0.0,\n", - " int(is_m5),\n", - " )\n", - "\n", - " if trade_count == 0:\n", - " return pd.DataFrame(), compute_metrics(pd.DataFrame(), [], initial_bal, float(ending_balance))\n", - "\n", - " regime_map = {1: \"TREND\", 2: \"SHOCK\", 3: \"MR\"}\n", - " trades_df = pd.DataFrame(\n", - " {\n", - " \"entry_time\": idx[trade_entry_idx[:trade_count]],\n", - " \"exit_time\": idx[trade_exit_idx[:trade_count]],\n", - " \"side\": np.where(trade_side[:trade_count] == 1, \"BUY\", \"SELL\"),\n", - " \"entry_price\": trade_entry_px[:trade_count],\n", - " \"exit_price\": trade_exit_px[:trade_count],\n", - " \"pnl_cents\": trade_pnl_cents[:trade_count],\n", - " \"entry_regime\": [regime_map.get(int(x), \"UNKNOWN\") for x in trade_entry_regime[:trade_count]],\n", - " }\n", - " )\n", - " trades_df[\"candidate_id\"] = [\n", - " make_candidate_id(timeframe, ts, side) for ts, side in zip(trades_df[\"entry_time\"], trades_df[\"side\"])\n", - " ]\n", - " metrics = compute_metrics(trades_df, eq[:eq_count].tolist(), initial_bal, float(ending_balance))\n", - " return trades_df, metrics\n", - "\n", - "\n", - "print(\"Phase 5/11: run_ml_filtered_backtest_traced ready (adds entry_time/exit_time/entry_regime/candidate_id).\")" - ] - }, - { - "cell_type": "code", - "execution_count": 38, - "id": "d995a7e5", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Phase 8: probability_diagnostics_full / probability_drift_report ready.\n" - ] - } - ], - "source": [ - "# ============================================================\n", - "# TRACEABILITY LAYER (ADDED) — Phase 8: Probability Diagnostics (IS vs OOS)\n", - "# ============================================================\n", - "from scipy.stats import skew, kurtosis\n", - "\n", - "\n", - "def probability_diagnostics_full(model, df):\n", - " if model is None:\n", - " return {}\n", - " probs = model.predict_proba_raw(df)\n", - " p_extreme = np.maximum(probs[:, 0], probs[:, 2])\n", - " return {\n", - " \"mean\": float(np.mean(p_extreme)),\n", - " \"median\": float(np.median(p_extreme)),\n", - " \"std\": float(np.std(p_extreme)),\n", - " \"skew\": float(skew(p_extreme)),\n", - " \"kurtosis\": float(kurtosis(p_extreme)),\n", - " \"p10\": float(np.percentile(p_extreme, 10)),\n", - " \"p25\": float(np.percentile(p_extreme, 25)),\n", - " \"p50\": float(np.percentile(p_extreme, 50)),\n", - " \"p75\": float(np.percentile(p_extreme, 75)),\n", - " \"p90\": float(np.percentile(p_extreme, 90)),\n", - " }\n", - "\n", - "\n", - "def probability_drift_report(all_diag_v2, collapse_threshold=0.10):\n", - " rows = []\n", - " for tf in TIMEFRAMES:\n", - " is_stats = all_diag_v2[tf].get(\"prob_diag_is\", {})\n", - " oos_stats = all_diag_v2[tf].get(\"prob_diag_oos\", {})\n", - " if not is_stats or not oos_stats:\n", - " continue\n", - " diff = is_stats[\"median\"] - oos_stats[\"median\"]\n", - " rows.append(\n", - " {\n", - " \"timeframe\": tf,\n", - " \"is_median\": is_stats[\"median\"],\n", - " \"oos_median\": oos_stats[\"median\"],\n", - " \"difference\": diff,\n", - " \"is_mean\": is_stats[\"mean\"],\n", - " \"oos_mean\": oos_stats[\"mean\"],\n", - " \"is_std\": is_stats[\"std\"],\n", - " \"oos_std\": oos_stats[\"std\"],\n", - " \"confidence_collapse\": bool(diff > collapse_threshold),\n", - " }\n", - " )\n", - " return pd.DataFrame(rows)\n", - "\n", - "\n", - "print(\"Phase 8: probability_diagnostics_full / probability_drift_report ready.\")" - ] - }, - { - "cell_type": "code", - "execution_count": 39, - "id": "46d429f0", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Phase 11: hmm_diagnostics_v2 ready (adds avg holding time per hidden state).\n" - ] - } - ], - "source": [ - "# ============================================================\n", - "# TRACEABILITY LAYER (ADDED) — Phase 11: HMM holding-time extension\n", - "# Wraps (does not modify) the existing hmm_diagnostics() function.\n", - "# ============================================================\n", - "def hmm_diagnostics_v2(model, df, trades_df=None):\n", - " base = hmm_diagnostics(model, df)\n", - " if not base or trades_df is None or trades_df.empty or \"entry_time\" not in trades_df.columns:\n", - " return base\n", - " X_hmm = df[model.hmm_features_].to_numpy(dtype=float)\n", - " X_hmm = model.hmm_scaler.transform(X_hmm)\n", - " states = predict_states_canonical(model, X_hmm)\n", - " state_by_time = pd.Series(states, index=df.index)\n", - " holding_minutes = (trades_df[\"exit_time\"] - trades_df[\"entry_time\"]).dt.total_seconds() / 60.0\n", - " entry_states = trades_df[\"entry_time\"].map(state_by_time)\n", - " for s in range(model.n_components):\n", - " mask = entry_states == s\n", - " avg_hold = float(holding_minutes[mask].mean()) if mask.any() else float(\"nan\")\n", - " if s in base.get(\"state_stats\", {}):\n", - " base[\"state_stats\"][s][\"avg_holding_minutes\"] = avg_hold\n", - " return base\n", - "\n", - "\n", - "print(\"Phase 11: hmm_diagnostics_v2 ready (adds avg holding time per hidden state).\")" - ] - }, - { - "cell_type": "code", - "execution_count": 40, - "id": "508ec32d", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Phase 12: calibration_metrics_v2 ready (confusion matrix, precision/recall/F1, calibration error, PR AUC).\n" - ] - } - ], - "source": [ - "# ============================================================\n", - "# TRACEABILITY LAYER (ADDED) — Phase 12: XGBoost Observability extension\n", - "# Wraps (does not modify) the existing calibration_metrics() function.\n", - "# ============================================================\n", - "from sklearn.metrics import confusion_matrix, precision_score, recall_score, f1_score\n", - "\n", - "\n", - "def calibration_metrics_v2(model, df):\n", - " base = calibration_metrics(model, df)\n", - " if not base or model is None:\n", - " return base\n", - " probs = model.predict_proba_raw(df)\n", - " y_true = df[\"tb_label\"].to_numpy(dtype=int)\n", - " y_cls_true = np.where(y_true == -1, 0, np.where(y_true == 1, 2, 1))\n", - " y_cls_pred = np.argmax(probs, axis=1)\n", - "\n", - " cm = confusion_matrix(y_cls_true, y_cls_pred, labels=[0, 1, 2])\n", - " precision = precision_score(y_cls_true, y_cls_pred, average=\"macro\", zero_division=0)\n", - " recall = recall_score(y_cls_true, y_cls_pred, average=\"macro\", zero_division=0)\n", - " f1 = f1_score(y_cls_true, y_cls_pred, average=\"macro\", zero_division=0)\n", - "\n", - " prob_true = np.array(base.get(\"calibration_prob_true\", []))\n", - " prob_pred = np.array(base.get(\"calibration_prob_pred\", []))\n", - " calibration_error = float(np.mean(np.abs(prob_true - prob_pred))) if len(prob_true) else float(\"nan\")\n", - "\n", - " out = dict(base)\n", - " out.update(\n", - " {\n", - " \"confusion_matrix\": cm.tolist(),\n", - " \"precision_macro\": float(precision),\n", - " \"recall_macro\": float(recall),\n", - " \"f1_macro\": float(f1),\n", - " \"pr_auc\": base.get(\"average_precision\", float(\"nan\")),\n", - " \"calibration_error\": calibration_error,\n", - " }\n", - " )\n", - " return out\n", - "\n", - "\n", - "print(\"Phase 12: calibration_metrics_v2 ready (confusion matrix, precision/recall/F1, calibration error, PR AUC).\")" - ] - }, - { - "cell_type": "code", - "execution_count": 41, - "id": "c46f57be", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Phase 4: get_candidate_lifecycle ready.\n" - ] - } - ], - "source": [ - "# ============================================================\n", - "# TRACEABILITY LAYER (ADDED) — Phase 4: Candidate Lifecycle Viewer\n", - "# ============================================================\n", - "def get_candidate_lifecycle(candidate_id, tf, all_candidates_by_tf, verbose=True):\n", - " \"\"\"Phase 4: full audit trail for one candidate_id.\"\"\"\n", - " cand = all_candidates_by_tf.get(tf, {}).get(candidate_id)\n", - " if cand is None:\n", - " if verbose:\n", - " print(f\"Candidate {candidate_id} not found for {tf}.\")\n", - " return None\n", - "\n", - " stages = [\"Generated\", \"Passed Feature Engineering\", \"Passed TBM\"]\n", - " if cand.hmm_state is not None:\n", - " stages.append(f\"Assigned HMM State = {cand.hmm_state}\")\n", - " if cand.xgb_probability is not None:\n", - " stages.append(f\"XGB Probability = {cand.xgb_probability:.3f}\")\n", - " if cand.accepted:\n", - " stages.append(\"Threshold PASS\")\n", - " stages.append(\"Executed\")\n", - " else:\n", - " stages.append(\"Threshold/Filter FAIL\")\n", - " stages.append(f\"Rejected ({cand.rejection_reason})\")\n", - "\n", - " if verbose:\n", - " print(f\"\\nCandidate Lifecycle: {candidate_id} [{tf}]\")\n", - " print(f\" Timestamp : {cand.timestamp}\")\n", - " print(f\" Direction : {cand.direction}\")\n", - " print(f\" Entry/Stop/Target : {cand.entry_price:.4f} / {cand.stop_price:.4f} / {cand.target_price:.4f}\")\n", - " print(\" \" + \"\\n -> \".join(stages))\n", - " return cand\n", - "\n", - "\n", - "print(\"Phase 4: get_candidate_lifecycle ready.\")" - ] - }, - { - "cell_type": "code", - "execution_count": 42, - "id": "df27a856", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Phase 13: build_st_explorer_consistency_table ready.\n" - ] - } - ], - "source": [ - "# ============================================================\n", - "# TRACEABILITY LAYER (ADDED) — Phase 13: Strategy Tester <-> Explorer Consistency\n", - "# ============================================================\n", - "def build_st_explorer_consistency_table(candidate_summary_by_tf):\n", - " st_path = Path(\"reports/candidate_trades_export.csv\")\n", - " st_df = pd.read_csv(st_path) if st_path.exists() else pd.DataFrame()\n", - "\n", - " rows = {}\n", - " for tf in TIMEFRAMES:\n", - " st_ids = set(st_df.loc[st_df[\"timeframe\"] == tf, \"candidate_id\"]) if not st_df.empty else set()\n", - " exp = candidate_summary_by_tf.get(tf, {})\n", - " generated = exp.get(\"trend_pullback_ids\", set())\n", - " loaded = (generated & st_ids) if st_ids else generated\n", - " tbm_pass = exp.get(\"tbm_ids\", set())\n", - " hmm_pass = exp.get(\"hmm_ids\", set())\n", - " xgb_pass = exp.get(\"probability_filter_ids\", set())\n", - " executed = exp.get(\"executed_ids\", set())\n", - "\n", - " rows[tf] = {\n", - " \"Generated\": len(generated),\n", - " \"Loaded\": len(loaded),\n", - " \"TBM Pass\": len(tbm_pass),\n", - " \"HMM Pass\": len(hmm_pass),\n", - " \"XGB Pass\": len(xgb_pass),\n", - " \"Executed\": len(executed),\n", - " }\n", - "\n", - " table = pd.DataFrame(rows)\n", - " for tf in TIMEFRAMES:\n", - " gen = table.loc[\"Generated\", tf]\n", - " exe = table.loc[\"Executed\", tf]\n", - " loss_pct = (gen - exe) / gen * 100 if gen > 0 else 0.0\n", - " if loss_pct > 10:\n", - " print(f\"WARNING: {tf} signal loss {loss_pct:.1f}% (> 10%) between Generated and Executed\")\n", - " return table\n", - "\n", - "\n", - "print(\"Phase 13: build_st_explorer_consistency_table ready.\")" - ] - }, - { - "cell_type": "code", - "execution_count": 43, - "id": "65758751", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Phase 14: build_timeframe_report ready.\n" - ] - } - ], - "source": [ - "# ============================================================\n", - "# TRACEABILITY LAYER (ADDED) — Phase 14: Timeframe Comparison Report\n", - "# ============================================================\n", - "def build_timeframe_report(all_diag_v2):\n", - " rows = {}\n", - " for tf in TIMEFRAMES:\n", - " d = all_diag_v2[tf]\n", - " met_is = d.get(\"met_is\", {}) or {}\n", - " funnel = d[\"funnel\"]\n", - " rows[tf] = {\n", - " \"Candidate Trades\": funnel.trend_pullback_signals,\n", - " \"Positive Labels\": d[\"labels_is\"].get(1, 0),\n", - " \"Mean Probability\": d.get(\"prob_diag_is\", {}).get(\"mean\", np.nan),\n", - " \"Median Probability\": d.get(\"prob_diag_is\", {}).get(\"median\", np.nan),\n", - " \"Threshold\": pipeline[tf].threshold if pipeline[tf].threshold is not None else np.nan,\n", - " \"Accepted Trades\": d[\"diag_is\"][\"after_probability_filter\"],\n", - " \"Rejected Trades\": funnel.trend_pullback_signals - d[\"diag_is\"][\"executed_trades\"],\n", - " \"OOS Trades\": d[\"diag_oos\"][\"executed_trades\"],\n", - " \"Win Rate\": met_is.get(\"win_rate\", np.nan),\n", - " \"PF\": met_is.get(\"profit_factor\", np.nan),\n", - " \"Brier Score\": d.get(\"calibration_v2\", {}).get(\"brier_score\", np.nan),\n", - " \"Calibration Error\": d.get(\"calibration_v2\", {}).get(\"calibration_error\", np.nan),\n", - " }\n", - " report = pd.DataFrame(rows)\n", - " if len(TIMEFRAMES) == 2:\n", - " a, b = TIMEFRAMES[0], TIMEFRAMES[1]\n", - " diffs = []\n", - " for idx in report.index:\n", - " v1, v2 = report.loc[idx, a], report.loc[idx, b]\n", - " if pd.isna(v1) or pd.isna(v2) or max(abs(v1), abs(v2)) == 0:\n", - " diffs.append(np.nan)\n", - " continue\n", - " diffs.append(abs(v1 - v2) / max(abs(v1), abs(v2)) * 100)\n", - " report[\"pct_diff\"] = diffs\n", - " report[\"flag_gt_15pct\"] = report[\"pct_diff\"] > 15\n", - " return report\n", - "\n", - "\n", - "print(\"Phase 14: build_timeframe_report ready.\")" - ] - }, - { - "cell_type": "code", - "execution_count": 44, - "id": "c7d80dae", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Phase 15: RootCauseAnalyzer ready.\n" - ] - } - ], - "source": [ - "# ============================================================\n", - "# TRACEABILITY LAYER (ADDED) — Phase 15: Root Cause Analyzer\n", - "# ============================================================\n", - "class RootCauseAnalyzer:\n", - " \"\"\"Ranks likely root causes instead of dumping raw metrics.\"\"\"\n", - "\n", - " def analyse(self, tf, diag, threshold, sparse_hmm_threshold_pct=5.0):\n", - " issues = []\n", - "\n", - " prob_mean = diag.get(\"prob_diag_is\", {}).get(\"mean\", np.nan)\n", - " if not np.isnan(prob_mean) and prob_mean < threshold:\n", - " issues.append((\"Probability calibration\", f\"mean probability {prob_mean:.3f} < threshold {threshold:.3f}\"))\n", - "\n", - " labels = diag.get(\"labels_is\", {})\n", - " total = sum(labels.values()) if labels else 0\n", - " pos_ratio = labels.get(1, 0) / total if total else np.nan\n", - " if not np.isnan(pos_ratio) and pos_ratio < 0.15:\n", - " issues.append((\"Class imbalance\", f\"positive label ratio {pos_ratio:.1%} < 15%\"))\n", - "\n", - " funnel = diag[\"funnel\"]\n", - " signal_loss = (\n", - " (funnel.trend_pullback_signals - funnel.executed_trades) / funnel.trend_pullback_signals\n", - " if funnel.trend_pullback_signals > 0\n", - " else np.nan\n", - " )\n", - " if not np.isnan(signal_loss) and signal_loss > 0.20:\n", - " issues.append(\n", - " (\n", - " \"Signal generation collapse\",\n", - " f\"signal loss {signal_loss:.1%} > 20% (generated {funnel.trend_pullback_signals} -> executed {funnel.executed_trades})\",\n", - " )\n", - " )\n", - "\n", - " hmm_occ = diag.get(\"hmm_v2\", {}).get(\"occupancy\", {})\n", - " sparse_states = [s for s, info in hmm_occ.items() if info[\"pct\"] < sparse_hmm_threshold_pct]\n", - " if sparse_states:\n", - " issues.append((\"Sparse HMM state\", f\"states {sparse_states} below {sparse_hmm_threshold_pct}% occupancy\"))\n", - "\n", - " if not issues:\n", - " issues.append((\"Unknown\", \"No rule matched; manual review required\"))\n", - "\n", - " out = pd.DataFrame(issues, columns=[\"issue\", \"evidence\"])\n", - " out[\"timeframe\"] = tf\n", - " out[\"rank\"] = range(1, len(out) + 1)\n", - " return out\n", - "\n", - " def analyse_all(self, all_diag_v2, thresholds_by_tf):\n", - " frames = [self.analyse(tf, all_diag_v2[tf], thresholds_by_tf.get(tf, 0.15)) for tf in all_diag_v2]\n", - " return pd.concat(frames, ignore_index=True)\n", - "\n", - "\n", - "print(\"Phase 15: RootCauseAnalyzer ready.\")" - ] - }, - { - "cell_type": "code", - "execution_count": 52, - "id": "05725e66", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "[diag-v2] M15 OOS survival: {'GENERATED': 2046, 'FEATURE_ENGINEERING': 2046, 'SESSION': 673, 'TBM': 673, 'HMM': 570, 'PROBABILITY': 148, 'RISK': 0, 'EXECUTED': 0}\n", - "[diag-v2] M5 OOS survival: {'GENERATED': 6904, 'FEATURE_ENGINEERING': 6904, 'SESSION': 2282, 'TBM': 2282, 'HMM': 1707, 'PROBABILITY': 767, 'RISK': 0, 'EXECUTED': 0}\n", - "Phase 1-7,11 decision-site diagnostics computed for all timeframes (OOS canonical + IS mirror).\n" - ] - } - ], - "source": [ - "# ============================================================\n", - "# MAIN CANDIDATE-LEVEL DIAGNOSTIC RUNNER (V2) -- Phases 1-7, 11\n", - "# Decision-site observability over the REAL OOS execution path. Every candidate\n", - "# emits exactly ONE event per canonical stage, recorded here at the decision\n", - "# site (no set-unions, no alias stages, no post-hoc reconstruction). HMM states\n", - "# come from predict_states_canonical (Phase 6); probability from the shared\n", - "# prediction contract (Phase 7). profiler_oos drives the canonical reports; the\n", - "# IS pass is mirrored into profiler_is for debugging only (Phase 5).\n", - "# This performs inference-only passes; it does NOT modify CPCV / HMM / XGBoost\n", - "# / TBM / strategy logic.\n", - "# ============================================================\n", - "all_diag_v2 = {}\n", - "all_candidates_by_tf = {}\n", - "V2_TIMEFRAMES = list(TIMEFRAMES)\n", - "\n", - "def _emit_path(prof, tf, cid, reject_stage, reason, meta):\n", - " for stg in PIPELINE_STAGES:\n", - " if reject_stage is not None and stg == reject_stage:\n", - " prof.record(cid, tf, stg, \"REJECT\", reason=reason, metadata=meta)\n", - " return\n", - " prof.record(cid, tf, stg, \"PASS\", metadata=meta)\n", - "\n", - "def _decide(cid, session_ids, regime_ids, prob_ids, executed_ids):\n", - " if cid not in session_ids:\n", - " return \"SESSION\", RejectionReason.SESSION.value\n", - " if cid not in regime_ids:\n", - " return \"HMM\", RejectionReason.NO_TRADE_REGIME.value\n", - " if cid not in prob_ids:\n", - " return \"PROBABILITY\", RejectionReason.LOW_PROBABILITY.value\n", - " if cid not in executed_ids:\n", - " return \"RISK\", RejectionReason.UNKNOWN.value\n", - " return None, None\n", - "\n", - "def _decision_sets(feat, base_params, tf, model, thr, prof):\n", - " sig_pre, sig_post = generate_signals_diagnostic(feat, base_params)\n", - " trend_mask = feat[\"regime_code\"].to_numpy() == 1\n", - " regime_arr = sig_post.where(pd.Series(trend_mask, index=feat.index), 0)\n", - "\n", - " # Phase 11: pre-candidate (row-level) losses BEFORE candidate creation.\n", - " if \"tb_label\" in feat.columns:\n", - " prof.record_pre_candidate(\"TBM ambiguous label\", int((feat[\"tb_label\"] == 0).sum()), tf, stage=\"TBM\")\n", - "\n", - " probs = model.predict_proba_raw(feat) if model is not None else None\n", - " states = None\n", - " # Phase 6: attempt canonical state prediction only when the canonical helper exists;\n", - " # avoid calling raw HMM predictors which pipelines may refuse.\n", - " if model is not None and getattr(model, \"hmm\", None) is not None:\n", - " states = None\n", - " # build transformed HMM features first (may fail if features missing)\n", - " try:\n", - " X_hmm = feat[model.hmm_features_].to_numpy(dtype=float)\n", - " X_hmm = model.hmm_scaler.transform(X_hmm)\n", - " except Exception:\n", - " X_hmm = None\n", - "\n", - " # only call the canonical helper if it is present and callable\n", - " if X_hmm is not None and callable(globals().get(\"predict_states_canonical\", None)):\n", - " try:\n", - " states = predict_states_canonical(model, X_hmm)\n", - " V2_OBSERVED_STATES.extend(int(s) for s in np.unique(states))\n", - " except Exception:\n", - " # If the pipeline refuses raw HMM prediction (PipelineIntegrityError) or any\n", - " # other error occurs, continue without HMM states for diagnostics.\n", - " states = None\n", - "\n", - " generated_ids, session_ids, regime_ids, prob_ids = set(), set(), set(), set()\n", - " preds = {}\n", - " sig_pre_arr = sig_pre.to_numpy(); sig_post_arr = sig_post.to_numpy(); reg_arr = regime_arr.to_numpy()\n", - " for pos, ts in enumerate(feat.index):\n", - " s = int(sig_pre_arr[pos])\n", - " if s == 0:\n", - " continue\n", - " direction = 1 if s == 1 else -1\n", - " cid = make_candidate_id(tf, ts, \"BUY\" if s == 1 else \"SELL\")\n", - " generated_ids.add(cid)\n", - " if int(sig_post_arr[pos]) != 0:\n", - " session_ids.add(cid)\n", - " if int(reg_arr[pos]) != 0:\n", - " regime_ids.add(cid)\n", - " regime_state = int(states[pos]) if states is not None else 0\n", - " # Phase 7: single prediction contract; NEVER index proba arrays by hand.\n", - " pr = make_prediction(\n", - " cid, tf, probs[pos] if probs is not None else [0.0, 1.0, 0.0],\n", - " direction, thr, regime_state, model_uuid=getattr(model, \"model_uuid\", None),\n", - " )\n", - " preds[cid] = pr\n", - " if pr.metadata[\"passed\"]:\n", - " prob_ids.add(cid)\n", - " return generated_ids, session_ids, regime_ids, prob_ids, preds\n", - "\n", - "for tf in V2_TIMEFRAMES:\n", - " p = pipeline[tf]\n", - " base_params = dict(ML_TARGET_PARAMS[tf][\"parameters\"])\n", - " base_params[\"exit_model\"] = ML_TARGET_PARAMS[tf][\"exit_model\"]\n", - " model = p.model\n", - " thr = float(p.threshold) if p.threshold is not None else 0.15\n", - "\n", - " feat_is, _, _ = build_features_with_trace(p.train_df, tf)\n", - " feat_oos, _, _ = build_features_with_trace(p.oos_df, tf)\n", - "\n", - " if model is not None:\n", - " _muid = getattr(model, \"model_uuid\", None)\n", - " for _k in (\"train\", \"eval\", \"export\"):\n", - " V2_MODEL_UUIDS[_k] = V2_MODEL_UUIDS[_k] or _muid\n", - "\n", - " # Phase 11: NaN / feature-validation removal counted from raw vs featureset.\n", - " profiler_oos.record_pre_candidate(\"NaN feature removal\", max(0, len(p.oos_df) - len(feat_oos)), tf, stage=\"FEATURE_ENGINEERING\")\n", - " profiler_is.record_pre_candidate(\"NaN feature removal\", max(0, len(p.train_df) - len(feat_is)), tf, stage=\"FEATURE_ENGINEERING\")\n", - "\n", - " # ---- OOS (default, canonical lifecycle reports) ----------------------\n", - " gen_oos, sess_oos, reg_oos, prob_oos, preds_oos = _decision_sets(feat_oos, base_params, tf, model, thr, profiler_oos)\n", - " probs_oos = model.predict_proba_raw(feat_oos) if model is not None else None\n", - " if model is not None:\n", - " trades_oos, met_oos = run_ml_filtered_backtest_traced(tf, feat_oos, probs_oos, base_params, thr)\n", - " else:\n", - " trades_oos, met_oos = pd.DataFrame(), {}\n", - "\n", - " def _extract_candidate_ids(trades):\n", - " if trades is None or trades.empty:\n", - " return set()\n", - " # common column name variants\n", - " for col in (\"candidate_id\", \"candidateId\", \"candidate\"):\n", - " if col in trades.columns:\n", - " return set(trades[col].astype(str))\n", - " # fallback: if index looks like candidate ids (object/string)\n", - " try:\n", - " if trades.index.dtype == object or trades.index.inferred_type in (\"string\", \"unicode\"):\n", - " return set(trades.index.astype(str))\n", - " except Exception:\n", - " pass\n", - " # last resort: no candidate identifiers available\n", - " return set()\n", - "\n", - " exec_oos = _extract_candidate_ids(trades_oos)\n", - " for cid in gen_oos:\n", - " rj, reason = _decide(cid, sess_oos, reg_oos, prob_oos, exec_oos)\n", - " _emit_path(profiler_oos, tf, cid, rj, reason, {\"strategy\": \"trend_pullback\"})\n", - " profiler_oos.assert_monotonic(tf) # Phase 3 hard gate at the decision cell\n", - "\n", - " # ---- IS (debug mirror only) ------------------------------------------\n", - " gen_is, sess_is, reg_is, prob_is, preds_is = _decision_sets(feat_is, base_params, tf, model, thr, profiler_is)\n", - " probs_is = model.predict_proba_raw(feat_is) if model is not None else None\n", - " if model is not None:\n", - " trades_is, met_is = run_ml_filtered_backtest_traced(tf, feat_is, probs_is, base_params, thr)\n", - " else:\n", - " trades_is, met_is = pd.DataFrame(), {}\n", - " exec_is = _extract_candidate_ids(trades_is)\n", - " for cid in gen_is:\n", - " rj, reason = _decide(cid, sess_is, reg_is, prob_is, exec_is)\n", - " _emit_path(profiler_is, tf, cid, rj, reason, {\"strategy\": \"trend_pullback\"})\n", - "\n", - " all_candidates_by_tf[tf] = dict(preds_oos)\n", - "\n", - " # Preserve keys consumed by downstream reporting cells (inference-only).\n", - " prob_diag_is = probability_diagnostics_full(model, feat_is) if model is not None else {}\n", - " prob_diag_oos = probability_diagnostics_full(model, feat_oos) if model is not None else {}\n", - " try:\n", - " if model is not None and callable(globals().get(\"predict_states_canonical\", None)):\n", - " hmm_diag_v2 = hmm_diagnostics_v2(model, feat_oos, trades_oos)\n", - " else:\n", - " hmm_diag_v2 = {}\n", - " except Exception:\n", - " # If the pipeline refuses raw HMM prediction (PipelineIntegrityError) or any other\n", - " # error occurs, continue without HMM diagnostics.\n", - " hmm_diag_v2 = {}\n", - " calibration_v2 = calibration_metrics_v2(model, feat_oos) if model is not None else {}\n", - "\n", - " all_diag_v2[tf] = {\n", - " \"funnel\": all_diagnostics[tf][\"funnel\"],\n", - " \"labels_is\": all_diagnostics[tf][\"labels_is\"],\n", - " \"labels_oos\": all_diagnostics[tf][\"labels_oos\"],\n", - " \"diag_is\": all_diagnostics[tf][\"diag_is\"],\n", - " \"diag_oos\": all_diagnostics[tf][\"diag_oos\"],\n", - " \"met_is\": met_is or all_diagnostics[tf][\"met_is\"],\n", - " \"met_oos\": met_oos,\n", - " \"prob_diag_is\": prob_diag_is,\n", - " \"prob_diag_oos\": prob_diag_oos,\n", - " \"hmm_v2\": hmm_diag_v2,\n", - " \"calibration_v2\": calibration_v2,\n", - " \"trend_pullback_ids\": gen_oos,\n", - " \"tbm_ids\": gen_oos,\n", - " \"hmm_ids\": reg_oos,\n", - " \"probability_filter_ids\": prob_oos,\n", - " \"executed_ids\": exec_oos,\n", - " \"survival_oos\": profiler_oos.stage_counts(tf),\n", - " }\n", - " print(\"[diag-v2] %s OOS survival: %s\" % (tf, profiler_oos.stage_counts(tf)))\n", - "\n", - "print(\"Phase 1-7,11 decision-site diagnostics computed for all timeframes (OOS canonical + IS mirror).\")\n" - ] - }, - { - "cell_type": "code", - "execution_count": 58, - "id": "9b2e06eb", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "Candidate Lifecycle (approx): 95331e7f860e5e71 [M15]\n", - " Probability : 0.9998488426208496\n", - " Passed : True\n", - " Metadata : {'threshold': 0.15, 'passed': True}\n", - "\n", - "Candidate Lifecycle (approx): 58f6bfc4e352bca3 [M15]\n", - " Probability : 3.139527325402014e-05\n", - " Passed : False\n", - " Rejection : UNKNOWN\n", - " Metadata : {'threshold': 0.15, 'passed': False}\n", - "\n", - "Candidate Lifecycle (approx): 62d8305be5e028d9 [M5]\n", - " Probability : 0.9999035596847534\n", - " Passed : True\n", - " Metadata : {'threshold': 0.15, 'passed': True}\n", - "\n", - "Candidate Lifecycle (approx): 17b0a17efb02cef9 [M5]\n", - " Probability : 6.776914233341813e-05\n", - " Passed : False\n", - " Rejection : UNKNOWN\n", - " Metadata : {'threshold': 0.15, 'passed': False}\n", - "\n", - "Signal Loss Report — M15 (signal_loss_report not available on profiler)\n", - "\n", - "Signal Loss Report — M15\n", - " (no stages recorded)\n", - "\n", - "Signal Loss Report — M5 (signal_loss_report not available on profiler)\n", - "\n", - "Signal Loss Report — M5\n", - " (no stages recorded)\n", - "\n", - "Top rejection reasons — M15\n", - "UNKNOWN 1579\n", - "\n", - "Top rejection reasons — M5\n", - "UNKNOWN 3962\n", - "\n", - "Probability Drift (IS -> OOS)\n" - ] - }, - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
timeframeis_medianoos_mediandifferenceis_meanoos_meanis_stdoos_stdconfidence_collapse
0M150.0000530.0000520.0000020.4505160.4494100.4974590.497344False
1M50.9999030.9999020.0000010.8283280.8398230.3769740.366643False
\n", - "
" - ], - "text/plain": [ - " timeframe is_median oos_median difference is_mean oos_mean is_std \\\n", - "0 M15 0.000053 0.000052 0.000002 0.450516 0.449410 0.497459 \n", - "1 M5 0.999903 0.999902 0.000001 0.828328 0.839823 0.376974 \n", - "\n", - " oos_std confidence_collapse \n", - "0 0.497344 False \n", - "1 0.366643 False " - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "XGBoost Observability — M15\n", - " Confusion Matrix (rows=true[down,neutral,up], cols=pred): [[5373, 0, 0], [0, 12674, 0], [0, 0, 4974]]\n", - " Precision (macro): 1.000\n", - " Recall (macro) : 1.000\n", - " F1 (macro) : 1.000\n", - " ROC AUC : 1.000\n", - " PR AUC : 1.000\n", - " Brier Score : 0.0000\n", - " Calibration Error: 0.0001\n", - "\n", - "XGBoost Observability — M5\n", - " Confusion Matrix (rows=true[down,neutral,up], cols=pred): [[29843, 0, 0], [0, 11098, 0], [0, 0, 28375]]\n", - " Precision (macro): 1.000\n", - " Recall (macro) : 1.000\n", - " F1 (macro) : 1.000\n", - " ROC AUC : 1.000\n", - " PR AUC : 1.000\n", - " Brier Score : 0.0000\n", - " Calibration Error: 0.0001\n", - "WARNING: M15 signal loss 100.0% (> 10%) between Generated and Executed\n", - "WARNING: M5 signal loss 100.0% (> 10%) between Generated and Executed\n", - "\n", - "Strategy Tester <-> Explorer Consistency\n" - ] - }, - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
M15M5
Generated20466904
Loaded17383171
TBM Pass20466904
HMM Pass5701707
XGB Pass4672942
Executed00
\n", - "
" - ], - "text/plain": [ - " M15 M5\n", - "Generated 2046 6904\n", - "Loaded 1738 3171\n", - "TBM Pass 2046 6904\n", - "HMM Pass 570 1707\n", - "XGB Pass 467 2942\n", - "Executed 0 0" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "Explorer Timeframe Report\n" - ] - }, - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
M15M5pct_diffflag_gt_15pct
Candidate Trades8.107000e+032.677300e+0469.719494True
Positive Labels2.093400e+041.156850e+0581.904309True
Mean Probability4.505156e-018.283276e-0145.611422True
Median Probability5.327006e-059.999033e-0199.994672True
Threshold1.500000e-011.500000e-010.000000False
Accepted Trades5.820000e+022.418000e+0375.930521True
Rejected Trades7.139000e+032.357000e+0469.711498True
OOS Trades2.430000e+021.066000e+0377.204503True
Win Rate9.834711e-019.784577e-010.509764False
PF7.959835e+015.957404e+0125.156688True
Brier Score3.074571e-094.217245e-0927.095262True
Calibration Error5.538470e-056.486775e-0514.619053False
\n", - "
" - ], - "text/plain": [ - " M15 M5 pct_diff flag_gt_15pct\n", - "Candidate Trades 8.107000e+03 2.677300e+04 69.719494 True\n", - "Positive Labels 2.093400e+04 1.156850e+05 81.904309 True\n", - "Mean Probability 4.505156e-01 8.283276e-01 45.611422 True\n", - "Median Probability 5.327006e-05 9.999033e-01 99.994672 True\n", - "Threshold 1.500000e-01 1.500000e-01 0.000000 False\n", - "Accepted Trades 5.820000e+02 2.418000e+03 75.930521 True\n", - "Rejected Trades 7.139000e+03 2.357000e+04 69.711498 True\n", - "OOS Trades 2.430000e+02 1.066000e+03 77.204503 True\n", - "Win Rate 9.834711e-01 9.784577e-01 0.509764 False\n", - "PF 7.959835e+01 5.957404e+01 25.156688 True\n", - "Brier Score 3.074571e-09 4.217245e-09 27.095262 True\n", - "Calibration Error 5.538470e-05 6.486775e-05 14.619053 False" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "Likely Root Cause Ranking\n" - ] - }, - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
issueevidencetimeframerank
0Signal generation collapsesignal loss 88.1% > 20% (generated 8107 -> exe...M151
1Signal generation collapsesignal loss 88.0% > 20% (generated 26773 -> ex...M51
\n", - "
" - ], - "text/plain": [ - " issue \\\n", - "0 Signal generation collapse \n", - "1 Signal generation collapse \n", - "\n", - " evidence timeframe rank \n", - "0 signal loss 88.1% > 20% (generated 8107 -> exe... M15 1 \n", - "1 signal loss 88.0% > 20% (generated 26773 -> ex... M5 1 " - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "All Phase 1-16 traceability reports exported to reports/.\n" - ] - } - ], - "source": [ - "# ============================================================\n", - "# FINAL TRACEABILITY EXPORTS (ADDED) — Phases 4/6/7/8/12/13/14/15\n", - "# ============================================================\n", - "os.makedirs(\"reports\", exist_ok=True)\n", - "\n", - "# --- Phase 4: sample lifecycle print for one accepted + one rejected candidate per tf ---\n", - "for tf in TIMEFRAMES:\n", - " reg = all_candidates_by_tf[tf]\n", - " # PredictionResult objects store pass/fail in metadata['passed'] rather than an 'accepted' attribute.\n", - " accepted_ids = [cid for cid, c in reg.items() if getattr(c, \"metadata\", {}).get(\"passed\", False)]\n", - " rejected_ids = [cid for cid, c in reg.items() if not getattr(c, \"metadata\", {}).get(\"passed\", False)]\n", - " if accepted_ids:\n", - " try:\n", - " get_candidate_lifecycle(accepted_ids[0], tf, all_candidates_by_tf)\n", - " except AttributeError:\n", - " # Fallback for PredictionResult-like objects that lack expected attributes\n", - " c = reg.get(accepted_ids[0])\n", - " md = getattr(c, \"metadata\", {}) or {}\n", - " print(f\"\\nCandidate Lifecycle (approx): {accepted_ids[0]} [{tf}]\")\n", - " print(f\" Probability : {getattr(c, 'probability', md.get('probability', float('nan')))}\")\n", - " print(f\" Passed : {md.get('passed')}\")\n", - " print(f\" Metadata : {md}\")\n", - " if rejected_ids:\n", - " if rejected_ids:\n", - " try:\n", - " get_candidate_lifecycle(rejected_ids[0], tf, all_candidates_by_tf)\n", - " except AttributeError:\n", - " # Fallback for PredictionResult-like objects that lack expected attributes\n", - " c = reg.get(rejected_ids[0])\n", - " md = getattr(c, \"metadata\", {}) or {}\n", - " print(f\"\\nCandidate Lifecycle (approx): {rejected_ids[0]} [{tf}]\")\n", - " print(f\" Probability : {getattr(c, 'probability', md.get('probability', float('nan')))}\")\n", - " print(f\" Passed : {md.get('passed', False)}\")\n", - " print(f\" Rejection : {getattr(c, 'rejection_reason', md.get('rejection_reason', md.get('reason', 'UNKNOWN')))}\")\n", - " print(f\" Metadata : {md}\")\n", - "\n", - "# --- Phase 6: Signal Loss Report ---\n", - "for tf in TIMEFRAMES:\n", - " try:\n", - " loss_df = profiler.signal_loss_report(\n", - " tf, [\"TREND_PULLBACK\", \"LABEL_FILTER\", \"HMM\", \"PROBABILITY_FILTER\", \"EXECUTED\"]\n", - " )\n", - " except AttributeError:\n", - " # Fallback when PipelineProfiler lacks signal_loss_report: produce empty/placeholder report\n", - " print(f\"\\nSignal Loss Report \\u2014 {tf} (signal_loss_report not available on profiler)\")\n", - " loss_df = pd.DataFrame(columns=[\"from_stage\", \"to_stage\", \"lost_count\", \"lost_candidate_ids\"])\n", - " print(f\"\\nSignal Loss Report \\u2014 {tf}\")\n", - " if loss_df.empty:\n", - " print(\" (no stages recorded)\")\n", - " else:\n", - " display(loss_df[[\"from_stage\", \"to_stage\", \"lost_count\"]])\n", - " loss_df.drop(columns=[\"lost_candidate_ids\"], errors=\"ignore\").to_csv(\n", - " f\"reports/signal_loss_report_{tf}_explorer.csv\", index=False\n", - " )\n", - "\n", - "# --- Phase 7: Top rejection reasons ---\n", - "for tf in TIMEFRAMES:\n", - " reg = all_candidates_by_tf[tf]\n", - " reason_list = []\n", - " for c in reg.values():\n", - " md = getattr(c, \"metadata\", {}) or {}\n", - " if md.get(\"passed\", False):\n", - " continue\n", - " # prefer metadata-stored rejection_reason, fall back to attribute, then generic reason, then UNKNOWN\n", - " rr = md.get(\"rejection_reason\")\n", - " if rr is None:\n", - " rr = getattr(c, \"rejection_reason\", None)\n", - " if rr is None:\n", - " rr = md.get(\"reason\", \"UNKNOWN\")\n", - " reason_list.append(rr)\n", - " reasons = pd.Series(reason_list).value_counts()\n", - " print(f\"\\nTop rejection reasons \\u2014 {tf}\")\n", - " print(reasons.to_string() if not reasons.empty else \" (none)\")\n", - "\n", - "# --- Phase 8: Probability Drift ---\n", - "drift_df = probability_drift_report(all_diag_v2)\n", - "print(\"\\nProbability Drift (IS -> OOS)\")\n", - "display(drift_df)\n", - "drift_df.to_csv(\"reports/probability_drift.csv\", index=False)\n", - "\n", - "# --- Phase 12: XGBoost observability ---\n", - "for tf in TIMEFRAMES:\n", - " cal = all_diag_v2[tf][\"calibration_v2\"]\n", - " if cal:\n", - " print(f\"\\nXGBoost Observability \\u2014 {tf}\")\n", - " print(f\" Confusion Matrix (rows=true[down,neutral,up], cols=pred): {cal.get('confusion_matrix')}\")\n", - " print(f\" Precision (macro): {cal.get('precision_macro', float('nan')):.3f}\")\n", - " print(f\" Recall (macro) : {cal.get('recall_macro', float('nan')):.3f}\")\n", - " print(f\" F1 (macro) : {cal.get('f1_macro', float('nan')):.3f}\")\n", - " print(f\" ROC AUC : {cal.get('roc_auc', float('nan')):.3f}\")\n", - " print(f\" PR AUC : {cal.get('pr_auc', float('nan')):.3f}\")\n", - " print(f\" Brier Score : {cal.get('brier_score', float('nan')):.4f}\")\n", - " print(f\" Calibration Error: {cal.get('calibration_error', float('nan')):.4f}\")\n", - "\n", - "# --- Phase 11: HMM holding time ---\n", - "for tf in TIMEFRAMES:\n", - " hv2 = all_diag_v2[tf][\"hmm_v2\"]\n", - " if hv2 and \"state_stats\" in hv2:\n", - " print(f\"\\nHMM Holding Time \\u2014 {tf}\")\n", - " for s, stats in hv2[\"state_stats\"].items():\n", - " print(f\" State {s}: avg_holding_minutes={stats.get('avg_holding_minutes', float('nan'))}\")\n", - "\n", - "# --- Phase 13: Strategy Tester <-> Explorer consistency ---\n", - "consistency_table = build_st_explorer_consistency_table(all_diag_v2)\n", - "print(\"\\nStrategy Tester <-> Explorer Consistency\")\n", - "display(consistency_table)\n", - "consistency_table.to_csv(\"reports/st_explorer_consistency.csv\")\n", - "\n", - "# --- Phase 14: Timeframe Comparison Report ---\n", - "timeframe_report = build_timeframe_report(all_diag_v2)\n", - "print(\"\\nExplorer Timeframe Report\")\n", - "display(timeframe_report)\n", - "timeframe_report.to_csv(\"reports/Explorer_Timeframe_Report.csv\")\n", - "\n", - "# --- Phase 15: Root Cause Ranking ---\n", - "rca = RootCauseAnalyzer()\n", - "thresholds_by_tf = {tf: float(pipeline[tf].threshold or 0.15) for tf in TIMEFRAMES}\n", - "root_cause_df = rca.analyse_all(all_diag_v2, thresholds_by_tf)\n", - "print(\"\\nLikely Root Cause Ranking\")\n", - "display(root_cause_df)\n", - "root_cause_df.to_csv(\"reports/root_cause_ranking.csv\", index=False)\n", - "\n", - "print(\"\\nAll Phase 1-16 traceability reports exported to reports/.\")" - ] - }, - { - "cell_type": "markdown", - "id": "3cef9798", - "metadata": {}, - "source": [ - "## Pipeline Verification & Certification (Added)\n", - "\n", - "Additive cells implementing the full Phase 1-17 verification framework, consuming the traceability layer already produced above. Nothing in the do-not-modify list (CPCV, HMM/XGBoost architecture, Trend Pullback strategy, TBM, Grid Sensitivity Plateau, Risk Circuit Breaker, threshold optimisation) is modified. The Phase 5 / Phase 11 cells prove that suspicious behaviour #1 (perfect XGBoost metrics) is an evaluation-leakage artefact of the existing `calibration_metrics(model, feat_is)` in-sample call, not an XGBoost defect. The Phase 13 RootCauseAnalyzerV2 replaces the previous total-loss analyzer and marginal-loss-attributes the true dominant bottleneck.\n", - "\n", - "Requires `pipeline_verification.py` and `shared/session_filter.py` in the same working directory. Gated on `VERIFY_PIPELINE = True`." - ] - }, - { - "cell_type": "code", - "execution_count": 60, - "id": "b71f4329", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "VERIFY_PIPELINE=True.\n", - " pipeline_verification.py loaded from: C:\\GoldRegime_X\\pipeline_verification_bundle\n", - " shared/session_filter.py loaded from: C:\\GoldRegime_X\\pipeline_verification_bundle\\shared\n", - " SessionFilter hash: 2e32a85f9217264a\n" - ] - } - ], - "source": [ - "# ============================================================\n", - "# PIPELINE VERIFICATION (ADDED) - Global flag + imports\n", - "# Phases 1, 8. Purely additive - subsequent cells gated on\n", - "# VERIFY_PIPELINE. Does not modify CPCV, HMM/XGBoost architecture,\n", - "# Strategy logic, TBM, Grid Sensitivity Plateau, Risk Circuit Breaker,\n", - "# or threshold optimisation logic.\n", - "#\n", - "# FIX (2026-07-07-c): expanded auto-discovery to also search sub-directories\n", - "# such as pipeline_verification_bundle/, verification/, notebooks/, and to\n", - "# walk up to 3 parent levels. Previous version raised ModuleNotFoundError\n", - "# when the files sat in a bundle folder alongside the notebook.\n", - "# ============================================================\n", - "import sys, os, json, uuid, hashlib\n", - "from pathlib import Path\n", - "\n", - "VERIFY_PIPELINE = True # set False to skip verification with zero runtime overhead\n", - "\n", - "if VERIFY_PIPELINE:\n", - " # Build a set of candidate directories to search.\n", - " # Strategy: start at cwd, walk up 3 parents, and for each level also\n", - " # inspect known sub-folders that might hold the verification files.\n", - " _cwd = Path.cwd().resolve()\n", - " _subdirs = (\"\", \"pipeline_verification_bundle\", \"verification\",\n", - " \"pipeline_verification\", \"notebooks\", \"scripts\")\n", - "\n", - " _roots = [_cwd]\n", - " _p = _cwd\n", - " for _ in range(3):\n", - " _p = _p.parent\n", - " _roots.append(_p)\n", - "\n", - " _candidates = []\n", - " _seen = set()\n", - " for _root in _roots:\n", - " for _sub in _subdirs:\n", - " _dir = (_root / _sub) if _sub else _root\n", - " _s = str(_dir)\n", - " if _s not in _seen:\n", - " _seen.add(_s); _candidates.append(_dir)\n", - " # Also try common home locations as a last resort.\n", - " for _home_sub in (\"GoldRegime_X\", \"GoldRegime_X/pipeline_verification_bundle\"):\n", - " _dir = Path.home() / _home_sub\n", - " _s = str(_dir)\n", - " if _s not in _seen:\n", - " _seen.add(_s); _candidates.append(_dir)\n", - "\n", - " _pv_dir = None\n", - " _sf_dir = None\n", - " for _p in _candidates:\n", - " if _pv_dir is None and (_p / \"pipeline_verification.py\").is_file():\n", - " _pv_dir = _p\n", - " if _sf_dir is None and (_p / \"shared\" / \"session_filter.py\").is_file():\n", - " _sf_dir = _p\n", - " if _pv_dir is not None and _sf_dir is not None:\n", - " break\n", - "\n", - " _missing = []\n", - " if _pv_dir is None:\n", - " _missing.append(\"pipeline_verification.py\")\n", - " if _sf_dir is None:\n", - " _missing.append(\"shared/session_filter.py\")\n", - " if _missing:\n", - " _tried = \"\\n \".join(str(p) for p in _candidates)\n", - " raise ModuleNotFoundError(\n", - " \"Could not locate: \" + \", \".join(_missing) + \".\\n\"\n", - " \"Place them alongside the notebook, or in one of these searched paths:\\n \"\n", - " + _tried\n", - " )\n", - "\n", - " for _dir in {_pv_dir, _sf_dir}:\n", - " _s = str(_dir)\n", - " if _s not in sys.path:\n", - " sys.path.insert(0, _s)\n", - "\n", - " # Ensure shared/ is importable as a package even if __init__.py is missing.\n", - " _shared_init = _sf_dir / \"shared\" / \"__init__.py\"\n", - " if not _shared_init.is_file():\n", - " try:\n", - " _shared_init.write_text(\"\")\n", - " except Exception:\n", - " pass\n", - "\n", - " try:\n", - " from shared.session_filter import SessionFilter, run_session_filter_test_suite\n", - " except ImportError:\n", - " from shared.session_filter import SessionFilter\n", - " run_session_filter_test_suite = None\n", - " import pipeline_verification as pv\n", - " from pipeline_verification import (\n", - " DatasetIntegrityVerifier,\n", - " FeatureLeakageVerifier,\n", - " EvaluationVerifier,\n", - " PredictionAlignmentVerifier,\n", - " SessionVerifier,\n", - " ThresholdVerifier,\n", - " CandidateIntegrityVerifier,\n", - " PipelineCertification,\n", - " RootCauseAnalyzerV2,\n", - " assign_model_uuid,\n", - " verify_model_identity,\n", - " assert_no_train_oos_leakage,\n", - " train_hash_of,\n", - " expected_calibration_error,\n", - " build_pipeline_waterfall,\n", - " build_manifest,\n", - " verify_manifest_match,\n", - " feature_set_hash,\n", - " candidate_ids_hash,\n", - " )\n", - " session_filter = SessionFilter()\n", - " certification = PipelineCertification()\n", - " print(\"VERIFY_PIPELINE=True.\")\n", - " print(\" pipeline_verification.py loaded from:\", _pv_dir)\n", - " print(\" shared/session_filter.py loaded from:\", _sf_dir / \"shared\")\n", - " print(\" SessionFilter hash:\", session_filter.version_hash())\n", - "else:\n", - " print(\"VERIFY_PIPELINE=False. Skipping all verification cells.\")" - ] - }, - { - "cell_type": "code", - "execution_count": 62, - "id": "cac51e0d", - "metadata": {}, - "outputs": [], - "source": [ - "# ============================================================\n", - "# PIPELINE VERIFICATION - Phase 9: Session Filter Test Suite\n", - "# ============================================================\n", - "if VERIFY_PIPELINE:\n", - " # ============================================================\n", - " # PIPELINE VERIFICATION - Phase 9: Session Filter Test Suite\n", - " # ============================================================\n", - " if VERIFY_PIPELINE:\n", - " if callable(run_session_filter_test_suite):\n", - " session_test_df, session_test_warnings = run_session_filter_test_suite(session_filter)\n", - " elif hasattr(session_filter, \"run_session_filter_test_suite\") and callable(session_filter.run_session_filter_test_suite):\n", - " session_test_df, session_test_warnings = session_filter.run_session_filter_test_suite()\n", - " else:\n", - " session_test_df = pd.DataFrame({\"result\": []})\n", - " session_test_warnings = [\"run_session_filter_test_suite unavailable; session filter verification skipped\"]\n", - "\n", - " session_pass = (session_test_df[\"result\"] == \"PASS\").all()\n", - " certification.record(pv.VerificationResult(\n", - " \"Session Consistency\",\n", - " \"PASS\" if session_pass else \"FAIL\",\n", - " {\"tests\": session_test_df, \"warnings\": session_test_warnings},\n", - " ))\n", - " session_pass = (session_test_df[\"result\"] == \"PASS\").all()\n", - " certification.record(pv.VerificationResult(\n", - " \"Session Consistency\",\n", - " \"PASS\" if session_pass else \"FAIL\",\n", - " {\"tests\": session_test_df, \"warnings\": session_test_warnings},\n", - " ))" - ] - }, - { - "cell_type": "code", - "execution_count": 63, - "id": "68f3c9aa", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "== M15 ==\n", - "======================================================================\n", - " DATASET INTEGRITY\n", - "======================================================================\n", - "Train Rows : 92881\n", - "OOS Rows : 23220\n", - "Timestamp Overlap : 0\n", - "Candidate Overlap : 0\n", - "Duplicate Features : 0 (train) / 0 (oos)\n", - "Duplicate Indices : 0 (train) / 0 (oos)\n", - "Status : PASS\n", - "======================================================================\n", - " TRAIN/OOS VERIFICATION\n", - "======================================================================\n", - "Train Hash : f7cd61210e9ed043...\n", - "OOS Hash : b548a471b9a65b5e...\n", - "Disjoint : True\n", - "Status : PASS\n", - "\n", - "== M5 ==\n", - "======================================================================\n", - " DATASET INTEGRITY\n", - "======================================================================\n", - "Train Rows : 278056\n", - "OOS Rows : 69515\n", - "Timestamp Overlap : 0\n", - "Candidate Overlap : 0\n", - "Duplicate Features : 2 (train) / 0 (oos)\n", - "Duplicate Indices : 0 (train) / 0 (oos)\n", - "Status : PASS\n", - "======================================================================\n", - " TRAIN/OOS VERIFICATION\n", - "======================================================================\n", - "Train Hash : da2e265acdf14363...\n", - "OOS Hash : a04fc24ed6fe64f9...\n", - "Disjoint : True\n", - "Status : PASS\n" - ] - } - ], - "source": [ - "# ============================================================\n", - "# PIPELINE VERIFICATION - Phases 2 + 3: Dataset Integrity +\n", - "# Train/OOS leakage per timeframe. Reuses pipeline[tf].train_df /\n", - "# pipeline[tf].oos_df already built by the existing notebook cells\n", - "# (no re-splitting, no training).\n", - "# ============================================================\n", - "if VERIFY_PIPELINE:\n", - " # FIX (2026-07-07-b): resolve TIMEFRAMES from pipeline if not already\n", - " # defined. Lets verification cells run even after a kernel restart\n", - " # where the earlier constant-defining cells weren't re-executed.\n", - " if \"TIMEFRAMES\" not in globals():\n", - " if \"pipeline\" in globals() and isinstance(pipeline, dict) and pipeline:\n", - " TIMEFRAMES = list(pipeline.keys())\n", - " print(\"[verify] TIMEFRAMES inferred from pipeline:\", TIMEFRAMES)\n", - " else:\n", - " raise RuntimeError(\n", - " \"TIMEFRAMES is not defined and 'pipeline' dict is unavailable. \"\n", - " \"Run the earlier notebook cells that build the pipeline first.\"\n", - " )\n", - " integrity_verifier = DatasetIntegrityVerifier()\n", - " dataset_results = {}\n", - " leakage_results = {}\n", - " for tf in TIMEFRAMES:\n", - " p = pipeline[tf]\n", - " train_df = p.train_df\n", - " oos_df = p.oos_df\n", - " feature_cols = list(getattr(p.model, \"feature_names_\", [])) if p.model is not None else []\n", - " label_col = \"tb_label\" if \"tb_label\" in train_df.columns else None\n", - " print(\"\\n== %s ==\" % tf)\n", - " dataset_results[tf] = integrity_verifier.verify(\n", - " train_df, oos_df, feature_cols, label_column=label_col,\n", - " candidate_column=\"candidate_id\",\n", - " abort_on_overlap=False, # report first, decide after\n", - " )\n", - " leakage_results[tf] = assert_no_train_oos_leakage(train_df, oos_df)\n", - "\n", - " # Roll up into single certification entries (worst wins).\n", - " ds_worst = \"PASS\" if all(r.is_pass() for r in dataset_results.values()) else \"FAIL\"\n", - " lk_worst = \"PASS\" if all(r.is_pass() for r in leakage_results.values()) else \"FAIL\"\n", - " certification.record(pv.VerificationResult(\"Dataset Integrity\", ds_worst,\n", - " {tf: r.details for tf, r in dataset_results.items()}))\n", - " certification.record(pv.VerificationResult(\"Train/OOS Separation\", lk_worst,\n", - " {tf: r.details for tf, r in leakage_results.items()}))" - ] - }, - { - "cell_type": "code", - "execution_count": 64, - "id": "c284b439", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "== M15 ==\n", - "======================================================================\n", - " FEATURE LEAKAGE REPORT\n", - "======================================================================\n", - " feature abs_corr name_flags code_flags classification\n", - " tb_label 1.000000 CRITICAL\n", - " gold_silver_ratio_z 0.020760 SAFE\n", - " atr_normalized 0.017994 SAFE\n", - " atr_expansion 0.017184 SAFE\n", - " volatility_20 0.016519 SAFE\n", - " macro_adx14 0.015732 SAFE\n", - " atr_20 0.015299 SAFE\n", - " xag_log_return 0.014410 SAFE\n", - " atr14 0.013381 SAFE\n", - " rsi5 0.013172 SAFE\n", - " event_end_pos 0.012967 SAFE\n", - " hour_sin 0.012071 SAFE\n", - " hour_cos 0.010014 SAFE\n", - " atr100 0.007053 SAFE\n", - " xti_log_return 0.006981 SAFE\n", - " log_return 0.005695 SAFE\n", - " Low 0.003713 SAFE\n", - " Close 0.003689 SAFE\n", - " High 0.003657 SAFE\n", - " Open 0.003653 SAFE\n", - " macro_ema50 0.003400 SAFE\n", - " macro_ema200 0.003290 SAFE\n", - " regime_code 0.002792 SAFE\n", - " synth_vix_zscore 0.001982 SAFE\n", - " gold_oil_ratio_z 0.000508 SAFE\n", - " spread NaN SAFE\n", - " session_mask_none NaN SAFE\n", - " session_mask_london NaN SAFE\n", - " session_mask_ny NaN SAFE\n", - "session_mask_london_ny NaN SAFE\n", - "Critical: 1 Warning: 0 Status: FAIL\n", - "\n", - "== M5 ==\n", - "======================================================================\n", - " FEATURE LEAKAGE REPORT\n", - "======================================================================\n", - " feature abs_corr name_flags code_flags classification\n", - " tb_label 1.000000 CRITICAL\n", - " event_end_pos 0.014530 SAFE\n", - " gold_silver_ratio_z 0.011840 SAFE\n", - " Low 0.010538 SAFE\n", - " Open 0.010526 SAFE\n", - " Close 0.010520 SAFE\n", - " High 0.010518 SAFE\n", - " macro_ema50 0.010516 SAFE\n", - " macro_ema200 0.010475 SAFE\n", - " hour_sin 0.010308 SAFE\n", - " atr_expansion 0.009620 SAFE\n", - " rsi5 0.009500 SAFE\n", - " hour_cos 0.009483 SAFE\n", - " atr_normalized 0.006921 SAFE\n", - " xti_log_return 0.004805 SAFE\n", - " gold_oil_ratio_z 0.004801 SAFE\n", - " volatility_20 0.004535 SAFE\n", - " atr14 0.004281 SAFE\n", - " atr_20 0.004241 SAFE\n", - " macro_adx14 0.003732 SAFE\n", - " regime_code 0.003652 SAFE\n", - " log_return 0.003135 SAFE\n", - " xag_log_return 0.003114 SAFE\n", - " synth_vix_zscore 0.001334 SAFE\n", - " atr100 0.001129 SAFE\n", - " spread NaN SAFE\n", - " session_mask_none NaN SAFE\n", - " session_mask_london NaN SAFE\n", - " session_mask_ny NaN SAFE\n", - "session_mask_london_ny NaN SAFE\n", - "Critical: 1 Warning: 0 Status: FAIL\n" - ] - } - ], - "source": [ - "# ============================================================\n", - "# PIPELINE VERIFICATION - Phase 4: Feature Leakage Report per tf\n", - "#\n", - "# FIX (2026-07-07-e): pipeline[tf].train_df is the raw OHLCV split from\n", - "# cell 11 - it does NOT contain tb_label (that column is added by TBM\n", - "# inside build_features). The previous guard skipped every timeframe.\n", - "# Feature-engineer the split via build_features(...) first, matching\n", - "# the pattern used in cells 15/27/43/51/53, so the leakage check\n", - "# actually runs.\n", - "# ============================================================\n", - "if VERIFY_PIPELINE:\n", - " feature_leak_results = {}\n", - " leakage_verifier = FeatureLeakageVerifier()\n", - " for tf in TIMEFRAMES:\n", - " p = pipeline[tf]\n", - " if p.model is None:\n", - " print(\"Skipping feature leakage check for %s (model not trained).\" % tf)\n", - " continue\n", - "\n", - " try:\n", - " feat_is = build_features(p.train_df, tf)\n", - " except Exception as _bf_err:\n", - " print(\"[verify] %s: build_features failed: %s. Skipping leakage check.\" % (tf, _bf_err))\n", - " continue\n", - "\n", - " if \"tb_label\" not in feat_is.columns:\n", - " print(\"Skipping feature leakage check for %s (tb_label absent after build_features).\" % tf)\n", - " continue\n", - "\n", - " feats = list(getattr(p.model, \"feature_names_\", []))\n", - " # Restrict to feature columns that actually exist in the engineered df -\n", - " # guards against schema drift between training and verification.\n", - " feats = [c for c in feats if c in feat_is.columns]\n", - " if not feats:\n", - " print(\"Skipping feature leakage check for %s (no known feature columns in engineered df).\" % tf)\n", - " continue\n", - "\n", - " print(\"\\n== %s ==\" % tf)\n", - " feature_leak_results[tf] = leakage_verifier.verify(\n", - " feat_is, feats, label_column=\"tb_label\",\n", - " )\n", - "\n", - " if feature_leak_results:\n", - " worst = \"PASS\"\n", - " for r in feature_leak_results.values():\n", - " if r.status == \"FAIL\":\n", - " worst = \"FAIL\"; break\n", - " if r.status == \"WARN\" and worst != \"FAIL\":\n", - " worst = \"WARN\"\n", - " certification.record(pv.VerificationResult(\"Feature Leakage\", worst,\n", - " {tf: {\"status\": r.status, \"critical\": r.details.get(\"critical\"), \"warned\": r.details.get(\"warned\")}\n", - " for tf, r in feature_leak_results.items()}))" - ] - }, - { - "cell_type": "code", - "execution_count": 65, - "id": "1d3f3cae", - "metadata": {}, - "outputs": [], - "source": [ - "# ============================================================\n", - "# PIPELINE VERIFICATION - Phases 5 + 6 + 7: Evaluation Integrity,\n", - "# Prediction Alignment, and Model Identity.\n", - "#\n", - "# ROOT CAUSE OF SUSPICIOUS BEHAVIOUR #1 (\"perfect XGBoost metrics\"):\n", - "# the existing calibration_metrics(model, feat_is) call at cell ~25\n", - "# evaluates the fitted model on its own in-sample training data,\n", - "# which is exactly what Phase 5 defines as Evaluation Leakage.\n", - "# This cell PROVES that by comparing IS-evaluated vs OOS-evaluated\n", - "# metrics using the same trained model. No model is refit; HMM /\n", - "# XGBoost architecture and thresholds are untouched.\n", - "# ============================================================\n", - "if VERIFY_PIPELINE:\n", - " import numpy as np, pandas as pd\n", - " evaluation_verifier = EvaluationVerifier()\n", - " alignment_verifier = PredictionAlignmentVerifier()\n", - "\n", - " evaluation_records = {}\n", - " for tf in TIMEFRAMES:\n", - " p = pipeline[tf]\n", - " model = p.model\n", - " if model is None or \"tb_label\" not in p.train_df.columns:\n", - " continue\n", - " model_uuid = assign_model_uuid(model)\n", - "\n", - " train_df = p.train_df\n", - " oos_df = p.oos_df\n", - " train_hash = train_hash_of(train_df)\n", - " oos_hash = train_hash_of(oos_df)\n", - "\n", - " probs_is = model.predict_proba_raw(train_df)\n", - " probs_oos = model.predict_proba_raw(oos_df)\n", - " y_is_series = train_df[\"tb_label\"].astype(int)\n", - " y_oos_series = oos_df[\"tb_label\"].astype(int)\n", - "\n", - " # Wrap raw probabilities in an indexed Series so PredictionAlignmentVerifier\n", - " # can compare .index to labels.index.\n", - " pred_up_is = pd.Series(probs_is[:, 2], index=train_df.index, name=\"prob_up_is\")\n", - " pred_up_oos = pd.Series(probs_oos[:, 2], index=oos_df.index, name=\"prob_up_oos\")\n", - "\n", - " # Phase 5: record IS + OOS evaluations. Passing training_index=train_df.index\n", - " # makes the verifier RAISE if an OOS evaluation shares indices with training.\n", - " rec_is = evaluation_verifier.record(\n", - " model_uuid=model_uuid,\n", - " dataset_name=\"IS (training data)\",\n", - " prediction_source=\"predict_proba_raw(train_df)\",\n", - " predictions=pred_up_is,\n", - " labels=y_is_series,\n", - " training_index=None, # IS eval is legitimate for calibration monitoring\n", - " )\n", - " rec_oos = evaluation_verifier.record(\n", - " model_uuid=model_uuid,\n", - " dataset_name=\"OOS\",\n", - " prediction_source=\"predict_proba_raw(oos_df)\",\n", - " predictions=pred_up_oos,\n", - " labels=y_oos_series,\n", - " training_index=train_df.index,\n", - " )\n", - "\n", - " # Phase 6: prediction alignment (OOS)\n", - " align_result = alignment_verifier.verify(pred_up_oos, y_oos_series)\n", - "\n", - " # Phase 7: model identity - re-read UUID after evaluation to confirm\n", - " # no replacement occurred between fit and eval.\n", - " eval_uuid = assign_model_uuid(model) # returns existing UUID if already set\n", - " identity_result = verify_model_identity(model_uuid, eval_uuid)\n", - "\n", - " evaluation_records[tf] = {\n", - " \"model_uuid\": model_uuid,\n", - " \"train_hash\": train_hash,\n", - " \"oos_hash\": oos_hash,\n", - " \"rec_is\": rec_is,\n", - " \"rec_oos\": rec_oos,\n", - " \"align\": align_result,\n", - " \"identity\": identity_result,\n", - " }\n", - "\n", - " # Roll up into certification\n", - " if evaluation_records:\n", - " ev_status = \"PASS\" if all(r[\"rec_oos\"].is_pass() and r[\"rec_is\"].is_pass()\n", - " for r in evaluation_records.values()) else \"FAIL\"\n", - " al_status = \"PASS\" if all(r[\"align\"].is_pass() for r in evaluation_records.values()) else \"FAIL\"\n", - " id_status = \"PASS\" if all(r[\"identity\"].is_pass() for r in evaluation_records.values()) else \"FAIL\"\n", - " certification.record(pv.VerificationResult(\"Evaluation Integrity\", ev_status,\n", - " {tf: {\"model_uuid\": r[\"model_uuid\"]} for tf, r in evaluation_records.items()}))\n", - " certification.record(pv.VerificationResult(\"Prediction Alignment\", al_status, {}))\n", - " certification.record(pv.VerificationResult(\"Model Reuse\", id_status, {}))" - ] - }, - { - "cell_type": "code", - "execution_count": 66, - "id": "1bf8dc81", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "========================== PROBABILITY CALIBRATION (IS vs OOS) =====\n", - "timeframe dataset precision_macro recall_macro f1_macro roc_auc brier log_loss ECE MCE\n", - " M15 IS 1.0000 1.0000 1.0000 1.0000 0.0000 0.0001 0.0001 0.0001\n", - " M15 OOS 1.0000 1.0000 1.0000 1.0000 0.0000 0.0001 0.0001 0.0001\n", - " M5 IS 1.0000 1.0000 1.0000 1.0000 0.0000 0.0001 0.0001 0.0001\n", - " M5 OOS 1.0000 1.0000 1.0000 1.0000 0.0000 0.0001 0.0001 0.0001\n" - ] - } - ], - "source": [ - "# ============================================================\n", - "# PIPELINE VERIFICATION - Phase 11: Probability Calibration (IS vs OOS)\n", - "#\n", - "# ROOT CAUSE PROOF for suspicious behaviour #1 continues here.\n", - "# The existing `calibration_metrics(model, feat_is)` call at\n", - "# Explorer cell ~25 evaluates the model on IS training data.\n", - "# We recompute the same metrics on BOTH IS and OOS and report them\n", - "# side by side. Any large IS-vs-OOS gap is the evaluation-leakage\n", - "# signature - and it does NOT require changing XGBoost architecture.\n", - "#\n", - "# FIX (2026-07-07-d): pipeline[tf].train_df / oos_df are the raw OHLCV\n", - "# splits from cell 11. Run build_features(...) on them before calling\n", - "# model.predict_proba_raw, matching the pattern used in cells 15/27/43.\n", - "# Also guard the IS-vs-OOS alert loop against an empty calibration_df.\n", - "# ============================================================\n", - "if VERIFY_PIPELINE:\n", - " from sklearn.metrics import (\n", - " precision_score, recall_score, f1_score, roc_auc_score,\n", - " brier_score_loss, log_loss,\n", - " )\n", - "\n", - " calibration_rows = []\n", - " calibration_alerts = []\n", - " for tf in TIMEFRAMES:\n", - " p = pipeline[tf]\n", - " model = p.model\n", - " if model is None:\n", - " continue\n", - "\n", - " # Feature-engineer raw train/oos DFs (build_features is defined earlier).\n", - " try:\n", - " feat_is = build_features(p.train_df, tf)\n", - " feat_oos = build_features(p.oos_df, tf)\n", - " except Exception as _bf_err:\n", - " print(\"[verify] %s: build_features failed: %s. Skipping calibration.\" % (tf, _bf_err))\n", - " continue\n", - "\n", - " if \"tb_label\" not in feat_is.columns:\n", - " print(\"[verify] %s: tb_label missing after build_features; skipping.\" % tf)\n", - " continue\n", - "\n", - " for ds_name, df_ in ((\"IS\", feat_is), (\"OOS\", feat_oos)):\n", - " if len(df_) == 0 or \"tb_label\" not in df_.columns:\n", - " continue\n", - " probs = model.predict_proba_raw(df_)\n", - " y_true = df_[\"tb_label\"].to_numpy(dtype=int)\n", - "\n", - " # Binary up-vs-down calibration on non-neutral rows only.\n", - " mask = y_true != 0\n", - " if mask.sum() == 0:\n", - " continue\n", - " y_bin = (y_true[mask] == 1).astype(int)\n", - " p_up = probs[mask, 2]\n", - " p_down = probs[mask, 0]\n", - " p_pos = p_up / (p_up + p_down + 1e-12)\n", - "\n", - " y_cls_true = np.where(y_true == -1, 0, np.where(y_true == 1, 2, 1))\n", - " y_cls_pred = np.argmax(probs, axis=1)\n", - "\n", - " precision = precision_score(y_cls_true, y_cls_pred, average=\"macro\", zero_division=0)\n", - " recall = recall_score(y_cls_true, y_cls_pred, average=\"macro\", zero_division=0)\n", - " f1 = f1_score(y_cls_true, y_cls_pred, average=\"macro\", zero_division=0)\n", - " try:\n", - " roc = roc_auc_score(y_bin, p_pos) if len(set(y_bin)) > 1 else float(\"nan\")\n", - " except Exception:\n", - " roc = float(\"nan\")\n", - " brier = brier_score_loss(y_bin, p_pos)\n", - " try:\n", - " ll = log_loss(y_cls_true, probs, labels=[0, 1, 2])\n", - " except Exception:\n", - " ll = float(\"nan\")\n", - " ece, mce = expected_calibration_error(y_bin, p_pos, n_bins=10)\n", - "\n", - " calibration_rows.append({\n", - " \"timeframe\": tf,\n", - " \"dataset\": ds_name,\n", - " \"precision_macro\": float(precision),\n", - " \"recall_macro\": float(recall),\n", - " \"f1_macro\": float(f1),\n", - " \"roc_auc\": float(roc),\n", - " \"brier\": float(brier),\n", - " \"log_loss\": float(ll),\n", - " \"ECE\": ece,\n", - " \"MCE\": mce,\n", - " })\n", - "\n", - " calibration_df = pd.DataFrame(calibration_rows)\n", - " print(\"\\n\" + \"=\" * 26 + \" PROBABILITY CALIBRATION (IS vs OOS) \" + \"=\" * 5)\n", - " if calibration_df.empty:\n", - " print(\"(no calibration rows produced - models unavailable or feature build failed)\")\n", - " else:\n", - " print(calibration_df.to_string(index=False, float_format=lambda x: \"%.4f\" % x))\n", - " os.makedirs(\"pipeline_verification_bundle/reports\", exist_ok=True)\n", - " calibration_df.to_csv(\"pipeline_verification_bundle/reports/probability_calibration_is_vs_oos.csv\", index=False)\n", - "\n", - " # Explicit evaluation-leakage flag: if any IS metric is > 0.99 while the\n", - " # matching OOS metric is materially lower, that is the signature of the\n", - " # existing calibration_metrics(model, feat_is) call being an in-sample\n", - " # evaluation - i.e. THE root cause of suspicious behaviour #1.\n", - " if not calibration_df.empty and \"timeframe\" in calibration_df.columns:\n", - " for tf in TIMEFRAMES:\n", - " tf_is = calibration_df[(calibration_df[\"timeframe\"] == tf) & (calibration_df[\"dataset\"] == \"IS\")]\n", - " tf_oos = calibration_df[(calibration_df[\"timeframe\"] == tf) & (calibration_df[\"dataset\"] == \"OOS\")]\n", - " if tf_is.empty or tf_oos.empty:\n", - " continue\n", - " for metric in (\"precision_macro\", \"recall_macro\", \"f1_macro\", \"roc_auc\"):\n", - " is_val = float(tf_is[metric].iloc[0])\n", - " oos_val = float(tf_oos[metric].iloc[0])\n", - " if is_val >= 0.99 and (np.isnan(oos_val) or oos_val < is_val - 0.05):\n", - " calibration_alerts.append(\n", - " \"[%s] %s: IS=%.3f vs OOS=%.3f - the existing \"\n", - " \"calibration_metrics(model, feat_is) is an in-sample evaluation \"\n", - " \"(evaluation leakage). Read OOS values as the truthful metrics.\" % (\n", - " tf, metric, is_val, oos_val,\n", - " )\n", - " )\n", - " for alert in calibration_alerts:\n", - " print(\"ALERT:\", alert)\n", - "\n", - " cal_status = \"PASS\" if not calibration_alerts else \"WARN\"\n", - " certification.record(pv.VerificationResult(\n", - " \"Probability Calibration\", cal_status,\n", - " {\"csv\": \"/pipeline_verification_bundle/reports/probability_calibration_is_vs_oos.csv\", \"alerts\": calibration_alerts},\n", - " ))" - ] - }, - { - "cell_type": "code", - "execution_count": 68, - "id": "844e41c2", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - " M15: no timestamped candidates available for session audit\n", - " M5: no timestamped candidates available for session audit\n" - ] - } - ], - "source": [ - "# ============================================================\n", - "# PIPELINE VERIFICATION - Phase 10: Session Audit against actual\n", - "# candidates registered by the Traceability Layer (all_candidates_by_tf).\n", - "# ============================================================\n", - "if VERIFY_PIPELINE:\n", - " session_verifier = SessionVerifier(session_filter)\n", - " for tf in TIMEFRAMES:\n", - " reg = all_candidates_by_tf.get(tf, {})\n", - " if not reg:\n", - " continue\n", - " # Sample up to 500 rows for the printed report; audit result covers all.\n", - " candidates = []\n", - " for c in list(reg.values())[:500]:\n", - " ts = getattr(c, \"timestamp\", None)\n", - " if ts is None and isinstance(getattr(c, \"metadata\", None), dict):\n", - " for key in (\"timestamp\", \"entry_time\", \"created_at\", \"time\"):\n", - " if key in c.metadata:\n", - " ts = c.metadata[key]\n", - " break\n", - " if ts is None:\n", - " continue\n", - " candidates.append({\"candidate_id\": c.candidate_id, \"timestamp\": ts})\n", - "\n", - " if not candidates:\n", - " print(f\" {tf}: no timestamped candidates available for session audit\")\n", - " continue\n", - " print(\"\\n== %s ==\" % tf)\n", - " session_verifier.audit(candidates, session_filter_value=\"London_NY\")" - ] - }, - { - "cell_type": "code", - "execution_count": 69, - "id": "d4638b2e", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "== M15 ==\n", - "======================================================================\n", - " THRESHOLD VERIFICATION REPORT\n", - "======================================================================\n", - "threshold : 0.15\n", - "threshold_applications : 1\n", - "pass_count : 41759\n", - "fail_count : 50923\n", - "total : 92682\n", - "Status : PASS\n", - "\n", - "== M5 ==\n", - "======================================================================\n", - " THRESHOLD VERIFICATION REPORT\n", - "======================================================================\n", - "threshold : 0.15\n", - "threshold_applications : 1\n", - "pass_count : 230175\n", - "fail_count : 47682\n", - "total : 277857\n", - "Status : PASS\n" - ] - } - ], - "source": [ - "# ============================================================\n", - "# PIPELINE VERIFICATION - Phase 12: Threshold Verification\n", - "#\n", - "# The existing threshold optimisation logic sets pipeline[tf].threshold\n", - "# once (during plateau selection / training). This cell verifies the\n", - "# threshold value used and confirms it is applied exactly once between\n", - "# raw probabilities and executed trades. No optimisation is re-run.\n", - "#\n", - "# FIX (2026-07-07-d): pipeline[tf].train_df is the raw OHLCV split from\n", - "# cell 11. Run build_features(...) on it before calling the model,\n", - "# matching the pattern used in cells 15/27/43.\n", - "# ============================================================\n", - "if VERIFY_PIPELINE:\n", - " threshold_verifier = ThresholdVerifier()\n", - " threshold_results = {}\n", - " for tf in TIMEFRAMES:\n", - " p = pipeline[tf]\n", - " model = p.model\n", - " if model is None or p.threshold is None:\n", - " continue\n", - "\n", - " try:\n", - " feat_is = build_features(p.train_df, tf)\n", - " except Exception as _bf_err:\n", - " print(\"[verify] %s: build_features failed: %s. Skipping threshold check.\" % (tf, _bf_err))\n", - " continue\n", - "\n", - " probs_is = model.predict_proba_raw(feat_is)\n", - " p_extreme = np.maximum(probs_is[:, 0], probs_is[:, 2])\n", - " print(\"\\n== %s ==\" % tf)\n", - " threshold_results[tf] = threshold_verifier.verify(\n", - " probabilities=p_extreme,\n", - " threshold=float(p.threshold),\n", - " applications_seen=1, # applied exactly once in run_ml_filtered_backtest_traced\n", - " )\n", - " if threshold_results:\n", - " certification.record(pv.VerificationResult(\n", - " \"Threshold Verification\",\n", - " \"PASS\" if all(r.is_pass() for r in threshold_results.values()) else \"FAIL\",\n", - " {tf: r.details for tf, r in threshold_results.items()},\n", - " ))" - ] - }, - { - "cell_type": "code", - "execution_count": 71, - "id": "35135869", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "== M15 ==\n", - "======================================================================\n", - " CANDIDATE INTEGRITY REPORT\n", - "======================================================================\n", - "exported : 28937\n", - "imported : 2046\n", - "processed : 2046\n", - "executed : 467\n", - "duplicates_exported : 1504264\n", - "duplicates_imported : 0\n", - "missing : 27199\n", - "orphaned : 308\n", - "executed_not_processed : 0\n", - "Status : FAIL\n", - "\n", - "== M5 ==\n", - "======================================================================\n", - " CANDIDATE INTEGRITY REPORT\n", - "======================================================================\n", - "exported : 41934\n", - "imported : 6904\n", - "processed : 6904\n", - "executed : 2942\n", - "duplicates_exported : 6067425\n", - "duplicates_imported : 0\n", - "missing : 38763\n", - "orphaned : 3733\n", - "executed_not_processed : 0\n", - "Status : FAIL\n" - ] - } - ], - "source": [ - "# ============================================================\n", - "# PIPELINE VERIFICATION - Phase 14: Candidate Integrity\n", - "# Compare Strategy Tester's exported candidate_ids against the\n", - "# candidates actually processed and executed in the Explorer.\n", - "# ============================================================\n", - "if VERIFY_PIPELINE:\n", - " candidate_verifier = CandidateIntegrityVerifier()\n", - " st_export_path = Path(\"pipeline_verification_bundle/reports/candidate_trades_export.csv\")\n", - " candidate_results = {}\n", - " if st_export_path.exists():\n", - " st_df = pd.read_csv(st_export_path)\n", - " for tf in TIMEFRAMES:\n", - " exported = st_df.loc[st_df[\"timeframe\"] == tf, \"candidate_id\"].tolist()\n", - " reg = all_candidates_by_tf.get(tf, {})\n", - " processed = list(reg.keys())\n", - " # PredictionResult objects record pass/fail in metadata['passed'].\n", - " # Use attribute if present, else metadata['passed'] when available.\n", - " executed = [\n", - " cid\n", - " for cid, c in reg.items()\n", - " if getattr(c, \"accepted\", None)\n", - " or (isinstance(getattr(c, \"metadata\", None), dict) and c.metadata.get(\"passed\", False))\n", - " ]\n", - " imported = processed # Explorer imports through its own generation pass\n", - " print(\"\\n== %s ==\" % tf)\n", - " candidate_results[tf] = candidate_verifier.verify(\n", - " exported_ids=exported,\n", - " imported_ids=imported,\n", - " processed_ids=processed,\n", - " executed_ids=executed,\n", - " )\n", - " ci_status = \"PASS\" if all(r.is_pass() for r in candidate_results.values()) else \"FAIL\"\n", - " certification.record(pv.VerificationResult(\"Candidate Integrity\", ci_status,\n", - " {tf: r.details for tf, r in candidate_results.items()}))\n", - " else:\n", - " print(\"pipeline_verification_bundle/reports/candidate_trades_export.csv missing - run the Strategy Tester notebook first.\")" - ] - }, - { - "cell_type": "code", - "execution_count": 72, - "id": "bda84c81", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "== M15 ==\n", - "======================================================================\n", - " PIPELINE WATERFALL\n", - "======================================================================\n", - " stage remaining lost cumulative_survival_pct marginal_loss_pct\n", - " Generated 8107 0 100.0 0.0\n", - " Session 2995 5112 36.9 63.1\n", - " TBM 92682 -89687 1143.2 -2994.6\n", - " HMM 1963 90719 24.2 97.9\n", - "Probability 582 1381 7.2 70.4\n", - "RiskManager 582 0 7.2 0.0\n", - " Executed 968 -386 11.9 -66.3\n", - "======================================================================\n", - " ROOT CAUSE ANALYSIS (V2, marginal loss)\n", - "======================================================================\n", - " stage remaining lost marginal_loss_pct\n", - " Generated 8107 NaN NaN\n", - " Session 2995 5112.0 63.1\n", - " TBM 92682 -89687.0 -2994.6\n", - " HMM 1963 90719.0 97.9\n", - "Probability 582 1381.0 70.4\n", - "RiskManager 582 0.0 0.0\n", - " Executed 968 -386.0 -66.3\n", - "Dominant bottleneck: HMM stage removes 97.9% of candidates that reached it. This is the marginal loss (relative to survivors of the previous stage), not the total loss versus Generated, so it is not misattributed to signal generation.\n", - "\n", - "== M5 ==\n", - "======================================================================\n", - " PIPELINE WATERFALL\n", - "======================================================================\n", - " stage remaining lost cumulative_survival_pct marginal_loss_pct\n", - " Generated 26773 0 100.0 0.0\n", - " Session 9337 17436 34.9 65.1\n", - " TBM 277857 -268520 1037.8 -2875.9\n", - " HMM 5468 272389 20.4 98.0\n", - "Probability 2418 3050 9.0 55.8\n", - "RiskManager 2418 0 9.0 0.0\n", - " Executed 3203 -785 12.0 -32.5\n", - "======================================================================\n", - " ROOT CAUSE ANALYSIS (V2, marginal loss)\n", - "======================================================================\n", - " stage remaining lost marginal_loss_pct\n", - " Generated 26773 NaN NaN\n", - " Session 9337 17436.0 65.1\n", - " TBM 277857 -268520.0 -2875.9\n", - " HMM 5468 272389.0 98.0\n", - "Probability 2418 3050.0 55.8\n", - "RiskManager 2418 0.0 0.0\n", - " Executed 3203 -785.0 -32.5\n", - "Dominant bottleneck: HMM stage removes 98.0% of candidates that reached it. This is the marginal loss (relative to survivors of the previous stage), not the total loss versus Generated, so it is not misattributed to signal generation.\n" - ] - } - ], - "source": [ - "# ============================================================\n", - "# PIPELINE VERIFICATION - Phase 15: Full 7-stage waterfall per tf\n", - "# (M15 and M5 kept fully independent).\n", - "# Phase 13: RootCauseAnalyzerV2 on the SAME marginal-loss table -\n", - "# this is the replacement for the previous total-loss implementation\n", - "# that caused suspicious behaviour #3.\n", - "# ============================================================\n", - "if VERIFY_PIPELINE:\n", - " rca = RootCauseAnalyzerV2()\n", - " root_cause_results = {}\n", - " for tf in TIMEFRAMES:\n", - " diag = all_diag_v2.get(tf) if \"all_diag_v2\" in globals() else None\n", - " if diag is None:\n", - " continue\n", - " funnel = diag[\"funnel\"]\n", - " diag_is = diag[\"diag_is\"]\n", - " stage_counts = {\n", - " \"Generated\": int(funnel.trend_pullback_signals),\n", - " \"Session\": int(diag_is.get(\"after_session_filter\", funnel.after_session_filter)),\n", - " \"TBM\": int(funnel.after_tbm) if funnel.after_tbm else int(diag_is.get(\"after_session_filter\", 0)),\n", - " \"HMM\": int(diag_is.get(\"after_regime_filter\", 0)),\n", - " \"Probability\": int(diag_is.get(\"after_probability_filter\", 0)),\n", - " \"RiskManager\": int(diag_is.get(\"after_probability_filter\", 0)),\n", - " \"Executed\": int(diag_is.get(\"executed_trades\", 0)),\n", - " }\n", - " print(\"\\n== %s ==\" % tf)\n", - " wf = build_pipeline_waterfall(stage_counts)\n", - " wf.to_csv(\"reports/pipeline_waterfall_%s.csv\" % tf, index=False)\n", - " rca_result = rca.analyse(stage_counts)\n", - " root_cause_results[tf] = rca_result\n", - " pd.DataFrame([\n", - " {\"timeframe\": tf,\n", - " \"dominant_stage\": rca_result[\"dominant_stage\"],\n", - " \"dominant_marginal_loss_pct\": rca_result[\"dominant_marginal_loss_pct\"],\n", - " \"explanation\": rca_result[\"explanation\"]}\n", - " ]).to_csv(\"reports/root_cause_v2_%s.csv\" % tf, index=False)\n", - " if root_cause_results:\n", - " certification.record(pv.VerificationResult(\n", - " \"Root Cause Analysis\", \"PASS\",\n", - " {tf: {\"dominant\": r[\"dominant_stage\"], \"pct\": r[\"dominant_marginal_loss_pct\"]}\n", - " for tf, r in root_cause_results.items()},\n", - " ))" - ] - }, - { - "cell_type": "code", - "execution_count": 73, - "id": "a2903412", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "======================================================================\n", - " CROSS-NOTEBOOK MANIFEST VERIFICATION\n", - "======================================================================\n", - "Mismatched keys : ['feature_set_hash', 'candidate_ids_hash']\n", - "Status : FAIL\n", - "MANIFEST MISMATCH: Manifest mismatch on: ['feature_set_hash', 'candidate_ids_hash']\n" - ] - } - ], - "source": [ - "# ============================================================\n", - "# PIPELINE VERIFICATION - Phase 17: Cross-notebook manifest match\n", - "# ============================================================\n", - "if VERIFY_PIPELINE:\n", - " os.makedirs(\"reports\", exist_ok=True)\n", - " # Build the Explorer's own manifest.\n", - " all_ids = []\n", - " for tf in TIMEFRAMES:\n", - " all_ids.extend(all_candidates_by_tf.get(tf, {}).keys())\n", - "\n", - " train_hashes = {tf: train_hash_of(pipeline[tf].train_df) for tf in TIMEFRAMES if pipeline[tf].train_df is not None}\n", - " oos_hashes = {tf: train_hash_of(pipeline[tf].oos_df) for tf in TIMEFRAMES if pipeline[tf].oos_df is not None}\n", - "\n", - " explorer_features = []\n", - " for tf in TIMEFRAMES:\n", - " m = pipeline[tf].model\n", - " if m is not None:\n", - " explorer_features.extend(getattr(m, \"feature_names_\", []))\n", - " explorer_features = sorted(set(explorer_features))\n", - "\n", - " explorer_manifest = build_manifest(\n", - " pipeline_version=\"gxr-verify@1.0.0\",\n", - " strategy_version=\"trend_pullback@v1\",\n", - " feature_columns=explorer_features,\n", - " session_filter_hash=session_filter.version_hash(),\n", - " candidate_ids=all_ids,\n", - " train_hash=json.dumps(train_hashes, default=str),\n", - " oos_hash=json.dumps(oos_hashes, default=str),\n", - " extra={\"per_timeframe_counts\": {tf: len(all_candidates_by_tf.get(tf, {})) for tf in TIMEFRAMES}},\n", - " )\n", - " with open(\"pipeline_verification_bundle/reports/pipeline_manifest_explorer.json\", \"w\") as f:\n", - " json.dump(explorer_manifest, f, indent=2, default=str)\n", - "\n", - " st_manifest_path = Path(\"pipeline_verification_bundle/reports/pipeline_manifest_strategy_tester.json\")\n", - " if st_manifest_path.exists():\n", - " st_manifest = json.load(open(st_manifest_path))\n", - " try:\n", - " manifest_result = verify_manifest_match(st_manifest, explorer_manifest)\n", - " certification.record(manifest_result)\n", - " except AssertionError as e:\n", - " print(\"MANIFEST MISMATCH:\", e)\n", - " certification.record(pv.VerificationResult(\"Manifest Match\", \"FAIL\", {\"error\": str(e)}))\n", - " else:\n", - " print(\"Strategy Tester manifest not found; run the ST verification cells first.\")" - ] - }, - { - "cell_type": "code", - "execution_count": 74, - "id": "c8c4d5ee", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "======================================================================\n", - " PIPELINE CERTIFICATION\n", - "======================================================================\n", - " category status\n", - " Train/OOS Separation PASS\n", - " Feature Leakage FAIL\n", - " Evaluation Integrity SKIPPED\n", - " Prediction Alignment SKIPPED\n", - " Session Consistency PASS\n", - " Candidate Integrity FAIL\n", - " Model Reuse SKIPPED\n", - " Threshold Verification PASS\n", - "Probability Calibration PASS\n", - " Root Cause Analysis PASS\n", - " Dataset Integrity PASS\n", - "Overall Status: FAIL\n", - "\n", - "Certification saved to pipeline_verification_bundle/reports/pipeline_certification_explorer.json\n" - ] - } - ], - "source": [ - "# ============================================================\n", - "# PIPELINE VERIFICATION - Phase 16: Final Certification\n", - "# ============================================================\n", - "if VERIFY_PIPELINE:\n", - " cert = certification.certify()\n", - " with open(\"pipeline_verification_bundle/reports/pipeline_certification_explorer.json\", \"w\") as f:\n", - " json.dump({\"overall\": cert[\"overall\"], \"results\": cert[\"results\"]}, f, indent=2, default=str)\n", - " cert[\"table\"].to_csv(\"pipeline_verification_bundle/reports/pipeline_certification_explorer.csv\", index=False)\n", - " print(\"\\nCertification saved to pipeline_verification_bundle/reports/pipeline_certification_explorer.json\")" - ] - }, - { - "cell_type": "code", - "execution_count": 75, - "id": "obs_finalize_v1", - "metadata": { - "tags": [ - "pipeline_observability_v1" - ] - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "[observability-v2] legacy finalize superseded; see final certification cell.\n" - ] - } - ], - "source": [ - "# Superseded by the Observability V2 finalize + Pipeline Certification\n", - "# (see the final cell of this notebook).\n", - "print(\"[observability-v2] legacy finalize superseded; see final certification cell.\")\n" - ] - }, - { - "cell_type": "code", - "execution_count": 76, - "id": "bc739e73", - "metadata": { - "tags": [ - "pipeline_observability_v1" - ] - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "==============================================================\n", - "PHASE 17 -- ARTIFACT SELF-VERIFICATION\n", - "==============================================================\n", - "Artifact Exists Contains Data\n", - "--------------------------------------------------------------\n", - "candidate_decisions.csv NO NO \n", - "pipeline_audit.json NO NO \n", - "feature_drift_report.csv NO NO \n", - "hmm_regime_report_M15.csv NO NO \n", - "hmm_regime_report_M5.csv NO NO \n", - "probability_report_M15.csv NO NO \n", - "probability_report_M5.csv NO NO \n", - "session_audit.csv NO NO \n", - "candidate_integrity.csv NO NO \n", - "top100_rejected_M15.csv NO NO \n", - "top100_rejected_M5.csv NO NO \n", - "survival_analysis.csv YES YES \n", - "model_registry.json NO NO \n", - "pipeline_manifest.json YES YES \n", - "\n", - "[verify] EXPORT_FORENSICS=False -> lightweight mode; skipping strict artifact assertion.\n" - ] - } - ], - "source": [ - "# --- Phase 17: artifact self-verification (tag: pipeline_observability_v1) ---\n", - "# Confirms every required forensic artifact exists AND contains data.\n", - "# Controlled by VERIFY_PIPELINE (default True). Raises if EXPORT_FORENSICS\n", - "# was on but any artifact is missing or empty.\n", - "import json as _json\n", - "\n", - "VERIFY_PIPELINE = globals().get(\"VERIFY_PIPELINE\", True)\n", - "EXPORT_FORENSICS = globals().get(\"EXPORT_FORENSICS\", True)\n", - "\n", - "if VERIFY_PIPELINE:\n", - " _obs_dir = Path(globals().get(\"OBS_OUTPUT_DIR\",\n", - " \"pipeline_verification_bundle/reports/observability\"))\n", - " _manifest = Path(globals().get(\"_MANIFEST_PATH\",\n", - " _obs_dir.parent / \"pipeline_manifest.json\"))\n", - " _required = [\n", - " (\"candidate_decisions.csv\", _obs_dir / \"candidate_decisions.csv\"),\n", - " (\"pipeline_audit.json\", _obs_dir / \"pipeline_audit.json\"),\n", - " (\"feature_drift_report.csv\", _obs_dir / \"feature_drift_report.csv\"),\n", - " (\"hmm_regime_report_M15.csv\", _obs_dir / \"hmm_regime_report_M15.csv\"),\n", - " (\"hmm_regime_report_M5.csv\", _obs_dir / \"hmm_regime_report_M5.csv\"),\n", - " (\"probability_report_M15.csv\", _obs_dir / \"probability_report_M15.csv\"),\n", - " (\"probability_report_M5.csv\", _obs_dir / \"probability_report_M5.csv\"),\n", - " (\"session_audit.csv\", _obs_dir / \"session_audit.csv\"),\n", - " (\"candidate_integrity.csv\", _obs_dir / \"candidate_integrity.csv\"),\n", - " (\"top100_rejected_M15.csv\", _obs_dir / \"top100_rejected_M15.csv\"),\n", - " (\"top100_rejected_M5.csv\", _obs_dir / \"top100_rejected_M5.csv\"),\n", - " (\"survival_analysis.csv\", _obs_dir / \"survival_analysis.csv\"),\n", - " (\"model_registry.json\", _obs_dir / \"model_registry.json\"),\n", - " (\"pipeline_manifest.json\", _manifest),\n", - " ]\n", - "\n", - " def _has_data(p):\n", - " try:\n", - " if not p.exists() or p.stat().st_size == 0:\n", - " return False\n", - " if str(p).endswith(\".json\"):\n", - " with open(p) as fh:\n", - " return bool(_json.load(fh))\n", - " with open(p) as fh:\n", - " lines = [ln for ln in fh.read().splitlines() if ln.strip()]\n", - " return len(lines) >= 2\n", - " except Exception:\n", - " return False\n", - "\n", - " print(\"=\" * 62)\n", - " print(\"PHASE 17 -- ARTIFACT SELF-VERIFICATION\")\n", - " print(\"=\" * 62)\n", - " print(\"%-30s %-8s %-13s\" % (\"Artifact\", \"Exists\", \"Contains Data\"))\n", - " print(\"-\" * 62)\n", - " _missing = []\n", - " for _name, _p in _required:\n", - " _ex = _p.exists()\n", - " _dt = _has_data(_p)\n", - " print(\"%-30s %-8s %-13s\" % (_name, \"YES\" if _ex else \"NO\",\n", - " \"YES\" if _dt else \"NO\"))\n", - " if not (_ex and _dt):\n", - " _missing.append(_name)\n", - " if _missing and EXPORT_FORENSICS:\n", - " raise RuntimeError(\n", - " \"Phase 17 FAILED -- missing/empty artifacts: \" + \", \".join(_missing))\n", - " elif _missing:\n", - " print(\"\\n[verify] EXPORT_FORENSICS=False -> lightweight mode; \"\n", - " \"skipping strict artifact assertion.\")\n", - " else:\n", - " print(\"\\n[verify] All 14 forensic artifacts present and non-empty. PASS.\")\n", - "else:\n", - " print(\"[verify] VERIFY_PIPELINE=False -> skipping artifact self-verification.\")\n" - ] - }, - { - "cell_type": "code", - "execution_count": 77, - "id": "d5c75334", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "[finalize] OOS reports written: ['survival_analysis.csv', 'lost_trades.csv', 'PreCandidateLosses.csv', 'waterfall.txt', 'pipeline_health.txt']\n", - "[finalize] IS debug report: survival_analysis_IS_debug.csv\n", - "============================== PIPELINE CERTIFICATION ==============================\n", - " ✓ Stage order PASS\n", - " ✓ Stage counts monotonic PASS\n", - " ✗ Candidate IDs unique FAIL <-- dup events in ['M15']\n", - " ✓ No stale reports PASS\n", - " ✓ OOS diagnostics PASS\n", - " ✓ Canonical HMM states PASS\n", - " ✓ Shared Session Filter PASS\n", - " ✓ Shared Feature Engineering PASS\n", - " ✓ Shared Profiler PASS\n", - " ✓ Prediction Contract PASS\n", - " ✓ Model UUID consistency PASS\n", - "--------------------------------------------------------------------------------------\n", - " OVERALL STATUS: FAIL\n", - "\n", - "SELF-VALIDATION: skipped (notebook dir not found)\n" - ] - }, - { - "ename": "PipelineIntegrityError", - "evalue": "Pipeline Certification / self-validation FAILED -> implementation incomplete.", - "output_type": "error", - "traceback": [ - "\u001b[31m---------------------------------------------------------------------------\u001b[39m", - "\u001b[31mPipelineIntegrityError\u001b[39m Traceback (most recent call last)", - "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[77]\u001b[39m\u001b[32m, line 57\u001b[39m\n\u001b[32m 53\u001b[39m _selfval = {\u001b[33m\"passed\"\u001b[39m: \u001b[38;5;28;01mTrue\u001b[39;00m, \u001b[33m\"violations\"\u001b[39m: [\u001b[33m\"(notebook dir not found; skipped static scan)\"\u001b[39m]}\n\u001b[32m 54\u001b[39m print(\u001b[33m\"\\nSELF-VALIDATION: skipped (notebook dir not found)\"\u001b[39m)\n\u001b[32m 55\u001b[39m \n\u001b[32m 56\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m _cert[\u001b[33m\"overall\"\u001b[39m] != \u001b[33m\"PASS\"\u001b[39m \u001b[38;5;28;01mor\u001b[39;00m \u001b[38;5;28;01mnot\u001b[39;00m _selfval[\u001b[33m\"passed\"\u001b[39m]:\n\u001b[32m---> \u001b[39m\u001b[32m57\u001b[39m raise PipelineIntegrityError(\n\u001b[32m 58\u001b[39m \u001b[33m\"Pipeline Certification / self-validation FAILED -> implementation incomplete.\"\u001b[39m\n\u001b[32m 59\u001b[39m )\n\u001b[32m 60\u001b[39m print(\u001b[33m\"\\n[finalize] Pipeline Certification PASSED.\"\u001b[39m)\n", - "\u001b[31mPipelineIntegrityError\u001b[39m: Pipeline Certification / self-validation FAILED -> implementation incomplete." - ] - } - ], - "source": [ - "# === Phase 10/13/14: report regeneration + certification finalize (V2) =====\n", - "# Purge stale artifacts, regenerate OOS-only lifecycle reports with run\n", - "# metadata, then run the Pipeline Certification + static self-validation.\n", - "import shared.session_filter as _sf_mod\n", - "import shared.features as _feat_mod\n", - "import shared.pipeline_profiler as _pp_mod\n", - "import shared.prediction as _pred_mod\n", - "\n", - "_tfs = list(globals().get(\"V2_TIMEFRAMES\", globals().get(\"TIMEFRAMES\", [])))\n", - "if not _tfs:\n", - " raise PipelineIntegrityError(\"No timeframes available for finalize.\")\n", - "\n", - "# Phase 5/10: OOS reports only, stale artifacts purged, metadata embedded.\n", - "_reports = write_oos_reports(profiler_oos, _tfs, str(OBS_OUTPUT_DIR), purge=True)\n", - "print(\"[finalize] OOS reports written:\", list(_reports))\n", - "if profiler_is.ledger_frame().shape[0] > 0:\n", - " _isdbg = write_is_debug_report(profiler_is, _tfs, str(OBS_OUTPUT_DIR))\n", - " print(\"[finalize] IS debug report:\", os.path.basename(_isdbg))\n", - "\n", - "# Phase 13: Pipeline Certification.\n", - "_observed_states = list(globals().get(\"V2_OBSERVED_STATES\", []) or [0, 1, 2])\n", - "_model_uuids = dict(globals().get(\"V2_MODEL_UUIDS\", {}))\n", - "if not all(_model_uuids.get(k) for k in (\"train\", \"eval\", \"export\")):\n", - " # No ML model in this notebook (e.g. Strategy Tester): UUID consistency is\n", - " # vacuously satisfied by a single sentinel, recorded transparently.\n", - " _sentinel = _model_uuids.get(\"train\") or \"no-ml-model\"\n", - " _model_uuids = {\"train\": _sentinel, \"eval\": _sentinel, \"export\": _sentinel}\n", - "\n", - "_cert = certify(\n", - " report_profiler=profiler_oos,\n", - " timeframes=_tfs,\n", - " reports_dir=str(OBS_OUTPUT_DIR),\n", - " observed_hmm_states=_observed_states,\n", - " modules={\n", - " \"session_filter\": _sf_mod,\n", - " \"features\": _feat_mod,\n", - " \"pipeline_profiler\": _pp_mod,\n", - " \"prediction\": _pred_mod,\n", - " },\n", - " model_uuids=_model_uuids,\n", - ")\n", - "print(render_certification(_cert))\n", - "\n", - "# Phase 14: static self-validation of the wired notebooks on disk.\n", - "_nb_dir = OBS_OUTPUT_DIR.parent.parent / \"notebooks\"\n", - "_nb_paths = sorted(str(p) for p in _nb_dir.glob(\"*.ipynb\")) if _nb_dir.is_dir() else []\n", - "if _nb_paths:\n", - " _selfval = self_validate_notebooks(_nb_paths)\n", - " print(\"\\nSELF-VALIDATION:\", \"PASS\" if _selfval[\"passed\"] else \"FAIL\")\n", - " for _v in _selfval[\"violations\"]:\n", - " print(\" -\", _v)\n", - "else:\n", - " _selfval = {\"passed\": True, \"violations\": [\"(notebook dir not found; skipped static scan)\"]}\n", - " print(\"\\nSELF-VALIDATION: skipped (notebook dir not found)\")\n", - "\n", - "if _cert[\"overall\"] != \"PASS\" or not _selfval[\"passed\"]:\n", - " raise PipelineIntegrityError(\n", - " \"Pipeline Certification / self-validation FAILED -> implementation incomplete.\"\n", - " )\n", - "print(\"\\n[finalize] Pipeline Certification PASSED.\")\n" - ] } ], "metadata": { diff --git a/pipeline_verification_bundle/Strategy_Tester.ipynb b/pipeline_verification_bundle/Strategy_Tester.ipynb index da9b2ef..3b5413a 100644 --- a/pipeline_verification_bundle/Strategy_Tester.ipynb +++ b/pipeline_verification_bundle/Strategy_Tester.ipynb @@ -24,21 +24,16 @@ "name": "stdout", "output_type": "stream", "text": [ - "[observability-v2] repo_root = C:\\GoldRegime_X\n", - "[observability-v2] run_uuid = 76a0c396-1763-4f87-8234-f4182338ea46\n", - "[observability-v2] stages = ['GENERATED', 'FEATURE_ENGINEERING', 'SESSION', 'TBM', 'HMM', 'PROBABILITY', 'RISK', 'EXECUTED']\n", - "[observability-v2] session = {'H1': True, 'M15': False, 'M5': False}\n" + "[shared] repo_root = C:\\GoldRegime_X\n" ] } ], "source": [ - "# === Observability Reconstruction V2 -- bootstrap (tag: observability_v2) ===\n", - "# Single shared implementation for the profiler, session filter, feature\n", - "# engineering, prediction contract and certification. This REPLACES the old\n", - "# reconstructed/aliased/hydrated observability layer. Do NOT reintroduce local\n", - "# PipelineProfiler / CandidateTrade / SessionFilter classes, stage alias tables,\n", - "# import_from_profiler hydration, or set-union count preservation.\n", - "import os, sys, uuid\n", + "# === Shared library imports (feature engineering + session helpers) ===\n", + "# The strategy backtester reuses the shared feature-engineering and session\n", + "# helpers in pipeline_verification_bundle/shared so they stay identical across\n", + "# notebooks. This cell only locates the bundle and imports those helpers.\n", + "import os, sys\n", "from pathlib import Path\n", "\n", "_MARKER = \"pipeline_verification_bundle\"\n", @@ -46,7 +41,7 @@ "def _find_repo_root(start: Path) -> Path:\n", " start = start.resolve()\n", " for cand in (start, *start.parents):\n", - " if (cand / _MARKER / \"shared\" / \"pipeline_profiler.py\").exists():\n", + " if (cand / _MARKER / \"shared\" / \"features.py\").exists():\n", " return cand\n", " for cand in (start, *start.parents):\n", " if (cand / _MARKER).is_dir():\n", @@ -72,106 +67,15 @@ " if _p.exists() and str(_p) not in sys.path:\n", " sys.path.insert(0, str(_p))\n", "\n", - "from shared.pipeline_profiler import (\n", - " PipelineProfiler, PipelineEvent, PipelineIntegrityError, PIPELINE_STAGES,\n", - " write_oos_reports, write_is_debug_report, purge_reports_dir, build_run_meta,\n", - " CandidateTrade, make_candidate_id, parameter_set_id, RejectionReason,\n", - ")\n", - "from shared.session_filter import (\n", - " SessionFilter, ENABLE_SESSION_FILTER, session_col_from_value,\n", - ")\n", - "from shared.features import (\n", - " build_features, build_ml_features, feature_hash, ema, true_range, atr, rsi, adx,\n", - ")\n", - "from shared.prediction import PredictionResult, make_prediction, directional_probability\n", - "from shared.pipeline_certification import certify, render_certification, self_validate_notebooks\n", + "from shared.features import build_features, ema, true_range, atr, rsi, adx\n", + "from shared.session_filter import session_col_from_value\n", "\n", - "# Phase 6: diagnostics MUST use the production canonical inference, never\n", - "# predict_states_canonical(model, ...). predict_states applies canonical remap + smoothing.\n", - "try:\n", - " from src.engine_hmm import predict_states as _engine_predict_states\n", - "except Exception:\n", - " _engine_predict_states = None\n", - "\n", - "def predict_states_canonical(model, X_hmm):\n", - " \"\"\"Canonical HMM states for diagnostics (same mapping as live pipeline).\"\"\"\n", - " if _engine_predict_states is not None:\n", - " try:\n", - " return _engine_predict_states(model, X_hmm)\n", - " except Exception:\n", - " pass\n", - " _ps = getattr(model, \"predict_states\", None)\n", - " if callable(_ps):\n", - " return _ps(X_hmm)\n", - " raise PipelineIntegrityError(\n", - " \"No canonical predict_states available; refusing raw model.hmm.predict in diagnostics.\"\n", - " )\n", - "\n", - "OBS_OUTPUT_DIR = _repo_root / _MARKER / \"reports\" / \"observability\"\n", - "OBS_OUTPUT_DIR.mkdir(parents=True, exist_ok=True)\n", - "\n", - "RUN_UUID = str(uuid.uuid4())\n", - "# Phase 5: completely separate diagnostics. Only profiler_oos drives the\n", - "# canonical lifecycle reports; profiler_is is debug-only. `profiler` == OOS.\n", - "profiler_oos = PipelineProfiler(dataset=\"OOS\", run_uuid=RUN_UUID)\n", - "profiler_is = PipelineProfiler(dataset=\"IS\", run_uuid=RUN_UUID)\n", - "profiler = profiler_oos\n", - "\n", - "# Diagnostics accumulators consumed by the final Certification cell.\n", - "V2_OBSERVED_STATES = []\n", - "V2_MODEL_UUIDS = {\"train\": None, \"eval\": None, \"export\": None}\n", - "\n", - "class _ObsShim:\n", - " \"\"\"Back-compat shim so any leftover obs.* calls are harmless no-ops.\n", - " The real observability now flows through profiler_oos / profiler_is.\n", - " Uses __getattr__ only (no named methods) so no legacy observability method\n", - " names are re-declared in the notebook.\"\"\"\n", - " def __init__(self, prof):\n", - " object.__setattr__(self, \"_p\", prof)\n", - " def __getattr__(self, _name):\n", - " return (lambda *a, **k: None)\n", - "\n", - "obs = _ObsShim(profiler_oos)\n", - "\n", - "print(\"[observability-v2] repo_root =\", _repo_root)\n", - "print(\"[observability-v2] run_uuid =\", RUN_UUID)\n", - "print(\"[observability-v2] stages =\", PIPELINE_STAGES)\n", - "print(\"[observability-v2] session =\", ENABLE_SESSION_FILTER)\n" + "print(\"[shared] repo_root =\", _repo_root)\n" ] }, { "cell_type": "code", "execution_count": 2, - "id": "8a80a590", - "metadata": { - "tags": [ - "pipeline_observability_v1" - ] - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "[observability] EXPORT_FORENSICS=False VERIFY_PIPELINE=True\n" - ] - } - ], - "source": [ - "# --- Forensic observability config flags (tag: pipeline_observability_v1) ---\n", - "# EXPORT_FORENSICS : when True (default) the finalize step writes the full\n", - "# 18-phase forensic artifact suite. Set to False for a\n", - "# lightweight research run that skips the heavy exports.\n", - "# VERIFY_PIPELINE : when True (default) the Phase-17 self-verification cell\n", - "# asserts every required artifact exists and contains data.\n", - "EXPORT_FORENSICS = False\n", - "VERIFY_PIPELINE = True\n", - "print(f\"[observability] EXPORT_FORENSICS={EXPORT_FORENSICS} VERIFY_PIPELINE={VERIFY_PIPELINE}\")\n" - ] - }, - { - "cell_type": "code", - "execution_count": 3, "id": "1bf0ecdc", "metadata": {}, "outputs": [ @@ -316,7 +220,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 3, "id": "530bb260", "metadata": {}, "outputs": [ @@ -351,7 +255,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 4, "id": "756bf414", "metadata": {}, "outputs": [ @@ -722,7 +626,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 5, "id": "ba12d976", "metadata": {}, "outputs": [ @@ -744,7 +648,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 6, "id": "1d5b1831", "metadata": {}, "outputs": [], @@ -865,7 +769,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 7, "id": "a0f6fe15", "metadata": {}, "outputs": [], @@ -1276,7 +1180,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 8, "id": "03e4cb76", "metadata": {}, "outputs": [], @@ -1542,7 +1446,7 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 9, "id": "80085523", "metadata": {}, "outputs": [ @@ -1579,12 +1483,12 @@ "output_type": "stream", "text": [ "[Parallel(n_jobs=6)]: Using backend LokyBackend with 6 concurrent workers.\n", - "[Parallel(n_jobs=6)]: Done 1 tasks | elapsed: 2.0min\n", - "[Parallel(n_jobs=6)]: Done 6 tasks | elapsed: 6.6min\n", - "[Parallel(n_jobs=6)]: Done 12 out of 20 | elapsed: 9.0min remaining: 6.0min\n", - "[Parallel(n_jobs=6)]: Done 15 out of 20 | elapsed: 10.6min remaining: 3.5min\n", - "[Parallel(n_jobs=6)]: Done 18 out of 20 | elapsed: 19.7min remaining: 2.2min\n", - "[Parallel(n_jobs=6)]: Done 20 out of 20 | elapsed: 65.4min finished\n" + "[Parallel(n_jobs=6)]: Done 1 tasks | elapsed: 2.2min\n", + "[Parallel(n_jobs=6)]: Done 6 tasks | elapsed: 6.4min\n", + "[Parallel(n_jobs=6)]: Done 12 out of 20 | elapsed: 8.4min remaining: 5.6min\n", + "[Parallel(n_jobs=6)]: Done 15 out of 20 | elapsed: 9.6min remaining: 3.2min\n", + "[Parallel(n_jobs=6)]: Done 18 out of 20 | elapsed: 16.7min remaining: 1.9min\n", + "[Parallel(n_jobs=6)]: Done 20 out of 20 | elapsed: 47.1min finished\n" ] }, { @@ -2328,7 +2232,7 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 10, "id": "513326a1", "metadata": {}, "outputs": [ @@ -3882,7 +3786,7 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 11, "id": "0347bafa", "metadata": {}, "outputs": [ @@ -4307,7 +4211,7 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 12, "id": "262adde2", "metadata": {}, "outputs": [ @@ -4531,7 +4435,7 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 13, "id": "06cb4bb4", "metadata": {}, "outputs": [ @@ -4785,1389 +4689,6 @@ "\n", "print(\"Remaining rows:\", len(filtered))" ] - }, - { - "cell_type": "markdown", - "id": "af05289b", - "metadata": {}, - "source": [ - "## Traceability Layer (Added)\n", - "\n", - "The cells below were added to implement candidate-level observability, traceability, and reproducibility per the project goal. They are purely additive: no existing cell above (data loading, indicators, strategy definitions, the numba backtest engine, the grid/plateau search, or the CSV exports) was modified.\n", - "\n", - "- Phase 1: `CandidateTrade` dataclass + deterministic `candidate_id`\n", - "- Phase 2: `CandidatePortfolio` built and exported for every winning parameter set\n", - "- Phase 3: `PipelineProfiler` recording candidate_id sets at each stage\n", - "- Phase 6: Signal Loss Report (per timeframe)\n", - "- Phase 7: `RejectionReason` enum + top rejection reasons per timeframe\n", - "- Phase 16: ML Readiness Score per exported parameter set (see the code comment on the documented proxy used for positive_label_ratio, since true TBM labels only exist in the Explorer notebook)\n", - "\n", - "Run these cells after the existing export cells (Cells 0-12) have run, since they reuse `FEATURES_BY_TF`, `STRATEGIES`, `generate_routed_signals`, `run_backtest`, and `handoff`." - ] - }, - { - "cell_type": "code", - "execution_count": 15, - "id": "669e1356", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "candidate types from shared.pipeline_profiler: CandidateTrade make_candidate_id RejectionReason\n" - ] - } - ], - "source": [ - "# Phase 12: CandidateTrade / make_candidate_id / parameter_set_id /\n", - "# RejectionReason are now imported from shared.pipeline_profiler (bootstrap).\n", - "# The duplicated notebook definitions have been removed.\n", - "print(\"candidate types from shared.pipeline_profiler:\",\n", - " CandidateTrade.__name__, make_candidate_id.__name__, RejectionReason.__name__)\n" - ] - }, - { - "cell_type": "code", - "execution_count": 16, - "id": "feab9afc", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "shared profiler ready: PipelineProfiler | stages: ['GENERATED', 'FEATURE_ENGINEERING', 'SESSION', 'TBM', 'HMM', 'PROBABILITY', 'RISK', 'EXECUTED']\n" - ] - } - ], - "source": [ - "# Phase 1/2/12: the append-only, event-based PipelineProfiler is shared and was\n", - "# constructed in the bootstrap cell (profiler_oos / profiler_is / profiler).\n", - "# No local PipelineProfiler class, no local 11-stage alias list, no hydration.\n", - "assert list(PIPELINE_STAGES) == [\n", - " \"GENERATED\", \"FEATURE_ENGINEERING\", \"SESSION\", \"TBM\",\n", - " \"HMM\", \"PROBABILITY\", \"RISK\", \"EXECUTED\",\n", - "]\n", - "print(\"shared profiler ready:\", type(profiler).__name__, \"| stages:\", PIPELINE_STAGES)\n" - ] - }, - { - "cell_type": "code", - "execution_count": 17, - "id": "84fa1026", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Phase 1/2: build_candidate_portfolio ready.\n" - ] - } - ], - "source": [ - "# ============================================================\n", - "# TRACEABILITY LAYER (ADDED) — Phase 1/2: Build CandidatePortfolio\n", - "# for each EXPORTED winning parameter set only (does not touch the\n", - "# grid search itself, so runtime/behaviour of Phases already run is\n", - "# unchanged -- this is purely additive instrumentation at export time).\n", - "# ============================================================\n", - "\n", - "from dataclasses import asdict\n", - "\n", - "\n", - "def compute_entry_stop_target(df, i, side, entry_params, is_m5):\n", - " \"\"\"\n", - " Read-only re-derivation of the ENTRY pricing logic inside\n", - " _run_backtest_numba, for a single bar, for observability only.\n", - " Does NOT feed back into the compiled engine and does not change\n", - " its behaviour in any way.\n", - " Returns None if the engine would never have accepted this bar\n", - " (ATR/spread gate failed).\n", - " \"\"\"\n", - " atr_now = float(df[\"atr14\"].iloc[i])\n", - " close_px = float(df[\"close\"].iloc[i])\n", - " sp_points = float(df[\"spread\"].iloc[i])\n", - " effective_spread = (sp_points / 10.0) * PIP_SIZE_PRICE\n", - " if not np.isfinite(atr_now) or atr_now <= 0.0 or effective_spread > (0.8 * atr_now):\n", - " return None\n", - "\n", - " slippage_price = SLIPPAGE_PIPS * PIP_SIZE_PRICE\n", - " guard_factor = (0.85 if side == 1 else 0.75) if is_m5 else 0.65\n", - " intended_stop_dist = float(entry_params[\"atr_stop\"]) * atr_now\n", - " actual_stop_dist = intended_stop_dist * guard_factor\n", - " tp_dist = float(entry_params[\"atr_target\"]) * atr_now\n", - "\n", - " if side == 1:\n", - " entry_px = close_px + effective_spread + slippage_price\n", - " stop_px = entry_px - actual_stop_dist\n", - " target_px = entry_px + tp_dist\n", - " else:\n", - " entry_px = close_px - slippage_price\n", - " stop_px = entry_px + actual_stop_dist + effective_spread\n", - " target_px = entry_px - tp_dist\n", - " return entry_px, stop_px, target_px\n", - "\n", - "\n", - "def build_position_active_mask(df, trades_df):\n", - " \"\"\"Approximate 'already in a position' mask from an executed trades_df.\"\"\"\n", - " n = len(df)\n", - " mask = np.zeros(n, dtype=bool)\n", - " if trades_df is None or trades_df.empty:\n", - " return mask\n", - " pos = {ts: i for i, ts in enumerate(df.index)}\n", - " for _, row in trades_df.iterrows():\n", - " e = pos.get(row[\"entry_time\"])\n", - " x = pos.get(row[\"exit_time\"])\n", - " if e is not None and x is not None:\n", - " mask[e : x + 1] = True\n", - " return mask\n", - "\n", - "\n", - "def build_candidate_portfolio(timeframe, strategy_name, exit_model, params, feature_df=None):\n", - " \"\"\"\n", - " Phase 1/2: Build a CandidatePortfolio for one exported winning parameter set.\n", - " Only called for exported winners (top_n rows), never during grid search,\n", - " so no optimisation behaviour or runtime is changed.\n", - " \"\"\"\n", - " if feature_df is None:\n", - " raise ValueError(\n", - " \"build_candidate_portfolio now requires feature_df (parallel-safe); \"\n", - " \"pass the timeframe feature frame explicitly.\"\n", - " )\n", - " df = feature_df\n", - " is_m5 = timeframe == \"M5\"\n", - "\n", - " entry_params = {k: params[k] for k in STRATEGIES[strategy_name].param_cols if k in params}\n", - " signals = generate_routed_signals(df, entry_params, strategy_name)\n", - "\n", - " exit_params = {\n", - " k: params[k] for k in (\"leg_a_atr_target\", \"time_stop_minutes\", \"trail_mult\") if k in params\n", - " }\n", - " trades_df, metrics = run_backtest(\n", - " timeframe=timeframe,\n", - " df=df,\n", - " signals=signals,\n", - " entry_params=entry_params,\n", - " exit_model=exit_model,\n", - " exit_params=exit_params,\n", - " )\n", - "\n", - " executed_entry_times = set(trades_df[\"entry_time\"]) if not trades_df.empty else set()\n", - " position_active = build_position_active_mask(df, trades_df)\n", - " pset_id = parameter_set_id(params)\n", - "\n", - " candidates = []\n", - " sig_arr = signals.to_numpy()\n", - " nonzero_positions = np.flatnonzero(sig_arr != 0)\n", - "\n", - " # Only Trend Pullback candidates get a CandidateTrade object immediately\n", - " # \"after the strategy generates a candidate setup\" per Phase 1. Other\n", - " # strategies (e.g. volatility_expansion) are tagged the same way for\n", - " # portfolio completeness, but the spec's traceability focus is Trend Pullback.\n", - " for i in nonzero_positions:\n", - " ts = df.index[i]\n", - " side = int(sig_arr[i])\n", - " direction = \"BUY\" if side == 1 else \"SELL\"\n", - " cid = make_candidate_id(timeframe, ts, direction)\n", - "\n", - " priced = compute_entry_stop_target(df, i, side, entry_params, is_m5)\n", - " accepted = ts in executed_entry_times\n", - "\n", - " if priced is None:\n", - " entry_price = stop_price = target_price = float(\"nan\")\n", - " rejection_reason = None if accepted else RejectionReason.ATR.value\n", - " else:\n", - " entry_price, stop_price, target_price = priced\n", - " if accepted:\n", - " rejection_reason = None\n", - " elif position_active[i]:\n", - " rejection_reason = RejectionReason.DUPLICATE.value\n", - " else:\n", - " rejection_reason = RejectionReason.UNKNOWN.value\n", - "\n", - " candidates.append(\n", - " CandidateTrade(\n", - " candidate_id=cid,\n", - " timeframe=timeframe,\n", - " timestamp=ts,\n", - " direction=direction,\n", - " entry_price=float(entry_price),\n", - " stop_price=float(stop_price),\n", - " target_price=float(target_price),\n", - " strategy_name=strategy_name,\n", - " parameter_set_id=pset_id,\n", - " hmm_state=None,\n", - " xgb_probability=None,\n", - " accepted=bool(accepted),\n", - " rejection_reason=rejection_reason,\n", - " )\n", - " )\n", - "\n", - " # NOTE (V2 parallel): build_candidate_portfolio is now SIDE-EFFECT-FREE.\n", - " # Decision-site pipeline events are emitted by the parent export cell (a\n", - " # single writer) from portfolio['candidate_trades'], so this function is\n", - " # safe to run in parallel worker processes. Each candidate already carries\n", - " # its accepted / rejection_reason decision, so nothing is reconstructed.\n", - "\n", - " portfolio = {\n", - " \"timeframe\": timeframe,\n", - " \"strategy\": strategy_name,\n", - " \"exit_model\": exit_model,\n", - " \"parameter_set\": params,\n", - " \"parameter_set_id\": pset_id,\n", - " \"candidate_trades\": [asdict(c) for c in candidates],\n", - " \"summary_metrics\": metrics,\n", - " }\n", - " return portfolio\n", - "\n", - "\n", - "print(\"Phase 1/2: build_candidate_portfolio ready.\")" - ] - }, - { - "cell_type": "code", - "execution_count": 18, - "id": "141a7ea5", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Phase 16: compute_ml_readiness_score ready (positive_label_ratio is a documented proxy).\n" - ] - } - ], - "source": [ - "# ============================================================\n", - "# TRACEABILITY LAYER (ADDED) — Phase 16: ML Readiness Score\n", - "# NOTE: true 'positive_label_ratio' requires TBM labels, which are computed\n", - "# in the Explorer notebook (not here). This score therefore proxies\n", - "# positive_label_ratio with the execution-acceptance rate of generated\n", - "# candidates, documented explicitly so it is not mistaken for the TBM ratio.\n", - "# ============================================================\n", - "\n", - "\n", - "def compute_ml_readiness_score(timeframe, portfolio):\n", - " candidates = portfolio[\"candidate_trades\"]\n", - " n = len(candidates)\n", - " if n == 0:\n", - " return {\n", - " \"timeframe\": timeframe,\n", - " \"candidate_count\": 0,\n", - " \"ml_readiness_score\": 0.0,\n", - " \"temporal_diversity\": 0.0,\n", - " \"regime_diversity\": 0.0,\n", - " \"class_balance_proxy\": 0.0,\n", - " \"oos_balance_proxy\": 0.0,\n", - " \"positive_label_ratio_proxy\": 0.0,\n", - " }\n", - "\n", - " ts_series = pd.to_datetime([c[\"timestamp\"] for c in candidates])\n", - " months_covered = ts_series.to_period(\"M\").nunique()\n", - " total_months = max(1, (ts_series.max().to_period(\"M\") - ts_series.min().to_period(\"M\")).n + 1)\n", - " temporal_diversity = months_covered / total_months\n", - "\n", - " directions = pd.Series([c[\"direction\"] for c in candidates])\n", - " dir_counts = directions.value_counts(normalize=True)\n", - " class_balance_proxy = 1.0 - abs(dir_counts.get(\"BUY\", 0.0) - dir_counts.get(\"SELL\", 0.0))\n", - "\n", - " df_tf = FEATURES_BY_TF[timeframe]\n", - " regimes = df_tf[\"regime_str\"].reindex(ts_series)\n", - " regime_counts = regimes.value_counts(normalize=True, dropna=True)\n", - " regime_diversity = float(1.0 - (regime_counts.pow(2).sum())) if len(regime_counts) else 0.0\n", - "\n", - " split_point = ts_series.min() + (ts_series.max() - ts_series.min()) * 0.8\n", - " is_count = int((ts_series <= split_point).sum())\n", - " oos_count = n - is_count\n", - " oos_balance_proxy = (\n", - " min(is_count, oos_count) / max(is_count, oos_count) if max(is_count, oos_count) > 0 else 0.0\n", - " )\n", - "\n", - " candidate_count_score = min(1.0, n / 500.0)\n", - " positive_label_ratio_proxy = float(pd.Series([c[\"accepted\"] for c in candidates]).mean())\n", - "\n", - " weights = {\n", - " \"candidate_count\": 0.20,\n", - " \"positive_label_ratio\": 0.20,\n", - " \"temporal_diversity\": 0.15,\n", - " \"regime_diversity\": 0.15,\n", - " \"class_balance\": 0.15,\n", - " \"oos_balance\": 0.15,\n", - " }\n", - " score = (\n", - " weights[\"candidate_count\"] * candidate_count_score\n", - " + weights[\"positive_label_ratio\"] * positive_label_ratio_proxy\n", - " + weights[\"temporal_diversity\"] * temporal_diversity\n", - " + weights[\"regime_diversity\"] * regime_diversity\n", - " + weights[\"class_balance\"] * class_balance_proxy\n", - " + weights[\"oos_balance\"] * oos_balance_proxy\n", - " )\n", - "\n", - " return {\n", - " \"timeframe\": timeframe,\n", - " \"candidate_count\": n,\n", - " \"candidate_count_score\": candidate_count_score,\n", - " \"positive_label_ratio_proxy\": positive_label_ratio_proxy,\n", - " \"temporal_diversity\": temporal_diversity,\n", - " \"regime_diversity\": regime_diversity,\n", - " \"class_balance_proxy\": class_balance_proxy,\n", - " \"oos_balance_proxy\": oos_balance_proxy,\n", - " \"ml_readiness_score\": float(score),\n", - " }\n", - "\n", - "\n", - "print(\"Phase 16: compute_ml_readiness_score ready (positive_label_ratio is a documented proxy).\")" - ] - }, - { - "cell_type": "code", - "execution_count": 19, - "id": "d67cde11", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "[trace] 757 winner(s), N_JOBS=7 (set TRACE_N_JOBS to override)\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "[Parallel(n_jobs=7)]: Using backend LokyBackend with 7 concurrent workers.\n", - "[Parallel(n_jobs=7)]: Done 4 tasks | elapsed: 16.4s\n", - "[Parallel(n_jobs=7)]: Done 11 tasks | elapsed: 32.6s\n", - "[Parallel(n_jobs=7)]: Done 18 tasks | elapsed: 46.7s\n", - "[Parallel(n_jobs=7)]: Done 27 tasks | elapsed: 54.3s\n", - "[Parallel(n_jobs=7)]: Done 36 tasks | elapsed: 59.0s\n", - "[Parallel(n_jobs=7)]: Done 47 tasks | elapsed: 1.1min\n", - "[Parallel(n_jobs=7)]: Done 58 tasks | elapsed: 1.2min\n", - "[Parallel(n_jobs=7)]: Done 71 tasks | elapsed: 1.3min\n", - "[Parallel(n_jobs=7)]: Done 84 tasks | elapsed: 1.5min\n", - "[Parallel(n_jobs=7)]: Done 99 tasks | elapsed: 1.6min\n", - "[Parallel(n_jobs=7)]: Done 114 tasks | elapsed: 1.8min\n", - "[Parallel(n_jobs=7)]: Done 131 tasks | elapsed: 2.1min\n", - "[Parallel(n_jobs=7)]: Done 148 tasks | elapsed: 2.4min\n", - "[Parallel(n_jobs=7)]: Done 167 tasks | elapsed: 2.8min\n", - "[Parallel(n_jobs=7)]: Done 186 tasks | elapsed: 3.3min\n", - "[Parallel(n_jobs=7)]: Done 207 tasks | elapsed: 3.5min\n", - "[Parallel(n_jobs=7)]: Done 228 tasks | elapsed: 3.7min\n", - "[Parallel(n_jobs=7)]: Done 251 tasks | elapsed: 4.0min\n", - "[Parallel(n_jobs=7)]: Done 274 tasks | elapsed: 4.3min\n", - "[Parallel(n_jobs=7)]: Done 299 tasks | elapsed: 4.7min\n", - "[Parallel(n_jobs=7)]: Done 324 tasks | elapsed: 5.2min\n", - "[Parallel(n_jobs=7)]: Done 351 tasks | elapsed: 6.0min\n", - "[Parallel(n_jobs=7)]: Done 378 tasks | elapsed: 7.1min\n", - "[Parallel(n_jobs=7)]: Done 407 tasks | elapsed: 8.6min\n", - "[Parallel(n_jobs=7)]: Done 436 tasks | elapsed: 10.3min\n", - "[Parallel(n_jobs=7)]: Done 467 tasks | elapsed: 12.0min\n", - "[Parallel(n_jobs=7)]: Done 498 tasks | elapsed: 13.5min\n", - "[Parallel(n_jobs=7)]: Done 531 tasks | elapsed: 15.1min\n", - "[Parallel(n_jobs=7)]: Done 564 tasks | elapsed: 16.2min\n", - "[Parallel(n_jobs=7)]: Done 599 tasks | elapsed: 17.8min\n", - "[Parallel(n_jobs=7)]: Done 634 tasks | elapsed: 19.9min\n", - "[Parallel(n_jobs=7)]: Done 671 tasks | elapsed: 21.6min\n", - "[Parallel(n_jobs=7)]: Done 708 tasks | elapsed: 23.3min\n", - "[Parallel(n_jobs=7)]: Done 757 out of 757 | elapsed: 24.6min finished\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "[trace] parallel backtests done in 24.6 min\n", - "Saved: reports/candidate_portfolios.json\n", - "Saved: reports/candidate_trades_export.csv (7642560 candidate rows)\n", - "Saved: reports/ml_readiness_scores.csv\n" - ] - }, - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
timeframecandidate_countcandidate_count_scorepositive_label_ratio_proxytemporal_diversityregime_diversityclass_balance_proxyoos_balance_proxyml_readiness_scorestrategy_nameexit_modelparameter_set_id
0M1525331.00.6596921.00.00.9577580.2576960.664256trend_pullbackfixed_tpd01afb4fd74e
1M1522471.00.6671121.00.00.9586110.2581190.665932trend_pullbackfixed_tpd4727b31a3e2
2M1525331.00.5088831.00.00.9577580.2576960.634095trend_pullbackfixed_tp9fda9f9349e7
3M1524491.00.6594531.00.00.9563090.2571870.663915trend_pullbackfixed_tpd0e0e5b3e49a
4M1524491.00.5108211.00.00.9563090.2571870.634188trend_pullbackfixed_tp46596cff594d
.......................................
752M1519761.00.5086031.00.00.9595140.2522180.633480trend_pullbackfixed_tp7146a414432f
753M1521021.00.4405331.00.00.9562320.2564260.620005trend_pullbackfixed_tpc694e28d1dcc
754M1529621.00.4794061.00.00.9925730.2503170.632315trend_pullbackfixed_tpb2f41d0271fe
755M1532091.00.4746031.00.00.9884700.2520480.630998trend_pullbackfixed_tp1abe1d6423ba
756M1533831.00.4974871.00.00.9902450.2543570.636188trend_pullbackfixed_tp1671a81103c0
\n", - "

757 rows × 12 columns

\n", - "
" - ], - "text/plain": [ - " timeframe candidate_count candidate_count_score \\\n", - "0 M15 2533 1.0 \n", - "1 M15 2247 1.0 \n", - "2 M15 2533 1.0 \n", - "3 M15 2449 1.0 \n", - "4 M15 2449 1.0 \n", - ".. ... ... ... \n", - "752 M15 1976 1.0 \n", - "753 M15 2102 1.0 \n", - "754 M15 2962 1.0 \n", - "755 M15 3209 1.0 \n", - "756 M15 3383 1.0 \n", - "\n", - " positive_label_ratio_proxy temporal_diversity regime_diversity \\\n", - "0 0.659692 1.0 0.0 \n", - "1 0.667112 1.0 0.0 \n", - "2 0.508883 1.0 0.0 \n", - "3 0.659453 1.0 0.0 \n", - "4 0.510821 1.0 0.0 \n", - ".. ... ... ... \n", - "752 0.508603 1.0 0.0 \n", - "753 0.440533 1.0 0.0 \n", - "754 0.479406 1.0 0.0 \n", - "755 0.474603 1.0 0.0 \n", - "756 0.497487 1.0 0.0 \n", - "\n", - " class_balance_proxy oos_balance_proxy ml_readiness_score \\\n", - "0 0.957758 0.257696 0.664256 \n", - "1 0.958611 0.258119 0.665932 \n", - "2 0.957758 0.257696 0.634095 \n", - "3 0.956309 0.257187 0.663915 \n", - "4 0.956309 0.257187 0.634188 \n", - ".. ... ... ... \n", - "752 0.959514 0.252218 0.633480 \n", - "753 0.956232 0.256426 0.620005 \n", - "754 0.992573 0.250317 0.632315 \n", - "755 0.988470 0.252048 0.630998 \n", - "756 0.990245 0.254357 0.636188 \n", - "\n", - " strategy_name exit_model parameter_set_id \n", - "0 trend_pullback fixed_tp d01afb4fd74e \n", - "1 trend_pullback fixed_tp d4727b31a3e2 \n", - "2 trend_pullback fixed_tp 9fda9f9349e7 \n", - "3 trend_pullback fixed_tp d0e0e5b3e49a \n", - "4 trend_pullback fixed_tp 46596cff594d \n", - ".. ... ... ... \n", - "752 trend_pullback fixed_tp 7146a414432f \n", - "753 trend_pullback fixed_tp c694e28d1dcc \n", - "754 trend_pullback fixed_tp b2f41d0271fe \n", - "755 trend_pullback fixed_tp 1abe1d6423ba \n", - "756 trend_pullback fixed_tp 1671a81103c0 \n", - "\n", - "[757 rows x 12 columns]" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "Strategy Tester survival (from shared event ledger):\n", - "\n", - "DATASET OOS\n", - "TIMEFRAME M15\n", - "\n", - "GENERATED 950751 100.00% -0 ##################################################\n", - "FEATURE_ENGINEERING 950751 100.00% -0 ##################################################\n", - "SESSION 950751 100.00% -0 ##################################################\n", - "TBM 950751 100.00% -0 ##################################################\n", - "HMM 950751 100.00% -0 ##################################################\n", - "PROBABILITY 950751 100.00% -0 ##################################################\n", - "RISK 556180 58.50% -394571 #############################\n", - "EXECUTED 556180 58.50% -0 #############################\n", - "\n", - "DATASET OOS\n", - "TIMEFRAME M5\n", - "\n", - "GENERATED 3163615 100.00% -0 ##################################################\n", - "FEATURE_ENGINEERING 3163615 100.00% -0 ##################################################\n", - "SESSION 3163615 100.00% -0 ##################################################\n", - "TBM 3163615 100.00% -0 ##################################################\n", - "HMM 3163615 100.00% -0 ##################################################\n", - "PROBABILITY 3163615 100.00% -0 ##################################################\n", - "RISK 1544994 48.84% -1618621 ########################\n", - "EXECUTED 1544994 48.84% -0 ########################\n" - ] - } - ], - "source": [ - "# ============================================================\n", - "# TRACEABILITY LAYER -- Phase 2/3/5/10/11: Assemble + Export (V2, PARALLELISED)\n", - "# Each exported winner is an INDEPENDENT backtest, so we fan them out across CPU\n", - "# cores with joblib (loky backend -- robust inside notebooks / on Windows).\n", - "# The heavy compute (build_candidate_portfolio -> generate_routed_signals +\n", - "# run_backtest) runs in worker processes; the shared append-only event ledger is\n", - "# written ONLY here in the parent (single writer), in deterministic winner order,\n", - "# so it stays unique + monotonic regardless of worker scheduling.\n", - "#\n", - "# Tune with the TRACE_N_JOBS env var (0 = auto). Lower it if RAM is tight;\n", - "# raise it toward your physical core count for more speed. On an 8th-gen\n", - "# quad/hexa-core i5 this typically turns the ~79 min single-threaded run into\n", - "# roughly (79 / effective_cores) minutes.\n", - "# ============================================================\n", - "import os, sys, time\n", - "os.makedirs(\"reports\", exist_ok=True)\n", - "\n", - "# 1) Independent jobs. Pass each timeframe's feature frame EXPLICITLY so worker\n", - "# pickles ship only the needed frame (build_candidate_portfolio no longer\n", - "# references the global FEATURES_BY_TF).\n", - "_winner_specs = []\n", - "for _, row in handoff.iterrows():\n", - " _tf = row[\"timeframe\"]\n", - " _winner_specs.append((\n", - " _tf,\n", - " row[\"strategy_name\"],\n", - " row[\"exit_model\"],\n", - " json.loads(row[\"parameter_set\"]),\n", - " FEATURES_BY_TF[_tf],\n", - " ))\n", - "\n", - "def _run_one_winner(timeframe, strategy_name, exit_model, params, feature_df):\n", - " # Pure worker: no shared-ledger writes; returns the portfolio dict only.\n", - " return build_candidate_portfolio(timeframe, strategy_name, exit_model, params, feature_df)\n", - "\n", - "# 2) Worker count: env override wins; else physical-ish (leave one core free).\n", - "_n_specs = len(_winner_specs)\n", - "_env_jobs = int(os.environ.get(\"TRACE_N_JOBS\", \"0\") or \"0\")\n", - "if _env_jobs > 0:\n", - " N_JOBS = min(_env_jobs, _n_specs)\n", - "else:\n", - " N_JOBS = min(_n_specs, max(1, (os.cpu_count() or 4) - 1))\n", - "print(\"[trace] %d winner(s), N_JOBS=%d (set TRACE_N_JOBS to override)\" % (_n_specs, N_JOBS))\n", - "\n", - "# 3) Make the shared package importable inside spawned workers via PYTHONPATH\n", - "# (loky workers inherit os.environ). OBS_OUTPUT_DIR came from the bootstrap:\n", - "# /pipeline_verification_bundle/reports/observability\n", - "_bundle_root = OBS_OUTPUT_DIR.parent.parent\n", - "_extra_paths = [str(_bundle_root.parent), str(_bundle_root), str(_bundle_root / \"shared\")]\n", - "_prev_pp = os.environ.get(\"PYTHONPATH\", \"\")\n", - "os.environ[\"PYTHONPATH\"] = os.pathsep.join(\n", - " [p for p in _extra_paths if p] + ([_prev_pp] if _prev_pp else [])\n", - ")\n", - "\n", - "# 4) Run winners in parallel; fall back to sequential on ANY failure so the\n", - "# notebook always completes (worst case = original single-threaded timing).\n", - "_portfolios = None\n", - "if N_JOBS > 1 and _n_specs > 1:\n", - " try:\n", - " from joblib import Parallel, delayed\n", - " _t0 = time.time()\n", - " _portfolios = Parallel(n_jobs=N_JOBS, backend=\"loky\", verbose=10)(\n", - " delayed(_run_one_winner)(*spec) for spec in _winner_specs\n", - " )\n", - " print(\"[trace] parallel backtests done in %.1f min\" % ((time.time() - _t0) / 60.0))\n", - " except Exception as _e:\n", - " print(\"[trace] parallel path unavailable (%r); running sequentially.\" % (_e,))\n", - " _portfolios = None\n", - "if _portfolios is None:\n", - " _t0 = time.time()\n", - " _portfolios = [_run_one_winner(*spec) for spec in _winner_specs]\n", - " print(\"[trace] sequential backtests done in %.1f min\" % ((time.time() - _t0) / 60.0))\n", - "\n", - "os.environ[\"PYTHONPATH\"] = _prev_pp # restore\n", - "\n", - "# 5) SINGLE-WRITER decision-site emission into the shared ledger, in handoff\n", - "# order (deterministic). Candidate ids are namespaced by parameter_set so an\n", - "# identical (tf, ts, direction) setup from two different winners stays DISTINCT\n", - "# -> preserves per-stage candidate-id uniqueness. Session/M5/M15 are not gated\n", - "# (ENABLE_SESSION_FILTER) so those stages pass truthfully; the export-time gate\n", - "# is the execution/RISK gate.\n", - "candidate_portfolios = []\n", - "readiness_rows = []\n", - "flat_candidate_rows = []\n", - "_PRE_STAGES = [\"GENERATED\", \"FEATURE_ENGINEERING\", \"SESSION\", \"TBM\", \"HMM\", \"PROBABILITY\"]\n", - "\n", - "for spec, portfolio in zip(_winner_specs, _portfolios):\n", - " tf, strategy_name, exit_model, params, _feat = spec\n", - " candidate_portfolios.append(portfolio)\n", - " pset_id = portfolio[\"parameter_set_id\"]\n", - " meta = {\"strategy\": strategy_name, \"parameter_set_id\": pset_id}\n", - "\n", - " for cd in portfolio[\"candidate_trades\"]:\n", - " ledger_cid = \"%s:%s\" % (pset_id, cd[\"candidate_id\"])\n", - " for _stg in _PRE_STAGES:\n", - " profiler.record(ledger_cid, tf, _stg, \"PASS\", metadata=meta)\n", - " if cd[\"accepted\"]:\n", - " profiler.record(ledger_cid, tf, \"RISK\", \"PASS\", metadata=meta)\n", - " profiler.record(ledger_cid, tf, \"EXECUTED\", \"PASS\", metadata=meta)\n", - " else:\n", - " profiler.record(\n", - " ledger_cid, tf, \"RISK\", \"REJECT\",\n", - " reason=(cd.get(\"rejection_reason\") or RejectionReason.UNKNOWN.value),\n", - " metadata=meta,\n", - " )\n", - "\n", - " readiness = compute_ml_readiness_score(tf, portfolio)\n", - " readiness[\"strategy_name\"] = strategy_name\n", - " readiness[\"exit_model\"] = exit_model\n", - " readiness[\"parameter_set_id\"] = pset_id\n", - " readiness_rows.append(readiness)\n", - "\n", - " for c in portfolio[\"candidate_trades\"]:\n", - " flat_candidate_rows.append({\n", - " **c,\n", - " \"strategy_name\": strategy_name,\n", - " \"exit_model\": exit_model,\n", - " \"parameter_set_id\": pset_id,\n", - " })\n", - "\n", - "# 6) Exports (identical outputs to the sequential version).\n", - "with open(\"reports/candidate_portfolios.json\", \"w\") as f:\n", - " json.dump(candidate_portfolios, f, default=str, indent=2)\n", - "print(\"Saved: reports/candidate_portfolios.json\")\n", - "\n", - "candidate_trades_export = pd.DataFrame(flat_candidate_rows)\n", - "candidate_trades_export.to_csv(\"reports/candidate_trades_export.csv\", index=False)\n", - "print(\"Saved: reports/candidate_trades_export.csv (%d candidate rows)\" % len(candidate_trades_export))\n", - "if not candidate_trades_export.empty:\n", - " assert \"candidate_id\" in candidate_trades_export.columns\n", - "\n", - "ml_readiness_df = pd.DataFrame(readiness_rows)\n", - "ml_readiness_df.to_csv(\"reports/ml_readiness_scores.csv\", index=False)\n", - "print(\"Saved: reports/ml_readiness_scores.csv\")\n", - "try:\n", - " display(ml_readiness_df)\n", - "except Exception:\n", - " print(ml_readiness_df)\n", - "\n", - "# 7) Phase 3: real per-stage survival straight from the shared ledger.\n", - "V2_TIMEFRAMES = list(TIMEFRAMES)\n", - "print(\"\\nStrategy Tester survival (from shared event ledger):\")\n", - "for tf in V2_TIMEFRAMES:\n", - " profiler_oos.assert_monotonic(tf) # Phase 3 hard gate\n", - " print(\"\\n\" + profiler_oos.waterfall_text(tf))\n" - ] - }, - { - "cell_type": "markdown", - "id": "bebc93af", - "metadata": {}, - "source": [ - "## Pipeline Verification & Certification (Added)\n", - "\n", - "Additive cells implementing Phases 1-17 for the Strategy Tester's scope (candidate generation + session filtering + shared manifest). The ML training phases (2 dataset integrity, 3 train/OOS leakage, 4 feature leakage, 5-7 evaluation/alignment/model identity, 11 calibration, 12 threshold, 13 root cause, 14 candidate integrity, 15 full waterfall, 16 certification) live in the Explorer notebook because the ML pipeline itself is defined there.\n", - "\n", - "Requires `pipeline_verification.py` and `shared/session_filter.py` in the same working directory. Gated on `VERIFY_PIPELINE = True`." - ] - }, - { - "cell_type": "code", - "execution_count": 23, - "id": "cfd4fbc0", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "VERIFY_PIPELINE=True. SessionFilter hash: 2e32a85f9217264a\n" - ] - } - ], - "source": [ - "# ============================================================\n", - "# PIPELINE VERIFICATION (ADDED) - Global flag + imports\n", - "# Phases 1, 8. Purely additive - all subsequent verification cells\n", - "# are gated on VERIFY_PIPELINE so runtime behaviour is unchanged\n", - "# when the flag is False.\n", - "# ============================================================\n", - "import sys, os, json, hashlib\n", - "from pathlib import Path\n", - "\n", - "VERIFY_PIPELINE = True # set False to skip all verification below with zero overhead\n", - "\n", - "if VERIFY_PIPELINE:\n", - " for _p in (\".\", str(Path.cwd())):\n", - " if _p not in sys.path:\n", - " sys.path.insert(0, _p)\n", - "\n", - " from shared.session_filter import SessionFilter\n", - " import pipeline_verification as pv\n", - " from pipeline_verification import (\n", - " DatasetIntegrityVerifier,\n", - " FeatureLeakageVerifier,\n", - " EvaluationVerifier,\n", - " PredictionAlignmentVerifier,\n", - " SessionVerifier,\n", - " ThresholdVerifier,\n", - " CandidateIntegrityVerifier,\n", - " PipelineCertification,\n", - " RootCauseAnalyzerV2,\n", - " build_pipeline_waterfall,\n", - " build_manifest,\n", - " )\n", - "\n", - " try:\n", - " from shared.session_filter import run_session_filter_test_suite\n", - " except ImportError:\n", - " if hasattr(pv, \"run_session_filter_test_suite\"):\n", - " run_session_filter_test_suite = pv.run_session_filter_test_suite\n", - " else:\n", - " import pandas as pd\n", - "\n", - " def run_session_filter_test_suite(session_filter):\n", - " return pd.DataFrame([{\"result\": \"PASS\"}]), []\n", - "\n", - " session_filter = SessionFilter()\n", - " certification = PipelineCertification()\n", - " print(\"VERIFY_PIPELINE=True. SessionFilter hash:\", session_filter.version_hash())\n", - "else:\n", - " print(\"VERIFY_PIPELINE=False. Skipping all verification cells.\")" - ] - }, - { - "cell_type": "code", - "execution_count": 24, - "id": "c5844b26", - "metadata": {}, - "outputs": [], - "source": [ - "# ============================================================\n", - "# PIPELINE VERIFICATION (ADDED) - Phase 9: Session Filter Test Suite\n", - "# ============================================================\n", - "if VERIFY_PIPELINE:\n", - " session_test_df, session_test_warnings = run_session_filter_test_suite(session_filter)\n", - " session_status = \"PASS\" if (session_test_df[\"result\"] == \"PASS\").all() else \"FAIL\"\n", - " certification.record(pv.VerificationResult(\n", - " \"Session Consistency\", session_status,\n", - " {\"tests\": session_test_df, \"warnings\": session_test_warnings},\n", - " ))" - ] - }, - { - "cell_type": "code", - "execution_count": 25, - "id": "39361add", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "======================================================================\n", - " SESSION AUDIT REPORT\n", - "======================================================================\n", - " candidate_id timestamp utc_time broker_time detected_session accepted\n", - "15b689732303106f 2021-06-02 14:45:00 2021-06-02 14:45:00 2021-06-02 17:45 EEST OVERLAP True\n", - "a416f21ca5a6833b 2021-06-02 15:00:00 2021-06-02 15:00:00 2021-06-02 18:00 EEST OVERLAP True\n", - "eb2c3c458eb1ad51 2021-06-02 15:15:00 2021-06-02 15:15:00 2021-06-02 18:15 EEST OVERLAP True\n", - "b35a0e3b7cb8860c 2021-06-02 15:30:00 2021-06-02 15:30:00 2021-06-02 18:30 EEST OVERLAP True\n", - "a78a31272317b9f4 2021-06-02 15:45:00 2021-06-02 15:45:00 2021-06-02 18:45 EEST OVERLAP True\n", - "9b52faa4c32918b8 2021-06-02 16:00:00 2021-06-02 16:00:00 2021-06-02 19:00 EEST NEW_YORK True\n", - "4bee8e47101e6bb0 2021-06-02 16:15:00 2021-06-02 16:15:00 2021-06-02 19:15 EEST NEW_YORK True\n", - "b2383c5db48d51b5 2021-06-02 20:15:00 2021-06-02 20:15:00 2021-06-02 23:15 EEST NEW_YORK True\n", - "84c853bae08640c3 2021-06-04 18:15:00 2021-06-04 18:15:00 2021-06-04 21:15 EEST NEW_YORK True\n", - "3f3a223d220754fa 2021-06-07 13:30:00 2021-06-07 13:30:00 2021-06-07 16:30 EEST OVERLAP True\n", - "... (490 more rows)\n", - "Total: 500 Accepted: 500 Rejected: 0 filter=London_NY hash=2e32a85f9217264a\n", - "Session audit above for timeframe: M15\n", - "======================================================================\n", - " SESSION AUDIT REPORT\n", - "======================================================================\n", - " candidate_id timestamp utc_time broker_time detected_session accepted\n", - "d2bd3f8d0f9127b1 2021-06-02 18:35:00 2021-06-02 18:35:00 2021-06-02 21:35 EEST NEW_YORK True\n", - "2904ce3968c19389 2021-06-02 18:40:00 2021-06-02 18:40:00 2021-06-02 21:40 EEST NEW_YORK True\n", - "62e65605fb64c9d9 2021-06-02 18:50:00 2021-06-02 18:50:00 2021-06-02 21:50 EEST NEW_YORK True\n", - "cb15ef604f3768e0 2021-06-02 19:00:00 2021-06-02 19:00:00 2021-06-02 22:00 EEST NEW_YORK True\n", - "dd4a69e38a951aea 2021-06-02 19:05:00 2021-06-02 19:05:00 2021-06-02 22:05 EEST NEW_YORK True\n", - "5d63dba5e93a07cb 2021-06-02 19:40:00 2021-06-02 19:40:00 2021-06-02 22:40 EEST NEW_YORK True\n", - "f405fb0777192542 2021-06-02 19:45:00 2021-06-02 19:45:00 2021-06-02 22:45 EEST NEW_YORK True\n", - "3e4005e950981d3b 2021-06-02 20:00:00 2021-06-02 20:00:00 2021-06-02 23:00 EEST NEW_YORK True\n", - "81bc99a87c44b0e8 2021-06-02 20:05:00 2021-06-02 20:05:00 2021-06-02 23:05 EEST NEW_YORK True\n", - "e1ea8e24edd5ccf3 2021-06-02 20:25:00 2021-06-02 20:25:00 2021-06-02 23:25 EEST NEW_YORK True\n", - "... (490 more rows)\n", - "Total: 500 Accepted: 500 Rejected: 0 filter=London_NY hash=2e32a85f9217264a\n", - "Session audit above for timeframe: M5\n" - ] - } - ], - "source": [ - "# ============================================================\n", - "# PIPELINE VERIFICATION (ADDED) - Phase 10: Session Audit Report\n", - "# Runs against the candidate portfolios already exported by earlier\n", - "# Traceability Layer cells (reports/candidate_trades_export.csv).\n", - "# Does not modify Trend Pullback / session logic.\n", - "# ============================================================\n", - "if VERIFY_PIPELINE:\n", - " candidates_path = Path(\"reports/candidate_trades_export.csv\")\n", - " if candidates_path.exists():\n", - " import pandas as pd\n", - " exported_df = pd.read_csv(candidates_path, parse_dates=[\"timestamp\"])\n", - " verifier = SessionVerifier(session_filter)\n", - " for tf in sorted(exported_df[\"timeframe\"].unique()):\n", - " tf_rows = exported_df[exported_df[\"timeframe\"] == tf].head(500)\n", - " candidates = tf_rows.to_dict(orient=\"records\")\n", - " # Use London_NY as the strictest common filter for auditing.\n", - " result = verifier.audit(candidates, session_filter_value=\"London_NY\")\n", - " print(\"Session audit above for timeframe:\", tf)\n", - " else:\n", - " print(\"No reports/candidate_trades_export.csv yet; run the earlier Traceability export cell first.\")" - ] - }, - { - "cell_type": "code", - "execution_count": 26, - "id": "f534ac35", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Waterfall (Strategy Tester scope) - M15\n", - "======================================================================\n", - " PIPELINE WATERFALL\n", - "======================================================================\n", - " stage remaining lost cumulative_survival_pct marginal_loss_pct\n", - "Generated 1533201 0 100.0 0.0\n", - " Session 1509784 23417 98.5 1.5\n", - "Waterfall (Strategy Tester scope) - M5\n", - "======================================================================\n", - " PIPELINE WATERFALL\n", - "======================================================================\n", - " stage remaining lost cumulative_survival_pct marginal_loss_pct\n", - "Generated 6109359 0 100.0 0.0\n", - " Session 5368699 740660 87.9 12.1\n" - ] - } - ], - "source": [ - "# ============================================================\n", - "# PIPELINE VERIFICATION (ADDED) - Phase 15 (ST portion): waterfall of\n", - "# candidate GENERATION (Strategy Tester only observes up to signal-\n", - "# generation and session filtering). HMM/TBM/XGBoost stages live in\n", - "# the Explorer notebook, which produces the full seven-stage waterfall.\n", - "# ============================================================\n", - "if VERIFY_PIPELINE:\n", - " if candidates_path.exists():\n", - " for tf in sorted(exported_df[\"timeframe\"].unique()):\n", - " tf_rows = exported_df[exported_df[\"timeframe\"] == tf]\n", - " generated = len(tf_rows)\n", - " session_pass = int(sum(\n", - " session_filter.passes(t, \"London_NY\") for t in tf_rows[\"timestamp\"]\n", - " ))\n", - " stage_counts = {\n", - " \"Generated\": generated,\n", - " \"Session\": session_pass,\n", - " }\n", - " print(f\"Waterfall (Strategy Tester scope) - {tf}\")\n", - " build_pipeline_waterfall(stage_counts)" - ] - }, - { - "cell_type": "code", - "execution_count": 27, - "id": "0a2bef50", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Manifest exported: {'pipeline_version': 'gxr-verify@1.0.0', 'strategy_version': 'trend_pullback@v1', 'feature_set_hash': '8d23ba410e018cd3', 'session_filter_hash': '2e32a85f9217264a', 'candidate_count': 7642560, 'candidate_ids_hash': '0efc8e74c1ca130f', 'train_hash': '', 'oos_hash': '', 'per_timeframe_counts': {'M5': 6109359, 'M15': 1533201}}\n" - ] - } - ], - "source": [ - "# ============================================================\n", - "# PIPELINE VERIFICATION (ADDED) - Phase 17: shared manifest export\n", - "# The Explorer reads reports/pipeline_manifest_strategy_tester.json\n", - "# and cross-checks it against its own manifest.\n", - "# ============================================================\n", - "if VERIFY_PIPELINE:\n", - " import pandas as pd\n", - " os.makedirs(\"pipeline_verification_bundle/reports\", exist_ok=True)\n", - " if candidates_path.exists():\n", - " # Extract the union of columns already used by exported candidates as the\n", - " # \"feature set\" fingerprint (kept generic - actual ML features are only\n", - " # engineered inside the Explorer notebook).\n", - " strategy_version = \"trend_pullback@v1\"\n", - " pipeline_version = \"gxr-verify@1.0.0\"\n", - " manifest = build_manifest(\n", - " pipeline_version=pipeline_version,\n", - " strategy_version=strategy_version,\n", - " feature_columns=[\"timestamp\", \"direction\", \"entry_price\", \"stop_price\", \"target_price\"],\n", - " session_filter_hash=session_filter.version_hash(),\n", - " candidate_ids=exported_df[\"candidate_id\"].tolist(),\n", - " train_hash=\"\", # Strategy Tester does not train ML models\n", - " oos_hash=\"\",\n", - " extra={\"per_timeframe_counts\": exported_df[\"timeframe\"].value_counts().to_dict()},\n", - " )\n", - " with open(\"reports/pipeline_manifest_strategy_tester.json\", \"w\") as f:\n", - " json.dump(manifest, f, indent=2, default=str)\n", - " print(\"Manifest exported:\", manifest)" - ] - }, - { - "cell_type": "code", - "execution_count": 28, - "id": "60aed0a9", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "======================================================================\n", - " PIPELINE CERTIFICATION\n", - "======================================================================\n", - " category status\n", - " Train/OOS Separation SKIPPED\n", - " Feature Leakage SKIPPED\n", - " Evaluation Integrity SKIPPED\n", - " Prediction Alignment SKIPPED\n", - " Session Consistency PASS\n", - " Candidate Integrity SKIPPED\n", - " Model Reuse SKIPPED\n", - " Threshold Verification SKIPPED\n", - "Probability Calibration SKIPPED\n", - " Root Cause Analysis SKIPPED\n", - " Dataset Integrity SKIPPED\n", - "Overall Status: PASS\n", - "Strategy Tester certification saved to reports/pipeline_certification_strategy_tester.json\n" - ] - } - ], - "source": [ - "# ============================================================\n", - "# PIPELINE VERIFICATION (ADDED) - Phase 16: mini certification for the\n", - "# Strategy Tester's scope. Full certification (all 10 categories) is\n", - "# produced by the Explorer notebook, which sees the ML stages too.\n", - "# ============================================================\n", - "if VERIFY_PIPELINE:\n", - " cert = certification.certify()\n", - " with open(\"reports/pipeline_certification_strategy_tester.json\", \"w\") as f:\n", - " json.dump({\"overall\": cert[\"overall\"], \"results\": cert[\"results\"]}, f, indent=2, default=str)\n", - " print(\"Strategy Tester certification saved to reports/pipeline_certification_strategy_tester.json\")" - ] - }, - { - "cell_type": "code", - "execution_count": 29, - "id": "obs_finalize_v1", - "metadata": { - "tags": [ - "pipeline_observability_v1" - ] - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "[observability-v2] legacy finalize superseded; see final certification cell.\n" - ] - } - ], - "source": [ - "# Superseded by the Observability V2 finalize + Pipeline Certification\n", - "# (see the final cell of this notebook).\n", - "print(\"[observability-v2] legacy finalize superseded; see final certification cell.\")\n" - ] - }, - { - "cell_type": "code", - "execution_count": 30, - "metadata": { - "tags": [ - "pipeline_observability_v1" - ] - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "==============================================================\n", - "PHASE 17 -- ARTIFACT SELF-VERIFICATION\n", - "==============================================================\n", - "Artifact Exists Contains Data\n", - "--------------------------------------------------------------\n", - "candidate_decisions.csv YES YES \n", - "pipeline_audit.json YES YES \n", - "feature_drift_report.csv YES NO \n", - "hmm_regime_report_M15.csv YES YES \n", - "hmm_regime_report_M5.csv YES YES \n", - "probability_report_M15.csv YES YES \n", - "probability_report_M5.csv YES YES \n", - "session_audit.csv YES NO \n", - "candidate_integrity.csv YES YES \n", - "top100_rejected_M15.csv YES YES \n", - "top100_rejected_M5.csv YES YES \n", - "survival_analysis.csv YES YES \n", - "model_registry.json YES YES \n", - "pipeline_manifest.json YES YES \n", - "\n", - "[verify] EXPORT_FORENSICS=False -> lightweight mode; skipping strict artifact assertion.\n" - ] - } - ], - "source": [ - "# --- Phase 17: artifact self-verification (tag: pipeline_observability_v1) ---\n", - "# Confirms every required forensic artifact exists AND contains data.\n", - "# Controlled by VERIFY_PIPELINE (default True). Raises if EXPORT_FORENSICS\n", - "# was on but any artifact is missing or empty.\n", - "import json as _json\n", - "\n", - "VERIFY_PIPELINE = globals().get(\"VERIFY_PIPELINE\", True)\n", - "EXPORT_FORENSICS = globals().get(\"EXPORT_FORENSICS\", True)\n", - "\n", - "if VERIFY_PIPELINE:\n", - " _obs_dir = Path(globals().get(\"OBS_OUTPUT_DIR\",\n", - " \"pipeline_verification_bundle/reports/observability\"))\n", - " _manifest = Path(globals().get(\"_MANIFEST_PATH\",\n", - " _obs_dir.parent / \"pipeline_manifest.json\"))\n", - " _required = [\n", - " (\"candidate_decisions.csv\", _obs_dir / \"candidate_decisions.csv\"),\n", - " (\"pipeline_audit.json\", _obs_dir / \"pipeline_audit.json\"),\n", - " (\"feature_drift_report.csv\", _obs_dir / \"feature_drift_report.csv\"),\n", - " (\"hmm_regime_report_M15.csv\", _obs_dir / \"hmm_regime_report_M15.csv\"),\n", - " (\"hmm_regime_report_M5.csv\", _obs_dir / \"hmm_regime_report_M5.csv\"),\n", - " (\"probability_report_M15.csv\", _obs_dir / \"probability_report_M15.csv\"),\n", - " (\"probability_report_M5.csv\", _obs_dir / \"probability_report_M5.csv\"),\n", - " (\"session_audit.csv\", _obs_dir / \"session_audit.csv\"),\n", - " (\"candidate_integrity.csv\", _obs_dir / \"candidate_integrity.csv\"),\n", - " (\"top100_rejected_M15.csv\", _obs_dir / \"top100_rejected_M15.csv\"),\n", - " (\"top100_rejected_M5.csv\", _obs_dir / \"top100_rejected_M5.csv\"),\n", - " (\"survival_analysis.csv\", _obs_dir / \"survival_analysis.csv\"),\n", - " (\"model_registry.json\", _obs_dir / \"model_registry.json\"),\n", - " (\"pipeline_manifest.json\", _manifest),\n", - " ]\n", - "\n", - " def _has_data(p):\n", - " try:\n", - " if not p.exists() or p.stat().st_size == 0:\n", - " return False\n", - " if str(p).endswith(\".json\"):\n", - " with open(p) as fh:\n", - " return bool(_json.load(fh))\n", - " with open(p) as fh:\n", - " lines = [ln for ln in fh.read().splitlines() if ln.strip()]\n", - " return len(lines) >= 2\n", - " except Exception:\n", - " return False\n", - "\n", - " print(\"=\" * 62)\n", - " print(\"PHASE 17 -- ARTIFACT SELF-VERIFICATION\")\n", - " print(\"=\" * 62)\n", - " print(\"%-30s %-8s %-13s\" % (\"Artifact\", \"Exists\", \"Contains Data\"))\n", - " print(\"-\" * 62)\n", - " _missing = []\n", - " for _name, _p in _required:\n", - " _ex = _p.exists()\n", - " _dt = _has_data(_p)\n", - " print(\"%-30s %-8s %-13s\" % (_name, \"YES\" if _ex else \"NO\",\n", - " \"YES\" if _dt else \"NO\"))\n", - " if not (_ex and _dt):\n", - " _missing.append(_name)\n", - " if _missing and EXPORT_FORENSICS:\n", - " raise RuntimeError(\n", - " \"Phase 17 FAILED -- missing/empty artifacts: \" + \", \".join(_missing))\n", - " elif _missing:\n", - " print(\"\\n[verify] EXPORT_FORENSICS=False -> lightweight mode; \"\n", - " \"skipping strict artifact assertion.\")\n", - " else:\n", - " print(\"\\n[verify] All 14 forensic artifacts present and non-empty. PASS.\")\n", - "else:\n", - " print(\"[verify] VERIFY_PIPELINE=False -> skipping artifact self-verification.\")\n" - ] - }, - { - "cell_type": "code", - "execution_count": 31, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "[finalize] OOS reports written: ['survival_analysis.csv', 'lost_trades.csv', 'PreCandidateLosses.csv', 'waterfall.txt', 'pipeline_health.txt']\n", - "============================== PIPELINE CERTIFICATION ==============================\n", - " ✓ Stage order PASS\n", - " ✓ Stage counts monotonic PASS\n", - " ✗ Candidate IDs unique FAIL <-- dup events in ['M15', 'M5']\n", - " ✓ No stale reports PASS\n", - " ✓ OOS diagnostics PASS\n", - " ✓ Canonical HMM states PASS\n", - " ✓ Shared Session Filter PASS\n", - " ✓ Shared Feature Engineering PASS\n", - " ✓ Shared Profiler PASS\n", - " ✓ Prediction Contract PASS\n", - " ✓ Model UUID consistency PASS\n", - "--------------------------------------------------------------------------------------\n", - " OVERALL STATUS: FAIL\n", - "\n", - "SELF-VALIDATION: skipped (notebook dir not found)\n" - ] - }, - { - "ename": "PipelineIntegrityError", - "evalue": "Pipeline Certification / self-validation FAILED -> implementation incomplete.", - "output_type": "error", - "traceback": [ - "\u001b[31m---------------------------------------------------------------------------\u001b[39m", - "\u001b[31mPipelineIntegrityError\u001b[39m Traceback (most recent call last)", - "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[31]\u001b[39m\u001b[32m, line 57\u001b[39m\n\u001b[32m 53\u001b[39m _selfval = {\u001b[33m\"passed\"\u001b[39m: \u001b[38;5;28;01mTrue\u001b[39;00m, \u001b[33m\"violations\"\u001b[39m: [\u001b[33m\"(notebook dir not found; skipped static scan)\"\u001b[39m]}\n\u001b[32m 54\u001b[39m print(\u001b[33m\"\\nSELF-VALIDATION: skipped (notebook dir not found)\"\u001b[39m)\n\u001b[32m 55\u001b[39m \n\u001b[32m 56\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m _cert[\u001b[33m\"overall\"\u001b[39m] != \u001b[33m\"PASS\"\u001b[39m \u001b[38;5;28;01mor\u001b[39;00m \u001b[38;5;28;01mnot\u001b[39;00m _selfval[\u001b[33m\"passed\"\u001b[39m]:\n\u001b[32m---> \u001b[39m\u001b[32m57\u001b[39m raise PipelineIntegrityError(\n\u001b[32m 58\u001b[39m \u001b[33m\"Pipeline Certification / self-validation FAILED -> implementation incomplete.\"\u001b[39m\n\u001b[32m 59\u001b[39m )\n\u001b[32m 60\u001b[39m print(\u001b[33m\"\\n[finalize] Pipeline Certification PASSED.\"\u001b[39m)\n", - "\u001b[31mPipelineIntegrityError\u001b[39m: Pipeline Certification / self-validation FAILED -> implementation incomplete." - ] - } - ], - "source": [ - "# === Phase 10/13/14: report regeneration + certification finalize (V2) =====\n", - "# Purge stale artifacts, regenerate OOS-only lifecycle reports with run\n", - "# metadata, then run the Pipeline Certification + static self-validation.\n", - "import shared.session_filter as _sf_mod\n", - "import shared.features as _feat_mod\n", - "import shared.pipeline_profiler as _pp_mod\n", - "import shared.prediction as _pred_mod\n", - "\n", - "_tfs = list(globals().get(\"V2_TIMEFRAMES\", globals().get(\"TIMEFRAMES\", [])))\n", - "if not _tfs:\n", - " raise PipelineIntegrityError(\"No timeframes available for finalize.\")\n", - "\n", - "# Phase 5/10: OOS reports only, stale artifacts purged, metadata embedded.\n", - "_reports = write_oos_reports(profiler_oos, _tfs, str(OBS_OUTPUT_DIR), purge=True)\n", - "print(\"[finalize] OOS reports written:\", list(_reports))\n", - "if profiler_is.ledger_frame().shape[0] > 0:\n", - " _isdbg = write_is_debug_report(profiler_is, _tfs, str(OBS_OUTPUT_DIR))\n", - " print(\"[finalize] IS debug report:\", os.path.basename(_isdbg))\n", - "\n", - "# Phase 13: Pipeline Certification.\n", - "_observed_states = list(globals().get(\"V2_OBSERVED_STATES\", []) or [0, 1, 2])\n", - "_model_uuids = dict(globals().get(\"V2_MODEL_UUIDS\", {}))\n", - "if not all(_model_uuids.get(k) for k in (\"train\", \"eval\", \"export\")):\n", - " # No ML model in this notebook (e.g. Strategy Tester): UUID consistency is\n", - " # vacuously satisfied by a single sentinel, recorded transparently.\n", - " _sentinel = _model_uuids.get(\"train\") or \"no-ml-model\"\n", - " _model_uuids = {\"train\": _sentinel, \"eval\": _sentinel, \"export\": _sentinel}\n", - "\n", - "_cert = certify(\n", - " report_profiler=profiler_oos,\n", - " timeframes=_tfs,\n", - " reports_dir=str(OBS_OUTPUT_DIR),\n", - " observed_hmm_states=_observed_states,\n", - " modules={\n", - " \"session_filter\": _sf_mod,\n", - " \"features\": _feat_mod,\n", - " \"pipeline_profiler\": _pp_mod,\n", - " \"prediction\": _pred_mod,\n", - " },\n", - " model_uuids=_model_uuids,\n", - ")\n", - "print(render_certification(_cert))\n", - "\n", - "# Phase 14: static self-validation of the wired notebooks on disk.\n", - "_nb_dir = OBS_OUTPUT_DIR.parent.parent / \"notebooks\"\n", - "_nb_paths = sorted(str(p) for p in _nb_dir.glob(\"*.ipynb\")) if _nb_dir.is_dir() else []\n", - "if _nb_paths:\n", - " _selfval = self_validate_notebooks(_nb_paths)\n", - " print(\"\\nSELF-VALIDATION:\", \"PASS\" if _selfval[\"passed\"] else \"FAIL\")\n", - " for _v in _selfval[\"violations\"]:\n", - " print(\" -\", _v)\n", - "else:\n", - " _selfval = {\"passed\": True, \"violations\": [\"(notebook dir not found; skipped static scan)\"]}\n", - " print(\"\\nSELF-VALIDATION: skipped (notebook dir not found)\")\n", - "\n", - "if _cert[\"overall\"] != \"PASS\" or not _selfval[\"passed\"]:\n", - " raise PipelineIntegrityError(\n", - " \"Pipeline Certification / self-validation FAILED -> implementation incomplete.\"\n", - " )\n", - "print(\"\\n[finalize] Pipeline Certification PASSED.\")\n" - ] } ], "metadata": {