From 9f32948cafb5e7694ea0c5005dd2668c6f2b7ed5 Mon Sep 17 00:00:00 2001 From: Brent Neale Date: Fri, 20 Feb 2026 23:28:17 +1000 Subject: [PATCH] Update correlation analysis for 4-strategy portfolio Avg pairwise PnL correlation: 0.028 (excellent diversification). S7/S3 signal overlap on GBP_JPY only 6.5%. Portfolio: 289 trades, PF=1.37, Sharpe=1.29, +1,732 pips ($+23,424), max DD -21%. Co-Authored-By: Claude Opus 4.6 --- results/phase2/correlation_analysis.json | 74 ++++-- src/run_correlation_analysis.py | 322 +++++++++++++++-------- 2 files changed, 274 insertions(+), 122 deletions(-) diff --git a/results/phase2/correlation_analysis.json b/results/phase2/correlation_analysis.json index db18cc4..9befee9 100644 --- a/results/phase2/correlation_analysis.json +++ b/results/phase2/correlation_analysis.json @@ -1,23 +1,65 @@ { - "S7_S3_overlap": { - "overlap_count": 21, - "same_dir": 21, + "S7_S3_signal_overlap": { + "overlap_count": 5, + "same_dir": 5, "opposite_dir": 0, - "ratio": 0.169 + "ratio": 0.065 }, - "S9_S9F_temporal": { - "s9_trade_days": 275, - "s9f_trade_days": 65, - "shared_trade_days": 33, - "temporal_overlap_ratio": 0.12 + "pnl_correlation": { + "S7_Tight_vs_S9_Filtered": -0.054, + "S7_Tight_vs_S3": 0.072, + "S7_Tight_vs_S8_OB": 0.055, + "S9_Filtered_vs_S3": -0.05, + "S9_Filtered_vs_S8_OB": 0.036, + "S3_vs_S8_OB": 0.109 + }, + "avg_pairwise_correlation": 0.028, + "temporal_overlap": { + "S7_Tight_vs_S9_Filtered": { + "trade_days_a": 38, + "trade_days_b": 50, + "shared_days": 5, + "jaccard_index": 0.06 + }, + "S7_Tight_vs_S3": { + "trade_days_a": 38, + "trade_days_b": 111, + "shared_days": 6, + "jaccard_index": 0.042 + }, + "S7_Tight_vs_S8_OB": { + "trade_days_a": 38, + "trade_days_b": 51, + "shared_days": 2, + "jaccard_index": 0.023 + }, + "S9_Filtered_vs_S3": { + "trade_days_a": 50, + "trade_days_b": 111, + "shared_days": 11, + "jaccard_index": 0.073 + }, + "S9_Filtered_vs_S8_OB": { + "trade_days_a": 50, + "trade_days_b": 51, + "shared_days": 7, + "jaccard_index": 0.074 + }, + "S3_vs_S8_OB": { + "trade_days_a": 111, + "trade_days_b": 51, + "shared_days": 11, + "jaccard_index": 0.073 + } }, "portfolio": { - "total_trades": 694, - "win_rate_pct": 53.3, - "profit_factor": 0.99, - "total_pnl_pips": -175.1, - "total_pnl_dollars": -23036.66, - "max_drawdown_pct": -47.8, - "sharpe_ratio": -0.71 + "total_trades": 289, + "win_rate_pct": 54.0, + "profit_factor": 1.37, + "total_pnl_pips": 1732.1, + "total_pnl_dollars": 23424.02, + "max_drawdown_pct": -21.06, + "sharpe_ratio": 1.29, + "expectancy_pips": 5.99 } } \ No newline at end of file diff --git a/src/run_correlation_analysis.py b/src/run_correlation_analysis.py index ac18e4f..4031b02 100644 --- a/src/run_correlation_analysis.py +++ b/src/run_correlation_analysis.py @@ -1,12 +1,17 @@ """ -Phase 2 — Correlation Analysis (Step 8). +Phase 2 — Correlation Analysis (4-Strategy Portfolio). -For strategies sharing a pair (S7_Tight + S3 on GBP_JPY): -- Compute signal overlap and simultaneous position frequency. -- Combined equity curve analysis. +Analyzes diversification across the portfolio: + S7_Tight / GBP_JPY / H1 — Liquidity Sweep + S9_Filtered/ GBP_AUD / H1 — London Session + S3 / GBP_JPY / H1 — Key Level Breakout + S8_OB / GBP_USD / M15 — Order Block Retest -Also compute portfolio-level metrics: combined PF, combined max DD, -Sharpe of combined equity curve. +Computes: + 1. Signal overlap: S7 vs S3 (same pair GBP_JPY) + 2. Daily PnL correlation matrix (all strategy pairs) + 3. Temporal clustering (same-day trade entries) + 4. Portfolio-level metrics and combined equity curve Output: results/phase2/correlation_analysis.json """ @@ -22,25 +27,31 @@ from src.backtester.engine import Backtester # Strategy imports from src.strategies_pkg.s7_liquidity_sweep import S7_Liquidity_Sweep from src.strategies_pkg.s9_london_session import S9_London_Session -from src.strategies_pkg.s4f_ema_ribbon import S4F_EMA_Ribbon from src.strategies_pkg.s3_key_level_breakout import S3_KeyLevel_Breakout +from src.strategies_pkg.s8_order_block import S8_Order_Block PROCESSED_DIR = os.path.join(os.path.dirname(os.path.dirname(__file__)), "data", "processed") RESULTS_DIR = os.path.join(os.path.dirname(os.path.dirname(__file__)), "results", "phase2") os.makedirs(RESULTS_DIR, exist_ok=True) -# All Phase 2 strategies + +def _s8_tuned(): + s = S8_Order_Block() + s.DISPLACEMENT_ATR = 2.5 + s.TP1_ATR_MULT = 2.0 + s.OB_RETEST_WINDOW = 40 + return s + + CONFIGS = [ - {"name": "S7_Tight", "pair": "GBP_JPY", "tf": "H1", + {"name": "S7_Tight", "pair": "GBP_JPY", "tf": "H1", "htf_tf": "H1", "factory": lambda: S7_Liquidity_Sweep()}, - {"name": "S9", "pair": "GBP_USD", "tf": "H1", - "factory": lambda: S9_London_Session()}, - {"name": "S9_Filtered", "pair": "GBP_AUD", "tf": "H1", + {"name": "S9_Filtered", "pair": "GBP_AUD", "tf": "H1", "htf_tf": "H1", "factory": lambda: S9_London_Session(pair="GBP_AUD", filtered=True)}, - {"name": "S4F", "pair": "EUR_AUD", "tf": "M15", - "factory": lambda: S4F_EMA_Ribbon()}, - {"name": "S3", "pair": "GBP_JPY", "tf": "H1", + {"name": "S3", "pair": "GBP_JPY", "tf": "H1", "htf_tf": "H1", "factory": lambda: S3_KeyLevel_Breakout()}, + {"name": "S8_OB", "pair": "GBP_USD", "tf": "M15", "htf_tf": "H1", + "factory": lambda: _s8_tuned()}, ] @@ -53,15 +64,23 @@ def load_data(pair, tf): return compute_all_indicators(df) -def run_backtest(cfg): +def run_backtest(cfg, data_cache): """Run backtest for a config, return trade log and equity curve.""" pair = cfg["pair"] tf = cfg["tf"] - data = load_data(pair, tf) + htf_tf = cfg["htf_tf"] + + cache_key = f"{pair}_{tf}" + if cache_key not in data_cache: + data_cache[cache_key] = load_data(pair, tf) + data = data_cache[cache_key] if data is None: return None, None, None - htf_data = data.copy() if tf == "H1" else load_data(pair, "H1") + htf_cache_key = f"{pair}_{htf_tf}" + if htf_cache_key not in data_cache: + data_cache[htf_cache_key] = load_data(pair, htf_tf) + htf_data = data_cache[htf_cache_key] if htf_tf != tf else data strategy = cfg["factory"]() bt = Backtester(data=data, strategy=strategy, pair=pair, @@ -72,15 +91,8 @@ def run_backtest(cfg): return report, trade_log, eq_curve -def compute_signal_overlap(log_a, log_b, pair): - """Compute signal overlap between two strategies on the same pair. - - Returns: - - overlap_count: trades that are open at the same time - - same_direction_count: overlapping trades in same direction - - opposite_direction_count: overlapping trades in opposite direction - - overlap_ratio: fraction of trades that overlap - """ +def compute_signal_overlap(log_a, log_b): + """Compute signal overlap between two strategies (time-based).""" if log_a.empty or log_b.empty: return {"overlap_count": 0, "same_dir": 0, "opposite_dir": 0, "ratio": 0} @@ -96,7 +108,6 @@ def compute_signal_overlap(log_a, log_b, pair): b_start = pd.Timestamp(trade_b["timestamp"]) b_end = pd.Timestamp(trade_b["exit_time"]) if pd.notna(trade_b.get("exit_time")) else b_start - # Check if time ranges overlap if a_start <= b_end and b_start <= a_end: overlap += 1 if trade_a["signal_direction"] == trade_b["signal_direction"]: @@ -115,29 +126,61 @@ def compute_signal_overlap(log_a, log_b, pair): } -def compute_combined_equity(eq_curves: list[pd.DataFrame]) -> pd.DataFrame: - """Combine equity curves from multiple strategies into portfolio equity.""" - combined = None - for eq in eq_curves: - if eq is None or eq.empty: +def compute_daily_pnl_series(trade_logs): + """Build daily PnL series per strategy for correlation analysis.""" + daily_pnl = {} + for name, log in trade_logs.items(): + if log is None or log.empty: continue - eq = eq.set_index("timestamp")["equity"] - # Convert to returns relative to starting equity - returns = eq - 100_000.0 - if combined is None: - combined = returns - else: - combined = combined.add(returns, fill_value=0) + df = log.copy() + df["date"] = pd.to_datetime(df["timestamp"]).dt.date + daily = df.groupby("date")["pnl_pips"].sum() + daily_pnl[name] = daily + return daily_pnl - if combined is None: + +def compute_pnl_correlation(daily_pnl): + """Compute pairwise correlation of daily PnL between strategies.""" + if len(daily_pnl) < 2: return pd.DataFrame() - # Add back starting equity (100k per slot, or just use combined returns) - combined = combined + 100_000.0 - return combined.reset_index() + combined = pd.DataFrame(daily_pnl) + combined = combined.fillna(0) + + return combined.corr() -def compute_portfolio_metrics(all_trade_logs: list[pd.DataFrame]) -> dict: +def compute_temporal_overlap(trade_logs): + """Compute pairwise temporal overlap (same-day entries) between all strategies.""" + names = list(trade_logs.keys()) + results = {} + + for i in range(len(names)): + for j in range(i + 1, len(names)): + a_name, b_name = names[i], names[j] + log_a = trade_logs[a_name] + log_b = trade_logs[b_name] + + if log_a is None or log_a.empty or log_b is None or log_b.empty: + continue + + dates_a = set(pd.to_datetime(log_a["timestamp"]).dt.date) + dates_b = set(pd.to_datetime(log_b["timestamp"]).dt.date) + shared = dates_a & dates_b + union = dates_a | dates_b + + key = f"{a_name}_vs_{b_name}" + results[key] = { + "trade_days_a": len(dates_a), + "trade_days_b": len(dates_b), + "shared_days": len(shared), + "jaccard_index": round(len(shared) / len(union), 3) if union else 0, + } + + return results + + +def compute_portfolio_metrics(all_trade_logs): """Compute portfolio-level metrics from combined trade logs.""" combined = pd.concat([log for log in all_trade_logs if not log.empty], ignore_index=True) @@ -153,18 +196,20 @@ def compute_portfolio_metrics(all_trade_logs: list[pd.DataFrame]) -> dict: gross_loss = abs(losses["pnl_pips"].sum()) if len(losses) > 0 else 0 pf = gross_profit / gross_loss if gross_loss > 0 else float("inf") total_pnl = combined["pnl_pips"].sum() + expectancy = total_pnl / n if n > 0 else 0 # Combined max drawdown - cum_pnl = combined.sort_values("timestamp")["pnl_dollars"].cumsum() + sorted_trades = combined.sort_values("timestamp") + cum_pnl = sorted_trades["pnl_dollars"].cumsum() peak = cum_pnl.cummax() dd = cum_pnl - peak max_dd = dd.min() max_dd_pct = max_dd / 100_000 * 100 if max_dd < 0 else 0 - # Sharpe ratio - daily_pnl = combined.copy() - daily_pnl["date"] = pd.to_datetime(daily_pnl["timestamp"]).dt.date - daily = daily_pnl.groupby("date")["pnl_dollars"].sum() + # Sharpe ratio from daily PnL + sorted_trades = sorted_trades.copy() + sorted_trades["date"] = pd.to_datetime(sorted_trades["timestamp"]).dt.date + daily = sorted_trades.groupby("date")["pnl_dollars"].sum() if len(daily) > 1 and daily.std() > 0: sharpe = (daily.mean() / daily.std()) * np.sqrt(252) else: @@ -178,92 +223,137 @@ def compute_portfolio_metrics(all_trade_logs: list[pd.DataFrame]) -> dict: "total_pnl_dollars": round(combined["pnl_dollars"].sum(), 2), "max_drawdown_pct": round(max_dd_pct, 2), "sharpe_ratio": round(sharpe, 2), + "expectancy_pips": round(expectancy, 2), } def main(): + t0 = time.time() + print(f"{'='*80}") - print("PHASE 2 — CORRELATION ANALYSIS") + print("PHASE 2 — CORRELATION ANALYSIS (4-Strategy Portfolio)") print(f"{'='*80}") results = {} trade_logs = {} eq_curves = {} + data_cache = {} # Run all backtests for cfg in CONFIGS: name = cfg["name"] pair = cfg["pair"] - print(f"\nRunning {name} / {pair}...", end=" ", flush=True) - t0 = time.time() - report, log, eq = run_backtest(cfg) - elapsed = time.time() - t0 + tf = cfg["tf"] + print(f"\n Running {name} / {pair} ({tf})...", end=" ", flush=True) + t1 = time.time() + report, log, eq = run_backtest(cfg, data_cache) + elapsed = time.time() - t1 n_trades = len(log) if log is not None and not log.empty else 0 print(f"{n_trades} trades ({elapsed:.0f}s)") trade_logs[name] = log eq_curves[name] = eq - # --- Signal Overlap: S7_Tight vs S3 on GBP_JPY --- - print(f"\n{'#'*60}") - print("# Signal Overlap: S7_Tight vs S3 on GBP_JPY") - print(f"{'#'*60}") + # ===================================================================== + # 1. Signal Overlap: S7_Tight vs S3 (both on GBP_JPY) + # ===================================================================== + print(f"\n{'#'*70}") + print("# 1. SIGNAL OVERLAP: S7_Tight vs S3 (GBP_JPY)") + print(f"{'#'*70}") log_s7 = trade_logs.get("S7_Tight", pd.DataFrame()) log_s3 = trade_logs.get("S3", pd.DataFrame()) if not log_s7.empty and not log_s3.empty: - overlap = compute_signal_overlap(log_s7, log_s3, "GBP_JPY") - results["S7_S3_overlap"] = overlap + overlap = compute_signal_overlap(log_s7, log_s3) + results["S7_S3_signal_overlap"] = overlap - print(f" S7 trades: {len(log_s7)}") - print(f" S3 trades: {len(log_s3)}") + print(f" S7_Tight trades: {len(log_s7)}") + print(f" S3 trades: {len(log_s3)}") print(f" Overlapping periods: {overlap['overlap_count']}") - print(f" Same direction: {overlap['same_dir']}") + print(f" Same direction: {overlap['same_dir']}") print(f" Opposite direction: {overlap['opposite_dir']}") print(f" Overlap ratio: {overlap['ratio']:.1%}") if overlap['ratio'] < 0.15: - print(" => LOW overlap: Good diversification!") + print(" => LOW overlap: Good diversification within GBP_JPY") elif overlap['ratio'] < 0.30: - print(" => MODERATE overlap: Some clustering.") + print(" => MODERATE overlap: Some clustering on GBP_JPY") else: - print(" => HIGH overlap: Significant clustering risk.") + print(" => HIGH overlap: Significant clustering risk on GBP_JPY") else: print(" Insufficient data for overlap analysis.") - # --- S9 vs S9_Filtered (different pairs, should be independent) --- - print(f"\n{'#'*60}") - print("# Independence Check: S9 (GBP_USD) vs S9_Filtered (GBP_AUD)") - print(f"{'#'*60}") + # ===================================================================== + # 2. Daily PnL Correlation Matrix + # ===================================================================== + print(f"\n{'#'*70}") + print("# 2. DAILY PnL CORRELATION MATRIX") + print(f"{'#'*70}") - log_s9 = trade_logs.get("S9", pd.DataFrame()) - log_s9f = trade_logs.get("S9_Filtered", pd.DataFrame()) + daily_pnl = compute_daily_pnl_series(trade_logs) + corr_matrix = compute_pnl_correlation(daily_pnl) - if not log_s9.empty and not log_s9f.empty: - # Check temporal clustering (same-day entries) - s9_dates = set(pd.to_datetime(log_s9["timestamp"]).dt.date) - s9f_dates = set(pd.to_datetime(log_s9f["timestamp"]).dt.date) - shared_dates = s9_dates & s9f_dates - temporal_overlap = len(shared_dates) / max(len(s9_dates), 1) - - results["S9_S9F_temporal"] = { - "s9_trade_days": len(s9_dates), - "s9f_trade_days": len(s9f_dates), - "shared_trade_days": len(shared_dates), - "temporal_overlap_ratio": round(temporal_overlap, 3), + if not corr_matrix.empty: + results["pnl_correlation"] = { + f"{a}_vs_{b}": round(corr_matrix.loc[a, b], 3) + for i, a in enumerate(corr_matrix.index) + for j, b in enumerate(corr_matrix.columns) + if j > i } - print(f" S9 trade days: {len(s9_dates)}") - print(f" S9_Filtered trade days: {len(s9f_dates)}") - print(f" Shared trade days: {len(shared_dates)}") - print(f" Temporal overlap: {temporal_overlap:.1%}") + print(f"\n {'':>14}", end="") + for name in corr_matrix.columns: + print(f" {name:>12}", end="") + print() + + for row_name in corr_matrix.index: + print(f" {row_name:>14}", end="") + for col_name in corr_matrix.columns: + val = corr_matrix.loc[row_name, col_name] + print(f" {val:>12.3f}", end="") + print() + + # Average pairwise correlation + pairs = [] + for i, a in enumerate(corr_matrix.index): + for j, b in enumerate(corr_matrix.columns): + if j > i: + pairs.append(corr_matrix.loc[a, b]) + avg_corr = np.mean(pairs) if pairs else 0 + results["avg_pairwise_correlation"] = round(avg_corr, 3) + + print(f"\n Average pairwise correlation: {avg_corr:.3f}") + if avg_corr < 0.20: + print(" => LOW correlation: Excellent diversification") + elif avg_corr < 0.40: + print(" => MODERATE correlation: Decent diversification") + else: + print(" => HIGH correlation: Limited diversification benefit") else: print(" Insufficient data.") - # --- Portfolio Metrics --- - print(f"\n{'#'*60}") - print("# Portfolio-Level Metrics (All 5 Strategies Combined)") - print(f"{'#'*60}") + # ===================================================================== + # 3. Temporal Overlap (Same-Day Entry Clustering) + # ===================================================================== + print(f"\n{'#'*70}") + print("# 3. TEMPORAL OVERLAP (Same-Day Entries)") + print(f"{'#'*70}") + + temporal = compute_temporal_overlap(trade_logs) + results["temporal_overlap"] = temporal + + print(f"\n {'Pair':<30} {'Days A':>7} {'Days B':>7} {'Shared':>7} {'Jaccard':>8}") + print(f" {'-'*65}") + for key, val in temporal.items(): + print(f" {key:<30} {val['trade_days_a']:>7} {val['trade_days_b']:>7} " + f"{val['shared_days']:>7} {val['jaccard_index']:>7.3f}") + + # ===================================================================== + # 4. Portfolio Metrics + # ===================================================================== + print(f"\n{'#'*70}") + print("# 4. PORTFOLIO METRICS (All 4 Strategies Combined)") + print(f"{'#'*70}") all_logs = [log for log in trade_logs.values() if log is not None and not log.empty] @@ -271,27 +361,32 @@ def main(): portfolio = compute_portfolio_metrics(all_logs) results["portfolio"] = portfolio - print(f" Total trades: {portfolio['total_trades']}") - print(f" Win rate: {portfolio['win_rate_pct']}%") - print(f" Profit factor: {portfolio['profit_factor']}") - print(f" Total PnL (pips): {portfolio['total_pnl_pips']:+.1f}") - print(f" Total PnL ($): {portfolio['total_pnl_dollars']:+,.2f}") - print(f" Max drawdown: {portfolio['max_drawdown_pct']:.2f}%") - print(f" Sharpe ratio: {portfolio['sharpe_ratio']:.2f}") + print(f" Total trades: {portfolio['total_trades']}") + print(f" Win rate: {portfolio['win_rate_pct']}%") + print(f" Profit factor: {portfolio['profit_factor']}") + print(f" Expectancy: {portfolio['expectancy_pips']:+.2f} pips/trade") + print(f" Total PnL: {portfolio['total_pnl_pips']:+.1f} pips " + f"(${portfolio['total_pnl_dollars']:+,.2f})") + print(f" Max drawdown: {portfolio['max_drawdown_pct']:.2f}%") + print(f" Sharpe ratio: {portfolio['sharpe_ratio']:.2f}") - # --- Per-Strategy Summary --- + # ===================================================================== + # 5. Per-Strategy Summary + # ===================================================================== print(f"\n{'='*80}") print("STRATEGY SUMMARY") print(f"{'='*80}") - print(f"{'Strategy':<16} {'Pair':<10} {'Trades':>6} {'WR%':>6} {'PF':>6} {'PnL(p)':>9}") - print(f"{'-'*60}") + print(f" {'Strategy':<14} {'Pair':<10} {'TF':<4} {'Trades':>6} {'WR%':>6} " + f"{'PF':>6} {'PnL(p)':>9} {'Exp':>7}") + print(f" {'-'*70}") for cfg in CONFIGS: name = cfg["name"] pair = cfg["pair"] + tf = cfg["tf"] log = trade_logs.get(name, pd.DataFrame()) if log.empty: - print(f"{name:<16} {pair:<10} {'N/A':>6}") + print(f" {name:<14} {pair:<10} {tf:<4} {'N/A':>6}") continue n = len(log) wins = log[log["win"] == True] @@ -300,14 +395,29 @@ def main(): gl = abs(log[log["win"] == False]["pnl_pips"].sum()) pf = gp / gl if gl > 0 else 0 pnl = log["pnl_pips"].sum() - print(f"{name:<16} {pair:<10} {n:>6} {wr:>5.1f}% {pf:>5.2f} {pnl:>+8.1f}") + exp = pnl / n if n > 0 else 0 + print(f" {name:<14} {pair:<10} {tf:<4} {n:>6} {wr:>5.1f}% " + f"{pf:>5.2f} {pnl:>+8.1f} {exp:>+6.2f}") # Save out_path = os.path.join(RESULTS_DIR, "correlation_analysis.json") + + def json_default(obj): + if isinstance(obj, (np.integer,)): + return int(obj) + if isinstance(obj, (np.floating,)): + return float(obj) + if isinstance(obj, (np.bool_,)): + return bool(obj) + return str(obj) + with open(out_path, "w") as f: - json.dump(results, f, indent=2, default=str) + json.dump(results, f, indent=2, default=json_default) print(f"\nResults saved: {out_path}") + elapsed = time.time() - t0 + print(f"Total runtime: {elapsed:.1f}s") + if __name__ == "__main__": main()