Update correlation analysis for 4-strategy portfolio

Avg pairwise PnL correlation: 0.028 (excellent diversification).
S7/S3 signal overlap on GBP_JPY only 6.5%. Portfolio: 289 trades,
PF=1.37, Sharpe=1.29, +1,732 pips ($+23,424), max DD -21%.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Brent Neale
2026-02-20 23:28:17 +10:00
parent 98cc843d14
commit 9f32948caf
2 changed files with 274 additions and 122 deletions
+58 -16
View File
@@ -1,23 +1,65 @@
{
"S7_S3_overlap": {
"overlap_count": 21,
"same_dir": 21,
"S7_S3_signal_overlap": {
"overlap_count": 5,
"same_dir": 5,
"opposite_dir": 0,
"ratio": 0.169
"ratio": 0.065
},
"S9_S9F_temporal": {
"s9_trade_days": 275,
"s9f_trade_days": 65,
"shared_trade_days": 33,
"temporal_overlap_ratio": 0.12
"pnl_correlation": {
"S7_Tight_vs_S9_Filtered": -0.054,
"S7_Tight_vs_S3": 0.072,
"S7_Tight_vs_S8_OB": 0.055,
"S9_Filtered_vs_S3": -0.05,
"S9_Filtered_vs_S8_OB": 0.036,
"S3_vs_S8_OB": 0.109
},
"avg_pairwise_correlation": 0.028,
"temporal_overlap": {
"S7_Tight_vs_S9_Filtered": {
"trade_days_a": 38,
"trade_days_b": 50,
"shared_days": 5,
"jaccard_index": 0.06
},
"S7_Tight_vs_S3": {
"trade_days_a": 38,
"trade_days_b": 111,
"shared_days": 6,
"jaccard_index": 0.042
},
"S7_Tight_vs_S8_OB": {
"trade_days_a": 38,
"trade_days_b": 51,
"shared_days": 2,
"jaccard_index": 0.023
},
"S9_Filtered_vs_S3": {
"trade_days_a": 50,
"trade_days_b": 111,
"shared_days": 11,
"jaccard_index": 0.073
},
"S9_Filtered_vs_S8_OB": {
"trade_days_a": 50,
"trade_days_b": 51,
"shared_days": 7,
"jaccard_index": 0.074
},
"S3_vs_S8_OB": {
"trade_days_a": 111,
"trade_days_b": 51,
"shared_days": 11,
"jaccard_index": 0.073
}
},
"portfolio": {
"total_trades": 694,
"win_rate_pct": 53.3,
"profit_factor": 0.99,
"total_pnl_pips": -175.1,
"total_pnl_dollars": -23036.66,
"max_drawdown_pct": -47.8,
"sharpe_ratio": -0.71
"total_trades": 289,
"win_rate_pct": 54.0,
"profit_factor": 1.37,
"total_pnl_pips": 1732.1,
"total_pnl_dollars": 23424.02,
"max_drawdown_pct": -21.06,
"sharpe_ratio": 1.29,
"expectancy_pips": 5.99
}
}
+216 -106
View File
@@ -1,12 +1,17 @@
"""
Phase 2 — Correlation Analysis (Step 8).
Phase 2 — Correlation Analysis (4-Strategy Portfolio).
For strategies sharing a pair (S7_Tight + S3 on GBP_JPY):
- Compute signal overlap and simultaneous position frequency.
- Combined equity curve analysis.
Analyzes diversification across the portfolio:
S7_Tight / GBP_JPY / H1 — Liquidity Sweep
S9_Filtered/ GBP_AUD / H1 — London Session
S3 / GBP_JPY / H1 — Key Level Breakout
S8_OB / GBP_USD / M15 — Order Block Retest
Also compute portfolio-level metrics: combined PF, combined max DD,
Sharpe of combined equity curve.
Computes:
1. Signal overlap: S7 vs S3 (same pair GBP_JPY)
2. Daily PnL correlation matrix (all strategy pairs)
3. Temporal clustering (same-day trade entries)
4. Portfolio-level metrics and combined equity curve
Output: results/phase2/correlation_analysis.json
"""
@@ -22,25 +27,31 @@ from src.backtester.engine import Backtester
# Strategy imports
from src.strategies_pkg.s7_liquidity_sweep import S7_Liquidity_Sweep
from src.strategies_pkg.s9_london_session import S9_London_Session
from src.strategies_pkg.s4f_ema_ribbon import S4F_EMA_Ribbon
from src.strategies_pkg.s3_key_level_breakout import S3_KeyLevel_Breakout
from src.strategies_pkg.s8_order_block import S8_Order_Block
PROCESSED_DIR = os.path.join(os.path.dirname(os.path.dirname(__file__)), "data", "processed")
RESULTS_DIR = os.path.join(os.path.dirname(os.path.dirname(__file__)), "results", "phase2")
os.makedirs(RESULTS_DIR, exist_ok=True)
# All Phase 2 strategies
def _s8_tuned():
s = S8_Order_Block()
s.DISPLACEMENT_ATR = 2.5
s.TP1_ATR_MULT = 2.0
s.OB_RETEST_WINDOW = 40
return s
CONFIGS = [
{"name": "S7_Tight", "pair": "GBP_JPY", "tf": "H1",
{"name": "S7_Tight", "pair": "GBP_JPY", "tf": "H1", "htf_tf": "H1",
"factory": lambda: S7_Liquidity_Sweep()},
{"name": "S9", "pair": "GBP_USD", "tf": "H1",
"factory": lambda: S9_London_Session()},
{"name": "S9_Filtered", "pair": "GBP_AUD", "tf": "H1",
{"name": "S9_Filtered", "pair": "GBP_AUD", "tf": "H1", "htf_tf": "H1",
"factory": lambda: S9_London_Session(pair="GBP_AUD", filtered=True)},
{"name": "S4F", "pair": "EUR_AUD", "tf": "M15",
"factory": lambda: S4F_EMA_Ribbon()},
{"name": "S3", "pair": "GBP_JPY", "tf": "H1",
{"name": "S3", "pair": "GBP_JPY", "tf": "H1", "htf_tf": "H1",
"factory": lambda: S3_KeyLevel_Breakout()},
{"name": "S8_OB", "pair": "GBP_USD", "tf": "M15", "htf_tf": "H1",
"factory": lambda: _s8_tuned()},
]
@@ -53,15 +64,23 @@ def load_data(pair, tf):
return compute_all_indicators(df)
def run_backtest(cfg):
def run_backtest(cfg, data_cache):
"""Run backtest for a config, return trade log and equity curve."""
pair = cfg["pair"]
tf = cfg["tf"]
data = load_data(pair, tf)
htf_tf = cfg["htf_tf"]
cache_key = f"{pair}_{tf}"
if cache_key not in data_cache:
data_cache[cache_key] = load_data(pair, tf)
data = data_cache[cache_key]
if data is None:
return None, None, None
htf_data = data.copy() if tf == "H1" else load_data(pair, "H1")
htf_cache_key = f"{pair}_{htf_tf}"
if htf_cache_key not in data_cache:
data_cache[htf_cache_key] = load_data(pair, htf_tf)
htf_data = data_cache[htf_cache_key] if htf_tf != tf else data
strategy = cfg["factory"]()
bt = Backtester(data=data, strategy=strategy, pair=pair,
@@ -72,15 +91,8 @@ def run_backtest(cfg):
return report, trade_log, eq_curve
def compute_signal_overlap(log_a, log_b, pair):
"""Compute signal overlap between two strategies on the same pair.
Returns:
- overlap_count: trades that are open at the same time
- same_direction_count: overlapping trades in same direction
- opposite_direction_count: overlapping trades in opposite direction
- overlap_ratio: fraction of trades that overlap
"""
def compute_signal_overlap(log_a, log_b):
"""Compute signal overlap between two strategies (time-based)."""
if log_a.empty or log_b.empty:
return {"overlap_count": 0, "same_dir": 0, "opposite_dir": 0, "ratio": 0}
@@ -96,7 +108,6 @@ def compute_signal_overlap(log_a, log_b, pair):
b_start = pd.Timestamp(trade_b["timestamp"])
b_end = pd.Timestamp(trade_b["exit_time"]) if pd.notna(trade_b.get("exit_time")) else b_start
# Check if time ranges overlap
if a_start <= b_end and b_start <= a_end:
overlap += 1
if trade_a["signal_direction"] == trade_b["signal_direction"]:
@@ -115,29 +126,61 @@ def compute_signal_overlap(log_a, log_b, pair):
}
def compute_combined_equity(eq_curves: list[pd.DataFrame]) -> pd.DataFrame:
"""Combine equity curves from multiple strategies into portfolio equity."""
combined = None
for eq in eq_curves:
if eq is None or eq.empty:
def compute_daily_pnl_series(trade_logs):
"""Build daily PnL series per strategy for correlation analysis."""
daily_pnl = {}
for name, log in trade_logs.items():
if log is None or log.empty:
continue
eq = eq.set_index("timestamp")["equity"]
# Convert to returns relative to starting equity
returns = eq - 100_000.0
if combined is None:
combined = returns
else:
combined = combined.add(returns, fill_value=0)
df = log.copy()
df["date"] = pd.to_datetime(df["timestamp"]).dt.date
daily = df.groupby("date")["pnl_pips"].sum()
daily_pnl[name] = daily
return daily_pnl
if combined is None:
def compute_pnl_correlation(daily_pnl):
"""Compute pairwise correlation of daily PnL between strategies."""
if len(daily_pnl) < 2:
return pd.DataFrame()
# Add back starting equity (100k per slot, or just use combined returns)
combined = combined + 100_000.0
return combined.reset_index()
combined = pd.DataFrame(daily_pnl)
combined = combined.fillna(0)
return combined.corr()
def compute_portfolio_metrics(all_trade_logs: list[pd.DataFrame]) -> dict:
def compute_temporal_overlap(trade_logs):
"""Compute pairwise temporal overlap (same-day entries) between all strategies."""
names = list(trade_logs.keys())
results = {}
for i in range(len(names)):
for j in range(i + 1, len(names)):
a_name, b_name = names[i], names[j]
log_a = trade_logs[a_name]
log_b = trade_logs[b_name]
if log_a is None or log_a.empty or log_b is None or log_b.empty:
continue
dates_a = set(pd.to_datetime(log_a["timestamp"]).dt.date)
dates_b = set(pd.to_datetime(log_b["timestamp"]).dt.date)
shared = dates_a & dates_b
union = dates_a | dates_b
key = f"{a_name}_vs_{b_name}"
results[key] = {
"trade_days_a": len(dates_a),
"trade_days_b": len(dates_b),
"shared_days": len(shared),
"jaccard_index": round(len(shared) / len(union), 3) if union else 0,
}
return results
def compute_portfolio_metrics(all_trade_logs):
"""Compute portfolio-level metrics from combined trade logs."""
combined = pd.concat([log for log in all_trade_logs if not log.empty],
ignore_index=True)
@@ -153,18 +196,20 @@ def compute_portfolio_metrics(all_trade_logs: list[pd.DataFrame]) -> dict:
gross_loss = abs(losses["pnl_pips"].sum()) if len(losses) > 0 else 0
pf = gross_profit / gross_loss if gross_loss > 0 else float("inf")
total_pnl = combined["pnl_pips"].sum()
expectancy = total_pnl / n if n > 0 else 0
# Combined max drawdown
cum_pnl = combined.sort_values("timestamp")["pnl_dollars"].cumsum()
sorted_trades = combined.sort_values("timestamp")
cum_pnl = sorted_trades["pnl_dollars"].cumsum()
peak = cum_pnl.cummax()
dd = cum_pnl - peak
max_dd = dd.min()
max_dd_pct = max_dd / 100_000 * 100 if max_dd < 0 else 0
# Sharpe ratio
daily_pnl = combined.copy()
daily_pnl["date"] = pd.to_datetime(daily_pnl["timestamp"]).dt.date
daily = daily_pnl.groupby("date")["pnl_dollars"].sum()
# Sharpe ratio from daily PnL
sorted_trades = sorted_trades.copy()
sorted_trades["date"] = pd.to_datetime(sorted_trades["timestamp"]).dt.date
daily = sorted_trades.groupby("date")["pnl_dollars"].sum()
if len(daily) > 1 and daily.std() > 0:
sharpe = (daily.mean() / daily.std()) * np.sqrt(252)
else:
@@ -178,92 +223,137 @@ def compute_portfolio_metrics(all_trade_logs: list[pd.DataFrame]) -> dict:
"total_pnl_dollars": round(combined["pnl_dollars"].sum(), 2),
"max_drawdown_pct": round(max_dd_pct, 2),
"sharpe_ratio": round(sharpe, 2),
"expectancy_pips": round(expectancy, 2),
}
def main():
t0 = time.time()
print(f"{'='*80}")
print("PHASE 2 — CORRELATION ANALYSIS")
print("PHASE 2 — CORRELATION ANALYSIS (4-Strategy Portfolio)")
print(f"{'='*80}")
results = {}
trade_logs = {}
eq_curves = {}
data_cache = {}
# Run all backtests
for cfg in CONFIGS:
name = cfg["name"]
pair = cfg["pair"]
print(f"\nRunning {name} / {pair}...", end=" ", flush=True)
t0 = time.time()
report, log, eq = run_backtest(cfg)
elapsed = time.time() - t0
tf = cfg["tf"]
print(f"\n Running {name} / {pair} ({tf})...", end=" ", flush=True)
t1 = time.time()
report, log, eq = run_backtest(cfg, data_cache)
elapsed = time.time() - t1
n_trades = len(log) if log is not None and not log.empty else 0
print(f"{n_trades} trades ({elapsed:.0f}s)")
trade_logs[name] = log
eq_curves[name] = eq
# --- Signal Overlap: S7_Tight vs S3 on GBP_JPY ---
print(f"\n{'#'*60}")
print("# Signal Overlap: S7_Tight vs S3 on GBP_JPY")
print(f"{'#'*60}")
# =====================================================================
# 1. Signal Overlap: S7_Tight vs S3 (both on GBP_JPY)
# =====================================================================
print(f"\n{'#'*70}")
print("# 1. SIGNAL OVERLAP: S7_Tight vs S3 (GBP_JPY)")
print(f"{'#'*70}")
log_s7 = trade_logs.get("S7_Tight", pd.DataFrame())
log_s3 = trade_logs.get("S3", pd.DataFrame())
if not log_s7.empty and not log_s3.empty:
overlap = compute_signal_overlap(log_s7, log_s3, "GBP_JPY")
results["S7_S3_overlap"] = overlap
overlap = compute_signal_overlap(log_s7, log_s3)
results["S7_S3_signal_overlap"] = overlap
print(f" S7 trades: {len(log_s7)}")
print(f" S3 trades: {len(log_s3)}")
print(f" S7_Tight trades: {len(log_s7)}")
print(f" S3 trades: {len(log_s3)}")
print(f" Overlapping periods: {overlap['overlap_count']}")
print(f" Same direction: {overlap['same_dir']}")
print(f" Same direction: {overlap['same_dir']}")
print(f" Opposite direction: {overlap['opposite_dir']}")
print(f" Overlap ratio: {overlap['ratio']:.1%}")
if overlap['ratio'] < 0.15:
print(" => LOW overlap: Good diversification!")
print(" => LOW overlap: Good diversification within GBP_JPY")
elif overlap['ratio'] < 0.30:
print(" => MODERATE overlap: Some clustering.")
print(" => MODERATE overlap: Some clustering on GBP_JPY")
else:
print(" => HIGH overlap: Significant clustering risk.")
print(" => HIGH overlap: Significant clustering risk on GBP_JPY")
else:
print(" Insufficient data for overlap analysis.")
# --- S9 vs S9_Filtered (different pairs, should be independent) ---
print(f"\n{'#'*60}")
print("# Independence Check: S9 (GBP_USD) vs S9_Filtered (GBP_AUD)")
print(f"{'#'*60}")
# =====================================================================
# 2. Daily PnL Correlation Matrix
# =====================================================================
print(f"\n{'#'*70}")
print("# 2. DAILY PnL CORRELATION MATRIX")
print(f"{'#'*70}")
log_s9 = trade_logs.get("S9", pd.DataFrame())
log_s9f = trade_logs.get("S9_Filtered", pd.DataFrame())
daily_pnl = compute_daily_pnl_series(trade_logs)
corr_matrix = compute_pnl_correlation(daily_pnl)
if not log_s9.empty and not log_s9f.empty:
# Check temporal clustering (same-day entries)
s9_dates = set(pd.to_datetime(log_s9["timestamp"]).dt.date)
s9f_dates = set(pd.to_datetime(log_s9f["timestamp"]).dt.date)
shared_dates = s9_dates & s9f_dates
temporal_overlap = len(shared_dates) / max(len(s9_dates), 1)
results["S9_S9F_temporal"] = {
"s9_trade_days": len(s9_dates),
"s9f_trade_days": len(s9f_dates),
"shared_trade_days": len(shared_dates),
"temporal_overlap_ratio": round(temporal_overlap, 3),
if not corr_matrix.empty:
results["pnl_correlation"] = {
f"{a}_vs_{b}": round(corr_matrix.loc[a, b], 3)
for i, a in enumerate(corr_matrix.index)
for j, b in enumerate(corr_matrix.columns)
if j > i
}
print(f" S9 trade days: {len(s9_dates)}")
print(f" S9_Filtered trade days: {len(s9f_dates)}")
print(f" Shared trade days: {len(shared_dates)}")
print(f" Temporal overlap: {temporal_overlap:.1%}")
print(f"\n {'':>14}", end="")
for name in corr_matrix.columns:
print(f" {name:>12}", end="")
print()
for row_name in corr_matrix.index:
print(f" {row_name:>14}", end="")
for col_name in corr_matrix.columns:
val = corr_matrix.loc[row_name, col_name]
print(f" {val:>12.3f}", end="")
print()
# Average pairwise correlation
pairs = []
for i, a in enumerate(corr_matrix.index):
for j, b in enumerate(corr_matrix.columns):
if j > i:
pairs.append(corr_matrix.loc[a, b])
avg_corr = np.mean(pairs) if pairs else 0
results["avg_pairwise_correlation"] = round(avg_corr, 3)
print(f"\n Average pairwise correlation: {avg_corr:.3f}")
if avg_corr < 0.20:
print(" => LOW correlation: Excellent diversification")
elif avg_corr < 0.40:
print(" => MODERATE correlation: Decent diversification")
else:
print(" => HIGH correlation: Limited diversification benefit")
else:
print(" Insufficient data.")
# --- Portfolio Metrics ---
print(f"\n{'#'*60}")
print("# Portfolio-Level Metrics (All 5 Strategies Combined)")
print(f"{'#'*60}")
# =====================================================================
# 3. Temporal Overlap (Same-Day Entry Clustering)
# =====================================================================
print(f"\n{'#'*70}")
print("# 3. TEMPORAL OVERLAP (Same-Day Entries)")
print(f"{'#'*70}")
temporal = compute_temporal_overlap(trade_logs)
results["temporal_overlap"] = temporal
print(f"\n {'Pair':<30} {'Days A':>7} {'Days B':>7} {'Shared':>7} {'Jaccard':>8}")
print(f" {'-'*65}")
for key, val in temporal.items():
print(f" {key:<30} {val['trade_days_a']:>7} {val['trade_days_b']:>7} "
f"{val['shared_days']:>7} {val['jaccard_index']:>7.3f}")
# =====================================================================
# 4. Portfolio Metrics
# =====================================================================
print(f"\n{'#'*70}")
print("# 4. PORTFOLIO METRICS (All 4 Strategies Combined)")
print(f"{'#'*70}")
all_logs = [log for log in trade_logs.values()
if log is not None and not log.empty]
@@ -271,27 +361,32 @@ def main():
portfolio = compute_portfolio_metrics(all_logs)
results["portfolio"] = portfolio
print(f" Total trades: {portfolio['total_trades']}")
print(f" Win rate: {portfolio['win_rate_pct']}%")
print(f" Profit factor: {portfolio['profit_factor']}")
print(f" Total PnL (pips): {portfolio['total_pnl_pips']:+.1f}")
print(f" Total PnL ($): {portfolio['total_pnl_dollars']:+,.2f}")
print(f" Max drawdown: {portfolio['max_drawdown_pct']:.2f}%")
print(f" Sharpe ratio: {portfolio['sharpe_ratio']:.2f}")
print(f" Total trades: {portfolio['total_trades']}")
print(f" Win rate: {portfolio['win_rate_pct']}%")
print(f" Profit factor: {portfolio['profit_factor']}")
print(f" Expectancy: {portfolio['expectancy_pips']:+.2f} pips/trade")
print(f" Total PnL: {portfolio['total_pnl_pips']:+.1f} pips "
f"(${portfolio['total_pnl_dollars']:+,.2f})")
print(f" Max drawdown: {portfolio['max_drawdown_pct']:.2f}%")
print(f" Sharpe ratio: {portfolio['sharpe_ratio']:.2f}")
# --- Per-Strategy Summary ---
# =====================================================================
# 5. Per-Strategy Summary
# =====================================================================
print(f"\n{'='*80}")
print("STRATEGY SUMMARY")
print(f"{'='*80}")
print(f"{'Strategy':<16} {'Pair':<10} {'Trades':>6} {'WR%':>6} {'PF':>6} {'PnL(p)':>9}")
print(f"{'-'*60}")
print(f" {'Strategy':<14} {'Pair':<10} {'TF':<4} {'Trades':>6} {'WR%':>6} "
f"{'PF':>6} {'PnL(p)':>9} {'Exp':>7}")
print(f" {'-'*70}")
for cfg in CONFIGS:
name = cfg["name"]
pair = cfg["pair"]
tf = cfg["tf"]
log = trade_logs.get(name, pd.DataFrame())
if log.empty:
print(f"{name:<16} {pair:<10} {'N/A':>6}")
print(f" {name:<14} {pair:<10} {tf:<4} {'N/A':>6}")
continue
n = len(log)
wins = log[log["win"] == True]
@@ -300,14 +395,29 @@ def main():
gl = abs(log[log["win"] == False]["pnl_pips"].sum())
pf = gp / gl if gl > 0 else 0
pnl = log["pnl_pips"].sum()
print(f"{name:<16} {pair:<10} {n:>6} {wr:>5.1f}% {pf:>5.2f} {pnl:>+8.1f}")
exp = pnl / n if n > 0 else 0
print(f" {name:<14} {pair:<10} {tf:<4} {n:>6} {wr:>5.1f}% "
f"{pf:>5.2f} {pnl:>+8.1f} {exp:>+6.2f}")
# Save
out_path = os.path.join(RESULTS_DIR, "correlation_analysis.json")
def json_default(obj):
if isinstance(obj, (np.integer,)):
return int(obj)
if isinstance(obj, (np.floating,)):
return float(obj)
if isinstance(obj, (np.bool_,)):
return bool(obj)
return str(obj)
with open(out_path, "w") as f:
json.dump(results, f, indent=2, default=str)
json.dump(results, f, indent=2, default=json_default)
print(f"\nResults saved: {out_path}")
elapsed = time.time() - t0
print(f"Total runtime: {elapsed:.1f}s")
if __name__ == "__main__":
main()