#!/usr/bin/env python3 """Portfolio Optimizer — combine uncorrelated strategies for 15% monthly target. Given N strategies with daily returns, find the optimal combination that: - Maximizes monthly return - Keeps max drawdown within RiskMgmt limits (10% total, 5% daily) - Diversifies across uncorrelated strategies """ import json import os from pathlib import Path import numpy as np import pandas as pd PROJECT = Path(__file__).resolve().parent.parent RESULTS_DIR = PROJECT / "results" / "strategies_new" STRATEGIES_DIR = PROJECT / "results" / "strategies" FACTORS_DIR = PROJECT / "results" / "factors" VALUES_DIR = FACTORS_DIR / "values" OHLCV_PATH = Path(os.getenv("PREDIX_OHLCV_PATH", str(PROJECT / "git_ignore_folder" / "intraday_pv_all.h5"))) TARGET_MONTHLY = 15.0 MAX_DD = 0.10 # RiskMgmt: 10% max total drawdown MAX_DAILY_DD = 0.05 # RiskMgmt: 5% max daily drawdown MIN_TRADES = 30 MIN_SHARPE = 0.5 def load_strategies() -> list[dict]: """Load all strategy JSONs with real (non-fabricated) verified metrics.""" strategies = [] seen = set() for d in (STRATEGIES_DIR, RESULTS_DIR): if not d.exists(): continue for p in d.glob("*.json"): try: r = json.loads(p.read_text()) except Exception: continue if not isinstance(r, dict): continue name = r.get("strategy_name", p.stem) if name in seen: continue seen.add(name) s = r.get("summary", {}) if not isinstance(s, dict): s = {} m = r.get("metrics", {}) if not isinstance(m, dict): m = {} # Extract metrics (prefer summary, fallback to metrics) sharpe = float(s.get("sharpe") or m.get("sharpe") or 0) mon_pct = float(s.get("monthly_return_pct") or s.get("oos_monthly_return_pct") or m.get("monthly_return_pct") or 0) max_dd = float(s.get("max_drawdown") or s.get("oos_max_drawdown") or m.get("max_drawdown") or 0) win_rate = float(s.get("win_rate") or s.get("oos_win_rate") or m.get("win_rate") or 0) n_trades = int(s.get("n_trades") or s.get("oos_n_trades") or s.get("real_n_trades") or m.get("n_trades") or 0) total_ret = float(s.get("total_return") or m.get("total_return") or 0) # Filter fabricated if mon_pct == 200 and sharpe == 3.0 and abs(max_dd + 0.167) < 0.01: continue if mon_pct == -20 and max_dd == -1.0: continue if sharpe == 200: continue # Filter quality if n_trades < MIN_TRADES or sharpe < MIN_SHARPE: continue if mon_pct <= 0: continue strategies.append({ "name": name, "file": str(p), "sharpe": sharpe, "monthly_pct": mon_pct, "max_dd": max_dd, "win_rate": win_rate, "n_trades": n_trades, "total_return": total_ret, "factors": r.get("factor_names") or r.get("factors_used") or [], "code": r.get("code", ""), }) return strategies def load_strategy_returns(strategy: dict, close_daily: pd.Series) -> pd.Series | None: """Reconstruct daily strategy returns from code and factor data.""" code = strategy.get("code", "") if not code: return None factors_list = strategy.get("factors", []) if not factors_list: return None # Load factor values factor_series = {} for fname in factors_list: safe = str(fname).replace("/", "_").replace("\\", "_").replace(" ", "_")[:150] parq = VALUES_DIR / f"{safe}.parquet" if not parq.exists(): continue try: s = pd.read_parquet(str(parq)) if isinstance(s.index, pd.MultiIndex): s = s.xs("EURUSD", level="instrument")[s.columns[0]] # Align to close_daily index s = s.resample("D").last().reindex(close_daily.index).ffill(limit=5) factor_series[fname] = s except Exception: continue if len(factor_series) < 2: return None df_factors = pd.DataFrame(factor_series).dropna() if len(df_factors) < 100: return None # Execute strategy code on daily data local_vars = {"factors": df_factors, "close": close_daily.reindex(df_factors.index)} try: exec(code, {"np": np, "pd": pd, "numpy": np}, local_vars) except Exception: # Can't execute — use simple IC-weighted signal as fallback return None signal = local_vars.get("signal") if signal is None or not isinstance(signal, pd.Series): return None # Compute daily returns from signal common = close_daily.index.intersection(signal.index) c = close_daily.loc[common] s = signal.loc[common].clip(-1, 1).fillna(0) fwd_ret = c.pct_change().shift(-1) strat_ret = s.shift(1) * fwd_ret strat_ret = strat_ret.dropna() if len(strat_ret) < 30: return None return strat_ret def build_simple_signal(factors_list: list[str], close_daily: pd.Series) -> tuple[pd.Series, pd.Series]: """Build simple IC-weighted daily signal (fallback when code fails).""" import json as _json factor_series = {} ic_values = {} for fname in factors_list: safe = str(fname).replace("/", "_").replace("\\", "_").replace(" ", "_")[:150] parq = VALUES_DIR / f"{safe}.parquet" jf = FACTORS_DIR / f"{safe}.json" if not parq.exists(): continue ic = 0.0 if jf.exists(): ic = float(_json.loads(jf.read_text()).get("ic", 0)) try: s = pd.read_parquet(str(parq)) if isinstance(s.index, pd.MultiIndex): s = s.xs("EURUSD", level="instrument")[s.columns[0]] s = s.resample("D").last().reindex(close_daily.index).ffill(limit=5) factor_series[fname] = s ic_values[fname] = ic except Exception: continue df = pd.DataFrame(factor_series).dropna() if len(df) < 50: return pd.Series(), pd.Series() # z-score composite window = 20 z = (df - df.rolling(window).mean()) / (df.rolling(window).std() + 1e-8) composite = pd.Series(0.0, index=df.index) total_ic = sum(abs(v) for v in ic_values.values()) if total_ic == 0: total_ic = 1.0 for col in df.columns: ic = ic_values.get(col, 0) w = abs(ic) / total_ic sign = -1 if ic < 0 else 1 composite += sign * w * z[col] signal = pd.Series(0, index=df.index) signal[composite > 0.5] = 1 signal[composite < -0.5] = -1 # Compute returns common = close_daily.index.intersection(signal.index) c = close_daily.loc[common] s = signal.loc[common].clip(-1, 1).fillna(0) fwd_ret = c.pct_change().shift(-1) strat_ret = s.shift(1) * fwd_ret return signal, strat_ret.dropna() def compute_portfolio_metrics(returns: list[pd.Series], weights: list[float], close_daily: pd.Series) -> dict: """Compute portfolio-level metrics from weighted strategy returns.""" if not returns: return {"monthly_pct": 0, "max_dd": 0, "sharpe": 0} # Align all return series common_idx = returns[0].index for r in returns[1:]: common_idx = common_idx.intersection(r.index) if len(common_idx) < 50: return {"monthly_pct": 0, "max_dd": 0, "sharpe": 0} aligned = pd.DataFrame({i: r.loc[common_idx] for i, r in enumerate(returns)}).dropna() if len(aligned) < 30: return {"monthly_pct": 0, "max_dd": 0, "sharpe": 0} # Weighted portfolio return port_ret = pd.Series(0.0, index=aligned.index) for i in range(len(returns)): port_ret += weights[i] * aligned[i] # Equity curve eq = (1 + port_ret).cumprod() peak = eq.cummax() max_dd = float(((eq - peak) / peak).min()) total_ret = float(eq.iloc[-1] - 1) n_days = (port_ret.index[-1] - port_ret.index[0]).days n_months = max(n_days / 30.44, 1) monthly = float((1 + total_ret) ** (1 / n_months) - 1) sharpe = float(port_ret.mean() / port_ret.std() * np.sqrt(252)) if port_ret.std() > 0 else 0 daily_dd = float(port_ret.min()) # Worst daily return return { "monthly_pct": monthly * 100, "max_dd": max_dd, "sharpe": sharpe, "daily_worst": daily_dd, "n_days": len(port_ret), "n_months": n_months, } def main(): print("=" * 60) print(" Portfolio Optimizer — 15% Monthly Target") print("=" * 60) # Load OHLCV daily print("\nLoading data...") df = pd.read_hdf(OHLCV_PATH, key="data") close = df.xs("EURUSD", level="instrument")["$close"].sort_index() close_daily = close.resample("D").last().dropna() print(f" Daily bars: {len(close_daily)}") # Load strategies strategies = load_strategies() print(f" Real strategies: {len(strategies)}") # Build daily returns for each strategy print("\nBuilding strategy returns...") strat_returns = [] strat_names = [] for s in strategies[:50]: # Limit to top 50 for speed rets = load_strategy_returns(s, close_daily) if rets is None or len(rets) < 30: # Use simple signal as fallback _, rets = build_simple_signal(s["factors"], close_daily) if rets is not None and len(rets) >= 30: strat_returns.append(rets) strat_names.append(s["name"]) print(f" [{len(strat_returns)}] {s['name'][:40]:40s} " f"Sh={s['sharpe']:.1f} Mon={s['monthly_pct']:.1f}% Tr={s['n_trades']}") if len(strat_returns) < 2: print("\n Not enough valid strategies.") return print(f"\n Valid return series: {len(strat_returns)}") # Find best portfolio via greedy selection (low correlation, high return) print("\n--- Greedy Portfolio Selection ---") print(f" Target: {TARGET_MONTHLY}% monthly | Max DD: {MAX_DD:.0%} | Max Daily DD: {MAX_DAILY_DD:.0%}") print() # Compute individual metrics individual = [] for i, (rets, name) in enumerate(zip(strat_returns, strat_names)): eq = (1 + rets).cumprod() dd = float(((eq - eq.cummax()) / eq.cummax()).min()) total = float(eq.iloc[-1] - 1) n = max((rets.index[-1] - rets.index[0]).days / 30.44, 1) mon = float((1 + total) ** (1 / n) - 1) * 100 individual.append({"idx": i, "name": name, "monthly": mon, "dd": dd, "n": len(rets)}) individual.sort(key=lambda x: x["monthly"], reverse=True) # Greedy: add strategies one by one if they don't increase correlation too much selected = [] selected_rets = [] for s in individual: if len(selected) >= 8: break # Check correlation with existing portfolio new_ret = strat_returns[s["idx"]] if selected_rets: common = new_ret.index for r in selected_rets: common = common.intersection(r.index) if len(common) < 30: continue cors = [] for r in selected_rets: aligned_new = new_ret.loc[common] aligned_r = r.loc[common] if len(aligned_new) >= 30: cors.append(abs(aligned_new.corr(aligned_r))) if cors and max(cors) > 0.5: print(f" SKIP {s['name'][:40]} (max_corr={max(cors):.2f})") continue selected.append(s) selected_rets.append(new_ret) print(f" ADD {s['name'][:40]:40s} Mon={s['monthly']:+.1f}% DD={s['dd']:.3f} corr<0.5") # Evaluate portfolio if len(selected) >= 2: print(f"\n Portfolio: {len(selected)} strategies") weights = [1.0 / len(selected)] * len(selected) rets = [strat_returns[s["idx"]] for s in selected] pm = compute_portfolio_metrics(rets, weights, close_daily) print(f" Equal-weight metrics:") print(f" Monthly return: {pm['monthly_pct']:.2f}%") print(f" Max drawdown: {pm['max_dd']:.3f}") print(f" Sharpe: {pm['sharpe']:.2f}") print(f" Worst day: {pm['daily_worst']:.3%}") print(f" Period: {pm['n_months']:.1f} months ({pm['n_days']} days)") # Leverage scaling max_safe_lev = min( MAX_DD / abs(pm["max_dd"]) if pm["max_dd"] != 0 else 30, MAX_DAILY_DD / abs(pm["daily_worst"]) if pm["daily_worst"] != 0 else 30, 30, ) leveraged_monthly = pm["monthly_pct"] * max_safe_lev print(f"\n Max safe leverage: {max_safe_lev:.1f}× (limited by max DD {MAX_DD:.0%})") print(f" Leveraged monthly: {leveraged_monthly:.1f}%") if leveraged_monthly >= TARGET_MONTHLY: print(f"\n ✓ MEETS TARGET! {leveraged_monthly:.1f}% ≥ {TARGET_MONTHLY}%") else: gap = TARGET_MONTHLY - leveraged_monthly needed_strategies = int(np.ceil(len(selected) * TARGET_MONTHLY / max(leveraged_monthly, 0.1))) print(f"\n ✗ Below target. Need ~{needed_strategies} strategies or {TARGET_MONTHLY/max(pm['monthly_pct'],0.01):.1f}× better monthly.") # Save portfolio config out = { "target_monthly": TARGET_MONTHLY, "selected": [{"name": s["name"], "monthly": s["monthly"], "dd": s["dd"]} for s in selected], "portfolio": pm if len(selected) >= 2 else {}, } out_path = RESULTS_DIR / "portfolio_config.json" out_path.write_text(json.dumps(out, indent=2, default=str)) print(f"\n Saved → {out_path}") if __name__ == "__main__": main()