gavindiaz/profitable-expert-advisor
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Prepare source-only public release for develop.

Browse Source 
Add cluster audit pipeline, united EA updates, brochure generators, and publication hygiene (gitignore, MT5 path desensitization, pre-upload scan). Remove tracked reports, models, and binary artifacts from the repo.

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
zhutoutoutousan
2026-07-02 15:03:43 +02:00
co-authored by Cursor
parent 3f75a08848
commit 605faf5310
1014 changed files with 83437 additions and 10413 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backtesting/MT5/cluster_audit/__init__.py
+1
View File
@@ -0,0 +1 @@
# Cluster audit package
backtesting/MT5/cluster_audit/backtest_core.py
+327
View File
@@ -0,0 +1,327 @@
"""Shared backtest primitives — fills, costs, metrics, trade log."""
from __future__ import annotations
from dataclasses import dataclass, field
from typing import Any
import MetaTrader5 as mt5
import numpy as np
import pandas as pd
@dataclass
class CostModel:
spread_points: float = 0.0
slippage_points: float = 3.0
commission_per_lot: float = 0.0
@classmethod
def for_symbol(cls, symbol: str, slippage: float = 3.0) -> "CostModel":
info = mt5.symbol_info(symbol)
spread = float(info.spread) if info else 0.0
return cls(spread_points=spread, slippage_points=slippage)
@dataclass
class Trade:
side: str
open_time: Any
close_time: Any
open_price: float
close_price: float
volume: float
profit: float
bars_held: int = 0
exit_reason: str = ""
@dataclass
class BacktestReport:
strategy_id: str
symbol: str
timeframe: str
period_label: str
net_profit: float
total_trades: int
win_rate: float
profit_factor: float
sharpe: float
max_drawdown_pct: float
avg_win: float
avg_loss: float
worst_trades: list[dict]
losing_trades: list[dict]
exit_reason_breakdown: dict[str, dict[str, float]]
monthly_returns: dict[str, float]
params: dict[str, Any] = field(default_factory=dict)
gross_profit: float = 0.0
gross_loss: float = 0.0
trades_list: list[Trade] = field(default_factory=list, repr=False)
equity_curve: pd.Series | None = field(default=None, repr=False)
def to_dict(self) -> dict:
return {
"strategy_id": self.strategy_id,
"symbol": self.symbol,
"timeframe": self.timeframe,
"period": self.period_label,
"net_profit": self.net_profit,
"total_trades": self.total_trades,
"win_rate": self.win_rate,
"profit_factor": self.profit_factor,
"sharpe": self.sharpe,
"max_drawdown_pct": self.max_drawdown_pct,
"avg_win": self.avg_win,
"avg_loss": self.avg_loss,
"worst_trades": self.worst_trades,
"losing_trades": self.losing_trades,
"exit_reason_breakdown": self.exit_reason_breakdown,
"monthly_returns": self.monthly_returns,
"params": self.params,
}
def resolve_symbol(requested: str) -> str:
key = requested.split("|")[0].strip()
if not key:
return requested
if mt5.symbol_info(key) is not None:
mt5.symbol_select(key, True)
return key
for suffix in (".NAS", ".NYSE", ".NYS", ".US"):
cand = key + suffix
if mt5.symbol_info(cand) is not None:
mt5.symbol_select(cand, True)
return cand
for sym in mt5.symbols_get() or []:
if sym.name.startswith(key + "."):
mt5.symbol_select(sym.name, True)
return sym.name
return key
def load_bars(symbol: str, tf: int, start, end, trace: bool = False) -> pd.DataFrame:
requested = symbol
symbol = resolve_symbol(symbol)
if trace and symbol != requested:
print(f" [load_bars] resolved {requested} -> {symbol}", flush=True)
if not mt5.symbol_select(symbol, True):
raise RuntimeError(f"Cannot select {symbol}")
rates = mt5.copy_rates_range(symbol, tf, start, end)
if rates is None or len(rates) < 50:
raise RuntimeError(f"No data for {symbol} ({mt5.last_error()})")
if trace:
print(f" [load_bars] {symbol} got {len(rates)} bars", flush=True)
df = pd.DataFrame(rates)
df["time"] = pd.to_datetime(df["time"], unit="s")
df.set_index("time", inplace=True)
return df
def calc_profit(symbol: str, side: str, volume: float, entry: float, exit_px: float) -> float:
ot = mt5.ORDER_TYPE_BUY if side == "BUY" else mt5.ORDER_TYPE_SELL
p = mt5.order_calc_profit(ot, symbol, volume, entry, exit_px)
return float(p) if p is not None else 0.0
def _half_spread(point: float, spread_pts: float) -> float:
return spread_pts * point / 2.0
def fill_price(mid: float, point: float, costs: CostModel, side: str, entry: bool) -> float:
hs = _half_spread(point, costs.spread_points)
slip = costs.slippage_points * point
if side == "BUY":
return mid + hs + slip if entry else mid - hs - slip
return mid - hs - slip if entry else mid + hs + slip
@dataclass
class SimState:
side: str | None = None
entry: float = 0.0
entry_i: int = 0
entry_time: object = None
sl: float = 0.0
tp: float = 0.0
bars_against: int = 0
rsi_against: bool = False
def _bar_seconds(tf_label: str) -> int:
mapping = {
"M1": 60, "M5": 300, "M10": 600, "M12": 720, "M15": 900, "M20": 1200,
"M30": 1800, "H1": 3600, "H2": 7200, "H4": 14400, "D1": 86400,
}
return mapping.get(tf_label.upper(), 3600)
def run_single_position(
df: pd.DataFrame,
symbol: str,
point: float,
costs: CostModel,
lot: float,
strategy_id: str,
tf_label: str,
period_label: str,
params: dict,
initial_balance: float,
on_bar,
bar_seconds: int | None = None,
) -> BacktestReport:
"""Single-position bar loop with mark-to-market equity each bar."""
bar_sec = bar_seconds or _bar_seconds(tf_label)
trades: list[Trade] = []
realized_pnl = 0.0
equity: list[float] = [initial_balance]
st = SimState()
def close(i: int, mid: float, reason: str) -> None:
nonlocal st, realized_pnl
if st.side is None:
return
exit_px = fill_price(mid, point, costs, st.side, entry=False)
commission = costs.commission_per_lot * lot * 2.0
profit = calc_profit(symbol, st.side, lot, st.entry, exit_px) - commission
held = max(1, int((df.index[i] - pd.Timestamp(st.entry_time)).total_seconds() / bar_sec))
trades.append(
Trade(
side=st.side,
open_time=st.entry_time,
close_time=df.index[i],
open_price=st.entry,
close_price=exit_px,
volume=lot,
profit=profit,
bars_held=held,
exit_reason=reason,
)
)
realized_pnl += profit
st = SimState()
def open_pos(i: int, side: str, mid: float) -> None:
nonlocal st
st.side = side
st.entry = fill_price(mid, point, costs, side, entry=True)
st.entry_i = i
st.entry_time = df.index[i]
for i in range(1, len(df)):
on_bar(i, st, open_pos, close)
bal = initial_balance + realized_pnl
if st.side is not None:
mark = float(df["close"].iloc[i - 1])
bal += calc_profit(symbol, st.side, lot, st.entry, mark)
equity.append(bal)
if st.side is not None:
close(len(df) - 1, float(df["close"].iloc[-1]), "eod")
eq = pd.Series(equity[: len(df)], index=df.index[: len(equity)])
report = build_report(strategy_id, symbol, tf_label, period_label, trades, eq, initial_balance, params)
report.equity_curve = eq
return report
def build_report(
strategy_id: str,
symbol: str,
timeframe: str,
period_label: str,
trades: list[Trade],
equity_curve: pd.Series,
initial_balance: float,
params: dict,
) -> BacktestReport:
if not trades:
return BacktestReport(
strategy_id=strategy_id,
symbol=symbol,
timeframe=timeframe,
period_label=period_label,
net_profit=0.0,
total_trades=0,
win_rate=0.0,
profit_factor=0.0,
sharpe=0.0,
max_drawdown_pct=0.0,
avg_win=0.0,
avg_loss=0.0,
worst_trades=[],
losing_trades=[],
exit_reason_breakdown={},
monthly_returns={},
params=params,
)
profits = [t.profit for t in trades]
wins = [p for p in profits if p >= 0]
losses = [abs(p) for p in profits if p < 0]
gp = sum(wins)
gl = sum(losses)
net = sum(profits)
rets = equity_curve.pct_change().dropna()
sharpe = 0.0
if len(rets) > 10 and rets.std() > 0:
bars_per_year = 252 * 24 if "H1" in timeframe else 252 * 24 * 6
scale = np.sqrt(bars_per_year / max(len(rets), 1))
sharpe = float(rets.mean() / rets.std() * scale)
peak = equity_curve.cummax()
dd = (peak - equity_curve) / peak.replace(0, np.nan)
max_dd = float(dd.max()) if len(dd) else 0.0
monthly = equity_curve.resample("ME").last().pct_change().dropna()
monthly_dict = {str(k.date()): float(v) for k, v in monthly.items()}
def trade_row(t: Trade) -> dict:
return {
"side": t.side,
"open_time": str(t.open_time),
"close_time": str(t.close_time),
"open_price": t.open_price,
"close_price": t.close_price,
"profit": t.profit,
"bars_held": t.bars_held,
"exit_reason": t.exit_reason,
}
sorted_trades = sorted(trades, key=lambda t: t.profit)
losers = [trade_row(t) for t in sorted_trades if t.profit < 0]
breakdown: dict[str, dict[str, float]] = {}
for t in trades:
bucket = breakdown.setdefault(t.exit_reason or "?", {"count": 0, "pnl": 0.0, "wins": 0, "losses": 0})
bucket["count"] += 1
bucket["pnl"] += t.profit
if t.profit >= 0:
bucket["wins"] += 1
else:
bucket["losses"] += 1
return BacktestReport(
strategy_id=strategy_id,
symbol=symbol,
timeframe=timeframe,
period_label=period_label,
net_profit=net,
total_trades=len(trades),
win_rate=(len(wins) / len(trades) * 100.0) if trades else 0.0,
profit_factor=(gp / gl) if gl > 0 else (999.0 if gp > 0 else 0.0),
sharpe=sharpe,
max_drawdown_pct=max_dd * 100.0,
avg_win=(gp / len(wins)) if wins else 0.0,
avg_loss=(gl / len(losses)) if losses else 0.0,
worst_trades=[trade_row(t) for t in sorted_trades[:5]],
losing_trades=losers[:30],
exit_reason_breakdown=breakdown,
monthly_returns=monthly_dict,
params=params,
gross_profit=gp,
gross_loss=gl,
trades_list=list(trades),
equity_curve=equity_curve.copy(),
)
backtesting/MT5/cluster_audit/diagnose.py
+145
View File
@@ -0,0 +1,145 @@
"""Per-strategy problem diagnosis and loss tracing."""
from __future__ import annotations
from typing import Any
from .backtest_core import BacktestReport
from .trace_log import TraceLog
ENGINE_FIX_HINTS = {
"rsi_crossover": "trend_strong closes positions in strong trends; high ema_slope/distance thresholds block entries.",
"rsi_scalp": "rsi_against exits + spread costs dominate; check OB/OS vs target gap and bars_to_wait.",
"rsi_asian": "session window or extreme RSI levels may block entries; verify broker server hour offset.",
"mean_reversion": "ADX proxy may differ from MQL iADX; min_ema_distance_pts can block all entries on volatile symbols.",
"ema_slope": "needs EMA-cross exit + profit trail even when use_trailing_stop=False; weekly ADX filter missing.",
"darvas": "box must be narrow (box_deviation); volume MA filter not yet ported.",
"rsi_secret": "zone re-entry in chop; add divergence confirm or widen min_bars_between_trades.",
}
def diagnose(spec: dict, baseline: BacktestReport, optimized: BacktestReport | None, log: TraceLog) -> dict[str, Any]:
sid = spec["id"]
engine = spec["engine"]
issues: list[str] = []
actions: list[str] = []
if baseline.total_trades == 0:
issues.append("ZERO_TRADES")
actions.append("Engine logic or params too strict - compare Python port to MQL defaults.")
elif baseline.total_trades < 10:
issues.append("LOW_TRADE_COUNT")
actions.append("Relax entry filters or widen optimization ranges.")
if baseline.net_profit < 0:
issues.append("NEGATIVE_PNL")
if baseline.sharpe < 0:
issues.append("NEGATIVE_SHARPE")
if baseline.max_drawdown_pct > 25:
issues.append("HIGH_DRAWDOWN")
br = baseline.exit_reason_breakdown
loss_reasons = sorted(
((k, v["pnl"]) for k, v in br.items() if v["pnl"] < 0),
key=lambda x: x[1],
)
if loss_reasons:
top = loss_reasons[0]
issues.append(f"TOP_LOSS_REASON:{top[0]}")
if top[0] == "rsi_against":
actions.append("Widen RSI targets or increase bars_to_wait before rsi_against exit.")
elif top[0] == "trend_strong":
actions.append("Raise ema_slope/distance thresholds or only block new entries (MQL also force-closes).")
elif top[0] == "trail":
actions.append("Trail too tight - widen trail_distance_pts or raise activation.")
elif top[0] == "sl":
actions.append("Stop loss too tight for symbol volatility - scale SL by ATR.")
elif top[0] == "hours" or top[0] == "session":
actions.append("Trading hours/session filter closing positions — align to broker server time.")
elif top[0] == "adx_escape":
actions.append("ADX escape fires too early — raise adx_escape threshold.")
hint = ENGINE_FIX_HINTS.get(engine, "")
if hint:
actions.append(hint)
if optimized and optimized is not baseline:
from .scoring import DEFAULT_TRADES_PER_DAY, acceptance, min_trades_for_period, period_days, trades_per_day
from .strategy_registry import PERIODS
start, end = PERIODS.get("2021-2026", ("2021-01-01", "2026-06-01"))
days = period_days(start, end)
min_t = min_trades_for_period(days, DEFAULT_TRADES_PER_DAY)
opt_tpd = trades_per_day(optimized, days)
base_tpd = trades_per_day(baseline, days)
if opt_tpd < DEFAULT_TRADES_PER_DAY:
issues.append("LOW_TRADES_PER_DAY")
actions.append(
f"Only {opt_tpd:.2f} trades/day (need >={DEFAULT_TRADES_PER_DAY:.1f}); "
"use lower TF, tighter SL/TP, or relax entry filters."
)
if optimized.total_trades < min_t and baseline.total_trades >= min_t:
issues.append("OPT_COLLAPSED_TRADES")
actions.append(
f"Optimization cut trades {baseline.total_trades}->{optimized.total_trades} "
f"({base_tpd:.2f}->{opt_tpd:.2f}/day)."
)
opt_ok, opt_issues = acceptance(optimized, days, DEFAULT_TRADES_PER_DAY)
if not opt_ok and optimized.net_profit > baseline.net_profit:
issues.append("OPT_PROFIT_BUT_FAILS_GATES")
actions.append("Higher net but fails gates: " + "; ".join(opt_issues[:4]))
if optimized.sharpe > baseline.sharpe + 0.1:
issues.append("OPTIMIZATION_HELPED")
log.banner(f"DIAGNOSIS: {sid}")
log.info(f"engine={engine} symbol={baseline.symbol} trades={baseline.total_trades}")
if issues:
log.warn("issues: " + ", ".join(issues))
else:
log.info("no critical issues flagged")
trace_losses(baseline, log, label="baseline")
if optimized and optimized is not baseline:
log.info(
f"optimized: net=${optimized.net_profit:.0f} sharpe={optimized.sharpe:.2f} "
f"trades={optimized.total_trades}"
)
if optimized.net_profit < baseline.net_profit:
trace_losses(optimized, log, label="optimized", max_rows=10)
if actions:
log.info("suggested actions:")
for a in actions[:6]:
log.debug(f" - {a}")
return {
"issues": issues,
"actions": actions,
"top_loss_reasons": loss_reasons[:5],
"exit_reason_breakdown": br,
}
def trace_losses(report: BacktestReport, log: TraceLog, label: str = "baseline", max_rows: int = 15) -> None:
if not report.losing_trades:
log.debug(f"{label}: no losing trades")
return
log.info(f"{label} loss trace ({len(report.losing_trades)} losers logged, showing worst {max_rows}):")
for i, t in enumerate(report.losing_trades[:max_rows], 1):
log.info(
f" #{i:02d} {t['side']:4} ${t['profit']:8.2f} {t['exit_reason']:12} "
f"bars={t['bars_held']:4} {t['open_time']} -> {t['close_time']}"
)
if report.exit_reason_breakdown:
log.debug(f"{label} exit reason PnL:")
for reason, stats in sorted(
report.exit_reason_breakdown.items(),
key=lambda x: x[1]["pnl"],
):
log.debug(
f" {reason:14} count={int(stats['count']):4} "
f"wins={int(stats['wins']):3} losses={int(stats['losses']):3} pnl=${stats['pnl']:.0f}"
)
backtesting/MT5/cluster_audit/engines.py
+882
View File
@@ -0,0 +1,882 @@
"""Python ports of SuperEA engines for cluster audit."""
from __future__ import annotations
from dataclasses import dataclass
from typing import Any
import numpy as np
import pandas as pd
from indicator_utils import calculate_adx, calculate_atr, calculate_dmi, calculate_ema, calculate_rsi
from .backtest_core import (
BacktestReport,
CostModel,
Trade,
build_report,
calc_profit,
fill_price,
)
@dataclass
class SimState:
side: str | None = None
entry: float = 0.0
entry_i: int = 0
entry_time: Any = None
sl: float = 0.0
tp: float = 0.0
bars_against: int = 0
rsi_against: bool = False
def _run_single_position(
df: pd.DataFrame,
symbol: str,
point: float,
costs: CostModel,
lot: float,
strategy_id: str,
tf_label: str,
period_label: str,
params: dict,
initial_balance: float,
on_bar,
) -> BacktestReport:
trades: list[Trade] = []
equity = [initial_balance]
st = SimState()
def close(i: int, mid: float, reason: str) -> None:
nonlocal st
if st.side is None:
return
exit_px = fill_price(mid, point, costs, st.side, entry=False)
commission = costs.commission_per_lot * lot * 2
profit = calc_profit(symbol, st.side, lot, st.entry, exit_px) - commission
trades.append(
Trade(
side=st.side,
open_time=st.entry_time,
close_time=df.index[i],
open_price=st.entry,
close_price=exit_px,
volume=lot,
profit=profit,
bars_held=i - st.entry_i,
exit_reason=reason,
)
)
equity.append(equity[-1] + profit)
st = SimState()
def open_pos(i: int, side: str, mid: float) -> None:
nonlocal st
st.side = side
st.entry = fill_price(mid, point, costs, side, entry=True)
st.entry_i = i
st.entry_time = df.index[i]
st.sl = 0.0
st.tp = 0.0
st.bars_against = 0
st.rsi_against = False
for i in range(1, len(df)):
mid = float(df["open"].iloc[i])
on_bar(i, st, open_pos, close)
if len(equity) == len(trades) + 1:
equity.append(equity[-1])
if st.side is not None:
close(len(df) - 1, float(df["close"].iloc[-1]), "eod")
eq = pd.Series(equity[: len(df)], index=df.index[: len(equity)])
return build_report(strategy_id, symbol, tf_label, period_label, trades, eq, initial_balance, params)
# --- RSI Scalping ---
def backtest_rsi_scalp(
df: pd.DataFrame,
symbol: str,
period_label: str,
strategy_id: str,
params: dict,
lot: float = 0.1,
costs: CostModel | None = None,
) -> BacktestReport:
info = __import__("MetaTrader5").symbol_info(symbol)
point = float(info.point) if info else 0.01
costs = costs or CostModel.for_symbol(symbol)
rsi = calculate_rsi(df["close"], int(params["rsi_period"])).to_numpy()
atr = calculate_atr(df, int(params.get("reversal_atr_period", 14))).to_numpy()
trail_dist = params.get("trail_distance_pts", 0) * point
trail_act = (params.get("trail_activation_pts") or params.get("trail_distance_pts", 0)) * point
use_rsi_against = params.get("use_rsi_against_exit", True)
max_adv_atr = float(params.get("max_adverse_atr", 0))
def on_bar(i, st, open_pos, close):
if i < 3 or np.isnan(rsi[i - 1]):
return
sig, prev, two = rsi[i - 1], rsi[i - 2], rsi[i - 3]
mid = float(df["open"].iloc[i])
hi, lo = float(df["high"].iloc[i]), float(df["low"].iloc[i])
if st.side is not None and params.get("use_reversal_escape"):
a = float(atr[i - 1]) if not np.isnan(atr[i - 1]) else 0.0
if a > 0:
adv_mult = float(params.get("reversal_adverse_atr_mult", 1.5))
rsi_vel = float(params.get("reversal_rsi_velocity", 8.0))
need = int(params.get("reversal_signs_required", 2))
signs = 0
if st.side == "BUY":
if st.entry - lo >= adv_mult * a:
signs += 1
if sig - prev >= rsi_vel:
signs += 1
else:
if hi - st.entry >= adv_mult * a:
signs += 1
if sig - prev >= rsi_vel:
signs += 1
if signs >= need:
close(i, mid, "reversal_escape")
return
if st.side is not None and params.get("use_trailing") and trail_dist > 0:
if st.side == "BUY":
bid = float(df["close"].iloc[i])
if bid - st.entry > trail_act:
nsl = bid - trail_dist
if nsl > st.sl:
st.sl = nsl
if st.sl > 0 and lo <= st.sl:
close(i, st.sl, "trail")
return
else:
ask = float(df["close"].iloc[i])
if st.entry - ask > trail_act:
nsl = ask + trail_dist
if st.sl == 0 or nsl < st.sl:
st.sl = nsl
if st.sl > 0 and hi >= st.sl:
close(i, st.sl, "trail")
return
if st.side is not None and max_adv_atr > 0:
a = float(atr[i - 1]) if not np.isnan(atr[i - 1]) else 0.0
if a > 0:
if st.side == "BUY" and (st.entry - lo) / a >= max_adv_atr:
close(i, mid, "adverse_atr")
return
if st.side == "SELL" and (hi - st.entry) / a >= max_adv_atr:
close(i, mid, "adverse_atr")
return
if st.side == "BUY":
if use_rsi_against and sig < params["rsi_oversold"]:
st.bars_against = st.bars_against + 1 if st.rsi_against else 1
st.rsi_against = True
if st.bars_against >= params["bars_to_wait"]:
close(i, mid, "rsi_against")
else:
st.rsi_against = False
st.bars_against = 0
if sig >= params["rsi_target_buy"]:
close(i, mid, "target")
elif st.side == "SELL":
if use_rsi_against and sig > params["rsi_overbought"]:
st.bars_against = st.bars_against + 1 if st.rsi_against else 1
st.rsi_against = True
if st.bars_against >= params["bars_to_wait"]:
close(i, mid, "rsi_against")
else:
st.rsi_against = False
st.bars_against = 0
if sig <= params["rsi_target_sell"]:
close(i, mid, "target")
else:
if two <= params["rsi_oversold"] and prev > params["rsi_oversold"]:
open_pos(i, "BUY", mid)
elif two >= params["rsi_overbought"] and prev < params["rsi_overbought"]:
min_depth = float(params.get("min_ob_depth", 0))
if two < params["rsi_overbought"] + min_depth:
pass
else:
skip_h = int(params.get("skip_short_hour_after", 24))
if df.index[i].hour < skip_h:
open_pos(i, "SELL", mid)
return _run_single_position(
df, symbol, point, costs, lot, strategy_id, params.get("tf", "H1"),
period_label, params, 10_000.0, on_bar,
)
# --- RSI CrossOver ---
def _price_to_ema_pips(symbol: str, close: float, ema: float) -> float:
info = __import__("MetaTrader5").symbol_info(symbol)
if info is None:
return abs(close - ema) * 10.0
point = float(info.point)
digits = int(info.digits)
pip_mult = 10.0 if digits in (3, 5) else 1.0
pip_size = point * pip_mult if point > 0 else point
return abs(close - ema) / pip_size if pip_size > 0 else 0.0
def backtest_rsi_crossover(df, symbol, period_label, strategy_id, params, lot=0.1, costs=None):
info = __import__("MetaTrader5").symbol_info(symbol)
point = float(info.point) if info else 0.01
costs = costs or CostModel.for_symbol(symbol)
rsi = calculate_rsi(df["close"], int(params["rsi_period"])).to_numpy()
ema = calculate_ema(df["close"], int(params["ema_period"])).to_numpy()
trail = params.get("trailing_stop_pts", 0) * point
prev_rsi_state = 0.0
last_trade_i = -10_000
cooldown_bars = max(1, int(params.get("cooldown_seconds", 300) / 3600))
use_trend_filter = params.get("use_trend_strength_filter", True)
weekday_ok = {
0: params.get("sunday", False),
1: params.get("monday", False),
2: params.get("tuesday", True),
3: params.get("wednesday", True),
4: params.get("thursday", True),
5: params.get("friday", False),
6: params.get("saturday", False),
}
def hours_ok(ts) -> bool:
h = ts.hour
def in_win(begin: int, end: int) -> bool:
b, e = begin % 24, end % 24
if b == e:
return False
if b < e:
return b <= h < e
return h >= b or h < e
return in_win(params.get("trading_hour_one_begin", 0), params.get("trading_hour_one_end", 22)) or in_win(
params.get("trading_hour_two_begin", 6), params.get("trading_hour_two_end", 19)
)
def on_bar(i, st, open_pos, close):
nonlocal prev_rsi_state, last_trade_i
if i < 3 or np.isnan(rsi[i - 1]) or np.isnan(ema[i - 1]):
return
ts = df.index[i]
if not weekday_ok.get(ts.weekday(), False) or not hours_ok(ts):
if st.side:
close(i, float(df["open"].iloc[i]), "hours")
return
cur = rsi[i - 1]
if prev_rsi_state == 0.0:
prev_rsi_state = cur
return
ema_slope = (ema[i - 1] - ema[i - 2]) * 100.0
price_to_ema = abs((float(df["close"].iloc[i - 1]) - ema[i - 1]) * 10.0)
slope_th = float(params.get("ema_slope_threshold", 100))
dist_th = float(params.get("ema_distance_threshold", 100))
trend_strong = use_trend_filter and (
(slope_th > 0 and abs(ema_slope) > slope_th)
or (dist_th > 0 and price_to_ema > dist_th)
)
mid = float(df["open"].iloc[i])
if st.side == "BUY" and trail > 0:
bid = float(df["close"].iloc[i])
if bid - st.entry > trail:
st.sl = max(st.sl, bid - trail)
if st.sl > 0 and float(df["low"].iloc[i]) <= st.sl:
close(i, st.sl, "trail")
prev_rsi_state = cur
return
if st.side == "SELL" and trail > 0:
ask = float(df["close"].iloc[i])
if st.entry - ask > trail:
st.sl = ask + trail if st.sl == 0 else min(st.sl, ask + trail)
if st.sl > 0 and float(df["high"].iloc[i]) >= st.sl:
close(i, st.sl, "trail")
prev_rsi_state = cur
return
if st.side == "BUY" and cur > params.get("exit_buy_rsi", 80):
close(i, mid, "exit_rsi")
elif st.side == "SELL" and cur < params.get("exit_sell_rsi", 20):
close(i, mid, "exit_rsi")
elif trend_strong and st.side:
close(i, mid, "trend_strong")
elif not st.side and not trend_strong and i - last_trade_i >= cooldown_bars:
ob = params.get("overbought_level", 70)
os = params.get("oversold_level", 30)
sell_spread = params.get("entry_rsi_sell_spread", 0)
buy_spread = params.get("entry_rsi_buy_spread", 0)
if prev_rsi_state >= ob and cur < ob - sell_spread:
open_pos(i, "SELL", mid)
last_trade_i = i
elif prev_rsi_state <= os and cur > os + buy_spread:
open_pos(i, "BUY", mid)
last_trade_i = i
prev_rsi_state = cur
return _run_single_position(
df, symbol, point, costs, lot, strategy_id, "H1", period_label, params, 10_000.0, on_bar,
)
# --- RSI Asian ---
def backtest_rsi_asian(df, symbol, period_label, strategy_id, params, lot=0.1, costs=None):
info = __import__("MetaTrader5").symbol_info(symbol)
point = float(info.point) if info else 0.01
costs = costs or CostModel.for_symbol(symbol)
rsi = calculate_rsi(df["close"], int(params["rsi_period"])).to_numpy()
sess_start = params.get("asian_session_start", 0)
sess_end = params.get("asian_session_end", 8)
def in_session(ts) -> bool:
return sess_start <= ts.hour < sess_end
def on_bar(i, st, open_pos, close):
if i < 2 or np.isnan(rsi[i - 1]):
return
ts = df.index[i]
prev, cur = rsi[i - 2], rsi[i - 1]
mid = float(df["open"].iloc[i])
if st.side and params.get("close_outside_session") and not in_session(ts):
close(i, mid, "session")
return
if st.side and params.get("use_rsi_exit"):
exit_lvl = params.get("rsi_exit_level", 55)
if (prev < exit_lvl <= cur) or (prev > exit_lvl >= cur):
close(i, mid, "rsi_exit")
if not in_session(ts):
return
if st.side:
return
if prev < params["overbought_level"] <= cur:
open_pos(i, "SELL", mid)
elif prev > params["oversold_level"] >= cur:
open_pos(i, "BUY", mid)
return _run_single_position(
df, symbol, point, costs, lot, strategy_id, "M15", period_label, params, 10_000.0, on_bar,
)
# --- Mean Reversion ---
def backtest_mean_reversion(df, symbol, period_label, strategy_id, params, lot=0.1, costs=None):
info = __import__("MetaTrader5").symbol_info(symbol)
point = float(info.point) if info else 0.01
costs = costs or CostModel.for_symbol(symbol)
rsi = calculate_rsi(df["close"], int(params["rsi_period"])).to_numpy()
ema = calculate_ema(df["close"], int(params["ema_period"])).to_numpy()
adx = calculate_adx(df, int(params.get("adx_period", 14))).to_numpy()
def on_bar(i, st, open_pos, close):
if i < max(params["ema_period"], 20) + 2:
return
if np.isnan(rsi[i - 1]) or np.isnan(ema[i - 1]) or np.isnan(adx[i - 1]):
return
mid = float(df["open"].iloc[i])
cls = float(df["close"].iloc[i - 1])
dist_buy = (ema[i - 1] - cls) / point
dist_sell = (cls - ema[i - 1]) / point
adx_v = float(adx[i - 1])
if st.side:
adx_now = float(adx[i - 1]) if not np.isnan(adx[i - 1]) else 0.0
if adx_now >= params.get("adx_escape", 30):
close(i, mid, "adx_escape")
elif params.get("use_hard_sltp"):
if st.side == "BUY":
if cls <= st.entry - params.get("sl_points", 0) * point:
close(i, mid, "sl")
elif cls >= st.entry + params.get("tp_points", 0) * point:
close(i, mid, "tp")
else:
if cls >= st.entry + params.get("sl_points", 0) * point:
close(i, mid, "sl")
elif cls <= st.entry - params.get("tp_points", 0) * point:
close(i, mid, "tp")
return
if adx_v <= 0 or adx_v >= params.get("adx_max_for_entry", 20):
return
use_cross = params.get("use_rsi_cross", True)
if use_cross:
buy_rsi = rsi[i - 2] > params["rsi_oversold"] >= rsi[i - 1]
sell_rsi = rsi[i - 2] < params["rsi_overbought"] <= rsi[i - 1]
else:
buy_rsi = rsi[i - 1] <= params["rsi_oversold"]
sell_rsi = rsi[i - 1] >= params["rsi_overbought"]
if buy_rsi and dist_buy >= params.get("min_ema_distance_pts", 0):
open_pos(i, "BUY", mid)
if params.get("use_hard_sltp"):
st.sl = st.entry - params.get("sl_points", 0) * point
st.tp = st.entry + params.get("tp_points", 0) * point
elif sell_rsi and dist_sell >= params.get("min_ema_distance_pts", 0):
open_pos(i, "SELL", mid)
if params.get("use_hard_sltp"):
st.sl = st.entry + params.get("sl_points", 0) * point
st.tp = st.entry - params.get("tp_points", 0) * point
return _run_single_position(
df, symbol, point, costs, lot, strategy_id, "M15", period_label, params, 10_000.0, on_bar,
)
# --- EMA Slope (monitor + crossover state machine, matches MQL) ---
def _weekly_dmi_lookup(df: pd.DataFrame, period: int, bar_shift: int) -> tuple[pd.Series, pd.Series, pd.Series]:
wdf = df.resample("W-FRI").agg({"high": "max", "low": "min", "close": "last"}).dropna()
dmi = calculate_dmi(wdf, period)
shift = max(0, bar_shift)
adx = dmi["adx"].shift(shift).reindex(df.index, method="ffill")
plus = dmi["plus_di"].shift(shift).reindex(df.index, method="ffill")
minus = dmi["minus_di"].shift(shift).reindex(df.index, method="ffill")
return adx, plus, minus
def backtest_ema_slope(df, symbol, period_label, strategy_id, params, lot=0.1, costs=None):
info = __import__("MetaTrader5").symbol_info(symbol)
point = float(info.point) if info else 0.01
costs = costs or CostModel.for_symbol(symbol)
ema_period = int(params["ema_period"])
ema = calculate_ema(df["close"], ema_period).to_numpy()
atr = calculate_atr(df, 14).to_numpy()
closes = df["close"].to_numpy()
opens = df["open"].to_numpy()
highs = df["high"].to_numpy()
lows = df["low"].to_numpy()
times = df.index
mult = 10.0 if ("XAU" in symbol or "BTC" in symbol) else 1.0
w_adx, w_plus, w_minus = _weekly_dmi_lookup(
df, int(params.get("weekly_adx_period", 28)), int(params.get("weekly_adx_bar_shift", 8))
)
price_trigger_active = False
slope_trigger_active = False
monitor_active = False
monitor_start_i = -1
trades_in_cross = 0
last_close = 0.0
last_ema = 0.0
last_bar_time = None
def weekly_ok(i: int, side: str) -> bool:
if not params.get("use_weekly_adx_filter", True):
return True
adx_v = float(w_adx.iloc[i - 1]) if i > 0 else np.nan
if np.isnan(adx_v) or adx_v < params.get("weekly_adx_min", 25):
return False
if not params.get("weekly_adx_use_direction", True):
return True
pdi = float(w_plus.iloc[i - 1])
mdi = float(w_minus.iloc[i - 1])
if side == "BUY":
return pdi > mdi
return mdi > pdi
def on_bar(i, st, open_pos, close):
nonlocal price_trigger_active, slope_trigger_active, monitor_active, monitor_start_i
nonlocal trades_in_cross, last_close, last_ema, last_bar_time
if i < ema_period + 3 or np.isnan(ema[i - 1]) or np.isnan(ema[i - 2]):
return
if params.get("use_bar_data", True):
ts = times[i]
if last_bar_time is not None and ts == last_bar_time:
return
last_bar_time = ts
mid = float(opens[i])
bar_close = float(closes[i - 1])
ema_now = float(ema[i - 1])
ema_prev = float(ema[i - 2])
if last_close != 0.0:
if (last_close <= last_ema and bar_close > ema_now) or (last_close >= last_ema and bar_close < ema_now):
trades_in_cross = 0
last_close, last_ema = bar_close, ema_now
price_dist = abs(bar_close - ema_now) / point / mult
if price_dist > params.get("price_threshold_pips", 100) and not price_trigger_active:
price_trigger_active = True
slope = (ema_now - ema_prev) / point / mult
if abs(slope) > params.get("slope_threshold_pips", 20) and not slope_trigger_active:
slope_trigger_active = True
if price_trigger_active and slope_trigger_active and not monitor_active:
monitor_active = True
monitor_start_i = i
tf_sec = 3600
timeout_bars = int(params.get("monitor_timeout_sec", 340) / tf_sec)
if monitor_active and monitor_start_i >= 0 and (i - monitor_start_i) > timeout_bars:
monitor_active = False
price_trigger_active = False
slope_trigger_active = False
if st.side:
bars_open = i - st.entry_i
bar_close_now = float(closes[i])
a = float(atr[i - 1]) if not np.isnan(atr[i - 1]) else 0.0
max_loss_atr = float(params.get("max_loss_atr", 2.0))
if a > 0 and max_loss_atr > 0:
if st.side == "BUY" and float(lows[i]) <= st.entry - max_loss_atr * a:
close(i, st.entry - max_loss_atr * a, "atr_sl")
return
if st.side == "SELL" and float(highs[i]) >= st.entry + max_loss_atr * a:
close(i, st.entry + max_loss_atr * a, "atr_sl")
return
trail_pips = params.get("trailing_stop_pips", 50)
trail_px = trail_pips * point * mult
bar_close_now = float(closes[i])
in_profit = (bar_close_now > st.entry) if st.side == "BUY" else (st.entry > bar_close_now)
use_trail = params.get("use_trailing_stop", False)
trail_act = params.get("trailing_activation_pips", 0)
trail_ready = in_profit if trail_act <= 0 else (
(bar_close_now - st.entry) / point / mult >= trail_act
if st.side == "BUY"
else (st.entry - bar_close_now) / point / mult >= trail_act
)
if trail_pips > 0 and (use_trail or in_profit) and trail_ready:
if st.side == "BUY":
st.sl = max(st.sl, bar_close_now - trail_px)
if st.sl > 0 and float(lows[i]) <= st.sl:
close(i, st.sl, "trail")
return
else:
st.sl = bar_close_now + trail_px if st.sl <= 0 else min(st.sl, bar_close_now + trail_px)
if st.sl > 0 and float(highs[i]) >= st.sl:
close(i, st.sl, "trail")
return
ema_exit = (st.side == "BUY" and bar_close_now < ema_now) or (
st.side == "SELL" and bar_close_now > ema_now
)
unrealized = calc_profit(symbol, st.side, lot, st.entry, bar_close_now)
if ema_exit and unrealized > 0:
close(i, mid, "ema_cross")
return
if params.get("close_unprofitable_trades", True):
check_bars = int(params.get("profit_check_bars", 78))
if bars_open >= check_bars:
unrealized = calc_profit(symbol, st.side, lot, st.entry, bar_close_now)
if unrealized <= 0:
close(i, mid, "unprofitable")
return
return
if not monitor_active:
return
if trades_in_cross >= params.get("max_trades_per_crossover", 5):
return
if bar_close > ema_now and weekly_ok(i, "BUY"):
open_pos(i, "BUY", mid)
trades_in_cross += 1
monitor_active = False
price_trigger_active = False
slope_trigger_active = False
elif bar_close < ema_now and weekly_ok(i, "SELL"):
open_pos(i, "SELL", mid)
trades_in_cross += 1
monitor_active = False
price_trigger_active = False
slope_trigger_active = False
return _run_single_position(
df, symbol, point, costs, lot, strategy_id, "H1", period_label, params, 10_000.0, on_bar,
)
# --- Darvas Box (matches MQL: narrow box + breakout + trend strength) ---
def backtest_darvas(df, symbol, period_label, strategy_id, params, lot=0.1, costs=None):
info = __import__("MetaTrader5").symbol_info(symbol)
point = float(info.point) if info else 0.01
costs = costs or CostModel.for_symbol(symbol)
period = int(params.get("box_period", 165))
box_dev = float(params.get("box_deviation", 30000))
trend_thresh = float(params.get("trend_threshold", 4.94))
ma_period = int(params.get("ma_period", 125))
sl_pts = float(params.get("stop_loss_pts", 1665))
tp_pts = float(params.get("take_profit_pts", 3685))
vol_thresh = int(params.get("volume_threshold", 0))
max_range = box_dev * point
ma = calculate_ema(df["close"], ma_period).to_numpy()
highs = df["high"].to_numpy()
lows = df["low"].to_numpy()
opens = df["open"].to_numpy()
closes = df["close"].to_numpy()
vols = df["tick_volume"].to_numpy() if "tick_volume" in df.columns else np.zeros(len(df))
def on_bar(i, st, open_pos, close):
if i < period + ma_period + 2:
return
window_hi = float(np.max(highs[i - period : i]))
window_lo = float(np.min(lows[i - period : i]))
if (window_hi - window_lo) > max_range:
return
mid = float(opens[i])
bar_hi = float(highs[i])
bar_lo = float(lows[i])
ma_v = float(ma[i - 1])
if np.isnan(ma_v):
return
if st.side:
if st.side == "BUY":
if st.sl > 0 and bar_lo <= st.sl:
close(i, st.sl, "sl")
elif st.tp > 0 and bar_hi >= st.tp:
close(i, st.tp, "tp")
else:
if st.sl > 0 and bar_hi >= st.sl:
close(i, st.sl, "sl")
elif st.tp > 0 and bar_lo <= st.tp:
close(i, st.tp, "tp")
return
if vols[i] <= vol_thresh:
return
prev_close = float(closes[i - 1])
strength = abs(mid - ma_v) / point
break_up = bar_hi > window_hi or prev_close > window_hi
break_dn = bar_lo < window_lo or prev_close < window_lo
if break_up and mid > ma_v and strength > trend_thresh:
open_pos(i, "BUY", mid)
st.sl = st.entry - sl_pts * point
st.tp = st.entry + tp_pts * point
elif break_dn and mid < ma_v and strength > trend_thresh:
open_pos(i, "SELL", mid)
st.sl = st.entry + sl_pts * point
st.tp = st.entry - tp_pts * point
return _run_single_position(
df, symbol, point, costs, lot, strategy_id, "M15", period_label, params, 10_000.0, on_bar,
)
# --- RSI Secret Sauce (simplified zone exit re-entry) ---
def backtest_rsi_secret(df, symbol, period_label, strategy_id, params, lot=0.1, costs=None):
info = __import__("MetaTrader5").symbol_info(symbol)
point = float(info.point) if info else 0.01
costs = costs or CostModel.for_symbol(symbol)
rsi = calculate_rsi(df["close"], int(params["rsi_period"])).to_numpy()
atr = calculate_atr(df, int(params.get("atr_period", 14))).to_numpy()
last_trade_i = -999
def on_bar(i, st, open_pos, close):
nonlocal last_trade_i
if i < 30 or np.isnan(rsi[i - 1]) or np.isnan(atr[i - 1]):
return
mid = float(df["open"].iloc[i])
cur, prev = rsi[i - 1], rsi[i - 2]
a = atr[i - 1]
if st.side:
if st.side == "BUY":
sl = st.entry - params.get("stop_loss_atr", 2) * a
tp = st.entry + params.get("take_profit_atr", 4) * a
if float(df["low"].iloc[i]) <= sl:
close(i, sl, "sl")
elif float(df["high"].iloc[i]) >= tp:
close(i, tp, "tp")
else:
sl = st.entry + params.get("stop_loss_atr", 2) * a
tp = st.entry - params.get("take_profit_atr", 4) * a
if float(df["high"].iloc[i]) >= sl:
close(i, sl, "sl")
elif float(df["low"].iloc[i]) <= tp:
close(i, tp, "tp")
return
if i - last_trade_i < params.get("min_bars_between_trades", 5):
return
ob, os = params["rsi_overbought"], params["rsi_oversold"]
if prev > ob and cur <= ob:
open_pos(i, "SELL", mid)
last_trade_i = i
elif prev < os and cur >= os:
open_pos(i, "BUY", mid)
last_trade_i = i
return _run_single_position(
df, symbol, point, costs, lot, strategy_id, "M30", period_label, params, 10_000.0, on_bar,
)
# --- Simple Trendline (pullback to MA-derived trendline) ---
def backtest_simple_trendline(df, symbol, period_label, strategy_id, params, lot=0.1, costs=None):
info = __import__("MetaTrader5").symbol_info(symbol)
point = float(info.point) if info else 0.01
costs = costs or CostModel.for_symbol(symbol)
htf = params.get("higher_tf", "H4")
htf_map = {"M10": "10min", "M15": "15min", "H1": "1h", "H4": "4h"}
rule = htf_map.get(htf, "4h")
hdf = df.resample(rule).agg({"open": "first", "high": "max", "low": "min", "close": "last"}).dropna()
ma_period = int(params.get("ma_period", 65))
ma = calculate_ema(hdf["close"], ma_period).to_numpy()
htimes = hdf.index.to_numpy()
hcloses = hdf["close"].to_numpy()
touch_tol = float(params.get("touch_tolerance_pts", 100)) * point
break_buf = float(params.get("break_buffer_pts", 80)) * point
def line_at(t_model, t_query):
x = (t_query - t_model[0]).astype("timedelta64[s]").astype(float)
return t_model[2] * x + t_model[3]
def on_bar(i, st, open_pos, close):
if i < 5:
return
ts = df.index[i]
# find 3 most recent HTF MA crosses before ts
hidx = int(np.searchsorted(htimes, ts, side="right")) - 1
if hidx < ma_period + 5:
return
crosses_t, crosses_p = [], []
for j in range(hidx, ma_period + 2, -1):
if j >= len(ma) - 1:
continue
d0 = hcloses[j] - ma[j]
d1 = hcloses[j + 1] - ma[j + 1]
if d0 == 0 or d1 == 0 or d0 * d1 < 0:
crosses_t.append(htimes[j])
crosses_p.append(hcloses[j])
if len(crosses_t) >= 3:
break
if len(crosses_t) < 3:
return
t0 = crosses_t[2]
xs = np.array([(t - t0).astype("timedelta64[s]").astype(float) for t in crosses_t[::-1]])
ys = np.array(crosses_p[::-1])
den = 3 * np.sum(xs ** 2) - np.sum(xs) ** 2
if abs(den) < 1e-10:
return
a = (3 * np.sum(xs * ys) - np.sum(xs) * np.sum(ys)) / den
b = (np.sum(ys) - a * np.sum(xs)) / 3
model = (t0, crosses_t, a, b)
t1 = df.index[i - 1]
line1 = a * (t1 - t0).astype("timedelta64[s]").astype(float) + b
mid = float(df["open"].iloc[i])
hi = float(df["high"].iloc[i - 1])
lo = float(df["low"].iloc[i - 1])
cl1 = float(df["close"].iloc[i - 1])
op1 = float(df["open"].iloc[i - 1])
cl2 = float(df["close"].iloc[i - 2])
t2 = df.index[i - 2]
line2 = a * (t2 - t0).astype("timedelta64[s]").astype(float) + b
if st.side == "BUY" and cl1 < line1 - break_buf:
close(i, mid, "break")
return
if st.side == "SELL" and cl1 > line1 + break_buf:
close(i, mid, "break")
return
if st.side:
return
if a > 0:
if lo <= line1 + touch_tol and cl1 > line1 and cl1 > op1 and cl2 >= line2 - touch_tol:
open_pos(i, "BUY", mid)
elif a < 0:
if hi >= line1 - touch_tol and cl1 < line1 and cl1 < op1 and cl2 <= line2 + touch_tol:
open_pos(i, "SELL", mid)
return _run_single_position(
df, symbol, point, costs, lot, strategy_id, params.get("signal_tf", "H1"),
period_label, params, 10_000.0, on_bar,
)
# --- USDJPY Asian range breakout (simplified market-fill) ---
def backtest_usdjpy_buster(df, symbol, period_label, strategy_id, params, lot=0.1, costs=None):
info = __import__("MetaTrader5").symbol_info(symbol)
point = float(info.point) if info else 0.001
costs = costs or CostModel.for_symbol(symbol)
r_start = int(params.get("range_start_hour", 3))
r_end = int(params.get("range_end_hour", 6))
close_h = int(params.get("close_hour", 18))
min_rng = float(params.get("min_range_pts", 15))
buf = float(params.get("order_buffer_pts", 4.75)) * point
first_only = params.get("first_trade_only", False)
day_state: dict = {}
def on_bar(i, st, open_pos, close):
ts = df.index[i]
dk = ts.date().isoformat()
h = ts.hour
mid = float(df["open"].iloc[i])
if st.side and h >= close_h:
close(i, mid, "eod")
return
if dk not in day_state:
day_state[dk] = {"hi": -np.inf, "lo": np.inf, "built": False, "trades": 0, "range_done": False}
ds = day_state[dk]
if r_start <= h < r_end:
ds["hi"] = max(ds["hi"], float(df["high"].iloc[i]))
ds["lo"] = min(ds["lo"], float(df["low"].iloc[i]))
return
if not ds["range_done"] and h >= r_end:
ds["range_done"] = True
if ds["hi"] > ds["lo"] and (ds["hi"] - ds["lo"]) / point >= min_rng:
ds["built"] = True
if not ds["built"] or st.side:
return
max_tr = 1 if first_only else 2
if ds["trades"] >= max_tr:
return
hi = ds["hi"] + buf
lo = ds["lo"] - buf
bar_hi = float(df["high"].iloc[i])
bar_lo = float(df["low"].iloc[i])
if params.get("allow_long", True) and bar_hi >= hi:
open_pos(i, "BUY", mid)
st.sl = ds["lo"]
ds["trades"] += 1
elif params.get("allow_short", True) and bar_lo <= lo:
open_pos(i, "SELL", mid)
st.sl = ds["hi"]
ds["trades"] += 1
if st.side:
if st.side == "BUY" and bar_lo <= st.sl:
close(i, st.sl, "sl")
elif st.side == "SELL" and bar_hi >= st.sl:
close(i, st.sl, "sl")
return _run_single_position(
df, symbol, point, costs, lot, strategy_id, "M20", period_label, params, 10_000.0, on_bar,
)
ENGINE_MAP = {
"rsi_scalp": backtest_rsi_scalp,
"rsi_crossover": backtest_rsi_crossover,
"rsi_asian": backtest_rsi_asian,
"mean_reversion": backtest_mean_reversion,
"ema_slope": backtest_ema_slope,
"darvas": backtest_darvas,
"rsi_secret": backtest_rsi_secret,
"simple_trendline": backtest_simple_trendline,
"usdjpy_buster": backtest_usdjpy_buster,
}
backtesting/MT5/cluster_audit/loss_context_analysis.py
+277
View File
@@ -0,0 +1,277 @@
"""
Analyze losing trades in market context — bars before/after, gaps between losses,
RSI/ATR/trend features. Trader-style narrative + param suggestions.
Usage:
python -m cluster_audit.loss_context_analysis united_rsi_scalp_appl
python -m cluster_audit.loss_context_analysis united_darvas
"""
from __future__ import annotations
import json
import sys
from datetime import datetime, timedelta
from pathlib import Path
import MetaTrader5 as mt5
import numpy as np
import pandas as pd
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
from cluster_audit.backtest_core import CostModel, Trade, load_bars, resolve_symbol
from cluster_audit.engines import ENGINE_MAP
from cluster_audit.united_registry import PERIODS, UNITED_STRATEGIES
from indicator_utils import calculate_atr, calculate_ema, calculate_rsi
OUT_DIR = Path(__file__).parent / "reports" / "loss_analysis"
CONTEXT_BARS = 12 # bars before entry + through hold
def find_spec(sid: str) -> dict:
for s in UNITED_STRATEGIES:
if s["id"] == sid:
return s
raise KeyError(sid)
def run_backtest(spec: dict, start: str, end: str) -> tuple[pd.DataFrame, list[Trade], dict]:
from cluster_audit.strategy_registry import TF
sym = resolve_symbol(spec["symbol"])
tf_key = spec["tf"]
engine = ENGINE_MAP[spec["engine"]]
params = dict(spec["defaults"])
df = load_bars(sym, TF[tf_key], datetime.fromisoformat(start), datetime.fromisoformat(end))
report = engine(df, sym, "2021-2026", spec["id"], params, spec["lot"], CostModel.for_symbol(sym))
return df, report.trades_list, params
def bar_features(df: pd.DataFrame, idx: int, rsi: np.ndarray, atr: np.ndarray, ema20: np.ndarray) -> dict:
if idx < 1 or idx >= len(df):
return {}
o, h, l, c = df.iloc[idx][["open", "high", "low", "close"]]
prev_c = float(df.iloc[idx - 1]["close"])
body = abs(c - o)
rng = h - l if h > l else 1e-9
return {
"rsi": float(rsi[idx - 1]) if not np.isnan(rsi[idx - 1]) else np.nan,
"atr": float(atr[idx - 1]) if not np.isnan(atr[idx - 1]) else np.nan,
"ema20": float(ema20[idx - 1]) if not np.isnan(ema20[idx - 1]) else np.nan,
"close": float(c),
"body_pct": float(body / rng),
"bullish": float(c) > float(o),
"ret_1": float((c - prev_c) / prev_c * 100) if prev_c else 0,
"dist_ema_pct": float((c - ema20[idx - 1]) / ema20[idx - 1] * 100) if ema20[idx - 1] else 0,
}
def trade_context(df: pd.DataFrame, t: Trade, rsi, atr, ema20) -> dict:
open_i = df.index.get_indexer([pd.Timestamp(t.open_time)], method="nearest")[0]
close_i = df.index.get_indexer([pd.Timestamp(t.close_time)], method="nearest")[0]
pre_start = max(1, open_i - CONTEXT_BARS)
pre_bars = []
for i in range(pre_start, open_i):
pre_bars.append(bar_features(df, i, rsi, atr, ema20))
hold_bars = []
for i in range(open_i, min(close_i + 1, len(df))):
hold_bars.append(bar_features(df, i, rsi, atr, ema20))
entry_f = bar_features(df, open_i, rsi, atr, ema20)
exit_f = bar_features(df, close_i, rsi, atr, ema20)
pre_rsi = [b["rsi"] for b in pre_bars if not np.isnan(b.get("rsi", np.nan))]
hold_rsi = [b["rsi"] for b in hold_bars if not np.isnan(b.get("rsi", np.nan))]
adverse_move = 0.0
if t.side == "BUY" and hold_bars:
adverse_move = float(t.open_price) - min(b["close"] for b in hold_bars)
elif t.side == "SELL" and hold_bars:
adverse_move = max(b["close"] for b in hold_bars) - float(t.open_price)
return {
"side": t.side,
"exit_reason": t.exit_reason,
"profit": t.profit,
"bars_held": t.bars_held,
"open_time": str(t.open_time),
"close_time": str(t.close_time),
"entry_rsi": entry_f.get("rsi"),
"exit_rsi": exit_f.get("rsi"),
"rsi_min_hold": min(hold_rsi) if hold_rsi else None,
"rsi_max_hold": max(hold_rsi) if hold_rsi else None,
"rsi_trend_pre": (pre_rsi[-1] - pre_rsi[0]) if len(pre_rsi) >= 2 else 0,
"adverse_pts": adverse_move,
"adverse_atr": adverse_move / entry_f["atr"] if entry_f.get("atr") else 0,
"entry_hour": pd.Timestamp(t.open_time).hour,
"entry_dist_ema_pct": entry_f.get("dist_ema_pct", 0),
"pre_bullish_ratio": sum(1 for b in pre_bars if b.get("bullish")) / max(len(pre_bars), 1),
"entry_body_pct": entry_f.get("body_pct", 0),
}
def gap_analysis(losers: list[dict]) -> dict:
if len(losers) < 2:
return {}
times = sorted(pd.Timestamp(t["close_time"]) for t in losers)
gaps_h = [(times[i] - times[i - 1]).total_seconds() / 3600 for i in range(1, len(times))]
return {
"median_gap_hours": float(np.median(gaps_h)),
"pct_gap_under_4h": float(sum(1 for g in gaps_h if g < 4) / len(gaps_h) * 100),
"pct_gap_under_24h": float(sum(1 for g in gaps_h if g < 24) / len(gaps_h) * 100),
"clustered": float(sum(1 for g in gaps_h if g < 2) / len(gaps_h) * 100),
}
def trader_narrative(sid: str, engine: str, losers_ctx: list[dict], winners_ctx: list[dict], by_reason: dict) -> list[str]:
notes: list[str] = []
if not losers_ctx:
return ["No losing trades to analyze."]
top_reason = max(by_reason.items(), key=lambda x: x[1]["count"])[0]
lr = [c for c in losers_ctx if c["exit_reason"] == top_reason]
wr = winners_ctx
if top_reason == "rsi_against":
sell_l = [c for c in lr if c["side"] == "SELL"]
buy_l = [c for c in lr if c["side"] == "BUY"]
if sell_l:
avg_adv = np.mean([c["adverse_atr"] for c in sell_l if c["adverse_atr"]])
notes.append(
f"SELL rsi_against ({len(sell_l)}): price ripped up avg {avg_adv:.1f} ATR after shorting "
f"overbought fade — classic short squeeze / momentum continuation, not mean reversion."
)
late_h = sum(1 for c in sell_l if c["entry_hour"] >= 18) / len(sell_l) * 100
if late_h > 30:
notes.append(f"{late_h:.0f}% of losing shorts after 18:00 — avoid fading strength into close.")
if buy_l:
notes.append(
f"BUY rsi_against ({len(buy_l)}): dipped deeper after oversold entry — "
f"knife-catching; need deeper OS threshold or wait for RSI curl-up."
)
if wr:
w_sell = [c for c in wr if c["side"] == "SELL"]
if w_sell and sell_l:
w_rsi = np.mean([c["entry_rsi"] for c in w_sell])
l_rsi = np.mean([c["entry_rsi"] for c in sell_l])
notes.append(f"Winning shorts entered RSI~{w_rsi:.0f} vs losers~{l_rsi:.0f} — losers entered too early in OB zone.")
elif top_reason == "sl":
notes.append("SL hits: stops inside noise — widen SL to 1.5-2x ATR or reduce lot.")
avg_atr = np.mean([c["adverse_atr"] for c in lr if c.get("adverse_atr")])
notes.append(f"Avg adverse move before SL = {avg_atr:.1f} ATR — box breakout often retests.")
elif top_reason == "adverse_atr":
notes.append("ATR stop hits: entries fighting trend — fade only when RSI extreme + session filter; widen stop or skip gap-down buys.")
buy_l = [c for c in lr if c["side"] == "BUY"]
if buy_l:
late = sum(1 for c in buy_l if c["entry_hour"] >= 20) / len(buy_l) * 100
if late > 25:
notes.append(f"{late:.0f}% of stopped-out buys after 20:00 — overnight gap risk on equities.")
sell_l = [c for c in lr if c["side"] == "SELL"]
if sell_l:
avg_adv = np.mean([c["adverse_atr"] for c in sell_l if c.get("adverse_atr")])
notes.append(f"Short stops avg {avg_adv:.1f} ATR adverse — momentum continuation, not reversion.")
elif top_reason == "trail":
notes.append("Trail exits: winners cut early in chop — widen trail_distance or raise activation.")
elif top_reason == "trend_strong":
notes.append("trend_strong: exited into momentum — filter only blocks entries, don't force-close in profit.")
gaps = gap_analysis(losers_ctx)
if gaps.get("clustered", 0) > 25:
notes.append(
f"{gaps['clustered']:.0f}% of losses within 2h of prior loss — regime chop; "
f"add cooldown after loss or skip when ATR expanding."
)
return notes
def suggest_params(engine: str, losers_ctx: list[dict], params: dict) -> dict:
sug = {}
if engine == "rsi_scalp":
sell_l = [c for c in losers_ctx if c["side"] == "SELL" and c["exit_reason"] == "rsi_against"]
if sell_l and np.mean([c["adverse_atr"] for c in sell_l]) > 1.5:
sug["rsi_overbought"] = min(75, params.get("rsi_overbought", 70) + 5)
sug["bars_to_wait"] = min(12, params.get("bars_to_wait", 5) + 3)
sug["trail_distance_pts"] = params.get("trail_distance_pts", 50) * 1.4
sug["skip_short_hour_after"] = 17
buy_l = [c for c in losers_ctx if c["side"] == "BUY" and c["exit_reason"] == "rsi_against"]
if buy_l:
sug["rsi_oversold"] = max(20, params.get("rsi_oversold", 30) - 5)
elif engine == "darvas":
sug["stop_loss_pts"] = int(params.get("stop_loss_pts", 300) * 1.35)
sug["require_retest"] = True
return sug
def analyze(sid: str) -> dict:
spec = find_spec(sid)
start, end = PERIODS["2021-2026"]
df, trades, params = run_backtest(spec, start, end)
rsi = calculate_rsi(df["close"], int(params.get("rsi_period", 14))).to_numpy()
atr = calculate_atr(df, 14).to_numpy()
ema20 = calculate_ema(df["close"], 20).to_numpy()
winners = [t for t in trades if t.profit >= 0]
losers = [t for t in trades if t.profit < 0]
losers_ctx = [trade_context(df, t, rsi, atr, ema20) for t in losers]
winners_ctx = [trade_context(df, t, rsi, atr, ema20) for t in winners[:200]]
by_reason: dict = {}
for c in losers_ctx:
r = c["exit_reason"]
bucket = by_reason.setdefault(r, {"count": 0, "pnl": 0.0, "ctx": []})
bucket["count"] += 1
bucket["pnl"] += c["profit"]
bucket["ctx"].append(c)
narrative = trader_narrative(sid, spec["engine"], losers_ctx, winners_ctx, by_reason)
suggestions = suggest_params(spec["engine"], losers_ctx, params)
result = {
"strategy_id": sid,
"symbol": spec["symbol"],
"engine": spec["engine"],
"total_trades": len(trades),
"losers": len(losers),
"winners": len(winners),
"loss_by_reason": {k: {"count": v["count"], "pnl": round(v["pnl"], 2)} for k, v in by_reason.items()},
"gap_stats": gap_analysis(losers_ctx),
"trader_notes": narrative,
"suggested_param_tweaks": suggestions,
"sample_losers": sorted(losers_ctx, key=lambda x: x["profit"])[:8],
}
return result
def main() -> None:
sid = sys.argv[1] if len(sys.argv) > 1 else "united_rsi_scalp_appl"
if not mt5.initialize():
raise SystemExit(f"MT5 init failed: {mt5.last_error()}")
try:
result = analyze(sid)
OUT_DIR.mkdir(parents=True, exist_ok=True)
path = OUT_DIR / f"{sid}_loss_context.json"
path.write_text(json.dumps(result, indent=2), encoding="utf-8")
print(f"Wrote {path}\n")
print(f"=== {sid} loss context ({result['losers']} losers / {result['total_trades']} trades) ===\n")
for reason, stats in sorted(result["loss_by_reason"].items(), key=lambda x: x[1]["pnl"]):
print(f" {reason:14} count={stats['count']:4} pnl=${stats['pnl']:,.0f}")
print("\nTrader read:")
for n in result["trader_notes"]:
print(f" - {n}")
if result["suggested_param_tweaks"]:
print("\nSuggested tweaks:", result["suggested_param_tweaks"])
finally:
mt5.shutdown()
if __name__ == "__main__":
main()
backtesting/MT5/cluster_audit/lot_sizing.py
+88
View File
@@ -0,0 +1,88 @@
"""
Lot sizing guidance for United EA combined portfolio.
Compares per-strategy risk at 123.set nominal lots and suggests relative weights.
Usage:
python -m cluster_audit.lot_sizing
"""
from __future__ import annotations
import json
from pathlib import Path
from cluster_audit.united_registry import UNITED_STRATEGIES
REPORTS = Path(__file__).parent / "reports" / "united_sequential"
MANIFEST = REPORTS / "united_manifest.json"
OUT = REPORTS / "lot_sizing.json"
REF_BALANCE = 1000.0
def main() -> None:
manifest = {}
if MANIFEST.exists():
manifest = json.loads(MANIFEST.read_text(encoding="utf-8"))
rows: list[dict] = []
for spec in UNITED_STRATEGIES:
sid = spec["id"]
info = manifest.get("strategies", {}).get(sid, {})
if not info.get("passed"):
continue
o_net = float(info.get("net_profit", 0))
trades = int(info.get("trades", 1))
dd = float(info.get("max_drawdown_pct", info.get("issues", [""])[0] if False else 5))
lot = float(spec["lot"])
per_trade = o_net / max(trades, 1)
# risk proxy: lot * avg loss magnitude; use net/trades as PnL per trade signal
rows.append({
"id": sid,
"symbol": spec["symbol"],
"lot_123set": lot,
"net_profit": o_net,
"trades": trades,
"pnl_per_trade": round(per_trade, 2),
"max_dd_pct": dd,
})
if not rows:
print("No passed strategies in manifest — run united sequential audit first.")
return
# Target: equal risk contribution via inverse DD weighting
inv_dd = [1.0 / max(r.get("max_dd_pct", 5), 0.5) for r in rows]
total_inv = sum(inv_dd)
for i, r in enumerate(rows):
weight = inv_dd[i] / total_inv
r["risk_weight"] = round(weight, 4)
r["suggested_lot_vs_darvas"] = round(weight / (inv_dd[0] / total_inv), 3) if rows else 1.0
# Scale all lots so combined net at ref balance ~ sum of individuals / sqrt(N)
import math
n = len(rows)
diversification = math.sqrt(n)
base_lot = rows[0]["lot_123set"]
for r in rows:
r["suggested_lot_at_1k"] = round(base_lot * r["suggested_lot_vs_darvas"] / diversification, 4)
result = {
"reference_balance": REF_BALANCE,
"passed_count": n,
"diversification_factor": diversification,
"note": "suggested_lot_at_1k scales 123.set lots by inverse-DD weight / sqrt(N)",
"strategies": rows,
}
OUT.write_text(json.dumps(result, indent=2), encoding="utf-8")
print(f"Wrote {OUT}")
print(f"\nLot sizing for {n} passed strategies @ ${REF_BALANCE:,.0f} reference:\n")
for r in rows:
print(
f" {r['id']:28} lot={r['lot_123set']:8} -> suggested={r['suggested_lot_at_1k']:8} "
f"(weight={r['risk_weight']:.2%}, pnl/trade=${r['pnl_per_trade']:.2f})"
)
if __name__ == "__main__":
main()
backtesting/MT5/cluster_audit/margin.py
+53
View File
@@ -0,0 +1,53 @@
"""Margin helpers — uses MT5 order_calc_margin for realistic portfolio sizing."""
from __future__ import annotations
import MetaTrader5 as mt5
def calc_margin(symbol: str, side: str, volume: float, price: float) -> float:
ot = mt5.ORDER_TYPE_BUY if side == "BUY" else mt5.ORDER_TYPE_SELL
m = mt5.order_calc_margin(ot, symbol, volume, price)
return float(m) if m is not None and m > 0 else 0.0
def max_lot_for_margin(
symbol: str,
side: str,
price: float,
free_margin: float,
leverage: int = 0,
) -> float:
"""Binary search max lot that fits in free_margin (with 5% buffer)."""
info = mt5.symbol_info(symbol)
if info is None or free_margin <= 0:
return 0.0
vmin = float(info.volume_min)
vmax = float(info.volume_max)
step = float(info.volume_step) or vmin
budget = free_margin * 0.95
lo, hi = vmin, vmax
best = 0.0
for _ in range(24):
mid = (lo + hi) / 2
m = calc_margin(symbol, side, mid, price)
if m <= budget:
best = mid
lo = mid
else:
hi = mid
if step > 0 and best > 0:
best = max(vmin, (int(best / step)) * step)
return best
def normalize_volume(symbol: str, volume: float) -> float:
info = mt5.symbol_info(symbol)
if info is None:
return volume
vmin = float(info.volume_min)
vmax = float(info.volume_max)
step = float(info.volume_step) or vmin
if step > 0:
volume = (int(volume / step)) * step
return max(vmin, min(vmax, volume))
backtesting/MT5/cluster_audit/portfolio_build.py
+213
View File
@@ -0,0 +1,213 @@
"""Progressive portfolio build: combine 1, 2, 3 ... N strategies with lot optimization."""
from __future__ import annotations
import random
from datetime import datetime
from typing import Any
import numpy as np
import pandas as pd
from .backtest_core import CostModel, build_report, load_bars, resolve_symbol
from .engines import ENGINE_MAP
from .strategy_registry import TF
from .trace_log import TraceLog
def _score_report(report) -> float:
if report.total_trades < 5:
return float("-inf")
return report.sharpe * 0.6 + (report.net_profit / 1000.0) * 0.3 - report.max_drawdown_pct * 0.1
def _align_equity_curves(curves: list[pd.Series], initial: float = 10_000.0) -> pd.Series:
if not curves:
return pd.Series([initial])
idx = curves[0].index
for c in curves[1:]:
idx = idx.union(c.index)
idx = idx.sort_values()
combined = pd.Series(0.0, index=idx)
for c in curves:
delta = c - c.iloc[0]
combined = combined.add(delta.reindex(idx, method="ffill").fillna(0.0), fill_value=0.0)
return initial + combined
def run_single_cached(
spec: dict,
params: dict,
df: pd.DataFrame,
sym: str,
period_label: str,
lot_mult: float = 1.0,
) -> tuple[Any, pd.Series]:
engine = ENGINE_MAP[spec["engine"]]
lot = spec["lot"] * lot_mult
costs = CostModel.for_symbol(sym)
report = engine(df, sym, period_label, spec["id"], params, lot, costs)
# Reconstruct equity from trades is hard; re-run stores equity internally.
# Use monthly returns proxy: build flat equity from trade PnL timeline.
eq = pd.Series(10_000.0, index=df.index)
pnl = 0.0
trade_idx = 0
trades_sorted = sorted(
getattr(report, "_trades", []) or [],
key=lambda t: t.close_time if hasattr(t, "close_time") else "",
)
# Fallback: approximate equity from net profit linearly (weak) — engines don't export eq.
# Better: patch engines to return equity. For now use per-strategy report sharpe weighting only.
if report.total_trades > 0:
step = report.net_profit / max(len(df), 1)
eq = eq + np.arange(len(df)) * (step / len(df))
return report, eq
def backtest_portfolio(
members: list[dict],
period_label: str,
start: str,
end: str,
lot_mults: dict[str, float] | None = None,
data_cache: dict | None = None,
) -> dict[str, Any]:
"""members: list of {spec, params, lot_mult}"""
lot_mults = lot_mults or {}
data_cache = data_cache or {}
curves: list[pd.Series] = []
member_reports = []
all_trades = []
start_dt = datetime.fromisoformat(start)
end_dt = datetime.fromisoformat(end)
for m in members:
spec = m["spec"]
params = m["params"]
sid = spec["id"]
sym = resolve_symbol(spec["symbol"])
cache_key = (sym, spec["tf"])
if cache_key not in data_cache:
data_cache[cache_key] = load_bars(sym, TF[spec["tf"]], start_dt, end_dt)
df = data_cache[cache_key]
engine = ENGINE_MAP[spec["engine"]]
lot = spec["lot"] * lot_mults.get(sid, m.get("lot_mult", 1.0))
costs = CostModel.for_symbol(sym)
report = engine(df, sym, period_label, sid, params, lot, costs)
member_reports.append(report)
# Build equity from trade close events on this df's index
eq = pd.Series(10_000.0, index=df.index, dtype=float)
running = 10_000.0
# We don't have trade list in report — use net profit distributed at bar closes via worst_trades timing
# Simpler: daily PnL from member net / days
if report.total_trades > 0 and report.net_profit != 0:
daily_ret = report.net_profit / len(df)
eq = eq + pd.Series(np.cumsum([daily_ret] * len(df)), index=df.index)
curves.append(eq)
all_trades.append(report.total_trades)
combined_eq = _align_equity_curves(curves)
combined_report = build_report(
"portfolio",
"MIXED",
"H1",
period_label,
[],
combined_eq,
10_000.0,
{"members": [m["spec"]["id"] for m in members]},
)
# Override with summed stats
net = sum(r.net_profit for r in member_reports)
trades = sum(r.total_trades for r in member_reports)
sharpes = [r.sharpe for r in member_reports if r.total_trades >= 5]
combined_report.net_profit = net
combined_report.total_trades = trades
combined_report.sharpe = float(np.mean(sharpes)) if sharpes else 0.0
return {
"members": [m["spec"]["id"] for m in members],
"member_reports": [r.to_dict() for r in member_reports],
"net_profit": net,
"total_trades": trades,
"sharpe_proxy": combined_report.sharpe,
"lot_mults": {m["spec"]["id"]: lot_mults.get(m["spec"]["id"], m.get("lot_mult", 1.0)) for m in members},
}
def optimize_portfolio_lots(
members: list[dict],
period_label: str,
start: str,
end: str,
trials: int,
rng: random.Random,
log: TraceLog,
) -> dict[str, Any]:
best_mults = {m["spec"]["id"]: 1.0 for m in members}
best = backtest_portfolio(members, period_label, start, end, best_mults)
best_score = _score_proxy(best)
for n in range(1, trials + 1):
mults = {sid: round(rng.uniform(0.25, 1.5), 2) for sid in best_mults}
r = backtest_portfolio(members, period_label, start, end, mults)
sc = _score_proxy(r)
if sc > best_score:
best_score = sc
best = r
best_mults = dict(mults)
log.info(f" portfolio trial {n}/{trials}: NEW BEST net=${r['net_profit']:.0f} mults={mults}")
best["optimized_score"] = best_score
return best
def _score_proxy(portfolio_result: dict) -> float:
net = portfolio_result["net_profit"]
trades = portfolio_result["total_trades"]
sh = portfolio_result.get("sharpe_proxy", 0.0)
if trades < 5:
return float("-inf")
return sh * 0.6 + (net / 1000.0) * 0.3
def build_progressive_portfolios(
ranked_results: list[dict],
period_label: str,
start: str,
end: str,
trials_per_step: int,
rng: random.Random,
log: TraceLog,
) -> list[dict]:
"""ranked_results: sorted best-first, each has spec + optimized params."""
steps: list[dict] = []
members: list[dict] = []
for i, r in enumerate(ranked_results, 1):
members.append({"spec": r["spec"], "params": r["optimized_params"], "lot_mult": 1.0})
log.banner(f"PORTFOLIO STEP {i}/{len(ranked_results)}: +{r['spec']['id']}")
log.info(f"members: {[m['spec']['id'] for m in members]}")
optimized = optimize_portfolio_lots(members, period_label, start, end, trials_per_step, rng, log)
baseline = backtest_portfolio(members, period_label, start, end)
step = {
"step": i,
"member_ids": [m["spec"]["id"] for m in members],
"baseline_net": baseline["net_profit"],
"baseline_trades": baseline["total_trades"],
"optimized_net": optimized["net_profit"],
"optimized_trades": optimized["total_trades"],
"lot_mults": optimized["lot_mults"],
"member_reports": optimized["member_reports"],
}
steps.append(step)
log.info(
f"step {i}: baseline net=${baseline['net_profit']:.0f} -> "
f"optimized net=${optimized['net_profit']:.0f} mults={optimized['lot_mults']}"
)
return steps
backtesting/MT5/cluster_audit/portfolio_sim.py
+303
View File
@@ -0,0 +1,303 @@
"""
Margin-aware portfolio simulator — merges strategy trades chronologically.
Rejects new entries when margin level would drop below min_margin_level_pct.
"""
from __future__ import annotations
from dataclasses import dataclass, field
from datetime import datetime
from typing import Any
import numpy as np
import pandas as pd
from .backtest_core import BacktestReport, CostModel, Trade, build_report, load_bars, resolve_symbol
from .engines import ENGINE_MAP
from .margin import calc_margin, normalize_volume
from .united_registry import TF
@dataclass
class OpenPosition:
strategy_id: str
symbol: str
side: str
volume: float
entry_price: float
entry_time: Any
margin: float
@dataclass
class PortfolioSimResult:
members: list[str]
lot_scales: dict[str, float]
initial_balance: float
net_profit: float
total_trades: int
rejected_margin: int
min_margin_level_pct: float
lowest_margin_level_pct: float
max_drawdown_pct: float
sharpe: float
equity_curve: pd.Series = field(repr=False)
member_reports: list[dict] = field(default_factory=list)
def _scale_trade(t: Trade, scale: float) -> Trade:
if scale == 1.0:
return t
return Trade(
side=t.side,
open_time=t.open_time,
close_time=t.close_time,
open_price=t.open_price,
close_price=t.close_price,
volume=t.volume * scale,
profit=t.profit * scale,
bars_held=t.bars_held,
exit_reason=t.exit_reason,
)
def run_member_backtest(
spec: dict,
params: dict,
lot: float,
df: pd.DataFrame,
sym: str,
period_label: str,
) -> BacktestReport:
engine = ENGINE_MAP[spec["engine"]]
costs = CostModel.for_symbol(sym)
return engine(df, sym, period_label, spec["id"], params, lot, costs)
def simulate_portfolio(
members: list[dict],
period_label: str,
start: str,
end: str,
initial_balance: float = 1000.0,
lot_scales: dict[str, float] | None = None,
min_margin_level_pct: float = 150.0,
data_cache: dict | None = None,
) -> PortfolioSimResult:
"""
members: [{spec, params, lot}] — lot = nominal from 123.set
lot_scales: per-strategy multiplier on nominal lot
"""
lot_scales = lot_scales or {}
data_cache = data_cache or {}
start_dt = datetime.fromisoformat(start)
end_dt = datetime.fromisoformat(end)
events: list[tuple[Any, str, str, Trade]] = []
member_reports: list[dict] = []
for m in members:
spec = m["spec"]
sid = spec["id"]
sym = resolve_symbol(spec["symbol"])
cache_key = (sym, spec["tf"])
if cache_key not in data_cache:
data_cache[cache_key] = load_bars(sym, TF[spec["tf"]], start_dt, end_dt)
df = data_cache[cache_key]
scale = lot_scales.get(sid, m.get("lot_scale", 1.0))
lot = spec["lot"] * scale
report = run_member_backtest(spec, m["params"], lot, df, sym, period_label)
member_reports.append({**report.to_dict(), "lot_used": lot, "lot_scale": scale})
for t in report.trades_list:
events.append((t.open_time, "open", sid, t))
events.append((t.close_time, "close", sid, t))
events.sort(key=lambda x: (pd.Timestamp(x[0]), 0 if x[1] == "close" else 1))
balance = initial_balance
equity = initial_balance
open_pos: dict[str, OpenPosition] = {}
realized: list[Trade] = []
rejected = 0
equity_points: list[tuple[Any, float]] = [(events[0][0] if events else start_dt, initial_balance)]
lowest_ml = 9999.0
for ts, kind, sid, raw_t in events:
m = next(x for x in members if x["spec"]["id"] == sid)
spec = m["spec"]
sym = resolve_symbol(spec["symbol"])
scale = lot_scales.get(sid, m.get("lot_scale", 1.0))
t = _scale_trade(raw_t, scale / (raw_t.volume / spec["lot"]) if raw_t.volume else scale)
if kind == "close":
key = f"{sid}"
if key not in open_pos:
continue
op = open_pos.pop(key)
balance += t.profit
equity = balance + sum(
calc_profit(op2.symbol, op2.side, op2.volume, op2.entry_price, t.close_price)
for op2 in open_pos.values()
if op2.symbol == sym
)
# simpler: balance only on close
balance = equity_points[-1][1] + t.profit if equity_points else balance + t.profit
realized.append(t)
equity_points.append((ts, balance))
continue
# open
vol = normalize_volume(sym, spec["lot"] * lot_scales.get(sid, 1.0))
if vol <= 0:
rejected += 1
continue
margin_req = calc_margin(sym, t.side, vol, t.open_price)
used_margin = sum(p.margin for p in open_pos.values())
free = balance - used_margin
if margin_req > free:
rejected += 1
continue
new_used = used_margin + margin_req
equity = balance # simplified
ml = (equity / new_used * 100.0) if new_used > 0 else 9999.0
if ml < min_margin_level_pct:
rejected += 1
continue
lowest_ml = min(lowest_ml, ml)
open_pos[sid] = OpenPosition(sid, sym, t.side, vol, t.open_price, ts, margin_req)
if not equity_points:
equity_points = [(start_dt, initial_balance)]
eq = pd.Series(
[p[1] for p in equity_points],
index=pd.DatetimeIndex([p[0] for p in equity_points]),
)
combined = build_report(
"portfolio",
"MIXED",
"H1",
period_label,
realized,
eq,
initial_balance,
{"members": [m["spec"]["id"] for m in members], "lot_scales": lot_scales},
)
return PortfolioSimResult(
members=[m["spec"]["id"] for m in members],
lot_scales={m["spec"]["id"]: lot_scales.get(m["spec"]["id"], 1.0) for m in members},
initial_balance=initial_balance,
net_profit=combined.net_profit,
total_trades=len(realized),
rejected_margin=rejected,
min_margin_level_pct=min_margin_level_pct,
lowest_margin_level_pct=lowest_ml if lowest_ml < 9999 else 0.0,
max_drawdown_pct=combined.max_drawdown_pct,
sharpe=combined.sharpe,
equity_curve=eq,
member_reports=member_reports,
)
def optimize_lot_scales(
members: list[dict],
period_label: str,
start: str,
end: str,
initial_balance: float,
min_margin_level_pct: float,
trials: int,
rng,
) -> PortfolioSimResult:
"""Grid-search lot scales down from 1.0 — margin-safe, maximize net."""
best_scales = {m["spec"]["id"]: 1.0 for m in members}
best = simulate_portfolio(
members, period_label, start, end, initial_balance, best_scales, min_margin_level_pct
)
best_score = _portfolio_score(best)
# Coarse: try uniform scale factors
for factor in [1.0, 0.75, 0.5, 0.35, 0.25, 0.15, 0.1]:
scales = {m["spec"]["id"]: factor for m in members}
r = simulate_portfolio(members, period_label, start, end, initial_balance, scales, min_margin_level_pct)
sc = _portfolio_score(r)
if sc > best_score:
best_score = sc
best = r
best_scales = dict(scales)
# Fine-tune per member around best uniform
for _ in range(trials):
scales = {}
for m in members:
sid = m["spec"]["id"]
base = best_scales.get(sid, 1.0)
scales[sid] = round(max(0.05, min(1.5, base * rng.uniform(0.7, 1.3))), 3)
r = simulate_portfolio(members, period_label, start, end, initial_balance, scales, min_margin_level_pct)
sc = _portfolio_score(r)
if sc > best_score and r.lowest_margin_level_pct >= min_margin_level_pct * 0.9:
best_score = sc
best = r
best_scales = dict(scales)
best.lot_scales = best_scales
return best
def _portfolio_score(r: PortfolioSimResult) -> float:
if r.net_profit <= 0:
return float("-inf")
if r.lowest_margin_level_pct < r.min_margin_level_pct:
return float("-inf")
return r.sharpe * 0.4 + (r.net_profit / 500.0) * 0.4 - r.max_drawdown_pct * 0.15 - r.rejected_margin * 0.001
def build_progressive_margin_portfolio(
ranked: list[dict],
period_label: str,
start: str,
end: str,
initial_balance: float,
min_margin_level_pct: float,
trials_per_step: int,
rng,
) -> list[dict]:
"""ranked: [{spec, params, lot, baseline_report}] sorted best-first."""
steps: list[dict] = []
members: list[dict] = []
for i, r in enumerate(ranked, 1):
members.append({
"spec": r["spec"],
"params": r["params"],
"lot_scale": 1.0,
})
baseline = simulate_portfolio(
members, period_label, start, end, initial_balance,
{m["spec"]["id"]: 1.0 for m in members}, min_margin_level_pct,
)
optimized = optimize_lot_scales(
members, period_label, start, end, initial_balance,
min_margin_level_pct, trials_per_step, rng,
)
steps.append({
"step": i,
"members": [m["spec"]["id"] for m in members],
"baseline_net": baseline.net_profit,
"baseline_trades": baseline.total_trades,
"baseline_lowest_margin_pct": baseline.lowest_margin_level_pct,
"optimized_net": optimized.net_profit,
"optimized_trades": optimized.total_trades,
"optimized_sharpe": optimized.sharpe,
"optimized_max_dd_pct": optimized.max_drawdown_pct,
"lowest_margin_level_pct": optimized.lowest_margin_level_pct,
"rejected_margin": optimized.rejected_margin,
"lot_scales": optimized.lot_scales,
"lots_final": {
m["spec"]["id"]: round(m["spec"]["lot"] * optimized.lot_scales.get(m["spec"]["id"], 1.0), 4)
for m in members
},
})
return steps
backtesting/MT5/cluster_audit/reports/.gitkeep
+1
View File
@@ -0,0 +1 @@
# Local MT5 audit outputs — see cluster_audit/*.py to regenerate.
backtesting/MT5/cluster_audit/run_audit.py
+646
View File
@@ -0,0 +1,646 @@
"""
Full cluster audit: baseline + optimize per strategy per period.
Outputs JSON report with worst losses and improvement hints.
Usage:
python -m cluster_audit.run_audit [trials] [--trial-every N] [--quiet]
Examples:
python -m cluster_audit.run_audit 80
python -m cluster_audit.run_audit 120 --trial-every 5
"""
from __future__ import annotations
import argparse
import json
import random
import sys
import time
from datetime import datetime
from pathlib import Path
import MetaTrader5 as mt5
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
from cluster_audit.backtest_core import CostModel, load_bars, resolve_symbol
from cluster_audit.engines import ENGINE_MAP
from cluster_audit.strategy_registry import PERIODS, STRATEGIES, TF
from cluster_audit.trace_log import TraceLog
LOG = TraceLog(enabled=True, trial_every=10)
def _sample_params(defaults: dict, opt_ranges: dict, rng: random.Random) -> dict:
p = dict(defaults)
for key, spec in opt_ranges.items():
if not isinstance(spec, tuple) or len(spec) != 3:
continue
lo, hi, step = spec
if isinstance(lo, int):
vals = list(range(int(lo), int(hi) + 1, int(step)))
p[key] = rng.choice(vals) if vals else p.get(key, lo)
else:
n = int((hi - lo) / step) + 1
idx = rng.randint(0, max(n - 1, 0))
p[key] = round(lo + idx * step, 4)
return p
from cluster_audit.scoring import DEFAULT_TRADES_PER_DAY, period_days, score_label, score_report
from cluster_audit.strategy_registry import PERIODS
def _audit_period_days() -> int:
start, end = PERIODS["2021-2026"]
return period_days(start, end)
def _score_label(score: float, trades: int) -> str:
return score_label(score, trades, _audit_period_days(), DEFAULT_TRADES_PER_DAY)
def _score(report) -> float:
return score_report(report, _audit_period_days(), DEFAULT_TRADES_PER_DAY)
def run_strategy(
spec: dict,
period_label: str,
start: str,
end: str,
trials: int,
rng: random.Random,
run_idx: int,
run_total: int,
) -> dict:
sid = spec["id"]
label = f"{sid} @ {period_label} ({run_idx}/{run_total})"
LOG.phase_start(label)
engine_name = spec["engine"]
if engine_name not in ENGINE_MAP:
LOG.error(f"Unknown engine '{engine_name}'")
return {"id": sid, "period": period_label, "error": f"unknown engine {engine_name}"}
engine = ENGINE_MAP[engine_name]
tf_key = spec["tf"]
tf = TF[tf_key]
LOG.debug(f"resolve symbol: requested={spec['symbol']}")
raw_sym = spec["symbol"]
sym = resolve_symbol(raw_sym)
if sym != raw_sym:
LOG.info(f"symbol mapped {raw_sym} -> {sym}")
else:
LOG.debug(f"symbol={sym}")
start_dt = datetime.fromisoformat(start)
end_dt = datetime.fromisoformat(end)
LOG.debug(f"load bars: tf={tf_key} from={start} to={end} lot={spec['lot']}")
t_load = time.perf_counter()
try:
df = load_bars(sym, tf, start_dt, end_dt)
except Exception as e:
LOG.error(f"data load failed: {e}")
LOG.phase_end(label, "SKIPPED")
return {"id": sid, "period": period_label, "error": str(e)}
load_ms = (time.perf_counter() - t_load) * 1000
LOG.info(
f"loaded {len(df)} bars in {load_ms:.0f}ms "
f"({df.index[0]} -> {df.index[-1]})"
)
costs = CostModel.for_symbol(sym)
LOG.debug(
f"costs: spread={costs.spread_points}pts slippage={costs.slippage_points}pts "
f"commission/lot={costs.commission_per_lot}"
)
defaults = dict(spec["defaults"])
opt_keys = list(spec.get("opt", {}).keys())
LOG.debug(f"baseline params keys: {list(defaults.keys())}")
LOG.debug(f"optimize keys ({len(opt_keys)}): {opt_keys}")
LOG.debug("running baseline backtest...")
t0 = time.perf_counter()
baseline = engine(df, sym, period_label, sid, defaults, spec["lot"], costs)
base_ms = (time.perf_counter() - t0) * 1000
LOG.info(
f"baseline done in {base_ms:.0f}ms: net=${baseline.net_profit:.2f} "
f"sharpe={baseline.sharpe:.2f} trades={baseline.total_trades} "
f"pf={baseline.profit_factor:.2f} max_dd={baseline.max_drawdown_pct:.1f}%"
)
if baseline.total_trades == 0:
LOG.warn(f"{sid}: 0 trades on baseline — engine may not match MQL or params too strict")
best = baseline
best_params = defaults
best_score = _score(baseline)
LOG.debug(f"baseline score={_score_label(best_score, baseline.total_trades)}")
if trials > 0:
LOG.info(f"optimizing: {trials} random trials...")
for n in range(1, trials + 1):
params = _sample_params(defaults, spec.get("opt", {}), rng)
r = engine(df, sym, period_label, sid, params, spec["lot"], costs)
sc = _score(r)
improved = sc > best_score
if improved:
best_score = sc
best = r
best_params = params
LOG.trial(n, trials, sc, r.net_profit, r.sharpe, improved=True)
LOG.debug(f" new best params sample: { {k: best_params[k] for k in opt_keys[:6] if k in best_params} }")
else:
LOG.trial(n, trials, sc, r.net_profit, r.sharpe, improved=False)
if trials > 0 and best is not baseline:
LOG.info(
f"optimized: net +${best.net_profit - baseline.net_profit:.0f} "
f"sharpe +{best.sharpe - baseline.sharpe:.2f}"
)
elif trials > 0:
LOG.info("optimization: no better params found (baseline kept)")
loss_reasons: dict[str, float] = {}
for t in baseline.worst_trades:
reason = t.get("exit_reason", "?")
loss_reasons[reason] = loss_reasons.get(reason, 0) + min(t["profit"], 0)
if loss_reasons:
top_loss = sorted(loss_reasons.items(), key=lambda x: x[1])[:3]
LOG.debug(f"baseline loss by exit reason: {top_loss}")
result = {
"id": sid,
"engine": engine_name,
"symbol": sym,
"timeframe": tf_key,
"period": period_label,
"bars": len(df),
"baseline": baseline.to_dict(),
"optimized": {**best.to_dict(), "params": best_params},
"improvement_net": best.net_profit - baseline.net_profit,
"improvement_sharpe": best.sharpe - baseline.sharpe,
"loss_reasons_baseline": loss_reasons,
}
LOG.report_line(sid, result["baseline"], result["optimized"], len(df))
LOG.phase_end(label)
return result
def build_improvement_plan(results: list[dict]) -> dict:
by_engine: dict[str, list] = {}
for r in results:
if "error" in r:
continue
by_engine.setdefault(r["engine"], []).append(r)
plan = {"per_engine": {}, "portfolio": []}
engine_notes = {
"rsi_crossover": "Trend-strong filter closes/blocks too aggressively (ema_slope 105 + distance 165). "
"Raise thresholds or only block entries, not force-exit. Add pip-based scaling per symbol.",
"rsi_scalp": "High trade count + rsi_against exits cause death by spread. Widen OB/OS gap, add ADX/session "
"filter, scale trail by ATR not fixed points. Stocks need .NAS symbols.",
"rsi_asian": "Narrow session + extreme RSI levels -> few trades or bad fills. Align session to broker server "
"time; add spread cap in points not pips.",
"mean_reversion": "ADX proxy weak in Python; large min_ema_distance on BTC blocks entries. "
"Use true ADX; cap concurrent positions; hard SL when ADX escapes.",
"ema_slope": "Re-enters on same crossover too often; weekly ADX filter missing in audit. "
"Add cooldown after loss; separate unit vs trail param sets.",
"darvas": "Box breakout without volume filter whipsaws. Add volume MA + trend MA filter from MQL.",
"rsi_secret": "Zone re-entry fires too often in chop. Require divergence or RSI momentum confirm.",
}
for eng, rows in by_engine.items():
sharpes = [x["baseline"]["sharpe"] for x in rows]
opt_sharpes = [x["optimized"]["sharpe"] for x in rows]
nets = [x["baseline"]["net_profit"] for x in rows]
plan["per_engine"][eng] = {
"count": len(rows),
"avg_baseline_sharpe": sum(sharpes) / len(sharpes) if sharpes else 0,
"avg_optimized_sharpe": sum(opt_sharpes) / len(opt_sharpes) if opt_sharpes else 0,
"avg_baseline_net": sum(nets) / len(nets) if nets else 0,
"logic_fixes": engine_notes.get(eng, ""),
"worst_strategies": sorted(rows, key=lambda x: x["baseline"]["sharpe"])[:3],
}
plan["portfolio"] = [
"Run correlation matrix on daily returns — disable highly correlated RSI scalps on same underlying (NVDA x3).",
"Portfolio-level max daily loss circuit breaker (pause new entries cluster-wide).",
"Per-asset-class lot caps: forex micro, gold 0.1, stocks margin-scaled not fixed 25 lots.",
"Split optimization windows: long 2021-2026 for structure, short 2024-2026 for recency; only deploy params that pass both.",
"Add core/ShockGuard.mqh: ATR spike pause + margin level gate before SE_TickAll.",
"Enable robots by regime: Asian RSI only 00-08 server; EMA slope only when W1 ADX > threshold.",
"Replace fixed magic collisions with 401xxx registry; log per-robot PnL for live attribution.",
]
return plan
def parse_args() -> argparse.Namespace:
p = argparse.ArgumentParser(description="SuperEA cluster audit with trace logging")
p.add_argument("trials", nargs="?", type=int, default=80, help="Random trials per strategy (default 80)")
p.add_argument("--trial-every", type=int, default=10, help="Log every N trials (default 10)")
p.add_argument("--quiet", action="store_true", help="Suppress trace output")
return p.parse_args()
def main() -> None:
args = parse_args()
global LOG
LOG = TraceLog(enabled=not args.quiet, trial_every=args.trial_every)
total_runs = len(STRATEGIES) * len(PERIODS)
LOG.banner(
f"CLUSTER AUDIT - {len(STRATEGIES)} strategies x {len(PERIODS)} periods "
f"= {total_runs} runs, {args.trials} trials each"
)
LOG.phase_start("MT5 initialize")
if not mt5.initialize():
LOG.error(f"MT5 init failed: {mt5.last_error()}")
raise SystemExit(1)
acc = mt5.account_info()
if acc:
LOG.info(f"MT5 connected: server={acc.server} (account redacted)")
LOG.phase_end("MT5 initialize")
out_dir = Path(__file__).parent / "reports"
out_dir.mkdir(exist_ok=True)
rng = random.Random(42)
all_results = []
run_idx = 0
errors = 0
try:
for period_label, (start, end) in PERIODS.items():
LOG.banner(f"PERIOD {period_label} ({start} -> {end})")
for spec in STRATEGIES:
run_idx += 1
LOG.progress(run_idx, total_runs, f"next: {spec['id']}")
r = run_strategy(spec, period_label, start, end, args.trials, rng, run_idx, total_runs)
all_results.append(r)
if "error" in r:
errors += 1
LOG.phase_start("build improvement plan")
plan = build_improvement_plan(all_results)
LOG.phase_end("build improvement plan")
report = {
"generated": datetime.now().isoformat(),
"trials_per_strategy": args.trials,
"strategies": len(STRATEGIES),
"periods": list(PERIODS.keys()),
"runs_total": total_runs,
"runs_failed": errors,
"results": all_results,
"improvement_plan": plan,
}
out_path = out_dir / "cluster_audit_report.json"
out_path.write_text(json.dumps(report, indent=2), encoding="utf-8")
LOG.info(f"report written: {out_path} ({out_path.stat().st_size // 1024} KB)")
LOG.banner("SUMMARY - worst baseline Sharpe (2021-2026)")
r21 = [x for x in all_results if x.get("period") == "2021-2026" and "error" not in x]
for x in sorted(r21, key=lambda z: z["baseline"]["sharpe"])[:10]:
b = x["baseline"]
w = b["worst_trades"][:1]
wtxt = f"${w[0]['profit']:.0f} {w[0]['exit_reason']}" if w else "n/a"
LOG.info(
f" {x['id']:28} sharpe={b['sharpe']:6.2f} net=${b['net_profit']:9.0f} "
f"trades={b['total_trades']:4} worst={wtxt}"
)
if errors:
LOG.warn(f"{errors}/{total_runs} runs failed - search report for \"error\" fields")
LOG.banner(f"DONE - {run_idx} runs in {LOG._elapsed()}")
finally:
LOG.phase_start("MT5 shutdown")
mt5.shutdown()
LOG.phase_end("MT5 shutdown")
if __name__ == "__main__":
main()
backtesting/MT5/cluster_audit/run_close_signal_audit.py
+277
View File
@@ -0,0 +1,277 @@
#!/usr/bin/env python3
"""
United EA MT5 audit: per-strategy solo runs, close-unprofitable A/B, lot/margin check.
Usage:
python -m cluster_audit.run_close_signal_audit
python -m cluster_audit.run_close_signal_audit --only DB
python -m cluster_audit.run_close_signal_audit --from RS_NVDA --combo
"""
from __future__ import annotations
import argparse
import json
import sys
from datetime import datetime
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
from set_parser import parse_set_file
from cluster_audit.united_mt5_manifest import (
ALL_ENABLE_KEYS,
PARAM_TWEAKS,
PRODUCTION_IDS,
UNITED_MT5_STRATEGIES,
)
from cluster_audit.united_mt5_runner import (
BASE_SET,
deploy_united,
mt5_context,
patch_set,
run_backtest,
)
OUT = Path(__file__).parent / "reports" / "close_signal_audit"
LOT_SUMMARY = Path(__file__).parent / "reports" / "lot_genetic" / "lot_genetic_summary.json"
REF_BALANCE = 3000.0
def load_best_lots() -> dict[str, float]:
if not LOT_SUMMARY.exists():
return {}
data = json.loads(LOT_SUMMARY.read_text(encoding="utf-8"))
return {k: float(v) for k, v in data.get("best_lots", {}).items()}
def solo_patches(spec: dict, *, close_on: bool, lots: dict[str, float]) -> dict:
ov = solo_overrides(spec, close_on=close_on)
ov["ORCH_ReferenceBalance"] = REF_BALANCE
ov["ORCH_ScaleLotsByBalance"] = True
if spec.get("lot") in lots:
ov[spec["lot"]] = lots[spec["lot"]]
return ov
def solo_overrides(target: dict, *, close_on: bool) -> dict[str, bool]:
o: dict[str, bool] = {k: False for k in ALL_ENABLE_KEYS}
o[target["enable"]] = True
o[target["close"]] = close_on
o["GAP_Enable"] = False
o["OPT_GuardOptimizationMode"] = True
return o
def audit_one(ctx: dict, spec: dict, base_params: dict, lots: dict[str, float]) -> dict:
sid = spec["id"]
print(f"\n{'='*60}\n[{sid}] {spec['name']}\n{'='*60}", flush=True)
off_ov = solo_patches(spec, close_on=False, lots=lots)
on_ov = solo_patches(spec, close_on=True, lots=lots)
off_body = patch_set(BASE_SET, off_ov)
on_body = patch_set(BASE_SET, on_ov)
ts = spec.get("test_symbol")
off = run_backtest(
ctx["data"], ctx["mt5_path"], ctx["login"], ctx["server"],
off_body, f"solo_{sid}_off.set", f"solo_{sid}_off",
test_symbol=ts,
)
print(f" OFF PF={off.get('profit_factor')} net={off.get('net_profit')} "
f"trades={off.get('total_trades')} sharpe={off.get('sharpe')} "
f"ready={off.get('ready')} ({off.get('elapsed_sec')}s)", flush=True)
on = run_backtest(
ctx["data"], ctx["mt5_path"], ctx["login"], ctx["server"],
on_body, f"solo_{sid}_on.set", f"solo_{sid}_on",
test_symbol=ts,
)
print(f" ON PF={on.get('profit_factor')} net={on.get('net_profit')} "
f"trades={on.get('total_trades')} sharpe={on.get('sharpe')} "
f"ready={on.get('ready')} ({on.get('elapsed_sec')}s)", flush=True)
d = delta_metrics(off, on)
v = verdict(d, off, on)
print(f" -> {v} dNet={d['net_profit_delta']:.0f} dSharpe={d['sharpe_delta']:.3f} "
f"dTrades={d['trades_delta']}", flush=True)
tweaks: list[dict] = []
if v == "POSITIVE" and sid in PARAM_TWEAKS:
for i, tw in enumerate(PARAM_TWEAKS[sid], 1):
ov = {**on_ov, **tw}
body = patch_set(BASE_SET, ov)
r = run_backtest(
ctx["data"], ctx["mt5_path"], ctx["login"], ctx["server"],
body, f"solo_{sid}_tw{i}.set", f"solo_{sid}_tw{i}",
test_symbol=ts,
)
tweaks.append({"tweak": tw, "metrics": r})
print(f" tweak{i} net={r.get('net_profit')} sharpe={r.get('sharpe')} "
f"trades={r.get('total_trades')}", flush=True)
lot_key = spec["lot"]
lot_val = lots.get(lot_key)
if lot_val is None and lot_key in base_params:
lot_val = base_params[lot_key].value
return {
"id": sid,
"name": spec["name"],
"enable": spec["enable"],
"close_key": spec["close"],
"lot_key": lot_key,
"lot_value": lot_val,
"close_off": off,
"close_on": on,
"delta": d,
"verdict": v,
"margin_ok_off": margin_ok(off),
"margin_ok_on": margin_ok(on),
"param_tweaks": tweaks,
"recommend_close_on": v == "POSITIVE",
}
def delta_metrics(off: dict, on: dict) -> dict:
def g(d, k):
v = d.get(k)
return v if v is not None else 0
return {
"net_profit_delta": g(on, "net_profit") - g(off, "net_profit"),
"pf_delta": (g(on, "profit_factor") - g(off, "profit_factor")),
"sharpe_delta": g(on, "sharpe") - g(off, "sharpe"),
"trades_delta": g(on, "total_trades") - g(off, "total_trades"),
}
def verdict(delta: dict, off: dict, on: dict) -> str:
if not off.get("ready") or not on.get("ready"):
return "BROKEN"
if off.get("total_trades", 0) == 0 and on.get("total_trades", 0) == 0:
return "NO_TRADES"
if delta["net_profit_delta"] > 50 and delta["sharpe_delta"] >= 0:
return "POSITIVE"
if delta["net_profit_delta"] < -50 or delta["sharpe_delta"] < -0.1:
return "NEGATIVE"
return "NEUTRAL"
def margin_ok(m: dict) -> bool:
ml = m.get("min_margin_level")
if not ml:
return True
if isinstance(ml, str) and "%" in ml:
try:
return float(ml.replace("%", "").strip()) >= 100.0
except ValueError:
return True
return True
def run_combo(ctx: dict, winners: list[dict], base_params: dict) -> dict:
"""Combo: enable all strategies that benefit from close-on, with flag set."""
overrides: dict = {k: False for k in ALL_ENABLE_KEYS}
overrides["GAP_Enable"] = False
for w in winners:
overrides[w["enable"]] = True
overrides[w["close_key"]] = True
for spec in UNITED_MT5_STRATEGIES:
if spec["enable"] not in overrides or not overrides[spec["enable"]]:
continue
if spec["id"] not in {w["id"] for w in winners}:
overrides[spec["close_key"]] = False
body = patch_set(BASE_SET, overrides)
m = run_backtest(
ctx["data"], ctx["mt5_path"], ctx["login"], ctx["server"],
body, "combo_close_winners.set", "combo_close_winners",
)
return {"members": [w["id"] for w in winners], "metrics": m}
def main() -> int:
p = argparse.ArgumentParser()
p.add_argument("--only", default=None)
p.add_argument("--from", dest="from_id", default=None)
p.add_argument("--production", action="store_true", help="Only PRODUCTION_IDS (19 strategies)")
p.add_argument("--combo", action="store_true")
p.add_argument("--enabled-only", action="store_true", help="Only strategies enabled in 123.set")
args = p.parse_args()
OUT.mkdir(parents=True, exist_ok=True)
base_params = parse_set_file(BASE_SET)
lots = load_best_lots()
sm = {s["id"]: s for s in UNITED_MT5_STRATEGIES}
if args.production:
strategies = [sm[sid] for sid in PRODUCTION_IDS if sid in sm]
elif args.enabled_only:
strategies = [
s for s in UNITED_MT5_STRATEGIES
if base_params.get(s["enable"], type("x", (), {"value": False})).value
]
else:
strategies = list(UNITED_MT5_STRATEGIES)
if args.only:
strategies = [s for s in strategies if s["id"] == args.only]
elif args.from_id:
found = False
filtered = []
for s in strategies:
if s["id"] == args.from_id:
found = True
if found:
filtered.append(s)
strategies = filtered if found else strategies
print(f"United EA close-signal audit | {len(strategies)} strategies | lots={len(lots)}")
import cluster_audit.united_mt5_runner as runner
runner.DEPOSIT = int(REF_BALANCE)
ctx = mt5_context()
print(f"MT5 server={ctx['server']} (account from local terminal)")
deploy_united(ctx["data"], ctx["mt5_path"])
print("Compiled main.ex5 OK")
results: list[dict] = []
for spec in strategies:
try:
results.append(audit_one(ctx, spec, base_params, lots))
except Exception as ex:
print(f" ERROR {spec['id']}: {ex}", flush=True)
results.append({"id": spec["id"], "error": str(ex), "verdict": "ERROR"})
positive = [r for r in results if r.get("verdict") == "POSITIVE"]
negative = [r for r in results if r.get("verdict") == "NEGATIVE"]
broken = [r for r in results if r.get("verdict") in ("BROKEN", "NO_TRADES", "ERROR")]
combo_result = None
if args.combo and positive:
combo_result = run_combo(ctx, positive, base_params)
print(f"\nCOMBO winners ({len(positive)}): net={combo_result['metrics'].get('net_profit')} "
f"sharpe={combo_result['metrics'].get('sharpe')}")
summary = {
"generated": datetime.now().isoformat(),
"base_set": str(BASE_SET),
"positive_close_on": [r["id"] for r in positive],
"negative_close_on": [r["id"] for r in negative],
"broken": [r["id"] for r in broken],
"results": results,
"combo": combo_result,
}
out_path = OUT / "summary.json"
out_path.write_text(json.dumps(summary, indent=2, default=str), encoding="utf-8")
print(f"\nSaved {out_path}")
print(f"POSITIVE ({len(positive)}): {[r['id'] for r in positive]}")
print(f"NEGATIVE ({len(negative)}): {[r['id'] for r in negative]}")
print(f"BROKEN ({len(broken)}): {[r['id'] for r in broken]}")
return 0
if __name__ == "__main__":
raise SystemExit(main())
backtesting/MT5/cluster_audit/run_disabled_audit.py
+198
View File
@@ -0,0 +1,198 @@
#!/usr/bin/env python3
"""
Solo MT5 audit for all currently-disabled sub-strategies (main.mq5 enable=false).
Finds profitable passers to add to the cluster; compares enhanced portfolio vs production baseline.
Usage:
python -m cluster_audit.run_disabled_audit
python -m cluster_audit.run_disabled_audit --min-trades 40
"""
from __future__ import annotations
import argparse
import json
import re
import sys
from datetime import datetime
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
from cluster_audit.united_mt5_manifest import (
ALL_ENABLE_KEYS,
HIGH_MARGIN_STOCK_ENABLES,
PRODUCTION_IDS,
UNITED_MT5_STRATEGIES,
)
from cluster_audit.united_mt5_runner import (
BASE_SET,
CLUSTER,
FROM_DATE,
TO_DATE,
deploy_united,
mt5_context,
patch_set,
run_backtest,
)
OUT = Path(__file__).resolve().parent / "reports" / "disabled_audit"
REF_BALANCE = 3000.0
def g(m: dict, k: str) -> float:
v = m.get(k)
return float(v) if v is not None else 0.0
def parse_dd_pct(dd: str | None) -> float | None:
if not dd:
return None
m = re.search(r"([\d.]+)\s*%", dd.replace(",", ""))
return float(m.group(1)) if m else None
def disabled_ids_from_mq5() -> list[str]:
text = (CLUSTER / "main.mq5").read_text(encoding="utf-8")
off: set[str] = set()
for m in re.finditer(r"input bool (Enable\w+) = false", text):
off.add(m.group(1))
for key in HIGH_MARGIN_STOCK_ENABLES:
off.discard(key)
ids: list[str] = []
for s in UNITED_MT5_STRATEGIES:
if s["enable"] in off:
ids.append(s["id"])
return ids
def common_patches() -> dict[str, float | bool]:
return {
"ORCH_ReferenceBalance": REF_BALANCE,
"ORCH_ScaleLotsByBalance": True,
"GAP_Enable": False,
"OPT_GuardOptimizationMode": True,
}
def production_enables() -> dict[str, bool]:
prod = set(PRODUCTION_IDS)
o: dict[str, bool] = {}
for s in UNITED_MT5_STRATEGIES:
o[s["enable"]] = s["id"] in prod
for key in HIGH_MARGIN_STOCK_ENABLES:
o[key] = False
return o
def solo_overrides(spec: dict) -> dict[str, bool]:
o: dict[str, bool] = {k: False for k in ALL_ENABLE_KEYS}
o[spec["enable"]] = True
return o
def classify(m: dict, min_trades: int) -> str:
if not m.get("ready"):
return "BROKEN"
trades = int(m.get("total_trades") or 0)
if trades < min_trades:
return "LOW_TRADES"
if g(m, "net_profit") > 0 and g(m, "profit_factor") >= 1.05:
return "PASS"
if g(m, "net_profit") > 50 and g(m, "profit_factor") >= 1.0:
return "MARGINAL"
return "FAIL"
def main() -> None:
p = argparse.ArgumentParser()
p.add_argument("--from", dest="from_date", default=FROM_DATE)
p.add_argument("--to", dest="to_date", default=TO_DATE)
p.add_argument("--min-trades", type=int, default=60)
args = p.parse_args()
import cluster_audit.united_mt5_runner as runner
runner.FROM_DATE = args.from_date.replace("-", ".")
runner.TO_DATE = args.to_date.replace("-", ".")
runner.DEPOSIT = int(REF_BALANCE)
sm = {s["id"]: s for s in UNITED_MT5_STRATEGIES}
disabled = disabled_ids_from_mq5()
OUT.mkdir(parents=True, exist_ok=True)
ctx = mt5_context()
deploy_united(ctx["data"], ctx["mt5_path"])
print(
f"Disabled audit n={len(disabled)} production={PRODUCTION_IDS} "
f"{runner.FROM_DATE}->{runner.TO_DATE}",
flush=True,
)
prod_ov = {**common_patches(), **production_enables()}
baseline = run_backtest(
ctx["data"], ctx["mt5_path"], ctx["login"], ctx["server"],
patch_set(BASE_SET, prod_ov), "dis_prod_baseline.set", "dis_prod_baseline",
)
print(
f"PROD baseline PF={baseline.get('profit_factor')} net={baseline.get('net_profit')} "
f"sharpe={baseline.get('sharpe')} dd={baseline.get('max_drawdown')}",
flush=True,
)
solo_rows: list[dict] = []
passed_ids: list[str] = []
for sid in disabled:
spec = sm[sid]
m = run_backtest(
ctx["data"], ctx["mt5_path"], ctx["login"], ctx["server"],
patch_set(BASE_SET, {**common_patches(), **solo_overrides(spec)}),
f"dis_solo_{sid}.set", f"dis_solo_{sid}",
test_symbol=spec.get("test_symbol"),
)
verdict = classify(m, args.min_trades)
dd_pct = parse_dd_pct(m.get("max_drawdown"))
print(
f" {sid:12} {verdict:10} PF={m.get('profit_factor')} net={m.get('net_profit')} "
f"sharpe={m.get('sharpe')} trades={m.get('total_trades')} dd={m.get('max_drawdown')}",
flush=True,
)
row = {"id": sid, "enable": spec["enable"], "verdict": verdict, "dd_pct": dd_pct, "metrics": m}
solo_rows.append(row)
if verdict in ("PASS", "MARGINAL"):
passed_ids.append(sid)
enhanced_ov = dict(prod_ov)
for sid in passed_ids:
enhanced_ov[sm[sid]["enable"]] = True
enhanced = run_backtest(
ctx["data"], ctx["mt5_path"], ctx["login"], ctx["server"],
patch_set(BASE_SET, enhanced_ov), "dis_enhanced.set", "dis_enhanced",
)
print(
f"ENHANCED +{len(passed_ids)} PF={enhanced.get('profit_factor')} net={enhanced.get('net_profit')} "
f"sharpe={enhanced.get('sharpe')} dd={enhanced.get('max_drawdown')}",
flush=True,
)
print(f"PASS/MARGINAL: {passed_ids}", flush=True)
summary = {
"timestamp": datetime.now().isoformat(timespec="seconds"),
"period": {"from": runner.FROM_DATE, "to": runner.TO_DATE},
"min_trades": args.min_trades,
"disabled_ids": disabled,
"production_baseline": baseline,
"solo": solo_rows,
"passed_ids": passed_ids,
"enhanced": enhanced,
}
path = OUT / "disabled_audit_summary.json"
path.write_text(json.dumps(summary, indent=2), encoding="utf-8")
print(f"Saved {path}", flush=True)
if __name__ == "__main__":
main()
backtesting/MT5/cluster_audit/run_hedge_audit.py
+233
View File
@@ -0,0 +1,233 @@
#!/usr/bin/env python3
"""
Symbol mining audit for cluster-latest (round 2 = indices, round 3 = low-margin stocks).
Baseline = 123.set + survivors only; solo each candidate round; enhanced = baseline + passers.
Usage:
python -m cluster_audit.run_hedge_audit --round 2
python -m cluster_audit.run_hedge_audit --round 3
"""
from __future__ import annotations
import argparse
import json
import sys
from datetime import datetime
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
from cluster_audit.united_mt5_manifest import (
ALL_ENABLE_KEYS,
EXPANSION_RETIRED_IDS,
HIGH_MARGIN_STOCK_ENABLES,
ROUND2_IDS,
ROUND3_IDS,
SURVIVOR_IDS,
UNITED_MT5_STRATEGIES,
)
from cluster_audit.united_mt5_runner import (
BASE_SET,
FROM_DATE,
TO_DATE,
deploy_united,
mt5_context,
patch_set,
run_backtest,
)
OUT = Path(__file__).resolve().parent / "reports" / "hedge_audit"
MIN_TRADES_DEFAULT = 60
ROUND_CONFIG = {
2: {"candidate_ids": ROUND2_IDS, "prefix": "r2"},
3: {"candidate_ids": ROUND3_IDS, "prefix": "r3"},
}
def g(m: dict, k: str) -> float:
v = m.get(k)
return float(v) if v is not None else 0.0
def spec_map() -> dict[str, dict]:
return {s["id"]: s for s in UNITED_MT5_STRATEGIES}
def candidate_ids_for_round(round_num: int) -> tuple[str, ...]:
return ROUND_CONFIG[round_num]["candidate_ids"]
def baseline_overrides(round_num: int) -> dict[str, bool]:
o: dict[str, bool] = {}
retired = set(EXPANSION_RETIRED_IDS)
survivors = set(SURVIVOR_IDS)
candidates = set(candidate_ids_for_round(round_num))
all_candidates = set(ROUND2_IDS) | set(ROUND3_IDS)
for s in UNITED_MT5_STRATEGIES:
sid = s["id"]
if sid in retired:
o[s["enable"]] = False
elif sid in survivors:
o[s["enable"]] = True
elif sid in all_candidates and sid not in candidates:
o[s["enable"]] = False
elif sid in candidates:
o[s["enable"]] = False
for key in HIGH_MARGIN_STOCK_ENABLES:
o[key] = False
o["GAP_Enable"] = False
o["OPT_GuardOptimizationMode"] = True
return o
def solo_overrides(spec: dict) -> dict:
o: dict[str, bool] = {k: False for k in ALL_ENABLE_KEYS}
o[spec["enable"]] = True
for key in HIGH_MARGIN_STOCK_ENABLES:
o[key] = False
o["GAP_Enable"] = False
o["OPT_GuardOptimizationMode"] = True
return o
def classify_solo(m: dict, min_trades: int) -> str:
if not m.get("ready"):
return "BROKEN"
trades = int(m.get("total_trades", 0) or 0)
if trades < min_trades:
return "LOW_TRADES"
if g(m, "net_profit") > 0 and g(m, "profit_factor") >= 1.05:
return "PASS"
if g(m, "net_profit") > 50 and g(m, "profit_factor") >= 1.0:
return "MARGINAL"
return "FAIL"
def delta(base: dict, var: dict) -> dict:
return {
"net_profit_delta": g(var, "net_profit") - g(base, "net_profit"),
"sharpe_delta": g(var, "sharpe") - g(base, "sharpe"),
"pf_delta": g(var, "profit_factor") - g(base, "profit_factor"),
"trades_delta": int(g(var, "total_trades") - g(base, "total_trades")),
}
def portfolio_verdict(d: dict) -> str:
if d["net_profit_delta"] > 100 and d["sharpe_delta"] >= -0.05:
return "IMPROVED"
if d["net_profit_delta"] < -150 or d["sharpe_delta"] < -0.15:
return "WORSE"
return "NEUTRAL"
def main() -> None:
p = argparse.ArgumentParser()
p.add_argument("--from", dest="from_date", default=FROM_DATE)
p.add_argument("--to", dest="to_date", default=TO_DATE)
p.add_argument("--min-trades", type=int, default=MIN_TRADES_DEFAULT)
p.add_argument("--round", type=int, default=3, choices=(2, 3))
args = p.parse_args()
import cluster_audit.united_mt5_runner as runner
runner.FROM_DATE = args.from_date.replace("-", ".")
runner.TO_DATE = args.to_date.replace("-", ".")
cfg = ROUND_CONFIG[args.round]
prefix = cfg["prefix"]
candidates = cfg["candidate_ids"]
OUT.mkdir(parents=True, exist_ok=True)
ctx = mt5_context()
deploy_united(ctx["data"], ctx["mt5_path"])
sm = spec_map()
print(
f"Round {args.round} {runner.FROM_DATE}->{runner.TO_DATE} "
f"min_trades={args.min_trades} survivor={SURVIVOR_IDS} "
f"candidates={candidates}",
flush=True,
)
baseline = run_backtest(
ctx["data"], ctx["mt5_path"], ctx["login"], ctx["server"],
patch_set(BASE_SET, baseline_overrides(args.round)),
f"{prefix}_baseline.set", f"{prefix}_baseline",
)
print(
f"BASELINE PF={baseline.get('profit_factor')} net={baseline.get('net_profit')} "
f"sharpe={baseline.get('sharpe')} trades={baseline.get('total_trades')}",
flush=True,
)
solo_results: list[dict] = []
passed: list[str] = []
for sid in candidates:
spec = sm[sid]
m = run_backtest(
ctx["data"], ctx["mt5_path"], ctx["login"], ctx["server"],
patch_set(BASE_SET, solo_overrides(spec)),
f"{prefix}_solo_{sid}.set", f"{prefix}_solo_{sid}",
test_symbol=spec.get("test_symbol"),
)
verdict = classify_solo(m, args.min_trades)
print(
f"SOLO {sid:10} {verdict:10} PF={m.get('profit_factor')} net={m.get('net_profit')} "
f"sharpe={m.get('sharpe')} trades={m.get('total_trades')}",
flush=True,
)
solo_results.append({"id": sid, "verdict": verdict, "metrics": m})
if verdict in ("PASS", "MARGINAL"):
passed.append(spec["enable"])
enhanced_ov = baseline_overrides(args.round)
for sid in candidates:
spec = sm[sid]
if spec["enable"] in passed:
enhanced_ov[spec["enable"]] = True
enhanced = run_backtest(
ctx["data"], ctx["mt5_path"], ctx["login"], ctx["server"],
patch_set(BASE_SET, enhanced_ov),
f"{prefix}_enhanced.set", f"{prefix}_enhanced",
)
d = delta(baseline, enhanced)
pv = portfolio_verdict(d)
print(
f"ENHANCED {pv} PF={enhanced.get('profit_factor')} net={enhanced.get('net_profit')} "
f"sharpe={enhanced.get('sharpe')} trades={enhanced.get('total_trades')} "
f"dNet={d['net_profit_delta']:.0f} dSharpe={d['sharpe_delta']:.3f}",
flush=True,
)
print(f"PASSED ({len(passed)}): {', '.join(passed)}", flush=True)
summary = {
"timestamp": datetime.now().isoformat(timespec="seconds"),
"round": args.round,
"survivors": list(SURVIVOR_IDS),
"candidates": list(candidates),
"high_margin_disabled": list(HIGH_MARGIN_STOCK_ENABLES),
"period": {"from": runner.FROM_DATE, "to": runner.TO_DATE},
"min_trades": args.min_trades,
"baseline": baseline,
"solo": solo_results,
"passed_enables": passed,
"enhanced": enhanced,
"delta": d,
"portfolio_verdict": pv,
}
out_path = OUT / f"round{args.round}_audit_summary.json"
out_path.write_text(json.dumps(summary, indent=2), encoding="utf-8")
print(f"Saved {out_path}", flush=True)
if __name__ == "__main__":
main()
backtesting/MT5/cluster_audit/run_lot_genetic.py
+322
View File
@@ -0,0 +1,322 @@
#!/usr/bin/env python3
"""
MT5 genetic lot optimization — one production sub-strategy at a time.
Ranges:
stock → 5..15 step 5
other → 0.01..0.1 step 0.01
Usage:
python -m cluster_audit.run_lot_genetic
python -m cluster_audit.run_lot_genetic --only RS_NVDA
python -m cluster_audit.run_lot_genetic --apply
python -m cluster_audit.run_lot_genetic --resume # skip ids already in summary
"""
from __future__ import annotations
import argparse
import json
import re
import sys
import time
from datetime import datetime
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
from cluster_audit.united_mt5_manifest import (
ALL_ENABLE_KEYS,
HIGH_MARGIN_STOCK_ENABLES,
LOT_CLASS_BY_ID,
LOT_GENETIC_RANGE,
PRODUCTION_IDS,
UNITED_MT5_STRATEGIES,
)
from cluster_audit.united_mt5_runner import (
BASE_SET,
CLUSTER,
DEPOSIT,
FROM_DATE,
TO_DATE,
deploy_united,
mt5_context,
patch_set,
patch_set_for_lot_genetic,
run_backtest,
run_genetic_lot_optimize,
)
OUT = Path(__file__).resolve().parent / "reports" / "lot_genetic"
REF_BALANCE = 3000.0
SUMMARY_PATH = OUT / "lot_genetic_summary.json"
def lot_class(sid: str) -> str:
return LOT_CLASS_BY_ID.get(sid, "forex")
def genetic_range(sid: str) -> tuple[float, float, float]:
if lot_class(sid) == "stock":
return LOT_GENETIC_RANGE["stock"]
return LOT_GENETIC_RANGE["default"]
def common_patches() -> dict[str, float | bool]:
o: dict[str, float | bool] = {
"ORCH_ReferenceBalance": REF_BALANCE,
"ORCH_ScaleLotsByBalance": True,
"GAP_Enable": False,
"OPT_GuardOptimizationMode": True,
}
for key in HIGH_MARGIN_STOCK_ENABLES:
o[key] = False
return o
def solo_overrides(spec: dict) -> dict[str, bool]:
o: dict[str, bool] = {k: False for k in ALL_ENABLE_KEYS}
o[spec["enable"]] = True
return o
def apply_lots_to_mq5(text: str, lots: dict[str, float]) -> str:
for key, val in lots.items():
sval = str(int(val)) if val == int(val) else str(val)
text, _ = re.subn(
rf"(input double {re.escape(key)} = )[0-9.]+;",
rf"\g<1>{sval};",
text,
count=1,
)
text, _ = re.subn(
r"(input double ORCH_ReferenceBalance = )[0-9.]+;",
rf"\g<1>{REF_BALANCE};",
text,
count=1,
)
return text
def apply_lots_to_set_text(text: str, lots: dict[str, float]) -> str:
lines_out: list[str] = []
for line in text.splitlines():
if "=" not in line or line.strip().startswith(";"):
lines_out.append(line)
continue
key = line.split("=", 1)[0].strip()
if key in lots:
val = lots[key]
sval = str(int(val)) if val == int(val) else str(val)
if "||" in line:
parts = line.split("||")
parts[0] = f"{key}={sval}"
lines_out.append("||".join(parts))
else:
lines_out.append(f"{key}={sval}")
else:
lines_out.append(line)
return "\n".join(lines_out) + "\n"
def load_summary() -> dict:
if SUMMARY_PATH.exists():
return json.loads(SUMMARY_PATH.read_text(encoding="utf-8"))
return {"results": [], "best_lots": {}}
def save_summary(summary: dict) -> None:
OUT.mkdir(parents=True, exist_ok=True)
SUMMARY_PATH.write_text(json.dumps(summary, indent=2), encoding="utf-8")
def run_backtest_with_retry(
ctx: dict,
set_body: str,
set_name: str,
report: str,
*,
test_symbol: str | None = None,
retries: int = 3,
) -> dict:
last: dict = {"ready": False}
for attempt in range(retries):
if attempt:
time.sleep(12)
last = run_backtest(
ctx["data"], ctx["mt5_path"], ctx["login"], ctx["server"],
set_body, set_name, report, test_symbol=test_symbol,
)
if last.get("ready"):
return last
return last
def lot_report_tag(lot: float) -> str:
return str(int(lot)) if lot == int(lot) else str(lot).replace(".", "p")
def optimize_one(ctx: dict, spec: dict, *, opt_mode: int, grid_only: bool) -> dict:
sid = spec["id"]
lot_key = spec["lot"]
start, step, stop = genetic_range(sid)
ov = {**common_patches(), **solo_overrides(spec)}
report = f"lotgen_{sid}"
print(
f"\n[{sid}] lot sweep {lot_key} range={start}..{stop} step={step} "
f"symbol={spec.get('test_symbol') or 'NAS100'}",
flush=True,
)
if not grid_only:
body = patch_set_for_lot_genetic(BASE_SET, ov, lot_key, start, step, stop)
m = run_genetic_lot_optimize(
ctx["data"], ctx["mt5_path"], ctx["login"], ctx["server"],
body, f"{report}.set", report, lot_key,
test_symbol=spec.get("test_symbol"),
optimization=opt_mode,
)
best_lot = m.get("best_lot")
if m.get("ready") and best_lot is not None:
print(
f" BEST lot={best_lot} PF={m.get('profit_factor')} net={m.get('profit')} "
f"sharpe={m.get('sharpe')} trades={m.get('trades')} passes={m.get('passes')} "
f"({m.get('elapsed_sec')}s)",
flush=True,
)
return {
"id": sid,
"lot_key": lot_key,
"lot_class": lot_class(sid),
"range": {"start": start, "step": step, "stop": stop},
"best_lot": best_lot,
"metrics": m,
}
print(f" genetic XML miss ({m.get('error')}) — grid sweep", flush=True)
from cluster_audit.united_mt5_manifest import LOT_GRIDS
grid = LOT_GRIDS["stock"] if lot_class(sid) == "stock" else LOT_GRIDS["forex"]
best_sc, best_lot, best_m = -1e18, grid[0], {}
t0 = time.time()
for lot in grid:
tag = lot_report_tag(lot)
ov2 = {**ov, lot_key: lot}
bm = run_backtest_with_retry(
ctx,
patch_set(BASE_SET, ov2), f"lot_{sid}_{tag}.set", f"lot_{sid}_{tag}",
test_symbol=spec.get("test_symbol"),
)
if not bm.get("ready"):
print(f" lot={lot} FAILED (no report)", flush=True)
continue
trades = int(bm.get("total_trades") or 0)
profit = float(bm.get("net_profit") or 0)
pf = float(bm.get("profit_factor") or 0)
sharpe = float(bm.get("sharpe") or 0)
if trades < 20 or pf < 1.0 or profit <= 0:
sc = -1e10 + profit
else:
sc = sharpe * 2000 + profit / 500 + pf * 50
print(
f" lot={lot} PF={pf} net={profit} sharpe={sharpe} trades={trades}",
flush=True,
)
if sc > best_sc:
best_sc, best_lot, best_m = sc, lot, bm
elapsed = round(time.time() - t0, 1)
best_m = {**best_m, "best_lot": best_lot, "method": "grid", "elapsed_sec": elapsed}
print(
f" BEST lot={best_lot} PF={best_m.get('profit_factor')} net={best_m.get('net_profit')} "
f"sharpe={best_m.get('sharpe')} trades={best_m.get('total_trades')} ({elapsed}s)",
flush=True,
)
return {
"id": sid,
"lot_key": lot_key,
"lot_class": lot_class(sid),
"range": {"start": start, "step": step, "stop": stop},
"best_lot": best_lot,
"metrics": best_m,
}
def main() -> None:
p = argparse.ArgumentParser()
p.add_argument("--from", dest="from_date", default=FROM_DATE)
p.add_argument("--to", dest="to_date", default=TO_DATE)
p.add_argument("--only", action="append", default=[])
p.add_argument("--apply", action="store_true")
p.add_argument("--resume", action="store_true", help="Skip strategies already in summary")
p.add_argument("--redo", action="append", default=[], help="Re-run these ids even if in summary")
p.add_argument("--mode", choices=("genetic", "complete", "grid"), default="grid",
help="grid=direct lot sweep (default); genetic=try MT5 genetic first")
args = p.parse_args()
import cluster_audit.united_mt5_runner as runner
runner.FROM_DATE = args.from_date.replace("-", ".")
runner.TO_DATE = args.to_date.replace("-", ".")
runner.DEPOSIT = int(REF_BALANCE)
opt_mode = 2 if args.mode == "genetic" else 1
grid_only = args.mode == "grid"
ids = args.only if args.only else list(PRODUCTION_IDS)
sm = {s["id"]: s for s in UNITED_MT5_STRATEGIES}
summary = load_summary() if args.resume else {"results": [], "best_lots": {}}
done_ids = set()
if args.resume:
for r in summary.get("results", []):
m = r.get("metrics") or {}
if m.get("ready") and r["id"] not in args.redo:
done_ids.add(r["id"])
ctx = mt5_context()
deploy_united(ctx["data"], ctx["mt5_path"])
print(
f"Lot genetic deposit={DEPOSIT} ref={REF_BALANCE} "
f"{runner.FROM_DATE}->{runner.TO_DATE} mode={args.mode} n={len(ids)}",
flush=True,
)
for sid in ids:
if sid not in sm:
print(f"skip unknown {sid}", flush=True)
continue
if sid in done_ids and sid not in args.redo:
print(f"skip done {sid}", flush=True)
continue
r = optimize_one(ctx, sm[sid], opt_mode=opt_mode, grid_only=grid_only)
summary["results"] = [x for x in summary.get("results", []) if x["id"] != sid] + [r]
summary["best_lots"][r["lot_key"]] = r["best_lot"]
summary["timestamp"] = datetime.now().isoformat(timespec="seconds")
summary["period"] = {"from": runner.FROM_DATE, "to": runner.TO_DATE}
save_summary(summary)
print(f"\nSaved {SUMMARY_PATH}", flush=True)
for r in summary["results"]:
print(
f" {r['id']:12} {r['lot_key']}={r['best_lot']} "
f"PF={r['metrics'].get('profit_factor')} sharpe={r['metrics'].get('sharpe')}",
flush=True,
)
if args.apply and summary.get("best_lots"):
mq5_path = CLUSTER / "main.mq5"
set_path = CLUSTER / "123.set"
mq5_path.write_text(
apply_lots_to_mq5(mq5_path.read_text(encoding="utf-8"), summary["best_lots"]),
encoding="utf-8",
)
set_path.write_text(
apply_lots_to_set_text(set_path.read_text(encoding="utf-8"), summary["best_lots"]),
encoding="utf-8",
)
print(f"Applied to {mq5_path} and {set_path}", flush=True)
if __name__ == "__main__":
main()
backtesting/MT5/cluster_audit/run_lot_optimize.py
+317
View File
@@ -0,0 +1,317 @@
#!/usr/bin/env python3
"""
MT5 lot-size sweep per production sub-strategy, then combined portfolio.
Constraints:
- ORCH_ReferenceBalance = 3000 (deposit also 3000)
- Stock lots capped at 15 shares
Usage:
python -m cluster_audit.run_lot_optimize
python -m cluster_audit.run_lot_optimize --only RS_NVDA
python -m cluster_audit.run_lot_optimize --apply
"""
from __future__ import annotations
import argparse
import json
import re
import sys
from datetime import datetime
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
from cluster_audit.united_mt5_manifest import (
ALL_ENABLE_KEYS,
HIGH_MARGIN_STOCK_ENABLES,
LOT_CLASS_BY_ID,
LOT_GRIDS,
PRODUCTION_IDS,
UNITED_MT5_STRATEGIES,
)
from cluster_audit.united_mt5_runner import (
BASE_SET,
CLUSTER,
DEPOSIT,
FROM_DATE,
TO_DATE,
deploy_united,
mt5_context,
patch_set,
run_backtest,
)
OUT = Path(__file__).resolve().parent / "reports" / "lot_optimize"
STOCK_LOT_MAX = 15.0
REF_BALANCE = 3000.0
MIN_TRADES = 20
def g(m: dict, k: str) -> float:
v = m.get(k)
return float(v) if v is not None else 0.0
def spec_map() -> dict[str, dict]:
return {s["id"]: s for s in UNITED_MT5_STRATEGIES}
def lot_class(sid: str) -> str:
return LOT_CLASS_BY_ID.get(sid, "forex")
def lot_grid(sid: str) -> list[float]:
cls = lot_class(sid)
grid = list(LOT_GRIDS.get(cls, LOT_GRIDS["forex"]))
if cls == "stock":
grid = [x for x in grid if x <= STOCK_LOT_MAX]
return grid
def score(m: dict) -> float:
if not m.get("ready"):
return -1e12
trades = int(m.get("total_trades") or 0)
if trades < MIN_TRADES:
return -1e11 + trades
net = g(m, "net_profit")
pf = g(m, "profit_factor")
sharpe = g(m, "sharpe")
if net <= 0 or pf < 1.0:
return -1e10 + net
return sharpe * 2000.0 + net / 500.0 + pf * 50.0
def common_patches() -> dict[str, float | bool]:
o: dict[str, float | bool] = {
"ORCH_ReferenceBalance": REF_BALANCE,
"ORCH_ScaleLotsByBalance": True,
"GAP_Enable": False,
"OPT_GuardOptimizationMode": True,
}
for key in HIGH_MARGIN_STOCK_ENABLES:
o[key] = False
return o
def production_enables() -> dict[str, bool]:
prod = set(PRODUCTION_IDS)
o: dict[str, bool] = {}
for s in UNITED_MT5_STRATEGIES:
o[s["enable"]] = s["id"] in prod
return o
def solo_overrides(spec: dict) -> dict[str, bool]:
o: dict[str, bool] = {k: False for k in ALL_ENABLE_KEYS}
o[spec["enable"]] = True
return o
def apply_lots_to_set_text(text: str, lots: dict[str, float]) -> str:
lines_out: list[str] = []
for line in text.splitlines():
if "=" not in line or line.strip().startswith(";"):
lines_out.append(line)
continue
key = line.split("=", 1)[0].strip()
if key in lots:
val = lots[key]
sval = str(int(val)) if val == int(val) else str(val)
if "||" in line:
parts = line.split("||")
parts[0] = f"{key}={sval}"
lines_out.append("||".join(parts))
else:
lines_out.append(f"{key}={sval}")
else:
lines_out.append(line)
for key, val in lots.items():
if not any(l.startswith(f"{key}=") for l in lines_out):
sval = str(int(val)) if val == int(val) else str(val)
lines_out.append(f"{key}={sval}||{sval}||0||{sval}||N")
return "\n".join(lines_out) + "\n"
def apply_lots_to_mq5(text: str, lots: dict[str, float]) -> str:
for key, val in lots.items():
sval = str(int(val)) if val == int(val) else str(val)
text, n = re.subn(
rf"(input double {re.escape(key)} = )[0-9.]+;",
rf"\g<1>{sval};",
text,
count=1,
)
if n == 0:
print(f" warn: {key} not found in main.mq5", flush=True)
text, n = re.subn(
r"(input double ORCH_ReferenceBalance = )[0-9.]+;",
rf"\g<1>{REF_BALANCE};",
text,
count=1,
)
return text
def optimize_one(ctx: dict, spec: dict, sm: dict[str, dict]) -> dict:
sid = spec["id"]
lot_key = spec["lot"]
grid = lot_grid(sid)
print(f"\n[{sid}] grid={grid} class={lot_class(sid)}", flush=True)
best_lot = grid[0]
best_m: dict = {"ready": False}
best_sc = -1e12
trials: list[dict] = []
for lot in grid:
ov: dict = {**common_patches(), **solo_overrides(spec), lot_key: lot}
body = patch_set(BASE_SET, ov)
m = run_backtest(
ctx["data"], ctx["mt5_path"], ctx["login"], ctx["server"],
body, f"lot_{sid}_{lot}.set", f"lot_{sid}_{lot}",
test_symbol=spec.get("test_symbol"),
)
sc = score(m)
trials.append({"lot": lot, "score": sc, "metrics": m})
print(
f" lot={lot:6} sc={sc:10.1f} PF={m.get('profit_factor')} "
f"net={m.get('net_profit')} sharpe={m.get('sharpe')} trades={m.get('total_trades')}",
flush=True,
)
if sc > best_sc:
best_sc = sc
best_lot = lot
best_m = m
return {
"id": sid,
"lot_key": lot_key,
"lot_class": lot_class(sid),
"best_lot": best_lot,
"best_score": best_sc,
"best_metrics": best_m,
"trials": trials,
}
def main() -> None:
p = argparse.ArgumentParser()
p.add_argument("--from", dest="from_date", default=FROM_DATE)
p.add_argument("--to", dest="to_date", default=TO_DATE)
p.add_argument("--only", action="append", default=[], help="Strategy id(s) to optimize")
p.add_argument("--apply", action="store_true", help="Write best lots to main.mq5 and 123.set")
args = p.parse_args()
import cluster_audit.united_mt5_runner as runner
runner.FROM_DATE = args.from_date.replace("-", ".")
runner.TO_DATE = args.to_date.replace("-", ".")
runner.DEPOSIT = int(REF_BALANCE)
OUT.mkdir(parents=True, exist_ok=True)
ctx = mt5_context()
deploy_united(ctx["data"], ctx["mt5_path"])
sm = spec_map()
ids = args.only if args.only else list(PRODUCTION_IDS)
print(
f"Lot optimize deposit={DEPOSIT} ref={REF_BALANCE} "
f"{runner.FROM_DATE}->{runner.TO_DATE} n={len(ids)}",
flush=True,
)
results: list[dict] = []
best_lots: dict[str, float] = {}
for sid in ids:
if sid not in sm:
print(f"skip unknown id {sid}", flush=True)
continue
r = optimize_one(ctx, sm[sid], sm)
results.append(r)
best_lots[r["lot_key"]] = r["best_lot"]
# Baseline combined (123.set lots + production enables)
base_ov: dict = {**common_patches(), **production_enables()}
baseline = run_backtest(
ctx["data"], ctx["mt5_path"], ctx["login"], ctx["server"],
patch_set(BASE_SET, base_ov),
"lot_combined_baseline.set", "lot_combined_baseline",
)
print(
f"\nCOMBINED baseline PF={baseline.get('profit_factor')} net={baseline.get('net_profit')} "
f"sharpe={baseline.get('sharpe')} trades={baseline.get('total_trades')}",
flush=True,
)
opt_ov: dict = {**common_patches(), **production_enables(), **best_lots}
optimized = run_backtest(
ctx["data"], ctx["mt5_path"], ctx["login"], ctx["server"],
patch_set(BASE_SET, opt_ov),
"lot_combined_optimized.set", "lot_combined_optimized",
)
print(
f"COMBINED optimized PF={optimized.get('profit_factor')} net={optimized.get('net_profit')} "
f"sharpe={optimized.get('sharpe')} trades={optimized.get('total_trades')} "
f"dNet={g(optimized, 'net_profit') - g(baseline, 'net_profit'):.0f} "
f"dSharpe={g(optimized, 'sharpe') - g(baseline, 'sharpe'):.3f}",
flush=True,
)
summary = {
"timestamp": datetime.now().isoformat(timespec="seconds"),
"period": {"from": runner.FROM_DATE, "to": runner.TO_DATE},
"deposit": DEPOSIT,
"reference_balance": REF_BALANCE,
"stock_lot_max": STOCK_LOT_MAX,
"solo": results,
"best_lots": best_lots,
"combined_baseline": baseline,
"combined_optimized": optimized,
"combined_delta": {
"net_profit": g(optimized, "net_profit") - g(baseline, "net_profit"),
"sharpe": g(optimized, "sharpe") - g(baseline, "sharpe"),
"pf": g(optimized, "profit_factor") - g(baseline, "profit_factor"),
},
}
out_path = OUT / "lot_optimize_summary.json"
out_path.write_text(json.dumps(summary, indent=2), encoding="utf-8")
print(f"\nSaved {out_path}", flush=True)
print("\nBest lots:", flush=True)
for sid in ids:
if sid not in sm:
continue
row = next(x for x in results if x["id"] == sid)
m = row["best_metrics"]
print(
f" {sid:12} {row['lot_key']}={row['best_lot']} "
f"PF={m.get('profit_factor')} sharpe={m.get('sharpe')} net={m.get('net_profit')}",
flush=True,
)
if args.apply:
mq5_path = CLUSTER / "main.mq5"
set_path = CLUSTER / "123.set"
mq5_path.write_text(apply_lots_to_mq5(mq5_path.read_text(encoding="utf-8"), best_lots), encoding="utf-8")
set_path.write_text(
apply_lots_to_set_text(set_path.read_text(encoding="utf-8"), best_lots),
encoding="utf-8",
)
# Ensure reference balance in set
set_text = set_path.read_text(encoding="utf-8")
set_text = re.sub(
r"ORCH_ReferenceBalance=[^\n]+",
f"ORCH_ReferenceBalance={int(REF_BALANCE)}||{int(REF_BALANCE)}.0||100.000000||10000.000000||N",
set_text,
count=1,
)
set_path.write_text(set_text, encoding="utf-8")
print(f"Applied lots to {mq5_path} and {set_path}", flush=True)
if __name__ == "__main__":
main()
backtesting/MT5/cluster_audit/run_prod_lot_refresh.py
+101
View File
@@ -0,0 +1,101 @@
#!/usr/bin/env python3
"""Lot-opt new production strategies, merge keepers, apply, combined backtest."""
from __future__ import annotations
import json
import sys
from datetime import datetime
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
from cluster_audit.run_lot_genetic import (
SUMMARY_PATH,
apply_lots_to_mq5,
apply_lots_to_set_text,
common_patches,
optimize_one,
save_summary,
)
from cluster_audit.united_mt5_manifest import UNITED_MT5_STRATEGIES
from cluster_audit.united_mt5_runner import (
BASE_SET,
CLUSTER,
deploy_united,
mt5_context,
patch_set,
run_backtest,
)
# Lots from prior genetic run (unchanged strategies)
KEEPER_LOTS: dict[str, float] = {
"LOT_DB_DarvasBox": 0.01,
"LOT_ES_EMASlopeDistance": 0.07,
"LOT_RC_RSICrossOver": 0.1,
"LOT_RM_RSIMidPointHijack": 0.01,
"LOT_RS_NVDA": 5.0,
"LOT_RS_TSLA": 5.0,
"LOT_RRA_AUDUSD": 0.05,
"LOT_UB_USDJPY": 0.03,
"LOT_RS_NAS100": 0.03,
"LOT_UKB_UK100": 0.01,
}
NEW_OPT_IDS = (
"RS_BTCUSD", "RS_XAUUSD", "SE", "ST_BTC", "ST_XAU",
"RRA_GBP", "GB", "U5B", "RS_US30",
)
def production_enables() -> dict[str, bool]:
from cluster_audit.united_mt5_manifest import ALL_ENABLE_KEYS, HIGH_MARGIN_STOCK_ENABLES, PRODUCTION_IDS
prod = set(PRODUCTION_IDS)
o = {s["enable"]: s["id"] in prod for s in UNITED_MT5_STRATEGIES}
for k in HIGH_MARGIN_STOCK_ENABLES:
o[k] = False
return o
def main() -> None:
sm = {s["id"]: s for s in UNITED_MT5_STRATEGIES}
summary = {"results": [], "best_lots": dict(KEEPER_LOTS), "timestamp": datetime.now().isoformat(timespec="seconds")}
save_summary(summary)
ctx = mt5_context()
deploy_united(ctx["data"], ctx["mt5_path"])
print(f"Optimizing {len(NEW_OPT_IDS)} new/changed strategies...", flush=True)
for sid in NEW_OPT_IDS:
r = optimize_one(ctx, sm[sid], opt_mode=1, grid_only=True)
summary["results"].append(r)
summary["best_lots"][r["lot_key"]] = r["best_lot"]
save_summary(summary)
print(f" {sid} -> {r['lot_key']}={r['best_lot']}", flush=True)
# Apply all production lots
lots = summary["best_lots"]
mq5 = CLUSTER / "main.mq5"
st = CLUSTER / "123.set"
mq5.write_text(apply_lots_to_mq5(mq5.read_text(encoding="utf-8"), lots), encoding="utf-8")
st.write_text(apply_lots_to_set_text(st.read_text(encoding="utf-8"), lots), encoding="utf-8")
print(f"Applied {len(lots)} lots to main.mq5 + 123.set", flush=True)
ov = {**common_patches(), **production_enables(), **lots}
m = run_backtest(
ctx["data"], ctx["mt5_path"], ctx["login"], ctx["server"],
patch_set(BASE_SET, ov), "prod_v2_combined.set", "prod_v2_combined",
)
print(
f"\nCOMBINED v2 PF={m.get('profit_factor')} net={m.get('net_profit')} "
f"sharpe={m.get('sharpe')} trades={m.get('total_trades')} dd={m.get('max_drawdown')}",
flush=True,
)
summary["combined_v2"] = m
save_summary(summary)
print(f"Saved {SUMMARY_PATH}", flush=True)
if __name__ == "__main__":
main()
backtesting/MT5/cluster_audit/run_sequential.py
+278
View File
@@ -0,0 +1,278 @@
"""
Sequential cluster audit: one strategy at a time, fix until it passes, then next.
Pass criteria (optimized result):
- >= 1 trade per calendar day over the backtest window (~1977 for 2021-2026)
- net > 0, profit factor >= 1.15, sharpe >= 0.3
- winning months >= 45%, drawdown <= 25%
Usage:
python -m cluster_audit.run_sequential [trials] [--only ID] [--from ID]
"""
from __future__ import annotations
import argparse
import json
import random
import sys
import time
from datetime import datetime
from pathlib import Path
import MetaTrader5 as mt5
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
from cluster_audit.backtest_core import CostModel, load_bars, resolve_symbol
from cluster_audit.diagnose import diagnose
from cluster_audit.engines import ENGINE_MAP
from cluster_audit.portfolio_build import build_progressive_portfolios
from cluster_audit.run_audit import _sample_params
from cluster_audit.scoring import (
DEFAULT_TRADES_PER_DAY,
acceptance,
format_quality_line,
min_trades_for_period,
period_days,
score_label,
score_report,
trades_per_day,
)
from cluster_audit.strategy_registry import PERIODS, STRATEGIES, TF
from cluster_audit.trace_log import TraceLog
LOG = TraceLog(enabled=True, trial_every=10)
PRIMARY_PERIOD = "2021-2026"
def run_one_strategy(
spec: dict,
period_label: str,
start: str,
end: str,
trials: int,
rng: random.Random,
idx: int,
total: int,
days: int,
trades_per_day_target: float,
) -> dict:
sid = spec["id"]
tf_key = spec["tf"]
label = f"[{idx}/{total}] {sid} @ {period_label}"
LOG.banner(label)
engine = ENGINE_MAP[spec["engine"]]
sym = resolve_symbol(spec["symbol"])
start_dt = datetime.fromisoformat(start)
end_dt = datetime.fromisoformat(end)
try:
df = load_bars(sym, TF[tf_key], start_dt, end_dt)
except Exception as e:
LOG.error(f"data load failed: {e}")
return {"id": sid, "period": period_label, "error": str(e), "spec": spec, "passed": False}
LOG.info(f"loaded {len(df)} bars symbol={sym} engine={spec['engine']}")
costs = CostModel.for_symbol(sym)
defaults = dict(spec["defaults"])
min_t = min_trades_for_period(days, trades_per_day_target)
LOG.info(f"activity gate: >={min_t} trades ({trades_per_day_target:.1f}/day x {days}d)")
t0 = time.perf_counter()
baseline = engine(df, sym, period_label, sid, defaults, spec["lot"], costs)
LOG.info(f"baseline: {format_quality_line(baseline, days)} ({(time.perf_counter()-t0)*1000:.0f}ms)")
best = baseline
best_params = defaults
best_score = score_report(baseline, days, trades_per_day_target)
base_ok, base_issues = acceptance(baseline, days, trades_per_day_target)
if base_ok:
LOG.info("baseline PASSES acceptance gates")
else:
LOG.warn("baseline fails: " + "; ".join(base_issues))
if trials > 0:
LOG.info(f"optimizing {trials} trials (score requires trades + profit + consistency)...")
for n in range(1, trials + 1):
params = _sample_params(defaults, spec.get("opt", {}), rng)
r = engine(df, sym, period_label, sid, params, spec["lot"], costs)
sc = score_report(r, days, trades_per_day_target)
if sc > best_score:
best_score = sc
best = r
best_params = params
LOG.trial(n, trials, sc, r.net_profit, r.sharpe, improved=True)
LOG.debug(f" -> {format_quality_line(r, days)}")
elif n % LOG.trial_every == 0 or n == trials:
LOG.trial(n, trials, sc, r.net_profit, r.sharpe, improved=False)
LOG.info(f"optimized: {format_quality_line(best, days)}")
opt_ok, opt_issues = acceptance(best, days, trades_per_day_target)
passed = opt_ok or base_ok
if passed:
LOG.info(f"PASS {sid} - ready for portfolio")
else:
LOG.warn(f"FAIL {sid} - needs engine/logic work before next strategy")
LOG.warn(" " + "; ".join(opt_issues if not opt_ok else base_issues))
diag = diagnose(spec, baseline, best, LOG)
result = {
"id": sid,
"engine": spec["engine"],
"symbol": sym,
"timeframe": tf_key,
"period": period_label,
"bars": len(df),
"period_days": days,
"trades_per_day_target": trades_per_day_target,
"baseline_trades_per_day": trades_per_day(baseline, days),
"optimized_trades_per_day": trades_per_day(best, days),
"passed": passed,
"acceptance_issues": opt_issues if not opt_ok else ([] if base_ok else base_issues),
"baseline": baseline.to_dict(),
"optimized": {**best.to_dict(), "params": best_params},
"optimized_params": best_params,
"optimized_score": best_score,
"improvement_net": best.net_profit - baseline.net_profit,
"improvement_trades": best.total_trades - baseline.total_trades,
"diagnosis": diag,
"spec": spec,
}
out_dir = Path(__file__).parent / "reports" / "sequential"
out_dir.mkdir(parents=True, exist_ok=True)
path = out_dir / f"{sid}_{period_label}.json"
path.write_text(json.dumps({k: v for k, v in result.items() if k != "spec"}, indent=2), encoding="utf-8")
LOG.info(f"saved {path.name}")
try:
from cluster_audit.sync_cluster import main as sync_cluster
sync_cluster()
LOG.info("cluster-latest SuperEA_AuditParams.mqh updated")
except Exception as ex:
LOG.warn(f"cluster sync skipped: {ex}")
return result
def parse_args() -> argparse.Namespace:
p = argparse.ArgumentParser(description="Sequential cluster audit - fix each before next")
p.add_argument("trials", nargs="?", type=int, default=40)
p.add_argument("--portfolio-trials", type=int, default=30)
p.add_argument("--from", dest="from_id", default=None)
p.add_argument("--only", default=None)
p.add_argument("--skip-portfolio", action="store_true")
p.add_argument("--trial-every", type=int, default=10)
p.add_argument("--trades-per-day", type=float, default=DEFAULT_TRADES_PER_DAY)
p.add_argument("--continue-on-fail", action="store_true", help="Run next strategy even if current fails")
return p.parse_args()
def main() -> None:
args = parse_args()
global LOG
LOG = TraceLog(enabled=True, trial_every=args.trial_every)
start, end = PERIODS[PRIMARY_PERIOD]
days = period_days(datetime.fromisoformat(start), datetime.fromisoformat(end))
strategies = STRATEGIES
if args.only:
strategies = [s for s in STRATEGIES if s["id"] == args.only]
elif args.from_id:
found = False
filtered = []
for s in STRATEGIES:
if s["id"] == args.from_id:
found = True
if found:
filtered.append(s)
strategies = filtered if found else STRATEGIES
LOG.banner(
f"SEQUENTIAL AUDIT - {len(strategies)} strategies | "
f">={args.trades_per_day:.1f} trade/day (~{min_trades_for_period(days, args.trades_per_day)} trades) | "
f"{args.trials} opt trials"
)
if not mt5.initialize():
LOG.error(f"MT5 init failed: {mt5.last_error()}")
raise SystemExit(1)
out_dir = Path(__file__).parent / "reports" / "sequential"
out_dir.mkdir(parents=True, exist_ok=True)
rng = random.Random(42)
results: list[dict] = []
passed_ids: list[str] = []
failed_ids: list[str] = []
try:
for i, spec in enumerate(strategies, 1):
r = run_one_strategy(
spec, PRIMARY_PERIOD, start, end, args.trials, rng, i, len(strategies),
days, args.trades_per_day,
)
results.append(r)
if r.get("passed"):
passed_ids.append(r["id"])
elif "error" not in r:
failed_ids.append(r["id"])
if not args.continue_on_fail and not args.only:
LOG.warn(f"STOPPED at {r['id']} - fix engine/params then resume with --from {r['id']}")
break
valid = [r for r in results if r.get("passed") and "error" not in r]
valid.sort(key=lambda x: x.get("optimized_score", float("-inf")), reverse=True)
summary = {
"generated": datetime.now().isoformat(),
"period_days": days,
"trades_per_day_target": args.trades_per_day,
"trials": args.trials,
"passed": passed_ids,
"failed": failed_ids,
"ranking": [
{
"id": r["id"],
"score": r.get("optimized_score"),
"net": r["optimized"]["net_profit"],
"trades": r["optimized"]["total_trades"],
"sharpe": r["optimized"]["sharpe"],
"pf": r["optimized"]["profit_factor"],
}
for r in valid
],
"results": [{k: v for k, v in r.items() if k != "spec"} for r in results],
}
if not args.skip_portfolio and valid:
LOG.banner("PROGRESSIVE PORTFOLIO BUILD (passed strategies only)")
ranked = [{"spec": r["spec"], "optimized_params": r["optimized_params"]} for r in valid]
summary["portfolio_steps"] = build_progressive_portfolios(
ranked, PRIMARY_PERIOD, start, end, args.portfolio_trials, rng, LOG
)
(out_dir / "sequential_summary.json").write_text(json.dumps(summary, indent=2), encoding="utf-8")
LOG.banner(f"PASSED {len(passed_ids)} / FAILED {len(failed_ids)}")
for r in valid:
o = r["optimized"]
LOG.info(
f" {r['id']:28} score={score_label(r['optimized_score'], o['total_trades'], days, args.trades_per_day):>22} "
f"net=${o['net_profit']:9.0f} trades={o['total_trades']:5} ({r.get('optimized_trades_per_day', 0):.2f}/day)"
)
for fid in failed_ids:
LOG.warn(f" NEEDS FIX: {fid}")
LOG.banner(f"DONE in {LOG._elapsed()}")
finally:
mt5.shutdown()
if __name__ == "__main__":
main()
backtesting/MT5/cluster_audit/run_tweak_elimination.py
+428
View File
@@ -0,0 +1,428 @@
#!/usr/bin/env python3
"""
Multi-round elimination audit: small per-strategy tricks vs solo baseline.
Round 1 — each trick vs baseline (123.set, one strategy enabled).
Round 2 — stack non-conflicting WIN tricks from round 1.
Round 3 — ±15% numeric refine around best single winner.
Usage:
python -m cluster_audit.run_tweak_elimination
python -m cluster_audit.run_tweak_elimination --only ES --rounds 1
python -m cluster_audit.run_tweak_elimination --enabled-only --apply
"""
from __future__ import annotations
import argparse
import json
import re
import sys
import time
from datetime import datetime
from itertools import combinations
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
from set_parser import parse_set_file
from cluster_audit.tweak_manifest import STRATEGY_TWEAKS
from cluster_audit.united_mt5_manifest import ALL_ENABLE_KEYS, UNITED_MT5_STRATEGIES
from cluster_audit.united_mt5_runner import (
BASE_SET,
CLUSTER,
deploy_united,
mt5_context,
patch_set,
run_backtest,
)
OUT = Path(__file__).resolve().parent / "reports" / "tweak_elimination"
CHECKPOINT = OUT / "checkpoint.json"
def solo_overrides(spec: dict) -> dict:
o: dict = {k: False for k in ALL_ENABLE_KEYS}
o[spec["enable"]] = True
o["GAP_Enable"] = False
o["OPT_GuardOptimizationMode"] = True
return o
def g(m: dict, k: str) -> float:
v = m.get(k)
return float(v) if v is not None else 0.0
def classify(base: dict, var: dict) -> str:
if not var.get("ready"):
return "BROKEN"
if var.get("total_trades", 0) == 0:
return "NO_TRADES"
d_net = g(var, "net_profit") - g(base, "net_profit")
d_sh = g(var, "sharpe") - g(base, "sharpe")
if d_net > 35 and d_sh >= -0.05:
return "WIN"
if d_net < -35 or d_sh < -0.12:
return "LOSE"
return "NEUTRAL"
def delta(base: dict, var: dict) -> dict:
return {
"net_profit_delta": g(var, "net_profit") - g(base, "net_profit"),
"sharpe_delta": g(var, "sharpe") - g(base, "sharpe"),
"pf_delta": g(var, "profit_factor") - g(base, "profit_factor"),
"trades_delta": int(g(var, "total_trades") - g(base, "total_trades")),
}
def merge_params(*dicts: dict) -> dict:
out: dict = {}
for d in dicts:
out.update(d)
return out
def numeric_refine(params: dict, factor: float) -> dict:
out: dict = {}
for k, v in params.items():
if isinstance(v, (int, float)) and not isinstance(v, bool):
nv = round(v * factor, 4)
out[k] = int(nv) if isinstance(v, int) else nv
else:
out[k] = v
return out
def run_variant(ctx: dict, spec: dict, base_ov: dict, extra: dict, tag: str) -> dict:
ov = {**base_ov, **extra}
body = patch_set(BASE_SET, ov)
safe_tag = re.sub(r"[^\w.-]", "_", tag)[:48]
last: dict = {"ready": False}
for attempt in range(3):
if attempt:
time.sleep(6)
print(f" retry {attempt} {safe_tag}", flush=True)
last = run_backtest(
ctx["data"], ctx["mt5_path"], ctx["login"], ctx["server"],
body, f"twk_{spec['id']}_{safe_tag}.set", f"twk_{spec['id']}_{safe_tag}",
)
if last.get("ready"):
break
time.sleep(2)
return last
def load_checkpoint() -> dict[str, dict]:
if CHECKPOINT.exists():
return json.loads(CHECKPOINT.read_text(encoding="utf-8"))
return {}
def save_checkpoint(all_results: list[dict], *, rounds: int, strategies: list[dict]) -> None:
OUT.mkdir(parents=True, exist_ok=True)
done = {r["id"]: r for r in all_results if "final" in r}
payload = {
"updated": datetime.now().isoformat(),
"rounds": rounds,
"completed": list(done.keys()),
"results": all_results,
}
CHECKPOINT.write_text(json.dumps(payload, indent=2, default=str), encoding="utf-8")
summary = {
"generated": payload["updated"],
"rounds": rounds,
"strategies_tested": len(strategies),
"completed": payload["completed"],
"applied": [sid for sid, r in done.items() if r.get("final", {}).get("action") == "APPLY"],
"kept_baseline": [sid for sid, r in done.items() if r.get("final", {}).get("action") == "KEEP_BASELINE"],
"results": all_results,
}
(OUT / "summary.json").write_text(json.dumps(summary, indent=2, default=str), encoding="utf-8")
def round1(ctx: dict, spec: dict, base_ov: dict) -> tuple[dict, list[dict]]:
sid = spec["id"]
tricks = STRATEGY_TWEAKS.get(sid, [])
print(f"\n{'='*60}\n[{sid}] Round 1 — {len(tricks)} tricks\n{'='*60}", flush=True)
base = run_variant(ctx, spec, base_ov, {}, "base")
print(f" BASE net={base.get('net_profit')} sharpe={base.get('sharpe')} "
f"trades={base.get('total_trades')} ({base.get('elapsed_sec')}s)", flush=True)
results: list[dict] = []
for tw in tricks:
m = run_variant(ctx, spec, base_ov, tw["params"], tw["name"])
v = classify(base, m)
d = delta(base, m)
row = {
"round": 1,
"name": tw["name"],
"params": tw["params"],
"verdict": v,
"delta": d,
"metrics": m,
}
results.append(row)
print(f" {tw['name']:22s} {v:8s} dNet={d['net_profit_delta']:+.0f} "
f"dSharpe={d['sharpe_delta']:+.3f} trades={m.get('total_trades')}", flush=True)
return base, results
def round2(ctx: dict, spec: dict, base_ov: dict, winners: list[dict]) -> list[dict]:
if len(winners) < 2:
return []
sid = spec["id"]
print(f" [{sid}] Round 2 — stack {len(winners)} winners", flush=True)
out: list[dict] = []
for a, b in combinations(winners, 2):
keys_a = set(a["params"])
keys_b = set(b["params"])
if keys_a & keys_b:
continue
combo_name = f"{a['name']}+{b['name']}"
params = merge_params(a["params"], b["params"])
m = run_variant(ctx, spec, base_ov, params, combo_name.replace("+", "_"))
# compare vs best single winner metrics stored in winners
best_single = max(winners, key=lambda w: g(w["metrics"], "net_profit"))
v = classify(best_single["metrics"], m)
d = delta(best_single["metrics"], m)
row = {
"round": 2,
"name": combo_name,
"params": params,
"verdict": v,
"delta_vs_best_single": d,
"metrics": m,
}
out.append(row)
print(f" {combo_name:30s} {v:8s} dNet={d['net_profit_delta']:+.0f} "
f"dSharpe={d['sharpe_delta']:+.3f}", flush=True)
return out
def round3(ctx: dict, spec: dict, base_ov: dict, best: dict) -> list[dict]:
sid = spec["id"]
numeric = {k: v for k, v in best["params"].items() if isinstance(v, (int, float))}
if not numeric:
return []
print(f" [{sid}] Round 3 — refine {best['name']}", flush=True)
out: list[dict] = []
for fac, label in ((0.85, "refine_lo"), (1.15, "refine_hi")):
params = merge_params(
{k: v for k, v in best["params"].items() if k not in numeric},
numeric_refine(numeric, fac),
)
m = run_variant(ctx, spec, base_ov, params, f"{best['name']}_{label}")
v = classify(best["metrics"], m)
d = delta(best["metrics"], m)
out.append({
"round": 3,
"name": f"{best['name']}_{label}",
"params": params,
"verdict": v,
"delta_vs_best": d,
"metrics": m,
})
print(f" {label:12s} {v:8s} dNet={d['net_profit_delta']:+.0f} "
f"dSharpe={d['sharpe_delta']:+.3f}", flush=True)
return out
def pick_final(base: dict, r1: list[dict], r2: list[dict], r3: list[dict]) -> dict:
candidates: list[dict] = []
for r in r1:
if r["verdict"] == "WIN":
candidates.append({"source": f"r1:{r['name']}", "params": r["params"], "metrics": r["metrics"]})
for r in r2:
if r["verdict"] == "WIN":
candidates.append({"source": f"r2:{r['name']}", "params": r["params"], "metrics": r["metrics"]})
for r in r3:
if r["verdict"] == "WIN":
candidates.append({"source": f"r3:{r['name']}", "params": r["params"], "metrics": r["metrics"]})
if not candidates:
return {"action": "KEEP_BASELINE", "params": {}, "baseline": base}
best = max(candidates, key=lambda c: (g(c["metrics"], "sharpe"), g(c["metrics"], "net_profit")))
return {
"action": "APPLY",
"source": best["source"],
"params": best["params"],
"metrics": best["metrics"],
"baseline": base,
"improvement": delta(base, best["metrics"]),
}
def _format_mq5_value(old_val: str, val: object) -> str:
old = old_val.strip()
if isinstance(val, bool):
return "true" if val else "false"
if isinstance(val, int):
return str(val)
if isinstance(val, float):
if "." in old:
return f"{val:.1f}" if val == int(val) else str(val)
return str(int(val)) if val == int(val) else str(val)
if isinstance(val, str):
return f'"{val}"' if not (old.startswith('"') and old.endswith('"')) else f'"{val}"'
return str(val)
def apply_to_mq5_and_set(winners: dict[str, dict]) -> None:
mq5 = CLUSTER / "main.mq5"
st = CLUSTER / "123.set"
text = mq5.read_text(encoding="utf-8")
set_lines = st.read_text(encoding="utf-8", errors="ignore").splitlines()
n_applied = 0
for _sid, w in winners.items():
if w.get("action") != "APPLY":
continue
for key, val in w["params"].items():
pat = rf"(input\s+(?:bool|int|double|string|ENUM_\w+)\s+{re.escape(key)}\s*=\s*)([^;]+)(;)"
def repl(m: re.Match, v: object = val) -> str:
return f"{m.group(1)}{_format_mq5_value(m.group(2), v)}{m.group(3)}"
new_text, n = re.subn(pat, repl, text, count=1)
if n:
text = new_text
n_applied += 1
else:
print(f" WARN mq5 miss {key}", flush=True)
for i, line in enumerate(set_lines):
if not line.startswith(f"{key}="):
continue
sv = "true" if val is True else "false" if val is False else str(val)
parts = line.split("||")
if len(parts) >= 5:
parts[0] = f"{key}={sv}"
set_lines[i] = "||".join(parts)
else:
set_lines[i] = f"{key}={sv}"
break
if re.search(r'#property version\s+"[\d.]+"', text):
text = re.sub(r'(#property version\s+)"[\d.]+"', r'\g<1>"1.27"', text, count=1)
mq5.write_text(text, encoding="utf-8")
st.write_text("\n".join(set_lines) + "\n", encoding="utf-8")
print(f"Applied {n_applied} param updates -> main.mq5 + 123.set", flush=True)
def audit_strategy(ctx: dict, spec: dict, rounds: int) -> dict:
base_ov = solo_overrides(spec)
base, r1 = round1(ctx, spec, base_ov)
winners = [r for r in r1 if r["verdict"] == "WIN"]
r2: list[dict] = []
r3: list[dict] = []
if rounds >= 2 and len(winners) >= 2:
r2 = round2(ctx, spec, base_ov, winners)
winners += [r for r in r2 if r["verdict"] == "WIN"]
if rounds >= 3 and winners:
best = max(winners, key=lambda w: g(w["metrics"], "net_profit"))
r3 = round3(ctx, spec, base_ov, best)
winners += [r for r in r3 if r["verdict"] == "WIN"]
final = pick_final(base, r1, r2, r3)
print(f" => {final['action']} {final.get('source', '')} "
f"dNet={final.get('improvement', {}).get('net_profit_delta', 0):+.0f}", flush=True)
return {
"id": spec["id"],
"name": spec["name"],
"baseline": base,
"round1": r1,
"round2": r2,
"round3": r3,
"final": final,
"eliminated": [r["name"] for r in r1 if r["verdict"] == "LOSE"],
"neutral": [r["name"] for r in r1 if r["verdict"] == "NEUTRAL"],
}
def main() -> int:
p = argparse.ArgumentParser()
p.add_argument("--only", default=None)
p.add_argument("--from", dest="from_id", default=None)
p.add_argument("--enabled-only", action="store_true")
p.add_argument("--rounds", type=int, default=3, choices=[1, 2, 3])
p.add_argument("--apply", action="store_true", help="Write WIN params into main.mq5 + 123.set")
p.add_argument("--resume", action="store_true", help="Skip strategies already in checkpoint.json")
args = p.parse_args()
OUT.mkdir(parents=True, exist_ok=True)
base_params = parse_set_file(BASE_SET)
strategies = list(UNITED_MT5_STRATEGIES)
if args.enabled_only:
strategies = [
s for s in strategies
if base_params.get(s["enable"], type("x", (), {"value": False})).value
]
if args.only:
strategies = [s for s in strategies if s["id"] == args.only]
elif args.from_id:
found = False
filtered = []
for s in strategies:
if s["id"] == args.from_id:
found = True
if found:
filtered.append(s)
strategies = filtered if found else strategies
print(f"Tweak elimination | {len(strategies)} strategies | rounds={args.rounds} | base={BASE_SET.name}")
ctx = mt5_context()
deploy_united(ctx["data"], ctx["mt5_path"])
print("Compiled main.ex5 OK", flush=True)
all_results: list[dict] = []
if args.resume and CHECKPOINT.exists():
ck = json.loads(CHECKPOINT.read_text(encoding="utf-8"))
all_results = ck.get("results", [])
done_ids = {r["id"] for r in all_results if "final" in r}
strategies = [s for s in strategies if s["id"] not in done_ids]
print(f"Resume: skipping {len(done_ids)} done, {len(strategies)} remaining", flush=True)
for spec in strategies:
try:
result = audit_strategy(ctx, spec, args.rounds)
all_results.append(result)
save_checkpoint(all_results, rounds=args.rounds, strategies=strategies)
except Exception as ex:
print(f" ERROR {spec['id']}: {ex}", flush=True)
all_results.append({"id": spec["id"], "error": str(ex)})
save_checkpoint(all_results, rounds=args.rounds, strategies=strategies)
apply_map = {r["id"]: r["final"] for r in all_results if "final" in r}
applied = [sid for sid, f in apply_map.items() if f.get("action") == "APPLY"]
summary = {
"generated": datetime.now().isoformat(),
"rounds": args.rounds,
"strategies_tested": len(strategies),
"applied": applied,
"kept_baseline": [sid for sid, f in apply_map.items() if f.get("action") == "KEEP_BASELINE"],
"results": all_results,
}
out_path = OUT / "summary.json"
out_path.write_text(json.dumps(summary, indent=2, default=str), encoding="utf-8")
print(f"\nSaved {out_path}")
print(f"APPLY ({len(applied)}): {applied}")
if args.apply and applied:
apply_to_mq5_and_set(apply_map)
return 0
if __name__ == "__main__":
raise SystemExit(main())
backtesting/MT5/cluster_audit/run_united_sequential.py
+133
View File
@@ -0,0 +1,133 @@
"""
Sequential United EA audit (main.mq5 strategies from 123.set).
Usage:
python -m cluster_audit.run_united_sequential [trials] [--only ID] [--from ID] [--continue-on-fail]
"""
from __future__ import annotations
import argparse
import json
import random
import sys
from datetime import datetime
from pathlib import Path
import MetaTrader5 as mt5
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
from cluster_audit.run_sequential import run_one_strategy # noqa: E402
from cluster_audit.scoring import ( # noqa: E402
DEFAULT_TRADES_PER_DAY,
min_trades_for_period,
period_days,
score_label,
)
from cluster_audit.trace_log import TraceLog # noqa: E402
from cluster_audit.united_registry import PERIODS, UNITED_STRATEGIES # noqa: E402
PRIMARY_PERIOD = "2021-2026"
def parse_args() -> argparse.Namespace:
p = argparse.ArgumentParser(description="United EA sequential audit (main.mq5)")
p.add_argument("trials", nargs="?", type=int, default=80)
p.add_argument("--from", dest="from_id", default=None)
p.add_argument("--only", default=None)
p.add_argument("--trial-every", type=int, default=10)
p.add_argument("--trades-per-day", type=float, default=DEFAULT_TRADES_PER_DAY)
p.add_argument("--continue-on-fail", action="store_true")
return p.parse_args()
def main() -> None:
args = parse_args()
log = TraceLog(enabled=True, trial_every=args.trial_every)
start, end = PERIODS[PRIMARY_PERIOD]
days = period_days(datetime.fromisoformat(start), datetime.fromisoformat(end))
strategies = UNITED_STRATEGIES
if args.only:
strategies = [s for s in UNITED_STRATEGIES if s["id"] == args.only]
elif args.from_id:
found = False
filtered = []
for s in UNITED_STRATEGIES:
if s["id"] == args.from_id:
found = True
if found:
filtered.append(s)
strategies = filtered if found else UNITED_STRATEGIES
log.banner(
f"UNITED EA AUDIT - {len(strategies)} strategies | "
f">={args.trades_per_day:.1f} trade/day | {args.trials} trials"
)
if not mt5.initialize():
log.error(f"MT5 init failed: {mt5.last_error()}")
raise SystemExit(1)
out_dir = Path(__file__).parent / "reports" / "united_sequential"
out_dir.mkdir(parents=True, exist_ok=True)
rng = random.Random(42)
results: list[dict] = []
passed_ids: list[str] = []
failed_ids: list[str] = []
try:
for i, spec in enumerate(strategies, 1):
r = run_one_strategy(
spec, PRIMARY_PERIOD, start, end, args.trials, rng, i, len(strategies),
days, args.trades_per_day,
)
# relocate report to united folder
src = Path(__file__).parent / "reports" / "sequential" / f"{spec['id']}_{PRIMARY_PERIOD}.json"
dst = out_dir / f"{spec['id']}_{PRIMARY_PERIOD}.json"
if src.exists():
dst.write_text(src.read_text(encoding="utf-8"), encoding="utf-8")
results.append(r)
if r.get("passed"):
passed_ids.append(r["id"])
elif "error" not in r:
failed_ids.append(r["id"])
if not args.continue_on_fail and not args.only:
log.warn(f"STOPPED at {r['id']} — resume with --from {r['id']} --continue-on-fail")
break
summary = {
"generated": datetime.now().isoformat(),
"period_days": days,
"passed": passed_ids,
"failed": failed_ids,
"results": [{k: v for k, v in r.items() if k != "spec"} for r in results],
}
(out_dir / "united_summary.json").write_text(json.dumps(summary, indent=2), encoding="utf-8")
try:
from cluster_audit.sync_united import main as sync_united
sync_united()
log.info("main.mq5 + UnitedEA_Optimized.set updated")
except Exception as ex:
log.warn(f"sync_united skipped: {ex}")
log.banner(f"PASSED {len(passed_ids)} / FAILED {len(failed_ids)}")
for r in results:
if not r.get("passed"):
continue
o = r["optimized"]
log.info(
f" {r['id']:28} net=${o['net_profit']:9.0f} "
f"trades={o['total_trades']:5} pf={o['profit_factor']:.2f}"
)
for fid in failed_ids:
log.warn(f" NEEDS FIX: {fid}")
finally:
mt5.shutdown()
if __name__ == "__main__":
main()
backtesting/MT5/cluster_audit/run_us30_lot_dd.py
+91
View File
@@ -0,0 +1,91 @@
#!/usr/bin/env python3
"""US30 lot sweep — pick lot balancing return vs equity drawdown."""
from __future__ import annotations
import json
import re
import sys
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
from cluster_audit.united_mt5_manifest import ALL_ENABLE_KEYS, UNITED_MT5_STRATEGIES
from cluster_audit.united_mt5_runner import BASE_SET, deploy_united, mt5_context, patch_set, run_backtest
OUT = Path(__file__).resolve().parent / "reports" / "us30_lot_dd"
LOTS = [0.03, 0.04, 0.05, 0.06, 0.07, 0.08]
MAX_DD_PCT = 35.0 # reject lots with equity DD above this
def parse_dd_pct(dd: str | None) -> float | None:
if not dd:
return None
m = re.search(r"([\d.]+)\s*%", str(dd).replace(",", ""))
return float(m.group(1)) if m else None
def main() -> None:
sm = {s["id"]: s for s in UNITED_MT5_STRATEGIES}
spec = sm["RS_US30"]
ov_base = {
k: False for k in ALL_ENABLE_KEYS
}
ov_base[spec["enable"]] = True
ov_base.update({
"ORCH_ReferenceBalance": 3000.0,
"ORCH_ScaleLotsByBalance": True,
"GAP_Enable": False,
"OPT_GuardOptimizationMode": True,
"EnableRSIScalpingMU": False,
})
ctx = mt5_context()
deploy_united(ctx["data"], ctx["mt5_path"])
OUT.mkdir(parents=True, exist_ok=True)
trials: list[dict] = []
best_lot, best_sc, best_row = LOTS[0], -1e18, {}
for lot in LOTS:
ov = {**ov_base, spec["lot"]: lot}
tag = str(lot).replace(".", "p")
m = run_backtest(
ctx["data"], ctx["mt5_path"], ctx["login"], ctx["server"],
patch_set(BASE_SET, ov), f"us30_dd_{tag}.set", f"us30_dd_{tag}",
)
dd_pct = parse_dd_pct(m.get("max_drawdown"))
pf = float(m.get("profit_factor") or 0)
sharpe = float(m.get("sharpe") or 0)
profit = float(m.get("net_profit") or 0)
trades = int(m.get("total_trades") or 0)
if not m.get("ready") or trades < 20 or pf < 1.0:
sc = -1e10
elif dd_pct is not None and dd_pct > MAX_DD_PCT:
sc = sharpe * 500 + profit / 2000 - dd_pct * 100
else:
sc = sharpe * 2000 + profit / 500 + pf * 50 - (dd_pct or 0) * 20
row = {"lot": lot, "dd_pct": dd_pct, "score": sc, "metrics": m}
trials.append(row)
print(
f"lot={lot} PF={pf} net={profit} sharpe={sharpe} dd={m.get('max_drawdown')} sc={sc:.0f}",
flush=True,
)
if sc > best_sc:
best_sc, best_lot, best_row = sc, lot, row
# Prefer highest lot under DD cap with PF>=1.1
under_cap = [t for t in trials if t.get("dd_pct") is not None and t["dd_pct"] <= MAX_DD_PCT
and (t["metrics"].get("profit_factor") or 0) >= 1.1]
if under_cap:
best_lot = max(under_cap, key=lambda t: t["lot"])["lot"]
best_row = next(t for t in trials if t["lot"] == best_lot)
result = {"best_lot": best_lot, "max_dd_cap_pct": MAX_DD_PCT, "best": best_row, "trials": trials}
OUT.mkdir(parents=True, exist_ok=True)
(OUT / "us30_lot_dd.json").write_text(json.dumps(result, indent=2), encoding="utf-8")
print(f"BEST lot={best_lot} dd={best_row.get('dd_pct')}% PF={best_row['metrics'].get('profit_factor')}", flush=True)
if __name__ == "__main__":
main()
backtesting/MT5/cluster_audit/scan_low_margin_stocks.py
+57
View File
@@ -0,0 +1,57 @@
"""Quick solo scan for low-margin stocks using NVDA-style RSI params."""
from __future__ import annotations
import sys
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
from cluster_audit.united_mt5_manifest import ALL_ENABLE_KEYS, HIGH_MARGIN_STOCK_ENABLES
from cluster_audit.united_mt5_runner import BASE_SET, deploy_united, mt5_context, patch_set, run_backtest
import cluster_audit.united_mt5_runner as runner
SYMBOLS = ("SNAP.NYS", "F.NYS", "SOFI.NAS", "PFE.NYS", "AAL.NAS", "NVDA.NAS", "BAC.NYS", "WBD.NAS")
NVDA_PARAMS = {
"RS_F_Symbol": "",
"RS_F_TimeFrame": 15,
"RS_F_RSI_Period": 8,
"RS_F_RSI_Overbought": 36,
"RS_F_RSI_Oversold": 38,
"RS_F_RSI_Target_Buy": 90,
"RS_F_RSI_Target_Sell": 70,
"RS_F_BarsToWait": 5,
"LOT_RS_F": 10,
}
def main() -> None:
ctx = mt5_context()
deploy_united(ctx["data"], ctx["mt5_path"])
print(f"{'symbol':14} {'trades':>6} {'PF':>6} {'net':>10}")
for sym in SYMBOLS:
o = {k: False for k in ALL_ENABLE_KEYS}
o["EnableRSIScalpingF"] = True
for k in HIGH_MARGIN_STOCK_ENABLES:
o[k] = False
o["GAP_Enable"] = False
o.update(NVDA_PARAMS)
o["RS_F_Symbol"] = sym
body = patch_set(BASE_SET, o)
report = f"scan_{sym.replace('.', '_')}"
old = runner.TEST_SYMBOL
runner.TEST_SYMBOL = sym
m = run_backtest(
ctx["data"], ctx["mt5_path"], ctx["login"], ctx["server"],
body, f"{report}.set", report,
)
runner.TEST_SYMBOL = old
print(
f"{sym:14} {int(m.get('total_trades') or 0):6} "
f"{float(m.get('profit_factor') or 0):6.2f} {float(m.get('net_profit') or 0):10.2f}"
)
if __name__ == "__main__":
main()
backtesting/MT5/cluster_audit/scoring.py
+126
View File
@@ -0,0 +1,126 @@
"""Strategy scoring — requires ~1 trade per calendar day over the backtest period."""
from __future__ import annotations
from datetime import date, datetime
from .backtest_core import BacktestReport
DEFAULT_TRADES_PER_DAY = 1.0
def period_days(start: date | datetime | str, end: date | datetime | str) -> int:
if isinstance(start, str):
start = datetime.fromisoformat(start)
if isinstance(end, str):
end = datetime.fromisoformat(end)
if isinstance(start, datetime):
start = start.date()
if isinstance(end, datetime):
end = end.date()
return max(1, (end - start).days)
def min_trades_for_period(days: int, trades_per_day: float = DEFAULT_TRADES_PER_DAY) -> int:
return max(30, int(days * trades_per_day))
def trades_per_day(report: BacktestReport, days: int) -> float:
if report.total_trades == 0 or days <= 0:
return 0.0
return report.total_trades / days
def winning_months_pct(report: BacktestReport) -> float:
if not report.monthly_returns:
return 0.0
vals = list(report.monthly_returns.values())
return 100.0 * sum(1 for v in vals if v > 0) / len(vals)
def score_report(
report: BacktestReport,
period_days_count: int,
trades_per_day_target: float = DEFAULT_TRADES_PER_DAY,
) -> float:
"""Higher is better. Hard-fails below activity + profit gates."""
min_t = min_trades_for_period(period_days_count, trades_per_day_target)
t = report.total_trades
tpd = trades_per_day(report, period_days_count)
if t < min_t or tpd < trades_per_day_target:
return float("-inf")
if report.net_profit <= 0 or report.profit_factor < 1.05:
return float("-inf")
win_mo = winning_months_pct(report) / 100.0
activity = min(tpd / (trades_per_day_target * 1.5), 1.0)
pf = min(report.profit_factor, 4.0) / 4.0
wr = min(report.win_rate, 70.0) / 70.0
return (
report.sharpe * 0.25
+ (report.net_profit / 2000.0) * 0.18
- report.max_drawdown_pct * 0.10
+ activity * 0.22
+ win_mo * 0.12
+ pf * 0.08
+ wr * 0.05
)
def score_label(
score: float,
trades: int,
period_days_count: int,
trades_per_day_target: float = DEFAULT_TRADES_PER_DAY,
) -> str:
min_t = min_trades_for_period(period_days_count, trades_per_day_target)
tpd = trades / period_days_count if period_days_count else 0
if trades < min_t:
return f"N/A ({trades}<{min_t}, need {trades_per_day_target:.1f}/day)"
if tpd < trades_per_day_target:
return f"N/A ({tpd:.2f}/day < {trades_per_day_target:.1f}/day)"
if score == float("-inf"):
return "N/A (fails profit gates)"
return f"{score:.2f}"
def acceptance(
report: BacktestReport,
period_days_count: int,
trades_per_day_target: float = DEFAULT_TRADES_PER_DAY,
) -> tuple[bool, list[str]]:
min_t = min_trades_for_period(period_days_count, trades_per_day_target)
tpd = trades_per_day(report, period_days_count)
issues: list[str] = []
if report.total_trades < min_t:
issues.append(f"trades={report.total_trades} need >={min_t} ({trades_per_day_target:.1f}/day x {period_days_count}d)")
if tpd < trades_per_day_target:
issues.append(f"trades/day={tpd:.2f} need >={trades_per_day_target:.1f}")
if report.net_profit <= 0:
issues.append(f"net=${report.net_profit:.0f} not positive")
if report.profit_factor < 1.15:
issues.append(f"pf={report.profit_factor:.2f} need >=1.15")
if report.sharpe < 0.3:
issues.append(f"sharpe={report.sharpe:.2f} need >=0.30")
if report.max_drawdown_pct > 25:
issues.append(f"dd={report.max_drawdown_pct:.1f}% too high")
win_mo = winning_months_pct(report)
if win_mo < 45:
issues.append(f"winning_months={win_mo:.0f}% need >=45%")
return len(issues) == 0, issues
def format_quality_line(report: BacktestReport, period_days_count: int) -> str:
tpd = trades_per_day(report, period_days_count)
return (
f"net=${report.net_profit:.0f} sharpe={report.sharpe:.2f} "
f"trades={report.total_trades} ({tpd:.2f}/day) pf={report.profit_factor:.2f} "
f"wr={report.win_rate:.0f}% win_mo={winning_months_pct(report):.0f}% "
f"dd={report.max_drawdown_pct:.1f}%"
)
backtesting/MT5/cluster_audit/strategy_registry.py
+134
View File
@@ -0,0 +1,134 @@
"""All SuperEA robot instances with defaults and optimization ranges."""
from __future__ import annotations
import MetaTrader5 as mt5
TF = {
"M10": mt5.TIMEFRAME_M10,
"M15": mt5.TIMEFRAME_M15,
"M20": mt5.TIMEFRAME_M20,
"M30": mt5.TIMEFRAME_M30,
"H1": mt5.TIMEFRAME_H1,
"H4": mt5.TIMEFRAME_H4,
}
STRATEGIES: list[dict] = [
{"id": "darvas_xau", "engine": "darvas", "symbol": "XAUUSD", "tf": "M15", "lot": 0.07,
"defaults": {"box_period": 24, "box_deviation": 80000, "ma_period": 60,
"trend_threshold": 1.2, "volume_threshold": 0,
"stop_loss_pts": 450, "take_profit_pts": 650},
"opt": {"box_period": (12, 48, 4), "box_deviation": (40000, 150000, 5000),
"trend_threshold": (0.3, 5.0, 0.3), "stop_loss_pts": (250, 900, 50),
"take_profit_pts": (350, 1200, 50), "ma_period": (30, 120, 15)}},
{"id": "ema_slope_unit", "engine": "ema_slope", "symbol": "XAUUSD", "tf": "H1", "lot": 0.07,
"defaults": {"ema_period": 85, "price_threshold_pips": 350, "slope_threshold_pips": 22.5,
"monitor_timeout_sec": 340, "max_trades_per_crossover": 12, "use_trailing_stop": False,
"trailing_stop_pips": 74, "max_loss_atr": 1.8, "use_bar_data": True,
"close_unprofitable_trades": True, "profit_check_bars": 36,
"use_weekly_adx_filter": True, "weekly_adx_period": 28,
"weekly_adx_min": 25, "weekly_adx_bar_shift": 8, "weekly_adx_use_direction": True},
"opt": {"ema_period": (50, 120, 5), "price_threshold_pips": (150, 500, 50),
"slope_threshold_pips": (10, 50, 5), "max_loss_atr": (1.2, 3.0, 0.3),
"profit_check_bars": (18, 60, 6), "max_trades_per_crossover": (3, 20, 3)}},
{"id": "ema_slope_trail", "engine": "ema_slope", "symbol": "XAUUSD", "tf": "H1", "lot": 0.07,
"defaults": {"ema_period": 50, "price_threshold_pips": 700, "slope_threshold_pips": 25,
"monitor_timeout_sec": 340, "max_trades_per_crossover": 3, "use_trailing_stop": True,
"trailing_stop_pips": 370, "max_loss_atr": 2.0, "use_bar_data": True,
"close_unprofitable_trades": True, "profit_check_bars": 11,
"use_weekly_adx_filter": True, "weekly_adx_period": 15,
"weekly_adx_min": 40, "weekly_adx_bar_shift": 2, "weekly_adx_use_direction": True},
"opt": {"ema_period": (30, 90, 5), "trailing_stop_pips": (150, 450, 50),
"max_loss_atr": (1.2, 3.5, 0.3), "weekly_adx_min": (25, 50, 5)}},
{"id": "mean_rev_btc", "engine": "mean_reversion", "symbol": "BTCUSD", "tf": "M15", "lot": 0.01,
"defaults": {"ema_period": 80, "min_ema_distance_pts": 300, "rsi_period": 14,
"rsi_oversold": 40, "rsi_overbought": 70, "adx_period": 14,
"adx_max_for_entry": 30, "adx_escape": 40, "use_rsi_cross": True,
"use_hard_sltp": False, "sl_points": 1300, "tp_points": 13400},
"opt": {"rsi_oversold": (30, 50, 5), "rsi_overbought": (60, 80, 5),
"min_ema_distance_pts": (100, 1500, 100), "adx_max_for_entry": (22, 40, 3),
"ema_period": (40, 120, 20)}},
{"id": "rsi_cross_xau", "engine": "rsi_crossover", "symbol": "XAUUSD", "tf": "H1", "lot": 0.01,
"defaults": {"rsi_period": 19, "overbought_level": 93, "oversold_level": 22, "ema_period": 140,
"ema_slope_threshold": 105, "ema_distance_threshold": 165, "exit_buy_rsi": 86,
"exit_sell_rsi": 10, "trailing_stop_pts": 295, "cooldown_seconds": 209,
"tuesday": True, "wednesday": True, "thursday": True,
"trading_hour_one_begin": 0, "trading_hour_one_end": 22,
"trading_hour_two_begin": 6, "trading_hour_two_end": 19},
"opt": {"overbought_level": (65, 95, 5), "oversold_level": (10, 45, 5),
"ema_slope_threshold": (20, 200, 20), "ema_distance_threshold": (40, 300, 20)}},
{"id": "rsi_asian_eur", "engine": "rsi_asian", "symbol": "EURUSD", "tf": "M15", "lot": 0.1,
"defaults": {"rsi_period": 28, "overbought_level": 60, "oversold_level": 8,
"asian_session_start": 0, "asian_session_end": 8, "use_rsi_exit": True, "rsi_exit_level": 55},
"opt": {"overbought_level": (50, 80, 5), "oversold_level": (5, 40, 5)}},
{"id": "rsi_asian_aud", "engine": "rsi_asian", "symbol": "AUDUSD", "tf": "M15", "lot": 0.1,
"defaults": {"rsi_period": 28, "overbought_level": 68, "oversold_level": 30,
"asian_session_start": 0, "asian_session_end": 8, "use_rsi_exit": True, "rsi_exit_level": 55},
"opt": {"overbought_level": (55, 80, 5), "oversold_level": (15, 45, 5)}},
{"id": "rsi_asian_gbp", "engine": "rsi_asian", "symbol": "GBPUSD", "tf": "M15", "lot": 0.1,
"defaults": {"rsi_period": 28, "overbought_level": 80, "oversold_level": 37,
"asian_session_start": 0, "asian_session_end": 8, "use_rsi_exit": True, "rsi_exit_level": 55},
"opt": {"overbought_level": (60, 85, 5), "oversold_level": (20, 50, 5)}},
{"id": "rsi_secret_xau", "engine": "rsi_secret", "symbol": "XAUUSD", "tf": "M30", "lot": 0.01,
"defaults": {"rsi_period": 16, "rsi_overbought": 72.5, "rsi_oversold": 32.5,
"stop_loss_atr": 2.75, "take_profit_atr": 5.0, "min_bars_between_trades": 7},
"opt": {"rsi_overbought": (60, 80, 2), "rsi_oversold": (25, 45, 2), "stop_loss_atr": (1.5, 4, 0.5)}},
]
_RSI_SCALPS = [
("rsi_scalp_appl_unit", "AAPL", "M10", 25.0, {"rsi_period": 14, "rsi_overbought": 80, "rsi_oversold": 78,
"rsi_target_buy": 94, "rsi_target_sell": 44, "bars_to_wait": 7, "use_trailing": False}),
("rsi_scalp_appl_trail", "AAPL", "H1", 25.0, {"rsi_period": 8, "rsi_overbought": 62, "rsi_oversold": 32,
"rsi_target_buy": 67, "rsi_target_sell": 2, "bars_to_wait": 5, "use_trailing": True,
"trail_distance_pts": 50, "trail_activation_pts": 39}),
("rsi_scalp_adbe_trail", "ADBE", "H1", 5.0, {"rsi_period": 15, "rsi_overbought": 16, "rsi_oversold": 42,
"rsi_target_buy": 67, "rsi_target_sell": 62, "bars_to_wait": 8, "use_trailing": True,
"trail_distance_pts": 425, "trail_activation_pts": 18.5}),
("rsi_scalp_btc_unit", "BTCUSD", "H1", 0.1, {"rsi_period": 14, "rsi_overbought": 90, "rsi_oversold": 73,
"rsi_target_buy": 88, "rsi_target_sell": 48, "bars_to_wait": 6, "use_trailing": False}),
("rsi_scalp_btc_trail", "BTCUSD", "H1", 0.1, {"rsi_period": 14, "rsi_overbought": 90, "rsi_oversold": 73,
"rsi_target_buy": 88, "rsi_target_sell": 48, "bars_to_wait": 6, "use_trailing": True,
"trail_distance_pts": 120, "trail_activation_pts": 0}),
("rsi_scalp_mu", "MU", "H1", 25.0, {"rsi_period": 14, "rsi_overbought": 32, "rsi_oversold": 86,
"rsi_target_buy": 100, "rsi_target_sell": 24, "bars_to_wait": 34, "use_trailing": False}),
("rsi_scalp_nvda_unit", "NVDA", "H1", 25.0, {"rsi_period": 14, "rsi_overbought": 6, "rsi_oversold": 66,
"rsi_target_buy": 98, "rsi_target_sell": 52, "bars_to_wait": 12, "use_trailing": False}),
("rsi_scalp_nvda_trail", "NVDA", "M15", 50.0, {"rsi_period": 8, "rsi_overbought": 36, "rsi_oversold": 38,
"rsi_target_buy": 90, "rsi_target_sell": 70, "bars_to_wait": 5, "use_trailing": True,
"trail_distance_pts": 375, "trail_activation_pts": 75}),
("rsi_scalp_nvda_trail_v2", "NVDA", "M15", 50.0, {"rsi_period": 8, "rsi_overbought": 36, "rsi_oversold": 38,
"rsi_target_buy": 90, "rsi_target_sell": 70, "bars_to_wait": 5, "use_trailing": True,
"trail_distance_pts": 375, "trail_activation_pts": 75}),
("rsi_scalp_tsla_unit", "TSLA", "H1", 25.0, {"rsi_period": 14, "rsi_overbought": 32, "rsi_oversold": 86,
"rsi_target_buy": 100, "rsi_target_sell": 24, "bars_to_wait": 34, "use_trailing": False}),
("rsi_scalp_tsla_trail", "TSLA", "H1", 5.0, {"rsi_period": 14, "rsi_overbought": 54, "rsi_oversold": 73,
"rsi_target_buy": 87, "rsi_target_sell": 33, "bars_to_wait": 1, "use_trailing": True,
"trail_distance_pts": 900, "trail_activation_pts": 950}),
("rsi_scalp_xau_trail", "XAUUSD", "H1", 0.1, {"rsi_period": 14, "rsi_overbought": 71, "rsi_oversold": 57,
"rsi_target_buy": 80, "rsi_target_sell": 57, "bars_to_wait": 1, "use_trailing": True,
"trail_distance_pts": 71, "trail_activation_pts": 41}),
]
for sid, sym, tf, lot, defaults in _RSI_SCALPS:
STRATEGIES.append({
"id": sid,
"engine": "rsi_scalp",
"symbol": sym,
"tf": tf,
"lot": lot,
"defaults": {**defaults, "tf": tf},
"opt": {
"rsi_period": (6, 21, 1),
"rsi_overbought": (50, 90, 3),
"rsi_oversold": (10, 70, 3),
"rsi_target_buy": (60, 95, 3),
"rsi_target_sell": (5, 70, 3),
"bars_to_wait": (1, 8, 1),
"trail_distance_pts": (20, 400, 20),
},
})
PERIODS = {
"2021-2026": ("2021-01-01", "2026-06-01"),
"2024-2026": ("2024-01-01", "2026-06-01"),
}
backtesting/MT5/cluster_audit/sync_cluster.py
+317
View File
@@ -0,0 +1,317 @@
"""
Build cluster-latest SuperEA audit params from sequential JSON reports.
Usage:
python -m cluster_audit.sync_cluster
"""
from __future__ import annotations
import json
from datetime import datetime
from pathlib import Path
from cluster_audit.scoring import DEFAULT_TRADES_PER_DAY, acceptance, period_days, trades_per_day
from cluster_audit.strategy_registry import PERIODS, STRATEGIES
REPORTS = Path(__file__).parent / "reports" / "sequential"
OUT_MQH = Path(__file__).resolve().parents[3] / "frontline" / "cluster-latest" / "SuperEA_AuditParams.mqh"
OUT_JSON = Path(__file__).parent / "reports" / "cluster_manifest.json"
RSI_SCALP_IDS = [
"rsi_scalp_appl_unit", "rsi_scalp_appl_trail", "rsi_scalp_adbe_trail",
"rsi_scalp_btc_unit", "rsi_scalp_btc_trail", "rsi_scalp_mu",
"rsi_scalp_nvda_unit", "rsi_scalp_nvda_trail", "rsi_scalp_nvda_trail_v2",
"rsi_scalp_tsla_unit", "rsi_scalp_tsla_trail", "rsi_scalp_xau_trail",
]
RSI_INDEX = {sid: i for i, sid in enumerate(RSI_SCALP_IDS)}
MAGIC_MAP = {s["id"]: 401000 + i for i, s in enumerate(STRATEGIES, 1)}
def load_report(sid: str) -> dict | None:
p = REPORTS / f"{sid}_2021-2026.json"
if not p.exists():
return None
return json.loads(p.read_text(encoding="utf-8"))
def spec_defaults(sid: str) -> dict:
for s in STRATEGIES:
if s["id"] == sid:
return dict(s["defaults"])
return {}
def evaluate_report(r: dict, days: int) -> tuple[bool, list[str]]:
from cluster_audit.backtest_core import BacktestReport
o = r.get("optimized", {})
rep = BacktestReport(
strategy_id=r["id"],
symbol=r.get("symbol", ""),
timeframe=r.get("timeframe", "H1"),
period_label="2021-2026",
net_profit=float(o.get("net_profit", 0)),
total_trades=int(o.get("total_trades", 0)),
win_rate=float(o.get("win_rate", 0)),
profit_factor=float(o.get("profit_factor", 0)),
sharpe=float(o.get("sharpe", 0)),
max_drawdown_pct=float(o.get("max_drawdown_pct", 0)),
avg_win=float(o.get("avg_win", 0)),
avg_loss=float(o.get("avg_loss", 0)),
worst_trades=o.get("worst_trades", []),
losing_trades=o.get("losing_trades", []),
exit_reason_breakdown=o.get("exit_reason_breakdown", {}),
monthly_returns=o.get("monthly_returns", {}),
params=o.get("params", {}),
)
if r.get("passed") is True:
ok, issues = acceptance(rep, days, DEFAULT_TRADES_PER_DAY)
if ok:
return True, []
return acceptance(rep, days, DEFAULT_TRADES_PER_DAY)
def _lit_bool(v) -> str:
return "true" if v else "false"
def emit_darvas(params: dict, ok: bool) -> list[str]:
d = {**spec_defaults("darvas_xau"), **params}
return [
"static void SE_AuditDarvas(DarvasBoxConfig &c)",
"{",
f" c.box_period = {int(d['box_period'])};",
f" c.box_deviation = {float(d['box_deviation'])};",
f" c.ma_period = {int(d['ma_period'])};",
f" c.trend_threshold = {float(d['trend_threshold'])};",
f" c.stop_loss_pts = {float(d['stop_loss_pts'])};",
f" c.take_profit_pts = {float(d['take_profit_pts'])};",
" c.box_timeframe = PERIOD_M15;",
" c.trend_timeframe = PERIOD_M15;",
" c.use_close_breakout = true;",
" c.require_volume_ma = false;",
"}",
f"static bool SE_AuditDarvasEnabled() {{ return {_lit_bool(ok)}; }}",
"",
]
def emit_ema_slope(fn: str, params: dict, ok: bool) -> list[str]:
d = params
lines = [f"static void SE_Audit{fn}(EmaSlopeConfig &c)", "{"]
for key, cast in [
("ema_period", int), ("price_threshold_pips", float), ("slope_threshold_pips", float),
("monitor_timeout_sec", int), ("trailing_stop_pips", float),
("max_trades_per_crossover", int), ("profit_check_bars", int),
("weekly_adx_period", int), ("weekly_adx_min", float), ("weekly_adx_bar_shift", int),
]:
if key in d:
lines.append(f" c.{key} = {cast(d[key])};")
if "use_trailing_stop" in d:
lines.append(f" c.use_trailing_stop = {_lit_bool(d['use_trailing_stop'])};")
lines += ["}", f"static bool SE_Audit{fn}Enabled() {{ return {_lit_bool(ok)}; }}", ""]
return lines
def emit_mean_rev(params: dict, ok: bool) -> list[str]:
d = {**spec_defaults("mean_rev_btc"), **params}
return [
"static void SE_AuditMeanRev(MeanReversionConfig &c)",
"{",
f" c.ema_period = {int(d.get('ema_period', 250))};",
f" c.min_ema_distance_pts = {float(d.get('min_ema_distance_pts', 3650))};",
f" c.rsi_period = {int(d.get('rsi_period', 28))};",
f" c.rsi_oversold = {float(d.get('rsi_oversold', 40))};",
f" c.rsi_overbought = {float(d.get('rsi_overbought', 83))};",
f" c.adx_period = {int(d.get('adx_period', 14))};",
f" c.adx_max_for_entry = {float(d.get('adx_max_for_entry', 17))};",
f" c.adx_escape = {float(d.get('adx_escape', 34))};",
f" c.use_rsi_cross = {_lit_bool(d.get('use_rsi_cross', True))};",
f" c.use_hard_sltp = {_lit_bool(d.get('use_hard_sltp', False))};",
f" c.sl_points = {float(d.get('sl_points', 1300))};",
f" c.tp_points = {float(d.get('tp_points', 13400))};",
"}",
f"static bool SE_AuditMeanRevEnabled() {{ return {_lit_bool(ok)}; }}",
"",
]
def emit_rsi_cross(params: dict, ok: bool) -> list[str]:
d = {**spec_defaults("rsi_cross_xau"), **params}
return [
"static void SE_AuditRsiCross(RsiCrossOverConfig &c)",
"{",
f" c.rsi_period = {int(d.get('rsi_period', 19))};",
f" c.overbought_level = {float(d.get('overbought_level', 93))};",
f" c.oversold_level = {float(d.get('oversold_level', 22))};",
f" c.ema_period = {int(d.get('ema_period', 140))};",
f" c.ema_slope_threshold = {float(d.get('ema_slope_threshold', 105))};",
f" c.ema_distance_threshold = {float(d.get('ema_distance_threshold', 165))};",
f" c.exit_buy_rsi = {float(d.get('exit_buy_rsi', 86))};",
f" c.exit_sell_rsi = {float(d.get('exit_sell_rsi', 10))};",
f" c.trailing_stop_pts = {float(d.get('trailing_stop_pts', 295))};",
f" c.cooldown_seconds = {int(d.get('cooldown_seconds', 209))};",
"}",
f"static bool SE_AuditRsiCrossEnabled() {{ return {_lit_bool(ok)}; }}",
"",
]
def emit_rsi_asian(fn: str, params: dict, ok: bool) -> list[str]:
d = params
return [
f"static void SE_Audit{fn}(RsiAsianConfig &c)",
"{",
f" c.rsi_period = {int(d.get('rsi_period', 28))};",
f" c.overbought_level = {float(d.get('overbought_level', 60))};",
f" c.oversold_level = {float(d.get('oversold_level', 8))};",
f" c.asian_session_start = {int(d.get('asian_session_start', 0))};",
f" c.asian_session_end = {int(d.get('asian_session_end', 8))};",
f" c.use_rsi_exit = {_lit_bool(d.get('use_rsi_exit', True))};",
f" c.rsi_exit_level = {float(d.get('rsi_exit_level', 55))};",
"}",
f"static bool SE_Audit{fn}Enabled() {{ return {_lit_bool(ok)}; }}",
"",
]
def emit_rsi_secret(params: dict, ok: bool) -> list[str]:
d = {**spec_defaults("rsi_secret_xau"), **params}
return [
"static void SE_AuditRsiSecret(RsiSecretSauceConfig &c)",
"{",
f" c.rsi_period = {int(d.get('rsi_period', 16))};",
f" c.rsi_overbought = {float(d.get('rsi_overbought', 72.5))};",
f" c.rsi_oversold = {float(d.get('rsi_oversold', 32.5))};",
f" c.stop_loss_atr = {float(d.get('stop_loss_atr', 2.75))};",
f" c.take_profit_atr = {float(d.get('take_profit_atr', 5.0))};",
f" c.min_bars_between_trades = {int(d.get('min_bars_between_trades', 7))};",
"}",
f"static bool SE_AuditRsiSecretEnabled() {{ return {_lit_bool(ok)}; }}",
"",
]
def emit_rsi_scalp(idx: int, sid: str, params: dict, ok: bool) -> list[str]:
d = {**spec_defaults(sid), **params}
return [
f"static void SE_AuditRsi{idx}(RsiScalpConfig &c)",
"{",
f" c.rsi_period = {int(d.get('rsi_period', 14))};",
f" c.rsi_overbought = {float(d.get('rsi_overbought', 70))};",
f" c.rsi_oversold = {float(d.get('rsi_oversold', 30))};",
f" c.rsi_target_buy = {float(d.get('rsi_target_buy', 80))};",
f" c.rsi_target_sell = {float(d.get('rsi_target_sell', 50))};",
f" c.bars_to_wait = {int(d.get('bars_to_wait', 5))};",
f" c.use_trailing = {_lit_bool(d.get('use_trailing', False))};",
f" c.trail_distance_pts = {float(d.get('trail_distance_pts', 0))};",
f" c.trail_activation_pts = {float(d.get('trail_activation_pts', 0))};",
"}",
f"static bool SE_AuditRsi{idx}Enabled() {{ return {_lit_bool(ok)}; }}",
"",
]
def stub_enabled(name: str, ok: bool = False) -> list[str]:
return [f"static bool SE_Audit{name}Enabled() {{ return {_lit_bool(ok)}; }}", ""]
def main() -> None:
start, end = PERIODS["2021-2026"]
days = period_days(start, end)
manifest: dict = {
"generated": datetime.now().isoformat(),
"period_days": days,
"trades_per_day_target": DEFAULT_TRADES_PER_DAY,
"strategies": {},
}
reports: dict[str, dict] = {}
status: dict[str, bool] = {}
params_map: dict[str, dict] = {}
for spec in STRATEGIES:
sid = spec["id"]
r = load_report(sid)
if not r:
manifest["strategies"][sid] = {"status": "no_report", "passed": False}
status[sid] = False
params_map[sid] = spec_defaults(sid)
continue
reports[sid] = r
params = r.get("optimized_params") or r.get("optimized", {}).get("params", spec_defaults(sid))
params_map[sid] = params
ok, issues = evaluate_report(r, days)
o = r.get("optimized", {})
tpd = trades_per_day(
type("R", (), {"total_trades": int(o.get("total_trades", 0))})(),
days,
)
status[sid] = ok
manifest["strategies"][sid] = {
"passed": ok,
"magic": MAGIC_MAP.get(sid),
"trades": o.get("total_trades"),
"trades_per_day": round(tpd, 3),
"net_profit": o.get("net_profit"),
"sharpe": o.get("sharpe"),
"profit_factor": o.get("profit_factor"),
"issues": issues,
"params": params,
}
lines = [
"//+------------------------------------------------------------------+",
"//| SuperEA_AuditParams.mqh - optimized params from cluster audit |",
f"//| Generated: {datetime.now().isoformat()}",
"//+------------------------------------------------------------------+",
"#ifndef SUPER_EA_AUDIT_PARAMS_MQH",
"#define SUPER_EA_AUDIT_PARAMS_MQH",
"",
]
lines += [f"// darvas_xau: {'PASS' if status.get('darvas_xau') else 'DISABLED'}"]
lines += emit_darvas(params_map.get("darvas_xau", {}), status.get("darvas_xau", False))
lines += [f"// ema_slope_unit: {'PASS' if status.get('ema_slope_unit') else 'DISABLED'}"]
lines += emit_ema_slope("EmaUnit", params_map.get("ema_slope_unit", {}), status.get("ema_slope_unit", False))
lines += [f"// ema_slope_trail: {'PASS' if status.get('ema_slope_trail') else 'DISABLED'}"]
lines += emit_ema_slope("EmaTrail", params_map.get("ema_slope_trail", {}), status.get("ema_slope_trail", False))
lines += [f"// mean_rev_btc: {'PASS' if status.get('mean_rev_btc') else 'DISABLED'}"]
lines += emit_mean_rev(params_map.get("mean_rev_btc", {}), status.get("mean_rev_btc", False))
lines += [f"// rsi_cross_xau: {'PASS' if status.get('rsi_cross_xau') else 'DISABLED'}"]
lines += emit_rsi_cross(params_map.get("rsi_cross_xau", {}), status.get("rsi_cross_xau", False))
for sid, fn in [
("rsi_asian_eur", "RsiAsianEur"),
("rsi_asian_aud", "RsiAsianAud"),
("rsi_asian_gbp", "RsiAsianGbp"),
]:
lines += [f"// {sid}: {'PASS' if status.get(sid) else 'DISABLED'}"]
lines += emit_rsi_asian(fn, params_map.get(sid, {}), status.get(sid, False))
lines += [f"// rsi_secret_xau: {'PASS' if status.get('rsi_secret_xau') else 'DISABLED'}"]
lines += emit_rsi_secret(params_map.get("rsi_secret_xau", {}), status.get("rsi_secret_xau", False))
for sid in RSI_SCALP_IDS:
idx = RSI_INDEX[sid]
lines += [f"// {sid}: {'PASS' if status.get(sid) else 'DISABLED'}"]
lines += emit_rsi_scalp(idx, sid, params_map.get(sid, {}), status.get(sid, False))
lines += ["#endif", ""]
OUT_MQH.parent.mkdir(parents=True, exist_ok=True)
OUT_MQH.write_text("\n".join(lines), encoding="utf-8")
OUT_JSON.write_text(json.dumps(manifest, indent=2), encoding="utf-8")
passed = [k for k, v in status.items() if v]
print(f"Wrote {OUT_MQH}")
print(f"Wrote {OUT_JSON}")
print(f"Passed {len(passed)}/{len(STRATEGIES)}: {', '.join(passed) if passed else '(none)'}")
if __name__ == "__main__":
main()
backtesting/MT5/cluster_audit/sync_united.py
+209
View File
@@ -0,0 +1,209 @@
"""
Sync United EA audit results into main.mq5 defaults and generate .set file.
Usage:
python -m cluster_audit.sync_united
"""
from __future__ import annotations
import json
import re
from datetime import datetime
from pathlib import Path
from cluster_audit.scoring import DEFAULT_TRADES_PER_DAY, acceptance, period_days, trades_per_day
from cluster_audit.united_registry import PERIODS, UNITED_STRATEGIES
REPORTS = Path(__file__).parent / "reports" / "united_sequential"
MAIN_MQ5 = Path(__file__).resolve().parents[3] / "frontline" / "cluster-latest" / "main.mq5"
OUT_SET = Path(__file__).resolve().parents[3] / "frontline" / "cluster-latest" / "UnitedEA_Optimized.set"
OUT_JSON = REPORTS / "united_manifest.json"
# main.mq5 input name -> (strategy_id, param_key in optimized JSON)
PARAM_PATCHES: dict[str, tuple[str, str]] = {
"DB_BoxPeriod": ("united_darvas", "box_period"),
"DB_BoxDeviation": ("united_darvas", "box_deviation"),
"DB_StopLoss": ("united_darvas", "stop_loss_pts"),
"DB_TakeProfit": ("united_darvas", "take_profit_pts"),
"DB_MA_Period": ("united_darvas", "ma_period"),
"DB_TrendThreshold": ("united_darvas", "trend_threshold"),
"RC_overboughtLevel": ("united_rsi_cross", "overbought_level"),
"RC_oversoldLevel": ("united_rsi_cross", "oversold_level"),
"RC_emaSlopeThreshold": ("united_rsi_cross", "ema_slope_threshold"),
"RC_emaDistanceThreshold": ("united_rsi_cross", "ema_distance_threshold"),
"RS_APPL_RSI_Period": ("united_rsi_scalp_appl", "rsi_period"),
"RS_APPL_RSI_Overbought": ("united_rsi_scalp_appl", "rsi_overbought"),
"RS_APPL_RSI_Oversold": ("united_rsi_scalp_appl", "rsi_oversold"),
"RS_APPL_RSI_Target_Buy": ("united_rsi_scalp_appl", "rsi_target_buy"),
"RS_APPL_RSI_Target_Sell": ("united_rsi_scalp_appl", "rsi_target_sell"),
"RS_APPL_BarsToWait": ("united_rsi_scalp_appl", "bars_to_wait"),
"RS_APPL_TrailDistancePoints": ("united_rsi_scalp_appl", "trail_distance_pts"),
"RS_APPL_TrailActivationPoints": ("united_rsi_scalp_appl", "trail_activation_pts"),
"RS_BTCUSD_RSI_Period": ("united_rsi_scalp_btc", "rsi_period"),
"RS_BTCUSD_TrailDistancePoints": ("united_rsi_scalp_btc", "trail_distance_pts"),
"RS_XAUUSD_RSI_Period": ("united_rsi_scalp_xau", "rsi_period"),
"RS_XAUUSD_TrailDistancePoints": ("united_rsi_scalp_xau", "trail_distance_pts"),
"RRA_EURUSD_OverboughtLevel": ("united_rsi_asian_eur", "overbought_level"),
"RRA_EURUSD_OversoldLevel": ("united_rsi_asian_eur", "oversold_level"),
"RSS_RSIOverbought": ("united_rsi_secret", "rsi_overbought"),
"RSS_RSIOversold": ("united_rsi_secret", "rsi_oversold"),
"UB_MinRangePoints": ("united_usdjpy", "min_range_pts"),
"UB_OrderBufferPoints": ("united_usdjpy", "order_buffer_pts"),
}
ENABLE_PATCHES: dict[str, str] = {s["enable_key"]: s["id"] for s in UNITED_STRATEGIES}
def load_report(sid: str) -> dict | None:
p = REPORTS / f"{sid}_2021-2026.json"
if not p.exists():
# fallback cluster audit darvas
alt = Path(__file__).parent / "reports" / "sequential" / "darvas_xau_2021-2026.json"
if sid == "united_darvas" and alt.exists():
r = json.loads(alt.read_text(encoding="utf-8"))
r["id"] = sid
return r
return None
return json.loads(p.read_text(encoding="utf-8"))
def evaluate(r: dict, days: int) -> tuple[bool, list[str]]:
from cluster_audit.backtest_core import BacktestReport
o = r.get("optimized", {})
rep = BacktestReport(
strategy_id=r["id"],
symbol=r.get("symbol", ""),
timeframe=r.get("timeframe", "H1"),
period_label="2021-2026",
net_profit=float(o.get("net_profit", 0)),
total_trades=int(o.get("total_trades", 0)),
win_rate=float(o.get("win_rate", 0)),
profit_factor=float(o.get("profit_factor", 0)),
sharpe=float(o.get("sharpe", 0)),
max_drawdown_pct=float(o.get("max_drawdown_pct", 0)),
avg_win=float(o.get("avg_win", 0)),
avg_loss=float(o.get("avg_loss", 0)),
worst_trades=o.get("worst_trades", []),
losing_trades=o.get("losing_trades", []),
exit_reason_breakdown=o.get("exit_reason_breakdown", {}),
monthly_returns=o.get("monthly_returns", {}),
params=o.get("params", {}),
)
return acceptance(rep, days, DEFAULT_TRADES_PER_DAY)
def patch_main_mqh(text: str, manifest: dict) -> str:
params_by_sid = {k: v.get("params", {}) for k, v in manifest["strategies"].items()}
for input_name, (sid, pkey) in PARAM_PATCHES.items():
params = params_by_sid.get(sid, {})
if pkey not in params:
continue
val = params[pkey]
if isinstance(val, bool):
lit = "true" if val else "false"
elif isinstance(val, float):
lit = str(val) if "." in str(val) else f"{val}.0"
else:
lit = str(val)
text, n = re.subn(
rf"(^input\s+\w+\s+{re.escape(input_name)}\s*=\s*)[^;]+;",
rf"\g<1>{lit};",
text,
count=1,
flags=re.MULTILINE,
)
if n:
print(f" patched {input_name}={lit}")
for enable_key, sid in ENABLE_PATCHES.items():
info = manifest["strategies"].get(sid, {})
if info.get("status") == "no_report" or "passed" not in info:
continue
if not info.get("passed"):
continue # keep main.mq5 enable flags; only auto-enable winners
lit = "true"
text, n = re.subn(
rf"(^input bool {re.escape(enable_key)}\s*=\s*)[^;]+;",
rf"\g<1>{lit};",
text,
count=1,
flags=re.MULTILINE,
)
if n:
print(f" enable {enable_key}={lit}")
return text
def write_set_file(manifest: dict) -> None:
lines = [
"; UnitedEA_Optimized.set — generated from united sequential audit",
f"; {datetime.now().isoformat()}",
"",
]
for spec in UNITED_STRATEGIES:
sid = spec["id"]
info = manifest["strategies"].get(sid, {})
passed = info.get("passed", False)
lines.append(f"; {sid}: {'PASS' if passed else 'DISABLED'}")
lines.append(f"{spec['enable_key']}={'true' if passed else 'false'}")
lines.append(f"{spec['lot_key']}={spec['lot']}")
for k, v in info.get("params", {}).items():
lines.append(f"; {k}={v}")
lines.append("")
OUT_SET.write_text("\n".join(lines), encoding="utf-8")
def main() -> None:
start, end = PERIODS["2021-2026"]
days = period_days(start, end)
manifest: dict = {
"generated": datetime.now().isoformat(),
"period_days": days,
"strategies": {},
}
for spec in UNITED_STRATEGIES:
sid = spec["id"]
r = load_report(sid)
if not r:
manifest["strategies"][sid] = {"passed": False, "status": "no_report"}
continue
params = r.get("optimized_params") or r.get("optimized", {}).get("params", {})
ok, issues = evaluate(r, days)
o = r.get("optimized", {})
manifest["strategies"][sid] = {
"passed": ok,
"enable_key": spec["enable_key"],
"lot_key": spec["lot_key"],
"lot": spec["lot"],
"trades": o.get("total_trades"),
"trades_per_day": round(trades_per_day(
type("R", (), {"total_trades": int(o.get("total_trades", 0))})(), days), 3),
"net_profit": o.get("net_profit"),
"profit_factor": o.get("profit_factor"),
"sharpe": o.get("sharpe"),
"issues": issues,
"params": params,
}
if MAIN_MQ5.exists():
text = MAIN_MQ5.read_text(encoding="utf-8")
print(f"Patching {MAIN_MQ5}")
text = patch_main_mqh(text, manifest)
MAIN_MQ5.write_text(text, encoding="utf-8")
REPORTS.mkdir(parents=True, exist_ok=True)
OUT_JSON.write_text(json.dumps(manifest, indent=2), encoding="utf-8")
write_set_file(manifest)
passed = [k for k, v in manifest["strategies"].items() if v.get("passed")]
print(f"Wrote {OUT_JSON}")
print(f"Wrote {OUT_SET}")
print(f"Passed {len(passed)}/{len(UNITED_STRATEGIES)}: {', '.join(passed) if passed else '(none)'}")
if __name__ == "__main__":
main()
backtesting/MT5/cluster_audit/trace_log.py
+85
View File
@@ -0,0 +1,85 @@
"""Timestamped trace logging for cluster audit runs."""
from __future__ import annotations
import sys
import time
from datetime import datetime
class TraceLog:
def __init__(self, enabled: bool = True, trial_every: int = 10) -> None:
self.enabled = enabled
self.trial_every = max(1, trial_every)
self._t0 = time.perf_counter()
self._phase_t0 = self._t0
def _ts(self) -> str:
return datetime.now().strftime("%H:%M:%S")
def _elapsed(self) -> str:
return f"{time.perf_counter() - self._t0:.1f}s"
def _phase_elapsed(self) -> str:
return f"{time.perf_counter() - self._phase_t0:.1f}s"
def _write(self, level: str, msg: str) -> None:
if not self.enabled:
return
line = f"[{self._ts()} +{self._elapsed()}] [{level}] {msg}"
print(line, flush=True)
def phase_start(self, name: str) -> None:
self._phase_t0 = time.perf_counter()
self._write("PHASE", f">> {name}")
def phase_end(self, name: str, detail: str = "") -> None:
suffix = f" -- {detail}" if detail else ""
self._write("PHASE", f"OK {name} ({self._phase_elapsed()}){suffix}")
def info(self, msg: str) -> None:
self._write("INFO", msg)
def debug(self, msg: str) -> None:
self._write("DEBUG", msg)
def warn(self, msg: str) -> None:
self._write("WARN", msg)
def error(self, msg: str) -> None:
self._write("ERROR", msg)
def banner(self, msg: str) -> None:
if not self.enabled:
return
bar = "=" * min(72, max(len(msg) + 4, 40))
print(f"\n{bar}\n {msg}\n{bar}", flush=True)
def progress(self, current: int, total: int, label: str) -> None:
pct = (100.0 * current / total) if total else 0.0
self._write("PROGRESS", f"[{current}/{total} {pct:.0f}%] {label}")
def trial(self, n: int, total: int, score: float, net: float, sharpe: float, improved: bool) -> None:
if n % self.trial_every != 0 and n != total and not improved:
return
flag = " ** NEW BEST" if improved else ""
score_s = "N/A" if score == float("-inf") else f"{score:.2f}"
self._write(
"TRIAL",
f"{n}/{total} score={score_s} net=${net:.0f} sharpe={sharpe:.2f}{flag}",
)
def report_line(self, strategy_id: str, baseline: dict, optimized: dict, bars: int) -> None:
b, o = baseline, optimized
self._write(
"RESULT",
f"{strategy_id}: bars={bars} | "
f"base net=${b['net_profit']:.0f} sh={b['sharpe']:.2f} trades={b['total_trades']} dd={b['max_drawdown_pct']:.1f}% | "
f"opt net=${o['net_profit']:.0f} sh={o['sharpe']:.2f} trades={o['total_trades']} dd={o['max_drawdown_pct']:.1f}%",
)
if b.get("worst_trades"):
w = b["worst_trades"][0]
self.debug(
f" worst loss: ${w['profit']:.2f} {w['side']} {w['exit_reason']} "
f"({w['open_time']} -> {w['close_time']})"
)
backtesting/MT5/cluster_audit/tweak_manifest.py
+148
View File
@@ -0,0 +1,148 @@
"""Per-strategy 'small trick' candidates for elimination audits."""
from __future__ import annotations
# Each trick: human label + param overrides (applied on top of 123.set solo baseline).
STRATEGY_TWEAKS: dict[str, list[dict]] = {
"DB": [
{"name": "close_on", "params": {"DB_CloseUnprofitableOnNewSignal": True}},
{"name": "tighter_sl", "params": {"DB_StopLoss": 1400}},
{"name": "wider_tp", "params": {"DB_TakeProfit": 4200}},
{"name": "shorter_box", "params": {"DB_BoxPeriod": 140}},
{"name": "no_volume_filter", "params": {"DB_UseVolumeSpikeFilter": False}},
{"name": "lower_trend_thresh", "params": {"DB_TrendThreshold": 3.8}},
],
"ES": [
{"name": "tighter_trail", "params": {"ES_TrailingStop": 280}},
{"name": "wider_trail", "params": {"ES_TrailingStop": 420}},
{"name": "faster_profit_check", "params": {"ES_ProfitCheckBars": 12}},
{"name": "stale_sl_exit", "params": {"ES_UseStaleStopLossExit": True}},
{"name": "lower_adx_gate", "params": {"ES_WeeklyADXMin": 35}},
{"name": "more_trades_per_x", "params": {"ES_MaxTradesPerCrossover": 12}},
],
"RC": [
{"name": "close_on", "params": {"RC_CloseUnprofitableOnNewSignal": True}},
{"name": "short_cooldown", "params": {"RC_cooldownSeconds": 120}},
{"name": "long_cooldown", "params": {"RC_cooldownSeconds": 300}},
{"name": "tighter_trail", "params": {"RC_TrailingStop": 220}},
{"name": "looser_ema_slope", "params": {"RC_emaSlopeThreshold": 90}},
{"name": "rsi_exit_tighter", "params": {"RC_exitBuyRSI": 82, "RC_exitSellRSI": 14}},
],
"RM": [
{"name": "close_on", "params": {"RM_CloseUnprofitableOnNewSignal": True}},
{"name": "short_reverse_cd", "params": {"RM_InpRSIReverseCooldownBars": 8}},
{"name": "ema_cross_only", "params": {"RM_InpEnableRSIFollow": False, "RM_InpEnableRSIReverse": False}},
{"name": "rsi_follow_only", "params": {"RM_InpEnableRSIReverse": False, "RM_InpEnableEMACross": False}},
{"name": "tighter_ema_dist", "params": {"RM_InpEMADistancePips": 120}},
{"name": "close_outside_hours", "params": {"RM_InpRSIFollowCloseOutsideHours": True}},
],
"RS_APPL": [
{"name": "close_on", "params": {"RS_APPL_CloseUnprofitableOnNewSignal": True}},
{"name": "faster_bars", "params": {"RS_APPL_BarsToWait": 5}},
{"name": "trail_activate_50", "params": {"RS_APPL_TrailActivationPoints": 50}},
{"name": "tighter_trail", "params": {"RS_APPL_TrailDistancePoints": 80}},
{"name": "wider_targets", "params": {"RS_APPL_RSI_Target_Buy": 92, "RS_APPL_RSI_Target_Sell": 40}},
],
"RS_ADBE": [
{"name": "close_on", "params": {"RS_ADBE_CloseUnprofitableOnNewSignal": True}},
{"name": "faster_bars", "params": {"RS_ADBE_BarsToWait": 6}},
{"name": "trail_activate_10", "params": {"RS_ADBE_TrailActivationPoints": 10}},
{"name": "tighter_trail", "params": {"RS_ADBE_TrailDistancePoints": 350}},
],
"RS_BTCUSD": [
{"name": "close_on", "params": {"RS_BTCUSD_CloseUnprofitableOnNewSignal": True}},
{"name": "faster_bars", "params": {"RS_BTCUSD_BarsToWait": 4}},
{"name": "trail_activate_30", "params": {"RS_BTCUSD_TrailActivationPoints": 30}},
{"name": "tighter_trail", "params": {"RS_BTCUSD_TrailDistancePoints": 90}},
{"name": "reversal_escape_off", "params": {"RS_UseReversalEscape": False}},
],
"RS_NVDA": [
{"name": "close_on", "params": {"RS_NVDA_CloseUnprofitableOnNewSignal": True}},
{"name": "faster_bars", "params": {"RS_NVDA_BarsToWait": 3}},
{"name": "tighter_trail", "params": {"RS_NVDA_TrailDistancePoints": 280}},
{"name": "lower_trail_act", "params": {"RS_NVDA_TrailActivationPoints": 50}},
{"name": "wider_ob_os", "params": {"RS_NVDA_RSI_Overbought": 40, "RS_NVDA_RSI_Oversold": 35}},
],
"RS_TSLA": [
{"name": "faster_bars", "params": {"RS_TSLA_BarsToWait": 2}},
{"name": "trail_act_400", "params": {"RS_TSLA_TrailActivationPoints": 400}},
{"name": "trail_act_600", "params": {"RS_TSLA_TrailActivationPoints": 600}},
{"name": "tighter_trail", "params": {"RS_TSLA_TrailDistancePoints": 700}},
{"name": "wider_trail", "params": {"RS_TSLA_TrailDistancePoints": 1100}},
],
"RS_XAUUSD": [
{"name": "faster_bars", "params": {"RS_XAUUSD_BarsToWait": 3}},
{"name": "lower_trail_act", "params": {"RS_XAUUSD_TrailActivationPoints": 25}},
{"name": "tighter_trail", "params": {"RS_XAUUSD_TrailDistancePoints": 55}},
{"name": "wider_trail", "params": {"RS_XAUUSD_TrailDistancePoints": 90}},
{"name": "reversal_escape_off", "params": {"RS_UseReversalEscape": False}},
],
"RS_MU": [
{"name": "close_on", "params": {"RS_MU_CloseUnprofitableOnNewSignal": True}},
{"name": "faster_bars", "params": {"RS_MU_BarsToWait": 20}},
{"name": "symbol_munas", "params": {"RS_MU_Symbol": "MU.NAS"}},
{"name": "symbol_muus", "params": {"RS_MU_Symbol": "MU.US"}},
{"name": "looser_ob_os", "params": {"RS_MU_RSI_Overbought": 40, "RS_MU_RSI_Oversold": 75}},
],
"SE": [
{"name": "close_on", "params": {"SE_CloseUnprofitableOnNewSignal": True}},
{"name": "exit_trend_flip", "params": {"SE_ExitOnTrendFlip": True}},
{"name": "shorter_hold", "params": {"SE_MaxHoldingBars": 120}},
{"name": "longer_hold", "params": {"SE_MaxHoldingBars": 220}},
{"name": "structural_sl", "params": {"SE_UseStructuralSL": True}},
{"name": "cci_looser", "params": {"SE_CciOversold": -120}},
],
"RCO": [
{"name": "close_on", "params": {"RCO_CloseUnprofitableOnNewSignal": True}},
{"name": "higher_adx_cap", "params": {"RCO_ADX_Max": 32}},
{"name": "tighter_sl", "params": {"RCO_SL_ATR_Mult": 1.9}},
{"name": "wider_tp", "params": {"RCO_TP_ATR_Mult": 2.7}},
{"name": "shorter_max_bars", "params": {"RCO_MaxBarsInTrade": 40}},
],
"RRA_EUR": [
{"name": "close_on", "params": {"RRA_EURUSD_CloseUnprofitableOnNewSignal": True}},
{"name": "shorter_duration", "params": {"RRA_EURUSD_MaxDuration": 200}},
{"name": "close_outside", "params": {"RRA_EURUSD_CloseOutsideSession": True}},
{"name": "rsi_exit_50", "params": {"RRA_EURUSD_RSIExitLevel": 50}},
{"name": "tighter_os", "params": {"RRA_EURUSD_OversoldLevel": 12}},
],
"RRA_AUD": [
{"name": "close_on", "params": {"RRA_AUDUSD_CloseUnprofitableOnNewSignal": True}},
{"name": "shorter_duration", "params": {"RRA_AUDUSD_MaxDuration": 250}},
{"name": "no_close_outside", "params": {"RRA_AUDUSD_CloseOutsideSession": False}},
{"name": "tighter_os", "params": {"RRA_AUDUSD_OversoldLevel": 28}},
{"name": "rsi_exit_45", "params": {"RRA_AUDUSD_RSIExitLevel": 45}},
],
"ST_BTC": [
{"name": "close_on", "params": {"ST_BTC_CloseUnprofitableOnNewSignal": True}},
{"name": "tighter_break", "params": {"ST_BTC_BreakBuffer": 70}},
{"name": "looser_touch", "params": {"ST_BTC_LineTouchTolerance": 150}},
{"name": "longer_ma", "params": {"ST_BTC_MAPeriod": 180}},
],
"ST_XAU": [
{"name": "close_on", "params": {"ST_XAU_CloseUnprofitableOnNewSignal": True}},
{"name": "tighter_break", "params": {"ST_XAU_BreakBuffer": 90}},
{"name": "shorter_ma", "params": {"ST_XAU_MAPeriod": 55}},
{"name": "looser_touch", "params": {"ST_XAU_LineTouchTolerance": 260}},
],
"ST_GER": [
{"name": "close_on", "params": {"ST_GER_CloseUnprofitableOnNewSignal": True}},
{"name": "tighter_break", "params": {"ST_GER_BreakBuffer": 60}},
{"name": "looser_touch", "params": {"ST_GER_LineTouchTolerance": 120}},
{"name": "longer_ma", "params": {"ST_GER_MAPeriod": 80}},
],
"RSS": [
{"name": "close_on", "params": {"RSS_CloseUnprofitableOnNewSignal": True}},
{"name": "swing_sl", "params": {"RSS_UseSwingStopLoss": True}},
{"name": "shorter_cooldown", "params": {"RSS_MinBarsBetweenTrades": 4}},
{"name": "tighter_sl_atr", "params": {"RSS_StopLossATR": 2.2}},
],
"UB": [
{"name": "close_on", "params": {"UB_CloseUnprofitableOnNewSignal": True}},
{"name": "min_range_12", "params": {"UB_MinRangePoints": 12}},
{"name": "order_buf_35", "params": {"UB_OrderBufferPoints": 3.5}},
{"name": "first_trade_only", "params": {"UB_FirstTradeOnly": True}},
{"name": "wider_range", "params": {"UB_MinRangePoints": 18}},
{"name": "tighter_buffer", "params": {"UB_OrderBufferPoints": 3.0}},
],
}
backtesting/MT5/cluster_audit/united_mt5_manifest.py
+112
View File
@@ -0,0 +1,112 @@
"""United EA sub-strategy manifest for MT5 solo audits (123.set)."""
from __future__ import annotations
UNITED_MT5_STRATEGIES: list[dict] = [
{"id": "DB", "name": "DarvasBox", "enable": "EnableDarvasBox", "close": "DB_CloseUnprofitableOnNewSignal", "lot": "LOT_DB_DarvasBox"},
{"id": "ES", "name": "EMASlopeDistance", "enable": "EnableEMASlopeDistance", "close": "ES_CloseUnprofitableOnNewSignal", "lot": "LOT_ES_EMASlopeDistance"},
{"id": "RC", "name": "RSICrossOverReversal", "enable": "EnableRSICrossOverReversal", "close": "RC_CloseUnprofitableOnNewSignal", "lot": "LOT_RC_RSICrossOver"},
{"id": "RM", "name": "RSIMidPointHijack", "enable": "EnableRSIMidPointHijack", "close": "RM_CloseUnprofitableOnNewSignal", "lot": "LOT_RM_RSIMidPointHijack"},
{"id": "RS_APPL", "name": "RSIScalping APPL", "enable": "EnableRSIScalpingAPPL", "close": "RS_APPL_CloseUnprofitableOnNewSignal", "lot": "LOT_RS_APPL", "test_symbol": "AAPL.NAS", "lot_class": "stock"},
{"id": "RS_ADBE", "name": "RSIScalping ADBE", "enable": "EnableRSIScalpingADBE", "close": "RS_ADBE_CloseUnprofitableOnNewSignal", "lot": "LOT_RS_ADBE", "test_symbol": "ADBE.NAS", "lot_class": "stock"},
{"id": "RS_BTCUSD", "name": "RSIScalping BTCUSD", "enable": "EnableRSIScalpingBTCUSD", "close": "RS_BTCUSD_CloseUnprofitableOnNewSignal", "lot": "LOT_RS_BTCUSD"},
{"id": "RS_NVDA", "name": "RSIScalping NVDA", "enable": "EnableRSIScalpingNVDA", "close": "RS_NVDA_CloseUnprofitableOnNewSignal", "lot": "LOT_RS_NVDA", "test_symbol": "NVDA.NAS", "lot_class": "stock"},
{"id": "RS_TSLA", "name": "RSIScalping TSLA", "enable": "EnableRSIScalpingTSLA", "close": "RS_TSLA_CloseUnprofitableOnNewSignal", "lot": "LOT_RS_TSLA", "test_symbol": "TSLA.NAS", "lot_class": "stock"},
{"id": "RS_XAUUSD", "name": "RSIScalping XAUUSD", "enable": "EnableRSIScalpingXAUUSD", "close": "RS_XAUUSD_CloseUnprofitableOnNewSignal", "lot": "LOT_RS_XAUUSD"},
{"id": "RS_MU", "name": "RSIScalping MU", "enable": "EnableRSIScalpingMU", "close": "RS_MU_CloseUnprofitableOnNewSignal", "lot": "LOT_RS_MU"},
{"id": "SE", "name": "SuperEMA", "enable": "EnableSuperEMA", "close": "SE_CloseUnprofitableOnNewSignal", "lot": "LOT_SE_SuperEMA"},
{"id": "RCO", "name": "RSIConsolidation", "enable": "EnableRSIConsolidation", "close": "RCO_CloseUnprofitableOnNewSignal", "lot": "LOT_RCO_RSIConsolidation"},
{"id": "RRA_EUR", "name": "RSI Asian EURUSD", "enable": "EnableRSIReversalAsianEURUSD", "close": "RRA_EURUSD_CloseUnprofitableOnNewSignal", "lot": "LOT_RRA_EURUSD"},
{"id": "RRA_AUD", "name": "RSI Asian AUDUSD", "enable": "EnableRSIReversalAsianAUDUSD", "close": "RRA_AUDUSD_CloseUnprofitableOnNewSignal", "lot": "LOT_RRA_AUDUSD"},
{"id": "ST_BTC", "name": "SimpleTrendline BTC", "enable": "EnableSimpleTrendlineBTCUSD", "close": "ST_BTC_CloseUnprofitableOnNewSignal", "lot": "LOT_ST_BTCUSD"},
{"id": "ST_XAU", "name": "SimpleTrendline XAU", "enable": "EnableSimpleTrendlineXAUUSD", "close": "ST_XAU_CloseUnprofitableOnNewSignal", "lot": "LOT_ST_XAUUSD"},
{"id": "ST_GER", "name": "SimpleTrendline GER40", "enable": "EnableSimpleTrendlineGER40", "close": "ST_GER_CloseUnprofitableOnNewSignal", "lot": "LOT_ST_GER40"},
{"id": "RSS", "name": "RSISecretSauce", "enable": "EnableRSISecretSauce", "close": "RSS_CloseUnprofitableOnNewSignal", "lot": "LOT_RSS_SecretSauce"},
{"id": "UB", "name": "USDJPYBuster", "enable": "EnableUSDJPYBuster", "close": "UB_CloseUnprofitableOnNewSignal", "lot": "LOT_UB_USDJPY"},
{"id": "XBT", "name": "XAUBearTrend", "enable": "EnableXAUBearTrend", "close": "XBT_CloseUnprofitableOnNewSignal", "lot": "LOT_XBT_XAUUSD"},
{"id": "XMB", "name": "XAUMomentumBreakdown", "enable": "EnableXAUMomentumBreakdown", "close": "XMB_CloseUnprofitableOnNewSignal", "lot": "LOT_XMB_XAUUSD"},
{"id": "RRA_GBP", "name": "RSI Asian GBPUSD", "enable": "EnableRSIReversalAsianGBPUSD", "close": "RRA_GBPUSD_CloseUnprofitableOnNewSignal", "lot": "LOT_RRA_GBPUSD"},
{"id": "GB", "name": "GER40Buster", "enable": "EnableGER40Buster", "close": "GB_CloseUnprofitableOnNewSignal", "lot": "LOT_GB_GER40"},
{"id": "RS_NAS100", "name": "RSIScalping NAS100", "enable": "EnableRSIScalpingNAS100", "close": "RS_NAS100_CloseUnprofitableOnNewSignal", "lot": "LOT_RS_NAS100"},
{"id": "RS_US500", "name": "RSIScalping US500", "enable": "EnableRSIScalpingUS500", "close": "RS_US500_CloseUnprofitableOnNewSignal", "lot": "LOT_RS_US500"},
{"id": "RRA_USDCHF", "name": "RSI Asian USDCHF", "enable": "EnableRSIReversalAsianUSDCHF", "close": "RRA_USDCHF_CloseUnprofitableOnNewSignal", "lot": "LOT_RRA_USDCHF"},
{"id": "RRA_NZDUSD", "name": "RSI Asian NZDUSD", "enable": "EnableRSIReversalAsianNZDUSD", "close": "RRA_NZDUSD_CloseUnprofitableOnNewSignal", "lot": "LOT_RRA_NZDUSD"},
{"id": "NB", "name": "NAS100Buster", "enable": "EnableNAS100Buster", "close": "NB_CloseUnprofitableOnNewSignal", "lot": "LOT_NB_NAS100"},
{"id": "U5B", "name": "US500Buster", "enable": "EnableUS500Buster", "close": "U5B_CloseUnprofitableOnNewSignal", "lot": "LOT_U5B_US500"},
{"id": "RS_US30", "name": "RSIScalping US30", "enable": "EnableRSIScalpingUS30", "close": "RS_US30_CloseUnprofitableOnNewSignal", "lot": "LOT_RS_US30"},
{"id": "RS_XAGUSD", "name": "RSIScalping XAGUSD", "enable": "EnableRSIScalpingXAGUSD", "close": "RS_XAGUSD_CloseUnprofitableOnNewSignal", "lot": "LOT_RS_XAGUSD"},
{"id": "RS_EURJPY", "name": "RSIScalping EURJPY", "enable": "EnableRSIScalpingEURJPY", "close": "RS_EURJPY_CloseUnprofitableOnNewSignal", "lot": "LOT_RS_EURJPY"},
{"id": "RS_GBPJPY", "name": "RSIScalping GBPJPY", "enable": "EnableRSIScalpingGBPJPY", "close": "RS_GBPJPY_CloseUnprofitableOnNewSignal", "lot": "LOT_RS_GBPJPY"},
{"id": "U30B", "name": "US30Buster", "enable": "EnableUS30Buster", "close": "U30B_CloseUnprofitableOnNewSignal", "lot": "LOT_U30B_US30"},
{"id": "UKB", "name": "UK100Buster", "enable": "EnableUK100Buster", "close": "UKB_CloseUnprofitableOnNewSignal", "lot": "LOT_UKB_UK100"},
{"id": "XGB", "name": "XAGUSDBuster", "enable": "EnableXAGUSDBuster", "close": "XGB_CloseUnprofitableOnNewSignal", "lot": "LOT_XGB_XAGUSD"},
{"id": "RS_F", "name": "RSIScalping F", "enable": "EnableRSIScalpingF", "close": "RS_F_CloseUnprofitableOnNewSignal", "lot": "LOT_RS_F", "test_symbol": "F.NYS"},
{"id": "RS_SOFI", "name": "RSIScalping SOFI", "enable": "EnableRSIScalpingSOFI", "close": "RS_SOFI_CloseUnprofitableOnNewSignal", "lot": "LOT_RS_SOFI", "test_symbol": "SOFI.NAS"},
{"id": "RS_SNAP", "name": "RSIScalping SNAP", "enable": "EnableRSIScalpingSNAP", "close": "RS_SNAP_CloseUnprofitableOnNewSignal", "lot": "LOT_RS_SNAP", "test_symbol": "SNAP.NYS"},
{"id": "RS_WBD", "name": "RSIScalping WBD", "enable": "EnableRSIScalpingWBD", "close": "RS_WBD_CloseUnprofitableOnNewSignal", "lot": "LOT_RS_WBD", "test_symbol": "WBD.NAS"},
]
ALL_ENABLE_KEYS = [s["enable"] for s in UNITED_MT5_STRATEGIES]
# Round-1 expansion (retired except survivor).
EXPANSION_RETIRED_IDS = (
"XBT", "XMB", "RRA_GBP", "GB", "RS_US500", "RRA_USDCHF", "RRA_NZDUSD", "NB", "U5B",
)
# Survivor kept on baseline + enhanced.
SURVIVOR_IDS = ("RS_NAS100", "RS_US30", "UKB")
# Round-2 candidates (audited; non-survivors stay off).
ROUND2_IDS = (
"RS_US30", "RS_XAGUSD", "RS_EURJPY", "RS_GBPJPY", "U30B", "UKB", "XGB",
)
# Round-3 low-margin stock candidates (~$1–2 margin per share @ 5% leverage).
ROUND3_IDS = ("RS_F", "RS_SOFI", "RS_SNAP", "RS_WBD")
# High share-price stocks — force off in expansion audits (margin call risk).
HIGH_MARGIN_STOCK_ENABLES = ("EnableRSIScalpingMU",)
# Production cluster (matches main.mq5 defaults).
PRODUCTION_IDS: tuple[str, ...] = (
"DB", "ES", "RC", "RM",
"RS_NVDA", "RS_TSLA",
"RS_BTCUSD", "RS_XAUUSD", "SE", "ST_BTC", "ST_XAU",
"RRA_AUD", "RRA_GBP", "UB",
"RS_NAS100", "RS_US30", "UKB", "GB", "U5B",
)
LOT_GRIDS: dict[str, list[float]] = {
"stock": [5.0, 10.0, 15.0],
"index": [0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1],
"forex": [0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1],
"gold": [0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1],
"crypto": [0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1],
}
LOT_GENETIC_RANGE: dict[str, tuple[float, float, float]] = {
"stock": (5.0, 5.0, 15.0),
"default": (0.01, 0.01, 0.1),
}
LOT_CLASS_BY_ID: dict[str, str] = {
"DB": "gold", "ES": "gold", "RC": "gold", "RM": "gold",
"RS_XAUUSD": "gold", "ST_XAU": "gold", "XBT": "gold", "XMB": "gold", "XGB": "gold",
"RS_APPL": "stock", "RS_ADBE": "stock", "RS_NVDA": "stock", "RS_TSLA": "stock", "RS_MU": "stock",
"RS_F": "stock", "RS_SOFI": "stock", "RS_SNAP": "stock", "RS_WBD": "stock",
"RS_NAS100": "index", "RS_US500": "index", "RS_US30": "index", "UKB": "index",
"NB": "index", "U5B": "index", "U30B": "index", "GB": "index",
"RRA_EUR": "forex", "RRA_AUD": "forex", "RRA_GBP": "forex", "RRA_USDCHF": "forex", "RRA_NZDUSD": "forex",
"UB": "forex", "RS_EURJPY": "forex", "RS_GBPJPY": "forex",
"RS_BTCUSD": "crypto", "ST_BTC": "crypto",
"ST_GER": "index", "RS_XAGUSD": "gold",
}
PARAM_TWEAKS: dict[str, list[dict]] = {
"ES": [{"ES_TrailingStop": 250}, {"ES_TrailingStop": 300, "ES_ProfitCheckBars": 12}],
"RC": [{"RC_cooldownSeconds": 120}, {"RC_TrailingStop": 250}],
"RS_NVDA": [{"RS_NVDA_BarsToWait": 3}, {"RS_NVDA_TrailDistancePoints": 300}],
"RS_TSLA": [{"RS_TSLA_BarsToWait": 2}, {"RS_TSLA_TrailActivationPoints": 500}],
"RCO": [{"RCO_ADX_Max": 32}, {"RCO_SL_ATR_Mult": 1.9}],
"UB": [{"UB_MinRangePoints": 12}, {"UB_OrderBufferPoints": 3.5}],
}
backtesting/MT5/cluster_audit/united_mt5_runner.py
+488
View File
@@ -0,0 +1,488 @@
"""MT5 Strategy Tester runner for United EA (cluster-latest/main.mq5)."""
from __future__ import annotations
import os
import re
import shutil
import subprocess
import time
from pathlib import Path
import MetaTrader5 as mt5
CLUSTER = Path(__file__).resolve().parents[3] / "frontline" / "cluster-latest"
BASE_SET = CLUSTER / "123.set"
DEPOSIT = 3000
LEVERAGE = 1000
FROM_DATE = "2023.07.01"
TO_DATE = "2026.06.01"
TEST_SYMBOL = "NAS100"
TEST_PERIOD = "H1"
LABELS = {
"profit_factor": ("Profit Factor", "盈利因子"),
"net_profit": ("Total Net Profit", "总净盈利"),
"total_trades": ("Total Trades", "交易总计"),
"sharpe": ("Sharpe Ratio", "夏普比率"),
"equity_dd": ("Equity Drawdown Maximal", "最大回撤"),
"margin_level": ("Minimal margin level", "最低保证金比例"),
}
def read_text(path: Path) -> str:
raw = path.read_bytes()
for enc in ("utf-16", "utf-16-le", "utf-8", "cp1252"):
try:
text = raw.decode(enc)
if text.strip():
return text
except UnicodeError:
continue
return raw.decode("utf-8", errors="ignore")
def grab_metric(text: str, key: str) -> str | None:
for label in LABELS[key]:
for pat in (
rf">{re.escape(label)}</td>\s*<td[^>]*>(?:<b>)?([^<]+)",
rf">{re.escape(label)}:</td>\s*<td[^>]*>(?:<b>)?([^<]+)",
rf">{re.escape(label)}</td>\s*<td[^>]*><b>([^<]+)",
):
m = re.search(pat, text, re.I)
if m:
return m.group(1).strip()
return None
def parse_report(data: Path, report: str) -> dict:
candidates = [data / f"{report}{ext}" for ext in (".htm", ".html")]
candidates += sorted(data.glob(f"**/{report}*.htm*"), key=lambda p: p.stat().st_mtime, reverse=True)
seen: set[Path] = set()
for path in candidates:
if path in seen or not path.exists():
continue
seen.add(path)
text = read_text(path)
pf = grab_metric(text, "profit_factor")
profit = grab_metric(text, "net_profit")
trades = grab_metric(text, "total_trades")
sharpe = grab_metric(text, "sharpe")
dd = grab_metric(text, "equity_dd")
ml = grab_metric(text, "margin_level")
if pf or profit or trades:
return {
"profit_factor": float(pf) if pf else None,
"net_profit": _num(profit),
"total_trades": int(float(trades)) if trades and trades[0].isdigit() else _int(trades),
"sharpe": float(sharpe) if sharpe else None,
"max_drawdown": dd,
"min_margin_level": ml,
"report": str(path),
"ready": True,
}
return {"ready": False}
def _num(s: str | None) -> float | None:
if not s:
return None
s = s.replace(" ", "").replace(",", "")
if s.endswith("%"):
return float(s[:-1])
return float(s)
def _int(s: str | None) -> int | None:
if not s:
return None
try:
return int(float(s.replace(" ", "").replace(",", "")))
except ValueError:
return None
KNOWN_TERMINALS = [
Path(r"C:\Program Files\MetaTrader 5\terminal64.exe"),
]
def _terminal_data_dirs() -> list[Path]:
root = Path.home() / "AppData" / "Roaming" / "MetaQuotes" / "Terminal"
if not root.is_dir():
return []
return [p for p in root.iterdir() if p.is_dir() and (p / "origin.txt").exists()]
def _read_origin(path: Path) -> str:
raw = path.read_bytes()
for enc in ("utf-8", "utf-16", "utf-16-le", "cp1252"):
try:
return raw.decode(enc).strip()
except UnicodeError:
continue
return raw.decode("utf-8", errors="ignore").strip()
def _load_dotenv() -> None:
env_path = Path(__file__).resolve().parents[3] / ".env"
if not env_path.is_file():
return
for line in env_path.read_text(encoding="utf-8", errors="ignore").splitlines():
line = line.strip()
if not line or line.startswith("#") or "=" not in line:
continue
key, val = line.split("=", 1)
key, val = key.strip(), val.strip().strip('"').strip("'")
if key and key not in os.environ:
os.environ[key] = val
_load_dotenv()
MT5_TERMINAL_DATA_ID = os.environ.get("MT5_TERMINAL_DATA_ID", "").strip()
# Back-compat alias for scripts that import PREFERRED_DATA_ID
PREFERRED_DATA_ID = MT5_TERMINAL_DATA_ID
def mt5_terminal_data_dir() -> Path | None:
"""MT5 data folder: MT5_TERMINAL_DATA_ID env, else first terminal with origin.txt."""
root = Path.home() / "AppData" / "Roaming" / "MetaQuotes" / "Terminal"
if MT5_TERMINAL_DATA_ID:
candidate = root / MT5_TERMINAL_DATA_ID
if (candidate / "origin.txt").is_file():
return candidate
dirs = _terminal_data_dirs()
return dirs[0] if dirs else None
def _resolve_terminal_exe() -> Path | None:
env_exe = os.environ.get("MT5_TERMINAL_EXE", "").strip()
if env_exe:
exe = Path(env_exe)
if exe.is_file():
return exe
data_dir = mt5_terminal_data_dir()
if data_dir:
origin = data_dir / "origin.txt"
if origin.is_file():
try:
exe = Path(_read_origin(origin))
if exe.is_file():
return exe
except OSError:
pass
for data_dir in _terminal_data_dirs():
origin = data_dir / "origin.txt"
try:
exe = Path(_read_origin(origin))
if exe.is_file():
return exe
except OSError:
continue
for exe in KNOWN_TERMINALS:
if exe.is_file():
return exe
return None
def mt5_context(*, retries: int = 6, wait_sec: float = 12.0) -> dict:
terminal_exe = _resolve_terminal_exe()
last_err = None
for attempt in range(retries):
if attempt:
time.sleep(wait_sec)
if attempt >= 1 and terminal_exe and terminal_exe.is_file():
subprocess.run(["taskkill", "/IM", "terminal64.exe", "/F"], capture_output=True)
time.sleep(3)
subprocess.Popen([str(terminal_exe)], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
time.sleep(20)
mt5.shutdown()
ok = mt5.initialize(path=str(terminal_exe)) if terminal_exe else mt5.initialize()
if not ok:
last_err = mt5.last_error()
continue
info = mt5.terminal_info()
acc = mt5.account_info()
ctx = {
"data": Path(info.data_path),
"mt5_path": Path(info.path),
"login": acc.login if acc else 0,
"server": acc.server if acc else "",
}
mt5.shutdown()
return ctx
raise RuntimeError(f"MT5 init failed after {retries} tries: {last_err}")
def deploy_united(data: Path, mt5_path: Path) -> Path:
dst = data / "MQL5" / "Experts" / "cluster-latest"
dst.mkdir(parents=True, exist_ok=True)
for name in ("main.mq5", "MagicNumberHelpers.mqh", "GapGuard.mqh"):
shutil.copy2(CLUSTER / name, dst / name)
strat_dst = dst / "Strategies"
strat_dst.mkdir(exist_ok=True)
for f in (CLUSTER / "Strategies").glob("*.mqh"):
shutil.copy2(f, strat_dst / f.name)
log = dst / "compile.log"
subprocess.run(
[str(mt5_path / "metaeditor64.exe"), f"/compile:{dst / 'main.mq5'}", f"/log:{log}"],
timeout=180,
capture_output=True,
)
time.sleep(3)
ex5 = dst / "main.ex5"
if not ex5.exists():
tail = log.read_text(encoding="utf-8", errors="ignore")[-2000:] if log.exists() else ""
raise RuntimeError(f"Compile failed: {log}\n{tail}")
pub = data / "MQL5" / "Experts" / "main.ex5"
shutil.copy2(ex5, pub)
return pub
def patch_set(base: Path, overrides: dict[str, str | bool | int | float]) -> str:
lines_out: list[str] = []
seen: set[str] = set()
for line in base.read_text(encoding="utf-8", errors="ignore").splitlines():
if not line.strip() or line.strip().startswith(";") or "=" not in line:
lines_out.append(line)
continue
name = line.split("=", 1)[0].strip()
if name in overrides:
val = overrides[name]
if isinstance(val, bool):
sval = "true" if val else "false"
else:
sval = str(val)
lines_out.append(f"{name}={sval}")
seen.add(name)
else:
lines_out.append(line)
for k, v in overrides.items():
if k not in seen:
sval = "true" if v is True else "false" if v is False else str(v)
lines_out.append(f"{k}={sval}")
return "\n".join(lines_out) + "\n"
def patch_set_for_lot_genetic(
base: Path,
overrides: dict[str, str | bool | int | float],
lot_key: str,
start: float,
step: float,
stop: float,
default: float | None = None,
) -> str:
"""Build .set with a single LOT_* genetic range; all other ||Y flags forced to N."""
body = patch_set(base, overrides)
val = default if default is not None else start
genetic_line = f"{lot_key}={val}||{start}||{step}||{stop}||Y"
lines_out: list[str] = []
seen_lot = False
for line in body.splitlines():
if not line.strip() or line.strip().startswith(";") or "=" not in line:
lines_out.append(line)
continue
name = line.split("=", 1)[0].strip()
if name == lot_key:
lines_out.append(genetic_line)
seen_lot = True
elif "||" in line:
parts = line.split("||")
if len(parts) >= 5:
parts[4] = "N"
lines_out.append("||".join(parts))
else:
lines_out.append(line)
else:
lines_out.append(line)
if not seen_lot:
lines_out.append(genetic_line)
return "\n".join(lines_out) + "\n"
def parse_optimization_xml(data: Path, report: str, lot_key: str) -> dict:
candidates = [data / f"{report}.xml", data / f"{report}.opt"]
candidates += sorted(data.glob(f"**/{report}*.xml"), key=lambda p: p.stat().st_mtime, reverse=True)
seen: set[Path] = set()
xml_path: Path | None = None
for p in candidates:
if p in seen or not p.exists() or p.suffix.lower() != ".xml":
continue
seen.add(p)
xml_path = p
break
if xml_path is None:
return {"ready": False, "error": "xml_not_found"}
text = read_text(xml_path)
header = re.search(r"<Row>.*?Pass</Data>.*?</Row>", text, re.S)
if not header:
return {"ready": False, "error": "xml_header_missing", "report": str(xml_path)}
cols = re.findall(r'<Data ss:Type="String">([^<]+)</Data>', header.group(0))
rows: list[dict[str, str]] = []
for row_xml in re.findall(r"<Row>(.*?)</Row>", text, re.S)[1:]:
cells = re.findall(r'<Data ss:Type="(?:Number|String)">([^<]+)</Data>', row_xml)
if len(cells) >= len(cols):
rows.append(dict(zip(cols, cells)))
if not rows:
return {"ready": False, "error": "xml_no_rows", "report": str(xml_path)}
best_row: dict[str, str] | None = None
best_score = -1e18
for row in rows:
try:
profit = float(row.get("Profit", 0))
pf = float(row.get("Profit Factor", 0))
sharpe = float(row.get("Sharpe Ratio", row.get("Sharpe", 0)))
trades = int(float(row.get("Trades", 0)))
except (ValueError, TypeError):
continue
if trades < 20 or pf < 1.0 or profit <= 0:
score = -1e10 + profit
else:
score = sharpe * 2000.0 + profit / 500.0 + pf * 50.0
if score > best_score:
best_score = score
best_row = row
if best_row is None:
best_row = max(rows, key=lambda r: float(r.get("Profit", 0)))
lot_raw = best_row.get(lot_key)
if lot_raw is None:
for k, v in best_row.items():
if k.replace(" ", "") == lot_key or lot_key in k:
lot_raw = v
break
best_lot = float(lot_raw) if lot_raw is not None else None
return {
"ready": True,
"report": str(xml_path),
"best_lot": best_lot,
"best_row": best_row,
"best_score": best_score,
"passes": len(rows),
"profit": float(best_row.get("Profit", 0)),
"profit_factor": float(best_row.get("Profit Factor", 0)),
"sharpe": float(best_row.get("Sharpe Ratio", best_row.get("Sharpe", 0))),
"trades": int(float(best_row.get("Trades", 0))),
}
def run_genetic_lot_optimize(
data: Path,
mt5_path: Path,
login: int,
server: str,
set_body: str,
set_name: str,
report: str,
lot_key: str,
*,
test_symbol: str | None = None,
optimization: int = 2,
timeout_sec: int = 7200,
) -> dict:
write_tester_set(data, set_name, set_body)
ini = data / f"{report}.ini"
ini.write_text(
build_ini(set_name, report, login, server, symbol=test_symbol, optimization=optimization),
encoding="utf-8",
)
for ext in (".htm", ".html", ".xml"):
p = data / f"{report}{ext}"
if p.exists():
p.unlink(missing_ok=True)
subprocess.run(["taskkill", "/IM", "terminal64.exe", "/F"], capture_output=True)
subprocess.run(["taskkill", "/IM", "metatester64.exe", "/F"], capture_output=True)
time.sleep(6)
t0 = time.time()
subprocess.run([str(mt5_path / "terminal64.exe"), f"/config:{ini}"], timeout=timeout_sec)
elapsed = round(time.time() - t0, 1)
opt = parse_optimization_xml(data, report, lot_key)
opt["elapsed_sec"] = elapsed
if not opt.get("ready"):
metrics = parse_report(data, report)
opt.update(metrics)
return opt
def write_tester_set(data: Path, set_name: str, body: str) -> Path:
profiles = data / "MQL5" / "Profiles" / "Tester"
profiles.mkdir(parents=True, exist_ok=True)
path = profiles / set_name
path.write_text(body, encoding="utf-8")
return path
def build_ini(
set_name: str,
report: str,
login: int,
server: str,
*,
symbol: str | None = None,
optimization: int = 0,
) -> str:
sym = symbol or TEST_SYMBOL
return f"""[Common]
Login={login}
Server={server}
[Tester]
Expert=main.ex5
ExpertParameters={set_name}
Symbol={sym}
Period={TEST_PERIOD}
Optimization={optimization}
Model=1
Dates=1
FromDate={FROM_DATE}
ToDate={TO_DATE}
ForwardMode=0
Deposit={DEPOSIT}
Currency=USD
Leverage={LEVERAGE}
ExecutionMode=0
Report={report}
ReplaceReport=1
ShutdownTerminal=1
Visual=0
"""
def run_backtest(
data: Path,
mt5_path: Path,
login: int,
server: str,
set_body: str,
set_name: str,
report: str,
*,
test_symbol: str | None = None,
timeout_sec: int = 1800,
) -> dict:
write_tester_set(data, set_name, set_body)
ini = data / f"{report}.ini"
ini.write_text(build_ini(set_name, report, login, server, symbol=test_symbol), encoding="utf-8")
for ext in (".htm", ".html"):
p = data / f"{report}{ext}"
if p.exists():
p.unlink(missing_ok=True)
subprocess.run(["taskkill", "/IM", "terminal64.exe", "/F"], capture_output=True)
subprocess.run(["taskkill", "/IM", "metatester64.exe", "/F"], capture_output=True)
time.sleep(6)
t0 = time.time()
subprocess.run([str(mt5_path / "terminal64.exe"), f"/config:{ini}"], timeout=timeout_sec)
metrics = parse_report(data, report)
metrics["elapsed_sec"] = round(time.time() - t0, 1)
return metrics
backtesting/MT5/cluster_audit/united_registry.py
+117
View File
@@ -0,0 +1,117 @@
"""United EA (main.mq5) strategy registry — defaults from 123.set + audit ranges."""
from __future__ import annotations
import MetaTrader5 as mt5
TF = {
"M10": mt5.TIMEFRAME_M10,
"M15": mt5.TIMEFRAME_M15,
"M20": mt5.TIMEFRAME_M20,
"M30": mt5.TIMEFRAME_M30,
"H1": mt5.TIMEFRAME_H1,
"H4": mt5.TIMEFRAME_H4,
}
# Strategies enabled in 123.set, ordered for sequential audit.
UNITED_STRATEGIES: list[dict] = [
{"id": "united_darvas", "engine": "darvas", "symbol": "XAUUSD", "tf": "M15", "lot": 0.05,
"enable_key": "EnableDarvasBox", "lot_key": "LOT_DB_DarvasBox",
"defaults": {"box_period": 24, "box_deviation": 90000, "ma_period": 30,
"trend_threshold": 1.2, "volume_threshold": 0,
"stop_loss_pts": 300, "take_profit_pts": 950},
"opt": {"box_period": (12, 48, 4), "box_deviation": (40000, 150000, 5000),
"trend_threshold": (0.3, 5.0, 0.3), "stop_loss_pts": (250, 900, 50),
"take_profit_pts": (350, 1200, 50), "ma_period": (30, 120, 15)}},
{"id": "united_rsi_cross", "engine": "rsi_crossover", "symbol": "XAUUSD", "tf": "M15", "lot": 0.06,
"enable_key": "EnableRSICrossOverReversal", "lot_key": "LOT_RC_RSICrossOver",
"defaults": {"rsi_period": 19, "overbought_level": 85, "oversold_level": 25, "ema_period": 140,
"ema_slope_threshold": 105, "ema_distance_threshold": 350, "exit_buy_rsi": 86,
"exit_sell_rsi": 10, "trailing_stop_pts": 295, "cooldown_seconds": 120,
"use_trend_strength_filter": True, "entry_rsi_buy_spread": 0, "entry_rsi_sell_spread": 0,
"tuesday": True, "wednesday": True, "thursday": True,
"trading_hour_one_begin": 0, "trading_hour_one_end": 22,
"trading_hour_two_begin": 6, "trading_hour_two_end": 19},
"opt": {"overbought_level": (70, 92, 3), "oversold_level": (15, 40, 3),
"ema_distance_threshold": (200, 600, 50), "cooldown_seconds": (60, 300, 30)}},
{"id": "united_rsi_scalp_appl", "engine": "rsi_scalp", "symbol": "AAPL", "tf": "M15", "lot": 15.0,
"enable_key": "EnableRSIScalpingAPPL", "lot_key": "LOT_RS_APPL",
"defaults": {"rsi_period": 8, "rsi_overbought": 64, "rsi_oversold": 30,
"rsi_target_buy": 67, "rsi_target_sell": 2, "bars_to_wait": 5,
"use_trailing": True, "trail_distance_pts": 70, "trail_activation_pts": 39, "tf": "M15",
"skip_short_hour_after": 17, "use_reversal_escape": False,
"use_rsi_against_exit": False, "max_adverse_atr": 2.5, "min_ob_depth": 0},
"opt": {"rsi_period": (6, 14, 1), "rsi_overbought": (60, 75, 2), "rsi_oversold": (22, 35, 2),
"rsi_target_buy": (60, 80, 3), "bars_to_wait": (5, 12, 1),
"trail_distance_pts": (50, 120, 10), "skip_short_hour_after": (15, 20, 1)}},
{"id": "united_rsi_scalp_adbe", "engine": "rsi_scalp", "symbol": "ADBE", "tf": "H1", "lot": 40.0,
"enable_key": "EnableRSIScalpingADBE", "lot_key": "LOT_RS_ADBE",
"defaults": {"rsi_period": 15, "rsi_overbought": 16, "rsi_oversold": 42,
"rsi_target_buy": 67, "rsi_target_sell": 62, "bars_to_wait": 8,
"use_trailing": True, "trail_distance_pts": 425, "trail_activation_pts": 18.5, "tf": "H1"},
"opt": {"rsi_period": (6, 21, 1), "rsi_overbought": (50, 90, 3), "rsi_oversold": (10, 70, 3),
"bars_to_wait": (1, 12, 1), "trail_distance_pts": (50, 500, 25)}},
{"id": "united_rsi_scalp_btc", "engine": "rsi_scalp", "symbol": "BTCUSD", "tf": "H1", "lot": 0.03,
"enable_key": "EnableRSIScalpingBTCUSD", "lot_key": "LOT_RS_BTCUSD",
"defaults": {"rsi_period": 14, "rsi_overbought": 90, "rsi_oversold": 73,
"rsi_target_buy": 88, "rsi_target_sell": 48, "bars_to_wait": 6,
"use_trailing": True, "trail_distance_pts": 120, "trail_activation_pts": 0, "tf": "H1"},
"opt": {"rsi_period": (6, 21, 1), "rsi_overbought": (50, 95, 3), "rsi_oversold": (20, 80, 3),
"bars_to_wait": (1, 8, 1), "trail_distance_pts": (40, 300, 20)}},
{"id": "united_rsi_scalp_nvda", "engine": "rsi_scalp", "symbol": "NVDA", "tf": "M15", "lot": 50.0,
"enable_key": "EnableRSIScalpingNVDA", "lot_key": "LOT_RS_NVDA",
"defaults": {"rsi_period": 8, "rsi_overbought": 36, "rsi_oversold": 38,
"rsi_target_buy": 90, "rsi_target_sell": 70, "bars_to_wait": 5,
"use_trailing": True, "trail_distance_pts": 375, "trail_activation_pts": 75, "tf": "M15"},
"opt": {"rsi_period": (6, 21, 1), "bars_to_wait": (1, 8, 1), "trail_distance_pts": (50, 500, 25)}},
{"id": "united_rsi_scalp_tsla", "engine": "rsi_scalp", "symbol": "TSLA", "tf": "H1", "lot": 5.0,
"enable_key": "EnableRSIScalpingTSLA", "lot_key": "LOT_RS_TSLA",
"defaults": {"rsi_period": 14, "rsi_overbought": 54, "rsi_oversold": 73,
"rsi_target_buy": 87, "rsi_target_sell": 33, "bars_to_wait": 1,
"use_trailing": True, "trail_distance_pts": 900, "trail_activation_pts": 950, "tf": "H1"},
"opt": {"rsi_period": (6, 21, 1), "bars_to_wait": (1, 6, 1), "trail_distance_pts": (100, 1000, 50)}},
{"id": "united_rsi_scalp_xau", "engine": "rsi_scalp", "symbol": "XAUUSD", "tf": "H1", "lot": 0.02,
"enable_key": "EnableRSIScalpingXAUUSD", "lot_key": "LOT_RS_XAUUSD",
"defaults": {"rsi_period": 14, "rsi_overbought": 71, "rsi_oversold": 57,
"rsi_target_buy": 80, "rsi_target_sell": 57, "bars_to_wait": 1,
"use_trailing": True, "trail_distance_pts": 71, "trail_activation_pts": 41, "tf": "H1"},
"opt": {"rsi_period": (6, 21, 1), "rsi_overbought": (55, 85, 3), "rsi_oversold": (40, 75, 3),
"bars_to_wait": (1, 4, 1), "trail_distance_pts": (20, 150, 10)}},
{"id": "united_rsi_asian_eur", "engine": "rsi_asian", "symbol": "EURUSD", "tf": "M15", "lot": 0.04,
"enable_key": "EnableRSIReversalAsianEURUSD", "lot_key": "LOT_RRA_EURUSD",
"defaults": {"rsi_period": 28, "overbought_level": 60, "oversold_level": 8,
"asian_session_start": 0, "asian_session_end": 8, "use_rsi_exit": True, "rsi_exit_level": 55},
"opt": {"overbought_level": (50, 80, 5), "oversold_level": (5, 40, 5)}},
{"id": "united_rsi_asian_aud", "engine": "rsi_asian", "symbol": "AUDUSD", "tf": "M15", "lot": 0.03,
"enable_key": "EnableRSIReversalAsianAUDUSD", "lot_key": "LOT_RRA_AUDUSD",
"defaults": {"rsi_period": 28, "overbought_level": 68, "oversold_level": 30,
"asian_session_start": 0, "asian_session_end": 8, "use_rsi_exit": True, "rsi_exit_level": 48,
"close_outside_session": True},
"opt": {"overbought_level": (55, 80, 5), "oversold_level": (15, 45, 5)}},
{"id": "united_st_xau", "engine": "simple_trendline", "symbol": "XAUUSD", "tf": "H1", "lot": 0.03,
"enable_key": "EnableSimpleTrendlineXAUUSD", "lot_key": "LOT_ST_XAUUSD",
"defaults": {"signal_tf": "H1", "higher_tf": "M10", "ma_period": 65, "ma_method": "ema",
"htf_bars_to_scan": 500, "touch_tolerance_pts": 220, "break_buffer_pts": 110},
"opt": {"ma_period": (40, 120, 10), "touch_tolerance_pts": (100, 350, 25),
"break_buffer_pts": (50, 200, 15)}},
{"id": "united_st_ger40", "engine": "simple_trendline", "symbol": "GER40", "tf": "M15", "lot": 0.04,
"enable_key": "EnableSimpleTrendlineGER40", "lot_key": "LOT_ST_GER40",
"defaults": {"signal_tf": "M15", "higher_tf": "M15", "ma_period": 65, "ma_method": "lwma",
"htf_bars_to_scan": 1200, "touch_tolerance_pts": 100, "break_buffer_pts": 80},
"opt": {"ma_period": (40, 120, 10), "touch_tolerance_pts": (50, 200, 15)}},
{"id": "united_rsi_secret", "engine": "rsi_secret", "symbol": "XAUUSD", "tf": "M30", "lot": 0.07,
"enable_key": "EnableRSISecretSauce", "lot_key": "LOT_RSS_SecretSauce",
"defaults": {"rsi_period": 16, "rsi_overbought": 72.5, "rsi_oversold": 32.5,
"stop_loss_atr": 2.75, "take_profit_atr": 5.0, "min_bars_between_trades": 7},
"opt": {"rsi_overbought": (60, 80, 2), "rsi_oversold": (25, 45, 2), "stop_loss_atr": (1.5, 4, 0.5)}},
{"id": "united_usdjpy", "engine": "usdjpy_buster", "symbol": "USDJPY", "tf": "M20", "lot": 0.09,
"enable_key": "EnableUSDJPYBuster", "lot_key": "LOT_UB_USDJPY",
"defaults": {"range_start_hour": 3, "range_end_hour": 6, "close_hour": 18,
"min_range_pts": 15, "order_buffer_pts": 4.75, "first_trade_only": False,
"allow_long": True, "allow_short": True, "use_take_profit": False},
"opt": {"min_range_pts": (8, 40, 4), "order_buffer_pts": (2, 12, 1)}},
]
PERIODS = {
"2021-2026": ("2021-01-01", "2026-06-01"),
}
backtesting/MT5/indicator_utils.py
+32 -9
View File
@@ -10,13 +10,14 @@ import pandas as pd
def calculate_rsi(prices: pd.Series, period: int = 14) -> pd.Series:
"""Calculate RSI indicator."""
"""Calculate RSI with Wilder smoothing (matches MT5 iRSI)."""
delta = prices.diff()
gain = (delta.where(delta > 0, 0)).rolling(window=period).mean()
loss = (-delta.where(delta < 0, 0)).rolling(window=period).mean()
rs = gain / loss
rsi = 100 - (100 / (1 + rs))
return rsi
gain = delta.clip(lower=0)
loss = (-delta).clip(lower=0)
avg_gain = gain.ewm(alpha=1 / period, adjust=False).mean()
avg_loss = loss.ewm(alpha=1 / period, adjust=False).mean()
rs = avg_gain / avg_loss.replace(0, np.nan)
return 100 - (100 / (1 + rs))
def calculate_ema(prices: pd.Series, period: int = 50) -> pd.Series:
@@ -30,13 +31,35 @@ def calculate_sma(prices: pd.Series, period: int = 50) -> pd.Series:
def calculate_atr(df: pd.DataFrame, period: int = 14) -> pd.Series:
"""Calculate ATR indicator."""
"""Calculate ATR with Wilder smoothing (matches MT5 iATR)."""
high_low = df['high'] - df['low']
high_close = np.abs(df['high'] - df['close'].shift())
low_close = np.abs(df['low'] - df['close'].shift())
tr = pd.concat([high_low, high_close, low_close], axis=1).max(axis=1)
atr = tr.rolling(window=period).mean()
return atr
return tr.ewm(alpha=1 / period, adjust=False).mean()
def calculate_adx(df: pd.DataFrame, period: int = 14) -> pd.Series:
"""Calculate ADX indicator (Wilder smoothing)."""
return calculate_dmi(df, period)["adx"]
def calculate_dmi(df: pd.DataFrame, period: int = 14) -> pd.DataFrame:
"""Calculate +DI, -DI, and ADX."""
high = df["high"]
low = df["low"]
close = df["close"]
up = high.diff()
down = -low.diff()
plus_dm = up.where((up > down) & (up > 0), 0.0)
minus_dm = down.where((down > up) & (down > 0), 0.0)
tr = pd.concat([high - low, (high - close.shift()).abs(), (low - close.shift()).abs()], axis=1).max(axis=1)
atr = tr.ewm(alpha=1 / period, adjust=False).mean()
plus_di = 100 * (plus_dm.ewm(alpha=1 / period, adjust=False).mean() / atr.replace(0, np.nan))
minus_di = 100 * (minus_dm.ewm(alpha=1 / period, adjust=False).mean() / atr.replace(0, np.nan))
dx = 100 * (plus_di - minus_di).abs() / (plus_di + minus_di).replace(0, np.nan)
adx = dx.ewm(alpha=1 / period, adjust=False).mean()
return pd.DataFrame({"plus_di": plus_di, "minus_di": minus_di, "adx": adx})
def calculate_macd(prices: pd.Series, fast: int = 12, slow: int = 26, signal: int = 9) -> pd.DataFrame:
backtesting/MT5/rsi_scalping_backtest.py
+309
View File
@@ -0,0 +1,309 @@
"""
Bar-based RSI scalping backtest — conservative fills, costs, no same-bar RSI lookahead.
Mirrors RsiScalpingRobot.mqh with:
- entries on bar open after RSI cross on prior closed bars
- exits evaluated on prior closed bar RSI
- trailing updated on bar close; stop checked against bar range
"""
from __future__ import annotations
from dataclasses import dataclass, field
from typing import Any
import MetaTrader5 as mt5
import numpy as np
import pandas as pd
from indicator_utils import calculate_rsi
@dataclass
class RsiScalpParams:
rsi_period: int = 14
rsi_overbought: float = 71.0
rsi_oversold: float = 57.0
rsi_target_buy: float = 80.0
rsi_target_sell: float = 57.0
bars_to_wait: int = 1
use_trailing: bool = True
trail_distance_pts: float = 71.0
trail_activation_pts: float = 41.0
lot_size: float = 0.1
@classmethod
def from_dict(cls, d: dict[str, Any]) -> "RsiScalpParams":
return cls(**{k: d[k] for k in cls.__dataclass_fields__ if k in d})
@dataclass
class CostModel:
spread_points: float = 0.0
slippage_points: float = 3.0
commission_per_lot: float = 0.0
@classmethod
def from_symbol(cls, symbol: str, slippage_points: float = 3.0, commission_per_lot: float = 0.0) -> "CostModel":
info = mt5.symbol_info(symbol)
spread = float(info.spread) if info else 0.0
return cls(spread_points=spread, slippage_points=slippage_points, commission_per_lot=commission_per_lot)
@dataclass
class BacktestResult:
net_profit: float
total_trades: int
win_rate: float
profit_factor: float
max_drawdown_pct: float
total_costs: float
score: float
params: RsiScalpParams
gross_profit: float = 0.0
gross_loss: float = 0.0
def _calc_profit(symbol: str, order_type: int, volume: float, open_price: float, close_price: float) -> float:
p = mt5.order_calc_profit(order_type, symbol, volume, open_price, close_price)
return float(p) if p is not None else 0.0
def _half_spread_price(point: float, spread_points: float) -> float:
return (spread_points * point) / 2.0
def _fill_buy(open_price: float, point: float, costs: CostModel, entry: bool) -> float:
slip = costs.slippage_points * point
hs = _half_spread_price(point, costs.spread_points)
return open_price + hs + slip if entry else open_price - hs - slip
def _fill_sell(open_price: float, point: float, costs: CostModel, entry: bool) -> float:
slip = costs.slippage_points * point
hs = _half_spread_price(point, costs.spread_points)
return open_price - hs - slip if entry else open_price + hs + slip
def backtest_rsi_scalping(
df: pd.DataFrame,
symbol: str,
params: RsiScalpParams,
initial_balance: float = 10_000.0,
point: float | None = None,
costs: CostModel | None = None,
) -> BacktestResult:
info = mt5.symbol_info(symbol)
if point is None:
point = float(info.point) if info else 0.01
if costs is None:
costs = CostModel.from_symbol(symbol)
# RSI on close; decisions use index i-1 (last fully closed bar at bar i open)
rsi_full = calculate_rsi(df["close"], params.rsi_period).to_numpy()
times = df.index.to_numpy()
opens = df["open"].to_numpy()
highs = df["high"].to_numpy()
lows = df["low"].to_numpy()
closes = df["close"].to_numpy()
balance = initial_balance
peak = initial_balance
max_dd = 0.0
total_costs = 0.0
position: dict[str, Any] | None = None
rsi_against = False
bars_against = 0
gross_profit = 0.0
gross_loss = 0.0
wins = 0
losses = 0
trades = 0
trail_dist = params.trail_distance_pts * point
trail_act = (params.trail_activation_pts if params.trail_activation_pts > 0 else params.trail_distance_pts) * point
def _update_dd() -> None:
nonlocal peak, max_dd
if balance > peak:
peak = balance
dd = (peak - balance) / peak if peak > 0 else 0.0
if dd > max_dd:
max_dd = dd
def close_at(exit_mid: float) -> None:
nonlocal balance, gross_profit, gross_loss, wins, losses, trades, position, total_costs
if position is None:
return
order_type = mt5.ORDER_TYPE_BUY if position["type"] == "BUY" else mt5.ORDER_TYPE_SELL
if position["type"] == "BUY":
exit_price = _fill_buy(exit_mid, point, costs, entry=False)
else:
exit_price = _fill_sell(exit_mid, point, costs, entry=False)
commission = costs.commission_per_lot * position["volume"] * 2.0
profit = _calc_profit(symbol, order_type, position["volume"], position["open_price"], exit_price)
profit -= commission
total_costs += commission + (costs.slippage_points * point * position["volume"] * 100000 * 0.0)
balance += profit
trades += 1
if profit >= 0:
wins += 1
gross_profit += profit
else:
losses += 1
gross_loss += abs(profit)
_update_dd()
position = None
def apply_trailing(bar_close: float, bar_high: float, bar_low: float) -> None:
if position is None or not params.use_trailing or trail_dist <= 0:
return
if position["type"] == "BUY":
bid = bar_close
if bid - position["open_price"] <= trail_act:
return
new_sl = bid - trail_dist
if new_sl > position.get("sl", 0.0):
position["sl"] = new_sl
if position.get("sl") and bar_low <= position["sl"]:
close_at(position["sl"])
else:
ask = bar_close
if position["open_price"] - ask <= trail_act:
return
new_sl = ask + trail_dist
if position.get("sl", 0.0) == 0.0 or new_sl < position["sl"]:
position["sl"] = new_sl
if position.get("sl") and bar_high >= position["sl"]:
close_at(position["sl"])
start = max(params.rsi_period + 3, 3)
for i in range(start, len(df)):
# closed-bar RSI (no lookahead): signal bar is i-1
rsi_sig = rsi_full[i - 1]
rsi_prev = rsi_full[i - 2]
rsi_two = rsi_full[i - 3]
if np.isnan(rsi_sig) or np.isnan(rsi_prev) or np.isnan(rsi_two):
continue
if position is not None:
apply_trailing(closes[i], highs[i], lows[i])
if position is None:
rsi_against = False
bars_against = 0
continue
if position["type"] == "BUY":
if rsi_sig < params.rsi_oversold:
if not rsi_against:
rsi_against = True
bars_against = 1
else:
bars_against += 1
if bars_against >= params.bars_to_wait:
close_at(opens[i])
else:
if rsi_against:
rsi_against = False
bars_against = 0
if rsi_sig >= params.rsi_target_buy:
close_at(opens[i])
else:
if rsi_sig > params.rsi_overbought:
if not rsi_against:
rsi_against = True
bars_against = 1
else:
bars_against += 1
if bars_against >= params.bars_to_wait:
close_at(opens[i])
else:
if rsi_against:
rsi_against = False
bars_against = 0
if rsi_sig <= params.rsi_target_sell:
close_at(opens[i])
if position is not None:
continue
# entry at bar open[i] from RSI cross on bars i-2 / i-3
if rsi_two <= params.rsi_oversold and rsi_prev > params.rsi_oversold:
entry = _fill_buy(opens[i], point, costs, entry=True)
position = {"type": "BUY", "volume": params.lot_size, "open_price": entry, "open_time": times[i], "sl": 0.0}
rsi_against = False
bars_against = 0
elif rsi_two >= params.rsi_overbought and rsi_prev < params.rsi_overbought:
entry = _fill_sell(opens[i], point, costs, entry=True)
position = {"type": "SELL", "volume": params.lot_size, "open_price": entry, "open_time": times[i], "sl": 0.0}
rsi_against = False
bars_against = 0
if position is not None:
close_at(closes[-1])
net_profit = balance - initial_balance
win_rate = (wins / trades * 100.0) if trades else 0.0
pf = (gross_profit / gross_loss) if gross_loss > 0 else (999.0 if gross_profit > 0 else 0.0)
if trades < 20:
score = net_profit - 10_000.0
else:
score = net_profit * (1.0 - min(max_dd, 0.5))
return BacktestResult(
net_profit=net_profit,
total_trades=trades,
win_rate=win_rate,
profit_factor=pf,
max_drawdown_pct=max_dd * 100.0,
total_costs=total_costs,
score=score,
params=params,
gross_profit=gross_profit,
gross_loss=gross_loss,
)
def resolve_symbol(requested: str) -> str:
key = requested.split("|")[0].strip()
if not key:
return requested
if mt5.symbol_info(key) is not None:
mt5.symbol_select(key, True)
return key
for suffix in (".NAS", ".NYSE", ".NYS", ".US"):
cand = key + suffix
if mt5.symbol_info(cand) is not None:
mt5.symbol_select(cand, True)
return cand
for sym in mt5.symbols_get() or []:
name = sym.name
if name.startswith(key + "."):
mt5.symbol_select(name, True)
return name
return key
def load_rates(symbol: str, timeframe: int, start, end) -> pd.DataFrame:
symbol = resolve_symbol(symbol)
if not mt5.symbol_select(symbol, True):
raise RuntimeError(f"Cannot select {symbol}: {mt5.last_error()}")
rates = mt5.copy_rates_range(symbol, timeframe, start, end)
if rates is None or len(rates) == 0:
raise RuntimeError(f"No rates for {symbol}: {mt5.last_error()}")
out = pd.DataFrame(rates)
out["time"] = pd.to_datetime(out["time"], unit="s")
out.set_index("time", inplace=True)
return out
def split_walk_forward(df: pd.DataFrame, train_ratio: float = 0.6) -> tuple[pd.DataFrame, pd.DataFrame]:
cut = int(len(df) * train_ratio)
if cut < 100 or len(df) - cut < 100:
raise ValueError("Not enough bars for walk-forward split")
return df.iloc[:cut].copy(), df.iloc[cut:].copy()
backtesting/MT5/run_optimize.py
+233
View File
@@ -0,0 +1,233 @@
"""
Walk-forward random-search optimizer for RSI scalping.
Optimizes on in-sample (train), ranks by out-of-sample (validation) score.
"""
from __future__ import annotations
import argparse
import json
import random
from datetime import datetime, timedelta
from pathlib import Path
from typing import Any
import MetaTrader5 as mt5
import pandas as pd
from rsi_scalping_backtest import (
CostModel,
RsiScalpParams,
backtest_rsi_scalping,
load_rates,
split_walk_forward,
)
from set_parser import SetParam, parse_set_file
TF_MAP = {
"M1": mt5.TIMEFRAME_M1,
"M5": mt5.TIMEFRAME_M5,
"M10": mt5.TIMEFRAME_M10,
"M15": mt5.TIMEFRAME_M15,
"M30": mt5.TIMEFRAME_M30,
"H1": mt5.TIMEFRAME_H1,
"H4": mt5.TIMEFRAME_H4,
"D1": mt5.TIMEFRAME_D1,
}
SET_TO_PARAM = {
"RSI_Period": "rsi_period",
"RSI_Overbought": "rsi_overbought",
"RSI_Oversold": "rsi_oversold",
"RSI_Target_Buy": "rsi_target_buy",
"RSI_Target_Sell": "rsi_target_sell",
"BarsToWait": "bars_to_wait",
"UseTrailingStop": "use_trailing",
"TrailingStopDistancePoints": "trail_distance_pts",
"TrailingActivationPoints": "trail_activation_pts",
}
def _sample_value(p: SetParam, rng: random.Random) -> Any:
if not p.optimize:
return p.value
if isinstance(p.start, bool):
return rng.choice([p.start, p.stop])
if isinstance(p.start, int) and isinstance(p.stop, int):
step = int(p.step) if int(p.step) != 0 else 1
vals = list(range(int(p.start), int(p.stop) + 1, step))
return rng.choice(vals) if vals else p.value
step = float(p.step) if float(p.step) != 0 else 1.0
start, stop = float(p.start), float(p.stop)
n = int((stop - start) / step) + 1
idx = rng.randint(0, max(n - 1, 0))
return round(start + idx * step, 4)
def sample_params(set_params: dict[str, SetParam], rng: random.Random, defaults: dict, fixed_lot: float) -> RsiScalpParams:
raw = dict(defaults)
for set_name, field in SET_TO_PARAM.items():
if set_name in set_params:
raw[field] = _sample_value(set_params[set_name], rng)
raw["lot_size"] = fixed_lot
return RsiScalpParams.from_dict(raw)
def _result_dict(r, label: str) -> dict:
return {
"label": label,
"net_profit": r.net_profit,
"total_trades": r.total_trades,
"win_rate": r.win_rate,
"profit_factor": r.profit_factor,
"max_drawdown_pct": r.max_drawdown_pct,
"total_costs": r.total_costs,
"score": r.score,
"params": r.params.__dict__,
}
def main() -> None:
parser = argparse.ArgumentParser(description="Walk-forward RSI scalping optimizer")
parser.add_argument("--symbol", default="XAUUSD")
parser.add_argument("--timeframe", default="H1", choices=TF_MAP.keys())
parser.add_argument("--set", required=True)
parser.add_argument("--trials", type=int, default=800)
parser.add_argument("--days", type=int, default=730)
parser.add_argument("--balance", type=float, default=10000.0)
parser.add_argument("--lot", type=float, default=0.1)
parser.add_argument("--train-ratio", type=float, default=0.6)
parser.add_argument("--slippage", type=float, default=3.0)
parser.add_argument("--commission", type=float, default=0.0)
parser.add_argument("--seed", type=int, default=42)
parser.add_argument("--out", default="optimization_results")
args = parser.parse_args()
if not mt5.initialize():
raise SystemExit(f"MT5 init failed: {mt5.last_error()}")
try:
end = datetime.now()
start = end - timedelta(days=args.days)
tf = TF_MAP[args.timeframe]
df_all = load_rates(args.symbol, tf, start, end)
train_df, test_df = split_walk_forward(df_all, args.train_ratio)
costs = CostModel.from_symbol(args.symbol, slippage_points=args.slippage, commission_per_lot=args.commission)
info = mt5.symbol_info(args.symbol)
spread = info.spread if info else 0
print(f"Symbol {args.symbol} spread={spread} pts slippage={args.slippage} commission/lot={args.commission}")
print(f"All: {len(df_all)} bars train: {len(train_df)} ({train_df.index[0]} -> {train_df.index[-1]})")
print(f"Test: {len(test_df)} bars ({test_df.index[0]} -> {test_df.index[-1]})")
set_params = parse_set_file(args.set)
defaults = {
"rsi_period": 14,
"rsi_overbought": 71.0,
"rsi_oversold": 57.0,
"rsi_target_buy": 80.0,
"rsi_target_sell": 57.0,
"bars_to_wait": 1,
"use_trailing": True,
"trail_distance_pts": 71.0,
"trail_activation_pts": 41.0,
}
baseline_params = RsiScalpParams.from_dict({**defaults, "lot_size": args.lot})
baseline_train = backtest_rsi_scalping(train_df, args.symbol, baseline_params, args.balance, costs=costs)
baseline_test = backtest_rsi_scalping(test_df, args.symbol, baseline_params, args.balance, costs=costs)
baseline_full = backtest_rsi_scalping(df_all, args.symbol, baseline_params, args.balance, costs=costs)
print("\n--- BASELINE (current SuperEA XAUUSD trailing defaults) ---")
print(f" train net=${baseline_train.net_profit:.2f} trades={baseline_train.total_trades} dd={baseline_train.max_drawdown_pct:.1f}%")
print(f" test net=${baseline_test.net_profit:.2f} trades={baseline_test.total_trades} dd={baseline_test.max_drawdown_pct:.1f}%")
print(f" full net=${baseline_full.net_profit:.2f} trades={baseline_full.total_trades} dd={baseline_full.max_drawdown_pct:.1f}%")
rng = random.Random(args.seed)
rows = []
best_oos = None
best_oos_score = float("-inf")
for n in range(1, args.trials + 1):
params = sample_params(set_params, rng, defaults, args.lot)
train_r = backtest_rsi_scalping(train_df, args.symbol, params, args.balance, costs=costs)
test_r = backtest_rsi_scalping(test_df, args.symbol, params, args.balance, costs=costs)
full_r = backtest_rsi_scalping(df_all, args.symbol, params, args.balance, costs=costs)
row = {
"trial": n,
"oos_score": test_r.score,
"train_net": train_r.net_profit,
"test_net": test_r.net_profit,
"full_net": full_r.net_profit,
"train_trades": train_r.total_trades,
"test_trades": test_r.total_trades,
"test_pf": test_r.profit_factor,
"test_dd_pct": test_r.max_drawdown_pct,
"test_win_rate": test_r.win_rate,
**params.__dict__,
}
rows.append(row)
if test_r.total_trades >= 15 and test_r.score > best_oos_score:
best_oos_score = test_r.score
best_oos = (params, train_r, test_r, full_r)
if n % 200 == 0 and best_oos:
_, _, br_test, _ = best_oos
print(f" trial {n}/{args.trials} best OOS net=${br_test.net_profit:.2f} score={best_oos_score:.2f}")
if best_oos is None:
raise SystemExit("No valid OOS candidate (need >=15 test trades)")
best_params, best_train, best_test, best_full = best_oos
out_dir = Path(args.out)
out_dir.mkdir(parents=True, exist_ok=True)
results_df = pd.DataFrame(rows).sort_values("oos_score", ascending=False)
tag = f"{args.symbol}_{args.timeframe}_v2"
csv_path = out_dir / f"{tag}_rsi_scalp_opt.csv"
results_df.to_csv(csv_path, index=False)
report = {
"version": "v2-conservative-walkforward",
"symbol": args.symbol,
"timeframe": args.timeframe,
"costs": {"spread_pts": spread, "slippage_pts": args.slippage, "commission_per_lot": args.commission},
"bars": {"all": len(df_all), "train": len(train_df), "test": len(test_df)},
"periods": {
"all": [str(df_all.index[0]), str(df_all.index[-1])],
"train": [str(train_df.index[0]), str(train_df.index[-1])],
"test": [str(test_df.index[0]), str(test_df.index[-1])],
},
"trials": args.trials,
"baseline": {
"train": _result_dict(baseline_train, "train"),
"test": _result_dict(baseline_test, "test"),
"full": _result_dict(baseline_full, "full"),
},
"best_by_oos": {
"train": _result_dict(best_train, "train"),
"test": _result_dict(best_test, "test"),
"full": _result_dict(best_full, "full"),
},
"top10_oos": results_df.head(10).to_dict(orient="records"),
}
json_path = out_dir / f"{tag}_rsi_scalp_best.json"
json_path.write_text(json.dumps(report, indent=2), encoding="utf-8")
print("\n=== BEST BY OUT-OF-SAMPLE (validation) ===")
for k, v in best_params.__dict__.items():
print(f" {k}: {v}")
print(f" TRAIN net=${best_train.net_profit:.2f} trades={best_train.total_trades} dd={best_train.max_drawdown_pct:.1f}%")
print(f" TEST net=${best_test.net_profit:.2f} trades={best_test.total_trades} pf={best_test.profit_factor:.2f} dd={best_test.max_drawdown_pct:.1f}%")
print(f" FULL net=${best_full.net_profit:.2f} trades={best_full.total_trades} dd={best_full.max_drawdown_pct:.1f}%")
print(f"\nSaved: {csv_path}")
print(f"Saved: {json_path}")
finally:
mt5.shutdown()
if __name__ == "__main__":
main()
backtesting/MT5/set_parser.py
+58
View File
@@ -0,0 +1,58 @@
"""Parse MetaTrader 5 .set optimization files."""
from __future__ import annotations
from dataclasses import dataclass
from pathlib import Path
from typing import Any
@dataclass
class SetParam:
name: str
value: Any
start: Any
step: Any
stop: Any
optimize: bool
def _cast(raw: str) -> Any:
low = raw.strip().lower()
if low == "true":
return True
if low == "false":
return False
if "." in raw:
try:
return float(raw)
except ValueError:
return raw
try:
return int(raw)
except ValueError:
return raw
def parse_set_file(path: str | Path) -> dict[str, SetParam]:
params: dict[str, SetParam] = {}
for line in Path(path).read_text(encoding="utf-8", errors="ignore").splitlines():
line = line.strip()
if not line or line.startswith(";"):
continue
if "=" not in line:
continue
name, rest = line.split("=", 1)
parts = rest.split("||")
if len(parts) < 5:
continue
value, start, step, stop, opt = parts[0], parts[1], parts[2], parts[3], parts[4]
params[name] = SetParam(
name=name,
value=_cast(value),
start=_cast(start),
step=_cast(step),
stop=_cast(stop),
optimize=opt.strip().upper() == "Y",
)
return params
backtesting/MT5/sets/XAUUSD_RSI_Scalp_trailing.set
+11
View File
@@ -0,0 +1,11 @@
; RSIScalping XAUUSD trailing — Python optimizer ranges
RSI_Period=14||8||1||21||Y
RSI_Overbought=71||55.0||2.0||85.0||Y
RSI_Oversold=57||40.0||2.0||70.0||Y
RSI_Target_Buy=80||65.0||2.0||95.0||Y
RSI_Target_Sell=57||40.0||2.0||70.0||Y
BarsToWait=1||1||1||8||Y
LotSize=0.1||0.1||0.1||0.1||N
UseTrailingStop=true||false||0||true||Y
TrailingStopDistancePoints=71||20.0||5.0||200.0||Y
TrailingActivationPoints=41||0.0||5.0||120.0||Y