Prepare source-only public release for develop.
Add cluster audit pipeline, united EA updates, brochure generators, and publication hygiene (gitignore, MT5 path desensitization, pre-upload scan). Remove tracked reports, models, and binary artifacts from the repo. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
zhutoutoutousan
co-authored by
Cursor
Cursor
parent
3f75a08848
commit
605faf5310
@@ -0,0 +1 @@
|
||||
# Cluster audit package
|
||||
@@ -0,0 +1,327 @@
|
||||
"""Shared backtest primitives — fills, costs, metrics, trade log."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
|
||||
import MetaTrader5 as mt5
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
|
||||
|
||||
@dataclass
|
||||
class CostModel:
|
||||
spread_points: float = 0.0
|
||||
slippage_points: float = 3.0
|
||||
commission_per_lot: float = 0.0
|
||||
|
||||
@classmethod
|
||||
def for_symbol(cls, symbol: str, slippage: float = 3.0) -> "CostModel":
|
||||
info = mt5.symbol_info(symbol)
|
||||
spread = float(info.spread) if info else 0.0
|
||||
return cls(spread_points=spread, slippage_points=slippage)
|
||||
|
||||
|
||||
@dataclass
|
||||
class Trade:
|
||||
side: str
|
||||
open_time: Any
|
||||
close_time: Any
|
||||
open_price: float
|
||||
close_price: float
|
||||
volume: float
|
||||
profit: float
|
||||
bars_held: int = 0
|
||||
exit_reason: str = ""
|
||||
|
||||
|
||||
@dataclass
|
||||
class BacktestReport:
|
||||
strategy_id: str
|
||||
symbol: str
|
||||
timeframe: str
|
||||
period_label: str
|
||||
net_profit: float
|
||||
total_trades: int
|
||||
win_rate: float
|
||||
profit_factor: float
|
||||
sharpe: float
|
||||
max_drawdown_pct: float
|
||||
avg_win: float
|
||||
avg_loss: float
|
||||
worst_trades: list[dict]
|
||||
losing_trades: list[dict]
|
||||
exit_reason_breakdown: dict[str, dict[str, float]]
|
||||
monthly_returns: dict[str, float]
|
||||
params: dict[str, Any] = field(default_factory=dict)
|
||||
gross_profit: float = 0.0
|
||||
gross_loss: float = 0.0
|
||||
trades_list: list[Trade] = field(default_factory=list, repr=False)
|
||||
equity_curve: pd.Series | None = field(default=None, repr=False)
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
return {
|
||||
"strategy_id": self.strategy_id,
|
||||
"symbol": self.symbol,
|
||||
"timeframe": self.timeframe,
|
||||
"period": self.period_label,
|
||||
"net_profit": self.net_profit,
|
||||
"total_trades": self.total_trades,
|
||||
"win_rate": self.win_rate,
|
||||
"profit_factor": self.profit_factor,
|
||||
"sharpe": self.sharpe,
|
||||
"max_drawdown_pct": self.max_drawdown_pct,
|
||||
"avg_win": self.avg_win,
|
||||
"avg_loss": self.avg_loss,
|
||||
"worst_trades": self.worst_trades,
|
||||
"losing_trades": self.losing_trades,
|
||||
"exit_reason_breakdown": self.exit_reason_breakdown,
|
||||
"monthly_returns": self.monthly_returns,
|
||||
"params": self.params,
|
||||
}
|
||||
|
||||
|
||||
def resolve_symbol(requested: str) -> str:
|
||||
key = requested.split("|")[0].strip()
|
||||
if not key:
|
||||
return requested
|
||||
if mt5.symbol_info(key) is not None:
|
||||
mt5.symbol_select(key, True)
|
||||
return key
|
||||
for suffix in (".NAS", ".NYSE", ".NYS", ".US"):
|
||||
cand = key + suffix
|
||||
if mt5.symbol_info(cand) is not None:
|
||||
mt5.symbol_select(cand, True)
|
||||
return cand
|
||||
for sym in mt5.symbols_get() or []:
|
||||
if sym.name.startswith(key + "."):
|
||||
mt5.symbol_select(sym.name, True)
|
||||
return sym.name
|
||||
return key
|
||||
|
||||
|
||||
def load_bars(symbol: str, tf: int, start, end, trace: bool = False) -> pd.DataFrame:
|
||||
requested = symbol
|
||||
symbol = resolve_symbol(symbol)
|
||||
if trace and symbol != requested:
|
||||
print(f" [load_bars] resolved {requested} -> {symbol}", flush=True)
|
||||
if not mt5.symbol_select(symbol, True):
|
||||
raise RuntimeError(f"Cannot select {symbol}")
|
||||
rates = mt5.copy_rates_range(symbol, tf, start, end)
|
||||
if rates is None or len(rates) < 50:
|
||||
raise RuntimeError(f"No data for {symbol} ({mt5.last_error()})")
|
||||
if trace:
|
||||
print(f" [load_bars] {symbol} got {len(rates)} bars", flush=True)
|
||||
df = pd.DataFrame(rates)
|
||||
df["time"] = pd.to_datetime(df["time"], unit="s")
|
||||
df.set_index("time", inplace=True)
|
||||
return df
|
||||
|
||||
|
||||
def calc_profit(symbol: str, side: str, volume: float, entry: float, exit_px: float) -> float:
|
||||
ot = mt5.ORDER_TYPE_BUY if side == "BUY" else mt5.ORDER_TYPE_SELL
|
||||
p = mt5.order_calc_profit(ot, symbol, volume, entry, exit_px)
|
||||
return float(p) if p is not None else 0.0
|
||||
|
||||
|
||||
def _half_spread(point: float, spread_pts: float) -> float:
|
||||
return spread_pts * point / 2.0
|
||||
|
||||
|
||||
def fill_price(mid: float, point: float, costs: CostModel, side: str, entry: bool) -> float:
|
||||
hs = _half_spread(point, costs.spread_points)
|
||||
slip = costs.slippage_points * point
|
||||
if side == "BUY":
|
||||
return mid + hs + slip if entry else mid - hs - slip
|
||||
return mid - hs - slip if entry else mid + hs + slip
|
||||
|
||||
|
||||
@dataclass
|
||||
class SimState:
|
||||
side: str | None = None
|
||||
entry: float = 0.0
|
||||
entry_i: int = 0
|
||||
entry_time: object = None
|
||||
sl: float = 0.0
|
||||
tp: float = 0.0
|
||||
bars_against: int = 0
|
||||
rsi_against: bool = False
|
||||
|
||||
|
||||
def _bar_seconds(tf_label: str) -> int:
|
||||
mapping = {
|
||||
"M1": 60, "M5": 300, "M10": 600, "M12": 720, "M15": 900, "M20": 1200,
|
||||
"M30": 1800, "H1": 3600, "H2": 7200, "H4": 14400, "D1": 86400,
|
||||
}
|
||||
return mapping.get(tf_label.upper(), 3600)
|
||||
|
||||
|
||||
def run_single_position(
|
||||
df: pd.DataFrame,
|
||||
symbol: str,
|
||||
point: float,
|
||||
costs: CostModel,
|
||||
lot: float,
|
||||
strategy_id: str,
|
||||
tf_label: str,
|
||||
period_label: str,
|
||||
params: dict,
|
||||
initial_balance: float,
|
||||
on_bar,
|
||||
bar_seconds: int | None = None,
|
||||
) -> BacktestReport:
|
||||
"""Single-position bar loop with mark-to-market equity each bar."""
|
||||
bar_sec = bar_seconds or _bar_seconds(tf_label)
|
||||
trades: list[Trade] = []
|
||||
realized_pnl = 0.0
|
||||
equity: list[float] = [initial_balance]
|
||||
st = SimState()
|
||||
|
||||
def close(i: int, mid: float, reason: str) -> None:
|
||||
nonlocal st, realized_pnl
|
||||
if st.side is None:
|
||||
return
|
||||
exit_px = fill_price(mid, point, costs, st.side, entry=False)
|
||||
commission = costs.commission_per_lot * lot * 2.0
|
||||
profit = calc_profit(symbol, st.side, lot, st.entry, exit_px) - commission
|
||||
held = max(1, int((df.index[i] - pd.Timestamp(st.entry_time)).total_seconds() / bar_sec))
|
||||
trades.append(
|
||||
Trade(
|
||||
side=st.side,
|
||||
open_time=st.entry_time,
|
||||
close_time=df.index[i],
|
||||
open_price=st.entry,
|
||||
close_price=exit_px,
|
||||
volume=lot,
|
||||
profit=profit,
|
||||
bars_held=held,
|
||||
exit_reason=reason,
|
||||
)
|
||||
)
|
||||
realized_pnl += profit
|
||||
st = SimState()
|
||||
|
||||
def open_pos(i: int, side: str, mid: float) -> None:
|
||||
nonlocal st
|
||||
st.side = side
|
||||
st.entry = fill_price(mid, point, costs, side, entry=True)
|
||||
st.entry_i = i
|
||||
st.entry_time = df.index[i]
|
||||
|
||||
for i in range(1, len(df)):
|
||||
on_bar(i, st, open_pos, close)
|
||||
bal = initial_balance + realized_pnl
|
||||
if st.side is not None:
|
||||
mark = float(df["close"].iloc[i - 1])
|
||||
bal += calc_profit(symbol, st.side, lot, st.entry, mark)
|
||||
equity.append(bal)
|
||||
|
||||
if st.side is not None:
|
||||
close(len(df) - 1, float(df["close"].iloc[-1]), "eod")
|
||||
|
||||
eq = pd.Series(equity[: len(df)], index=df.index[: len(equity)])
|
||||
report = build_report(strategy_id, symbol, tf_label, period_label, trades, eq, initial_balance, params)
|
||||
report.equity_curve = eq
|
||||
return report
|
||||
|
||||
|
||||
def build_report(
|
||||
strategy_id: str,
|
||||
symbol: str,
|
||||
timeframe: str,
|
||||
period_label: str,
|
||||
trades: list[Trade],
|
||||
equity_curve: pd.Series,
|
||||
initial_balance: float,
|
||||
params: dict,
|
||||
) -> BacktestReport:
|
||||
if not trades:
|
||||
return BacktestReport(
|
||||
strategy_id=strategy_id,
|
||||
symbol=symbol,
|
||||
timeframe=timeframe,
|
||||
period_label=period_label,
|
||||
net_profit=0.0,
|
||||
total_trades=0,
|
||||
win_rate=0.0,
|
||||
profit_factor=0.0,
|
||||
sharpe=0.0,
|
||||
max_drawdown_pct=0.0,
|
||||
avg_win=0.0,
|
||||
avg_loss=0.0,
|
||||
worst_trades=[],
|
||||
losing_trades=[],
|
||||
exit_reason_breakdown={},
|
||||
monthly_returns={},
|
||||
params=params,
|
||||
)
|
||||
|
||||
profits = [t.profit for t in trades]
|
||||
wins = [p for p in profits if p >= 0]
|
||||
losses = [abs(p) for p in profits if p < 0]
|
||||
gp = sum(wins)
|
||||
gl = sum(losses)
|
||||
net = sum(profits)
|
||||
|
||||
rets = equity_curve.pct_change().dropna()
|
||||
sharpe = 0.0
|
||||
if len(rets) > 10 and rets.std() > 0:
|
||||
bars_per_year = 252 * 24 if "H1" in timeframe else 252 * 24 * 6
|
||||
scale = np.sqrt(bars_per_year / max(len(rets), 1))
|
||||
sharpe = float(rets.mean() / rets.std() * scale)
|
||||
|
||||
peak = equity_curve.cummax()
|
||||
dd = (peak - equity_curve) / peak.replace(0, np.nan)
|
||||
max_dd = float(dd.max()) if len(dd) else 0.0
|
||||
|
||||
monthly = equity_curve.resample("ME").last().pct_change().dropna()
|
||||
monthly_dict = {str(k.date()): float(v) for k, v in monthly.items()}
|
||||
|
||||
def trade_row(t: Trade) -> dict:
|
||||
return {
|
||||
"side": t.side,
|
||||
"open_time": str(t.open_time),
|
||||
"close_time": str(t.close_time),
|
||||
"open_price": t.open_price,
|
||||
"close_price": t.close_price,
|
||||
"profit": t.profit,
|
||||
"bars_held": t.bars_held,
|
||||
"exit_reason": t.exit_reason,
|
||||
}
|
||||
|
||||
sorted_trades = sorted(trades, key=lambda t: t.profit)
|
||||
losers = [trade_row(t) for t in sorted_trades if t.profit < 0]
|
||||
breakdown: dict[str, dict[str, float]] = {}
|
||||
for t in trades:
|
||||
bucket = breakdown.setdefault(t.exit_reason or "?", {"count": 0, "pnl": 0.0, "wins": 0, "losses": 0})
|
||||
bucket["count"] += 1
|
||||
bucket["pnl"] += t.profit
|
||||
if t.profit >= 0:
|
||||
bucket["wins"] += 1
|
||||
else:
|
||||
bucket["losses"] += 1
|
||||
|
||||
return BacktestReport(
|
||||
strategy_id=strategy_id,
|
||||
symbol=symbol,
|
||||
timeframe=timeframe,
|
||||
period_label=period_label,
|
||||
net_profit=net,
|
||||
total_trades=len(trades),
|
||||
win_rate=(len(wins) / len(trades) * 100.0) if trades else 0.0,
|
||||
profit_factor=(gp / gl) if gl > 0 else (999.0 if gp > 0 else 0.0),
|
||||
sharpe=sharpe,
|
||||
max_drawdown_pct=max_dd * 100.0,
|
||||
avg_win=(gp / len(wins)) if wins else 0.0,
|
||||
avg_loss=(gl / len(losses)) if losses else 0.0,
|
||||
worst_trades=[trade_row(t) for t in sorted_trades[:5]],
|
||||
losing_trades=losers[:30],
|
||||
exit_reason_breakdown=breakdown,
|
||||
monthly_returns=monthly_dict,
|
||||
params=params,
|
||||
gross_profit=gp,
|
||||
gross_loss=gl,
|
||||
trades_list=list(trades),
|
||||
equity_curve=equity_curve.copy(),
|
||||
)
|
||||
@@ -0,0 +1,145 @@
|
||||
"""Per-strategy problem diagnosis and loss tracing."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from .backtest_core import BacktestReport
|
||||
from .trace_log import TraceLog
|
||||
|
||||
ENGINE_FIX_HINTS = {
|
||||
"rsi_crossover": "trend_strong closes positions in strong trends; high ema_slope/distance thresholds block entries.",
|
||||
"rsi_scalp": "rsi_against exits + spread costs dominate; check OB/OS vs target gap and bars_to_wait.",
|
||||
"rsi_asian": "session window or extreme RSI levels may block entries; verify broker server hour offset.",
|
||||
"mean_reversion": "ADX proxy may differ from MQL iADX; min_ema_distance_pts can block all entries on volatile symbols.",
|
||||
"ema_slope": "needs EMA-cross exit + profit trail even when use_trailing_stop=False; weekly ADX filter missing.",
|
||||
"darvas": "box must be narrow (box_deviation); volume MA filter not yet ported.",
|
||||
"rsi_secret": "zone re-entry in chop; add divergence confirm or widen min_bars_between_trades.",
|
||||
}
|
||||
|
||||
|
||||
def diagnose(spec: dict, baseline: BacktestReport, optimized: BacktestReport | None, log: TraceLog) -> dict[str, Any]:
|
||||
sid = spec["id"]
|
||||
engine = spec["engine"]
|
||||
issues: list[str] = []
|
||||
actions: list[str] = []
|
||||
|
||||
if baseline.total_trades == 0:
|
||||
issues.append("ZERO_TRADES")
|
||||
actions.append("Engine logic or params too strict - compare Python port to MQL defaults.")
|
||||
elif baseline.total_trades < 10:
|
||||
issues.append("LOW_TRADE_COUNT")
|
||||
actions.append("Relax entry filters or widen optimization ranges.")
|
||||
|
||||
if baseline.net_profit < 0:
|
||||
issues.append("NEGATIVE_PNL")
|
||||
if baseline.sharpe < 0:
|
||||
issues.append("NEGATIVE_SHARPE")
|
||||
if baseline.max_drawdown_pct > 25:
|
||||
issues.append("HIGH_DRAWDOWN")
|
||||
|
||||
br = baseline.exit_reason_breakdown
|
||||
loss_reasons = sorted(
|
||||
((k, v["pnl"]) for k, v in br.items() if v["pnl"] < 0),
|
||||
key=lambda x: x[1],
|
||||
)
|
||||
if loss_reasons:
|
||||
top = loss_reasons[0]
|
||||
issues.append(f"TOP_LOSS_REASON:{top[0]}")
|
||||
if top[0] == "rsi_against":
|
||||
actions.append("Widen RSI targets or increase bars_to_wait before rsi_against exit.")
|
||||
elif top[0] == "trend_strong":
|
||||
actions.append("Raise ema_slope/distance thresholds or only block new entries (MQL also force-closes).")
|
||||
elif top[0] == "trail":
|
||||
actions.append("Trail too tight - widen trail_distance_pts or raise activation.")
|
||||
elif top[0] == "sl":
|
||||
actions.append("Stop loss too tight for symbol volatility - scale SL by ATR.")
|
||||
elif top[0] == "hours" or top[0] == "session":
|
||||
actions.append("Trading hours/session filter closing positions — align to broker server time.")
|
||||
elif top[0] == "adx_escape":
|
||||
actions.append("ADX escape fires too early — raise adx_escape threshold.")
|
||||
|
||||
hint = ENGINE_FIX_HINTS.get(engine, "")
|
||||
if hint:
|
||||
actions.append(hint)
|
||||
|
||||
if optimized and optimized is not baseline:
|
||||
from .scoring import DEFAULT_TRADES_PER_DAY, acceptance, min_trades_for_period, period_days, trades_per_day
|
||||
from .strategy_registry import PERIODS
|
||||
|
||||
start, end = PERIODS.get("2021-2026", ("2021-01-01", "2026-06-01"))
|
||||
days = period_days(start, end)
|
||||
min_t = min_trades_for_period(days, DEFAULT_TRADES_PER_DAY)
|
||||
opt_tpd = trades_per_day(optimized, days)
|
||||
base_tpd = trades_per_day(baseline, days)
|
||||
if opt_tpd < DEFAULT_TRADES_PER_DAY:
|
||||
issues.append("LOW_TRADES_PER_DAY")
|
||||
actions.append(
|
||||
f"Only {opt_tpd:.2f} trades/day (need >={DEFAULT_TRADES_PER_DAY:.1f}); "
|
||||
"use lower TF, tighter SL/TP, or relax entry filters."
|
||||
)
|
||||
if optimized.total_trades < min_t and baseline.total_trades >= min_t:
|
||||
issues.append("OPT_COLLAPSED_TRADES")
|
||||
actions.append(
|
||||
f"Optimization cut trades {baseline.total_trades}->{optimized.total_trades} "
|
||||
f"({base_tpd:.2f}->{opt_tpd:.2f}/day)."
|
||||
)
|
||||
opt_ok, opt_issues = acceptance(optimized, days, DEFAULT_TRADES_PER_DAY)
|
||||
if not opt_ok and optimized.net_profit > baseline.net_profit:
|
||||
issues.append("OPT_PROFIT_BUT_FAILS_GATES")
|
||||
actions.append("Higher net but fails gates: " + "; ".join(opt_issues[:4]))
|
||||
if optimized.sharpe > baseline.sharpe + 0.1:
|
||||
issues.append("OPTIMIZATION_HELPED")
|
||||
|
||||
log.banner(f"DIAGNOSIS: {sid}")
|
||||
log.info(f"engine={engine} symbol={baseline.symbol} trades={baseline.total_trades}")
|
||||
if issues:
|
||||
log.warn("issues: " + ", ".join(issues))
|
||||
else:
|
||||
log.info("no critical issues flagged")
|
||||
|
||||
trace_losses(baseline, log, label="baseline")
|
||||
|
||||
if optimized and optimized is not baseline:
|
||||
log.info(
|
||||
f"optimized: net=${optimized.net_profit:.0f} sharpe={optimized.sharpe:.2f} "
|
||||
f"trades={optimized.total_trades}"
|
||||
)
|
||||
if optimized.net_profit < baseline.net_profit:
|
||||
trace_losses(optimized, log, label="optimized", max_rows=10)
|
||||
|
||||
if actions:
|
||||
log.info("suggested actions:")
|
||||
for a in actions[:6]:
|
||||
log.debug(f" - {a}")
|
||||
|
||||
return {
|
||||
"issues": issues,
|
||||
"actions": actions,
|
||||
"top_loss_reasons": loss_reasons[:5],
|
||||
"exit_reason_breakdown": br,
|
||||
}
|
||||
|
||||
|
||||
def trace_losses(report: BacktestReport, log: TraceLog, label: str = "baseline", max_rows: int = 15) -> None:
|
||||
if not report.losing_trades:
|
||||
log.debug(f"{label}: no losing trades")
|
||||
return
|
||||
|
||||
log.info(f"{label} loss trace ({len(report.losing_trades)} losers logged, showing worst {max_rows}):")
|
||||
for i, t in enumerate(report.losing_trades[:max_rows], 1):
|
||||
log.info(
|
||||
f" #{i:02d} {t['side']:4} ${t['profit']:8.2f} {t['exit_reason']:12} "
|
||||
f"bars={t['bars_held']:4} {t['open_time']} -> {t['close_time']}"
|
||||
)
|
||||
|
||||
if report.exit_reason_breakdown:
|
||||
log.debug(f"{label} exit reason PnL:")
|
||||
for reason, stats in sorted(
|
||||
report.exit_reason_breakdown.items(),
|
||||
key=lambda x: x[1]["pnl"],
|
||||
):
|
||||
log.debug(
|
||||
f" {reason:14} count={int(stats['count']):4} "
|
||||
f"wins={int(stats['wins']):3} losses={int(stats['losses']):3} pnl=${stats['pnl']:.0f}"
|
||||
)
|
||||
@@ -0,0 +1,882 @@
|
||||
"""Python ports of SuperEA engines for cluster audit."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
|
||||
from indicator_utils import calculate_adx, calculate_atr, calculate_dmi, calculate_ema, calculate_rsi
|
||||
|
||||
from .backtest_core import (
|
||||
BacktestReport,
|
||||
CostModel,
|
||||
Trade,
|
||||
build_report,
|
||||
calc_profit,
|
||||
fill_price,
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class SimState:
|
||||
side: str | None = None
|
||||
entry: float = 0.0
|
||||
entry_i: int = 0
|
||||
entry_time: Any = None
|
||||
sl: float = 0.0
|
||||
tp: float = 0.0
|
||||
bars_against: int = 0
|
||||
rsi_against: bool = False
|
||||
|
||||
|
||||
def _run_single_position(
|
||||
df: pd.DataFrame,
|
||||
symbol: str,
|
||||
point: float,
|
||||
costs: CostModel,
|
||||
lot: float,
|
||||
strategy_id: str,
|
||||
tf_label: str,
|
||||
period_label: str,
|
||||
params: dict,
|
||||
initial_balance: float,
|
||||
on_bar,
|
||||
) -> BacktestReport:
|
||||
trades: list[Trade] = []
|
||||
equity = [initial_balance]
|
||||
st = SimState()
|
||||
|
||||
def close(i: int, mid: float, reason: str) -> None:
|
||||
nonlocal st
|
||||
if st.side is None:
|
||||
return
|
||||
exit_px = fill_price(mid, point, costs, st.side, entry=False)
|
||||
commission = costs.commission_per_lot * lot * 2
|
||||
profit = calc_profit(symbol, st.side, lot, st.entry, exit_px) - commission
|
||||
trades.append(
|
||||
Trade(
|
||||
side=st.side,
|
||||
open_time=st.entry_time,
|
||||
close_time=df.index[i],
|
||||
open_price=st.entry,
|
||||
close_price=exit_px,
|
||||
volume=lot,
|
||||
profit=profit,
|
||||
bars_held=i - st.entry_i,
|
||||
exit_reason=reason,
|
||||
)
|
||||
)
|
||||
equity.append(equity[-1] + profit)
|
||||
st = SimState()
|
||||
|
||||
def open_pos(i: int, side: str, mid: float) -> None:
|
||||
nonlocal st
|
||||
st.side = side
|
||||
st.entry = fill_price(mid, point, costs, side, entry=True)
|
||||
st.entry_i = i
|
||||
st.entry_time = df.index[i]
|
||||
st.sl = 0.0
|
||||
st.tp = 0.0
|
||||
st.bars_against = 0
|
||||
st.rsi_against = False
|
||||
|
||||
for i in range(1, len(df)):
|
||||
mid = float(df["open"].iloc[i])
|
||||
on_bar(i, st, open_pos, close)
|
||||
if len(equity) == len(trades) + 1:
|
||||
equity.append(equity[-1])
|
||||
|
||||
if st.side is not None:
|
||||
close(len(df) - 1, float(df["close"].iloc[-1]), "eod")
|
||||
|
||||
eq = pd.Series(equity[: len(df)], index=df.index[: len(equity)])
|
||||
return build_report(strategy_id, symbol, tf_label, period_label, trades, eq, initial_balance, params)
|
||||
|
||||
|
||||
# --- RSI Scalping ---
|
||||
def backtest_rsi_scalp(
|
||||
df: pd.DataFrame,
|
||||
symbol: str,
|
||||
period_label: str,
|
||||
strategy_id: str,
|
||||
params: dict,
|
||||
lot: float = 0.1,
|
||||
costs: CostModel | None = None,
|
||||
) -> BacktestReport:
|
||||
info = __import__("MetaTrader5").symbol_info(symbol)
|
||||
point = float(info.point) if info else 0.01
|
||||
costs = costs or CostModel.for_symbol(symbol)
|
||||
rsi = calculate_rsi(df["close"], int(params["rsi_period"])).to_numpy()
|
||||
atr = calculate_atr(df, int(params.get("reversal_atr_period", 14))).to_numpy()
|
||||
trail_dist = params.get("trail_distance_pts", 0) * point
|
||||
trail_act = (params.get("trail_activation_pts") or params.get("trail_distance_pts", 0)) * point
|
||||
use_rsi_against = params.get("use_rsi_against_exit", True)
|
||||
max_adv_atr = float(params.get("max_adverse_atr", 0))
|
||||
|
||||
def on_bar(i, st, open_pos, close):
|
||||
if i < 3 or np.isnan(rsi[i - 1]):
|
||||
return
|
||||
sig, prev, two = rsi[i - 1], rsi[i - 2], rsi[i - 3]
|
||||
mid = float(df["open"].iloc[i])
|
||||
hi, lo = float(df["high"].iloc[i]), float(df["low"].iloc[i])
|
||||
|
||||
if st.side is not None and params.get("use_reversal_escape"):
|
||||
a = float(atr[i - 1]) if not np.isnan(atr[i - 1]) else 0.0
|
||||
if a > 0:
|
||||
adv_mult = float(params.get("reversal_adverse_atr_mult", 1.5))
|
||||
rsi_vel = float(params.get("reversal_rsi_velocity", 8.0))
|
||||
need = int(params.get("reversal_signs_required", 2))
|
||||
signs = 0
|
||||
if st.side == "BUY":
|
||||
if st.entry - lo >= adv_mult * a:
|
||||
signs += 1
|
||||
if sig - prev >= rsi_vel:
|
||||
signs += 1
|
||||
else:
|
||||
if hi - st.entry >= adv_mult * a:
|
||||
signs += 1
|
||||
if sig - prev >= rsi_vel:
|
||||
signs += 1
|
||||
if signs >= need:
|
||||
close(i, mid, "reversal_escape")
|
||||
return
|
||||
|
||||
if st.side is not None and params.get("use_trailing") and trail_dist > 0:
|
||||
if st.side == "BUY":
|
||||
bid = float(df["close"].iloc[i])
|
||||
if bid - st.entry > trail_act:
|
||||
nsl = bid - trail_dist
|
||||
if nsl > st.sl:
|
||||
st.sl = nsl
|
||||
if st.sl > 0 and lo <= st.sl:
|
||||
close(i, st.sl, "trail")
|
||||
return
|
||||
else:
|
||||
ask = float(df["close"].iloc[i])
|
||||
if st.entry - ask > trail_act:
|
||||
nsl = ask + trail_dist
|
||||
if st.sl == 0 or nsl < st.sl:
|
||||
st.sl = nsl
|
||||
if st.sl > 0 and hi >= st.sl:
|
||||
close(i, st.sl, "trail")
|
||||
return
|
||||
|
||||
if st.side is not None and max_adv_atr > 0:
|
||||
a = float(atr[i - 1]) if not np.isnan(atr[i - 1]) else 0.0
|
||||
if a > 0:
|
||||
if st.side == "BUY" and (st.entry - lo) / a >= max_adv_atr:
|
||||
close(i, mid, "adverse_atr")
|
||||
return
|
||||
if st.side == "SELL" and (hi - st.entry) / a >= max_adv_atr:
|
||||
close(i, mid, "adverse_atr")
|
||||
return
|
||||
|
||||
if st.side == "BUY":
|
||||
if use_rsi_against and sig < params["rsi_oversold"]:
|
||||
st.bars_against = st.bars_against + 1 if st.rsi_against else 1
|
||||
st.rsi_against = True
|
||||
if st.bars_against >= params["bars_to_wait"]:
|
||||
close(i, mid, "rsi_against")
|
||||
else:
|
||||
st.rsi_against = False
|
||||
st.bars_against = 0
|
||||
if sig >= params["rsi_target_buy"]:
|
||||
close(i, mid, "target")
|
||||
elif st.side == "SELL":
|
||||
if use_rsi_against and sig > params["rsi_overbought"]:
|
||||
st.bars_against = st.bars_against + 1 if st.rsi_against else 1
|
||||
st.rsi_against = True
|
||||
if st.bars_against >= params["bars_to_wait"]:
|
||||
close(i, mid, "rsi_against")
|
||||
else:
|
||||
st.rsi_against = False
|
||||
st.bars_against = 0
|
||||
if sig <= params["rsi_target_sell"]:
|
||||
close(i, mid, "target")
|
||||
else:
|
||||
if two <= params["rsi_oversold"] and prev > params["rsi_oversold"]:
|
||||
open_pos(i, "BUY", mid)
|
||||
elif two >= params["rsi_overbought"] and prev < params["rsi_overbought"]:
|
||||
min_depth = float(params.get("min_ob_depth", 0))
|
||||
if two < params["rsi_overbought"] + min_depth:
|
||||
pass
|
||||
else:
|
||||
skip_h = int(params.get("skip_short_hour_after", 24))
|
||||
if df.index[i].hour < skip_h:
|
||||
open_pos(i, "SELL", mid)
|
||||
|
||||
return _run_single_position(
|
||||
df, symbol, point, costs, lot, strategy_id, params.get("tf", "H1"),
|
||||
period_label, params, 10_000.0, on_bar,
|
||||
)
|
||||
|
||||
|
||||
# --- RSI CrossOver ---
|
||||
def _price_to_ema_pips(symbol: str, close: float, ema: float) -> float:
|
||||
info = __import__("MetaTrader5").symbol_info(symbol)
|
||||
if info is None:
|
||||
return abs(close - ema) * 10.0
|
||||
point = float(info.point)
|
||||
digits = int(info.digits)
|
||||
pip_mult = 10.0 if digits in (3, 5) else 1.0
|
||||
pip_size = point * pip_mult if point > 0 else point
|
||||
return abs(close - ema) / pip_size if pip_size > 0 else 0.0
|
||||
|
||||
|
||||
def backtest_rsi_crossover(df, symbol, period_label, strategy_id, params, lot=0.1, costs=None):
|
||||
info = __import__("MetaTrader5").symbol_info(symbol)
|
||||
point = float(info.point) if info else 0.01
|
||||
costs = costs or CostModel.for_symbol(symbol)
|
||||
rsi = calculate_rsi(df["close"], int(params["rsi_period"])).to_numpy()
|
||||
ema = calculate_ema(df["close"], int(params["ema_period"])).to_numpy()
|
||||
trail = params.get("trailing_stop_pts", 0) * point
|
||||
prev_rsi_state = 0.0
|
||||
last_trade_i = -10_000
|
||||
cooldown_bars = max(1, int(params.get("cooldown_seconds", 300) / 3600))
|
||||
use_trend_filter = params.get("use_trend_strength_filter", True)
|
||||
|
||||
weekday_ok = {
|
||||
0: params.get("sunday", False),
|
||||
1: params.get("monday", False),
|
||||
2: params.get("tuesday", True),
|
||||
3: params.get("wednesday", True),
|
||||
4: params.get("thursday", True),
|
||||
5: params.get("friday", False),
|
||||
6: params.get("saturday", False),
|
||||
}
|
||||
|
||||
def hours_ok(ts) -> bool:
|
||||
h = ts.hour
|
||||
def in_win(begin: int, end: int) -> bool:
|
||||
b, e = begin % 24, end % 24
|
||||
if b == e:
|
||||
return False
|
||||
if b < e:
|
||||
return b <= h < e
|
||||
return h >= b or h < e
|
||||
return in_win(params.get("trading_hour_one_begin", 0), params.get("trading_hour_one_end", 22)) or in_win(
|
||||
params.get("trading_hour_two_begin", 6), params.get("trading_hour_two_end", 19)
|
||||
)
|
||||
|
||||
def on_bar(i, st, open_pos, close):
|
||||
nonlocal prev_rsi_state, last_trade_i
|
||||
if i < 3 or np.isnan(rsi[i - 1]) or np.isnan(ema[i - 1]):
|
||||
return
|
||||
ts = df.index[i]
|
||||
if not weekday_ok.get(ts.weekday(), False) or not hours_ok(ts):
|
||||
if st.side:
|
||||
close(i, float(df["open"].iloc[i]), "hours")
|
||||
return
|
||||
|
||||
cur = rsi[i - 1]
|
||||
if prev_rsi_state == 0.0:
|
||||
prev_rsi_state = cur
|
||||
return
|
||||
|
||||
ema_slope = (ema[i - 1] - ema[i - 2]) * 100.0
|
||||
price_to_ema = abs((float(df["close"].iloc[i - 1]) - ema[i - 1]) * 10.0)
|
||||
slope_th = float(params.get("ema_slope_threshold", 100))
|
||||
dist_th = float(params.get("ema_distance_threshold", 100))
|
||||
trend_strong = use_trend_filter and (
|
||||
(slope_th > 0 and abs(ema_slope) > slope_th)
|
||||
or (dist_th > 0 and price_to_ema > dist_th)
|
||||
)
|
||||
|
||||
mid = float(df["open"].iloc[i])
|
||||
if st.side == "BUY" and trail > 0:
|
||||
bid = float(df["close"].iloc[i])
|
||||
if bid - st.entry > trail:
|
||||
st.sl = max(st.sl, bid - trail)
|
||||
if st.sl > 0 and float(df["low"].iloc[i]) <= st.sl:
|
||||
close(i, st.sl, "trail")
|
||||
prev_rsi_state = cur
|
||||
return
|
||||
if st.side == "SELL" and trail > 0:
|
||||
ask = float(df["close"].iloc[i])
|
||||
if st.entry - ask > trail:
|
||||
st.sl = ask + trail if st.sl == 0 else min(st.sl, ask + trail)
|
||||
if st.sl > 0 and float(df["high"].iloc[i]) >= st.sl:
|
||||
close(i, st.sl, "trail")
|
||||
prev_rsi_state = cur
|
||||
return
|
||||
|
||||
if st.side == "BUY" and cur > params.get("exit_buy_rsi", 80):
|
||||
close(i, mid, "exit_rsi")
|
||||
elif st.side == "SELL" and cur < params.get("exit_sell_rsi", 20):
|
||||
close(i, mid, "exit_rsi")
|
||||
elif trend_strong and st.side:
|
||||
close(i, mid, "trend_strong")
|
||||
elif not st.side and not trend_strong and i - last_trade_i >= cooldown_bars:
|
||||
ob = params.get("overbought_level", 70)
|
||||
os = params.get("oversold_level", 30)
|
||||
sell_spread = params.get("entry_rsi_sell_spread", 0)
|
||||
buy_spread = params.get("entry_rsi_buy_spread", 0)
|
||||
if prev_rsi_state >= ob and cur < ob - sell_spread:
|
||||
open_pos(i, "SELL", mid)
|
||||
last_trade_i = i
|
||||
elif prev_rsi_state <= os and cur > os + buy_spread:
|
||||
open_pos(i, "BUY", mid)
|
||||
last_trade_i = i
|
||||
prev_rsi_state = cur
|
||||
|
||||
return _run_single_position(
|
||||
df, symbol, point, costs, lot, strategy_id, "H1", period_label, params, 10_000.0, on_bar,
|
||||
)
|
||||
|
||||
|
||||
# --- RSI Asian ---
|
||||
def backtest_rsi_asian(df, symbol, period_label, strategy_id, params, lot=0.1, costs=None):
|
||||
info = __import__("MetaTrader5").symbol_info(symbol)
|
||||
point = float(info.point) if info else 0.01
|
||||
costs = costs or CostModel.for_symbol(symbol)
|
||||
rsi = calculate_rsi(df["close"], int(params["rsi_period"])).to_numpy()
|
||||
sess_start = params.get("asian_session_start", 0)
|
||||
sess_end = params.get("asian_session_end", 8)
|
||||
|
||||
def in_session(ts) -> bool:
|
||||
return sess_start <= ts.hour < sess_end
|
||||
|
||||
def on_bar(i, st, open_pos, close):
|
||||
if i < 2 or np.isnan(rsi[i - 1]):
|
||||
return
|
||||
ts = df.index[i]
|
||||
prev, cur = rsi[i - 2], rsi[i - 1]
|
||||
mid = float(df["open"].iloc[i])
|
||||
|
||||
if st.side and params.get("close_outside_session") and not in_session(ts):
|
||||
close(i, mid, "session")
|
||||
return
|
||||
if st.side and params.get("use_rsi_exit"):
|
||||
exit_lvl = params.get("rsi_exit_level", 55)
|
||||
if (prev < exit_lvl <= cur) or (prev > exit_lvl >= cur):
|
||||
close(i, mid, "rsi_exit")
|
||||
|
||||
if not in_session(ts):
|
||||
return
|
||||
if st.side:
|
||||
return
|
||||
if prev < params["overbought_level"] <= cur:
|
||||
open_pos(i, "SELL", mid)
|
||||
elif prev > params["oversold_level"] >= cur:
|
||||
open_pos(i, "BUY", mid)
|
||||
|
||||
return _run_single_position(
|
||||
df, symbol, point, costs, lot, strategy_id, "M15", period_label, params, 10_000.0, on_bar,
|
||||
)
|
||||
|
||||
|
||||
# --- Mean Reversion ---
|
||||
def backtest_mean_reversion(df, symbol, period_label, strategy_id, params, lot=0.1, costs=None):
|
||||
info = __import__("MetaTrader5").symbol_info(symbol)
|
||||
point = float(info.point) if info else 0.01
|
||||
costs = costs or CostModel.for_symbol(symbol)
|
||||
rsi = calculate_rsi(df["close"], int(params["rsi_period"])).to_numpy()
|
||||
ema = calculate_ema(df["close"], int(params["ema_period"])).to_numpy()
|
||||
adx = calculate_adx(df, int(params.get("adx_period", 14))).to_numpy()
|
||||
|
||||
def on_bar(i, st, open_pos, close):
|
||||
if i < max(params["ema_period"], 20) + 2:
|
||||
return
|
||||
if np.isnan(rsi[i - 1]) or np.isnan(ema[i - 1]) or np.isnan(adx[i - 1]):
|
||||
return
|
||||
mid = float(df["open"].iloc[i])
|
||||
cls = float(df["close"].iloc[i - 1])
|
||||
dist_buy = (ema[i - 1] - cls) / point
|
||||
dist_sell = (cls - ema[i - 1]) / point
|
||||
adx_v = float(adx[i - 1])
|
||||
|
||||
if st.side:
|
||||
adx_now = float(adx[i - 1]) if not np.isnan(adx[i - 1]) else 0.0
|
||||
if adx_now >= params.get("adx_escape", 30):
|
||||
close(i, mid, "adx_escape")
|
||||
elif params.get("use_hard_sltp"):
|
||||
if st.side == "BUY":
|
||||
if cls <= st.entry - params.get("sl_points", 0) * point:
|
||||
close(i, mid, "sl")
|
||||
elif cls >= st.entry + params.get("tp_points", 0) * point:
|
||||
close(i, mid, "tp")
|
||||
else:
|
||||
if cls >= st.entry + params.get("sl_points", 0) * point:
|
||||
close(i, mid, "sl")
|
||||
elif cls <= st.entry - params.get("tp_points", 0) * point:
|
||||
close(i, mid, "tp")
|
||||
return
|
||||
|
||||
if adx_v <= 0 or adx_v >= params.get("adx_max_for_entry", 20):
|
||||
return
|
||||
use_cross = params.get("use_rsi_cross", True)
|
||||
if use_cross:
|
||||
buy_rsi = rsi[i - 2] > params["rsi_oversold"] >= rsi[i - 1]
|
||||
sell_rsi = rsi[i - 2] < params["rsi_overbought"] <= rsi[i - 1]
|
||||
else:
|
||||
buy_rsi = rsi[i - 1] <= params["rsi_oversold"]
|
||||
sell_rsi = rsi[i - 1] >= params["rsi_overbought"]
|
||||
if buy_rsi and dist_buy >= params.get("min_ema_distance_pts", 0):
|
||||
open_pos(i, "BUY", mid)
|
||||
if params.get("use_hard_sltp"):
|
||||
st.sl = st.entry - params.get("sl_points", 0) * point
|
||||
st.tp = st.entry + params.get("tp_points", 0) * point
|
||||
elif sell_rsi and dist_sell >= params.get("min_ema_distance_pts", 0):
|
||||
open_pos(i, "SELL", mid)
|
||||
if params.get("use_hard_sltp"):
|
||||
st.sl = st.entry + params.get("sl_points", 0) * point
|
||||
st.tp = st.entry - params.get("tp_points", 0) * point
|
||||
|
||||
return _run_single_position(
|
||||
df, symbol, point, costs, lot, strategy_id, "M15", period_label, params, 10_000.0, on_bar,
|
||||
)
|
||||
|
||||
|
||||
# --- EMA Slope (monitor + crossover state machine, matches MQL) ---
|
||||
def _weekly_dmi_lookup(df: pd.DataFrame, period: int, bar_shift: int) -> tuple[pd.Series, pd.Series, pd.Series]:
|
||||
wdf = df.resample("W-FRI").agg({"high": "max", "low": "min", "close": "last"}).dropna()
|
||||
dmi = calculate_dmi(wdf, period)
|
||||
shift = max(0, bar_shift)
|
||||
adx = dmi["adx"].shift(shift).reindex(df.index, method="ffill")
|
||||
plus = dmi["plus_di"].shift(shift).reindex(df.index, method="ffill")
|
||||
minus = dmi["minus_di"].shift(shift).reindex(df.index, method="ffill")
|
||||
return adx, plus, minus
|
||||
|
||||
|
||||
def backtest_ema_slope(df, symbol, period_label, strategy_id, params, lot=0.1, costs=None):
|
||||
info = __import__("MetaTrader5").symbol_info(symbol)
|
||||
point = float(info.point) if info else 0.01
|
||||
costs = costs or CostModel.for_symbol(symbol)
|
||||
ema_period = int(params["ema_period"])
|
||||
ema = calculate_ema(df["close"], ema_period).to_numpy()
|
||||
atr = calculate_atr(df, 14).to_numpy()
|
||||
closes = df["close"].to_numpy()
|
||||
opens = df["open"].to_numpy()
|
||||
highs = df["high"].to_numpy()
|
||||
lows = df["low"].to_numpy()
|
||||
times = df.index
|
||||
mult = 10.0 if ("XAU" in symbol or "BTC" in symbol) else 1.0
|
||||
|
||||
w_adx, w_plus, w_minus = _weekly_dmi_lookup(
|
||||
df, int(params.get("weekly_adx_period", 28)), int(params.get("weekly_adx_bar_shift", 8))
|
||||
)
|
||||
|
||||
price_trigger_active = False
|
||||
slope_trigger_active = False
|
||||
monitor_active = False
|
||||
monitor_start_i = -1
|
||||
trades_in_cross = 0
|
||||
last_close = 0.0
|
||||
last_ema = 0.0
|
||||
last_bar_time = None
|
||||
|
||||
def weekly_ok(i: int, side: str) -> bool:
|
||||
if not params.get("use_weekly_adx_filter", True):
|
||||
return True
|
||||
adx_v = float(w_adx.iloc[i - 1]) if i > 0 else np.nan
|
||||
if np.isnan(adx_v) or adx_v < params.get("weekly_adx_min", 25):
|
||||
return False
|
||||
if not params.get("weekly_adx_use_direction", True):
|
||||
return True
|
||||
pdi = float(w_plus.iloc[i - 1])
|
||||
mdi = float(w_minus.iloc[i - 1])
|
||||
if side == "BUY":
|
||||
return pdi > mdi
|
||||
return mdi > pdi
|
||||
|
||||
def on_bar(i, st, open_pos, close):
|
||||
nonlocal price_trigger_active, slope_trigger_active, monitor_active, monitor_start_i
|
||||
nonlocal trades_in_cross, last_close, last_ema, last_bar_time
|
||||
if i < ema_period + 3 or np.isnan(ema[i - 1]) or np.isnan(ema[i - 2]):
|
||||
return
|
||||
|
||||
if params.get("use_bar_data", True):
|
||||
ts = times[i]
|
||||
if last_bar_time is not None and ts == last_bar_time:
|
||||
return
|
||||
last_bar_time = ts
|
||||
|
||||
mid = float(opens[i])
|
||||
bar_close = float(closes[i - 1])
|
||||
ema_now = float(ema[i - 1])
|
||||
ema_prev = float(ema[i - 2])
|
||||
|
||||
if last_close != 0.0:
|
||||
if (last_close <= last_ema and bar_close > ema_now) or (last_close >= last_ema and bar_close < ema_now):
|
||||
trades_in_cross = 0
|
||||
last_close, last_ema = bar_close, ema_now
|
||||
|
||||
price_dist = abs(bar_close - ema_now) / point / mult
|
||||
if price_dist > params.get("price_threshold_pips", 100) and not price_trigger_active:
|
||||
price_trigger_active = True
|
||||
slope = (ema_now - ema_prev) / point / mult
|
||||
if abs(slope) > params.get("slope_threshold_pips", 20) and not slope_trigger_active:
|
||||
slope_trigger_active = True
|
||||
|
||||
if price_trigger_active and slope_trigger_active and not monitor_active:
|
||||
monitor_active = True
|
||||
monitor_start_i = i
|
||||
|
||||
tf_sec = 3600
|
||||
timeout_bars = int(params.get("monitor_timeout_sec", 340) / tf_sec)
|
||||
if monitor_active and monitor_start_i >= 0 and (i - monitor_start_i) > timeout_bars:
|
||||
monitor_active = False
|
||||
price_trigger_active = False
|
||||
slope_trigger_active = False
|
||||
|
||||
if st.side:
|
||||
bars_open = i - st.entry_i
|
||||
bar_close_now = float(closes[i])
|
||||
a = float(atr[i - 1]) if not np.isnan(atr[i - 1]) else 0.0
|
||||
max_loss_atr = float(params.get("max_loss_atr", 2.0))
|
||||
if a > 0 and max_loss_atr > 0:
|
||||
if st.side == "BUY" and float(lows[i]) <= st.entry - max_loss_atr * a:
|
||||
close(i, st.entry - max_loss_atr * a, "atr_sl")
|
||||
return
|
||||
if st.side == "SELL" and float(highs[i]) >= st.entry + max_loss_atr * a:
|
||||
close(i, st.entry + max_loss_atr * a, "atr_sl")
|
||||
return
|
||||
|
||||
trail_pips = params.get("trailing_stop_pips", 50)
|
||||
trail_px = trail_pips * point * mult
|
||||
bar_close_now = float(closes[i])
|
||||
in_profit = (bar_close_now > st.entry) if st.side == "BUY" else (st.entry > bar_close_now)
|
||||
use_trail = params.get("use_trailing_stop", False)
|
||||
trail_act = params.get("trailing_activation_pips", 0)
|
||||
trail_ready = in_profit if trail_act <= 0 else (
|
||||
(bar_close_now - st.entry) / point / mult >= trail_act
|
||||
if st.side == "BUY"
|
||||
else (st.entry - bar_close_now) / point / mult >= trail_act
|
||||
)
|
||||
if trail_pips > 0 and (use_trail or in_profit) and trail_ready:
|
||||
if st.side == "BUY":
|
||||
st.sl = max(st.sl, bar_close_now - trail_px)
|
||||
if st.sl > 0 and float(lows[i]) <= st.sl:
|
||||
close(i, st.sl, "trail")
|
||||
return
|
||||
else:
|
||||
st.sl = bar_close_now + trail_px if st.sl <= 0 else min(st.sl, bar_close_now + trail_px)
|
||||
if st.sl > 0 and float(highs[i]) >= st.sl:
|
||||
close(i, st.sl, "trail")
|
||||
return
|
||||
|
||||
ema_exit = (st.side == "BUY" and bar_close_now < ema_now) or (
|
||||
st.side == "SELL" and bar_close_now > ema_now
|
||||
)
|
||||
unrealized = calc_profit(symbol, st.side, lot, st.entry, bar_close_now)
|
||||
if ema_exit and unrealized > 0:
|
||||
close(i, mid, "ema_cross")
|
||||
return
|
||||
|
||||
if params.get("close_unprofitable_trades", True):
|
||||
check_bars = int(params.get("profit_check_bars", 78))
|
||||
if bars_open >= check_bars:
|
||||
unrealized = calc_profit(symbol, st.side, lot, st.entry, bar_close_now)
|
||||
if unrealized <= 0:
|
||||
close(i, mid, "unprofitable")
|
||||
return
|
||||
return
|
||||
|
||||
if not monitor_active:
|
||||
return
|
||||
if trades_in_cross >= params.get("max_trades_per_crossover", 5):
|
||||
return
|
||||
|
||||
if bar_close > ema_now and weekly_ok(i, "BUY"):
|
||||
open_pos(i, "BUY", mid)
|
||||
trades_in_cross += 1
|
||||
monitor_active = False
|
||||
price_trigger_active = False
|
||||
slope_trigger_active = False
|
||||
elif bar_close < ema_now and weekly_ok(i, "SELL"):
|
||||
open_pos(i, "SELL", mid)
|
||||
trades_in_cross += 1
|
||||
monitor_active = False
|
||||
price_trigger_active = False
|
||||
slope_trigger_active = False
|
||||
|
||||
return _run_single_position(
|
||||
df, symbol, point, costs, lot, strategy_id, "H1", period_label, params, 10_000.0, on_bar,
|
||||
)
|
||||
|
||||
|
||||
# --- Darvas Box (matches MQL: narrow box + breakout + trend strength) ---
|
||||
def backtest_darvas(df, symbol, period_label, strategy_id, params, lot=0.1, costs=None):
|
||||
info = __import__("MetaTrader5").symbol_info(symbol)
|
||||
point = float(info.point) if info else 0.01
|
||||
costs = costs or CostModel.for_symbol(symbol)
|
||||
period = int(params.get("box_period", 165))
|
||||
box_dev = float(params.get("box_deviation", 30000))
|
||||
trend_thresh = float(params.get("trend_threshold", 4.94))
|
||||
ma_period = int(params.get("ma_period", 125))
|
||||
sl_pts = float(params.get("stop_loss_pts", 1665))
|
||||
tp_pts = float(params.get("take_profit_pts", 3685))
|
||||
vol_thresh = int(params.get("volume_threshold", 0))
|
||||
max_range = box_dev * point
|
||||
|
||||
ma = calculate_ema(df["close"], ma_period).to_numpy()
|
||||
highs = df["high"].to_numpy()
|
||||
lows = df["low"].to_numpy()
|
||||
opens = df["open"].to_numpy()
|
||||
closes = df["close"].to_numpy()
|
||||
vols = df["tick_volume"].to_numpy() if "tick_volume" in df.columns else np.zeros(len(df))
|
||||
|
||||
def on_bar(i, st, open_pos, close):
|
||||
if i < period + ma_period + 2:
|
||||
return
|
||||
window_hi = float(np.max(highs[i - period : i]))
|
||||
window_lo = float(np.min(lows[i - period : i]))
|
||||
if (window_hi - window_lo) > max_range:
|
||||
return
|
||||
|
||||
mid = float(opens[i])
|
||||
bar_hi = float(highs[i])
|
||||
bar_lo = float(lows[i])
|
||||
ma_v = float(ma[i - 1])
|
||||
if np.isnan(ma_v):
|
||||
return
|
||||
|
||||
if st.side:
|
||||
if st.side == "BUY":
|
||||
if st.sl > 0 and bar_lo <= st.sl:
|
||||
close(i, st.sl, "sl")
|
||||
elif st.tp > 0 and bar_hi >= st.tp:
|
||||
close(i, st.tp, "tp")
|
||||
else:
|
||||
if st.sl > 0 and bar_hi >= st.sl:
|
||||
close(i, st.sl, "sl")
|
||||
elif st.tp > 0 and bar_lo <= st.tp:
|
||||
close(i, st.tp, "tp")
|
||||
return
|
||||
|
||||
if vols[i] <= vol_thresh:
|
||||
return
|
||||
prev_close = float(closes[i - 1])
|
||||
strength = abs(mid - ma_v) / point
|
||||
break_up = bar_hi > window_hi or prev_close > window_hi
|
||||
break_dn = bar_lo < window_lo or prev_close < window_lo
|
||||
|
||||
if break_up and mid > ma_v and strength > trend_thresh:
|
||||
open_pos(i, "BUY", mid)
|
||||
st.sl = st.entry - sl_pts * point
|
||||
st.tp = st.entry + tp_pts * point
|
||||
elif break_dn and mid < ma_v and strength > trend_thresh:
|
||||
open_pos(i, "SELL", mid)
|
||||
st.sl = st.entry + sl_pts * point
|
||||
st.tp = st.entry - tp_pts * point
|
||||
|
||||
return _run_single_position(
|
||||
df, symbol, point, costs, lot, strategy_id, "M15", period_label, params, 10_000.0, on_bar,
|
||||
)
|
||||
|
||||
|
||||
# --- RSI Secret Sauce (simplified zone exit re-entry) ---
|
||||
def backtest_rsi_secret(df, symbol, period_label, strategy_id, params, lot=0.1, costs=None):
|
||||
info = __import__("MetaTrader5").symbol_info(symbol)
|
||||
point = float(info.point) if info else 0.01
|
||||
costs = costs or CostModel.for_symbol(symbol)
|
||||
rsi = calculate_rsi(df["close"], int(params["rsi_period"])).to_numpy()
|
||||
atr = calculate_atr(df, int(params.get("atr_period", 14))).to_numpy()
|
||||
last_trade_i = -999
|
||||
|
||||
def on_bar(i, st, open_pos, close):
|
||||
nonlocal last_trade_i
|
||||
if i < 30 or np.isnan(rsi[i - 1]) or np.isnan(atr[i - 1]):
|
||||
return
|
||||
mid = float(df["open"].iloc[i])
|
||||
cur, prev = rsi[i - 1], rsi[i - 2]
|
||||
a = atr[i - 1]
|
||||
|
||||
if st.side:
|
||||
if st.side == "BUY":
|
||||
sl = st.entry - params.get("stop_loss_atr", 2) * a
|
||||
tp = st.entry + params.get("take_profit_atr", 4) * a
|
||||
if float(df["low"].iloc[i]) <= sl:
|
||||
close(i, sl, "sl")
|
||||
elif float(df["high"].iloc[i]) >= tp:
|
||||
close(i, tp, "tp")
|
||||
else:
|
||||
sl = st.entry + params.get("stop_loss_atr", 2) * a
|
||||
tp = st.entry - params.get("take_profit_atr", 4) * a
|
||||
if float(df["high"].iloc[i]) >= sl:
|
||||
close(i, sl, "sl")
|
||||
elif float(df["low"].iloc[i]) <= tp:
|
||||
close(i, tp, "tp")
|
||||
return
|
||||
|
||||
if i - last_trade_i < params.get("min_bars_between_trades", 5):
|
||||
return
|
||||
ob, os = params["rsi_overbought"], params["rsi_oversold"]
|
||||
if prev > ob and cur <= ob:
|
||||
open_pos(i, "SELL", mid)
|
||||
last_trade_i = i
|
||||
elif prev < os and cur >= os:
|
||||
open_pos(i, "BUY", mid)
|
||||
last_trade_i = i
|
||||
|
||||
return _run_single_position(
|
||||
df, symbol, point, costs, lot, strategy_id, "M30", period_label, params, 10_000.0, on_bar,
|
||||
)
|
||||
|
||||
|
||||
# --- Simple Trendline (pullback to MA-derived trendline) ---
|
||||
def backtest_simple_trendline(df, symbol, period_label, strategy_id, params, lot=0.1, costs=None):
|
||||
info = __import__("MetaTrader5").symbol_info(symbol)
|
||||
point = float(info.point) if info else 0.01
|
||||
costs = costs or CostModel.for_symbol(symbol)
|
||||
htf = params.get("higher_tf", "H4")
|
||||
htf_map = {"M10": "10min", "M15": "15min", "H1": "1h", "H4": "4h"}
|
||||
rule = htf_map.get(htf, "4h")
|
||||
hdf = df.resample(rule).agg({"open": "first", "high": "max", "low": "min", "close": "last"}).dropna()
|
||||
ma_period = int(params.get("ma_period", 65))
|
||||
ma = calculate_ema(hdf["close"], ma_period).to_numpy()
|
||||
htimes = hdf.index.to_numpy()
|
||||
hcloses = hdf["close"].to_numpy()
|
||||
touch_tol = float(params.get("touch_tolerance_pts", 100)) * point
|
||||
break_buf = float(params.get("break_buffer_pts", 80)) * point
|
||||
|
||||
def line_at(t_model, t_query):
|
||||
x = (t_query - t_model[0]).astype("timedelta64[s]").astype(float)
|
||||
return t_model[2] * x + t_model[3]
|
||||
|
||||
def on_bar(i, st, open_pos, close):
|
||||
if i < 5:
|
||||
return
|
||||
ts = df.index[i]
|
||||
# find 3 most recent HTF MA crosses before ts
|
||||
hidx = int(np.searchsorted(htimes, ts, side="right")) - 1
|
||||
if hidx < ma_period + 5:
|
||||
return
|
||||
crosses_t, crosses_p = [], []
|
||||
for j in range(hidx, ma_period + 2, -1):
|
||||
if j >= len(ma) - 1:
|
||||
continue
|
||||
d0 = hcloses[j] - ma[j]
|
||||
d1 = hcloses[j + 1] - ma[j + 1]
|
||||
if d0 == 0 or d1 == 0 or d0 * d1 < 0:
|
||||
crosses_t.append(htimes[j])
|
||||
crosses_p.append(hcloses[j])
|
||||
if len(crosses_t) >= 3:
|
||||
break
|
||||
if len(crosses_t) < 3:
|
||||
return
|
||||
t0 = crosses_t[2]
|
||||
xs = np.array([(t - t0).astype("timedelta64[s]").astype(float) for t in crosses_t[::-1]])
|
||||
ys = np.array(crosses_p[::-1])
|
||||
den = 3 * np.sum(xs ** 2) - np.sum(xs) ** 2
|
||||
if abs(den) < 1e-10:
|
||||
return
|
||||
a = (3 * np.sum(xs * ys) - np.sum(xs) * np.sum(ys)) / den
|
||||
b = (np.sum(ys) - a * np.sum(xs)) / 3
|
||||
model = (t0, crosses_t, a, b)
|
||||
t1 = df.index[i - 1]
|
||||
line1 = a * (t1 - t0).astype("timedelta64[s]").astype(float) + b
|
||||
mid = float(df["open"].iloc[i])
|
||||
hi = float(df["high"].iloc[i - 1])
|
||||
lo = float(df["low"].iloc[i - 1])
|
||||
cl1 = float(df["close"].iloc[i - 1])
|
||||
op1 = float(df["open"].iloc[i - 1])
|
||||
cl2 = float(df["close"].iloc[i - 2])
|
||||
t2 = df.index[i - 2]
|
||||
line2 = a * (t2 - t0).astype("timedelta64[s]").astype(float) + b
|
||||
|
||||
if st.side == "BUY" and cl1 < line1 - break_buf:
|
||||
close(i, mid, "break")
|
||||
return
|
||||
if st.side == "SELL" and cl1 > line1 + break_buf:
|
||||
close(i, mid, "break")
|
||||
return
|
||||
|
||||
if st.side:
|
||||
return
|
||||
if a > 0:
|
||||
if lo <= line1 + touch_tol and cl1 > line1 and cl1 > op1 and cl2 >= line2 - touch_tol:
|
||||
open_pos(i, "BUY", mid)
|
||||
elif a < 0:
|
||||
if hi >= line1 - touch_tol and cl1 < line1 and cl1 < op1 and cl2 <= line2 + touch_tol:
|
||||
open_pos(i, "SELL", mid)
|
||||
|
||||
return _run_single_position(
|
||||
df, symbol, point, costs, lot, strategy_id, params.get("signal_tf", "H1"),
|
||||
period_label, params, 10_000.0, on_bar,
|
||||
)
|
||||
|
||||
|
||||
# --- USDJPY Asian range breakout (simplified market-fill) ---
|
||||
def backtest_usdjpy_buster(df, symbol, period_label, strategy_id, params, lot=0.1, costs=None):
|
||||
info = __import__("MetaTrader5").symbol_info(symbol)
|
||||
point = float(info.point) if info else 0.001
|
||||
costs = costs or CostModel.for_symbol(symbol)
|
||||
r_start = int(params.get("range_start_hour", 3))
|
||||
r_end = int(params.get("range_end_hour", 6))
|
||||
close_h = int(params.get("close_hour", 18))
|
||||
min_rng = float(params.get("min_range_pts", 15))
|
||||
buf = float(params.get("order_buffer_pts", 4.75)) * point
|
||||
first_only = params.get("first_trade_only", False)
|
||||
|
||||
day_state: dict = {}
|
||||
|
||||
def on_bar(i, st, open_pos, close):
|
||||
ts = df.index[i]
|
||||
dk = ts.date().isoformat()
|
||||
h = ts.hour
|
||||
mid = float(df["open"].iloc[i])
|
||||
|
||||
if st.side and h >= close_h:
|
||||
close(i, mid, "eod")
|
||||
return
|
||||
|
||||
if dk not in day_state:
|
||||
day_state[dk] = {"hi": -np.inf, "lo": np.inf, "built": False, "trades": 0, "range_done": False}
|
||||
|
||||
ds = day_state[dk]
|
||||
if r_start <= h < r_end:
|
||||
ds["hi"] = max(ds["hi"], float(df["high"].iloc[i]))
|
||||
ds["lo"] = min(ds["lo"], float(df["low"].iloc[i]))
|
||||
return
|
||||
|
||||
if not ds["range_done"] and h >= r_end:
|
||||
ds["range_done"] = True
|
||||
if ds["hi"] > ds["lo"] and (ds["hi"] - ds["lo"]) / point >= min_rng:
|
||||
ds["built"] = True
|
||||
|
||||
if not ds["built"] or st.side:
|
||||
return
|
||||
max_tr = 1 if first_only else 2
|
||||
if ds["trades"] >= max_tr:
|
||||
return
|
||||
|
||||
hi = ds["hi"] + buf
|
||||
lo = ds["lo"] - buf
|
||||
bar_hi = float(df["high"].iloc[i])
|
||||
bar_lo = float(df["low"].iloc[i])
|
||||
|
||||
if params.get("allow_long", True) and bar_hi >= hi:
|
||||
open_pos(i, "BUY", mid)
|
||||
st.sl = ds["lo"]
|
||||
ds["trades"] += 1
|
||||
elif params.get("allow_short", True) and bar_lo <= lo:
|
||||
open_pos(i, "SELL", mid)
|
||||
st.sl = ds["hi"]
|
||||
ds["trades"] += 1
|
||||
|
||||
if st.side:
|
||||
if st.side == "BUY" and bar_lo <= st.sl:
|
||||
close(i, st.sl, "sl")
|
||||
elif st.side == "SELL" and bar_hi >= st.sl:
|
||||
close(i, st.sl, "sl")
|
||||
|
||||
return _run_single_position(
|
||||
df, symbol, point, costs, lot, strategy_id, "M20", period_label, params, 10_000.0, on_bar,
|
||||
)
|
||||
|
||||
|
||||
ENGINE_MAP = {
|
||||
"rsi_scalp": backtest_rsi_scalp,
|
||||
"rsi_crossover": backtest_rsi_crossover,
|
||||
"rsi_asian": backtest_rsi_asian,
|
||||
"mean_reversion": backtest_mean_reversion,
|
||||
"ema_slope": backtest_ema_slope,
|
||||
"darvas": backtest_darvas,
|
||||
"rsi_secret": backtest_rsi_secret,
|
||||
"simple_trendline": backtest_simple_trendline,
|
||||
"usdjpy_buster": backtest_usdjpy_buster,
|
||||
}
|
||||
@@ -0,0 +1,277 @@
|
||||
"""
|
||||
Analyze losing trades in market context — bars before/after, gaps between losses,
|
||||
RSI/ATR/trend features. Trader-style narrative + param suggestions.
|
||||
|
||||
Usage:
|
||||
python -m cluster_audit.loss_context_analysis united_rsi_scalp_appl
|
||||
python -m cluster_audit.loss_context_analysis united_darvas
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sys
|
||||
from datetime import datetime, timedelta
|
||||
from pathlib import Path
|
||||
|
||||
import MetaTrader5 as mt5
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
|
||||
|
||||
from cluster_audit.backtest_core import CostModel, Trade, load_bars, resolve_symbol
|
||||
from cluster_audit.engines import ENGINE_MAP
|
||||
from cluster_audit.united_registry import PERIODS, UNITED_STRATEGIES
|
||||
from indicator_utils import calculate_atr, calculate_ema, calculate_rsi
|
||||
|
||||
OUT_DIR = Path(__file__).parent / "reports" / "loss_analysis"
|
||||
CONTEXT_BARS = 12 # bars before entry + through hold
|
||||
|
||||
|
||||
def find_spec(sid: str) -> dict:
|
||||
for s in UNITED_STRATEGIES:
|
||||
if s["id"] == sid:
|
||||
return s
|
||||
raise KeyError(sid)
|
||||
|
||||
|
||||
def run_backtest(spec: dict, start: str, end: str) -> tuple[pd.DataFrame, list[Trade], dict]:
|
||||
from cluster_audit.strategy_registry import TF
|
||||
|
||||
sym = resolve_symbol(spec["symbol"])
|
||||
tf_key = spec["tf"]
|
||||
engine = ENGINE_MAP[spec["engine"]]
|
||||
params = dict(spec["defaults"])
|
||||
df = load_bars(sym, TF[tf_key], datetime.fromisoformat(start), datetime.fromisoformat(end))
|
||||
report = engine(df, sym, "2021-2026", spec["id"], params, spec["lot"], CostModel.for_symbol(sym))
|
||||
return df, report.trades_list, params
|
||||
|
||||
|
||||
def bar_features(df: pd.DataFrame, idx: int, rsi: np.ndarray, atr: np.ndarray, ema20: np.ndarray) -> dict:
|
||||
if idx < 1 or idx >= len(df):
|
||||
return {}
|
||||
o, h, l, c = df.iloc[idx][["open", "high", "low", "close"]]
|
||||
prev_c = float(df.iloc[idx - 1]["close"])
|
||||
body = abs(c - o)
|
||||
rng = h - l if h > l else 1e-9
|
||||
return {
|
||||
"rsi": float(rsi[idx - 1]) if not np.isnan(rsi[idx - 1]) else np.nan,
|
||||
"atr": float(atr[idx - 1]) if not np.isnan(atr[idx - 1]) else np.nan,
|
||||
"ema20": float(ema20[idx - 1]) if not np.isnan(ema20[idx - 1]) else np.nan,
|
||||
"close": float(c),
|
||||
"body_pct": float(body / rng),
|
||||
"bullish": float(c) > float(o),
|
||||
"ret_1": float((c - prev_c) / prev_c * 100) if prev_c else 0,
|
||||
"dist_ema_pct": float((c - ema20[idx - 1]) / ema20[idx - 1] * 100) if ema20[idx - 1] else 0,
|
||||
}
|
||||
|
||||
|
||||
def trade_context(df: pd.DataFrame, t: Trade, rsi, atr, ema20) -> dict:
|
||||
open_i = df.index.get_indexer([pd.Timestamp(t.open_time)], method="nearest")[0]
|
||||
close_i = df.index.get_indexer([pd.Timestamp(t.close_time)], method="nearest")[0]
|
||||
pre_start = max(1, open_i - CONTEXT_BARS)
|
||||
pre_bars = []
|
||||
for i in range(pre_start, open_i):
|
||||
pre_bars.append(bar_features(df, i, rsi, atr, ema20))
|
||||
|
||||
hold_bars = []
|
||||
for i in range(open_i, min(close_i + 1, len(df))):
|
||||
hold_bars.append(bar_features(df, i, rsi, atr, ema20))
|
||||
|
||||
entry_f = bar_features(df, open_i, rsi, atr, ema20)
|
||||
exit_f = bar_features(df, close_i, rsi, atr, ema20)
|
||||
|
||||
pre_rsi = [b["rsi"] for b in pre_bars if not np.isnan(b.get("rsi", np.nan))]
|
||||
hold_rsi = [b["rsi"] for b in hold_bars if not np.isnan(b.get("rsi", np.nan))]
|
||||
|
||||
adverse_move = 0.0
|
||||
if t.side == "BUY" and hold_bars:
|
||||
adverse_move = float(t.open_price) - min(b["close"] for b in hold_bars)
|
||||
elif t.side == "SELL" and hold_bars:
|
||||
adverse_move = max(b["close"] for b in hold_bars) - float(t.open_price)
|
||||
|
||||
return {
|
||||
"side": t.side,
|
||||
"exit_reason": t.exit_reason,
|
||||
"profit": t.profit,
|
||||
"bars_held": t.bars_held,
|
||||
"open_time": str(t.open_time),
|
||||
"close_time": str(t.close_time),
|
||||
"entry_rsi": entry_f.get("rsi"),
|
||||
"exit_rsi": exit_f.get("rsi"),
|
||||
"rsi_min_hold": min(hold_rsi) if hold_rsi else None,
|
||||
"rsi_max_hold": max(hold_rsi) if hold_rsi else None,
|
||||
"rsi_trend_pre": (pre_rsi[-1] - pre_rsi[0]) if len(pre_rsi) >= 2 else 0,
|
||||
"adverse_pts": adverse_move,
|
||||
"adverse_atr": adverse_move / entry_f["atr"] if entry_f.get("atr") else 0,
|
||||
"entry_hour": pd.Timestamp(t.open_time).hour,
|
||||
"entry_dist_ema_pct": entry_f.get("dist_ema_pct", 0),
|
||||
"pre_bullish_ratio": sum(1 for b in pre_bars if b.get("bullish")) / max(len(pre_bars), 1),
|
||||
"entry_body_pct": entry_f.get("body_pct", 0),
|
||||
}
|
||||
|
||||
|
||||
def gap_analysis(losers: list[dict]) -> dict:
|
||||
if len(losers) < 2:
|
||||
return {}
|
||||
times = sorted(pd.Timestamp(t["close_time"]) for t in losers)
|
||||
gaps_h = [(times[i] - times[i - 1]).total_seconds() / 3600 for i in range(1, len(times))]
|
||||
return {
|
||||
"median_gap_hours": float(np.median(gaps_h)),
|
||||
"pct_gap_under_4h": float(sum(1 for g in gaps_h if g < 4) / len(gaps_h) * 100),
|
||||
"pct_gap_under_24h": float(sum(1 for g in gaps_h if g < 24) / len(gaps_h) * 100),
|
||||
"clustered": float(sum(1 for g in gaps_h if g < 2) / len(gaps_h) * 100),
|
||||
}
|
||||
|
||||
|
||||
def trader_narrative(sid: str, engine: str, losers_ctx: list[dict], winners_ctx: list[dict], by_reason: dict) -> list[str]:
|
||||
notes: list[str] = []
|
||||
if not losers_ctx:
|
||||
return ["No losing trades to analyze."]
|
||||
|
||||
top_reason = max(by_reason.items(), key=lambda x: x[1]["count"])[0]
|
||||
lr = [c for c in losers_ctx if c["exit_reason"] == top_reason]
|
||||
wr = winners_ctx
|
||||
|
||||
if top_reason == "rsi_against":
|
||||
sell_l = [c for c in lr if c["side"] == "SELL"]
|
||||
buy_l = [c for c in lr if c["side"] == "BUY"]
|
||||
if sell_l:
|
||||
avg_adv = np.mean([c["adverse_atr"] for c in sell_l if c["adverse_atr"]])
|
||||
notes.append(
|
||||
f"SELL rsi_against ({len(sell_l)}): price ripped up avg {avg_adv:.1f} ATR after shorting "
|
||||
f"overbought fade — classic short squeeze / momentum continuation, not mean reversion."
|
||||
)
|
||||
late_h = sum(1 for c in sell_l if c["entry_hour"] >= 18) / len(sell_l) * 100
|
||||
if late_h > 30:
|
||||
notes.append(f"{late_h:.0f}% of losing shorts after 18:00 — avoid fading strength into close.")
|
||||
if buy_l:
|
||||
notes.append(
|
||||
f"BUY rsi_against ({len(buy_l)}): dipped deeper after oversold entry — "
|
||||
f"knife-catching; need deeper OS threshold or wait for RSI curl-up."
|
||||
)
|
||||
if wr:
|
||||
w_sell = [c for c in wr if c["side"] == "SELL"]
|
||||
if w_sell and sell_l:
|
||||
w_rsi = np.mean([c["entry_rsi"] for c in w_sell])
|
||||
l_rsi = np.mean([c["entry_rsi"] for c in sell_l])
|
||||
notes.append(f"Winning shorts entered RSI~{w_rsi:.0f} vs losers~{l_rsi:.0f} — losers entered too early in OB zone.")
|
||||
|
||||
elif top_reason == "sl":
|
||||
notes.append("SL hits: stops inside noise — widen SL to 1.5-2x ATR or reduce lot.")
|
||||
avg_atr = np.mean([c["adverse_atr"] for c in lr if c.get("adverse_atr")])
|
||||
notes.append(f"Avg adverse move before SL = {avg_atr:.1f} ATR — box breakout often retests.")
|
||||
|
||||
elif top_reason == "adverse_atr":
|
||||
notes.append("ATR stop hits: entries fighting trend — fade only when RSI extreme + session filter; widen stop or skip gap-down buys.")
|
||||
buy_l = [c for c in lr if c["side"] == "BUY"]
|
||||
if buy_l:
|
||||
late = sum(1 for c in buy_l if c["entry_hour"] >= 20) / len(buy_l) * 100
|
||||
if late > 25:
|
||||
notes.append(f"{late:.0f}% of stopped-out buys after 20:00 — overnight gap risk on equities.")
|
||||
sell_l = [c for c in lr if c["side"] == "SELL"]
|
||||
if sell_l:
|
||||
avg_adv = np.mean([c["adverse_atr"] for c in sell_l if c.get("adverse_atr")])
|
||||
notes.append(f"Short stops avg {avg_adv:.1f} ATR adverse — momentum continuation, not reversion.")
|
||||
|
||||
elif top_reason == "trail":
|
||||
notes.append("Trail exits: winners cut early in chop — widen trail_distance or raise activation.")
|
||||
|
||||
elif top_reason == "trend_strong":
|
||||
notes.append("trend_strong: exited into momentum — filter only blocks entries, don't force-close in profit.")
|
||||
|
||||
gaps = gap_analysis(losers_ctx)
|
||||
if gaps.get("clustered", 0) > 25:
|
||||
notes.append(
|
||||
f"{gaps['clustered']:.0f}% of losses within 2h of prior loss — regime chop; "
|
||||
f"add cooldown after loss or skip when ATR expanding."
|
||||
)
|
||||
|
||||
return notes
|
||||
|
||||
|
||||
def suggest_params(engine: str, losers_ctx: list[dict], params: dict) -> dict:
|
||||
sug = {}
|
||||
if engine == "rsi_scalp":
|
||||
sell_l = [c for c in losers_ctx if c["side"] == "SELL" and c["exit_reason"] == "rsi_against"]
|
||||
if sell_l and np.mean([c["adverse_atr"] for c in sell_l]) > 1.5:
|
||||
sug["rsi_overbought"] = min(75, params.get("rsi_overbought", 70) + 5)
|
||||
sug["bars_to_wait"] = min(12, params.get("bars_to_wait", 5) + 3)
|
||||
sug["trail_distance_pts"] = params.get("trail_distance_pts", 50) * 1.4
|
||||
sug["skip_short_hour_after"] = 17
|
||||
buy_l = [c for c in losers_ctx if c["side"] == "BUY" and c["exit_reason"] == "rsi_against"]
|
||||
if buy_l:
|
||||
sug["rsi_oversold"] = max(20, params.get("rsi_oversold", 30) - 5)
|
||||
elif engine == "darvas":
|
||||
sug["stop_loss_pts"] = int(params.get("stop_loss_pts", 300) * 1.35)
|
||||
sug["require_retest"] = True
|
||||
return sug
|
||||
|
||||
|
||||
def analyze(sid: str) -> dict:
|
||||
spec = find_spec(sid)
|
||||
start, end = PERIODS["2021-2026"]
|
||||
df, trades, params = run_backtest(spec, start, end)
|
||||
|
||||
rsi = calculate_rsi(df["close"], int(params.get("rsi_period", 14))).to_numpy()
|
||||
atr = calculate_atr(df, 14).to_numpy()
|
||||
ema20 = calculate_ema(df["close"], 20).to_numpy()
|
||||
|
||||
winners = [t for t in trades if t.profit >= 0]
|
||||
losers = [t for t in trades if t.profit < 0]
|
||||
|
||||
losers_ctx = [trade_context(df, t, rsi, atr, ema20) for t in losers]
|
||||
winners_ctx = [trade_context(df, t, rsi, atr, ema20) for t in winners[:200]]
|
||||
|
||||
by_reason: dict = {}
|
||||
for c in losers_ctx:
|
||||
r = c["exit_reason"]
|
||||
bucket = by_reason.setdefault(r, {"count": 0, "pnl": 0.0, "ctx": []})
|
||||
bucket["count"] += 1
|
||||
bucket["pnl"] += c["profit"]
|
||||
bucket["ctx"].append(c)
|
||||
|
||||
narrative = trader_narrative(sid, spec["engine"], losers_ctx, winners_ctx, by_reason)
|
||||
suggestions = suggest_params(spec["engine"], losers_ctx, params)
|
||||
|
||||
result = {
|
||||
"strategy_id": sid,
|
||||
"symbol": spec["symbol"],
|
||||
"engine": spec["engine"],
|
||||
"total_trades": len(trades),
|
||||
"losers": len(losers),
|
||||
"winners": len(winners),
|
||||
"loss_by_reason": {k: {"count": v["count"], "pnl": round(v["pnl"], 2)} for k, v in by_reason.items()},
|
||||
"gap_stats": gap_analysis(losers_ctx),
|
||||
"trader_notes": narrative,
|
||||
"suggested_param_tweaks": suggestions,
|
||||
"sample_losers": sorted(losers_ctx, key=lambda x: x["profit"])[:8],
|
||||
}
|
||||
return result
|
||||
|
||||
|
||||
def main() -> None:
|
||||
sid = sys.argv[1] if len(sys.argv) > 1 else "united_rsi_scalp_appl"
|
||||
if not mt5.initialize():
|
||||
raise SystemExit(f"MT5 init failed: {mt5.last_error()}")
|
||||
try:
|
||||
result = analyze(sid)
|
||||
OUT_DIR.mkdir(parents=True, exist_ok=True)
|
||||
path = OUT_DIR / f"{sid}_loss_context.json"
|
||||
path.write_text(json.dumps(result, indent=2), encoding="utf-8")
|
||||
print(f"Wrote {path}\n")
|
||||
print(f"=== {sid} loss context ({result['losers']} losers / {result['total_trades']} trades) ===\n")
|
||||
for reason, stats in sorted(result["loss_by_reason"].items(), key=lambda x: x[1]["pnl"]):
|
||||
print(f" {reason:14} count={stats['count']:4} pnl=${stats['pnl']:,.0f}")
|
||||
print("\nTrader read:")
|
||||
for n in result["trader_notes"]:
|
||||
print(f" - {n}")
|
||||
if result["suggested_param_tweaks"]:
|
||||
print("\nSuggested tweaks:", result["suggested_param_tweaks"])
|
||||
finally:
|
||||
mt5.shutdown()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,88 @@
|
||||
"""
|
||||
Lot sizing guidance for United EA combined portfolio.
|
||||
|
||||
Compares per-strategy risk at 123.set nominal lots and suggests relative weights.
|
||||
|
||||
Usage:
|
||||
python -m cluster_audit.lot_sizing
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
from cluster_audit.united_registry import UNITED_STRATEGIES
|
||||
|
||||
REPORTS = Path(__file__).parent / "reports" / "united_sequential"
|
||||
MANIFEST = REPORTS / "united_manifest.json"
|
||||
OUT = REPORTS / "lot_sizing.json"
|
||||
REF_BALANCE = 1000.0
|
||||
|
||||
|
||||
def main() -> None:
|
||||
manifest = {}
|
||||
if MANIFEST.exists():
|
||||
manifest = json.loads(MANIFEST.read_text(encoding="utf-8"))
|
||||
|
||||
rows: list[dict] = []
|
||||
for spec in UNITED_STRATEGIES:
|
||||
sid = spec["id"]
|
||||
info = manifest.get("strategies", {}).get(sid, {})
|
||||
if not info.get("passed"):
|
||||
continue
|
||||
o_net = float(info.get("net_profit", 0))
|
||||
trades = int(info.get("trades", 1))
|
||||
dd = float(info.get("max_drawdown_pct", info.get("issues", [""])[0] if False else 5))
|
||||
lot = float(spec["lot"])
|
||||
per_trade = o_net / max(trades, 1)
|
||||
# risk proxy: lot * avg loss magnitude; use net/trades as PnL per trade signal
|
||||
rows.append({
|
||||
"id": sid,
|
||||
"symbol": spec["symbol"],
|
||||
"lot_123set": lot,
|
||||
"net_profit": o_net,
|
||||
"trades": trades,
|
||||
"pnl_per_trade": round(per_trade, 2),
|
||||
"max_dd_pct": dd,
|
||||
})
|
||||
|
||||
if not rows:
|
||||
print("No passed strategies in manifest — run united sequential audit first.")
|
||||
return
|
||||
|
||||
# Target: equal risk contribution via inverse DD weighting
|
||||
inv_dd = [1.0 / max(r.get("max_dd_pct", 5), 0.5) for r in rows]
|
||||
total_inv = sum(inv_dd)
|
||||
for i, r in enumerate(rows):
|
||||
weight = inv_dd[i] / total_inv
|
||||
r["risk_weight"] = round(weight, 4)
|
||||
r["suggested_lot_vs_darvas"] = round(weight / (inv_dd[0] / total_inv), 3) if rows else 1.0
|
||||
|
||||
# Scale all lots so combined net at ref balance ~ sum of individuals / sqrt(N)
|
||||
import math
|
||||
n = len(rows)
|
||||
diversification = math.sqrt(n)
|
||||
base_lot = rows[0]["lot_123set"]
|
||||
for r in rows:
|
||||
r["suggested_lot_at_1k"] = round(base_lot * r["suggested_lot_vs_darvas"] / diversification, 4)
|
||||
|
||||
result = {
|
||||
"reference_balance": REF_BALANCE,
|
||||
"passed_count": n,
|
||||
"diversification_factor": diversification,
|
||||
"note": "suggested_lot_at_1k scales 123.set lots by inverse-DD weight / sqrt(N)",
|
||||
"strategies": rows,
|
||||
}
|
||||
OUT.write_text(json.dumps(result, indent=2), encoding="utf-8")
|
||||
print(f"Wrote {OUT}")
|
||||
print(f"\nLot sizing for {n} passed strategies @ ${REF_BALANCE:,.0f} reference:\n")
|
||||
for r in rows:
|
||||
print(
|
||||
f" {r['id']:28} lot={r['lot_123set']:8} -> suggested={r['suggested_lot_at_1k']:8} "
|
||||
f"(weight={r['risk_weight']:.2%}, pnl/trade=${r['pnl_per_trade']:.2f})"
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,53 @@
|
||||
"""Margin helpers — uses MT5 order_calc_margin for realistic portfolio sizing."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import MetaTrader5 as mt5
|
||||
|
||||
|
||||
def calc_margin(symbol: str, side: str, volume: float, price: float) -> float:
|
||||
ot = mt5.ORDER_TYPE_BUY if side == "BUY" else mt5.ORDER_TYPE_SELL
|
||||
m = mt5.order_calc_margin(ot, symbol, volume, price)
|
||||
return float(m) if m is not None and m > 0 else 0.0
|
||||
|
||||
|
||||
def max_lot_for_margin(
|
||||
symbol: str,
|
||||
side: str,
|
||||
price: float,
|
||||
free_margin: float,
|
||||
leverage: int = 0,
|
||||
) -> float:
|
||||
"""Binary search max lot that fits in free_margin (with 5% buffer)."""
|
||||
info = mt5.symbol_info(symbol)
|
||||
if info is None or free_margin <= 0:
|
||||
return 0.0
|
||||
vmin = float(info.volume_min)
|
||||
vmax = float(info.volume_max)
|
||||
step = float(info.volume_step) or vmin
|
||||
budget = free_margin * 0.95
|
||||
lo, hi = vmin, vmax
|
||||
best = 0.0
|
||||
for _ in range(24):
|
||||
mid = (lo + hi) / 2
|
||||
m = calc_margin(symbol, side, mid, price)
|
||||
if m <= budget:
|
||||
best = mid
|
||||
lo = mid
|
||||
else:
|
||||
hi = mid
|
||||
if step > 0 and best > 0:
|
||||
best = max(vmin, (int(best / step)) * step)
|
||||
return best
|
||||
|
||||
|
||||
def normalize_volume(symbol: str, volume: float) -> float:
|
||||
info = mt5.symbol_info(symbol)
|
||||
if info is None:
|
||||
return volume
|
||||
vmin = float(info.volume_min)
|
||||
vmax = float(info.volume_max)
|
||||
step = float(info.volume_step) or vmin
|
||||
if step > 0:
|
||||
volume = (int(volume / step)) * step
|
||||
return max(vmin, min(vmax, volume))
|
||||
@@ -0,0 +1,213 @@
|
||||
"""Progressive portfolio build: combine 1, 2, 3 ... N strategies with lot optimization."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import random
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
|
||||
from .backtest_core import CostModel, build_report, load_bars, resolve_symbol
|
||||
from .engines import ENGINE_MAP
|
||||
from .strategy_registry import TF
|
||||
from .trace_log import TraceLog
|
||||
|
||||
|
||||
def _score_report(report) -> float:
|
||||
if report.total_trades < 5:
|
||||
return float("-inf")
|
||||
return report.sharpe * 0.6 + (report.net_profit / 1000.0) * 0.3 - report.max_drawdown_pct * 0.1
|
||||
|
||||
|
||||
def _align_equity_curves(curves: list[pd.Series], initial: float = 10_000.0) -> pd.Series:
|
||||
if not curves:
|
||||
return pd.Series([initial])
|
||||
idx = curves[0].index
|
||||
for c in curves[1:]:
|
||||
idx = idx.union(c.index)
|
||||
idx = idx.sort_values()
|
||||
combined = pd.Series(0.0, index=idx)
|
||||
for c in curves:
|
||||
delta = c - c.iloc[0]
|
||||
combined = combined.add(delta.reindex(idx, method="ffill").fillna(0.0), fill_value=0.0)
|
||||
return initial + combined
|
||||
|
||||
|
||||
def run_single_cached(
|
||||
spec: dict,
|
||||
params: dict,
|
||||
df: pd.DataFrame,
|
||||
sym: str,
|
||||
period_label: str,
|
||||
lot_mult: float = 1.0,
|
||||
) -> tuple[Any, pd.Series]:
|
||||
engine = ENGINE_MAP[spec["engine"]]
|
||||
lot = spec["lot"] * lot_mult
|
||||
costs = CostModel.for_symbol(sym)
|
||||
report = engine(df, sym, period_label, spec["id"], params, lot, costs)
|
||||
# Reconstruct equity from trades is hard; re-run stores equity internally.
|
||||
# Use monthly returns proxy: build flat equity from trade PnL timeline.
|
||||
eq = pd.Series(10_000.0, index=df.index)
|
||||
pnl = 0.0
|
||||
trade_idx = 0
|
||||
trades_sorted = sorted(
|
||||
getattr(report, "_trades", []) or [],
|
||||
key=lambda t: t.close_time if hasattr(t, "close_time") else "",
|
||||
)
|
||||
# Fallback: approximate equity from net profit linearly (weak) — engines don't export eq.
|
||||
# Better: patch engines to return equity. For now use per-strategy report sharpe weighting only.
|
||||
if report.total_trades > 0:
|
||||
step = report.net_profit / max(len(df), 1)
|
||||
eq = eq + np.arange(len(df)) * (step / len(df))
|
||||
return report, eq
|
||||
|
||||
|
||||
def backtest_portfolio(
|
||||
members: list[dict],
|
||||
period_label: str,
|
||||
start: str,
|
||||
end: str,
|
||||
lot_mults: dict[str, float] | None = None,
|
||||
data_cache: dict | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""members: list of {spec, params, lot_mult}"""
|
||||
lot_mults = lot_mults or {}
|
||||
data_cache = data_cache or {}
|
||||
curves: list[pd.Series] = []
|
||||
member_reports = []
|
||||
all_trades = []
|
||||
|
||||
start_dt = datetime.fromisoformat(start)
|
||||
end_dt = datetime.fromisoformat(end)
|
||||
|
||||
for m in members:
|
||||
spec = m["spec"]
|
||||
params = m["params"]
|
||||
sid = spec["id"]
|
||||
sym = resolve_symbol(spec["symbol"])
|
||||
cache_key = (sym, spec["tf"])
|
||||
if cache_key not in data_cache:
|
||||
data_cache[cache_key] = load_bars(sym, TF[spec["tf"]], start_dt, end_dt)
|
||||
df = data_cache[cache_key]
|
||||
engine = ENGINE_MAP[spec["engine"]]
|
||||
lot = spec["lot"] * lot_mults.get(sid, m.get("lot_mult", 1.0))
|
||||
costs = CostModel.for_symbol(sym)
|
||||
report = engine(df, sym, period_label, sid, params, lot, costs)
|
||||
member_reports.append(report)
|
||||
|
||||
# Build equity from trade close events on this df's index
|
||||
eq = pd.Series(10_000.0, index=df.index, dtype=float)
|
||||
running = 10_000.0
|
||||
# We don't have trade list in report — use net profit distributed at bar closes via worst_trades timing
|
||||
# Simpler: daily PnL from member net / days
|
||||
if report.total_trades > 0 and report.net_profit != 0:
|
||||
daily_ret = report.net_profit / len(df)
|
||||
eq = eq + pd.Series(np.cumsum([daily_ret] * len(df)), index=df.index)
|
||||
curves.append(eq)
|
||||
all_trades.append(report.total_trades)
|
||||
|
||||
combined_eq = _align_equity_curves(curves)
|
||||
combined_report = build_report(
|
||||
"portfolio",
|
||||
"MIXED",
|
||||
"H1",
|
||||
period_label,
|
||||
[],
|
||||
combined_eq,
|
||||
10_000.0,
|
||||
{"members": [m["spec"]["id"] for m in members]},
|
||||
)
|
||||
# Override with summed stats
|
||||
net = sum(r.net_profit for r in member_reports)
|
||||
trades = sum(r.total_trades for r in member_reports)
|
||||
sharpes = [r.sharpe for r in member_reports if r.total_trades >= 5]
|
||||
combined_report.net_profit = net
|
||||
combined_report.total_trades = trades
|
||||
combined_report.sharpe = float(np.mean(sharpes)) if sharpes else 0.0
|
||||
|
||||
return {
|
||||
"members": [m["spec"]["id"] for m in members],
|
||||
"member_reports": [r.to_dict() for r in member_reports],
|
||||
"net_profit": net,
|
||||
"total_trades": trades,
|
||||
"sharpe_proxy": combined_report.sharpe,
|
||||
"lot_mults": {m["spec"]["id"]: lot_mults.get(m["spec"]["id"], m.get("lot_mult", 1.0)) for m in members},
|
||||
}
|
||||
|
||||
|
||||
def optimize_portfolio_lots(
|
||||
members: list[dict],
|
||||
period_label: str,
|
||||
start: str,
|
||||
end: str,
|
||||
trials: int,
|
||||
rng: random.Random,
|
||||
log: TraceLog,
|
||||
) -> dict[str, Any]:
|
||||
best_mults = {m["spec"]["id"]: 1.0 for m in members}
|
||||
best = backtest_portfolio(members, period_label, start, end, best_mults)
|
||||
best_score = _score_proxy(best)
|
||||
|
||||
for n in range(1, trials + 1):
|
||||
mults = {sid: round(rng.uniform(0.25, 1.5), 2) for sid in best_mults}
|
||||
r = backtest_portfolio(members, period_label, start, end, mults)
|
||||
sc = _score_proxy(r)
|
||||
if sc > best_score:
|
||||
best_score = sc
|
||||
best = r
|
||||
best_mults = dict(mults)
|
||||
log.info(f" portfolio trial {n}/{trials}: NEW BEST net=${r['net_profit']:.0f} mults={mults}")
|
||||
|
||||
best["optimized_score"] = best_score
|
||||
return best
|
||||
|
||||
|
||||
def _score_proxy(portfolio_result: dict) -> float:
|
||||
net = portfolio_result["net_profit"]
|
||||
trades = portfolio_result["total_trades"]
|
||||
sh = portfolio_result.get("sharpe_proxy", 0.0)
|
||||
if trades < 5:
|
||||
return float("-inf")
|
||||
return sh * 0.6 + (net / 1000.0) * 0.3
|
||||
|
||||
|
||||
def build_progressive_portfolios(
|
||||
ranked_results: list[dict],
|
||||
period_label: str,
|
||||
start: str,
|
||||
end: str,
|
||||
trials_per_step: int,
|
||||
rng: random.Random,
|
||||
log: TraceLog,
|
||||
) -> list[dict]:
|
||||
"""ranked_results: sorted best-first, each has spec + optimized params."""
|
||||
steps: list[dict] = []
|
||||
members: list[dict] = []
|
||||
|
||||
for i, r in enumerate(ranked_results, 1):
|
||||
members.append({"spec": r["spec"], "params": r["optimized_params"], "lot_mult": 1.0})
|
||||
log.banner(f"PORTFOLIO STEP {i}/{len(ranked_results)}: +{r['spec']['id']}")
|
||||
log.info(f"members: {[m['spec']['id'] for m in members]}")
|
||||
|
||||
optimized = optimize_portfolio_lots(members, period_label, start, end, trials_per_step, rng, log)
|
||||
baseline = backtest_portfolio(members, period_label, start, end)
|
||||
|
||||
step = {
|
||||
"step": i,
|
||||
"member_ids": [m["spec"]["id"] for m in members],
|
||||
"baseline_net": baseline["net_profit"],
|
||||
"baseline_trades": baseline["total_trades"],
|
||||
"optimized_net": optimized["net_profit"],
|
||||
"optimized_trades": optimized["total_trades"],
|
||||
"lot_mults": optimized["lot_mults"],
|
||||
"member_reports": optimized["member_reports"],
|
||||
}
|
||||
steps.append(step)
|
||||
log.info(
|
||||
f"step {i}: baseline net=${baseline['net_profit']:.0f} -> "
|
||||
f"optimized net=${optimized['net_profit']:.0f} mults={optimized['lot_mults']}"
|
||||
)
|
||||
|
||||
return steps
|
||||
@@ -0,0 +1,303 @@
|
||||
"""
|
||||
Margin-aware portfolio simulator — merges strategy trades chronologically.
|
||||
|
||||
Rejects new entries when margin level would drop below min_margin_level_pct.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
|
||||
from .backtest_core import BacktestReport, CostModel, Trade, build_report, load_bars, resolve_symbol
|
||||
from .engines import ENGINE_MAP
|
||||
from .margin import calc_margin, normalize_volume
|
||||
from .united_registry import TF
|
||||
|
||||
|
||||
@dataclass
|
||||
class OpenPosition:
|
||||
strategy_id: str
|
||||
symbol: str
|
||||
side: str
|
||||
volume: float
|
||||
entry_price: float
|
||||
entry_time: Any
|
||||
margin: float
|
||||
|
||||
|
||||
@dataclass
|
||||
class PortfolioSimResult:
|
||||
members: list[str]
|
||||
lot_scales: dict[str, float]
|
||||
initial_balance: float
|
||||
net_profit: float
|
||||
total_trades: int
|
||||
rejected_margin: int
|
||||
min_margin_level_pct: float
|
||||
lowest_margin_level_pct: float
|
||||
max_drawdown_pct: float
|
||||
sharpe: float
|
||||
equity_curve: pd.Series = field(repr=False)
|
||||
member_reports: list[dict] = field(default_factory=list)
|
||||
|
||||
|
||||
def _scale_trade(t: Trade, scale: float) -> Trade:
|
||||
if scale == 1.0:
|
||||
return t
|
||||
return Trade(
|
||||
side=t.side,
|
||||
open_time=t.open_time,
|
||||
close_time=t.close_time,
|
||||
open_price=t.open_price,
|
||||
close_price=t.close_price,
|
||||
volume=t.volume * scale,
|
||||
profit=t.profit * scale,
|
||||
bars_held=t.bars_held,
|
||||
exit_reason=t.exit_reason,
|
||||
)
|
||||
|
||||
|
||||
def run_member_backtest(
|
||||
spec: dict,
|
||||
params: dict,
|
||||
lot: float,
|
||||
df: pd.DataFrame,
|
||||
sym: str,
|
||||
period_label: str,
|
||||
) -> BacktestReport:
|
||||
engine = ENGINE_MAP[spec["engine"]]
|
||||
costs = CostModel.for_symbol(sym)
|
||||
return engine(df, sym, period_label, spec["id"], params, lot, costs)
|
||||
|
||||
|
||||
def simulate_portfolio(
|
||||
members: list[dict],
|
||||
period_label: str,
|
||||
start: str,
|
||||
end: str,
|
||||
initial_balance: float = 1000.0,
|
||||
lot_scales: dict[str, float] | None = None,
|
||||
min_margin_level_pct: float = 150.0,
|
||||
data_cache: dict | None = None,
|
||||
) -> PortfolioSimResult:
|
||||
"""
|
||||
members: [{spec, params, lot}] — lot = nominal from 123.set
|
||||
lot_scales: per-strategy multiplier on nominal lot
|
||||
"""
|
||||
lot_scales = lot_scales or {}
|
||||
data_cache = data_cache or {}
|
||||
start_dt = datetime.fromisoformat(start)
|
||||
end_dt = datetime.fromisoformat(end)
|
||||
|
||||
events: list[tuple[Any, str, str, Trade]] = []
|
||||
member_reports: list[dict] = []
|
||||
|
||||
for m in members:
|
||||
spec = m["spec"]
|
||||
sid = spec["id"]
|
||||
sym = resolve_symbol(spec["symbol"])
|
||||
cache_key = (sym, spec["tf"])
|
||||
if cache_key not in data_cache:
|
||||
data_cache[cache_key] = load_bars(sym, TF[spec["tf"]], start_dt, end_dt)
|
||||
df = data_cache[cache_key]
|
||||
scale = lot_scales.get(sid, m.get("lot_scale", 1.0))
|
||||
lot = spec["lot"] * scale
|
||||
report = run_member_backtest(spec, m["params"], lot, df, sym, period_label)
|
||||
member_reports.append({**report.to_dict(), "lot_used": lot, "lot_scale": scale})
|
||||
for t in report.trades_list:
|
||||
events.append((t.open_time, "open", sid, t))
|
||||
events.append((t.close_time, "close", sid, t))
|
||||
|
||||
events.sort(key=lambda x: (pd.Timestamp(x[0]), 0 if x[1] == "close" else 1))
|
||||
|
||||
balance = initial_balance
|
||||
equity = initial_balance
|
||||
open_pos: dict[str, OpenPosition] = {}
|
||||
realized: list[Trade] = []
|
||||
rejected = 0
|
||||
equity_points: list[tuple[Any, float]] = [(events[0][0] if events else start_dt, initial_balance)]
|
||||
lowest_ml = 9999.0
|
||||
|
||||
for ts, kind, sid, raw_t in events:
|
||||
m = next(x for x in members if x["spec"]["id"] == sid)
|
||||
spec = m["spec"]
|
||||
sym = resolve_symbol(spec["symbol"])
|
||||
scale = lot_scales.get(sid, m.get("lot_scale", 1.0))
|
||||
t = _scale_trade(raw_t, scale / (raw_t.volume / spec["lot"]) if raw_t.volume else scale)
|
||||
|
||||
if kind == "close":
|
||||
key = f"{sid}"
|
||||
if key not in open_pos:
|
||||
continue
|
||||
op = open_pos.pop(key)
|
||||
balance += t.profit
|
||||
equity = balance + sum(
|
||||
calc_profit(op2.symbol, op2.side, op2.volume, op2.entry_price, t.close_price)
|
||||
for op2 in open_pos.values()
|
||||
if op2.symbol == sym
|
||||
)
|
||||
# simpler: balance only on close
|
||||
balance = equity_points[-1][1] + t.profit if equity_points else balance + t.profit
|
||||
realized.append(t)
|
||||
equity_points.append((ts, balance))
|
||||
continue
|
||||
|
||||
# open
|
||||
vol = normalize_volume(sym, spec["lot"] * lot_scales.get(sid, 1.0))
|
||||
if vol <= 0:
|
||||
rejected += 1
|
||||
continue
|
||||
margin_req = calc_margin(sym, t.side, vol, t.open_price)
|
||||
used_margin = sum(p.margin for p in open_pos.values())
|
||||
free = balance - used_margin
|
||||
if margin_req > free:
|
||||
rejected += 1
|
||||
continue
|
||||
new_used = used_margin + margin_req
|
||||
equity = balance # simplified
|
||||
ml = (equity / new_used * 100.0) if new_used > 0 else 9999.0
|
||||
if ml < min_margin_level_pct:
|
||||
rejected += 1
|
||||
continue
|
||||
lowest_ml = min(lowest_ml, ml)
|
||||
open_pos[sid] = OpenPosition(sid, sym, t.side, vol, t.open_price, ts, margin_req)
|
||||
|
||||
if not equity_points:
|
||||
equity_points = [(start_dt, initial_balance)]
|
||||
|
||||
eq = pd.Series(
|
||||
[p[1] for p in equity_points],
|
||||
index=pd.DatetimeIndex([p[0] for p in equity_points]),
|
||||
)
|
||||
combined = build_report(
|
||||
"portfolio",
|
||||
"MIXED",
|
||||
"H1",
|
||||
period_label,
|
||||
realized,
|
||||
eq,
|
||||
initial_balance,
|
||||
{"members": [m["spec"]["id"] for m in members], "lot_scales": lot_scales},
|
||||
)
|
||||
|
||||
return PortfolioSimResult(
|
||||
members=[m["spec"]["id"] for m in members],
|
||||
lot_scales={m["spec"]["id"]: lot_scales.get(m["spec"]["id"], 1.0) for m in members},
|
||||
initial_balance=initial_balance,
|
||||
net_profit=combined.net_profit,
|
||||
total_trades=len(realized),
|
||||
rejected_margin=rejected,
|
||||
min_margin_level_pct=min_margin_level_pct,
|
||||
lowest_margin_level_pct=lowest_ml if lowest_ml < 9999 else 0.0,
|
||||
max_drawdown_pct=combined.max_drawdown_pct,
|
||||
sharpe=combined.sharpe,
|
||||
equity_curve=eq,
|
||||
member_reports=member_reports,
|
||||
)
|
||||
|
||||
|
||||
def optimize_lot_scales(
|
||||
members: list[dict],
|
||||
period_label: str,
|
||||
start: str,
|
||||
end: str,
|
||||
initial_balance: float,
|
||||
min_margin_level_pct: float,
|
||||
trials: int,
|
||||
rng,
|
||||
) -> PortfolioSimResult:
|
||||
"""Grid-search lot scales down from 1.0 — margin-safe, maximize net."""
|
||||
best_scales = {m["spec"]["id"]: 1.0 for m in members}
|
||||
best = simulate_portfolio(
|
||||
members, period_label, start, end, initial_balance, best_scales, min_margin_level_pct
|
||||
)
|
||||
best_score = _portfolio_score(best)
|
||||
|
||||
# Coarse: try uniform scale factors
|
||||
for factor in [1.0, 0.75, 0.5, 0.35, 0.25, 0.15, 0.1]:
|
||||
scales = {m["spec"]["id"]: factor for m in members}
|
||||
r = simulate_portfolio(members, period_label, start, end, initial_balance, scales, min_margin_level_pct)
|
||||
sc = _portfolio_score(r)
|
||||
if sc > best_score:
|
||||
best_score = sc
|
||||
best = r
|
||||
best_scales = dict(scales)
|
||||
|
||||
# Fine-tune per member around best uniform
|
||||
for _ in range(trials):
|
||||
scales = {}
|
||||
for m in members:
|
||||
sid = m["spec"]["id"]
|
||||
base = best_scales.get(sid, 1.0)
|
||||
scales[sid] = round(max(0.05, min(1.5, base * rng.uniform(0.7, 1.3))), 3)
|
||||
r = simulate_portfolio(members, period_label, start, end, initial_balance, scales, min_margin_level_pct)
|
||||
sc = _portfolio_score(r)
|
||||
if sc > best_score and r.lowest_margin_level_pct >= min_margin_level_pct * 0.9:
|
||||
best_score = sc
|
||||
best = r
|
||||
best_scales = dict(scales)
|
||||
|
||||
best.lot_scales = best_scales
|
||||
return best
|
||||
|
||||
|
||||
def _portfolio_score(r: PortfolioSimResult) -> float:
|
||||
if r.net_profit <= 0:
|
||||
return float("-inf")
|
||||
if r.lowest_margin_level_pct < r.min_margin_level_pct:
|
||||
return float("-inf")
|
||||
return r.sharpe * 0.4 + (r.net_profit / 500.0) * 0.4 - r.max_drawdown_pct * 0.15 - r.rejected_margin * 0.001
|
||||
|
||||
|
||||
def build_progressive_margin_portfolio(
|
||||
ranked: list[dict],
|
||||
period_label: str,
|
||||
start: str,
|
||||
end: str,
|
||||
initial_balance: float,
|
||||
min_margin_level_pct: float,
|
||||
trials_per_step: int,
|
||||
rng,
|
||||
) -> list[dict]:
|
||||
"""ranked: [{spec, params, lot, baseline_report}] sorted best-first."""
|
||||
steps: list[dict] = []
|
||||
members: list[dict] = []
|
||||
|
||||
for i, r in enumerate(ranked, 1):
|
||||
members.append({
|
||||
"spec": r["spec"],
|
||||
"params": r["params"],
|
||||
"lot_scale": 1.0,
|
||||
})
|
||||
baseline = simulate_portfolio(
|
||||
members, period_label, start, end, initial_balance,
|
||||
{m["spec"]["id"]: 1.0 for m in members}, min_margin_level_pct,
|
||||
)
|
||||
optimized = optimize_lot_scales(
|
||||
members, period_label, start, end, initial_balance,
|
||||
min_margin_level_pct, trials_per_step, rng,
|
||||
)
|
||||
steps.append({
|
||||
"step": i,
|
||||
"members": [m["spec"]["id"] for m in members],
|
||||
"baseline_net": baseline.net_profit,
|
||||
"baseline_trades": baseline.total_trades,
|
||||
"baseline_lowest_margin_pct": baseline.lowest_margin_level_pct,
|
||||
"optimized_net": optimized.net_profit,
|
||||
"optimized_trades": optimized.total_trades,
|
||||
"optimized_sharpe": optimized.sharpe,
|
||||
"optimized_max_dd_pct": optimized.max_drawdown_pct,
|
||||
"lowest_margin_level_pct": optimized.lowest_margin_level_pct,
|
||||
"rejected_margin": optimized.rejected_margin,
|
||||
"lot_scales": optimized.lot_scales,
|
||||
"lots_final": {
|
||||
m["spec"]["id"]: round(m["spec"]["lot"] * optimized.lot_scales.get(m["spec"]["id"], 1.0), 4)
|
||||
for m in members
|
||||
},
|
||||
})
|
||||
return steps
|
||||
@@ -0,0 +1 @@
|
||||
# Local MT5 audit outputs — see cluster_audit/*.py to regenerate.
|
||||
@@ -0,0 +1,646 @@
|
||||
"""
|
||||
|
||||
Full cluster audit: baseline + optimize per strategy per period.
|
||||
|
||||
Outputs JSON report with worst losses and improvement hints.
|
||||
|
||||
|
||||
|
||||
Usage:
|
||||
|
||||
python -m cluster_audit.run_audit [trials] [--trial-every N] [--quiet]
|
||||
|
||||
|
||||
|
||||
Examples:
|
||||
|
||||
python -m cluster_audit.run_audit 80
|
||||
|
||||
python -m cluster_audit.run_audit 120 --trial-every 5
|
||||
|
||||
"""
|
||||
|
||||
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
|
||||
|
||||
import argparse
|
||||
|
||||
import json
|
||||
|
||||
import random
|
||||
|
||||
import sys
|
||||
|
||||
import time
|
||||
|
||||
from datetime import datetime
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
|
||||
import MetaTrader5 as mt5
|
||||
|
||||
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
|
||||
|
||||
|
||||
|
||||
from cluster_audit.backtest_core import CostModel, load_bars, resolve_symbol
|
||||
|
||||
from cluster_audit.engines import ENGINE_MAP
|
||||
|
||||
from cluster_audit.strategy_registry import PERIODS, STRATEGIES, TF
|
||||
|
||||
from cluster_audit.trace_log import TraceLog
|
||||
|
||||
|
||||
|
||||
LOG = TraceLog(enabled=True, trial_every=10)
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
def _sample_params(defaults: dict, opt_ranges: dict, rng: random.Random) -> dict:
|
||||
|
||||
p = dict(defaults)
|
||||
|
||||
for key, spec in opt_ranges.items():
|
||||
|
||||
if not isinstance(spec, tuple) or len(spec) != 3:
|
||||
|
||||
continue
|
||||
|
||||
lo, hi, step = spec
|
||||
|
||||
if isinstance(lo, int):
|
||||
|
||||
vals = list(range(int(lo), int(hi) + 1, int(step)))
|
||||
|
||||
p[key] = rng.choice(vals) if vals else p.get(key, lo)
|
||||
|
||||
else:
|
||||
|
||||
n = int((hi - lo) / step) + 1
|
||||
|
||||
idx = rng.randint(0, max(n - 1, 0))
|
||||
|
||||
p[key] = round(lo + idx * step, 4)
|
||||
|
||||
return p
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
from cluster_audit.scoring import DEFAULT_TRADES_PER_DAY, period_days, score_label, score_report
|
||||
from cluster_audit.strategy_registry import PERIODS
|
||||
|
||||
|
||||
def _audit_period_days() -> int:
|
||||
start, end = PERIODS["2021-2026"]
|
||||
return period_days(start, end)
|
||||
|
||||
|
||||
def _score_label(score: float, trades: int) -> str:
|
||||
return score_label(score, trades, _audit_period_days(), DEFAULT_TRADES_PER_DAY)
|
||||
|
||||
|
||||
def _score(report) -> float:
|
||||
return score_report(report, _audit_period_days(), DEFAULT_TRADES_PER_DAY)
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
def run_strategy(
|
||||
|
||||
spec: dict,
|
||||
|
||||
period_label: str,
|
||||
|
||||
start: str,
|
||||
|
||||
end: str,
|
||||
|
||||
trials: int,
|
||||
|
||||
rng: random.Random,
|
||||
|
||||
run_idx: int,
|
||||
|
||||
run_total: int,
|
||||
|
||||
) -> dict:
|
||||
|
||||
sid = spec["id"]
|
||||
|
||||
label = f"{sid} @ {period_label} ({run_idx}/{run_total})"
|
||||
|
||||
LOG.phase_start(label)
|
||||
|
||||
|
||||
|
||||
engine_name = spec["engine"]
|
||||
|
||||
if engine_name not in ENGINE_MAP:
|
||||
|
||||
LOG.error(f"Unknown engine '{engine_name}'")
|
||||
|
||||
return {"id": sid, "period": period_label, "error": f"unknown engine {engine_name}"}
|
||||
|
||||
|
||||
|
||||
engine = ENGINE_MAP[engine_name]
|
||||
|
||||
tf_key = spec["tf"]
|
||||
|
||||
tf = TF[tf_key]
|
||||
|
||||
|
||||
|
||||
LOG.debug(f"resolve symbol: requested={spec['symbol']}")
|
||||
|
||||
raw_sym = spec["symbol"]
|
||||
|
||||
sym = resolve_symbol(raw_sym)
|
||||
|
||||
if sym != raw_sym:
|
||||
|
||||
LOG.info(f"symbol mapped {raw_sym} -> {sym}")
|
||||
|
||||
else:
|
||||
|
||||
LOG.debug(f"symbol={sym}")
|
||||
|
||||
|
||||
|
||||
start_dt = datetime.fromisoformat(start)
|
||||
|
||||
end_dt = datetime.fromisoformat(end)
|
||||
|
||||
LOG.debug(f"load bars: tf={tf_key} from={start} to={end} lot={spec['lot']}")
|
||||
|
||||
|
||||
|
||||
t_load = time.perf_counter()
|
||||
|
||||
try:
|
||||
|
||||
df = load_bars(sym, tf, start_dt, end_dt)
|
||||
|
||||
except Exception as e:
|
||||
|
||||
LOG.error(f"data load failed: {e}")
|
||||
|
||||
LOG.phase_end(label, "SKIPPED")
|
||||
|
||||
return {"id": sid, "period": period_label, "error": str(e)}
|
||||
|
||||
|
||||
|
||||
load_ms = (time.perf_counter() - t_load) * 1000
|
||||
|
||||
LOG.info(
|
||||
|
||||
f"loaded {len(df)} bars in {load_ms:.0f}ms "
|
||||
|
||||
f"({df.index[0]} -> {df.index[-1]})"
|
||||
|
||||
)
|
||||
|
||||
|
||||
|
||||
costs = CostModel.for_symbol(sym)
|
||||
|
||||
LOG.debug(
|
||||
|
||||
f"costs: spread={costs.spread_points}pts slippage={costs.slippage_points}pts "
|
||||
|
||||
f"commission/lot={costs.commission_per_lot}"
|
||||
|
||||
)
|
||||
|
||||
|
||||
|
||||
defaults = dict(spec["defaults"])
|
||||
|
||||
opt_keys = list(spec.get("opt", {}).keys())
|
||||
|
||||
LOG.debug(f"baseline params keys: {list(defaults.keys())}")
|
||||
|
||||
LOG.debug(f"optimize keys ({len(opt_keys)}): {opt_keys}")
|
||||
|
||||
|
||||
|
||||
LOG.debug("running baseline backtest...")
|
||||
|
||||
t0 = time.perf_counter()
|
||||
|
||||
baseline = engine(df, sym, period_label, sid, defaults, spec["lot"], costs)
|
||||
|
||||
base_ms = (time.perf_counter() - t0) * 1000
|
||||
|
||||
LOG.info(
|
||||
|
||||
f"baseline done in {base_ms:.0f}ms: net=${baseline.net_profit:.2f} "
|
||||
|
||||
f"sharpe={baseline.sharpe:.2f} trades={baseline.total_trades} "
|
||||
|
||||
f"pf={baseline.profit_factor:.2f} max_dd={baseline.max_drawdown_pct:.1f}%"
|
||||
|
||||
)
|
||||
|
||||
if baseline.total_trades == 0:
|
||||
|
||||
LOG.warn(f"{sid}: 0 trades on baseline — engine may not match MQL or params too strict")
|
||||
|
||||
|
||||
|
||||
best = baseline
|
||||
|
||||
best_params = defaults
|
||||
|
||||
best_score = _score(baseline)
|
||||
|
||||
LOG.debug(f"baseline score={_score_label(best_score, baseline.total_trades)}")
|
||||
|
||||
|
||||
|
||||
if trials > 0:
|
||||
|
||||
LOG.info(f"optimizing: {trials} random trials...")
|
||||
|
||||
for n in range(1, trials + 1):
|
||||
|
||||
params = _sample_params(defaults, spec.get("opt", {}), rng)
|
||||
|
||||
r = engine(df, sym, period_label, sid, params, spec["lot"], costs)
|
||||
|
||||
sc = _score(r)
|
||||
|
||||
improved = sc > best_score
|
||||
|
||||
if improved:
|
||||
|
||||
best_score = sc
|
||||
|
||||
best = r
|
||||
|
||||
best_params = params
|
||||
|
||||
LOG.trial(n, trials, sc, r.net_profit, r.sharpe, improved=True)
|
||||
|
||||
LOG.debug(f" new best params sample: { {k: best_params[k] for k in opt_keys[:6] if k in best_params} }")
|
||||
|
||||
else:
|
||||
|
||||
LOG.trial(n, trials, sc, r.net_profit, r.sharpe, improved=False)
|
||||
|
||||
|
||||
|
||||
if trials > 0 and best is not baseline:
|
||||
|
||||
LOG.info(
|
||||
|
||||
f"optimized: net +${best.net_profit - baseline.net_profit:.0f} "
|
||||
|
||||
f"sharpe +{best.sharpe - baseline.sharpe:.2f}"
|
||||
|
||||
)
|
||||
|
||||
elif trials > 0:
|
||||
|
||||
LOG.info("optimization: no better params found (baseline kept)")
|
||||
|
||||
|
||||
|
||||
loss_reasons: dict[str, float] = {}
|
||||
|
||||
for t in baseline.worst_trades:
|
||||
|
||||
reason = t.get("exit_reason", "?")
|
||||
|
||||
loss_reasons[reason] = loss_reasons.get(reason, 0) + min(t["profit"], 0)
|
||||
|
||||
if loss_reasons:
|
||||
|
||||
top_loss = sorted(loss_reasons.items(), key=lambda x: x[1])[:3]
|
||||
|
||||
LOG.debug(f"baseline loss by exit reason: {top_loss}")
|
||||
|
||||
|
||||
|
||||
result = {
|
||||
|
||||
"id": sid,
|
||||
|
||||
"engine": engine_name,
|
||||
|
||||
"symbol": sym,
|
||||
|
||||
"timeframe": tf_key,
|
||||
|
||||
"period": period_label,
|
||||
|
||||
"bars": len(df),
|
||||
|
||||
"baseline": baseline.to_dict(),
|
||||
|
||||
"optimized": {**best.to_dict(), "params": best_params},
|
||||
|
||||
"improvement_net": best.net_profit - baseline.net_profit,
|
||||
|
||||
"improvement_sharpe": best.sharpe - baseline.sharpe,
|
||||
|
||||
"loss_reasons_baseline": loss_reasons,
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
||||
LOG.report_line(sid, result["baseline"], result["optimized"], len(df))
|
||||
|
||||
LOG.phase_end(label)
|
||||
|
||||
return result
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
def build_improvement_plan(results: list[dict]) -> dict:
|
||||
|
||||
by_engine: dict[str, list] = {}
|
||||
|
||||
for r in results:
|
||||
|
||||
if "error" in r:
|
||||
|
||||
continue
|
||||
|
||||
by_engine.setdefault(r["engine"], []).append(r)
|
||||
|
||||
|
||||
|
||||
plan = {"per_engine": {}, "portfolio": []}
|
||||
|
||||
|
||||
|
||||
engine_notes = {
|
||||
|
||||
"rsi_crossover": "Trend-strong filter closes/blocks too aggressively (ema_slope 105 + distance 165). "
|
||||
|
||||
"Raise thresholds or only block entries, not force-exit. Add pip-based scaling per symbol.",
|
||||
|
||||
"rsi_scalp": "High trade count + rsi_against exits cause death by spread. Widen OB/OS gap, add ADX/session "
|
||||
|
||||
"filter, scale trail by ATR not fixed points. Stocks need .NAS symbols.",
|
||||
|
||||
"rsi_asian": "Narrow session + extreme RSI levels -> few trades or bad fills. Align session to broker server "
|
||||
|
||||
"time; add spread cap in points not pips.",
|
||||
|
||||
"mean_reversion": "ADX proxy weak in Python; large min_ema_distance on BTC blocks entries. "
|
||||
|
||||
"Use true ADX; cap concurrent positions; hard SL when ADX escapes.",
|
||||
|
||||
"ema_slope": "Re-enters on same crossover too often; weekly ADX filter missing in audit. "
|
||||
|
||||
"Add cooldown after loss; separate unit vs trail param sets.",
|
||||
|
||||
"darvas": "Box breakout without volume filter whipsaws. Add volume MA + trend MA filter from MQL.",
|
||||
|
||||
"rsi_secret": "Zone re-entry fires too often in chop. Require divergence or RSI momentum confirm.",
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
||||
for eng, rows in by_engine.items():
|
||||
|
||||
sharpes = [x["baseline"]["sharpe"] for x in rows]
|
||||
|
||||
opt_sharpes = [x["optimized"]["sharpe"] for x in rows]
|
||||
|
||||
nets = [x["baseline"]["net_profit"] for x in rows]
|
||||
|
||||
plan["per_engine"][eng] = {
|
||||
|
||||
"count": len(rows),
|
||||
|
||||
"avg_baseline_sharpe": sum(sharpes) / len(sharpes) if sharpes else 0,
|
||||
|
||||
"avg_optimized_sharpe": sum(opt_sharpes) / len(opt_sharpes) if opt_sharpes else 0,
|
||||
|
||||
"avg_baseline_net": sum(nets) / len(nets) if nets else 0,
|
||||
|
||||
"logic_fixes": engine_notes.get(eng, ""),
|
||||
|
||||
"worst_strategies": sorted(rows, key=lambda x: x["baseline"]["sharpe"])[:3],
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
||||
plan["portfolio"] = [
|
||||
|
||||
"Run correlation matrix on daily returns — disable highly correlated RSI scalps on same underlying (NVDA x3).",
|
||||
|
||||
"Portfolio-level max daily loss circuit breaker (pause new entries cluster-wide).",
|
||||
|
||||
"Per-asset-class lot caps: forex micro, gold 0.1, stocks margin-scaled not fixed 25 lots.",
|
||||
|
||||
"Split optimization windows: long 2021-2026 for structure, short 2024-2026 for recency; only deploy params that pass both.",
|
||||
|
||||
"Add core/ShockGuard.mqh: ATR spike pause + margin level gate before SE_TickAll.",
|
||||
|
||||
"Enable robots by regime: Asian RSI only 00-08 server; EMA slope only when W1 ADX > threshold.",
|
||||
|
||||
"Replace fixed magic collisions with 401xxx registry; log per-robot PnL for live attribution.",
|
||||
|
||||
]
|
||||
|
||||
return plan
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
|
||||
p = argparse.ArgumentParser(description="SuperEA cluster audit with trace logging")
|
||||
|
||||
p.add_argument("trials", nargs="?", type=int, default=80, help="Random trials per strategy (default 80)")
|
||||
|
||||
p.add_argument("--trial-every", type=int, default=10, help="Log every N trials (default 10)")
|
||||
|
||||
p.add_argument("--quiet", action="store_true", help="Suppress trace output")
|
||||
|
||||
return p.parse_args()
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
def main() -> None:
|
||||
|
||||
args = parse_args()
|
||||
|
||||
global LOG
|
||||
|
||||
LOG = TraceLog(enabled=not args.quiet, trial_every=args.trial_every)
|
||||
|
||||
|
||||
|
||||
total_runs = len(STRATEGIES) * len(PERIODS)
|
||||
|
||||
LOG.banner(
|
||||
|
||||
f"CLUSTER AUDIT - {len(STRATEGIES)} strategies x {len(PERIODS)} periods "
|
||||
|
||||
f"= {total_runs} runs, {args.trials} trials each"
|
||||
|
||||
)
|
||||
|
||||
|
||||
|
||||
LOG.phase_start("MT5 initialize")
|
||||
|
||||
if not mt5.initialize():
|
||||
|
||||
LOG.error(f"MT5 init failed: {mt5.last_error()}")
|
||||
|
||||
raise SystemExit(1)
|
||||
|
||||
acc = mt5.account_info()
|
||||
|
||||
if acc:
|
||||
|
||||
LOG.info(f"MT5 connected: server={acc.server} (account redacted)")
|
||||
|
||||
LOG.phase_end("MT5 initialize")
|
||||
|
||||
|
||||
|
||||
out_dir = Path(__file__).parent / "reports"
|
||||
|
||||
out_dir.mkdir(exist_ok=True)
|
||||
|
||||
rng = random.Random(42)
|
||||
|
||||
all_results = []
|
||||
|
||||
run_idx = 0
|
||||
|
||||
errors = 0
|
||||
|
||||
|
||||
|
||||
try:
|
||||
|
||||
for period_label, (start, end) in PERIODS.items():
|
||||
|
||||
LOG.banner(f"PERIOD {period_label} ({start} -> {end})")
|
||||
|
||||
for spec in STRATEGIES:
|
||||
|
||||
run_idx += 1
|
||||
|
||||
LOG.progress(run_idx, total_runs, f"next: {spec['id']}")
|
||||
|
||||
r = run_strategy(spec, period_label, start, end, args.trials, rng, run_idx, total_runs)
|
||||
|
||||
all_results.append(r)
|
||||
|
||||
if "error" in r:
|
||||
|
||||
errors += 1
|
||||
|
||||
|
||||
|
||||
LOG.phase_start("build improvement plan")
|
||||
|
||||
plan = build_improvement_plan(all_results)
|
||||
|
||||
LOG.phase_end("build improvement plan")
|
||||
|
||||
|
||||
|
||||
report = {
|
||||
|
||||
"generated": datetime.now().isoformat(),
|
||||
|
||||
"trials_per_strategy": args.trials,
|
||||
|
||||
"strategies": len(STRATEGIES),
|
||||
|
||||
"periods": list(PERIODS.keys()),
|
||||
|
||||
"runs_total": total_runs,
|
||||
|
||||
"runs_failed": errors,
|
||||
|
||||
"results": all_results,
|
||||
|
||||
"improvement_plan": plan,
|
||||
|
||||
}
|
||||
|
||||
out_path = out_dir / "cluster_audit_report.json"
|
||||
|
||||
out_path.write_text(json.dumps(report, indent=2), encoding="utf-8")
|
||||
|
||||
LOG.info(f"report written: {out_path} ({out_path.stat().st_size // 1024} KB)")
|
||||
|
||||
|
||||
|
||||
LOG.banner("SUMMARY - worst baseline Sharpe (2021-2026)")
|
||||
|
||||
r21 = [x for x in all_results if x.get("period") == "2021-2026" and "error" not in x]
|
||||
|
||||
for x in sorted(r21, key=lambda z: z["baseline"]["sharpe"])[:10]:
|
||||
|
||||
b = x["baseline"]
|
||||
|
||||
w = b["worst_trades"][:1]
|
||||
|
||||
wtxt = f"${w[0]['profit']:.0f} {w[0]['exit_reason']}" if w else "n/a"
|
||||
|
||||
LOG.info(
|
||||
|
||||
f" {x['id']:28} sharpe={b['sharpe']:6.2f} net=${b['net_profit']:9.0f} "
|
||||
|
||||
f"trades={b['total_trades']:4} worst={wtxt}"
|
||||
|
||||
)
|
||||
|
||||
|
||||
|
||||
if errors:
|
||||
|
||||
LOG.warn(f"{errors}/{total_runs} runs failed - search report for \"error\" fields")
|
||||
LOG.banner(f"DONE - {run_idx} runs in {LOG._elapsed()}")
|
||||
|
||||
finally:
|
||||
|
||||
LOG.phase_start("MT5 shutdown")
|
||||
|
||||
mt5.shutdown()
|
||||
|
||||
LOG.phase_end("MT5 shutdown")
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
main()
|
||||
|
||||
|
||||
@@ -0,0 +1,277 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
United EA MT5 audit: per-strategy solo runs, close-unprofitable A/B, lot/margin check.
|
||||
|
||||
Usage:
|
||||
python -m cluster_audit.run_close_signal_audit
|
||||
python -m cluster_audit.run_close_signal_audit --only DB
|
||||
python -m cluster_audit.run_close_signal_audit --from RS_NVDA --combo
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
|
||||
|
||||
from set_parser import parse_set_file
|
||||
|
||||
from cluster_audit.united_mt5_manifest import (
|
||||
ALL_ENABLE_KEYS,
|
||||
PARAM_TWEAKS,
|
||||
PRODUCTION_IDS,
|
||||
UNITED_MT5_STRATEGIES,
|
||||
)
|
||||
from cluster_audit.united_mt5_runner import (
|
||||
BASE_SET,
|
||||
deploy_united,
|
||||
mt5_context,
|
||||
patch_set,
|
||||
run_backtest,
|
||||
)
|
||||
|
||||
OUT = Path(__file__).parent / "reports" / "close_signal_audit"
|
||||
LOT_SUMMARY = Path(__file__).parent / "reports" / "lot_genetic" / "lot_genetic_summary.json"
|
||||
REF_BALANCE = 3000.0
|
||||
|
||||
|
||||
def load_best_lots() -> dict[str, float]:
|
||||
if not LOT_SUMMARY.exists():
|
||||
return {}
|
||||
data = json.loads(LOT_SUMMARY.read_text(encoding="utf-8"))
|
||||
return {k: float(v) for k, v in data.get("best_lots", {}).items()}
|
||||
|
||||
|
||||
def solo_patches(spec: dict, *, close_on: bool, lots: dict[str, float]) -> dict:
|
||||
ov = solo_overrides(spec, close_on=close_on)
|
||||
ov["ORCH_ReferenceBalance"] = REF_BALANCE
|
||||
ov["ORCH_ScaleLotsByBalance"] = True
|
||||
if spec.get("lot") in lots:
|
||||
ov[spec["lot"]] = lots[spec["lot"]]
|
||||
return ov
|
||||
|
||||
|
||||
def solo_overrides(target: dict, *, close_on: bool) -> dict[str, bool]:
|
||||
o: dict[str, bool] = {k: False for k in ALL_ENABLE_KEYS}
|
||||
o[target["enable"]] = True
|
||||
o[target["close"]] = close_on
|
||||
o["GAP_Enable"] = False
|
||||
o["OPT_GuardOptimizationMode"] = True
|
||||
return o
|
||||
|
||||
|
||||
def audit_one(ctx: dict, spec: dict, base_params: dict, lots: dict[str, float]) -> dict:
|
||||
sid = spec["id"]
|
||||
print(f"\n{'='*60}\n[{sid}] {spec['name']}\n{'='*60}", flush=True)
|
||||
|
||||
off_ov = solo_patches(spec, close_on=False, lots=lots)
|
||||
on_ov = solo_patches(spec, close_on=True, lots=lots)
|
||||
|
||||
off_body = patch_set(BASE_SET, off_ov)
|
||||
on_body = patch_set(BASE_SET, on_ov)
|
||||
ts = spec.get("test_symbol")
|
||||
|
||||
off = run_backtest(
|
||||
ctx["data"], ctx["mt5_path"], ctx["login"], ctx["server"],
|
||||
off_body, f"solo_{sid}_off.set", f"solo_{sid}_off",
|
||||
test_symbol=ts,
|
||||
)
|
||||
print(f" OFF PF={off.get('profit_factor')} net={off.get('net_profit')} "
|
||||
f"trades={off.get('total_trades')} sharpe={off.get('sharpe')} "
|
||||
f"ready={off.get('ready')} ({off.get('elapsed_sec')}s)", flush=True)
|
||||
|
||||
on = run_backtest(
|
||||
ctx["data"], ctx["mt5_path"], ctx["login"], ctx["server"],
|
||||
on_body, f"solo_{sid}_on.set", f"solo_{sid}_on",
|
||||
test_symbol=ts,
|
||||
)
|
||||
print(f" ON PF={on.get('profit_factor')} net={on.get('net_profit')} "
|
||||
f"trades={on.get('total_trades')} sharpe={on.get('sharpe')} "
|
||||
f"ready={on.get('ready')} ({on.get('elapsed_sec')}s)", flush=True)
|
||||
|
||||
d = delta_metrics(off, on)
|
||||
v = verdict(d, off, on)
|
||||
print(f" -> {v} dNet={d['net_profit_delta']:.0f} dSharpe={d['sharpe_delta']:.3f} "
|
||||
f"dTrades={d['trades_delta']}", flush=True)
|
||||
|
||||
tweaks: list[dict] = []
|
||||
if v == "POSITIVE" and sid in PARAM_TWEAKS:
|
||||
for i, tw in enumerate(PARAM_TWEAKS[sid], 1):
|
||||
ov = {**on_ov, **tw}
|
||||
body = patch_set(BASE_SET, ov)
|
||||
r = run_backtest(
|
||||
ctx["data"], ctx["mt5_path"], ctx["login"], ctx["server"],
|
||||
body, f"solo_{sid}_tw{i}.set", f"solo_{sid}_tw{i}",
|
||||
test_symbol=ts,
|
||||
)
|
||||
tweaks.append({"tweak": tw, "metrics": r})
|
||||
print(f" tweak{i} net={r.get('net_profit')} sharpe={r.get('sharpe')} "
|
||||
f"trades={r.get('total_trades')}", flush=True)
|
||||
|
||||
lot_key = spec["lot"]
|
||||
lot_val = lots.get(lot_key)
|
||||
if lot_val is None and lot_key in base_params:
|
||||
lot_val = base_params[lot_key].value
|
||||
|
||||
return {
|
||||
"id": sid,
|
||||
"name": spec["name"],
|
||||
"enable": spec["enable"],
|
||||
"close_key": spec["close"],
|
||||
"lot_key": lot_key,
|
||||
"lot_value": lot_val,
|
||||
"close_off": off,
|
||||
"close_on": on,
|
||||
"delta": d,
|
||||
"verdict": v,
|
||||
"margin_ok_off": margin_ok(off),
|
||||
"margin_ok_on": margin_ok(on),
|
||||
"param_tweaks": tweaks,
|
||||
"recommend_close_on": v == "POSITIVE",
|
||||
}
|
||||
|
||||
|
||||
def delta_metrics(off: dict, on: dict) -> dict:
|
||||
def g(d, k):
|
||||
v = d.get(k)
|
||||
return v if v is not None else 0
|
||||
|
||||
return {
|
||||
"net_profit_delta": g(on, "net_profit") - g(off, "net_profit"),
|
||||
"pf_delta": (g(on, "profit_factor") - g(off, "profit_factor")),
|
||||
"sharpe_delta": g(on, "sharpe") - g(off, "sharpe"),
|
||||
"trades_delta": g(on, "total_trades") - g(off, "total_trades"),
|
||||
}
|
||||
|
||||
|
||||
def verdict(delta: dict, off: dict, on: dict) -> str:
|
||||
if not off.get("ready") or not on.get("ready"):
|
||||
return "BROKEN"
|
||||
if off.get("total_trades", 0) == 0 and on.get("total_trades", 0) == 0:
|
||||
return "NO_TRADES"
|
||||
if delta["net_profit_delta"] > 50 and delta["sharpe_delta"] >= 0:
|
||||
return "POSITIVE"
|
||||
if delta["net_profit_delta"] < -50 or delta["sharpe_delta"] < -0.1:
|
||||
return "NEGATIVE"
|
||||
return "NEUTRAL"
|
||||
|
||||
|
||||
def margin_ok(m: dict) -> bool:
|
||||
ml = m.get("min_margin_level")
|
||||
if not ml:
|
||||
return True
|
||||
if isinstance(ml, str) and "%" in ml:
|
||||
try:
|
||||
return float(ml.replace("%", "").strip()) >= 100.0
|
||||
except ValueError:
|
||||
return True
|
||||
return True
|
||||
|
||||
|
||||
def run_combo(ctx: dict, winners: list[dict], base_params: dict) -> dict:
|
||||
"""Combo: enable all strategies that benefit from close-on, with flag set."""
|
||||
overrides: dict = {k: False for k in ALL_ENABLE_KEYS}
|
||||
overrides["GAP_Enable"] = False
|
||||
for w in winners:
|
||||
overrides[w["enable"]] = True
|
||||
overrides[w["close_key"]] = True
|
||||
for spec in UNITED_MT5_STRATEGIES:
|
||||
if spec["enable"] not in overrides or not overrides[spec["enable"]]:
|
||||
continue
|
||||
if spec["id"] not in {w["id"] for w in winners}:
|
||||
overrides[spec["close_key"]] = False
|
||||
|
||||
body = patch_set(BASE_SET, overrides)
|
||||
m = run_backtest(
|
||||
ctx["data"], ctx["mt5_path"], ctx["login"], ctx["server"],
|
||||
body, "combo_close_winners.set", "combo_close_winners",
|
||||
)
|
||||
return {"members": [w["id"] for w in winners], "metrics": m}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
p = argparse.ArgumentParser()
|
||||
p.add_argument("--only", default=None)
|
||||
p.add_argument("--from", dest="from_id", default=None)
|
||||
p.add_argument("--production", action="store_true", help="Only PRODUCTION_IDS (19 strategies)")
|
||||
p.add_argument("--combo", action="store_true")
|
||||
p.add_argument("--enabled-only", action="store_true", help="Only strategies enabled in 123.set")
|
||||
args = p.parse_args()
|
||||
|
||||
OUT.mkdir(parents=True, exist_ok=True)
|
||||
base_params = parse_set_file(BASE_SET)
|
||||
lots = load_best_lots()
|
||||
sm = {s["id"]: s for s in UNITED_MT5_STRATEGIES}
|
||||
|
||||
if args.production:
|
||||
strategies = [sm[sid] for sid in PRODUCTION_IDS if sid in sm]
|
||||
elif args.enabled_only:
|
||||
strategies = [
|
||||
s for s in UNITED_MT5_STRATEGIES
|
||||
if base_params.get(s["enable"], type("x", (), {"value": False})).value
|
||||
]
|
||||
else:
|
||||
strategies = list(UNITED_MT5_STRATEGIES)
|
||||
|
||||
if args.only:
|
||||
strategies = [s for s in strategies if s["id"] == args.only]
|
||||
elif args.from_id:
|
||||
found = False
|
||||
filtered = []
|
||||
for s in strategies:
|
||||
if s["id"] == args.from_id:
|
||||
found = True
|
||||
if found:
|
||||
filtered.append(s)
|
||||
strategies = filtered if found else strategies
|
||||
|
||||
print(f"United EA close-signal audit | {len(strategies)} strategies | lots={len(lots)}")
|
||||
import cluster_audit.united_mt5_runner as runner
|
||||
runner.DEPOSIT = int(REF_BALANCE)
|
||||
ctx = mt5_context()
|
||||
print(f"MT5 server={ctx['server']} (account from local terminal)")
|
||||
deploy_united(ctx["data"], ctx["mt5_path"])
|
||||
print("Compiled main.ex5 OK")
|
||||
|
||||
results: list[dict] = []
|
||||
for spec in strategies:
|
||||
try:
|
||||
results.append(audit_one(ctx, spec, base_params, lots))
|
||||
except Exception as ex:
|
||||
print(f" ERROR {spec['id']}: {ex}", flush=True)
|
||||
results.append({"id": spec["id"], "error": str(ex), "verdict": "ERROR"})
|
||||
|
||||
positive = [r for r in results if r.get("verdict") == "POSITIVE"]
|
||||
negative = [r for r in results if r.get("verdict") == "NEGATIVE"]
|
||||
broken = [r for r in results if r.get("verdict") in ("BROKEN", "NO_TRADES", "ERROR")]
|
||||
|
||||
combo_result = None
|
||||
if args.combo and positive:
|
||||
combo_result = run_combo(ctx, positive, base_params)
|
||||
print(f"\nCOMBO winners ({len(positive)}): net={combo_result['metrics'].get('net_profit')} "
|
||||
f"sharpe={combo_result['metrics'].get('sharpe')}")
|
||||
|
||||
summary = {
|
||||
"generated": datetime.now().isoformat(),
|
||||
"base_set": str(BASE_SET),
|
||||
"positive_close_on": [r["id"] for r in positive],
|
||||
"negative_close_on": [r["id"] for r in negative],
|
||||
"broken": [r["id"] for r in broken],
|
||||
"results": results,
|
||||
"combo": combo_result,
|
||||
}
|
||||
out_path = OUT / "summary.json"
|
||||
out_path.write_text(json.dumps(summary, indent=2, default=str), encoding="utf-8")
|
||||
print(f"\nSaved {out_path}")
|
||||
print(f"POSITIVE ({len(positive)}): {[r['id'] for r in positive]}")
|
||||
print(f"NEGATIVE ({len(negative)}): {[r['id'] for r in negative]}")
|
||||
print(f"BROKEN ({len(broken)}): {[r['id'] for r in broken]}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,198 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Solo MT5 audit for all currently-disabled sub-strategies (main.mq5 enable=false).
|
||||
|
||||
Finds profitable passers to add to the cluster; compares enhanced portfolio vs production baseline.
|
||||
|
||||
Usage:
|
||||
python -m cluster_audit.run_disabled_audit
|
||||
python -m cluster_audit.run_disabled_audit --min-trades 40
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
|
||||
|
||||
from cluster_audit.united_mt5_manifest import (
|
||||
ALL_ENABLE_KEYS,
|
||||
HIGH_MARGIN_STOCK_ENABLES,
|
||||
PRODUCTION_IDS,
|
||||
UNITED_MT5_STRATEGIES,
|
||||
)
|
||||
from cluster_audit.united_mt5_runner import (
|
||||
BASE_SET,
|
||||
CLUSTER,
|
||||
FROM_DATE,
|
||||
TO_DATE,
|
||||
deploy_united,
|
||||
mt5_context,
|
||||
patch_set,
|
||||
run_backtest,
|
||||
)
|
||||
|
||||
OUT = Path(__file__).resolve().parent / "reports" / "disabled_audit"
|
||||
REF_BALANCE = 3000.0
|
||||
|
||||
|
||||
def g(m: dict, k: str) -> float:
|
||||
v = m.get(k)
|
||||
return float(v) if v is not None else 0.0
|
||||
|
||||
|
||||
def parse_dd_pct(dd: str | None) -> float | None:
|
||||
if not dd:
|
||||
return None
|
||||
m = re.search(r"([\d.]+)\s*%", dd.replace(",", ""))
|
||||
return float(m.group(1)) if m else None
|
||||
|
||||
|
||||
def disabled_ids_from_mq5() -> list[str]:
|
||||
text = (CLUSTER / "main.mq5").read_text(encoding="utf-8")
|
||||
off: set[str] = set()
|
||||
for m in re.finditer(r"input bool (Enable\w+) = false", text):
|
||||
off.add(m.group(1))
|
||||
for key in HIGH_MARGIN_STOCK_ENABLES:
|
||||
off.discard(key)
|
||||
ids: list[str] = []
|
||||
for s in UNITED_MT5_STRATEGIES:
|
||||
if s["enable"] in off:
|
||||
ids.append(s["id"])
|
||||
return ids
|
||||
|
||||
|
||||
def common_patches() -> dict[str, float | bool]:
|
||||
return {
|
||||
"ORCH_ReferenceBalance": REF_BALANCE,
|
||||
"ORCH_ScaleLotsByBalance": True,
|
||||
"GAP_Enable": False,
|
||||
"OPT_GuardOptimizationMode": True,
|
||||
}
|
||||
|
||||
|
||||
def production_enables() -> dict[str, bool]:
|
||||
prod = set(PRODUCTION_IDS)
|
||||
o: dict[str, bool] = {}
|
||||
for s in UNITED_MT5_STRATEGIES:
|
||||
o[s["enable"]] = s["id"] in prod
|
||||
for key in HIGH_MARGIN_STOCK_ENABLES:
|
||||
o[key] = False
|
||||
return o
|
||||
|
||||
|
||||
def solo_overrides(spec: dict) -> dict[str, bool]:
|
||||
o: dict[str, bool] = {k: False for k in ALL_ENABLE_KEYS}
|
||||
o[spec["enable"]] = True
|
||||
return o
|
||||
|
||||
|
||||
def classify(m: dict, min_trades: int) -> str:
|
||||
if not m.get("ready"):
|
||||
return "BROKEN"
|
||||
trades = int(m.get("total_trades") or 0)
|
||||
if trades < min_trades:
|
||||
return "LOW_TRADES"
|
||||
if g(m, "net_profit") > 0 and g(m, "profit_factor") >= 1.05:
|
||||
return "PASS"
|
||||
if g(m, "net_profit") > 50 and g(m, "profit_factor") >= 1.0:
|
||||
return "MARGINAL"
|
||||
return "FAIL"
|
||||
|
||||
|
||||
def main() -> None:
|
||||
p = argparse.ArgumentParser()
|
||||
p.add_argument("--from", dest="from_date", default=FROM_DATE)
|
||||
p.add_argument("--to", dest="to_date", default=TO_DATE)
|
||||
p.add_argument("--min-trades", type=int, default=60)
|
||||
args = p.parse_args()
|
||||
|
||||
import cluster_audit.united_mt5_runner as runner
|
||||
|
||||
runner.FROM_DATE = args.from_date.replace("-", ".")
|
||||
runner.TO_DATE = args.to_date.replace("-", ".")
|
||||
runner.DEPOSIT = int(REF_BALANCE)
|
||||
|
||||
sm = {s["id"]: s for s in UNITED_MT5_STRATEGIES}
|
||||
disabled = disabled_ids_from_mq5()
|
||||
OUT.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
ctx = mt5_context()
|
||||
deploy_united(ctx["data"], ctx["mt5_path"])
|
||||
|
||||
print(
|
||||
f"Disabled audit n={len(disabled)} production={PRODUCTION_IDS} "
|
||||
f"{runner.FROM_DATE}->{runner.TO_DATE}",
|
||||
flush=True,
|
||||
)
|
||||
|
||||
prod_ov = {**common_patches(), **production_enables()}
|
||||
baseline = run_backtest(
|
||||
ctx["data"], ctx["mt5_path"], ctx["login"], ctx["server"],
|
||||
patch_set(BASE_SET, prod_ov), "dis_prod_baseline.set", "dis_prod_baseline",
|
||||
)
|
||||
print(
|
||||
f"PROD baseline PF={baseline.get('profit_factor')} net={baseline.get('net_profit')} "
|
||||
f"sharpe={baseline.get('sharpe')} dd={baseline.get('max_drawdown')}",
|
||||
flush=True,
|
||||
)
|
||||
|
||||
solo_rows: list[dict] = []
|
||||
passed_ids: list[str] = []
|
||||
|
||||
for sid in disabled:
|
||||
spec = sm[sid]
|
||||
m = run_backtest(
|
||||
ctx["data"], ctx["mt5_path"], ctx["login"], ctx["server"],
|
||||
patch_set(BASE_SET, {**common_patches(), **solo_overrides(spec)}),
|
||||
f"dis_solo_{sid}.set", f"dis_solo_{sid}",
|
||||
test_symbol=spec.get("test_symbol"),
|
||||
)
|
||||
verdict = classify(m, args.min_trades)
|
||||
dd_pct = parse_dd_pct(m.get("max_drawdown"))
|
||||
print(
|
||||
f" {sid:12} {verdict:10} PF={m.get('profit_factor')} net={m.get('net_profit')} "
|
||||
f"sharpe={m.get('sharpe')} trades={m.get('total_trades')} dd={m.get('max_drawdown')}",
|
||||
flush=True,
|
||||
)
|
||||
row = {"id": sid, "enable": spec["enable"], "verdict": verdict, "dd_pct": dd_pct, "metrics": m}
|
||||
solo_rows.append(row)
|
||||
if verdict in ("PASS", "MARGINAL"):
|
||||
passed_ids.append(sid)
|
||||
|
||||
enhanced_ov = dict(prod_ov)
|
||||
for sid in passed_ids:
|
||||
enhanced_ov[sm[sid]["enable"]] = True
|
||||
enhanced = run_backtest(
|
||||
ctx["data"], ctx["mt5_path"], ctx["login"], ctx["server"],
|
||||
patch_set(BASE_SET, enhanced_ov), "dis_enhanced.set", "dis_enhanced",
|
||||
)
|
||||
print(
|
||||
f"ENHANCED +{len(passed_ids)} PF={enhanced.get('profit_factor')} net={enhanced.get('net_profit')} "
|
||||
f"sharpe={enhanced.get('sharpe')} dd={enhanced.get('max_drawdown')}",
|
||||
flush=True,
|
||||
)
|
||||
print(f"PASS/MARGINAL: {passed_ids}", flush=True)
|
||||
|
||||
summary = {
|
||||
"timestamp": datetime.now().isoformat(timespec="seconds"),
|
||||
"period": {"from": runner.FROM_DATE, "to": runner.TO_DATE},
|
||||
"min_trades": args.min_trades,
|
||||
"disabled_ids": disabled,
|
||||
"production_baseline": baseline,
|
||||
"solo": solo_rows,
|
||||
"passed_ids": passed_ids,
|
||||
"enhanced": enhanced,
|
||||
}
|
||||
path = OUT / "disabled_audit_summary.json"
|
||||
path.write_text(json.dumps(summary, indent=2), encoding="utf-8")
|
||||
print(f"Saved {path}", flush=True)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,233 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Symbol mining audit for cluster-latest (round 2 = indices, round 3 = low-margin stocks).
|
||||
|
||||
Baseline = 123.set + survivors only; solo each candidate round; enhanced = baseline + passers.
|
||||
|
||||
Usage:
|
||||
python -m cluster_audit.run_hedge_audit --round 2
|
||||
python -m cluster_audit.run_hedge_audit --round 3
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
|
||||
|
||||
from cluster_audit.united_mt5_manifest import (
|
||||
ALL_ENABLE_KEYS,
|
||||
EXPANSION_RETIRED_IDS,
|
||||
HIGH_MARGIN_STOCK_ENABLES,
|
||||
ROUND2_IDS,
|
||||
ROUND3_IDS,
|
||||
SURVIVOR_IDS,
|
||||
UNITED_MT5_STRATEGIES,
|
||||
)
|
||||
from cluster_audit.united_mt5_runner import (
|
||||
BASE_SET,
|
||||
FROM_DATE,
|
||||
TO_DATE,
|
||||
deploy_united,
|
||||
mt5_context,
|
||||
patch_set,
|
||||
run_backtest,
|
||||
)
|
||||
|
||||
OUT = Path(__file__).resolve().parent / "reports" / "hedge_audit"
|
||||
|
||||
MIN_TRADES_DEFAULT = 60
|
||||
|
||||
ROUND_CONFIG = {
|
||||
2: {"candidate_ids": ROUND2_IDS, "prefix": "r2"},
|
||||
3: {"candidate_ids": ROUND3_IDS, "prefix": "r3"},
|
||||
}
|
||||
|
||||
|
||||
def g(m: dict, k: str) -> float:
|
||||
v = m.get(k)
|
||||
return float(v) if v is not None else 0.0
|
||||
|
||||
|
||||
def spec_map() -> dict[str, dict]:
|
||||
return {s["id"]: s for s in UNITED_MT5_STRATEGIES}
|
||||
|
||||
|
||||
def candidate_ids_for_round(round_num: int) -> tuple[str, ...]:
|
||||
return ROUND_CONFIG[round_num]["candidate_ids"]
|
||||
|
||||
|
||||
def baseline_overrides(round_num: int) -> dict[str, bool]:
|
||||
o: dict[str, bool] = {}
|
||||
retired = set(EXPANSION_RETIRED_IDS)
|
||||
survivors = set(SURVIVOR_IDS)
|
||||
candidates = set(candidate_ids_for_round(round_num))
|
||||
all_candidates = set(ROUND2_IDS) | set(ROUND3_IDS)
|
||||
|
||||
for s in UNITED_MT5_STRATEGIES:
|
||||
sid = s["id"]
|
||||
if sid in retired:
|
||||
o[s["enable"]] = False
|
||||
elif sid in survivors:
|
||||
o[s["enable"]] = True
|
||||
elif sid in all_candidates and sid not in candidates:
|
||||
o[s["enable"]] = False
|
||||
elif sid in candidates:
|
||||
o[s["enable"]] = False
|
||||
|
||||
for key in HIGH_MARGIN_STOCK_ENABLES:
|
||||
o[key] = False
|
||||
|
||||
o["GAP_Enable"] = False
|
||||
o["OPT_GuardOptimizationMode"] = True
|
||||
return o
|
||||
|
||||
|
||||
def solo_overrides(spec: dict) -> dict:
|
||||
o: dict[str, bool] = {k: False for k in ALL_ENABLE_KEYS}
|
||||
o[spec["enable"]] = True
|
||||
for key in HIGH_MARGIN_STOCK_ENABLES:
|
||||
o[key] = False
|
||||
o["GAP_Enable"] = False
|
||||
o["OPT_GuardOptimizationMode"] = True
|
||||
return o
|
||||
|
||||
|
||||
def classify_solo(m: dict, min_trades: int) -> str:
|
||||
if not m.get("ready"):
|
||||
return "BROKEN"
|
||||
trades = int(m.get("total_trades", 0) or 0)
|
||||
if trades < min_trades:
|
||||
return "LOW_TRADES"
|
||||
if g(m, "net_profit") > 0 and g(m, "profit_factor") >= 1.05:
|
||||
return "PASS"
|
||||
if g(m, "net_profit") > 50 and g(m, "profit_factor") >= 1.0:
|
||||
return "MARGINAL"
|
||||
return "FAIL"
|
||||
|
||||
|
||||
def delta(base: dict, var: dict) -> dict:
|
||||
return {
|
||||
"net_profit_delta": g(var, "net_profit") - g(base, "net_profit"),
|
||||
"sharpe_delta": g(var, "sharpe") - g(base, "sharpe"),
|
||||
"pf_delta": g(var, "profit_factor") - g(base, "profit_factor"),
|
||||
"trades_delta": int(g(var, "total_trades") - g(base, "total_trades")),
|
||||
}
|
||||
|
||||
|
||||
def portfolio_verdict(d: dict) -> str:
|
||||
if d["net_profit_delta"] > 100 and d["sharpe_delta"] >= -0.05:
|
||||
return "IMPROVED"
|
||||
if d["net_profit_delta"] < -150 or d["sharpe_delta"] < -0.15:
|
||||
return "WORSE"
|
||||
return "NEUTRAL"
|
||||
|
||||
|
||||
def main() -> None:
|
||||
p = argparse.ArgumentParser()
|
||||
p.add_argument("--from", dest="from_date", default=FROM_DATE)
|
||||
p.add_argument("--to", dest="to_date", default=TO_DATE)
|
||||
p.add_argument("--min-trades", type=int, default=MIN_TRADES_DEFAULT)
|
||||
p.add_argument("--round", type=int, default=3, choices=(2, 3))
|
||||
args = p.parse_args()
|
||||
|
||||
import cluster_audit.united_mt5_runner as runner
|
||||
|
||||
runner.FROM_DATE = args.from_date.replace("-", ".")
|
||||
runner.TO_DATE = args.to_date.replace("-", ".")
|
||||
|
||||
cfg = ROUND_CONFIG[args.round]
|
||||
prefix = cfg["prefix"]
|
||||
candidates = cfg["candidate_ids"]
|
||||
|
||||
OUT.mkdir(parents=True, exist_ok=True)
|
||||
ctx = mt5_context()
|
||||
deploy_united(ctx["data"], ctx["mt5_path"])
|
||||
|
||||
sm = spec_map()
|
||||
print(
|
||||
f"Round {args.round} {runner.FROM_DATE}->{runner.TO_DATE} "
|
||||
f"min_trades={args.min_trades} survivor={SURVIVOR_IDS} "
|
||||
f"candidates={candidates}",
|
||||
flush=True,
|
||||
)
|
||||
|
||||
baseline = run_backtest(
|
||||
ctx["data"], ctx["mt5_path"], ctx["login"], ctx["server"],
|
||||
patch_set(BASE_SET, baseline_overrides(args.round)),
|
||||
f"{prefix}_baseline.set", f"{prefix}_baseline",
|
||||
)
|
||||
print(
|
||||
f"BASELINE PF={baseline.get('profit_factor')} net={baseline.get('net_profit')} "
|
||||
f"sharpe={baseline.get('sharpe')} trades={baseline.get('total_trades')}",
|
||||
flush=True,
|
||||
)
|
||||
|
||||
solo_results: list[dict] = []
|
||||
passed: list[str] = []
|
||||
|
||||
for sid in candidates:
|
||||
spec = sm[sid]
|
||||
m = run_backtest(
|
||||
ctx["data"], ctx["mt5_path"], ctx["login"], ctx["server"],
|
||||
patch_set(BASE_SET, solo_overrides(spec)),
|
||||
f"{prefix}_solo_{sid}.set", f"{prefix}_solo_{sid}",
|
||||
test_symbol=spec.get("test_symbol"),
|
||||
)
|
||||
verdict = classify_solo(m, args.min_trades)
|
||||
print(
|
||||
f"SOLO {sid:10} {verdict:10} PF={m.get('profit_factor')} net={m.get('net_profit')} "
|
||||
f"sharpe={m.get('sharpe')} trades={m.get('total_trades')}",
|
||||
flush=True,
|
||||
)
|
||||
solo_results.append({"id": sid, "verdict": verdict, "metrics": m})
|
||||
if verdict in ("PASS", "MARGINAL"):
|
||||
passed.append(spec["enable"])
|
||||
|
||||
enhanced_ov = baseline_overrides(args.round)
|
||||
for sid in candidates:
|
||||
spec = sm[sid]
|
||||
if spec["enable"] in passed:
|
||||
enhanced_ov[spec["enable"]] = True
|
||||
|
||||
enhanced = run_backtest(
|
||||
ctx["data"], ctx["mt5_path"], ctx["login"], ctx["server"],
|
||||
patch_set(BASE_SET, enhanced_ov),
|
||||
f"{prefix}_enhanced.set", f"{prefix}_enhanced",
|
||||
)
|
||||
d = delta(baseline, enhanced)
|
||||
pv = portfolio_verdict(d)
|
||||
print(
|
||||
f"ENHANCED {pv} PF={enhanced.get('profit_factor')} net={enhanced.get('net_profit')} "
|
||||
f"sharpe={enhanced.get('sharpe')} trades={enhanced.get('total_trades')} "
|
||||
f"dNet={d['net_profit_delta']:.0f} dSharpe={d['sharpe_delta']:.3f}",
|
||||
flush=True,
|
||||
)
|
||||
print(f"PASSED ({len(passed)}): {', '.join(passed)}", flush=True)
|
||||
|
||||
summary = {
|
||||
"timestamp": datetime.now().isoformat(timespec="seconds"),
|
||||
"round": args.round,
|
||||
"survivors": list(SURVIVOR_IDS),
|
||||
"candidates": list(candidates),
|
||||
"high_margin_disabled": list(HIGH_MARGIN_STOCK_ENABLES),
|
||||
"period": {"from": runner.FROM_DATE, "to": runner.TO_DATE},
|
||||
"min_trades": args.min_trades,
|
||||
"baseline": baseline,
|
||||
"solo": solo_results,
|
||||
"passed_enables": passed,
|
||||
"enhanced": enhanced,
|
||||
"delta": d,
|
||||
"portfolio_verdict": pv,
|
||||
}
|
||||
out_path = OUT / f"round{args.round}_audit_summary.json"
|
||||
out_path.write_text(json.dumps(summary, indent=2), encoding="utf-8")
|
||||
print(f"Saved {out_path}", flush=True)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,322 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
MT5 genetic lot optimization — one production sub-strategy at a time.
|
||||
|
||||
Ranges:
|
||||
stock → 5..15 step 5
|
||||
other → 0.01..0.1 step 0.01
|
||||
|
||||
Usage:
|
||||
python -m cluster_audit.run_lot_genetic
|
||||
python -m cluster_audit.run_lot_genetic --only RS_NVDA
|
||||
python -m cluster_audit.run_lot_genetic --apply
|
||||
python -m cluster_audit.run_lot_genetic --resume # skip ids already in summary
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
|
||||
|
||||
from cluster_audit.united_mt5_manifest import (
|
||||
ALL_ENABLE_KEYS,
|
||||
HIGH_MARGIN_STOCK_ENABLES,
|
||||
LOT_CLASS_BY_ID,
|
||||
LOT_GENETIC_RANGE,
|
||||
PRODUCTION_IDS,
|
||||
UNITED_MT5_STRATEGIES,
|
||||
)
|
||||
from cluster_audit.united_mt5_runner import (
|
||||
BASE_SET,
|
||||
CLUSTER,
|
||||
DEPOSIT,
|
||||
FROM_DATE,
|
||||
TO_DATE,
|
||||
deploy_united,
|
||||
mt5_context,
|
||||
patch_set,
|
||||
patch_set_for_lot_genetic,
|
||||
run_backtest,
|
||||
run_genetic_lot_optimize,
|
||||
)
|
||||
|
||||
OUT = Path(__file__).resolve().parent / "reports" / "lot_genetic"
|
||||
REF_BALANCE = 3000.0
|
||||
SUMMARY_PATH = OUT / "lot_genetic_summary.json"
|
||||
|
||||
|
||||
def lot_class(sid: str) -> str:
|
||||
return LOT_CLASS_BY_ID.get(sid, "forex")
|
||||
|
||||
|
||||
def genetic_range(sid: str) -> tuple[float, float, float]:
|
||||
if lot_class(sid) == "stock":
|
||||
return LOT_GENETIC_RANGE["stock"]
|
||||
return LOT_GENETIC_RANGE["default"]
|
||||
|
||||
|
||||
def common_patches() -> dict[str, float | bool]:
|
||||
o: dict[str, float | bool] = {
|
||||
"ORCH_ReferenceBalance": REF_BALANCE,
|
||||
"ORCH_ScaleLotsByBalance": True,
|
||||
"GAP_Enable": False,
|
||||
"OPT_GuardOptimizationMode": True,
|
||||
}
|
||||
for key in HIGH_MARGIN_STOCK_ENABLES:
|
||||
o[key] = False
|
||||
return o
|
||||
|
||||
|
||||
def solo_overrides(spec: dict) -> dict[str, bool]:
|
||||
o: dict[str, bool] = {k: False for k in ALL_ENABLE_KEYS}
|
||||
o[spec["enable"]] = True
|
||||
return o
|
||||
|
||||
|
||||
def apply_lots_to_mq5(text: str, lots: dict[str, float]) -> str:
|
||||
for key, val in lots.items():
|
||||
sval = str(int(val)) if val == int(val) else str(val)
|
||||
text, _ = re.subn(
|
||||
rf"(input double {re.escape(key)} = )[0-9.]+;",
|
||||
rf"\g<1>{sval};",
|
||||
text,
|
||||
count=1,
|
||||
)
|
||||
text, _ = re.subn(
|
||||
r"(input double ORCH_ReferenceBalance = )[0-9.]+;",
|
||||
rf"\g<1>{REF_BALANCE};",
|
||||
text,
|
||||
count=1,
|
||||
)
|
||||
return text
|
||||
|
||||
|
||||
def apply_lots_to_set_text(text: str, lots: dict[str, float]) -> str:
|
||||
lines_out: list[str] = []
|
||||
for line in text.splitlines():
|
||||
if "=" not in line or line.strip().startswith(";"):
|
||||
lines_out.append(line)
|
||||
continue
|
||||
key = line.split("=", 1)[0].strip()
|
||||
if key in lots:
|
||||
val = lots[key]
|
||||
sval = str(int(val)) if val == int(val) else str(val)
|
||||
if "||" in line:
|
||||
parts = line.split("||")
|
||||
parts[0] = f"{key}={sval}"
|
||||
lines_out.append("||".join(parts))
|
||||
else:
|
||||
lines_out.append(f"{key}={sval}")
|
||||
else:
|
||||
lines_out.append(line)
|
||||
return "\n".join(lines_out) + "\n"
|
||||
|
||||
|
||||
def load_summary() -> dict:
|
||||
if SUMMARY_PATH.exists():
|
||||
return json.loads(SUMMARY_PATH.read_text(encoding="utf-8"))
|
||||
return {"results": [], "best_lots": {}}
|
||||
|
||||
|
||||
def save_summary(summary: dict) -> None:
|
||||
OUT.mkdir(parents=True, exist_ok=True)
|
||||
SUMMARY_PATH.write_text(json.dumps(summary, indent=2), encoding="utf-8")
|
||||
|
||||
|
||||
def run_backtest_with_retry(
|
||||
ctx: dict,
|
||||
set_body: str,
|
||||
set_name: str,
|
||||
report: str,
|
||||
*,
|
||||
test_symbol: str | None = None,
|
||||
retries: int = 3,
|
||||
) -> dict:
|
||||
last: dict = {"ready": False}
|
||||
for attempt in range(retries):
|
||||
if attempt:
|
||||
time.sleep(12)
|
||||
last = run_backtest(
|
||||
ctx["data"], ctx["mt5_path"], ctx["login"], ctx["server"],
|
||||
set_body, set_name, report, test_symbol=test_symbol,
|
||||
)
|
||||
if last.get("ready"):
|
||||
return last
|
||||
return last
|
||||
|
||||
|
||||
def lot_report_tag(lot: float) -> str:
|
||||
return str(int(lot)) if lot == int(lot) else str(lot).replace(".", "p")
|
||||
|
||||
|
||||
def optimize_one(ctx: dict, spec: dict, *, opt_mode: int, grid_only: bool) -> dict:
|
||||
sid = spec["id"]
|
||||
lot_key = spec["lot"]
|
||||
start, step, stop = genetic_range(sid)
|
||||
ov = {**common_patches(), **solo_overrides(spec)}
|
||||
report = f"lotgen_{sid}"
|
||||
|
||||
print(
|
||||
f"\n[{sid}] lot sweep {lot_key} range={start}..{stop} step={step} "
|
||||
f"symbol={spec.get('test_symbol') or 'NAS100'}",
|
||||
flush=True,
|
||||
)
|
||||
|
||||
if not grid_only:
|
||||
body = patch_set_for_lot_genetic(BASE_SET, ov, lot_key, start, step, stop)
|
||||
m = run_genetic_lot_optimize(
|
||||
ctx["data"], ctx["mt5_path"], ctx["login"], ctx["server"],
|
||||
body, f"{report}.set", report, lot_key,
|
||||
test_symbol=spec.get("test_symbol"),
|
||||
optimization=opt_mode,
|
||||
)
|
||||
best_lot = m.get("best_lot")
|
||||
if m.get("ready") and best_lot is not None:
|
||||
print(
|
||||
f" BEST lot={best_lot} PF={m.get('profit_factor')} net={m.get('profit')} "
|
||||
f"sharpe={m.get('sharpe')} trades={m.get('trades')} passes={m.get('passes')} "
|
||||
f"({m.get('elapsed_sec')}s)",
|
||||
flush=True,
|
||||
)
|
||||
return {
|
||||
"id": sid,
|
||||
"lot_key": lot_key,
|
||||
"lot_class": lot_class(sid),
|
||||
"range": {"start": start, "step": step, "stop": stop},
|
||||
"best_lot": best_lot,
|
||||
"metrics": m,
|
||||
}
|
||||
print(f" genetic XML miss ({m.get('error')}) — grid sweep", flush=True)
|
||||
|
||||
from cluster_audit.united_mt5_manifest import LOT_GRIDS
|
||||
|
||||
grid = LOT_GRIDS["stock"] if lot_class(sid) == "stock" else LOT_GRIDS["forex"]
|
||||
best_sc, best_lot, best_m = -1e18, grid[0], {}
|
||||
t0 = time.time()
|
||||
for lot in grid:
|
||||
tag = lot_report_tag(lot)
|
||||
ov2 = {**ov, lot_key: lot}
|
||||
bm = run_backtest_with_retry(
|
||||
ctx,
|
||||
patch_set(BASE_SET, ov2), f"lot_{sid}_{tag}.set", f"lot_{sid}_{tag}",
|
||||
test_symbol=spec.get("test_symbol"),
|
||||
)
|
||||
if not bm.get("ready"):
|
||||
print(f" lot={lot} FAILED (no report)", flush=True)
|
||||
continue
|
||||
trades = int(bm.get("total_trades") or 0)
|
||||
profit = float(bm.get("net_profit") or 0)
|
||||
pf = float(bm.get("profit_factor") or 0)
|
||||
sharpe = float(bm.get("sharpe") or 0)
|
||||
if trades < 20 or pf < 1.0 or profit <= 0:
|
||||
sc = -1e10 + profit
|
||||
else:
|
||||
sc = sharpe * 2000 + profit / 500 + pf * 50
|
||||
print(
|
||||
f" lot={lot} PF={pf} net={profit} sharpe={sharpe} trades={trades}",
|
||||
flush=True,
|
||||
)
|
||||
if sc > best_sc:
|
||||
best_sc, best_lot, best_m = sc, lot, bm
|
||||
elapsed = round(time.time() - t0, 1)
|
||||
best_m = {**best_m, "best_lot": best_lot, "method": "grid", "elapsed_sec": elapsed}
|
||||
print(
|
||||
f" BEST lot={best_lot} PF={best_m.get('profit_factor')} net={best_m.get('net_profit')} "
|
||||
f"sharpe={best_m.get('sharpe')} trades={best_m.get('total_trades')} ({elapsed}s)",
|
||||
flush=True,
|
||||
)
|
||||
return {
|
||||
"id": sid,
|
||||
"lot_key": lot_key,
|
||||
"lot_class": lot_class(sid),
|
||||
"range": {"start": start, "step": step, "stop": stop},
|
||||
"best_lot": best_lot,
|
||||
"metrics": best_m,
|
||||
}
|
||||
|
||||
|
||||
def main() -> None:
|
||||
p = argparse.ArgumentParser()
|
||||
p.add_argument("--from", dest="from_date", default=FROM_DATE)
|
||||
p.add_argument("--to", dest="to_date", default=TO_DATE)
|
||||
p.add_argument("--only", action="append", default=[])
|
||||
p.add_argument("--apply", action="store_true")
|
||||
p.add_argument("--resume", action="store_true", help="Skip strategies already in summary")
|
||||
p.add_argument("--redo", action="append", default=[], help="Re-run these ids even if in summary")
|
||||
p.add_argument("--mode", choices=("genetic", "complete", "grid"), default="grid",
|
||||
help="grid=direct lot sweep (default); genetic=try MT5 genetic first")
|
||||
args = p.parse_args()
|
||||
|
||||
import cluster_audit.united_mt5_runner as runner
|
||||
|
||||
runner.FROM_DATE = args.from_date.replace("-", ".")
|
||||
runner.TO_DATE = args.to_date.replace("-", ".")
|
||||
runner.DEPOSIT = int(REF_BALANCE)
|
||||
|
||||
opt_mode = 2 if args.mode == "genetic" else 1
|
||||
grid_only = args.mode == "grid"
|
||||
ids = args.only if args.only else list(PRODUCTION_IDS)
|
||||
sm = {s["id"]: s for s in UNITED_MT5_STRATEGIES}
|
||||
|
||||
summary = load_summary() if args.resume else {"results": [], "best_lots": {}}
|
||||
done_ids = set()
|
||||
if args.resume:
|
||||
for r in summary.get("results", []):
|
||||
m = r.get("metrics") or {}
|
||||
if m.get("ready") and r["id"] not in args.redo:
|
||||
done_ids.add(r["id"])
|
||||
|
||||
ctx = mt5_context()
|
||||
deploy_united(ctx["data"], ctx["mt5_path"])
|
||||
|
||||
print(
|
||||
f"Lot genetic deposit={DEPOSIT} ref={REF_BALANCE} "
|
||||
f"{runner.FROM_DATE}->{runner.TO_DATE} mode={args.mode} n={len(ids)}",
|
||||
flush=True,
|
||||
)
|
||||
|
||||
for sid in ids:
|
||||
if sid not in sm:
|
||||
print(f"skip unknown {sid}", flush=True)
|
||||
continue
|
||||
if sid in done_ids and sid not in args.redo:
|
||||
print(f"skip done {sid}", flush=True)
|
||||
continue
|
||||
r = optimize_one(ctx, sm[sid], opt_mode=opt_mode, grid_only=grid_only)
|
||||
summary["results"] = [x for x in summary.get("results", []) if x["id"] != sid] + [r]
|
||||
summary["best_lots"][r["lot_key"]] = r["best_lot"]
|
||||
summary["timestamp"] = datetime.now().isoformat(timespec="seconds")
|
||||
summary["period"] = {"from": runner.FROM_DATE, "to": runner.TO_DATE}
|
||||
save_summary(summary)
|
||||
|
||||
print(f"\nSaved {SUMMARY_PATH}", flush=True)
|
||||
for r in summary["results"]:
|
||||
print(
|
||||
f" {r['id']:12} {r['lot_key']}={r['best_lot']} "
|
||||
f"PF={r['metrics'].get('profit_factor')} sharpe={r['metrics'].get('sharpe')}",
|
||||
flush=True,
|
||||
)
|
||||
|
||||
if args.apply and summary.get("best_lots"):
|
||||
mq5_path = CLUSTER / "main.mq5"
|
||||
set_path = CLUSTER / "123.set"
|
||||
mq5_path.write_text(
|
||||
apply_lots_to_mq5(mq5_path.read_text(encoding="utf-8"), summary["best_lots"]),
|
||||
encoding="utf-8",
|
||||
)
|
||||
set_path.write_text(
|
||||
apply_lots_to_set_text(set_path.read_text(encoding="utf-8"), summary["best_lots"]),
|
||||
encoding="utf-8",
|
||||
)
|
||||
print(f"Applied to {mq5_path} and {set_path}", flush=True)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,317 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
MT5 lot-size sweep per production sub-strategy, then combined portfolio.
|
||||
|
||||
Constraints:
|
||||
- ORCH_ReferenceBalance = 3000 (deposit also 3000)
|
||||
- Stock lots capped at 15 shares
|
||||
|
||||
Usage:
|
||||
python -m cluster_audit.run_lot_optimize
|
||||
python -m cluster_audit.run_lot_optimize --only RS_NVDA
|
||||
python -m cluster_audit.run_lot_optimize --apply
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
|
||||
|
||||
from cluster_audit.united_mt5_manifest import (
|
||||
ALL_ENABLE_KEYS,
|
||||
HIGH_MARGIN_STOCK_ENABLES,
|
||||
LOT_CLASS_BY_ID,
|
||||
LOT_GRIDS,
|
||||
PRODUCTION_IDS,
|
||||
UNITED_MT5_STRATEGIES,
|
||||
)
|
||||
from cluster_audit.united_mt5_runner import (
|
||||
BASE_SET,
|
||||
CLUSTER,
|
||||
DEPOSIT,
|
||||
FROM_DATE,
|
||||
TO_DATE,
|
||||
deploy_united,
|
||||
mt5_context,
|
||||
patch_set,
|
||||
run_backtest,
|
||||
)
|
||||
|
||||
OUT = Path(__file__).resolve().parent / "reports" / "lot_optimize"
|
||||
STOCK_LOT_MAX = 15.0
|
||||
REF_BALANCE = 3000.0
|
||||
MIN_TRADES = 20
|
||||
|
||||
|
||||
def g(m: dict, k: str) -> float:
|
||||
v = m.get(k)
|
||||
return float(v) if v is not None else 0.0
|
||||
|
||||
|
||||
def spec_map() -> dict[str, dict]:
|
||||
return {s["id"]: s for s in UNITED_MT5_STRATEGIES}
|
||||
|
||||
|
||||
def lot_class(sid: str) -> str:
|
||||
return LOT_CLASS_BY_ID.get(sid, "forex")
|
||||
|
||||
|
||||
def lot_grid(sid: str) -> list[float]:
|
||||
cls = lot_class(sid)
|
||||
grid = list(LOT_GRIDS.get(cls, LOT_GRIDS["forex"]))
|
||||
if cls == "stock":
|
||||
grid = [x for x in grid if x <= STOCK_LOT_MAX]
|
||||
return grid
|
||||
|
||||
|
||||
def score(m: dict) -> float:
|
||||
if not m.get("ready"):
|
||||
return -1e12
|
||||
trades = int(m.get("total_trades") or 0)
|
||||
if trades < MIN_TRADES:
|
||||
return -1e11 + trades
|
||||
net = g(m, "net_profit")
|
||||
pf = g(m, "profit_factor")
|
||||
sharpe = g(m, "sharpe")
|
||||
if net <= 0 or pf < 1.0:
|
||||
return -1e10 + net
|
||||
return sharpe * 2000.0 + net / 500.0 + pf * 50.0
|
||||
|
||||
|
||||
def common_patches() -> dict[str, float | bool]:
|
||||
o: dict[str, float | bool] = {
|
||||
"ORCH_ReferenceBalance": REF_BALANCE,
|
||||
"ORCH_ScaleLotsByBalance": True,
|
||||
"GAP_Enable": False,
|
||||
"OPT_GuardOptimizationMode": True,
|
||||
}
|
||||
for key in HIGH_MARGIN_STOCK_ENABLES:
|
||||
o[key] = False
|
||||
return o
|
||||
|
||||
|
||||
def production_enables() -> dict[str, bool]:
|
||||
prod = set(PRODUCTION_IDS)
|
||||
o: dict[str, bool] = {}
|
||||
for s in UNITED_MT5_STRATEGIES:
|
||||
o[s["enable"]] = s["id"] in prod
|
||||
return o
|
||||
|
||||
|
||||
def solo_overrides(spec: dict) -> dict[str, bool]:
|
||||
o: dict[str, bool] = {k: False for k in ALL_ENABLE_KEYS}
|
||||
o[spec["enable"]] = True
|
||||
return o
|
||||
|
||||
|
||||
def apply_lots_to_set_text(text: str, lots: dict[str, float]) -> str:
|
||||
lines_out: list[str] = []
|
||||
for line in text.splitlines():
|
||||
if "=" not in line or line.strip().startswith(";"):
|
||||
lines_out.append(line)
|
||||
continue
|
||||
key = line.split("=", 1)[0].strip()
|
||||
if key in lots:
|
||||
val = lots[key]
|
||||
sval = str(int(val)) if val == int(val) else str(val)
|
||||
if "||" in line:
|
||||
parts = line.split("||")
|
||||
parts[0] = f"{key}={sval}"
|
||||
lines_out.append("||".join(parts))
|
||||
else:
|
||||
lines_out.append(f"{key}={sval}")
|
||||
else:
|
||||
lines_out.append(line)
|
||||
for key, val in lots.items():
|
||||
if not any(l.startswith(f"{key}=") for l in lines_out):
|
||||
sval = str(int(val)) if val == int(val) else str(val)
|
||||
lines_out.append(f"{key}={sval}||{sval}||0||{sval}||N")
|
||||
return "\n".join(lines_out) + "\n"
|
||||
|
||||
|
||||
def apply_lots_to_mq5(text: str, lots: dict[str, float]) -> str:
|
||||
for key, val in lots.items():
|
||||
sval = str(int(val)) if val == int(val) else str(val)
|
||||
text, n = re.subn(
|
||||
rf"(input double {re.escape(key)} = )[0-9.]+;",
|
||||
rf"\g<1>{sval};",
|
||||
text,
|
||||
count=1,
|
||||
)
|
||||
if n == 0:
|
||||
print(f" warn: {key} not found in main.mq5", flush=True)
|
||||
text, n = re.subn(
|
||||
r"(input double ORCH_ReferenceBalance = )[0-9.]+;",
|
||||
rf"\g<1>{REF_BALANCE};",
|
||||
text,
|
||||
count=1,
|
||||
)
|
||||
return text
|
||||
|
||||
|
||||
def optimize_one(ctx: dict, spec: dict, sm: dict[str, dict]) -> dict:
|
||||
sid = spec["id"]
|
||||
lot_key = spec["lot"]
|
||||
grid = lot_grid(sid)
|
||||
print(f"\n[{sid}] grid={grid} class={lot_class(sid)}", flush=True)
|
||||
|
||||
best_lot = grid[0]
|
||||
best_m: dict = {"ready": False}
|
||||
best_sc = -1e12
|
||||
trials: list[dict] = []
|
||||
|
||||
for lot in grid:
|
||||
ov: dict = {**common_patches(), **solo_overrides(spec), lot_key: lot}
|
||||
body = patch_set(BASE_SET, ov)
|
||||
m = run_backtest(
|
||||
ctx["data"], ctx["mt5_path"], ctx["login"], ctx["server"],
|
||||
body, f"lot_{sid}_{lot}.set", f"lot_{sid}_{lot}",
|
||||
test_symbol=spec.get("test_symbol"),
|
||||
)
|
||||
sc = score(m)
|
||||
trials.append({"lot": lot, "score": sc, "metrics": m})
|
||||
print(
|
||||
f" lot={lot:6} sc={sc:10.1f} PF={m.get('profit_factor')} "
|
||||
f"net={m.get('net_profit')} sharpe={m.get('sharpe')} trades={m.get('total_trades')}",
|
||||
flush=True,
|
||||
)
|
||||
if sc > best_sc:
|
||||
best_sc = sc
|
||||
best_lot = lot
|
||||
best_m = m
|
||||
|
||||
return {
|
||||
"id": sid,
|
||||
"lot_key": lot_key,
|
||||
"lot_class": lot_class(sid),
|
||||
"best_lot": best_lot,
|
||||
"best_score": best_sc,
|
||||
"best_metrics": best_m,
|
||||
"trials": trials,
|
||||
}
|
||||
|
||||
|
||||
def main() -> None:
|
||||
p = argparse.ArgumentParser()
|
||||
p.add_argument("--from", dest="from_date", default=FROM_DATE)
|
||||
p.add_argument("--to", dest="to_date", default=TO_DATE)
|
||||
p.add_argument("--only", action="append", default=[], help="Strategy id(s) to optimize")
|
||||
p.add_argument("--apply", action="store_true", help="Write best lots to main.mq5 and 123.set")
|
||||
args = p.parse_args()
|
||||
|
||||
import cluster_audit.united_mt5_runner as runner
|
||||
|
||||
runner.FROM_DATE = args.from_date.replace("-", ".")
|
||||
runner.TO_DATE = args.to_date.replace("-", ".")
|
||||
runner.DEPOSIT = int(REF_BALANCE)
|
||||
|
||||
OUT.mkdir(parents=True, exist_ok=True)
|
||||
ctx = mt5_context()
|
||||
deploy_united(ctx["data"], ctx["mt5_path"])
|
||||
sm = spec_map()
|
||||
|
||||
ids = args.only if args.only else list(PRODUCTION_IDS)
|
||||
print(
|
||||
f"Lot optimize deposit={DEPOSIT} ref={REF_BALANCE} "
|
||||
f"{runner.FROM_DATE}->{runner.TO_DATE} n={len(ids)}",
|
||||
flush=True,
|
||||
)
|
||||
|
||||
results: list[dict] = []
|
||||
best_lots: dict[str, float] = {}
|
||||
|
||||
for sid in ids:
|
||||
if sid not in sm:
|
||||
print(f"skip unknown id {sid}", flush=True)
|
||||
continue
|
||||
r = optimize_one(ctx, sm[sid], sm)
|
||||
results.append(r)
|
||||
best_lots[r["lot_key"]] = r["best_lot"]
|
||||
|
||||
# Baseline combined (123.set lots + production enables)
|
||||
base_ov: dict = {**common_patches(), **production_enables()}
|
||||
baseline = run_backtest(
|
||||
ctx["data"], ctx["mt5_path"], ctx["login"], ctx["server"],
|
||||
patch_set(BASE_SET, base_ov),
|
||||
"lot_combined_baseline.set", "lot_combined_baseline",
|
||||
)
|
||||
print(
|
||||
f"\nCOMBINED baseline PF={baseline.get('profit_factor')} net={baseline.get('net_profit')} "
|
||||
f"sharpe={baseline.get('sharpe')} trades={baseline.get('total_trades')}",
|
||||
flush=True,
|
||||
)
|
||||
|
||||
opt_ov: dict = {**common_patches(), **production_enables(), **best_lots}
|
||||
optimized = run_backtest(
|
||||
ctx["data"], ctx["mt5_path"], ctx["login"], ctx["server"],
|
||||
patch_set(BASE_SET, opt_ov),
|
||||
"lot_combined_optimized.set", "lot_combined_optimized",
|
||||
)
|
||||
print(
|
||||
f"COMBINED optimized PF={optimized.get('profit_factor')} net={optimized.get('net_profit')} "
|
||||
f"sharpe={optimized.get('sharpe')} trades={optimized.get('total_trades')} "
|
||||
f"dNet={g(optimized, 'net_profit') - g(baseline, 'net_profit'):.0f} "
|
||||
f"dSharpe={g(optimized, 'sharpe') - g(baseline, 'sharpe'):.3f}",
|
||||
flush=True,
|
||||
)
|
||||
|
||||
summary = {
|
||||
"timestamp": datetime.now().isoformat(timespec="seconds"),
|
||||
"period": {"from": runner.FROM_DATE, "to": runner.TO_DATE},
|
||||
"deposit": DEPOSIT,
|
||||
"reference_balance": REF_BALANCE,
|
||||
"stock_lot_max": STOCK_LOT_MAX,
|
||||
"solo": results,
|
||||
"best_lots": best_lots,
|
||||
"combined_baseline": baseline,
|
||||
"combined_optimized": optimized,
|
||||
"combined_delta": {
|
||||
"net_profit": g(optimized, "net_profit") - g(baseline, "net_profit"),
|
||||
"sharpe": g(optimized, "sharpe") - g(baseline, "sharpe"),
|
||||
"pf": g(optimized, "profit_factor") - g(baseline, "profit_factor"),
|
||||
},
|
||||
}
|
||||
out_path = OUT / "lot_optimize_summary.json"
|
||||
out_path.write_text(json.dumps(summary, indent=2), encoding="utf-8")
|
||||
print(f"\nSaved {out_path}", flush=True)
|
||||
|
||||
print("\nBest lots:", flush=True)
|
||||
for sid in ids:
|
||||
if sid not in sm:
|
||||
continue
|
||||
row = next(x for x in results if x["id"] == sid)
|
||||
m = row["best_metrics"]
|
||||
print(
|
||||
f" {sid:12} {row['lot_key']}={row['best_lot']} "
|
||||
f"PF={m.get('profit_factor')} sharpe={m.get('sharpe')} net={m.get('net_profit')}",
|
||||
flush=True,
|
||||
)
|
||||
|
||||
if args.apply:
|
||||
mq5_path = CLUSTER / "main.mq5"
|
||||
set_path = CLUSTER / "123.set"
|
||||
mq5_path.write_text(apply_lots_to_mq5(mq5_path.read_text(encoding="utf-8"), best_lots), encoding="utf-8")
|
||||
set_path.write_text(
|
||||
apply_lots_to_set_text(set_path.read_text(encoding="utf-8"), best_lots),
|
||||
encoding="utf-8",
|
||||
)
|
||||
# Ensure reference balance in set
|
||||
set_text = set_path.read_text(encoding="utf-8")
|
||||
set_text = re.sub(
|
||||
r"ORCH_ReferenceBalance=[^\n]+",
|
||||
f"ORCH_ReferenceBalance={int(REF_BALANCE)}||{int(REF_BALANCE)}.0||100.000000||10000.000000||N",
|
||||
set_text,
|
||||
count=1,
|
||||
)
|
||||
set_path.write_text(set_text, encoding="utf-8")
|
||||
print(f"Applied lots to {mq5_path} and {set_path}", flush=True)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,101 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Lot-opt new production strategies, merge keepers, apply, combined backtest."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sys
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
|
||||
|
||||
from cluster_audit.run_lot_genetic import (
|
||||
SUMMARY_PATH,
|
||||
apply_lots_to_mq5,
|
||||
apply_lots_to_set_text,
|
||||
common_patches,
|
||||
optimize_one,
|
||||
save_summary,
|
||||
)
|
||||
|
||||
from cluster_audit.united_mt5_manifest import UNITED_MT5_STRATEGIES
|
||||
from cluster_audit.united_mt5_runner import (
|
||||
BASE_SET,
|
||||
CLUSTER,
|
||||
deploy_united,
|
||||
mt5_context,
|
||||
patch_set,
|
||||
run_backtest,
|
||||
)
|
||||
|
||||
# Lots from prior genetic run (unchanged strategies)
|
||||
KEEPER_LOTS: dict[str, float] = {
|
||||
"LOT_DB_DarvasBox": 0.01,
|
||||
"LOT_ES_EMASlopeDistance": 0.07,
|
||||
"LOT_RC_RSICrossOver": 0.1,
|
||||
"LOT_RM_RSIMidPointHijack": 0.01,
|
||||
"LOT_RS_NVDA": 5.0,
|
||||
"LOT_RS_TSLA": 5.0,
|
||||
"LOT_RRA_AUDUSD": 0.05,
|
||||
"LOT_UB_USDJPY": 0.03,
|
||||
"LOT_RS_NAS100": 0.03,
|
||||
"LOT_UKB_UK100": 0.01,
|
||||
}
|
||||
|
||||
NEW_OPT_IDS = (
|
||||
"RS_BTCUSD", "RS_XAUUSD", "SE", "ST_BTC", "ST_XAU",
|
||||
"RRA_GBP", "GB", "U5B", "RS_US30",
|
||||
)
|
||||
|
||||
|
||||
def production_enables() -> dict[str, bool]:
|
||||
from cluster_audit.united_mt5_manifest import ALL_ENABLE_KEYS, HIGH_MARGIN_STOCK_ENABLES, PRODUCTION_IDS
|
||||
prod = set(PRODUCTION_IDS)
|
||||
o = {s["enable"]: s["id"] in prod for s in UNITED_MT5_STRATEGIES}
|
||||
for k in HIGH_MARGIN_STOCK_ENABLES:
|
||||
o[k] = False
|
||||
return o
|
||||
|
||||
|
||||
def main() -> None:
|
||||
sm = {s["id"]: s for s in UNITED_MT5_STRATEGIES}
|
||||
summary = {"results": [], "best_lots": dict(KEEPER_LOTS), "timestamp": datetime.now().isoformat(timespec="seconds")}
|
||||
save_summary(summary)
|
||||
|
||||
ctx = mt5_context()
|
||||
deploy_united(ctx["data"], ctx["mt5_path"])
|
||||
|
||||
print(f"Optimizing {len(NEW_OPT_IDS)} new/changed strategies...", flush=True)
|
||||
for sid in NEW_OPT_IDS:
|
||||
r = optimize_one(ctx, sm[sid], opt_mode=1, grid_only=True)
|
||||
summary["results"].append(r)
|
||||
summary["best_lots"][r["lot_key"]] = r["best_lot"]
|
||||
save_summary(summary)
|
||||
print(f" {sid} -> {r['lot_key']}={r['best_lot']}", flush=True)
|
||||
|
||||
# Apply all production lots
|
||||
lots = summary["best_lots"]
|
||||
mq5 = CLUSTER / "main.mq5"
|
||||
st = CLUSTER / "123.set"
|
||||
mq5.write_text(apply_lots_to_mq5(mq5.read_text(encoding="utf-8"), lots), encoding="utf-8")
|
||||
st.write_text(apply_lots_to_set_text(st.read_text(encoding="utf-8"), lots), encoding="utf-8")
|
||||
print(f"Applied {len(lots)} lots to main.mq5 + 123.set", flush=True)
|
||||
|
||||
ov = {**common_patches(), **production_enables(), **lots}
|
||||
m = run_backtest(
|
||||
ctx["data"], ctx["mt5_path"], ctx["login"], ctx["server"],
|
||||
patch_set(BASE_SET, ov), "prod_v2_combined.set", "prod_v2_combined",
|
||||
)
|
||||
print(
|
||||
f"\nCOMBINED v2 PF={m.get('profit_factor')} net={m.get('net_profit')} "
|
||||
f"sharpe={m.get('sharpe')} trades={m.get('total_trades')} dd={m.get('max_drawdown')}",
|
||||
flush=True,
|
||||
)
|
||||
summary["combined_v2"] = m
|
||||
save_summary(summary)
|
||||
print(f"Saved {SUMMARY_PATH}", flush=True)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,278 @@
|
||||
"""
|
||||
Sequential cluster audit: one strategy at a time, fix until it passes, then next.
|
||||
|
||||
Pass criteria (optimized result):
|
||||
- >= 1 trade per calendar day over the backtest window (~1977 for 2021-2026)
|
||||
- net > 0, profit factor >= 1.15, sharpe >= 0.3
|
||||
- winning months >= 45%, drawdown <= 25%
|
||||
|
||||
Usage:
|
||||
python -m cluster_audit.run_sequential [trials] [--only ID] [--from ID]
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import random
|
||||
import sys
|
||||
import time
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
import MetaTrader5 as mt5
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
|
||||
|
||||
from cluster_audit.backtest_core import CostModel, load_bars, resolve_symbol
|
||||
from cluster_audit.diagnose import diagnose
|
||||
from cluster_audit.engines import ENGINE_MAP
|
||||
from cluster_audit.portfolio_build import build_progressive_portfolios
|
||||
from cluster_audit.run_audit import _sample_params
|
||||
from cluster_audit.scoring import (
|
||||
DEFAULT_TRADES_PER_DAY,
|
||||
acceptance,
|
||||
format_quality_line,
|
||||
min_trades_for_period,
|
||||
period_days,
|
||||
score_label,
|
||||
score_report,
|
||||
trades_per_day,
|
||||
)
|
||||
from cluster_audit.strategy_registry import PERIODS, STRATEGIES, TF
|
||||
from cluster_audit.trace_log import TraceLog
|
||||
|
||||
LOG = TraceLog(enabled=True, trial_every=10)
|
||||
PRIMARY_PERIOD = "2021-2026"
|
||||
|
||||
|
||||
def run_one_strategy(
|
||||
spec: dict,
|
||||
period_label: str,
|
||||
start: str,
|
||||
end: str,
|
||||
trials: int,
|
||||
rng: random.Random,
|
||||
idx: int,
|
||||
total: int,
|
||||
days: int,
|
||||
trades_per_day_target: float,
|
||||
) -> dict:
|
||||
sid = spec["id"]
|
||||
tf_key = spec["tf"]
|
||||
label = f"[{idx}/{total}] {sid} @ {period_label}"
|
||||
LOG.banner(label)
|
||||
|
||||
engine = ENGINE_MAP[spec["engine"]]
|
||||
sym = resolve_symbol(spec["symbol"])
|
||||
start_dt = datetime.fromisoformat(start)
|
||||
end_dt = datetime.fromisoformat(end)
|
||||
|
||||
try:
|
||||
df = load_bars(sym, TF[tf_key], start_dt, end_dt)
|
||||
except Exception as e:
|
||||
LOG.error(f"data load failed: {e}")
|
||||
return {"id": sid, "period": period_label, "error": str(e), "spec": spec, "passed": False}
|
||||
|
||||
LOG.info(f"loaded {len(df)} bars symbol={sym} engine={spec['engine']}")
|
||||
costs = CostModel.for_symbol(sym)
|
||||
defaults = dict(spec["defaults"])
|
||||
|
||||
min_t = min_trades_for_period(days, trades_per_day_target)
|
||||
LOG.info(f"activity gate: >={min_t} trades ({trades_per_day_target:.1f}/day x {days}d)")
|
||||
|
||||
t0 = time.perf_counter()
|
||||
baseline = engine(df, sym, period_label, sid, defaults, spec["lot"], costs)
|
||||
LOG.info(f"baseline: {format_quality_line(baseline, days)} ({(time.perf_counter()-t0)*1000:.0f}ms)")
|
||||
|
||||
best = baseline
|
||||
best_params = defaults
|
||||
best_score = score_report(baseline, days, trades_per_day_target)
|
||||
|
||||
base_ok, base_issues = acceptance(baseline, days, trades_per_day_target)
|
||||
if base_ok:
|
||||
LOG.info("baseline PASSES acceptance gates")
|
||||
else:
|
||||
LOG.warn("baseline fails: " + "; ".join(base_issues))
|
||||
|
||||
if trials > 0:
|
||||
LOG.info(f"optimizing {trials} trials (score requires trades + profit + consistency)...")
|
||||
for n in range(1, trials + 1):
|
||||
params = _sample_params(defaults, spec.get("opt", {}), rng)
|
||||
r = engine(df, sym, period_label, sid, params, spec["lot"], costs)
|
||||
sc = score_report(r, days, trades_per_day_target)
|
||||
if sc > best_score:
|
||||
best_score = sc
|
||||
best = r
|
||||
best_params = params
|
||||
LOG.trial(n, trials, sc, r.net_profit, r.sharpe, improved=True)
|
||||
LOG.debug(f" -> {format_quality_line(r, days)}")
|
||||
elif n % LOG.trial_every == 0 or n == trials:
|
||||
LOG.trial(n, trials, sc, r.net_profit, r.sharpe, improved=False)
|
||||
|
||||
LOG.info(f"optimized: {format_quality_line(best, days)}")
|
||||
opt_ok, opt_issues = acceptance(best, days, trades_per_day_target)
|
||||
passed = opt_ok or base_ok
|
||||
|
||||
if passed:
|
||||
LOG.info(f"PASS {sid} - ready for portfolio")
|
||||
else:
|
||||
LOG.warn(f"FAIL {sid} - needs engine/logic work before next strategy")
|
||||
LOG.warn(" " + "; ".join(opt_issues if not opt_ok else base_issues))
|
||||
|
||||
diag = diagnose(spec, baseline, best, LOG)
|
||||
|
||||
result = {
|
||||
"id": sid,
|
||||
"engine": spec["engine"],
|
||||
"symbol": sym,
|
||||
"timeframe": tf_key,
|
||||
"period": period_label,
|
||||
"bars": len(df),
|
||||
"period_days": days,
|
||||
"trades_per_day_target": trades_per_day_target,
|
||||
"baseline_trades_per_day": trades_per_day(baseline, days),
|
||||
"optimized_trades_per_day": trades_per_day(best, days),
|
||||
"passed": passed,
|
||||
"acceptance_issues": opt_issues if not opt_ok else ([] if base_ok else base_issues),
|
||||
"baseline": baseline.to_dict(),
|
||||
"optimized": {**best.to_dict(), "params": best_params},
|
||||
"optimized_params": best_params,
|
||||
"optimized_score": best_score,
|
||||
"improvement_net": best.net_profit - baseline.net_profit,
|
||||
"improvement_trades": best.total_trades - baseline.total_trades,
|
||||
"diagnosis": diag,
|
||||
"spec": spec,
|
||||
}
|
||||
|
||||
out_dir = Path(__file__).parent / "reports" / "sequential"
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
path = out_dir / f"{sid}_{period_label}.json"
|
||||
path.write_text(json.dumps({k: v for k, v in result.items() if k != "spec"}, indent=2), encoding="utf-8")
|
||||
LOG.info(f"saved {path.name}")
|
||||
|
||||
try:
|
||||
from cluster_audit.sync_cluster import main as sync_cluster
|
||||
sync_cluster()
|
||||
LOG.info("cluster-latest SuperEA_AuditParams.mqh updated")
|
||||
except Exception as ex:
|
||||
LOG.warn(f"cluster sync skipped: {ex}")
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
p = argparse.ArgumentParser(description="Sequential cluster audit - fix each before next")
|
||||
p.add_argument("trials", nargs="?", type=int, default=40)
|
||||
p.add_argument("--portfolio-trials", type=int, default=30)
|
||||
p.add_argument("--from", dest="from_id", default=None)
|
||||
p.add_argument("--only", default=None)
|
||||
p.add_argument("--skip-portfolio", action="store_true")
|
||||
p.add_argument("--trial-every", type=int, default=10)
|
||||
p.add_argument("--trades-per-day", type=float, default=DEFAULT_TRADES_PER_DAY)
|
||||
p.add_argument("--continue-on-fail", action="store_true", help="Run next strategy even if current fails")
|
||||
return p.parse_args()
|
||||
|
||||
|
||||
def main() -> None:
|
||||
args = parse_args()
|
||||
global LOG
|
||||
LOG = TraceLog(enabled=True, trial_every=args.trial_every)
|
||||
|
||||
start, end = PERIODS[PRIMARY_PERIOD]
|
||||
days = period_days(datetime.fromisoformat(start), datetime.fromisoformat(end))
|
||||
strategies = STRATEGIES
|
||||
if args.only:
|
||||
strategies = [s for s in STRATEGIES if s["id"] == args.only]
|
||||
elif args.from_id:
|
||||
found = False
|
||||
filtered = []
|
||||
for s in STRATEGIES:
|
||||
if s["id"] == args.from_id:
|
||||
found = True
|
||||
if found:
|
||||
filtered.append(s)
|
||||
strategies = filtered if found else STRATEGIES
|
||||
|
||||
LOG.banner(
|
||||
f"SEQUENTIAL AUDIT - {len(strategies)} strategies | "
|
||||
f">={args.trades_per_day:.1f} trade/day (~{min_trades_for_period(days, args.trades_per_day)} trades) | "
|
||||
f"{args.trials} opt trials"
|
||||
)
|
||||
|
||||
if not mt5.initialize():
|
||||
LOG.error(f"MT5 init failed: {mt5.last_error()}")
|
||||
raise SystemExit(1)
|
||||
|
||||
out_dir = Path(__file__).parent / "reports" / "sequential"
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
rng = random.Random(42)
|
||||
results: list[dict] = []
|
||||
passed_ids: list[str] = []
|
||||
failed_ids: list[str] = []
|
||||
|
||||
try:
|
||||
for i, spec in enumerate(strategies, 1):
|
||||
r = run_one_strategy(
|
||||
spec, PRIMARY_PERIOD, start, end, args.trials, rng, i, len(strategies),
|
||||
days, args.trades_per_day,
|
||||
)
|
||||
results.append(r)
|
||||
if r.get("passed"):
|
||||
passed_ids.append(r["id"])
|
||||
elif "error" not in r:
|
||||
failed_ids.append(r["id"])
|
||||
if not args.continue_on_fail and not args.only:
|
||||
LOG.warn(f"STOPPED at {r['id']} - fix engine/params then resume with --from {r['id']}")
|
||||
break
|
||||
|
||||
valid = [r for r in results if r.get("passed") and "error" not in r]
|
||||
valid.sort(key=lambda x: x.get("optimized_score", float("-inf")), reverse=True)
|
||||
|
||||
summary = {
|
||||
"generated": datetime.now().isoformat(),
|
||||
"period_days": days,
|
||||
"trades_per_day_target": args.trades_per_day,
|
||||
"trials": args.trials,
|
||||
"passed": passed_ids,
|
||||
"failed": failed_ids,
|
||||
"ranking": [
|
||||
{
|
||||
"id": r["id"],
|
||||
"score": r.get("optimized_score"),
|
||||
"net": r["optimized"]["net_profit"],
|
||||
"trades": r["optimized"]["total_trades"],
|
||||
"sharpe": r["optimized"]["sharpe"],
|
||||
"pf": r["optimized"]["profit_factor"],
|
||||
}
|
||||
for r in valid
|
||||
],
|
||||
"results": [{k: v for k, v in r.items() if k != "spec"} for r in results],
|
||||
}
|
||||
|
||||
if not args.skip_portfolio and valid:
|
||||
LOG.banner("PROGRESSIVE PORTFOLIO BUILD (passed strategies only)")
|
||||
ranked = [{"spec": r["spec"], "optimized_params": r["optimized_params"]} for r in valid]
|
||||
summary["portfolio_steps"] = build_progressive_portfolios(
|
||||
ranked, PRIMARY_PERIOD, start, end, args.portfolio_trials, rng, LOG
|
||||
)
|
||||
|
||||
(out_dir / "sequential_summary.json").write_text(json.dumps(summary, indent=2), encoding="utf-8")
|
||||
|
||||
LOG.banner(f"PASSED {len(passed_ids)} / FAILED {len(failed_ids)}")
|
||||
for r in valid:
|
||||
o = r["optimized"]
|
||||
LOG.info(
|
||||
f" {r['id']:28} score={score_label(r['optimized_score'], o['total_trades'], days, args.trades_per_day):>22} "
|
||||
f"net=${o['net_profit']:9.0f} trades={o['total_trades']:5} ({r.get('optimized_trades_per_day', 0):.2f}/day)"
|
||||
)
|
||||
for fid in failed_ids:
|
||||
LOG.warn(f" NEEDS FIX: {fid}")
|
||||
|
||||
LOG.banner(f"DONE in {LOG._elapsed()}")
|
||||
finally:
|
||||
mt5.shutdown()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,428 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Multi-round elimination audit: small per-strategy tricks vs solo baseline.
|
||||
|
||||
Round 1 — each trick vs baseline (123.set, one strategy enabled).
|
||||
Round 2 — stack non-conflicting WIN tricks from round 1.
|
||||
Round 3 — ±15% numeric refine around best single winner.
|
||||
|
||||
Usage:
|
||||
python -m cluster_audit.run_tweak_elimination
|
||||
python -m cluster_audit.run_tweak_elimination --only ES --rounds 1
|
||||
python -m cluster_audit.run_tweak_elimination --enabled-only --apply
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
from datetime import datetime
|
||||
from itertools import combinations
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
|
||||
|
||||
from set_parser import parse_set_file
|
||||
|
||||
from cluster_audit.tweak_manifest import STRATEGY_TWEAKS
|
||||
from cluster_audit.united_mt5_manifest import ALL_ENABLE_KEYS, UNITED_MT5_STRATEGIES
|
||||
from cluster_audit.united_mt5_runner import (
|
||||
BASE_SET,
|
||||
CLUSTER,
|
||||
deploy_united,
|
||||
mt5_context,
|
||||
patch_set,
|
||||
run_backtest,
|
||||
)
|
||||
|
||||
OUT = Path(__file__).resolve().parent / "reports" / "tweak_elimination"
|
||||
CHECKPOINT = OUT / "checkpoint.json"
|
||||
|
||||
|
||||
def solo_overrides(spec: dict) -> dict:
|
||||
o: dict = {k: False for k in ALL_ENABLE_KEYS}
|
||||
o[spec["enable"]] = True
|
||||
o["GAP_Enable"] = False
|
||||
o["OPT_GuardOptimizationMode"] = True
|
||||
return o
|
||||
|
||||
|
||||
def g(m: dict, k: str) -> float:
|
||||
v = m.get(k)
|
||||
return float(v) if v is not None else 0.0
|
||||
|
||||
|
||||
def classify(base: dict, var: dict) -> str:
|
||||
if not var.get("ready"):
|
||||
return "BROKEN"
|
||||
if var.get("total_trades", 0) == 0:
|
||||
return "NO_TRADES"
|
||||
d_net = g(var, "net_profit") - g(base, "net_profit")
|
||||
d_sh = g(var, "sharpe") - g(base, "sharpe")
|
||||
if d_net > 35 and d_sh >= -0.05:
|
||||
return "WIN"
|
||||
if d_net < -35 or d_sh < -0.12:
|
||||
return "LOSE"
|
||||
return "NEUTRAL"
|
||||
|
||||
|
||||
def delta(base: dict, var: dict) -> dict:
|
||||
return {
|
||||
"net_profit_delta": g(var, "net_profit") - g(base, "net_profit"),
|
||||
"sharpe_delta": g(var, "sharpe") - g(base, "sharpe"),
|
||||
"pf_delta": g(var, "profit_factor") - g(base, "profit_factor"),
|
||||
"trades_delta": int(g(var, "total_trades") - g(base, "total_trades")),
|
||||
}
|
||||
|
||||
|
||||
def merge_params(*dicts: dict) -> dict:
|
||||
out: dict = {}
|
||||
for d in dicts:
|
||||
out.update(d)
|
||||
return out
|
||||
|
||||
|
||||
def numeric_refine(params: dict, factor: float) -> dict:
|
||||
out: dict = {}
|
||||
for k, v in params.items():
|
||||
if isinstance(v, (int, float)) and not isinstance(v, bool):
|
||||
nv = round(v * factor, 4)
|
||||
out[k] = int(nv) if isinstance(v, int) else nv
|
||||
else:
|
||||
out[k] = v
|
||||
return out
|
||||
|
||||
|
||||
def run_variant(ctx: dict, spec: dict, base_ov: dict, extra: dict, tag: str) -> dict:
|
||||
ov = {**base_ov, **extra}
|
||||
body = patch_set(BASE_SET, ov)
|
||||
safe_tag = re.sub(r"[^\w.-]", "_", tag)[:48]
|
||||
last: dict = {"ready": False}
|
||||
for attempt in range(3):
|
||||
if attempt:
|
||||
time.sleep(6)
|
||||
print(f" retry {attempt} {safe_tag}", flush=True)
|
||||
last = run_backtest(
|
||||
ctx["data"], ctx["mt5_path"], ctx["login"], ctx["server"],
|
||||
body, f"twk_{spec['id']}_{safe_tag}.set", f"twk_{spec['id']}_{safe_tag}",
|
||||
)
|
||||
if last.get("ready"):
|
||||
break
|
||||
time.sleep(2)
|
||||
return last
|
||||
|
||||
|
||||
def load_checkpoint() -> dict[str, dict]:
|
||||
if CHECKPOINT.exists():
|
||||
return json.loads(CHECKPOINT.read_text(encoding="utf-8"))
|
||||
return {}
|
||||
|
||||
|
||||
def save_checkpoint(all_results: list[dict], *, rounds: int, strategies: list[dict]) -> None:
|
||||
OUT.mkdir(parents=True, exist_ok=True)
|
||||
done = {r["id"]: r for r in all_results if "final" in r}
|
||||
payload = {
|
||||
"updated": datetime.now().isoformat(),
|
||||
"rounds": rounds,
|
||||
"completed": list(done.keys()),
|
||||
"results": all_results,
|
||||
}
|
||||
CHECKPOINT.write_text(json.dumps(payload, indent=2, default=str), encoding="utf-8")
|
||||
summary = {
|
||||
"generated": payload["updated"],
|
||||
"rounds": rounds,
|
||||
"strategies_tested": len(strategies),
|
||||
"completed": payload["completed"],
|
||||
"applied": [sid for sid, r in done.items() if r.get("final", {}).get("action") == "APPLY"],
|
||||
"kept_baseline": [sid for sid, r in done.items() if r.get("final", {}).get("action") == "KEEP_BASELINE"],
|
||||
"results": all_results,
|
||||
}
|
||||
(OUT / "summary.json").write_text(json.dumps(summary, indent=2, default=str), encoding="utf-8")
|
||||
|
||||
|
||||
def round1(ctx: dict, spec: dict, base_ov: dict) -> tuple[dict, list[dict]]:
|
||||
sid = spec["id"]
|
||||
tricks = STRATEGY_TWEAKS.get(sid, [])
|
||||
print(f"\n{'='*60}\n[{sid}] Round 1 — {len(tricks)} tricks\n{'='*60}", flush=True)
|
||||
|
||||
base = run_variant(ctx, spec, base_ov, {}, "base")
|
||||
print(f" BASE net={base.get('net_profit')} sharpe={base.get('sharpe')} "
|
||||
f"trades={base.get('total_trades')} ({base.get('elapsed_sec')}s)", flush=True)
|
||||
|
||||
results: list[dict] = []
|
||||
for tw in tricks:
|
||||
m = run_variant(ctx, spec, base_ov, tw["params"], tw["name"])
|
||||
v = classify(base, m)
|
||||
d = delta(base, m)
|
||||
row = {
|
||||
"round": 1,
|
||||
"name": tw["name"],
|
||||
"params": tw["params"],
|
||||
"verdict": v,
|
||||
"delta": d,
|
||||
"metrics": m,
|
||||
}
|
||||
results.append(row)
|
||||
print(f" {tw['name']:22s} {v:8s} dNet={d['net_profit_delta']:+.0f} "
|
||||
f"dSharpe={d['sharpe_delta']:+.3f} trades={m.get('total_trades')}", flush=True)
|
||||
|
||||
return base, results
|
||||
|
||||
|
||||
def round2(ctx: dict, spec: dict, base_ov: dict, winners: list[dict]) -> list[dict]:
|
||||
if len(winners) < 2:
|
||||
return []
|
||||
sid = spec["id"]
|
||||
print(f" [{sid}] Round 2 — stack {len(winners)} winners", flush=True)
|
||||
out: list[dict] = []
|
||||
for a, b in combinations(winners, 2):
|
||||
keys_a = set(a["params"])
|
||||
keys_b = set(b["params"])
|
||||
if keys_a & keys_b:
|
||||
continue
|
||||
combo_name = f"{a['name']}+{b['name']}"
|
||||
params = merge_params(a["params"], b["params"])
|
||||
m = run_variant(ctx, spec, base_ov, params, combo_name.replace("+", "_"))
|
||||
# compare vs best single winner metrics stored in winners
|
||||
best_single = max(winners, key=lambda w: g(w["metrics"], "net_profit"))
|
||||
v = classify(best_single["metrics"], m)
|
||||
d = delta(best_single["metrics"], m)
|
||||
row = {
|
||||
"round": 2,
|
||||
"name": combo_name,
|
||||
"params": params,
|
||||
"verdict": v,
|
||||
"delta_vs_best_single": d,
|
||||
"metrics": m,
|
||||
}
|
||||
out.append(row)
|
||||
print(f" {combo_name:30s} {v:8s} dNet={d['net_profit_delta']:+.0f} "
|
||||
f"dSharpe={d['sharpe_delta']:+.3f}", flush=True)
|
||||
return out
|
||||
|
||||
|
||||
def round3(ctx: dict, spec: dict, base_ov: dict, best: dict) -> list[dict]:
|
||||
sid = spec["id"]
|
||||
numeric = {k: v for k, v in best["params"].items() if isinstance(v, (int, float))}
|
||||
if not numeric:
|
||||
return []
|
||||
print(f" [{sid}] Round 3 — refine {best['name']}", flush=True)
|
||||
out: list[dict] = []
|
||||
for fac, label in ((0.85, "refine_lo"), (1.15, "refine_hi")):
|
||||
params = merge_params(
|
||||
{k: v for k, v in best["params"].items() if k not in numeric},
|
||||
numeric_refine(numeric, fac),
|
||||
)
|
||||
m = run_variant(ctx, spec, base_ov, params, f"{best['name']}_{label}")
|
||||
v = classify(best["metrics"], m)
|
||||
d = delta(best["metrics"], m)
|
||||
out.append({
|
||||
"round": 3,
|
||||
"name": f"{best['name']}_{label}",
|
||||
"params": params,
|
||||
"verdict": v,
|
||||
"delta_vs_best": d,
|
||||
"metrics": m,
|
||||
})
|
||||
print(f" {label:12s} {v:8s} dNet={d['net_profit_delta']:+.0f} "
|
||||
f"dSharpe={d['sharpe_delta']:+.3f}", flush=True)
|
||||
return out
|
||||
|
||||
|
||||
def pick_final(base: dict, r1: list[dict], r2: list[dict], r3: list[dict]) -> dict:
|
||||
candidates: list[dict] = []
|
||||
for r in r1:
|
||||
if r["verdict"] == "WIN":
|
||||
candidates.append({"source": f"r1:{r['name']}", "params": r["params"], "metrics": r["metrics"]})
|
||||
for r in r2:
|
||||
if r["verdict"] == "WIN":
|
||||
candidates.append({"source": f"r2:{r['name']}", "params": r["params"], "metrics": r["metrics"]})
|
||||
for r in r3:
|
||||
if r["verdict"] == "WIN":
|
||||
candidates.append({"source": f"r3:{r['name']}", "params": r["params"], "metrics": r["metrics"]})
|
||||
|
||||
if not candidates:
|
||||
return {"action": "KEEP_BASELINE", "params": {}, "baseline": base}
|
||||
|
||||
best = max(candidates, key=lambda c: (g(c["metrics"], "sharpe"), g(c["metrics"], "net_profit")))
|
||||
return {
|
||||
"action": "APPLY",
|
||||
"source": best["source"],
|
||||
"params": best["params"],
|
||||
"metrics": best["metrics"],
|
||||
"baseline": base,
|
||||
"improvement": delta(base, best["metrics"]),
|
||||
}
|
||||
|
||||
|
||||
def _format_mq5_value(old_val: str, val: object) -> str:
|
||||
old = old_val.strip()
|
||||
if isinstance(val, bool):
|
||||
return "true" if val else "false"
|
||||
if isinstance(val, int):
|
||||
return str(val)
|
||||
if isinstance(val, float):
|
||||
if "." in old:
|
||||
return f"{val:.1f}" if val == int(val) else str(val)
|
||||
return str(int(val)) if val == int(val) else str(val)
|
||||
if isinstance(val, str):
|
||||
return f'"{val}"' if not (old.startswith('"') and old.endswith('"')) else f'"{val}"'
|
||||
return str(val)
|
||||
|
||||
|
||||
def apply_to_mq5_and_set(winners: dict[str, dict]) -> None:
|
||||
mq5 = CLUSTER / "main.mq5"
|
||||
st = CLUSTER / "123.set"
|
||||
text = mq5.read_text(encoding="utf-8")
|
||||
set_lines = st.read_text(encoding="utf-8", errors="ignore").splitlines()
|
||||
n_applied = 0
|
||||
|
||||
for _sid, w in winners.items():
|
||||
if w.get("action") != "APPLY":
|
||||
continue
|
||||
for key, val in w["params"].items():
|
||||
pat = rf"(input\s+(?:bool|int|double|string|ENUM_\w+)\s+{re.escape(key)}\s*=\s*)([^;]+)(;)"
|
||||
|
||||
def repl(m: re.Match, v: object = val) -> str:
|
||||
return f"{m.group(1)}{_format_mq5_value(m.group(2), v)}{m.group(3)}"
|
||||
|
||||
new_text, n = re.subn(pat, repl, text, count=1)
|
||||
if n:
|
||||
text = new_text
|
||||
n_applied += 1
|
||||
else:
|
||||
print(f" WARN mq5 miss {key}", flush=True)
|
||||
|
||||
for i, line in enumerate(set_lines):
|
||||
if not line.startswith(f"{key}="):
|
||||
continue
|
||||
sv = "true" if val is True else "false" if val is False else str(val)
|
||||
parts = line.split("||")
|
||||
if len(parts) >= 5:
|
||||
parts[0] = f"{key}={sv}"
|
||||
set_lines[i] = "||".join(parts)
|
||||
else:
|
||||
set_lines[i] = f"{key}={sv}"
|
||||
break
|
||||
|
||||
if re.search(r'#property version\s+"[\d.]+"', text):
|
||||
text = re.sub(r'(#property version\s+)"[\d.]+"', r'\g<1>"1.27"', text, count=1)
|
||||
|
||||
mq5.write_text(text, encoding="utf-8")
|
||||
st.write_text("\n".join(set_lines) + "\n", encoding="utf-8")
|
||||
print(f"Applied {n_applied} param updates -> main.mq5 + 123.set", flush=True)
|
||||
|
||||
|
||||
def audit_strategy(ctx: dict, spec: dict, rounds: int) -> dict:
|
||||
base_ov = solo_overrides(spec)
|
||||
base, r1 = round1(ctx, spec, base_ov)
|
||||
winners = [r for r in r1 if r["verdict"] == "WIN"]
|
||||
|
||||
r2: list[dict] = []
|
||||
r3: list[dict] = []
|
||||
if rounds >= 2 and len(winners) >= 2:
|
||||
r2 = round2(ctx, spec, base_ov, winners)
|
||||
winners += [r for r in r2 if r["verdict"] == "WIN"]
|
||||
|
||||
if rounds >= 3 and winners:
|
||||
best = max(winners, key=lambda w: g(w["metrics"], "net_profit"))
|
||||
r3 = round3(ctx, spec, base_ov, best)
|
||||
winners += [r for r in r3 if r["verdict"] == "WIN"]
|
||||
|
||||
final = pick_final(base, r1, r2, r3)
|
||||
print(f" => {final['action']} {final.get('source', '')} "
|
||||
f"dNet={final.get('improvement', {}).get('net_profit_delta', 0):+.0f}", flush=True)
|
||||
return {
|
||||
"id": spec["id"],
|
||||
"name": spec["name"],
|
||||
"baseline": base,
|
||||
"round1": r1,
|
||||
"round2": r2,
|
||||
"round3": r3,
|
||||
"final": final,
|
||||
"eliminated": [r["name"] for r in r1 if r["verdict"] == "LOSE"],
|
||||
"neutral": [r["name"] for r in r1 if r["verdict"] == "NEUTRAL"],
|
||||
}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
p = argparse.ArgumentParser()
|
||||
p.add_argument("--only", default=None)
|
||||
p.add_argument("--from", dest="from_id", default=None)
|
||||
p.add_argument("--enabled-only", action="store_true")
|
||||
p.add_argument("--rounds", type=int, default=3, choices=[1, 2, 3])
|
||||
p.add_argument("--apply", action="store_true", help="Write WIN params into main.mq5 + 123.set")
|
||||
p.add_argument("--resume", action="store_true", help="Skip strategies already in checkpoint.json")
|
||||
args = p.parse_args()
|
||||
|
||||
OUT.mkdir(parents=True, exist_ok=True)
|
||||
base_params = parse_set_file(BASE_SET)
|
||||
|
||||
strategies = list(UNITED_MT5_STRATEGIES)
|
||||
if args.enabled_only:
|
||||
strategies = [
|
||||
s for s in strategies
|
||||
if base_params.get(s["enable"], type("x", (), {"value": False})).value
|
||||
]
|
||||
if args.only:
|
||||
strategies = [s for s in strategies if s["id"] == args.only]
|
||||
elif args.from_id:
|
||||
found = False
|
||||
filtered = []
|
||||
for s in strategies:
|
||||
if s["id"] == args.from_id:
|
||||
found = True
|
||||
if found:
|
||||
filtered.append(s)
|
||||
strategies = filtered if found else strategies
|
||||
|
||||
print(f"Tweak elimination | {len(strategies)} strategies | rounds={args.rounds} | base={BASE_SET.name}")
|
||||
ctx = mt5_context()
|
||||
deploy_united(ctx["data"], ctx["mt5_path"])
|
||||
print("Compiled main.ex5 OK", flush=True)
|
||||
|
||||
all_results: list[dict] = []
|
||||
if args.resume and CHECKPOINT.exists():
|
||||
ck = json.loads(CHECKPOINT.read_text(encoding="utf-8"))
|
||||
all_results = ck.get("results", [])
|
||||
done_ids = {r["id"] for r in all_results if "final" in r}
|
||||
strategies = [s for s in strategies if s["id"] not in done_ids]
|
||||
print(f"Resume: skipping {len(done_ids)} done, {len(strategies)} remaining", flush=True)
|
||||
|
||||
for spec in strategies:
|
||||
try:
|
||||
result = audit_strategy(ctx, spec, args.rounds)
|
||||
all_results.append(result)
|
||||
save_checkpoint(all_results, rounds=args.rounds, strategies=strategies)
|
||||
except Exception as ex:
|
||||
print(f" ERROR {spec['id']}: {ex}", flush=True)
|
||||
all_results.append({"id": spec["id"], "error": str(ex)})
|
||||
save_checkpoint(all_results, rounds=args.rounds, strategies=strategies)
|
||||
|
||||
apply_map = {r["id"]: r["final"] for r in all_results if "final" in r}
|
||||
applied = [sid for sid, f in apply_map.items() if f.get("action") == "APPLY"]
|
||||
|
||||
summary = {
|
||||
"generated": datetime.now().isoformat(),
|
||||
"rounds": args.rounds,
|
||||
"strategies_tested": len(strategies),
|
||||
"applied": applied,
|
||||
"kept_baseline": [sid for sid, f in apply_map.items() if f.get("action") == "KEEP_BASELINE"],
|
||||
"results": all_results,
|
||||
}
|
||||
out_path = OUT / "summary.json"
|
||||
out_path.write_text(json.dumps(summary, indent=2, default=str), encoding="utf-8")
|
||||
print(f"\nSaved {out_path}")
|
||||
print(f"APPLY ({len(applied)}): {applied}")
|
||||
|
||||
if args.apply and applied:
|
||||
apply_to_mq5_and_set(apply_map)
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,133 @@
|
||||
"""
|
||||
Sequential United EA audit (main.mq5 strategies from 123.set).
|
||||
|
||||
Usage:
|
||||
python -m cluster_audit.run_united_sequential [trials] [--only ID] [--from ID] [--continue-on-fail]
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import random
|
||||
import sys
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
import MetaTrader5 as mt5
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
|
||||
|
||||
from cluster_audit.run_sequential import run_one_strategy # noqa: E402
|
||||
from cluster_audit.scoring import ( # noqa: E402
|
||||
DEFAULT_TRADES_PER_DAY,
|
||||
min_trades_for_period,
|
||||
period_days,
|
||||
score_label,
|
||||
)
|
||||
from cluster_audit.trace_log import TraceLog # noqa: E402
|
||||
from cluster_audit.united_registry import PERIODS, UNITED_STRATEGIES # noqa: E402
|
||||
|
||||
PRIMARY_PERIOD = "2021-2026"
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
p = argparse.ArgumentParser(description="United EA sequential audit (main.mq5)")
|
||||
p.add_argument("trials", nargs="?", type=int, default=80)
|
||||
p.add_argument("--from", dest="from_id", default=None)
|
||||
p.add_argument("--only", default=None)
|
||||
p.add_argument("--trial-every", type=int, default=10)
|
||||
p.add_argument("--trades-per-day", type=float, default=DEFAULT_TRADES_PER_DAY)
|
||||
p.add_argument("--continue-on-fail", action="store_true")
|
||||
return p.parse_args()
|
||||
|
||||
|
||||
def main() -> None:
|
||||
args = parse_args()
|
||||
log = TraceLog(enabled=True, trial_every=args.trial_every)
|
||||
|
||||
start, end = PERIODS[PRIMARY_PERIOD]
|
||||
days = period_days(datetime.fromisoformat(start), datetime.fromisoformat(end))
|
||||
strategies = UNITED_STRATEGIES
|
||||
if args.only:
|
||||
strategies = [s for s in UNITED_STRATEGIES if s["id"] == args.only]
|
||||
elif args.from_id:
|
||||
found = False
|
||||
filtered = []
|
||||
for s in UNITED_STRATEGIES:
|
||||
if s["id"] == args.from_id:
|
||||
found = True
|
||||
if found:
|
||||
filtered.append(s)
|
||||
strategies = filtered if found else UNITED_STRATEGIES
|
||||
|
||||
log.banner(
|
||||
f"UNITED EA AUDIT - {len(strategies)} strategies | "
|
||||
f">={args.trades_per_day:.1f} trade/day | {args.trials} trials"
|
||||
)
|
||||
|
||||
if not mt5.initialize():
|
||||
log.error(f"MT5 init failed: {mt5.last_error()}")
|
||||
raise SystemExit(1)
|
||||
|
||||
out_dir = Path(__file__).parent / "reports" / "united_sequential"
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
rng = random.Random(42)
|
||||
results: list[dict] = []
|
||||
passed_ids: list[str] = []
|
||||
failed_ids: list[str] = []
|
||||
|
||||
try:
|
||||
for i, spec in enumerate(strategies, 1):
|
||||
r = run_one_strategy(
|
||||
spec, PRIMARY_PERIOD, start, end, args.trials, rng, i, len(strategies),
|
||||
days, args.trades_per_day,
|
||||
)
|
||||
# relocate report to united folder
|
||||
src = Path(__file__).parent / "reports" / "sequential" / f"{spec['id']}_{PRIMARY_PERIOD}.json"
|
||||
dst = out_dir / f"{spec['id']}_{PRIMARY_PERIOD}.json"
|
||||
if src.exists():
|
||||
dst.write_text(src.read_text(encoding="utf-8"), encoding="utf-8")
|
||||
|
||||
results.append(r)
|
||||
if r.get("passed"):
|
||||
passed_ids.append(r["id"])
|
||||
elif "error" not in r:
|
||||
failed_ids.append(r["id"])
|
||||
if not args.continue_on_fail and not args.only:
|
||||
log.warn(f"STOPPED at {r['id']} — resume with --from {r['id']} --continue-on-fail")
|
||||
break
|
||||
|
||||
summary = {
|
||||
"generated": datetime.now().isoformat(),
|
||||
"period_days": days,
|
||||
"passed": passed_ids,
|
||||
"failed": failed_ids,
|
||||
"results": [{k: v for k, v in r.items() if k != "spec"} for r in results],
|
||||
}
|
||||
(out_dir / "united_summary.json").write_text(json.dumps(summary, indent=2), encoding="utf-8")
|
||||
|
||||
try:
|
||||
from cluster_audit.sync_united import main as sync_united
|
||||
sync_united()
|
||||
log.info("main.mq5 + UnitedEA_Optimized.set updated")
|
||||
except Exception as ex:
|
||||
log.warn(f"sync_united skipped: {ex}")
|
||||
|
||||
log.banner(f"PASSED {len(passed_ids)} / FAILED {len(failed_ids)}")
|
||||
for r in results:
|
||||
if not r.get("passed"):
|
||||
continue
|
||||
o = r["optimized"]
|
||||
log.info(
|
||||
f" {r['id']:28} net=${o['net_profit']:9.0f} "
|
||||
f"trades={o['total_trades']:5} pf={o['profit_factor']:.2f}"
|
||||
)
|
||||
for fid in failed_ids:
|
||||
log.warn(f" NEEDS FIX: {fid}")
|
||||
finally:
|
||||
mt5.shutdown()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,91 @@
|
||||
#!/usr/bin/env python3
|
||||
"""US30 lot sweep — pick lot balancing return vs equity drawdown."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
|
||||
|
||||
from cluster_audit.united_mt5_manifest import ALL_ENABLE_KEYS, UNITED_MT5_STRATEGIES
|
||||
from cluster_audit.united_mt5_runner import BASE_SET, deploy_united, mt5_context, patch_set, run_backtest
|
||||
|
||||
OUT = Path(__file__).resolve().parent / "reports" / "us30_lot_dd"
|
||||
LOTS = [0.03, 0.04, 0.05, 0.06, 0.07, 0.08]
|
||||
MAX_DD_PCT = 35.0 # reject lots with equity DD above this
|
||||
|
||||
|
||||
def parse_dd_pct(dd: str | None) -> float | None:
|
||||
if not dd:
|
||||
return None
|
||||
m = re.search(r"([\d.]+)\s*%", str(dd).replace(",", ""))
|
||||
return float(m.group(1)) if m else None
|
||||
|
||||
|
||||
def main() -> None:
|
||||
sm = {s["id"]: s for s in UNITED_MT5_STRATEGIES}
|
||||
spec = sm["RS_US30"]
|
||||
ov_base = {
|
||||
k: False for k in ALL_ENABLE_KEYS
|
||||
}
|
||||
ov_base[spec["enable"]] = True
|
||||
ov_base.update({
|
||||
"ORCH_ReferenceBalance": 3000.0,
|
||||
"ORCH_ScaleLotsByBalance": True,
|
||||
"GAP_Enable": False,
|
||||
"OPT_GuardOptimizationMode": True,
|
||||
"EnableRSIScalpingMU": False,
|
||||
})
|
||||
|
||||
ctx = mt5_context()
|
||||
deploy_united(ctx["data"], ctx["mt5_path"])
|
||||
OUT.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
trials: list[dict] = []
|
||||
best_lot, best_sc, best_row = LOTS[0], -1e18, {}
|
||||
|
||||
for lot in LOTS:
|
||||
ov = {**ov_base, spec["lot"]: lot}
|
||||
tag = str(lot).replace(".", "p")
|
||||
m = run_backtest(
|
||||
ctx["data"], ctx["mt5_path"], ctx["login"], ctx["server"],
|
||||
patch_set(BASE_SET, ov), f"us30_dd_{tag}.set", f"us30_dd_{tag}",
|
||||
)
|
||||
dd_pct = parse_dd_pct(m.get("max_drawdown"))
|
||||
pf = float(m.get("profit_factor") or 0)
|
||||
sharpe = float(m.get("sharpe") or 0)
|
||||
profit = float(m.get("net_profit") or 0)
|
||||
trades = int(m.get("total_trades") or 0)
|
||||
if not m.get("ready") or trades < 20 or pf < 1.0:
|
||||
sc = -1e10
|
||||
elif dd_pct is not None and dd_pct > MAX_DD_PCT:
|
||||
sc = sharpe * 500 + profit / 2000 - dd_pct * 100
|
||||
else:
|
||||
sc = sharpe * 2000 + profit / 500 + pf * 50 - (dd_pct or 0) * 20
|
||||
row = {"lot": lot, "dd_pct": dd_pct, "score": sc, "metrics": m}
|
||||
trials.append(row)
|
||||
print(
|
||||
f"lot={lot} PF={pf} net={profit} sharpe={sharpe} dd={m.get('max_drawdown')} sc={sc:.0f}",
|
||||
flush=True,
|
||||
)
|
||||
if sc > best_sc:
|
||||
best_sc, best_lot, best_row = sc, lot, row
|
||||
|
||||
# Prefer highest lot under DD cap with PF>=1.1
|
||||
under_cap = [t for t in trials if t.get("dd_pct") is not None and t["dd_pct"] <= MAX_DD_PCT
|
||||
and (t["metrics"].get("profit_factor") or 0) >= 1.1]
|
||||
if under_cap:
|
||||
best_lot = max(under_cap, key=lambda t: t["lot"])["lot"]
|
||||
best_row = next(t for t in trials if t["lot"] == best_lot)
|
||||
|
||||
result = {"best_lot": best_lot, "max_dd_cap_pct": MAX_DD_PCT, "best": best_row, "trials": trials}
|
||||
OUT.mkdir(parents=True, exist_ok=True)
|
||||
(OUT / "us30_lot_dd.json").write_text(json.dumps(result, indent=2), encoding="utf-8")
|
||||
print(f"BEST lot={best_lot} dd={best_row.get('dd_pct')}% PF={best_row['metrics'].get('profit_factor')}", flush=True)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,57 @@
|
||||
"""Quick solo scan for low-margin stocks using NVDA-style RSI params."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
|
||||
|
||||
from cluster_audit.united_mt5_manifest import ALL_ENABLE_KEYS, HIGH_MARGIN_STOCK_ENABLES
|
||||
from cluster_audit.united_mt5_runner import BASE_SET, deploy_united, mt5_context, patch_set, run_backtest
|
||||
import cluster_audit.united_mt5_runner as runner
|
||||
|
||||
SYMBOLS = ("SNAP.NYS", "F.NYS", "SOFI.NAS", "PFE.NYS", "AAL.NAS", "NVDA.NAS", "BAC.NYS", "WBD.NAS")
|
||||
|
||||
NVDA_PARAMS = {
|
||||
"RS_F_Symbol": "",
|
||||
"RS_F_TimeFrame": 15,
|
||||
"RS_F_RSI_Period": 8,
|
||||
"RS_F_RSI_Overbought": 36,
|
||||
"RS_F_RSI_Oversold": 38,
|
||||
"RS_F_RSI_Target_Buy": 90,
|
||||
"RS_F_RSI_Target_Sell": 70,
|
||||
"RS_F_BarsToWait": 5,
|
||||
"LOT_RS_F": 10,
|
||||
}
|
||||
|
||||
|
||||
def main() -> None:
|
||||
ctx = mt5_context()
|
||||
deploy_united(ctx["data"], ctx["mt5_path"])
|
||||
print(f"{'symbol':14} {'trades':>6} {'PF':>6} {'net':>10}")
|
||||
for sym in SYMBOLS:
|
||||
o = {k: False for k in ALL_ENABLE_KEYS}
|
||||
o["EnableRSIScalpingF"] = True
|
||||
for k in HIGH_MARGIN_STOCK_ENABLES:
|
||||
o[k] = False
|
||||
o["GAP_Enable"] = False
|
||||
o.update(NVDA_PARAMS)
|
||||
o["RS_F_Symbol"] = sym
|
||||
body = patch_set(BASE_SET, o)
|
||||
report = f"scan_{sym.replace('.', '_')}"
|
||||
old = runner.TEST_SYMBOL
|
||||
runner.TEST_SYMBOL = sym
|
||||
m = run_backtest(
|
||||
ctx["data"], ctx["mt5_path"], ctx["login"], ctx["server"],
|
||||
body, f"{report}.set", report,
|
||||
)
|
||||
runner.TEST_SYMBOL = old
|
||||
print(
|
||||
f"{sym:14} {int(m.get('total_trades') or 0):6} "
|
||||
f"{float(m.get('profit_factor') or 0):6.2f} {float(m.get('net_profit') or 0):10.2f}"
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,126 @@
|
||||
"""Strategy scoring — requires ~1 trade per calendar day over the backtest period."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import date, datetime
|
||||
|
||||
from .backtest_core import BacktestReport
|
||||
|
||||
DEFAULT_TRADES_PER_DAY = 1.0
|
||||
|
||||
|
||||
def period_days(start: date | datetime | str, end: date | datetime | str) -> int:
|
||||
if isinstance(start, str):
|
||||
start = datetime.fromisoformat(start)
|
||||
if isinstance(end, str):
|
||||
end = datetime.fromisoformat(end)
|
||||
if isinstance(start, datetime):
|
||||
start = start.date()
|
||||
if isinstance(end, datetime):
|
||||
end = end.date()
|
||||
return max(1, (end - start).days)
|
||||
|
||||
|
||||
def min_trades_for_period(days: int, trades_per_day: float = DEFAULT_TRADES_PER_DAY) -> int:
|
||||
return max(30, int(days * trades_per_day))
|
||||
|
||||
|
||||
def trades_per_day(report: BacktestReport, days: int) -> float:
|
||||
if report.total_trades == 0 or days <= 0:
|
||||
return 0.0
|
||||
return report.total_trades / days
|
||||
|
||||
|
||||
def winning_months_pct(report: BacktestReport) -> float:
|
||||
if not report.monthly_returns:
|
||||
return 0.0
|
||||
vals = list(report.monthly_returns.values())
|
||||
return 100.0 * sum(1 for v in vals if v > 0) / len(vals)
|
||||
|
||||
|
||||
def score_report(
|
||||
report: BacktestReport,
|
||||
period_days_count: int,
|
||||
trades_per_day_target: float = DEFAULT_TRADES_PER_DAY,
|
||||
) -> float:
|
||||
"""Higher is better. Hard-fails below activity + profit gates."""
|
||||
min_t = min_trades_for_period(period_days_count, trades_per_day_target)
|
||||
t = report.total_trades
|
||||
tpd = trades_per_day(report, period_days_count)
|
||||
|
||||
if t < min_t or tpd < trades_per_day_target:
|
||||
return float("-inf")
|
||||
|
||||
if report.net_profit <= 0 or report.profit_factor < 1.05:
|
||||
return float("-inf")
|
||||
|
||||
win_mo = winning_months_pct(report) / 100.0
|
||||
activity = min(tpd / (trades_per_day_target * 1.5), 1.0)
|
||||
pf = min(report.profit_factor, 4.0) / 4.0
|
||||
wr = min(report.win_rate, 70.0) / 70.0
|
||||
|
||||
return (
|
||||
report.sharpe * 0.25
|
||||
+ (report.net_profit / 2000.0) * 0.18
|
||||
- report.max_drawdown_pct * 0.10
|
||||
+ activity * 0.22
|
||||
+ win_mo * 0.12
|
||||
+ pf * 0.08
|
||||
+ wr * 0.05
|
||||
)
|
||||
|
||||
|
||||
def score_label(
|
||||
score: float,
|
||||
trades: int,
|
||||
period_days_count: int,
|
||||
trades_per_day_target: float = DEFAULT_TRADES_PER_DAY,
|
||||
) -> str:
|
||||
min_t = min_trades_for_period(period_days_count, trades_per_day_target)
|
||||
tpd = trades / period_days_count if period_days_count else 0
|
||||
if trades < min_t:
|
||||
return f"N/A ({trades}<{min_t}, need {trades_per_day_target:.1f}/day)"
|
||||
if tpd < trades_per_day_target:
|
||||
return f"N/A ({tpd:.2f}/day < {trades_per_day_target:.1f}/day)"
|
||||
if score == float("-inf"):
|
||||
return "N/A (fails profit gates)"
|
||||
return f"{score:.2f}"
|
||||
|
||||
|
||||
def acceptance(
|
||||
report: BacktestReport,
|
||||
period_days_count: int,
|
||||
trades_per_day_target: float = DEFAULT_TRADES_PER_DAY,
|
||||
) -> tuple[bool, list[str]]:
|
||||
min_t = min_trades_for_period(period_days_count, trades_per_day_target)
|
||||
tpd = trades_per_day(report, period_days_count)
|
||||
issues: list[str] = []
|
||||
|
||||
if report.total_trades < min_t:
|
||||
issues.append(f"trades={report.total_trades} need >={min_t} ({trades_per_day_target:.1f}/day x {period_days_count}d)")
|
||||
if tpd < trades_per_day_target:
|
||||
issues.append(f"trades/day={tpd:.2f} need >={trades_per_day_target:.1f}")
|
||||
if report.net_profit <= 0:
|
||||
issues.append(f"net=${report.net_profit:.0f} not positive")
|
||||
if report.profit_factor < 1.15:
|
||||
issues.append(f"pf={report.profit_factor:.2f} need >=1.15")
|
||||
if report.sharpe < 0.3:
|
||||
issues.append(f"sharpe={report.sharpe:.2f} need >=0.30")
|
||||
if report.max_drawdown_pct > 25:
|
||||
issues.append(f"dd={report.max_drawdown_pct:.1f}% too high")
|
||||
|
||||
win_mo = winning_months_pct(report)
|
||||
if win_mo < 45:
|
||||
issues.append(f"winning_months={win_mo:.0f}% need >=45%")
|
||||
|
||||
return len(issues) == 0, issues
|
||||
|
||||
|
||||
def format_quality_line(report: BacktestReport, period_days_count: int) -> str:
|
||||
tpd = trades_per_day(report, period_days_count)
|
||||
return (
|
||||
f"net=${report.net_profit:.0f} sharpe={report.sharpe:.2f} "
|
||||
f"trades={report.total_trades} ({tpd:.2f}/day) pf={report.profit_factor:.2f} "
|
||||
f"wr={report.win_rate:.0f}% win_mo={winning_months_pct(report):.0f}% "
|
||||
f"dd={report.max_drawdown_pct:.1f}%"
|
||||
)
|
||||
@@ -0,0 +1,134 @@
|
||||
"""All SuperEA robot instances with defaults and optimization ranges."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import MetaTrader5 as mt5
|
||||
|
||||
TF = {
|
||||
"M10": mt5.TIMEFRAME_M10,
|
||||
"M15": mt5.TIMEFRAME_M15,
|
||||
"M20": mt5.TIMEFRAME_M20,
|
||||
"M30": mt5.TIMEFRAME_M30,
|
||||
"H1": mt5.TIMEFRAME_H1,
|
||||
"H4": mt5.TIMEFRAME_H4,
|
||||
}
|
||||
|
||||
STRATEGIES: list[dict] = [
|
||||
{"id": "darvas_xau", "engine": "darvas", "symbol": "XAUUSD", "tf": "M15", "lot": 0.07,
|
||||
"defaults": {"box_period": 24, "box_deviation": 80000, "ma_period": 60,
|
||||
"trend_threshold": 1.2, "volume_threshold": 0,
|
||||
"stop_loss_pts": 450, "take_profit_pts": 650},
|
||||
"opt": {"box_period": (12, 48, 4), "box_deviation": (40000, 150000, 5000),
|
||||
"trend_threshold": (0.3, 5.0, 0.3), "stop_loss_pts": (250, 900, 50),
|
||||
"take_profit_pts": (350, 1200, 50), "ma_period": (30, 120, 15)}},
|
||||
{"id": "ema_slope_unit", "engine": "ema_slope", "symbol": "XAUUSD", "tf": "H1", "lot": 0.07,
|
||||
"defaults": {"ema_period": 85, "price_threshold_pips": 350, "slope_threshold_pips": 22.5,
|
||||
"monitor_timeout_sec": 340, "max_trades_per_crossover": 12, "use_trailing_stop": False,
|
||||
"trailing_stop_pips": 74, "max_loss_atr": 1.8, "use_bar_data": True,
|
||||
"close_unprofitable_trades": True, "profit_check_bars": 36,
|
||||
"use_weekly_adx_filter": True, "weekly_adx_period": 28,
|
||||
"weekly_adx_min": 25, "weekly_adx_bar_shift": 8, "weekly_adx_use_direction": True},
|
||||
"opt": {"ema_period": (50, 120, 5), "price_threshold_pips": (150, 500, 50),
|
||||
"slope_threshold_pips": (10, 50, 5), "max_loss_atr": (1.2, 3.0, 0.3),
|
||||
"profit_check_bars": (18, 60, 6), "max_trades_per_crossover": (3, 20, 3)}},
|
||||
{"id": "ema_slope_trail", "engine": "ema_slope", "symbol": "XAUUSD", "tf": "H1", "lot": 0.07,
|
||||
"defaults": {"ema_period": 50, "price_threshold_pips": 700, "slope_threshold_pips": 25,
|
||||
"monitor_timeout_sec": 340, "max_trades_per_crossover": 3, "use_trailing_stop": True,
|
||||
"trailing_stop_pips": 370, "max_loss_atr": 2.0, "use_bar_data": True,
|
||||
"close_unprofitable_trades": True, "profit_check_bars": 11,
|
||||
"use_weekly_adx_filter": True, "weekly_adx_period": 15,
|
||||
"weekly_adx_min": 40, "weekly_adx_bar_shift": 2, "weekly_adx_use_direction": True},
|
||||
"opt": {"ema_period": (30, 90, 5), "trailing_stop_pips": (150, 450, 50),
|
||||
"max_loss_atr": (1.2, 3.5, 0.3), "weekly_adx_min": (25, 50, 5)}},
|
||||
{"id": "mean_rev_btc", "engine": "mean_reversion", "symbol": "BTCUSD", "tf": "M15", "lot": 0.01,
|
||||
"defaults": {"ema_period": 80, "min_ema_distance_pts": 300, "rsi_period": 14,
|
||||
"rsi_oversold": 40, "rsi_overbought": 70, "adx_period": 14,
|
||||
"adx_max_for_entry": 30, "adx_escape": 40, "use_rsi_cross": True,
|
||||
"use_hard_sltp": False, "sl_points": 1300, "tp_points": 13400},
|
||||
"opt": {"rsi_oversold": (30, 50, 5), "rsi_overbought": (60, 80, 5),
|
||||
"min_ema_distance_pts": (100, 1500, 100), "adx_max_for_entry": (22, 40, 3),
|
||||
"ema_period": (40, 120, 20)}},
|
||||
{"id": "rsi_cross_xau", "engine": "rsi_crossover", "symbol": "XAUUSD", "tf": "H1", "lot": 0.01,
|
||||
"defaults": {"rsi_period": 19, "overbought_level": 93, "oversold_level": 22, "ema_period": 140,
|
||||
"ema_slope_threshold": 105, "ema_distance_threshold": 165, "exit_buy_rsi": 86,
|
||||
"exit_sell_rsi": 10, "trailing_stop_pts": 295, "cooldown_seconds": 209,
|
||||
"tuesday": True, "wednesday": True, "thursday": True,
|
||||
"trading_hour_one_begin": 0, "trading_hour_one_end": 22,
|
||||
"trading_hour_two_begin": 6, "trading_hour_two_end": 19},
|
||||
"opt": {"overbought_level": (65, 95, 5), "oversold_level": (10, 45, 5),
|
||||
"ema_slope_threshold": (20, 200, 20), "ema_distance_threshold": (40, 300, 20)}},
|
||||
{"id": "rsi_asian_eur", "engine": "rsi_asian", "symbol": "EURUSD", "tf": "M15", "lot": 0.1,
|
||||
"defaults": {"rsi_period": 28, "overbought_level": 60, "oversold_level": 8,
|
||||
"asian_session_start": 0, "asian_session_end": 8, "use_rsi_exit": True, "rsi_exit_level": 55},
|
||||
"opt": {"overbought_level": (50, 80, 5), "oversold_level": (5, 40, 5)}},
|
||||
{"id": "rsi_asian_aud", "engine": "rsi_asian", "symbol": "AUDUSD", "tf": "M15", "lot": 0.1,
|
||||
"defaults": {"rsi_period": 28, "overbought_level": 68, "oversold_level": 30,
|
||||
"asian_session_start": 0, "asian_session_end": 8, "use_rsi_exit": True, "rsi_exit_level": 55},
|
||||
"opt": {"overbought_level": (55, 80, 5), "oversold_level": (15, 45, 5)}},
|
||||
{"id": "rsi_asian_gbp", "engine": "rsi_asian", "symbol": "GBPUSD", "tf": "M15", "lot": 0.1,
|
||||
"defaults": {"rsi_period": 28, "overbought_level": 80, "oversold_level": 37,
|
||||
"asian_session_start": 0, "asian_session_end": 8, "use_rsi_exit": True, "rsi_exit_level": 55},
|
||||
"opt": {"overbought_level": (60, 85, 5), "oversold_level": (20, 50, 5)}},
|
||||
{"id": "rsi_secret_xau", "engine": "rsi_secret", "symbol": "XAUUSD", "tf": "M30", "lot": 0.01,
|
||||
"defaults": {"rsi_period": 16, "rsi_overbought": 72.5, "rsi_oversold": 32.5,
|
||||
"stop_loss_atr": 2.75, "take_profit_atr": 5.0, "min_bars_between_trades": 7},
|
||||
"opt": {"rsi_overbought": (60, 80, 2), "rsi_oversold": (25, 45, 2), "stop_loss_atr": (1.5, 4, 0.5)}},
|
||||
]
|
||||
|
||||
_RSI_SCALPS = [
|
||||
("rsi_scalp_appl_unit", "AAPL", "M10", 25.0, {"rsi_period": 14, "rsi_overbought": 80, "rsi_oversold": 78,
|
||||
"rsi_target_buy": 94, "rsi_target_sell": 44, "bars_to_wait": 7, "use_trailing": False}),
|
||||
("rsi_scalp_appl_trail", "AAPL", "H1", 25.0, {"rsi_period": 8, "rsi_overbought": 62, "rsi_oversold": 32,
|
||||
"rsi_target_buy": 67, "rsi_target_sell": 2, "bars_to_wait": 5, "use_trailing": True,
|
||||
"trail_distance_pts": 50, "trail_activation_pts": 39}),
|
||||
("rsi_scalp_adbe_trail", "ADBE", "H1", 5.0, {"rsi_period": 15, "rsi_overbought": 16, "rsi_oversold": 42,
|
||||
"rsi_target_buy": 67, "rsi_target_sell": 62, "bars_to_wait": 8, "use_trailing": True,
|
||||
"trail_distance_pts": 425, "trail_activation_pts": 18.5}),
|
||||
("rsi_scalp_btc_unit", "BTCUSD", "H1", 0.1, {"rsi_period": 14, "rsi_overbought": 90, "rsi_oversold": 73,
|
||||
"rsi_target_buy": 88, "rsi_target_sell": 48, "bars_to_wait": 6, "use_trailing": False}),
|
||||
("rsi_scalp_btc_trail", "BTCUSD", "H1", 0.1, {"rsi_period": 14, "rsi_overbought": 90, "rsi_oversold": 73,
|
||||
"rsi_target_buy": 88, "rsi_target_sell": 48, "bars_to_wait": 6, "use_trailing": True,
|
||||
"trail_distance_pts": 120, "trail_activation_pts": 0}),
|
||||
("rsi_scalp_mu", "MU", "H1", 25.0, {"rsi_period": 14, "rsi_overbought": 32, "rsi_oversold": 86,
|
||||
"rsi_target_buy": 100, "rsi_target_sell": 24, "bars_to_wait": 34, "use_trailing": False}),
|
||||
("rsi_scalp_nvda_unit", "NVDA", "H1", 25.0, {"rsi_period": 14, "rsi_overbought": 6, "rsi_oversold": 66,
|
||||
"rsi_target_buy": 98, "rsi_target_sell": 52, "bars_to_wait": 12, "use_trailing": False}),
|
||||
("rsi_scalp_nvda_trail", "NVDA", "M15", 50.0, {"rsi_period": 8, "rsi_overbought": 36, "rsi_oversold": 38,
|
||||
"rsi_target_buy": 90, "rsi_target_sell": 70, "bars_to_wait": 5, "use_trailing": True,
|
||||
"trail_distance_pts": 375, "trail_activation_pts": 75}),
|
||||
("rsi_scalp_nvda_trail_v2", "NVDA", "M15", 50.0, {"rsi_period": 8, "rsi_overbought": 36, "rsi_oversold": 38,
|
||||
"rsi_target_buy": 90, "rsi_target_sell": 70, "bars_to_wait": 5, "use_trailing": True,
|
||||
"trail_distance_pts": 375, "trail_activation_pts": 75}),
|
||||
("rsi_scalp_tsla_unit", "TSLA", "H1", 25.0, {"rsi_period": 14, "rsi_overbought": 32, "rsi_oversold": 86,
|
||||
"rsi_target_buy": 100, "rsi_target_sell": 24, "bars_to_wait": 34, "use_trailing": False}),
|
||||
("rsi_scalp_tsla_trail", "TSLA", "H1", 5.0, {"rsi_period": 14, "rsi_overbought": 54, "rsi_oversold": 73,
|
||||
"rsi_target_buy": 87, "rsi_target_sell": 33, "bars_to_wait": 1, "use_trailing": True,
|
||||
"trail_distance_pts": 900, "trail_activation_pts": 950}),
|
||||
("rsi_scalp_xau_trail", "XAUUSD", "H1", 0.1, {"rsi_period": 14, "rsi_overbought": 71, "rsi_oversold": 57,
|
||||
"rsi_target_buy": 80, "rsi_target_sell": 57, "bars_to_wait": 1, "use_trailing": True,
|
||||
"trail_distance_pts": 71, "trail_activation_pts": 41}),
|
||||
]
|
||||
|
||||
for sid, sym, tf, lot, defaults in _RSI_SCALPS:
|
||||
STRATEGIES.append({
|
||||
"id": sid,
|
||||
"engine": "rsi_scalp",
|
||||
"symbol": sym,
|
||||
"tf": tf,
|
||||
"lot": lot,
|
||||
"defaults": {**defaults, "tf": tf},
|
||||
"opt": {
|
||||
"rsi_period": (6, 21, 1),
|
||||
"rsi_overbought": (50, 90, 3),
|
||||
"rsi_oversold": (10, 70, 3),
|
||||
"rsi_target_buy": (60, 95, 3),
|
||||
"rsi_target_sell": (5, 70, 3),
|
||||
"bars_to_wait": (1, 8, 1),
|
||||
"trail_distance_pts": (20, 400, 20),
|
||||
},
|
||||
})
|
||||
|
||||
PERIODS = {
|
||||
"2021-2026": ("2021-01-01", "2026-06-01"),
|
||||
"2024-2026": ("2024-01-01", "2026-06-01"),
|
||||
}
|
||||
@@ -0,0 +1,317 @@
|
||||
"""
|
||||
Build cluster-latest SuperEA audit params from sequential JSON reports.
|
||||
|
||||
Usage:
|
||||
python -m cluster_audit.sync_cluster
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
from cluster_audit.scoring import DEFAULT_TRADES_PER_DAY, acceptance, period_days, trades_per_day
|
||||
from cluster_audit.strategy_registry import PERIODS, STRATEGIES
|
||||
|
||||
REPORTS = Path(__file__).parent / "reports" / "sequential"
|
||||
OUT_MQH = Path(__file__).resolve().parents[3] / "frontline" / "cluster-latest" / "SuperEA_AuditParams.mqh"
|
||||
OUT_JSON = Path(__file__).parent / "reports" / "cluster_manifest.json"
|
||||
|
||||
RSI_SCALP_IDS = [
|
||||
"rsi_scalp_appl_unit", "rsi_scalp_appl_trail", "rsi_scalp_adbe_trail",
|
||||
"rsi_scalp_btc_unit", "rsi_scalp_btc_trail", "rsi_scalp_mu",
|
||||
"rsi_scalp_nvda_unit", "rsi_scalp_nvda_trail", "rsi_scalp_nvda_trail_v2",
|
||||
"rsi_scalp_tsla_unit", "rsi_scalp_tsla_trail", "rsi_scalp_xau_trail",
|
||||
]
|
||||
RSI_INDEX = {sid: i for i, sid in enumerate(RSI_SCALP_IDS)}
|
||||
|
||||
MAGIC_MAP = {s["id"]: 401000 + i for i, s in enumerate(STRATEGIES, 1)}
|
||||
|
||||
|
||||
def load_report(sid: str) -> dict | None:
|
||||
p = REPORTS / f"{sid}_2021-2026.json"
|
||||
if not p.exists():
|
||||
return None
|
||||
return json.loads(p.read_text(encoding="utf-8"))
|
||||
|
||||
|
||||
def spec_defaults(sid: str) -> dict:
|
||||
for s in STRATEGIES:
|
||||
if s["id"] == sid:
|
||||
return dict(s["defaults"])
|
||||
return {}
|
||||
|
||||
|
||||
def evaluate_report(r: dict, days: int) -> tuple[bool, list[str]]:
|
||||
from cluster_audit.backtest_core import BacktestReport
|
||||
|
||||
o = r.get("optimized", {})
|
||||
rep = BacktestReport(
|
||||
strategy_id=r["id"],
|
||||
symbol=r.get("symbol", ""),
|
||||
timeframe=r.get("timeframe", "H1"),
|
||||
period_label="2021-2026",
|
||||
net_profit=float(o.get("net_profit", 0)),
|
||||
total_trades=int(o.get("total_trades", 0)),
|
||||
win_rate=float(o.get("win_rate", 0)),
|
||||
profit_factor=float(o.get("profit_factor", 0)),
|
||||
sharpe=float(o.get("sharpe", 0)),
|
||||
max_drawdown_pct=float(o.get("max_drawdown_pct", 0)),
|
||||
avg_win=float(o.get("avg_win", 0)),
|
||||
avg_loss=float(o.get("avg_loss", 0)),
|
||||
worst_trades=o.get("worst_trades", []),
|
||||
losing_trades=o.get("losing_trades", []),
|
||||
exit_reason_breakdown=o.get("exit_reason_breakdown", {}),
|
||||
monthly_returns=o.get("monthly_returns", {}),
|
||||
params=o.get("params", {}),
|
||||
)
|
||||
if r.get("passed") is True:
|
||||
ok, issues = acceptance(rep, days, DEFAULT_TRADES_PER_DAY)
|
||||
if ok:
|
||||
return True, []
|
||||
return acceptance(rep, days, DEFAULT_TRADES_PER_DAY)
|
||||
|
||||
|
||||
def _lit_bool(v) -> str:
|
||||
return "true" if v else "false"
|
||||
|
||||
|
||||
def emit_darvas(params: dict, ok: bool) -> list[str]:
|
||||
d = {**spec_defaults("darvas_xau"), **params}
|
||||
return [
|
||||
"static void SE_AuditDarvas(DarvasBoxConfig &c)",
|
||||
"{",
|
||||
f" c.box_period = {int(d['box_period'])};",
|
||||
f" c.box_deviation = {float(d['box_deviation'])};",
|
||||
f" c.ma_period = {int(d['ma_period'])};",
|
||||
f" c.trend_threshold = {float(d['trend_threshold'])};",
|
||||
f" c.stop_loss_pts = {float(d['stop_loss_pts'])};",
|
||||
f" c.take_profit_pts = {float(d['take_profit_pts'])};",
|
||||
" c.box_timeframe = PERIOD_M15;",
|
||||
" c.trend_timeframe = PERIOD_M15;",
|
||||
" c.use_close_breakout = true;",
|
||||
" c.require_volume_ma = false;",
|
||||
"}",
|
||||
f"static bool SE_AuditDarvasEnabled() {{ return {_lit_bool(ok)}; }}",
|
||||
"",
|
||||
]
|
||||
|
||||
|
||||
def emit_ema_slope(fn: str, params: dict, ok: bool) -> list[str]:
|
||||
d = params
|
||||
lines = [f"static void SE_Audit{fn}(EmaSlopeConfig &c)", "{"]
|
||||
for key, cast in [
|
||||
("ema_period", int), ("price_threshold_pips", float), ("slope_threshold_pips", float),
|
||||
("monitor_timeout_sec", int), ("trailing_stop_pips", float),
|
||||
("max_trades_per_crossover", int), ("profit_check_bars", int),
|
||||
("weekly_adx_period", int), ("weekly_adx_min", float), ("weekly_adx_bar_shift", int),
|
||||
]:
|
||||
if key in d:
|
||||
lines.append(f" c.{key} = {cast(d[key])};")
|
||||
if "use_trailing_stop" in d:
|
||||
lines.append(f" c.use_trailing_stop = {_lit_bool(d['use_trailing_stop'])};")
|
||||
lines += ["}", f"static bool SE_Audit{fn}Enabled() {{ return {_lit_bool(ok)}; }}", ""]
|
||||
return lines
|
||||
|
||||
|
||||
def emit_mean_rev(params: dict, ok: bool) -> list[str]:
|
||||
d = {**spec_defaults("mean_rev_btc"), **params}
|
||||
return [
|
||||
"static void SE_AuditMeanRev(MeanReversionConfig &c)",
|
||||
"{",
|
||||
f" c.ema_period = {int(d.get('ema_period', 250))};",
|
||||
f" c.min_ema_distance_pts = {float(d.get('min_ema_distance_pts', 3650))};",
|
||||
f" c.rsi_period = {int(d.get('rsi_period', 28))};",
|
||||
f" c.rsi_oversold = {float(d.get('rsi_oversold', 40))};",
|
||||
f" c.rsi_overbought = {float(d.get('rsi_overbought', 83))};",
|
||||
f" c.adx_period = {int(d.get('adx_period', 14))};",
|
||||
f" c.adx_max_for_entry = {float(d.get('adx_max_for_entry', 17))};",
|
||||
f" c.adx_escape = {float(d.get('adx_escape', 34))};",
|
||||
f" c.use_rsi_cross = {_lit_bool(d.get('use_rsi_cross', True))};",
|
||||
f" c.use_hard_sltp = {_lit_bool(d.get('use_hard_sltp', False))};",
|
||||
f" c.sl_points = {float(d.get('sl_points', 1300))};",
|
||||
f" c.tp_points = {float(d.get('tp_points', 13400))};",
|
||||
"}",
|
||||
f"static bool SE_AuditMeanRevEnabled() {{ return {_lit_bool(ok)}; }}",
|
||||
"",
|
||||
]
|
||||
|
||||
|
||||
def emit_rsi_cross(params: dict, ok: bool) -> list[str]:
|
||||
d = {**spec_defaults("rsi_cross_xau"), **params}
|
||||
return [
|
||||
"static void SE_AuditRsiCross(RsiCrossOverConfig &c)",
|
||||
"{",
|
||||
f" c.rsi_period = {int(d.get('rsi_period', 19))};",
|
||||
f" c.overbought_level = {float(d.get('overbought_level', 93))};",
|
||||
f" c.oversold_level = {float(d.get('oversold_level', 22))};",
|
||||
f" c.ema_period = {int(d.get('ema_period', 140))};",
|
||||
f" c.ema_slope_threshold = {float(d.get('ema_slope_threshold', 105))};",
|
||||
f" c.ema_distance_threshold = {float(d.get('ema_distance_threshold', 165))};",
|
||||
f" c.exit_buy_rsi = {float(d.get('exit_buy_rsi', 86))};",
|
||||
f" c.exit_sell_rsi = {float(d.get('exit_sell_rsi', 10))};",
|
||||
f" c.trailing_stop_pts = {float(d.get('trailing_stop_pts', 295))};",
|
||||
f" c.cooldown_seconds = {int(d.get('cooldown_seconds', 209))};",
|
||||
"}",
|
||||
f"static bool SE_AuditRsiCrossEnabled() {{ return {_lit_bool(ok)}; }}",
|
||||
"",
|
||||
]
|
||||
|
||||
|
||||
def emit_rsi_asian(fn: str, params: dict, ok: bool) -> list[str]:
|
||||
d = params
|
||||
return [
|
||||
f"static void SE_Audit{fn}(RsiAsianConfig &c)",
|
||||
"{",
|
||||
f" c.rsi_period = {int(d.get('rsi_period', 28))};",
|
||||
f" c.overbought_level = {float(d.get('overbought_level', 60))};",
|
||||
f" c.oversold_level = {float(d.get('oversold_level', 8))};",
|
||||
f" c.asian_session_start = {int(d.get('asian_session_start', 0))};",
|
||||
f" c.asian_session_end = {int(d.get('asian_session_end', 8))};",
|
||||
f" c.use_rsi_exit = {_lit_bool(d.get('use_rsi_exit', True))};",
|
||||
f" c.rsi_exit_level = {float(d.get('rsi_exit_level', 55))};",
|
||||
"}",
|
||||
f"static bool SE_Audit{fn}Enabled() {{ return {_lit_bool(ok)}; }}",
|
||||
"",
|
||||
]
|
||||
|
||||
|
||||
def emit_rsi_secret(params: dict, ok: bool) -> list[str]:
|
||||
d = {**spec_defaults("rsi_secret_xau"), **params}
|
||||
return [
|
||||
"static void SE_AuditRsiSecret(RsiSecretSauceConfig &c)",
|
||||
"{",
|
||||
f" c.rsi_period = {int(d.get('rsi_period', 16))};",
|
||||
f" c.rsi_overbought = {float(d.get('rsi_overbought', 72.5))};",
|
||||
f" c.rsi_oversold = {float(d.get('rsi_oversold', 32.5))};",
|
||||
f" c.stop_loss_atr = {float(d.get('stop_loss_atr', 2.75))};",
|
||||
f" c.take_profit_atr = {float(d.get('take_profit_atr', 5.0))};",
|
||||
f" c.min_bars_between_trades = {int(d.get('min_bars_between_trades', 7))};",
|
||||
"}",
|
||||
f"static bool SE_AuditRsiSecretEnabled() {{ return {_lit_bool(ok)}; }}",
|
||||
"",
|
||||
]
|
||||
|
||||
|
||||
def emit_rsi_scalp(idx: int, sid: str, params: dict, ok: bool) -> list[str]:
|
||||
d = {**spec_defaults(sid), **params}
|
||||
return [
|
||||
f"static void SE_AuditRsi{idx}(RsiScalpConfig &c)",
|
||||
"{",
|
||||
f" c.rsi_period = {int(d.get('rsi_period', 14))};",
|
||||
f" c.rsi_overbought = {float(d.get('rsi_overbought', 70))};",
|
||||
f" c.rsi_oversold = {float(d.get('rsi_oversold', 30))};",
|
||||
f" c.rsi_target_buy = {float(d.get('rsi_target_buy', 80))};",
|
||||
f" c.rsi_target_sell = {float(d.get('rsi_target_sell', 50))};",
|
||||
f" c.bars_to_wait = {int(d.get('bars_to_wait', 5))};",
|
||||
f" c.use_trailing = {_lit_bool(d.get('use_trailing', False))};",
|
||||
f" c.trail_distance_pts = {float(d.get('trail_distance_pts', 0))};",
|
||||
f" c.trail_activation_pts = {float(d.get('trail_activation_pts', 0))};",
|
||||
"}",
|
||||
f"static bool SE_AuditRsi{idx}Enabled() {{ return {_lit_bool(ok)}; }}",
|
||||
"",
|
||||
]
|
||||
|
||||
|
||||
def stub_enabled(name: str, ok: bool = False) -> list[str]:
|
||||
return [f"static bool SE_Audit{name}Enabled() {{ return {_lit_bool(ok)}; }}", ""]
|
||||
|
||||
|
||||
def main() -> None:
|
||||
start, end = PERIODS["2021-2026"]
|
||||
days = period_days(start, end)
|
||||
manifest: dict = {
|
||||
"generated": datetime.now().isoformat(),
|
||||
"period_days": days,
|
||||
"trades_per_day_target": DEFAULT_TRADES_PER_DAY,
|
||||
"strategies": {},
|
||||
}
|
||||
|
||||
reports: dict[str, dict] = {}
|
||||
status: dict[str, bool] = {}
|
||||
params_map: dict[str, dict] = {}
|
||||
|
||||
for spec in STRATEGIES:
|
||||
sid = spec["id"]
|
||||
r = load_report(sid)
|
||||
if not r:
|
||||
manifest["strategies"][sid] = {"status": "no_report", "passed": False}
|
||||
status[sid] = False
|
||||
params_map[sid] = spec_defaults(sid)
|
||||
continue
|
||||
reports[sid] = r
|
||||
params = r.get("optimized_params") or r.get("optimized", {}).get("params", spec_defaults(sid))
|
||||
params_map[sid] = params
|
||||
ok, issues = evaluate_report(r, days)
|
||||
o = r.get("optimized", {})
|
||||
tpd = trades_per_day(
|
||||
type("R", (), {"total_trades": int(o.get("total_trades", 0))})(),
|
||||
days,
|
||||
)
|
||||
status[sid] = ok
|
||||
manifest["strategies"][sid] = {
|
||||
"passed": ok,
|
||||
"magic": MAGIC_MAP.get(sid),
|
||||
"trades": o.get("total_trades"),
|
||||
"trades_per_day": round(tpd, 3),
|
||||
"net_profit": o.get("net_profit"),
|
||||
"sharpe": o.get("sharpe"),
|
||||
"profit_factor": o.get("profit_factor"),
|
||||
"issues": issues,
|
||||
"params": params,
|
||||
}
|
||||
|
||||
lines = [
|
||||
"//+------------------------------------------------------------------+",
|
||||
"//| SuperEA_AuditParams.mqh - optimized params from cluster audit |",
|
||||
f"//| Generated: {datetime.now().isoformat()}",
|
||||
"//+------------------------------------------------------------------+",
|
||||
"#ifndef SUPER_EA_AUDIT_PARAMS_MQH",
|
||||
"#define SUPER_EA_AUDIT_PARAMS_MQH",
|
||||
"",
|
||||
]
|
||||
|
||||
lines += [f"// darvas_xau: {'PASS' if status.get('darvas_xau') else 'DISABLED'}"]
|
||||
lines += emit_darvas(params_map.get("darvas_xau", {}), status.get("darvas_xau", False))
|
||||
|
||||
lines += [f"// ema_slope_unit: {'PASS' if status.get('ema_slope_unit') else 'DISABLED'}"]
|
||||
lines += emit_ema_slope("EmaUnit", params_map.get("ema_slope_unit", {}), status.get("ema_slope_unit", False))
|
||||
|
||||
lines += [f"// ema_slope_trail: {'PASS' if status.get('ema_slope_trail') else 'DISABLED'}"]
|
||||
lines += emit_ema_slope("EmaTrail", params_map.get("ema_slope_trail", {}), status.get("ema_slope_trail", False))
|
||||
|
||||
lines += [f"// mean_rev_btc: {'PASS' if status.get('mean_rev_btc') else 'DISABLED'}"]
|
||||
lines += emit_mean_rev(params_map.get("mean_rev_btc", {}), status.get("mean_rev_btc", False))
|
||||
|
||||
lines += [f"// rsi_cross_xau: {'PASS' if status.get('rsi_cross_xau') else 'DISABLED'}"]
|
||||
lines += emit_rsi_cross(params_map.get("rsi_cross_xau", {}), status.get("rsi_cross_xau", False))
|
||||
|
||||
for sid, fn in [
|
||||
("rsi_asian_eur", "RsiAsianEur"),
|
||||
("rsi_asian_aud", "RsiAsianAud"),
|
||||
("rsi_asian_gbp", "RsiAsianGbp"),
|
||||
]:
|
||||
lines += [f"// {sid}: {'PASS' if status.get(sid) else 'DISABLED'}"]
|
||||
lines += emit_rsi_asian(fn, params_map.get(sid, {}), status.get(sid, False))
|
||||
|
||||
lines += [f"// rsi_secret_xau: {'PASS' if status.get('rsi_secret_xau') else 'DISABLED'}"]
|
||||
lines += emit_rsi_secret(params_map.get("rsi_secret_xau", {}), status.get("rsi_secret_xau", False))
|
||||
|
||||
for sid in RSI_SCALP_IDS:
|
||||
idx = RSI_INDEX[sid]
|
||||
lines += [f"// {sid}: {'PASS' if status.get(sid) else 'DISABLED'}"]
|
||||
lines += emit_rsi_scalp(idx, sid, params_map.get(sid, {}), status.get(sid, False))
|
||||
|
||||
lines += ["#endif", ""]
|
||||
OUT_MQH.parent.mkdir(parents=True, exist_ok=True)
|
||||
OUT_MQH.write_text("\n".join(lines), encoding="utf-8")
|
||||
OUT_JSON.write_text(json.dumps(manifest, indent=2), encoding="utf-8")
|
||||
passed = [k for k, v in status.items() if v]
|
||||
print(f"Wrote {OUT_MQH}")
|
||||
print(f"Wrote {OUT_JSON}")
|
||||
print(f"Passed {len(passed)}/{len(STRATEGIES)}: {', '.join(passed) if passed else '(none)'}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,209 @@
|
||||
"""
|
||||
Sync United EA audit results into main.mq5 defaults and generate .set file.
|
||||
|
||||
Usage:
|
||||
python -m cluster_audit.sync_united
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
from cluster_audit.scoring import DEFAULT_TRADES_PER_DAY, acceptance, period_days, trades_per_day
|
||||
from cluster_audit.united_registry import PERIODS, UNITED_STRATEGIES
|
||||
|
||||
REPORTS = Path(__file__).parent / "reports" / "united_sequential"
|
||||
MAIN_MQ5 = Path(__file__).resolve().parents[3] / "frontline" / "cluster-latest" / "main.mq5"
|
||||
OUT_SET = Path(__file__).resolve().parents[3] / "frontline" / "cluster-latest" / "UnitedEA_Optimized.set"
|
||||
OUT_JSON = REPORTS / "united_manifest.json"
|
||||
|
||||
# main.mq5 input name -> (strategy_id, param_key in optimized JSON)
|
||||
PARAM_PATCHES: dict[str, tuple[str, str]] = {
|
||||
"DB_BoxPeriod": ("united_darvas", "box_period"),
|
||||
"DB_BoxDeviation": ("united_darvas", "box_deviation"),
|
||||
"DB_StopLoss": ("united_darvas", "stop_loss_pts"),
|
||||
"DB_TakeProfit": ("united_darvas", "take_profit_pts"),
|
||||
"DB_MA_Period": ("united_darvas", "ma_period"),
|
||||
"DB_TrendThreshold": ("united_darvas", "trend_threshold"),
|
||||
"RC_overboughtLevel": ("united_rsi_cross", "overbought_level"),
|
||||
"RC_oversoldLevel": ("united_rsi_cross", "oversold_level"),
|
||||
"RC_emaSlopeThreshold": ("united_rsi_cross", "ema_slope_threshold"),
|
||||
"RC_emaDistanceThreshold": ("united_rsi_cross", "ema_distance_threshold"),
|
||||
"RS_APPL_RSI_Period": ("united_rsi_scalp_appl", "rsi_period"),
|
||||
"RS_APPL_RSI_Overbought": ("united_rsi_scalp_appl", "rsi_overbought"),
|
||||
"RS_APPL_RSI_Oversold": ("united_rsi_scalp_appl", "rsi_oversold"),
|
||||
"RS_APPL_RSI_Target_Buy": ("united_rsi_scalp_appl", "rsi_target_buy"),
|
||||
"RS_APPL_RSI_Target_Sell": ("united_rsi_scalp_appl", "rsi_target_sell"),
|
||||
"RS_APPL_BarsToWait": ("united_rsi_scalp_appl", "bars_to_wait"),
|
||||
"RS_APPL_TrailDistancePoints": ("united_rsi_scalp_appl", "trail_distance_pts"),
|
||||
"RS_APPL_TrailActivationPoints": ("united_rsi_scalp_appl", "trail_activation_pts"),
|
||||
"RS_BTCUSD_RSI_Period": ("united_rsi_scalp_btc", "rsi_period"),
|
||||
"RS_BTCUSD_TrailDistancePoints": ("united_rsi_scalp_btc", "trail_distance_pts"),
|
||||
"RS_XAUUSD_RSI_Period": ("united_rsi_scalp_xau", "rsi_period"),
|
||||
"RS_XAUUSD_TrailDistancePoints": ("united_rsi_scalp_xau", "trail_distance_pts"),
|
||||
"RRA_EURUSD_OverboughtLevel": ("united_rsi_asian_eur", "overbought_level"),
|
||||
"RRA_EURUSD_OversoldLevel": ("united_rsi_asian_eur", "oversold_level"),
|
||||
"RSS_RSIOverbought": ("united_rsi_secret", "rsi_overbought"),
|
||||
"RSS_RSIOversold": ("united_rsi_secret", "rsi_oversold"),
|
||||
"UB_MinRangePoints": ("united_usdjpy", "min_range_pts"),
|
||||
"UB_OrderBufferPoints": ("united_usdjpy", "order_buffer_pts"),
|
||||
}
|
||||
|
||||
ENABLE_PATCHES: dict[str, str] = {s["enable_key"]: s["id"] for s in UNITED_STRATEGIES}
|
||||
|
||||
|
||||
def load_report(sid: str) -> dict | None:
|
||||
p = REPORTS / f"{sid}_2021-2026.json"
|
||||
if not p.exists():
|
||||
# fallback cluster audit darvas
|
||||
alt = Path(__file__).parent / "reports" / "sequential" / "darvas_xau_2021-2026.json"
|
||||
if sid == "united_darvas" and alt.exists():
|
||||
r = json.loads(alt.read_text(encoding="utf-8"))
|
||||
r["id"] = sid
|
||||
return r
|
||||
return None
|
||||
return json.loads(p.read_text(encoding="utf-8"))
|
||||
|
||||
|
||||
def evaluate(r: dict, days: int) -> tuple[bool, list[str]]:
|
||||
from cluster_audit.backtest_core import BacktestReport
|
||||
|
||||
o = r.get("optimized", {})
|
||||
rep = BacktestReport(
|
||||
strategy_id=r["id"],
|
||||
symbol=r.get("symbol", ""),
|
||||
timeframe=r.get("timeframe", "H1"),
|
||||
period_label="2021-2026",
|
||||
net_profit=float(o.get("net_profit", 0)),
|
||||
total_trades=int(o.get("total_trades", 0)),
|
||||
win_rate=float(o.get("win_rate", 0)),
|
||||
profit_factor=float(o.get("profit_factor", 0)),
|
||||
sharpe=float(o.get("sharpe", 0)),
|
||||
max_drawdown_pct=float(o.get("max_drawdown_pct", 0)),
|
||||
avg_win=float(o.get("avg_win", 0)),
|
||||
avg_loss=float(o.get("avg_loss", 0)),
|
||||
worst_trades=o.get("worst_trades", []),
|
||||
losing_trades=o.get("losing_trades", []),
|
||||
exit_reason_breakdown=o.get("exit_reason_breakdown", {}),
|
||||
monthly_returns=o.get("monthly_returns", {}),
|
||||
params=o.get("params", {}),
|
||||
)
|
||||
return acceptance(rep, days, DEFAULT_TRADES_PER_DAY)
|
||||
|
||||
|
||||
def patch_main_mqh(text: str, manifest: dict) -> str:
|
||||
params_by_sid = {k: v.get("params", {}) for k, v in manifest["strategies"].items()}
|
||||
|
||||
for input_name, (sid, pkey) in PARAM_PATCHES.items():
|
||||
params = params_by_sid.get(sid, {})
|
||||
if pkey not in params:
|
||||
continue
|
||||
val = params[pkey]
|
||||
if isinstance(val, bool):
|
||||
lit = "true" if val else "false"
|
||||
elif isinstance(val, float):
|
||||
lit = str(val) if "." in str(val) else f"{val}.0"
|
||||
else:
|
||||
lit = str(val)
|
||||
text, n = re.subn(
|
||||
rf"(^input\s+\w+\s+{re.escape(input_name)}\s*=\s*)[^;]+;",
|
||||
rf"\g<1>{lit};",
|
||||
text,
|
||||
count=1,
|
||||
flags=re.MULTILINE,
|
||||
)
|
||||
if n:
|
||||
print(f" patched {input_name}={lit}")
|
||||
|
||||
for enable_key, sid in ENABLE_PATCHES.items():
|
||||
info = manifest["strategies"].get(sid, {})
|
||||
if info.get("status") == "no_report" or "passed" not in info:
|
||||
continue
|
||||
if not info.get("passed"):
|
||||
continue # keep main.mq5 enable flags; only auto-enable winners
|
||||
lit = "true"
|
||||
text, n = re.subn(
|
||||
rf"(^input bool {re.escape(enable_key)}\s*=\s*)[^;]+;",
|
||||
rf"\g<1>{lit};",
|
||||
text,
|
||||
count=1,
|
||||
flags=re.MULTILINE,
|
||||
)
|
||||
if n:
|
||||
print(f" enable {enable_key}={lit}")
|
||||
|
||||
return text
|
||||
|
||||
|
||||
def write_set_file(manifest: dict) -> None:
|
||||
lines = [
|
||||
"; UnitedEA_Optimized.set — generated from united sequential audit",
|
||||
f"; {datetime.now().isoformat()}",
|
||||
"",
|
||||
]
|
||||
for spec in UNITED_STRATEGIES:
|
||||
sid = spec["id"]
|
||||
info = manifest["strategies"].get(sid, {})
|
||||
passed = info.get("passed", False)
|
||||
lines.append(f"; {sid}: {'PASS' if passed else 'DISABLED'}")
|
||||
lines.append(f"{spec['enable_key']}={'true' if passed else 'false'}")
|
||||
lines.append(f"{spec['lot_key']}={spec['lot']}")
|
||||
for k, v in info.get("params", {}).items():
|
||||
lines.append(f"; {k}={v}")
|
||||
lines.append("")
|
||||
OUT_SET.write_text("\n".join(lines), encoding="utf-8")
|
||||
|
||||
|
||||
def main() -> None:
|
||||
start, end = PERIODS["2021-2026"]
|
||||
days = period_days(start, end)
|
||||
manifest: dict = {
|
||||
"generated": datetime.now().isoformat(),
|
||||
"period_days": days,
|
||||
"strategies": {},
|
||||
}
|
||||
|
||||
for spec in UNITED_STRATEGIES:
|
||||
sid = spec["id"]
|
||||
r = load_report(sid)
|
||||
if not r:
|
||||
manifest["strategies"][sid] = {"passed": False, "status": "no_report"}
|
||||
continue
|
||||
params = r.get("optimized_params") or r.get("optimized", {}).get("params", {})
|
||||
ok, issues = evaluate(r, days)
|
||||
o = r.get("optimized", {})
|
||||
manifest["strategies"][sid] = {
|
||||
"passed": ok,
|
||||
"enable_key": spec["enable_key"],
|
||||
"lot_key": spec["lot_key"],
|
||||
"lot": spec["lot"],
|
||||
"trades": o.get("total_trades"),
|
||||
"trades_per_day": round(trades_per_day(
|
||||
type("R", (), {"total_trades": int(o.get("total_trades", 0))})(), days), 3),
|
||||
"net_profit": o.get("net_profit"),
|
||||
"profit_factor": o.get("profit_factor"),
|
||||
"sharpe": o.get("sharpe"),
|
||||
"issues": issues,
|
||||
"params": params,
|
||||
}
|
||||
|
||||
if MAIN_MQ5.exists():
|
||||
text = MAIN_MQ5.read_text(encoding="utf-8")
|
||||
print(f"Patching {MAIN_MQ5}")
|
||||
text = patch_main_mqh(text, manifest)
|
||||
MAIN_MQ5.write_text(text, encoding="utf-8")
|
||||
|
||||
REPORTS.mkdir(parents=True, exist_ok=True)
|
||||
OUT_JSON.write_text(json.dumps(manifest, indent=2), encoding="utf-8")
|
||||
write_set_file(manifest)
|
||||
passed = [k for k, v in manifest["strategies"].items() if v.get("passed")]
|
||||
print(f"Wrote {OUT_JSON}")
|
||||
print(f"Wrote {OUT_SET}")
|
||||
print(f"Passed {len(passed)}/{len(UNITED_STRATEGIES)}: {', '.join(passed) if passed else '(none)'}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,85 @@
|
||||
"""Timestamped trace logging for cluster audit runs."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
import time
|
||||
from datetime import datetime
|
||||
|
||||
|
||||
class TraceLog:
|
||||
def __init__(self, enabled: bool = True, trial_every: int = 10) -> None:
|
||||
self.enabled = enabled
|
||||
self.trial_every = max(1, trial_every)
|
||||
self._t0 = time.perf_counter()
|
||||
self._phase_t0 = self._t0
|
||||
|
||||
def _ts(self) -> str:
|
||||
return datetime.now().strftime("%H:%M:%S")
|
||||
|
||||
def _elapsed(self) -> str:
|
||||
return f"{time.perf_counter() - self._t0:.1f}s"
|
||||
|
||||
def _phase_elapsed(self) -> str:
|
||||
return f"{time.perf_counter() - self._phase_t0:.1f}s"
|
||||
|
||||
def _write(self, level: str, msg: str) -> None:
|
||||
if not self.enabled:
|
||||
return
|
||||
line = f"[{self._ts()} +{self._elapsed()}] [{level}] {msg}"
|
||||
print(line, flush=True)
|
||||
|
||||
def phase_start(self, name: str) -> None:
|
||||
self._phase_t0 = time.perf_counter()
|
||||
self._write("PHASE", f">> {name}")
|
||||
|
||||
def phase_end(self, name: str, detail: str = "") -> None:
|
||||
suffix = f" -- {detail}" if detail else ""
|
||||
self._write("PHASE", f"OK {name} ({self._phase_elapsed()}){suffix}")
|
||||
|
||||
def info(self, msg: str) -> None:
|
||||
self._write("INFO", msg)
|
||||
|
||||
def debug(self, msg: str) -> None:
|
||||
self._write("DEBUG", msg)
|
||||
|
||||
def warn(self, msg: str) -> None:
|
||||
self._write("WARN", msg)
|
||||
|
||||
def error(self, msg: str) -> None:
|
||||
self._write("ERROR", msg)
|
||||
|
||||
def banner(self, msg: str) -> None:
|
||||
if not self.enabled:
|
||||
return
|
||||
bar = "=" * min(72, max(len(msg) + 4, 40))
|
||||
print(f"\n{bar}\n {msg}\n{bar}", flush=True)
|
||||
|
||||
def progress(self, current: int, total: int, label: str) -> None:
|
||||
pct = (100.0 * current / total) if total else 0.0
|
||||
self._write("PROGRESS", f"[{current}/{total} {pct:.0f}%] {label}")
|
||||
|
||||
def trial(self, n: int, total: int, score: float, net: float, sharpe: float, improved: bool) -> None:
|
||||
if n % self.trial_every != 0 and n != total and not improved:
|
||||
return
|
||||
flag = " ** NEW BEST" if improved else ""
|
||||
score_s = "N/A" if score == float("-inf") else f"{score:.2f}"
|
||||
self._write(
|
||||
"TRIAL",
|
||||
f"{n}/{total} score={score_s} net=${net:.0f} sharpe={sharpe:.2f}{flag}",
|
||||
)
|
||||
|
||||
def report_line(self, strategy_id: str, baseline: dict, optimized: dict, bars: int) -> None:
|
||||
b, o = baseline, optimized
|
||||
self._write(
|
||||
"RESULT",
|
||||
f"{strategy_id}: bars={bars} | "
|
||||
f"base net=${b['net_profit']:.0f} sh={b['sharpe']:.2f} trades={b['total_trades']} dd={b['max_drawdown_pct']:.1f}% | "
|
||||
f"opt net=${o['net_profit']:.0f} sh={o['sharpe']:.2f} trades={o['total_trades']} dd={o['max_drawdown_pct']:.1f}%",
|
||||
)
|
||||
if b.get("worst_trades"):
|
||||
w = b["worst_trades"][0]
|
||||
self.debug(
|
||||
f" worst loss: ${w['profit']:.2f} {w['side']} {w['exit_reason']} "
|
||||
f"({w['open_time']} -> {w['close_time']})"
|
||||
)
|
||||
@@ -0,0 +1,148 @@
|
||||
"""Per-strategy 'small trick' candidates for elimination audits."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
# Each trick: human label + param overrides (applied on top of 123.set solo baseline).
|
||||
STRATEGY_TWEAKS: dict[str, list[dict]] = {
|
||||
"DB": [
|
||||
{"name": "close_on", "params": {"DB_CloseUnprofitableOnNewSignal": True}},
|
||||
{"name": "tighter_sl", "params": {"DB_StopLoss": 1400}},
|
||||
{"name": "wider_tp", "params": {"DB_TakeProfit": 4200}},
|
||||
{"name": "shorter_box", "params": {"DB_BoxPeriod": 140}},
|
||||
{"name": "no_volume_filter", "params": {"DB_UseVolumeSpikeFilter": False}},
|
||||
{"name": "lower_trend_thresh", "params": {"DB_TrendThreshold": 3.8}},
|
||||
],
|
||||
"ES": [
|
||||
{"name": "tighter_trail", "params": {"ES_TrailingStop": 280}},
|
||||
{"name": "wider_trail", "params": {"ES_TrailingStop": 420}},
|
||||
{"name": "faster_profit_check", "params": {"ES_ProfitCheckBars": 12}},
|
||||
{"name": "stale_sl_exit", "params": {"ES_UseStaleStopLossExit": True}},
|
||||
{"name": "lower_adx_gate", "params": {"ES_WeeklyADXMin": 35}},
|
||||
{"name": "more_trades_per_x", "params": {"ES_MaxTradesPerCrossover": 12}},
|
||||
],
|
||||
"RC": [
|
||||
{"name": "close_on", "params": {"RC_CloseUnprofitableOnNewSignal": True}},
|
||||
{"name": "short_cooldown", "params": {"RC_cooldownSeconds": 120}},
|
||||
{"name": "long_cooldown", "params": {"RC_cooldownSeconds": 300}},
|
||||
{"name": "tighter_trail", "params": {"RC_TrailingStop": 220}},
|
||||
{"name": "looser_ema_slope", "params": {"RC_emaSlopeThreshold": 90}},
|
||||
{"name": "rsi_exit_tighter", "params": {"RC_exitBuyRSI": 82, "RC_exitSellRSI": 14}},
|
||||
],
|
||||
"RM": [
|
||||
{"name": "close_on", "params": {"RM_CloseUnprofitableOnNewSignal": True}},
|
||||
{"name": "short_reverse_cd", "params": {"RM_InpRSIReverseCooldownBars": 8}},
|
||||
{"name": "ema_cross_only", "params": {"RM_InpEnableRSIFollow": False, "RM_InpEnableRSIReverse": False}},
|
||||
{"name": "rsi_follow_only", "params": {"RM_InpEnableRSIReverse": False, "RM_InpEnableEMACross": False}},
|
||||
{"name": "tighter_ema_dist", "params": {"RM_InpEMADistancePips": 120}},
|
||||
{"name": "close_outside_hours", "params": {"RM_InpRSIFollowCloseOutsideHours": True}},
|
||||
],
|
||||
"RS_APPL": [
|
||||
{"name": "close_on", "params": {"RS_APPL_CloseUnprofitableOnNewSignal": True}},
|
||||
{"name": "faster_bars", "params": {"RS_APPL_BarsToWait": 5}},
|
||||
{"name": "trail_activate_50", "params": {"RS_APPL_TrailActivationPoints": 50}},
|
||||
{"name": "tighter_trail", "params": {"RS_APPL_TrailDistancePoints": 80}},
|
||||
{"name": "wider_targets", "params": {"RS_APPL_RSI_Target_Buy": 92, "RS_APPL_RSI_Target_Sell": 40}},
|
||||
],
|
||||
"RS_ADBE": [
|
||||
{"name": "close_on", "params": {"RS_ADBE_CloseUnprofitableOnNewSignal": True}},
|
||||
{"name": "faster_bars", "params": {"RS_ADBE_BarsToWait": 6}},
|
||||
{"name": "trail_activate_10", "params": {"RS_ADBE_TrailActivationPoints": 10}},
|
||||
{"name": "tighter_trail", "params": {"RS_ADBE_TrailDistancePoints": 350}},
|
||||
],
|
||||
"RS_BTCUSD": [
|
||||
{"name": "close_on", "params": {"RS_BTCUSD_CloseUnprofitableOnNewSignal": True}},
|
||||
{"name": "faster_bars", "params": {"RS_BTCUSD_BarsToWait": 4}},
|
||||
{"name": "trail_activate_30", "params": {"RS_BTCUSD_TrailActivationPoints": 30}},
|
||||
{"name": "tighter_trail", "params": {"RS_BTCUSD_TrailDistancePoints": 90}},
|
||||
{"name": "reversal_escape_off", "params": {"RS_UseReversalEscape": False}},
|
||||
],
|
||||
"RS_NVDA": [
|
||||
{"name": "close_on", "params": {"RS_NVDA_CloseUnprofitableOnNewSignal": True}},
|
||||
{"name": "faster_bars", "params": {"RS_NVDA_BarsToWait": 3}},
|
||||
{"name": "tighter_trail", "params": {"RS_NVDA_TrailDistancePoints": 280}},
|
||||
{"name": "lower_trail_act", "params": {"RS_NVDA_TrailActivationPoints": 50}},
|
||||
{"name": "wider_ob_os", "params": {"RS_NVDA_RSI_Overbought": 40, "RS_NVDA_RSI_Oversold": 35}},
|
||||
],
|
||||
"RS_TSLA": [
|
||||
{"name": "faster_bars", "params": {"RS_TSLA_BarsToWait": 2}},
|
||||
{"name": "trail_act_400", "params": {"RS_TSLA_TrailActivationPoints": 400}},
|
||||
{"name": "trail_act_600", "params": {"RS_TSLA_TrailActivationPoints": 600}},
|
||||
{"name": "tighter_trail", "params": {"RS_TSLA_TrailDistancePoints": 700}},
|
||||
{"name": "wider_trail", "params": {"RS_TSLA_TrailDistancePoints": 1100}},
|
||||
],
|
||||
"RS_XAUUSD": [
|
||||
{"name": "faster_bars", "params": {"RS_XAUUSD_BarsToWait": 3}},
|
||||
{"name": "lower_trail_act", "params": {"RS_XAUUSD_TrailActivationPoints": 25}},
|
||||
{"name": "tighter_trail", "params": {"RS_XAUUSD_TrailDistancePoints": 55}},
|
||||
{"name": "wider_trail", "params": {"RS_XAUUSD_TrailDistancePoints": 90}},
|
||||
{"name": "reversal_escape_off", "params": {"RS_UseReversalEscape": False}},
|
||||
],
|
||||
"RS_MU": [
|
||||
{"name": "close_on", "params": {"RS_MU_CloseUnprofitableOnNewSignal": True}},
|
||||
{"name": "faster_bars", "params": {"RS_MU_BarsToWait": 20}},
|
||||
{"name": "symbol_munas", "params": {"RS_MU_Symbol": "MU.NAS"}},
|
||||
{"name": "symbol_muus", "params": {"RS_MU_Symbol": "MU.US"}},
|
||||
{"name": "looser_ob_os", "params": {"RS_MU_RSI_Overbought": 40, "RS_MU_RSI_Oversold": 75}},
|
||||
],
|
||||
"SE": [
|
||||
{"name": "close_on", "params": {"SE_CloseUnprofitableOnNewSignal": True}},
|
||||
{"name": "exit_trend_flip", "params": {"SE_ExitOnTrendFlip": True}},
|
||||
{"name": "shorter_hold", "params": {"SE_MaxHoldingBars": 120}},
|
||||
{"name": "longer_hold", "params": {"SE_MaxHoldingBars": 220}},
|
||||
{"name": "structural_sl", "params": {"SE_UseStructuralSL": True}},
|
||||
{"name": "cci_looser", "params": {"SE_CciOversold": -120}},
|
||||
],
|
||||
"RCO": [
|
||||
{"name": "close_on", "params": {"RCO_CloseUnprofitableOnNewSignal": True}},
|
||||
{"name": "higher_adx_cap", "params": {"RCO_ADX_Max": 32}},
|
||||
{"name": "tighter_sl", "params": {"RCO_SL_ATR_Mult": 1.9}},
|
||||
{"name": "wider_tp", "params": {"RCO_TP_ATR_Mult": 2.7}},
|
||||
{"name": "shorter_max_bars", "params": {"RCO_MaxBarsInTrade": 40}},
|
||||
],
|
||||
"RRA_EUR": [
|
||||
{"name": "close_on", "params": {"RRA_EURUSD_CloseUnprofitableOnNewSignal": True}},
|
||||
{"name": "shorter_duration", "params": {"RRA_EURUSD_MaxDuration": 200}},
|
||||
{"name": "close_outside", "params": {"RRA_EURUSD_CloseOutsideSession": True}},
|
||||
{"name": "rsi_exit_50", "params": {"RRA_EURUSD_RSIExitLevel": 50}},
|
||||
{"name": "tighter_os", "params": {"RRA_EURUSD_OversoldLevel": 12}},
|
||||
],
|
||||
"RRA_AUD": [
|
||||
{"name": "close_on", "params": {"RRA_AUDUSD_CloseUnprofitableOnNewSignal": True}},
|
||||
{"name": "shorter_duration", "params": {"RRA_AUDUSD_MaxDuration": 250}},
|
||||
{"name": "no_close_outside", "params": {"RRA_AUDUSD_CloseOutsideSession": False}},
|
||||
{"name": "tighter_os", "params": {"RRA_AUDUSD_OversoldLevel": 28}},
|
||||
{"name": "rsi_exit_45", "params": {"RRA_AUDUSD_RSIExitLevel": 45}},
|
||||
],
|
||||
"ST_BTC": [
|
||||
{"name": "close_on", "params": {"ST_BTC_CloseUnprofitableOnNewSignal": True}},
|
||||
{"name": "tighter_break", "params": {"ST_BTC_BreakBuffer": 70}},
|
||||
{"name": "looser_touch", "params": {"ST_BTC_LineTouchTolerance": 150}},
|
||||
{"name": "longer_ma", "params": {"ST_BTC_MAPeriod": 180}},
|
||||
],
|
||||
"ST_XAU": [
|
||||
{"name": "close_on", "params": {"ST_XAU_CloseUnprofitableOnNewSignal": True}},
|
||||
{"name": "tighter_break", "params": {"ST_XAU_BreakBuffer": 90}},
|
||||
{"name": "shorter_ma", "params": {"ST_XAU_MAPeriod": 55}},
|
||||
{"name": "looser_touch", "params": {"ST_XAU_LineTouchTolerance": 260}},
|
||||
],
|
||||
"ST_GER": [
|
||||
{"name": "close_on", "params": {"ST_GER_CloseUnprofitableOnNewSignal": True}},
|
||||
{"name": "tighter_break", "params": {"ST_GER_BreakBuffer": 60}},
|
||||
{"name": "looser_touch", "params": {"ST_GER_LineTouchTolerance": 120}},
|
||||
{"name": "longer_ma", "params": {"ST_GER_MAPeriod": 80}},
|
||||
],
|
||||
"RSS": [
|
||||
{"name": "close_on", "params": {"RSS_CloseUnprofitableOnNewSignal": True}},
|
||||
{"name": "swing_sl", "params": {"RSS_UseSwingStopLoss": True}},
|
||||
{"name": "shorter_cooldown", "params": {"RSS_MinBarsBetweenTrades": 4}},
|
||||
{"name": "tighter_sl_atr", "params": {"RSS_StopLossATR": 2.2}},
|
||||
],
|
||||
"UB": [
|
||||
{"name": "close_on", "params": {"UB_CloseUnprofitableOnNewSignal": True}},
|
||||
{"name": "min_range_12", "params": {"UB_MinRangePoints": 12}},
|
||||
{"name": "order_buf_35", "params": {"UB_OrderBufferPoints": 3.5}},
|
||||
{"name": "first_trade_only", "params": {"UB_FirstTradeOnly": True}},
|
||||
{"name": "wider_range", "params": {"UB_MinRangePoints": 18}},
|
||||
{"name": "tighter_buffer", "params": {"UB_OrderBufferPoints": 3.0}},
|
||||
],
|
||||
}
|
||||
@@ -0,0 +1,112 @@
|
||||
"""United EA sub-strategy manifest for MT5 solo audits (123.set)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
UNITED_MT5_STRATEGIES: list[dict] = [
|
||||
{"id": "DB", "name": "DarvasBox", "enable": "EnableDarvasBox", "close": "DB_CloseUnprofitableOnNewSignal", "lot": "LOT_DB_DarvasBox"},
|
||||
{"id": "ES", "name": "EMASlopeDistance", "enable": "EnableEMASlopeDistance", "close": "ES_CloseUnprofitableOnNewSignal", "lot": "LOT_ES_EMASlopeDistance"},
|
||||
{"id": "RC", "name": "RSICrossOverReversal", "enable": "EnableRSICrossOverReversal", "close": "RC_CloseUnprofitableOnNewSignal", "lot": "LOT_RC_RSICrossOver"},
|
||||
{"id": "RM", "name": "RSIMidPointHijack", "enable": "EnableRSIMidPointHijack", "close": "RM_CloseUnprofitableOnNewSignal", "lot": "LOT_RM_RSIMidPointHijack"},
|
||||
{"id": "RS_APPL", "name": "RSIScalping APPL", "enable": "EnableRSIScalpingAPPL", "close": "RS_APPL_CloseUnprofitableOnNewSignal", "lot": "LOT_RS_APPL", "test_symbol": "AAPL.NAS", "lot_class": "stock"},
|
||||
{"id": "RS_ADBE", "name": "RSIScalping ADBE", "enable": "EnableRSIScalpingADBE", "close": "RS_ADBE_CloseUnprofitableOnNewSignal", "lot": "LOT_RS_ADBE", "test_symbol": "ADBE.NAS", "lot_class": "stock"},
|
||||
{"id": "RS_BTCUSD", "name": "RSIScalping BTCUSD", "enable": "EnableRSIScalpingBTCUSD", "close": "RS_BTCUSD_CloseUnprofitableOnNewSignal", "lot": "LOT_RS_BTCUSD"},
|
||||
{"id": "RS_NVDA", "name": "RSIScalping NVDA", "enable": "EnableRSIScalpingNVDA", "close": "RS_NVDA_CloseUnprofitableOnNewSignal", "lot": "LOT_RS_NVDA", "test_symbol": "NVDA.NAS", "lot_class": "stock"},
|
||||
{"id": "RS_TSLA", "name": "RSIScalping TSLA", "enable": "EnableRSIScalpingTSLA", "close": "RS_TSLA_CloseUnprofitableOnNewSignal", "lot": "LOT_RS_TSLA", "test_symbol": "TSLA.NAS", "lot_class": "stock"},
|
||||
{"id": "RS_XAUUSD", "name": "RSIScalping XAUUSD", "enable": "EnableRSIScalpingXAUUSD", "close": "RS_XAUUSD_CloseUnprofitableOnNewSignal", "lot": "LOT_RS_XAUUSD"},
|
||||
{"id": "RS_MU", "name": "RSIScalping MU", "enable": "EnableRSIScalpingMU", "close": "RS_MU_CloseUnprofitableOnNewSignal", "lot": "LOT_RS_MU"},
|
||||
{"id": "SE", "name": "SuperEMA", "enable": "EnableSuperEMA", "close": "SE_CloseUnprofitableOnNewSignal", "lot": "LOT_SE_SuperEMA"},
|
||||
{"id": "RCO", "name": "RSIConsolidation", "enable": "EnableRSIConsolidation", "close": "RCO_CloseUnprofitableOnNewSignal", "lot": "LOT_RCO_RSIConsolidation"},
|
||||
{"id": "RRA_EUR", "name": "RSI Asian EURUSD", "enable": "EnableRSIReversalAsianEURUSD", "close": "RRA_EURUSD_CloseUnprofitableOnNewSignal", "lot": "LOT_RRA_EURUSD"},
|
||||
{"id": "RRA_AUD", "name": "RSI Asian AUDUSD", "enable": "EnableRSIReversalAsianAUDUSD", "close": "RRA_AUDUSD_CloseUnprofitableOnNewSignal", "lot": "LOT_RRA_AUDUSD"},
|
||||
{"id": "ST_BTC", "name": "SimpleTrendline BTC", "enable": "EnableSimpleTrendlineBTCUSD", "close": "ST_BTC_CloseUnprofitableOnNewSignal", "lot": "LOT_ST_BTCUSD"},
|
||||
{"id": "ST_XAU", "name": "SimpleTrendline XAU", "enable": "EnableSimpleTrendlineXAUUSD", "close": "ST_XAU_CloseUnprofitableOnNewSignal", "lot": "LOT_ST_XAUUSD"},
|
||||
{"id": "ST_GER", "name": "SimpleTrendline GER40", "enable": "EnableSimpleTrendlineGER40", "close": "ST_GER_CloseUnprofitableOnNewSignal", "lot": "LOT_ST_GER40"},
|
||||
{"id": "RSS", "name": "RSISecretSauce", "enable": "EnableRSISecretSauce", "close": "RSS_CloseUnprofitableOnNewSignal", "lot": "LOT_RSS_SecretSauce"},
|
||||
{"id": "UB", "name": "USDJPYBuster", "enable": "EnableUSDJPYBuster", "close": "UB_CloseUnprofitableOnNewSignal", "lot": "LOT_UB_USDJPY"},
|
||||
{"id": "XBT", "name": "XAUBearTrend", "enable": "EnableXAUBearTrend", "close": "XBT_CloseUnprofitableOnNewSignal", "lot": "LOT_XBT_XAUUSD"},
|
||||
{"id": "XMB", "name": "XAUMomentumBreakdown", "enable": "EnableXAUMomentumBreakdown", "close": "XMB_CloseUnprofitableOnNewSignal", "lot": "LOT_XMB_XAUUSD"},
|
||||
{"id": "RRA_GBP", "name": "RSI Asian GBPUSD", "enable": "EnableRSIReversalAsianGBPUSD", "close": "RRA_GBPUSD_CloseUnprofitableOnNewSignal", "lot": "LOT_RRA_GBPUSD"},
|
||||
{"id": "GB", "name": "GER40Buster", "enable": "EnableGER40Buster", "close": "GB_CloseUnprofitableOnNewSignal", "lot": "LOT_GB_GER40"},
|
||||
{"id": "RS_NAS100", "name": "RSIScalping NAS100", "enable": "EnableRSIScalpingNAS100", "close": "RS_NAS100_CloseUnprofitableOnNewSignal", "lot": "LOT_RS_NAS100"},
|
||||
{"id": "RS_US500", "name": "RSIScalping US500", "enable": "EnableRSIScalpingUS500", "close": "RS_US500_CloseUnprofitableOnNewSignal", "lot": "LOT_RS_US500"},
|
||||
{"id": "RRA_USDCHF", "name": "RSI Asian USDCHF", "enable": "EnableRSIReversalAsianUSDCHF", "close": "RRA_USDCHF_CloseUnprofitableOnNewSignal", "lot": "LOT_RRA_USDCHF"},
|
||||
{"id": "RRA_NZDUSD", "name": "RSI Asian NZDUSD", "enable": "EnableRSIReversalAsianNZDUSD", "close": "RRA_NZDUSD_CloseUnprofitableOnNewSignal", "lot": "LOT_RRA_NZDUSD"},
|
||||
{"id": "NB", "name": "NAS100Buster", "enable": "EnableNAS100Buster", "close": "NB_CloseUnprofitableOnNewSignal", "lot": "LOT_NB_NAS100"},
|
||||
{"id": "U5B", "name": "US500Buster", "enable": "EnableUS500Buster", "close": "U5B_CloseUnprofitableOnNewSignal", "lot": "LOT_U5B_US500"},
|
||||
{"id": "RS_US30", "name": "RSIScalping US30", "enable": "EnableRSIScalpingUS30", "close": "RS_US30_CloseUnprofitableOnNewSignal", "lot": "LOT_RS_US30"},
|
||||
{"id": "RS_XAGUSD", "name": "RSIScalping XAGUSD", "enable": "EnableRSIScalpingXAGUSD", "close": "RS_XAGUSD_CloseUnprofitableOnNewSignal", "lot": "LOT_RS_XAGUSD"},
|
||||
{"id": "RS_EURJPY", "name": "RSIScalping EURJPY", "enable": "EnableRSIScalpingEURJPY", "close": "RS_EURJPY_CloseUnprofitableOnNewSignal", "lot": "LOT_RS_EURJPY"},
|
||||
{"id": "RS_GBPJPY", "name": "RSIScalping GBPJPY", "enable": "EnableRSIScalpingGBPJPY", "close": "RS_GBPJPY_CloseUnprofitableOnNewSignal", "lot": "LOT_RS_GBPJPY"},
|
||||
{"id": "U30B", "name": "US30Buster", "enable": "EnableUS30Buster", "close": "U30B_CloseUnprofitableOnNewSignal", "lot": "LOT_U30B_US30"},
|
||||
{"id": "UKB", "name": "UK100Buster", "enable": "EnableUK100Buster", "close": "UKB_CloseUnprofitableOnNewSignal", "lot": "LOT_UKB_UK100"},
|
||||
{"id": "XGB", "name": "XAGUSDBuster", "enable": "EnableXAGUSDBuster", "close": "XGB_CloseUnprofitableOnNewSignal", "lot": "LOT_XGB_XAGUSD"},
|
||||
{"id": "RS_F", "name": "RSIScalping F", "enable": "EnableRSIScalpingF", "close": "RS_F_CloseUnprofitableOnNewSignal", "lot": "LOT_RS_F", "test_symbol": "F.NYS"},
|
||||
{"id": "RS_SOFI", "name": "RSIScalping SOFI", "enable": "EnableRSIScalpingSOFI", "close": "RS_SOFI_CloseUnprofitableOnNewSignal", "lot": "LOT_RS_SOFI", "test_symbol": "SOFI.NAS"},
|
||||
{"id": "RS_SNAP", "name": "RSIScalping SNAP", "enable": "EnableRSIScalpingSNAP", "close": "RS_SNAP_CloseUnprofitableOnNewSignal", "lot": "LOT_RS_SNAP", "test_symbol": "SNAP.NYS"},
|
||||
{"id": "RS_WBD", "name": "RSIScalping WBD", "enable": "EnableRSIScalpingWBD", "close": "RS_WBD_CloseUnprofitableOnNewSignal", "lot": "LOT_RS_WBD", "test_symbol": "WBD.NAS"},
|
||||
]
|
||||
|
||||
ALL_ENABLE_KEYS = [s["enable"] for s in UNITED_MT5_STRATEGIES]
|
||||
|
||||
# Round-1 expansion (retired except survivor).
|
||||
EXPANSION_RETIRED_IDS = (
|
||||
"XBT", "XMB", "RRA_GBP", "GB", "RS_US500", "RRA_USDCHF", "RRA_NZDUSD", "NB", "U5B",
|
||||
)
|
||||
|
||||
# Survivor kept on baseline + enhanced.
|
||||
SURVIVOR_IDS = ("RS_NAS100", "RS_US30", "UKB")
|
||||
|
||||
# Round-2 candidates (audited; non-survivors stay off).
|
||||
ROUND2_IDS = (
|
||||
"RS_US30", "RS_XAGUSD", "RS_EURJPY", "RS_GBPJPY", "U30B", "UKB", "XGB",
|
||||
)
|
||||
|
||||
# Round-3 low-margin stock candidates (~$1–2 margin per share @ 5% leverage).
|
||||
ROUND3_IDS = ("RS_F", "RS_SOFI", "RS_SNAP", "RS_WBD")
|
||||
|
||||
# High share-price stocks — force off in expansion audits (margin call risk).
|
||||
HIGH_MARGIN_STOCK_ENABLES = ("EnableRSIScalpingMU",)
|
||||
|
||||
# Production cluster (matches main.mq5 defaults).
|
||||
PRODUCTION_IDS: tuple[str, ...] = (
|
||||
"DB", "ES", "RC", "RM",
|
||||
"RS_NVDA", "RS_TSLA",
|
||||
"RS_BTCUSD", "RS_XAUUSD", "SE", "ST_BTC", "ST_XAU",
|
||||
"RRA_AUD", "RRA_GBP", "UB",
|
||||
"RS_NAS100", "RS_US30", "UKB", "GB", "U5B",
|
||||
)
|
||||
|
||||
LOT_GRIDS: dict[str, list[float]] = {
|
||||
"stock": [5.0, 10.0, 15.0],
|
||||
"index": [0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1],
|
||||
"forex": [0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1],
|
||||
"gold": [0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1],
|
||||
"crypto": [0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1],
|
||||
}
|
||||
|
||||
LOT_GENETIC_RANGE: dict[str, tuple[float, float, float]] = {
|
||||
"stock": (5.0, 5.0, 15.0),
|
||||
"default": (0.01, 0.01, 0.1),
|
||||
}
|
||||
|
||||
LOT_CLASS_BY_ID: dict[str, str] = {
|
||||
"DB": "gold", "ES": "gold", "RC": "gold", "RM": "gold",
|
||||
"RS_XAUUSD": "gold", "ST_XAU": "gold", "XBT": "gold", "XMB": "gold", "XGB": "gold",
|
||||
"RS_APPL": "stock", "RS_ADBE": "stock", "RS_NVDA": "stock", "RS_TSLA": "stock", "RS_MU": "stock",
|
||||
"RS_F": "stock", "RS_SOFI": "stock", "RS_SNAP": "stock", "RS_WBD": "stock",
|
||||
"RS_NAS100": "index", "RS_US500": "index", "RS_US30": "index", "UKB": "index",
|
||||
"NB": "index", "U5B": "index", "U30B": "index", "GB": "index",
|
||||
"RRA_EUR": "forex", "RRA_AUD": "forex", "RRA_GBP": "forex", "RRA_USDCHF": "forex", "RRA_NZDUSD": "forex",
|
||||
"UB": "forex", "RS_EURJPY": "forex", "RS_GBPJPY": "forex",
|
||||
"RS_BTCUSD": "crypto", "ST_BTC": "crypto",
|
||||
"ST_GER": "index", "RS_XAGUSD": "gold",
|
||||
}
|
||||
|
||||
PARAM_TWEAKS: dict[str, list[dict]] = {
|
||||
"ES": [{"ES_TrailingStop": 250}, {"ES_TrailingStop": 300, "ES_ProfitCheckBars": 12}],
|
||||
"RC": [{"RC_cooldownSeconds": 120}, {"RC_TrailingStop": 250}],
|
||||
"RS_NVDA": [{"RS_NVDA_BarsToWait": 3}, {"RS_NVDA_TrailDistancePoints": 300}],
|
||||
"RS_TSLA": [{"RS_TSLA_BarsToWait": 2}, {"RS_TSLA_TrailActivationPoints": 500}],
|
||||
"RCO": [{"RCO_ADX_Max": 32}, {"RCO_SL_ATR_Mult": 1.9}],
|
||||
"UB": [{"UB_MinRangePoints": 12}, {"UB_OrderBufferPoints": 3.5}],
|
||||
}
|
||||
@@ -0,0 +1,488 @@
|
||||
"""MT5 Strategy Tester runner for United EA (cluster-latest/main.mq5)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
import MetaTrader5 as mt5
|
||||
|
||||
CLUSTER = Path(__file__).resolve().parents[3] / "frontline" / "cluster-latest"
|
||||
BASE_SET = CLUSTER / "123.set"
|
||||
DEPOSIT = 3000
|
||||
LEVERAGE = 1000
|
||||
FROM_DATE = "2023.07.01"
|
||||
TO_DATE = "2026.06.01"
|
||||
TEST_SYMBOL = "NAS100"
|
||||
TEST_PERIOD = "H1"
|
||||
|
||||
LABELS = {
|
||||
"profit_factor": ("Profit Factor", "盈利因子"),
|
||||
"net_profit": ("Total Net Profit", "总净盈利"),
|
||||
"total_trades": ("Total Trades", "交易总计"),
|
||||
"sharpe": ("Sharpe Ratio", "夏普比率"),
|
||||
"equity_dd": ("Equity Drawdown Maximal", "最大回撤"),
|
||||
"margin_level": ("Minimal margin level", "最低保证金比例"),
|
||||
}
|
||||
|
||||
|
||||
def read_text(path: Path) -> str:
|
||||
raw = path.read_bytes()
|
||||
for enc in ("utf-16", "utf-16-le", "utf-8", "cp1252"):
|
||||
try:
|
||||
text = raw.decode(enc)
|
||||
if text.strip():
|
||||
return text
|
||||
except UnicodeError:
|
||||
continue
|
||||
return raw.decode("utf-8", errors="ignore")
|
||||
|
||||
|
||||
def grab_metric(text: str, key: str) -> str | None:
|
||||
for label in LABELS[key]:
|
||||
for pat in (
|
||||
rf">{re.escape(label)}</td>\s*<td[^>]*>(?:<b>)?([^<]+)",
|
||||
rf">{re.escape(label)}:</td>\s*<td[^>]*>(?:<b>)?([^<]+)",
|
||||
rf">{re.escape(label)}</td>\s*<td[^>]*><b>([^<]+)",
|
||||
):
|
||||
m = re.search(pat, text, re.I)
|
||||
if m:
|
||||
return m.group(1).strip()
|
||||
return None
|
||||
|
||||
|
||||
def parse_report(data: Path, report: str) -> dict:
|
||||
candidates = [data / f"{report}{ext}" for ext in (".htm", ".html")]
|
||||
candidates += sorted(data.glob(f"**/{report}*.htm*"), key=lambda p: p.stat().st_mtime, reverse=True)
|
||||
seen: set[Path] = set()
|
||||
for path in candidates:
|
||||
if path in seen or not path.exists():
|
||||
continue
|
||||
seen.add(path)
|
||||
text = read_text(path)
|
||||
pf = grab_metric(text, "profit_factor")
|
||||
profit = grab_metric(text, "net_profit")
|
||||
trades = grab_metric(text, "total_trades")
|
||||
sharpe = grab_metric(text, "sharpe")
|
||||
dd = grab_metric(text, "equity_dd")
|
||||
ml = grab_metric(text, "margin_level")
|
||||
if pf or profit or trades:
|
||||
return {
|
||||
"profit_factor": float(pf) if pf else None,
|
||||
"net_profit": _num(profit),
|
||||
"total_trades": int(float(trades)) if trades and trades[0].isdigit() else _int(trades),
|
||||
"sharpe": float(sharpe) if sharpe else None,
|
||||
"max_drawdown": dd,
|
||||
"min_margin_level": ml,
|
||||
"report": str(path),
|
||||
"ready": True,
|
||||
}
|
||||
return {"ready": False}
|
||||
|
||||
|
||||
def _num(s: str | None) -> float | None:
|
||||
if not s:
|
||||
return None
|
||||
s = s.replace(" ", "").replace(",", "")
|
||||
if s.endswith("%"):
|
||||
return float(s[:-1])
|
||||
return float(s)
|
||||
|
||||
|
||||
def _int(s: str | None) -> int | None:
|
||||
if not s:
|
||||
return None
|
||||
try:
|
||||
return int(float(s.replace(" ", "").replace(",", "")))
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
KNOWN_TERMINALS = [
|
||||
Path(r"C:\Program Files\MetaTrader 5\terminal64.exe"),
|
||||
]
|
||||
|
||||
|
||||
def _terminal_data_dirs() -> list[Path]:
|
||||
root = Path.home() / "AppData" / "Roaming" / "MetaQuotes" / "Terminal"
|
||||
if not root.is_dir():
|
||||
return []
|
||||
return [p for p in root.iterdir() if p.is_dir() and (p / "origin.txt").exists()]
|
||||
|
||||
|
||||
def _read_origin(path: Path) -> str:
|
||||
raw = path.read_bytes()
|
||||
for enc in ("utf-8", "utf-16", "utf-16-le", "cp1252"):
|
||||
try:
|
||||
return raw.decode(enc).strip()
|
||||
except UnicodeError:
|
||||
continue
|
||||
return raw.decode("utf-8", errors="ignore").strip()
|
||||
|
||||
|
||||
def _load_dotenv() -> None:
|
||||
env_path = Path(__file__).resolve().parents[3] / ".env"
|
||||
if not env_path.is_file():
|
||||
return
|
||||
for line in env_path.read_text(encoding="utf-8", errors="ignore").splitlines():
|
||||
line = line.strip()
|
||||
if not line or line.startswith("#") or "=" not in line:
|
||||
continue
|
||||
key, val = line.split("=", 1)
|
||||
key, val = key.strip(), val.strip().strip('"').strip("'")
|
||||
if key and key not in os.environ:
|
||||
os.environ[key] = val
|
||||
|
||||
|
||||
_load_dotenv()
|
||||
MT5_TERMINAL_DATA_ID = os.environ.get("MT5_TERMINAL_DATA_ID", "").strip()
|
||||
# Back-compat alias for scripts that import PREFERRED_DATA_ID
|
||||
PREFERRED_DATA_ID = MT5_TERMINAL_DATA_ID
|
||||
|
||||
|
||||
def mt5_terminal_data_dir() -> Path | None:
|
||||
"""MT5 data folder: MT5_TERMINAL_DATA_ID env, else first terminal with origin.txt."""
|
||||
root = Path.home() / "AppData" / "Roaming" / "MetaQuotes" / "Terminal"
|
||||
if MT5_TERMINAL_DATA_ID:
|
||||
candidate = root / MT5_TERMINAL_DATA_ID
|
||||
if (candidate / "origin.txt").is_file():
|
||||
return candidate
|
||||
dirs = _terminal_data_dirs()
|
||||
return dirs[0] if dirs else None
|
||||
|
||||
|
||||
def _resolve_terminal_exe() -> Path | None:
|
||||
env_exe = os.environ.get("MT5_TERMINAL_EXE", "").strip()
|
||||
if env_exe:
|
||||
exe = Path(env_exe)
|
||||
if exe.is_file():
|
||||
return exe
|
||||
data_dir = mt5_terminal_data_dir()
|
||||
if data_dir:
|
||||
origin = data_dir / "origin.txt"
|
||||
if origin.is_file():
|
||||
try:
|
||||
exe = Path(_read_origin(origin))
|
||||
if exe.is_file():
|
||||
return exe
|
||||
except OSError:
|
||||
pass
|
||||
for data_dir in _terminal_data_dirs():
|
||||
origin = data_dir / "origin.txt"
|
||||
try:
|
||||
exe = Path(_read_origin(origin))
|
||||
if exe.is_file():
|
||||
return exe
|
||||
except OSError:
|
||||
continue
|
||||
for exe in KNOWN_TERMINALS:
|
||||
if exe.is_file():
|
||||
return exe
|
||||
return None
|
||||
|
||||
|
||||
def mt5_context(*, retries: int = 6, wait_sec: float = 12.0) -> dict:
|
||||
terminal_exe = _resolve_terminal_exe()
|
||||
last_err = None
|
||||
for attempt in range(retries):
|
||||
if attempt:
|
||||
time.sleep(wait_sec)
|
||||
if attempt >= 1 and terminal_exe and terminal_exe.is_file():
|
||||
subprocess.run(["taskkill", "/IM", "terminal64.exe", "/F"], capture_output=True)
|
||||
time.sleep(3)
|
||||
subprocess.Popen([str(terminal_exe)], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
||||
time.sleep(20)
|
||||
mt5.shutdown()
|
||||
ok = mt5.initialize(path=str(terminal_exe)) if terminal_exe else mt5.initialize()
|
||||
if not ok:
|
||||
last_err = mt5.last_error()
|
||||
continue
|
||||
info = mt5.terminal_info()
|
||||
acc = mt5.account_info()
|
||||
ctx = {
|
||||
"data": Path(info.data_path),
|
||||
"mt5_path": Path(info.path),
|
||||
"login": acc.login if acc else 0,
|
||||
"server": acc.server if acc else "",
|
||||
}
|
||||
mt5.shutdown()
|
||||
return ctx
|
||||
raise RuntimeError(f"MT5 init failed after {retries} tries: {last_err}")
|
||||
|
||||
|
||||
def deploy_united(data: Path, mt5_path: Path) -> Path:
|
||||
dst = data / "MQL5" / "Experts" / "cluster-latest"
|
||||
dst.mkdir(parents=True, exist_ok=True)
|
||||
for name in ("main.mq5", "MagicNumberHelpers.mqh", "GapGuard.mqh"):
|
||||
shutil.copy2(CLUSTER / name, dst / name)
|
||||
strat_dst = dst / "Strategies"
|
||||
strat_dst.mkdir(exist_ok=True)
|
||||
for f in (CLUSTER / "Strategies").glob("*.mqh"):
|
||||
shutil.copy2(f, strat_dst / f.name)
|
||||
log = dst / "compile.log"
|
||||
subprocess.run(
|
||||
[str(mt5_path / "metaeditor64.exe"), f"/compile:{dst / 'main.mq5'}", f"/log:{log}"],
|
||||
timeout=180,
|
||||
capture_output=True,
|
||||
)
|
||||
time.sleep(3)
|
||||
ex5 = dst / "main.ex5"
|
||||
if not ex5.exists():
|
||||
tail = log.read_text(encoding="utf-8", errors="ignore")[-2000:] if log.exists() else ""
|
||||
raise RuntimeError(f"Compile failed: {log}\n{tail}")
|
||||
pub = data / "MQL5" / "Experts" / "main.ex5"
|
||||
shutil.copy2(ex5, pub)
|
||||
return pub
|
||||
|
||||
|
||||
def patch_set(base: Path, overrides: dict[str, str | bool | int | float]) -> str:
|
||||
lines_out: list[str] = []
|
||||
seen: set[str] = set()
|
||||
for line in base.read_text(encoding="utf-8", errors="ignore").splitlines():
|
||||
if not line.strip() or line.strip().startswith(";") or "=" not in line:
|
||||
lines_out.append(line)
|
||||
continue
|
||||
name = line.split("=", 1)[0].strip()
|
||||
if name in overrides:
|
||||
val = overrides[name]
|
||||
if isinstance(val, bool):
|
||||
sval = "true" if val else "false"
|
||||
else:
|
||||
sval = str(val)
|
||||
lines_out.append(f"{name}={sval}")
|
||||
seen.add(name)
|
||||
else:
|
||||
lines_out.append(line)
|
||||
for k, v in overrides.items():
|
||||
if k not in seen:
|
||||
sval = "true" if v is True else "false" if v is False else str(v)
|
||||
lines_out.append(f"{k}={sval}")
|
||||
return "\n".join(lines_out) + "\n"
|
||||
|
||||
|
||||
def patch_set_for_lot_genetic(
|
||||
base: Path,
|
||||
overrides: dict[str, str | bool | int | float],
|
||||
lot_key: str,
|
||||
start: float,
|
||||
step: float,
|
||||
stop: float,
|
||||
default: float | None = None,
|
||||
) -> str:
|
||||
"""Build .set with a single LOT_* genetic range; all other ||Y flags forced to N."""
|
||||
body = patch_set(base, overrides)
|
||||
val = default if default is not None else start
|
||||
genetic_line = f"{lot_key}={val}||{start}||{step}||{stop}||Y"
|
||||
lines_out: list[str] = []
|
||||
seen_lot = False
|
||||
for line in body.splitlines():
|
||||
if not line.strip() or line.strip().startswith(";") or "=" not in line:
|
||||
lines_out.append(line)
|
||||
continue
|
||||
name = line.split("=", 1)[0].strip()
|
||||
if name == lot_key:
|
||||
lines_out.append(genetic_line)
|
||||
seen_lot = True
|
||||
elif "||" in line:
|
||||
parts = line.split("||")
|
||||
if len(parts) >= 5:
|
||||
parts[4] = "N"
|
||||
lines_out.append("||".join(parts))
|
||||
else:
|
||||
lines_out.append(line)
|
||||
else:
|
||||
lines_out.append(line)
|
||||
if not seen_lot:
|
||||
lines_out.append(genetic_line)
|
||||
return "\n".join(lines_out) + "\n"
|
||||
|
||||
|
||||
def parse_optimization_xml(data: Path, report: str, lot_key: str) -> dict:
|
||||
candidates = [data / f"{report}.xml", data / f"{report}.opt"]
|
||||
candidates += sorted(data.glob(f"**/{report}*.xml"), key=lambda p: p.stat().st_mtime, reverse=True)
|
||||
seen: set[Path] = set()
|
||||
xml_path: Path | None = None
|
||||
for p in candidates:
|
||||
if p in seen or not p.exists() or p.suffix.lower() != ".xml":
|
||||
continue
|
||||
seen.add(p)
|
||||
xml_path = p
|
||||
break
|
||||
if xml_path is None:
|
||||
return {"ready": False, "error": "xml_not_found"}
|
||||
|
||||
text = read_text(xml_path)
|
||||
header = re.search(r"<Row>.*?Pass</Data>.*?</Row>", text, re.S)
|
||||
if not header:
|
||||
return {"ready": False, "error": "xml_header_missing", "report": str(xml_path)}
|
||||
|
||||
cols = re.findall(r'<Data ss:Type="String">([^<]+)</Data>', header.group(0))
|
||||
rows: list[dict[str, str]] = []
|
||||
for row_xml in re.findall(r"<Row>(.*?)</Row>", text, re.S)[1:]:
|
||||
cells = re.findall(r'<Data ss:Type="(?:Number|String)">([^<]+)</Data>', row_xml)
|
||||
if len(cells) >= len(cols):
|
||||
rows.append(dict(zip(cols, cells)))
|
||||
|
||||
if not rows:
|
||||
return {"ready": False, "error": "xml_no_rows", "report": str(xml_path)}
|
||||
|
||||
best_row: dict[str, str] | None = None
|
||||
best_score = -1e18
|
||||
for row in rows:
|
||||
try:
|
||||
profit = float(row.get("Profit", 0))
|
||||
pf = float(row.get("Profit Factor", 0))
|
||||
sharpe = float(row.get("Sharpe Ratio", row.get("Sharpe", 0)))
|
||||
trades = int(float(row.get("Trades", 0)))
|
||||
except (ValueError, TypeError):
|
||||
continue
|
||||
if trades < 20 or pf < 1.0 or profit <= 0:
|
||||
score = -1e10 + profit
|
||||
else:
|
||||
score = sharpe * 2000.0 + profit / 500.0 + pf * 50.0
|
||||
if score > best_score:
|
||||
best_score = score
|
||||
best_row = row
|
||||
|
||||
if best_row is None:
|
||||
best_row = max(rows, key=lambda r: float(r.get("Profit", 0)))
|
||||
|
||||
lot_raw = best_row.get(lot_key)
|
||||
if lot_raw is None:
|
||||
for k, v in best_row.items():
|
||||
if k.replace(" ", "") == lot_key or lot_key in k:
|
||||
lot_raw = v
|
||||
break
|
||||
best_lot = float(lot_raw) if lot_raw is not None else None
|
||||
|
||||
return {
|
||||
"ready": True,
|
||||
"report": str(xml_path),
|
||||
"best_lot": best_lot,
|
||||
"best_row": best_row,
|
||||
"best_score": best_score,
|
||||
"passes": len(rows),
|
||||
"profit": float(best_row.get("Profit", 0)),
|
||||
"profit_factor": float(best_row.get("Profit Factor", 0)),
|
||||
"sharpe": float(best_row.get("Sharpe Ratio", best_row.get("Sharpe", 0))),
|
||||
"trades": int(float(best_row.get("Trades", 0))),
|
||||
}
|
||||
|
||||
|
||||
def run_genetic_lot_optimize(
|
||||
data: Path,
|
||||
mt5_path: Path,
|
||||
login: int,
|
||||
server: str,
|
||||
set_body: str,
|
||||
set_name: str,
|
||||
report: str,
|
||||
lot_key: str,
|
||||
*,
|
||||
test_symbol: str | None = None,
|
||||
optimization: int = 2,
|
||||
timeout_sec: int = 7200,
|
||||
) -> dict:
|
||||
write_tester_set(data, set_name, set_body)
|
||||
ini = data / f"{report}.ini"
|
||||
ini.write_text(
|
||||
build_ini(set_name, report, login, server, symbol=test_symbol, optimization=optimization),
|
||||
encoding="utf-8",
|
||||
)
|
||||
for ext in (".htm", ".html", ".xml"):
|
||||
p = data / f"{report}{ext}"
|
||||
if p.exists():
|
||||
p.unlink(missing_ok=True)
|
||||
|
||||
subprocess.run(["taskkill", "/IM", "terminal64.exe", "/F"], capture_output=True)
|
||||
subprocess.run(["taskkill", "/IM", "metatester64.exe", "/F"], capture_output=True)
|
||||
time.sleep(6)
|
||||
|
||||
t0 = time.time()
|
||||
subprocess.run([str(mt5_path / "terminal64.exe"), f"/config:{ini}"], timeout=timeout_sec)
|
||||
elapsed = round(time.time() - t0, 1)
|
||||
|
||||
opt = parse_optimization_xml(data, report, lot_key)
|
||||
opt["elapsed_sec"] = elapsed
|
||||
if not opt.get("ready"):
|
||||
metrics = parse_report(data, report)
|
||||
opt.update(metrics)
|
||||
return opt
|
||||
|
||||
|
||||
def write_tester_set(data: Path, set_name: str, body: str) -> Path:
|
||||
profiles = data / "MQL5" / "Profiles" / "Tester"
|
||||
profiles.mkdir(parents=True, exist_ok=True)
|
||||
path = profiles / set_name
|
||||
path.write_text(body, encoding="utf-8")
|
||||
return path
|
||||
|
||||
|
||||
def build_ini(
|
||||
set_name: str,
|
||||
report: str,
|
||||
login: int,
|
||||
server: str,
|
||||
*,
|
||||
symbol: str | None = None,
|
||||
optimization: int = 0,
|
||||
) -> str:
|
||||
sym = symbol or TEST_SYMBOL
|
||||
return f"""[Common]
|
||||
Login={login}
|
||||
Server={server}
|
||||
[Tester]
|
||||
Expert=main.ex5
|
||||
ExpertParameters={set_name}
|
||||
Symbol={sym}
|
||||
Period={TEST_PERIOD}
|
||||
Optimization={optimization}
|
||||
Model=1
|
||||
Dates=1
|
||||
FromDate={FROM_DATE}
|
||||
ToDate={TO_DATE}
|
||||
ForwardMode=0
|
||||
Deposit={DEPOSIT}
|
||||
Currency=USD
|
||||
Leverage={LEVERAGE}
|
||||
ExecutionMode=0
|
||||
Report={report}
|
||||
ReplaceReport=1
|
||||
ShutdownTerminal=1
|
||||
Visual=0
|
||||
"""
|
||||
|
||||
|
||||
def run_backtest(
|
||||
data: Path,
|
||||
mt5_path: Path,
|
||||
login: int,
|
||||
server: str,
|
||||
set_body: str,
|
||||
set_name: str,
|
||||
report: str,
|
||||
*,
|
||||
test_symbol: str | None = None,
|
||||
timeout_sec: int = 1800,
|
||||
) -> dict:
|
||||
write_tester_set(data, set_name, set_body)
|
||||
ini = data / f"{report}.ini"
|
||||
ini.write_text(build_ini(set_name, report, login, server, symbol=test_symbol), encoding="utf-8")
|
||||
for ext in (".htm", ".html"):
|
||||
p = data / f"{report}{ext}"
|
||||
if p.exists():
|
||||
p.unlink(missing_ok=True)
|
||||
|
||||
subprocess.run(["taskkill", "/IM", "terminal64.exe", "/F"], capture_output=True)
|
||||
subprocess.run(["taskkill", "/IM", "metatester64.exe", "/F"], capture_output=True)
|
||||
time.sleep(6)
|
||||
|
||||
t0 = time.time()
|
||||
subprocess.run([str(mt5_path / "terminal64.exe"), f"/config:{ini}"], timeout=timeout_sec)
|
||||
metrics = parse_report(data, report)
|
||||
metrics["elapsed_sec"] = round(time.time() - t0, 1)
|
||||
return metrics
|
||||
@@ -0,0 +1,117 @@
|
||||
"""United EA (main.mq5) strategy registry — defaults from 123.set + audit ranges."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import MetaTrader5 as mt5
|
||||
|
||||
TF = {
|
||||
"M10": mt5.TIMEFRAME_M10,
|
||||
"M15": mt5.TIMEFRAME_M15,
|
||||
"M20": mt5.TIMEFRAME_M20,
|
||||
"M30": mt5.TIMEFRAME_M30,
|
||||
"H1": mt5.TIMEFRAME_H1,
|
||||
"H4": mt5.TIMEFRAME_H4,
|
||||
}
|
||||
|
||||
# Strategies enabled in 123.set, ordered for sequential audit.
|
||||
UNITED_STRATEGIES: list[dict] = [
|
||||
{"id": "united_darvas", "engine": "darvas", "symbol": "XAUUSD", "tf": "M15", "lot": 0.05,
|
||||
"enable_key": "EnableDarvasBox", "lot_key": "LOT_DB_DarvasBox",
|
||||
"defaults": {"box_period": 24, "box_deviation": 90000, "ma_period": 30,
|
||||
"trend_threshold": 1.2, "volume_threshold": 0,
|
||||
"stop_loss_pts": 300, "take_profit_pts": 950},
|
||||
"opt": {"box_period": (12, 48, 4), "box_deviation": (40000, 150000, 5000),
|
||||
"trend_threshold": (0.3, 5.0, 0.3), "stop_loss_pts": (250, 900, 50),
|
||||
"take_profit_pts": (350, 1200, 50), "ma_period": (30, 120, 15)}},
|
||||
{"id": "united_rsi_cross", "engine": "rsi_crossover", "symbol": "XAUUSD", "tf": "M15", "lot": 0.06,
|
||||
"enable_key": "EnableRSICrossOverReversal", "lot_key": "LOT_RC_RSICrossOver",
|
||||
"defaults": {"rsi_period": 19, "overbought_level": 85, "oversold_level": 25, "ema_period": 140,
|
||||
"ema_slope_threshold": 105, "ema_distance_threshold": 350, "exit_buy_rsi": 86,
|
||||
"exit_sell_rsi": 10, "trailing_stop_pts": 295, "cooldown_seconds": 120,
|
||||
"use_trend_strength_filter": True, "entry_rsi_buy_spread": 0, "entry_rsi_sell_spread": 0,
|
||||
"tuesday": True, "wednesday": True, "thursday": True,
|
||||
"trading_hour_one_begin": 0, "trading_hour_one_end": 22,
|
||||
"trading_hour_two_begin": 6, "trading_hour_two_end": 19},
|
||||
"opt": {"overbought_level": (70, 92, 3), "oversold_level": (15, 40, 3),
|
||||
"ema_distance_threshold": (200, 600, 50), "cooldown_seconds": (60, 300, 30)}},
|
||||
{"id": "united_rsi_scalp_appl", "engine": "rsi_scalp", "symbol": "AAPL", "tf": "M15", "lot": 15.0,
|
||||
"enable_key": "EnableRSIScalpingAPPL", "lot_key": "LOT_RS_APPL",
|
||||
"defaults": {"rsi_period": 8, "rsi_overbought": 64, "rsi_oversold": 30,
|
||||
"rsi_target_buy": 67, "rsi_target_sell": 2, "bars_to_wait": 5,
|
||||
"use_trailing": True, "trail_distance_pts": 70, "trail_activation_pts": 39, "tf": "M15",
|
||||
"skip_short_hour_after": 17, "use_reversal_escape": False,
|
||||
"use_rsi_against_exit": False, "max_adverse_atr": 2.5, "min_ob_depth": 0},
|
||||
"opt": {"rsi_period": (6, 14, 1), "rsi_overbought": (60, 75, 2), "rsi_oversold": (22, 35, 2),
|
||||
"rsi_target_buy": (60, 80, 3), "bars_to_wait": (5, 12, 1),
|
||||
"trail_distance_pts": (50, 120, 10), "skip_short_hour_after": (15, 20, 1)}},
|
||||
{"id": "united_rsi_scalp_adbe", "engine": "rsi_scalp", "symbol": "ADBE", "tf": "H1", "lot": 40.0,
|
||||
"enable_key": "EnableRSIScalpingADBE", "lot_key": "LOT_RS_ADBE",
|
||||
"defaults": {"rsi_period": 15, "rsi_overbought": 16, "rsi_oversold": 42,
|
||||
"rsi_target_buy": 67, "rsi_target_sell": 62, "bars_to_wait": 8,
|
||||
"use_trailing": True, "trail_distance_pts": 425, "trail_activation_pts": 18.5, "tf": "H1"},
|
||||
"opt": {"rsi_period": (6, 21, 1), "rsi_overbought": (50, 90, 3), "rsi_oversold": (10, 70, 3),
|
||||
"bars_to_wait": (1, 12, 1), "trail_distance_pts": (50, 500, 25)}},
|
||||
{"id": "united_rsi_scalp_btc", "engine": "rsi_scalp", "symbol": "BTCUSD", "tf": "H1", "lot": 0.03,
|
||||
"enable_key": "EnableRSIScalpingBTCUSD", "lot_key": "LOT_RS_BTCUSD",
|
||||
"defaults": {"rsi_period": 14, "rsi_overbought": 90, "rsi_oversold": 73,
|
||||
"rsi_target_buy": 88, "rsi_target_sell": 48, "bars_to_wait": 6,
|
||||
"use_trailing": True, "trail_distance_pts": 120, "trail_activation_pts": 0, "tf": "H1"},
|
||||
"opt": {"rsi_period": (6, 21, 1), "rsi_overbought": (50, 95, 3), "rsi_oversold": (20, 80, 3),
|
||||
"bars_to_wait": (1, 8, 1), "trail_distance_pts": (40, 300, 20)}},
|
||||
{"id": "united_rsi_scalp_nvda", "engine": "rsi_scalp", "symbol": "NVDA", "tf": "M15", "lot": 50.0,
|
||||
"enable_key": "EnableRSIScalpingNVDA", "lot_key": "LOT_RS_NVDA",
|
||||
"defaults": {"rsi_period": 8, "rsi_overbought": 36, "rsi_oversold": 38,
|
||||
"rsi_target_buy": 90, "rsi_target_sell": 70, "bars_to_wait": 5,
|
||||
"use_trailing": True, "trail_distance_pts": 375, "trail_activation_pts": 75, "tf": "M15"},
|
||||
"opt": {"rsi_period": (6, 21, 1), "bars_to_wait": (1, 8, 1), "trail_distance_pts": (50, 500, 25)}},
|
||||
{"id": "united_rsi_scalp_tsla", "engine": "rsi_scalp", "symbol": "TSLA", "tf": "H1", "lot": 5.0,
|
||||
"enable_key": "EnableRSIScalpingTSLA", "lot_key": "LOT_RS_TSLA",
|
||||
"defaults": {"rsi_period": 14, "rsi_overbought": 54, "rsi_oversold": 73,
|
||||
"rsi_target_buy": 87, "rsi_target_sell": 33, "bars_to_wait": 1,
|
||||
"use_trailing": True, "trail_distance_pts": 900, "trail_activation_pts": 950, "tf": "H1"},
|
||||
"opt": {"rsi_period": (6, 21, 1), "bars_to_wait": (1, 6, 1), "trail_distance_pts": (100, 1000, 50)}},
|
||||
{"id": "united_rsi_scalp_xau", "engine": "rsi_scalp", "symbol": "XAUUSD", "tf": "H1", "lot": 0.02,
|
||||
"enable_key": "EnableRSIScalpingXAUUSD", "lot_key": "LOT_RS_XAUUSD",
|
||||
"defaults": {"rsi_period": 14, "rsi_overbought": 71, "rsi_oversold": 57,
|
||||
"rsi_target_buy": 80, "rsi_target_sell": 57, "bars_to_wait": 1,
|
||||
"use_trailing": True, "trail_distance_pts": 71, "trail_activation_pts": 41, "tf": "H1"},
|
||||
"opt": {"rsi_period": (6, 21, 1), "rsi_overbought": (55, 85, 3), "rsi_oversold": (40, 75, 3),
|
||||
"bars_to_wait": (1, 4, 1), "trail_distance_pts": (20, 150, 10)}},
|
||||
{"id": "united_rsi_asian_eur", "engine": "rsi_asian", "symbol": "EURUSD", "tf": "M15", "lot": 0.04,
|
||||
"enable_key": "EnableRSIReversalAsianEURUSD", "lot_key": "LOT_RRA_EURUSD",
|
||||
"defaults": {"rsi_period": 28, "overbought_level": 60, "oversold_level": 8,
|
||||
"asian_session_start": 0, "asian_session_end": 8, "use_rsi_exit": True, "rsi_exit_level": 55},
|
||||
"opt": {"overbought_level": (50, 80, 5), "oversold_level": (5, 40, 5)}},
|
||||
{"id": "united_rsi_asian_aud", "engine": "rsi_asian", "symbol": "AUDUSD", "tf": "M15", "lot": 0.03,
|
||||
"enable_key": "EnableRSIReversalAsianAUDUSD", "lot_key": "LOT_RRA_AUDUSD",
|
||||
"defaults": {"rsi_period": 28, "overbought_level": 68, "oversold_level": 30,
|
||||
"asian_session_start": 0, "asian_session_end": 8, "use_rsi_exit": True, "rsi_exit_level": 48,
|
||||
"close_outside_session": True},
|
||||
"opt": {"overbought_level": (55, 80, 5), "oversold_level": (15, 45, 5)}},
|
||||
{"id": "united_st_xau", "engine": "simple_trendline", "symbol": "XAUUSD", "tf": "H1", "lot": 0.03,
|
||||
"enable_key": "EnableSimpleTrendlineXAUUSD", "lot_key": "LOT_ST_XAUUSD",
|
||||
"defaults": {"signal_tf": "H1", "higher_tf": "M10", "ma_period": 65, "ma_method": "ema",
|
||||
"htf_bars_to_scan": 500, "touch_tolerance_pts": 220, "break_buffer_pts": 110},
|
||||
"opt": {"ma_period": (40, 120, 10), "touch_tolerance_pts": (100, 350, 25),
|
||||
"break_buffer_pts": (50, 200, 15)}},
|
||||
{"id": "united_st_ger40", "engine": "simple_trendline", "symbol": "GER40", "tf": "M15", "lot": 0.04,
|
||||
"enable_key": "EnableSimpleTrendlineGER40", "lot_key": "LOT_ST_GER40",
|
||||
"defaults": {"signal_tf": "M15", "higher_tf": "M15", "ma_period": 65, "ma_method": "lwma",
|
||||
"htf_bars_to_scan": 1200, "touch_tolerance_pts": 100, "break_buffer_pts": 80},
|
||||
"opt": {"ma_period": (40, 120, 10), "touch_tolerance_pts": (50, 200, 15)}},
|
||||
{"id": "united_rsi_secret", "engine": "rsi_secret", "symbol": "XAUUSD", "tf": "M30", "lot": 0.07,
|
||||
"enable_key": "EnableRSISecretSauce", "lot_key": "LOT_RSS_SecretSauce",
|
||||
"defaults": {"rsi_period": 16, "rsi_overbought": 72.5, "rsi_oversold": 32.5,
|
||||
"stop_loss_atr": 2.75, "take_profit_atr": 5.0, "min_bars_between_trades": 7},
|
||||
"opt": {"rsi_overbought": (60, 80, 2), "rsi_oversold": (25, 45, 2), "stop_loss_atr": (1.5, 4, 0.5)}},
|
||||
{"id": "united_usdjpy", "engine": "usdjpy_buster", "symbol": "USDJPY", "tf": "M20", "lot": 0.09,
|
||||
"enable_key": "EnableUSDJPYBuster", "lot_key": "LOT_UB_USDJPY",
|
||||
"defaults": {"range_start_hour": 3, "range_end_hour": 6, "close_hour": 18,
|
||||
"min_range_pts": 15, "order_buffer_pts": 4.75, "first_trade_only": False,
|
||||
"allow_long": True, "allow_short": True, "use_take_profit": False},
|
||||
"opt": {"min_range_pts": (8, 40, 4), "order_buffer_pts": (2, 12, 1)}},
|
||||
]
|
||||
|
||||
PERIODS = {
|
||||
"2021-2026": ("2021-01-01", "2026-06-01"),
|
||||
}
|
||||
Reference in New Issue
Block a user