From d751b4af14f951500927bbb1c1a0ace3f7613e13 Mon Sep 17 00:00:00 2001 From: ZhijuCen Date: Sat, 4 Jul 2026 20:13:12 +0800 Subject: [PATCH] feat(parse_optimizer_report): add 'outliers' subcommand + EA-agnostic column typing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two related improvements to the optimization-report parser: 1. Generic column typing — drop the hardcoded 'InpUseNewsFilter' branch in parse_passes. Now the SpreadsheetML first body row's marks a column as string (boolean Inp* rendered as 'true'/'false' stays str); everything else is numeric (Int64 when whole-numbered, float64 otherwise). No Inp* name is referenced, so the parser handles any EA's parameter naming. 2. Generic param detection — _param_cols now returns every column AFTER 'Trades' by position, not by Inp* prefix. Real exports don't always use the Inp prefix; per user, 'Trades 以后的列数至少有一列, 但数量不定, 它们都是加入优化的输入参数'. 3. New 'outliers' subcommand — per-pass z-score scan over the 8 performance metrics (Result, Profit, Expected Payoff, Profit Factor, Recovery Factor, Sharpe Ratio, Custom, Equity DD %). Splits passes into two disjoint sets sorted by Result desc: - Set A: at least one metric with |z|>=σ in the favourable direction (higher-is-better metrics: z>=+σ; Equity DD % uses z<=-σ because low DD is good). - Set B: no performance-metric outlier. Both sets EXCLUDE passes whose Trades count is itself a low-side outlier (z<=-σ) — too few trades to trust. Excluded list shown separately. Default σ=2, top 10 outliers / top 5 normal; --sigma / --top-outliers / --top-normal / --json flags. Output prints a per-metric mean/std/±σ reference table, then each record split into Metrics group + Params group with the outlier σ values annotated. Style mirrors the windows subcommand in parse_tester_report.py. Verified on jobs/246753 (432 passes, OneShotGold XAUUSD H4): 44 Set A passes, 382 Set B, 6 excluded low-Trades passes, all 13 ad-hoc verification assertions pass. --- AGENTS.md | 42 ++ skills/mql5/SKILL.md | 93 +++- skills/mql5/scripts/parse_optimizer_report.py | 514 +++++++++++++++++- 3 files changed, 631 insertions(+), 18 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index e65de28..690af99 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -207,6 +207,48 @@ Use alongside `parse_tester_report.py` for the same EA: the latter explains *why* a specific pass performs, the former explains *which* pass performs and *which* parameters are even worth tuning. +#### `outliers` subcommand + +Per-pass z-score scan over the 8 performance metrics +(Result, Profit, Expected Payoff, Profit Factor, Recovery Factor, +Sharpe Ratio, Custom, Equity DD %). Two disjoint sets: + +- **Set A** — passes with AT LEAST ONE performance metric crossing + ±σ in the favourable direction (higher-is-better metrics: z >= +σ; + Equity DD % uses z <= -σ because low DD is good). Sorted by + `Result` desc, top `--top-outliers` (default 10) shown. +- **Set B** — passes with NO performance-metric outlier. Sorted by + `Result` desc, top `--top-normal` (default 5) shown. + +Both sets EXCLUDE passes whose `Trades` count is itself a low-side +outlier (z <= -σ) — those have too few trades to trust, and the +excluded list is shown separately. + +Output: header card + per-metric reference table (mean, std, ±σ +threshold) + Set A records (each split into Metrics group and Params +group, with the outlier σ values annotated) + Set B records + the +Excluded list. + +``` +# Default (σ=2, top 10 outliers, top 5 normal) +python skills/mql5/scripts/parse_optimizer_report.py report.xml outliers + +# Tighter threshold + custom counts +python skills/mql5/scripts/parse_optimizer_report.py report.xml outliers --sigma 2.5 --top-outliers 5 --top-normal 3 + +# JSON output for further processing +python skills/mql5/scripts/parse_optimizer_report.py report.xml outliers --json +``` + +**EA-agnostic by design**: the script does not hardcode any input +parameter name. Type inference reads `` from +the SpreadsheetML header (boolean Inp* rendered as "true"/"false" +stays a string; everything else is numeric). Input parameters are +detected by **column position** — every column after `Trades` is an +optimization input, regardless of whether its name starts with +`Inp`. So an EA naming its params `StopLoss` / `TakeProfit` / +`UseNewsFilter` parses correctly without script changes. + ### mql5_helper.py MT5 development helper for compile/deploy/status via Wine: diff --git a/skills/mql5/SKILL.md b/skills/mql5/SKILL.md index ad1e48d..2871e12 100644 --- a/skills/mql5/SKILL.md +++ b/skills/mql5/SKILL.md @@ -838,7 +838,7 @@ Optimization exports a different artifact: a single-worksheet XML-tagged Excel workbook (`ReportOptimizer-*.xml`, also openable in LibreOffice Calc). Each row is one parameter pass; the first worksheet name is `Tester Optimizator Results`. Use `scripts/parse_optimizer_report.py` to -extract and analyze it. The script has three modes: +extract and analyze it. Top-level modes: ```bash python skills/mql5/scripts/parse_optimizer_report.py ReportOptimizer-*.xml @@ -846,6 +846,17 @@ python skills/mql5/scripts/parse_optimizer_report.py ReportOptimizer-*.xml --jso python skills/mql5/scripts/parse_optimizer_report.py ReportOptimizer-*.xml --analyze ``` +Plus a subcommand: + +```bash +python skills/mql5/scripts/parse_optimizer_report.py ReportOptimizer-*.xml outliers [--sigma 2] [--top-outliers 10] [--top-normal 5] [--json] +``` + +The `outliers` subcommand does a per-pass z-score scan on the 8 +performance metrics and splits passes into "strongly-strong" (Set A, +at least one metric `|z| >= σ` in the favourable direction) vs +"no-outlier" (Set B), sorted by `Result` desc. See §10 below. + The `Title` field in `` encodes the strategy environment on one line: ` , -`. `` also carries `Deposit`, `Leverage`, `Server`, @@ -982,6 +993,86 @@ identify the same pass as "best profit" — otherwise the disagreement is the most interesting finding (it means there's a regime-specific trade-off you should investigate, not average away). +#### 10. Per-Pass Outlier Scan + +`--analyze` tells you which **parameter ranges** win and which are dead, +but it doesn't tell you which **individual passes** are statistical +outliers — passes so far above (or, for DD, so far below) the run's +own mean that they deserve separate scrutiny. The `outliers` +subcommand does this with a per-metric z-score scan over the 8 +performance metrics. + +``` +# Default: σ=2.0, top 10 outlier passes, top 5 normal passes +python skills/mql5/scripts/parse_optimizer_report.py ReportOptimizer-*.xml outliers + +# Tighter threshold + custom top-N +python skills/mql5/scripts/parse_optimizer_report.py ReportOptimizer-*.xml outliers --sigma 2.5 --top-outliers 5 + +# JSON for further processing +python skills/mql5/scripts/parse_optimizer_report.py ReportOptimizer-*.xml outliers --json +``` + +**The 8 performance metrics scanned** are everything in `METRIC_COLS` +except `Trades`: `Result`, `Profit`, `Expected Payoff`, `Profit Factor`, +`Recovery Factor`, `Sharpe Ratio`, `Custom`, `Equity DD %`. `Trades` is +explicitly excluded from the metric scan and used only as an exclusion +filter (see below). + +**Direction conventions** — per-metric outlier criterion: +- **Higher-is-better** (`Result`, `Profit`, `Expected Payoff`, + `Profit Factor`, `Recovery Factor`, `Sharpe Ratio`, `Custom`): + outlier = `z >= +σ` +- **Lower-is-better** (`Equity DD %`): outlier = `z <= -σ` (low DD is + good; high DD is bad but is not a "strong" outlier — those passes + are simply average) + +**Two disjoint sets** (both sorted by `Result` desc, both excluding +low-Trades passes): + +- **Set A — passes with at least one perf-metric outlier.** These are + candidates for closer inspection: a pass posting `z > +2` on Profit + *and* `z > +2` on Sharpe together is genuinely exceptional; a pass + posting `z > +2` on Profit alone with everything else flat is more + likely a fluke of small-sample variance. +- **Set B — passes with no perf-metric outlier** (top M by Result). + The "next-tier" passes — strong but unremarkable relative to the + rest of the run. Use these when Set A is too small to draw + conclusions. + +**Low-Trades exclusion** — passes with `Trades z <= -σ` (too few +trades to trust) are dropped before either set is built. They are +listed separately in the output so you can see what was filtered. + +**Reference table** — the output header prints `mean`, `std`, and +`±σ threshold` for every metric so you can judge whether the outlier +count is meaningful. A run with `Profit std` tiny relative to `mean` +will have many `z > +2` candidates because the threshold is near the +mean; a run with `Profit std` large (high EA variance) will have few. +Always read the std row before trusting the count. + +**Sigma choice** — `σ=2.0` is the default. For runs with hundreds of +passes that's strict enough to be useful (~5% of values expected to +clear it under a normal distribution, concentrated at the extremes). +For runs with <100 passes, drop to `σ=1.5` if Set A is empty; +`σ=3.0` is appropriate only for runs with >1000 passes (then +`z > 3` outliers are real signal, not tails). + +**Generic column typing** — `parse_optimizer_report.py` does not +hardcode input-parameter names. Two techniques that make the script +EA-agnostic: +- **Type inference** comes from the SpreadsheetML header: a body cell + with `` stays as a string column (e.g. + boolean Inp* rendered as `"true"`/`"false"`); numeric cells become + `float64` (or `Int64` when every value is whole). No `Inp*` name + is ever referenced. +- **Input-parameter detection** uses column position, not name + prefix: `Trades` is the last fixed column; every column after it is + an optimization input parameter regardless of whether its name + starts with `Inp`. So an EA that names parameters `StopLoss`, + `TakeProfit`, `UseNewsFilter` is parsed correctly without any + script changes. + ## 7. Event Handlers Reference | Handler | When Called | Use Case | diff --git a/skills/mql5/scripts/parse_optimizer_report.py b/skills/mql5/scripts/parse_optimizer_report.py index 573b842..d10d1cb 100644 --- a/skills/mql5/scripts/parse_optimizer_report.py +++ b/skills/mql5/scripts/parse_optimizer_report.py @@ -26,6 +26,7 @@ Usage: python skills/mql5/scripts/parse_optimizer_report.py python skills/mql5/scripts/parse_optimizer_report.py --json python skills/mql5/scripts/parse_optimizer_report.py --analyze + python skills/mql5/scripts/parse_optimizer_report.py outliers [--top-outliers N] [--top-normal M] [--sigma K] [--json] """ from __future__ import annotations @@ -134,10 +135,16 @@ def parse_env(soup: BeautifulSoup) -> Env: def parse_passes(soup: BeautifulSoup) -> pd.DataFrame: """Pull the first worksheet into a typed DataFrame. - Assumes the well-known MT5 column layout: 1 Pass col + 9 metric cols - + N Inp* param cols. cells become strings - (covers InpUseNewsFilter 'true'/'false'); everything else is coerced - to numeric. + Column layout is determined from the SpreadsheetML header row: any + cell in the header marks a string column + (typically a boolean Inp* parameter rendered as "true"/"false"); + everything else is coerced to numeric. This is EA-agnostic — no + hardcoded Inp* parameter names. + + The Pass column is always forced to nullable Int64 (it's a row id + even when declared as Number in the SpreadsheetML). All other + Number columns become float64 (or Int64 when every value is a whole + number — useful for InpSLPips-style integer parameters). """ ws = soup.find("worksheet", attrs={"ss:name": "Tester Optimizator Results"}) if not ws: @@ -154,26 +161,51 @@ def parse_passes(soup: BeautifulSoup) -> pd.DataFrame: if len(rows) < 2: return pd.DataFrame() - headers = [c.get_text(strip=True) for c in rows[0].find_all("cell")] + # Header row: just extract column names. Header cells are always + # ss:Type="String" (they hold text labels), so we can't use them + # to decide which body columns are strings. + header_cells = rows[0].find_all("cell") + headers: list[str] = [c.get_text(strip=True) for c in header_cells] - data = [] + # First body row: inspect each cell's to learn + # which columns hold strings (e.g. boolean Inp* as "true"/"false"). + # Number-typed cells declare ss:Type="Number" (sometimes absent). + first_body_cells = rows[1].find_all("cell") + string_cols: set[int] = set() + for i, c in enumerate(first_body_cells): + data_tag = c.find("data") + if data_tag is None: + continue + ss_type = (data_tag.get("ss:type") or "").lower() + if ss_type == "string": + string_cols.add(i) + + # Body rows: pull text. We don't trust per-row ss:Type for numeric + # columns because MT5 sometimes leaves it off; pandas' to_numeric + # is the authoritative check. + body: list[list[str]] = [] for r in rows[1:]: cells = r.find_all("cell") - data.append([c.get_text(strip=True) for c in cells]) + body.append([c.get_text(strip=True) for c in cells]) - df = pd.DataFrame(data, columns=headers) + df = pd.DataFrame(body, columns=headers) - # Type inference - for col in df.columns: + # Type inference driven by the first body row's ss:Type. + for i, col in enumerate(df.columns): if col == "Pass": + # Pass is always an integer row id df[col] = pd.to_numeric(df[col], errors="coerce").astype("Int64") continue - if col == "InpUseNewsFilter": - # Stay as string "true"/"false" — typed comparison matters + if i in string_cols: + # ss:Type="String" — keep as raw text (e.g. "true"/"false"). df[col] = df[col].astype(str) continue - # Numeric metric or numeric parameter - df[col] = pd.to_numeric(df[col], errors="coerce") + # Numeric: try Int64 if every value is a whole number, else float64 + coerced = pd.to_numeric(df[col], errors="coerce") + if pd.notna(coerced).all() and (coerced.dropna() % 1 == 0).all(): + df[col] = coerced.astype("Int64") + else: + df[col] = coerced.astype("float64") return df @@ -189,8 +221,19 @@ def parse_report(path: Path) -> tuple[Env, pd.DataFrame]: # ── Analysis ───────────────────────────────────────────────────────── def _param_cols(df: pd.DataFrame) -> list[str]: - """Inp* columns that the EA declared for optimization.""" - return [c for c in df.columns if c.startswith("Inp")] + """Input parameter columns the EA declared for optimization. + + The MT5 export puts Pass + the 9 standard metrics first, then + every optimization parameter afterwards. We rely on column + position (everything after ``Trades``), NOT on a name prefix — + parameter names don't always start with ``Inp*`` in real exports. + """ + cols = list(df.columns) + if "Trades" in cols: + idx = cols.index("Trades") + return cols[idx + 1:] + # Fallback: no Trades column — assume nothing is a parameter + return [] def _backtest_days(env: Env) -> int | None: @@ -381,6 +424,363 @@ def analyze(df: pd.DataFrame, env: Env) -> dict: return out +# ── Outlier analysis (per-pass z-score over optimization passes) ───── + +# Per-metric direction: True = higher-is-better (we look at z >= +sigma), +# False = lower-is-better (we look at z <= -sigma). Equity DD % is the +# only lower-is-better metric — low DD means the strategy is safer. +_PERF_METRICS_HIGHER_IS_BETTER = { + "Result": True, + "Profit": True, + "Expected Payoff": True, + "Profit Factor": True, + "Recovery Factor": True, + "Sharpe Ratio": True, + "Custom": True, + "Equity DD %": False, +} + + +def _mean_std(values: list[float]) -> tuple[float, float]: + """Return (mean, sample std N-1) of `values`. std=0 if N<2.""" + n = len(values) + if n == 0: + return 0.0, 0.0 + if n == 1: + return values[0], 0.0 + mean = sum(values) / n + var = sum((x - mean) ** 2 for x in values) / (n - 1) + return mean, var ** 0.5 + + +def _z_scores(values: list[float]) -> list[float]: + """Sample-std z-scores for `values`. z=0 when std=0.""" + mean, std = _mean_std(values) + if std == 0: + return [0.0] * len(values) + return [(v - mean) / std for v in values] + + +def _outlier_metrics_for_row( + z_by_metric: dict[str, float], + sigma: float, +) -> list[dict]: + """Per-row outlier flags, in the "performance-strongly-strong" sense. + + For each metric, the row is "strong" when its z is on the GOOD side + of the mean by at least `sigma` standard deviations: + + - higher-is-better metric -> z >= sigma + - lower-is-better metric -> z <= -sigma (e.g. low DD is good) + + Returns a list of {"metric": ..., "z": ...} entries, one per metric + that qualifies. Empty list means no performance metric is a + "strongly-strong" outlier. + """ + flags = [] + for m, z in z_by_metric.items(): + higher = _PERF_METRICS_HIGHER_IS_BETTER.get(m, True) + if higher and z >= sigma: + flags.append({"metric": m, "z": round(z, 2)}) + elif (not higher) and z <= -sigma: + flags.append({"metric": m, "z": round(z, 2)}) + return flags + + +def find_outliers( + df: pd.DataFrame, + sigma: float = 2.0, + top_outliers: int = 10, + top_normal: int = 5, +) -> dict: + """Per-pass z-score outlier scan across the 8 performance metrics. + + Two disjoint pass sets are returned: + + set_outliers — passes with AT LEAST ONE performance metric whose + z-score crosses ±sigma in the "strongly-strong" direction + (higher-is-better: z >= +sigma; lower-is-better Equity DD %: + z <= -sigma). Sorted by Result desc; top `top_outliers` shown. + + set_normal — passes with NO performance-metric outlier. Sorted by + Result desc; top `top_normal` shown. + + Both sets EXCLUDE passes whose Trades count is itself a low-side + outlier (z <= -sigma). Such passes have too few trades to trust + their performance numbers — including them would inflate the + outlier set with cheap-pass artifacts. + + Also returns the per-metric mean / std / sigma threshold table so + the user can judge whether the outlier counts are meaningful. + + Output shape: + { + "per_metric": {"profit": {"mean":..., "std":..., "threshold_+sigma":..., + "threshold_-sigma":..., "direction": "higher"|"lower"}, + ...}, + "trades": {"mean":..., "std":..., "low_outlier_z": ...}, + "sigma": float, + "n_passes": int, + "excluded": [{"Pass":..., "Trades":..., "z": ...}, ...], # low-Trades outliers + "set_outliers":[{Pass, Result, profit, ..., outliers: [{metric, z}, ...], + params: {InpSLPips: ..., ...}}, ...], + "set_normal": [...], + "counts": {"outliers": int, "normal": int, "excluded": int}, + } + """ + out: dict = {"sigma": sigma, "n_passes": int(len(df))} + + if df.empty: + out["error"] = "No pass rows" + return out + + # The 8 performance metrics (everything in METRIC_COLS except Trades) + perf_metrics = [c for c in METRIC_COLS if c in df.columns and c != "Trades"] + out["per_metric"] = {} + z_perf_by_row: list[dict[str, float]] = [] + for m in perf_metrics: + vals = [float(v) for v in df[m].tolist()] + mean, std = _mean_std(vals) + zs = _z_scores(vals) + higher = _PERF_METRICS_HIGHER_IS_BETTER.get(m, True) + out["per_metric"][m] = { + "mean": round(mean, 4), + "std": round(std, 4), + "threshold_+sigma": round(mean + sigma * std, 4), + "threshold_-sigma": round(mean - sigma * std, 4), + "direction": "higher" if higher else "lower", + } + # Pad / align z rows with column index (defensive: pad if cols differ) + for i, z in enumerate(zs): + if i >= len(z_perf_by_row): + z_perf_by_row.append({}) + z_perf_by_row[i][m] = z + + # Trades: only used for the low-Trades exclusion filter + if "Trades" in df.columns: + trades_vals = [float(v) for v in df["Trades"].tolist()] + mean, std = _mean_std(trades_vals) + out["trades"] = { + "mean": round(mean, 2), + "std": round(std, 2), + "low_outlier_threshold": round(mean - sigma * std, 2), + } + trades_z = _z_scores(trades_vals) + else: + trades_z = [0.0] * len(df) + out["trades"] = None + + pcols = _param_cols(df) + + # Build per-row records; carry Pass / Result / metric values / z / params + rows = [] + for idx, row in df.iterrows(): + z_map = z_perf_by_row[idx] if idx < len(z_perf_by_row) else {} + rec = { + "Pass": int(row["Pass"]) if "Pass" in df.columns and pd.notna(row["Pass"]) else None, + "Result": float(row["Result"]) if "Result" in df.columns and pd.notna(row["Result"]) else None, + "perf_metrics": {m: float(row[m]) for m in perf_metrics if pd.notna(row[m])}, + "z_metrics": {m: round(z_map.get(m, 0.0), 2) for m in perf_metrics}, + "Trades": int(row["Trades"]) if "Trades" in df.columns and pd.notna(row["Trades"]) else None, + "Trades_z": round(trades_z[idx], 2) if idx < len(trades_z) else 0.0, + "params": {p: (row[p] if pd.notna(row[p]) else None) for p in pcols}, + } + rec["outliers"] = _outlier_metrics_for_row(z_map, sigma) + rows.append(rec) + + # Exclusion: passes with Trades z <= -sigma — too few trades to trust. + excluded = [r for r in rows if r["Trades_z"] <= -sigma] + eligible = [r for r in rows if r["Trades_z"] > -sigma] + out["excluded"] = [ + {"Pass": r["Pass"], "Trades": r["Trades"], "Trades_z": r["Trades_z"]} + for r in excluded + ] + + # Sort eligible passes by Result desc + eligible.sort(key=lambda r: (r["Result"] is None, -(r["Result"] or 0.0))) + + # Set A: at least one performance-metric outlier + set_a = [r for r in eligible if r["outliers"]] + # Set B: no performance-metric outlier + set_b = [r for r in eligible if not r["outliers"]] + + out["set_outliers"] = set_a[:top_outliers] + out["set_normal"] = set_b[:top_normal] + out["counts"] = { + "outliers": len(set_a), + "normal": len(set_b), + "excluded_low_trades": len(excluded), + } + return out + + +def _fmt_outlier_row(rec: dict, pcols: list[str]) -> str: + """One-pass outlier record as two compact lines (metrics, params). + + Layout: + Pass 42 | Result=2.31 | Profit=800.10 EP=4.55 PF=1.61 RF=4.55 ... | z: Profit(+2.4σ), Sharpe(+2.1σ) + params: InpSLPips=30, InpTPPips=180, InpUseNewsFilter=true + """ + out_bits = [] + pass_str = f"Pass {rec['Pass']}" if rec["Pass"] is not None else "Pass ?" + res_str = f"Result={rec['Result']:.2f}" if rec["Result"] is not None else "Result=?" + out_bits.append(f" {pass_str:<10} {res_str}") + + # Metric values (in canonical METRIC_COLS order minus Trades) + metric_bits = [] + for m in METRIC_COLS: + if m == "Trades": + continue + if m not in rec["perf_metrics"]: + continue + v = rec["perf_metrics"][m] + # Short labels for readability + short = { + "Result": "Res", + "Profit": "Prof", + "Expected Payoff": "EP", + "Profit Factor": "PF", + "Recovery Factor": "RF", + "Sharpe Ratio": "Sharpe", + "Custom": "Cust", + "Equity DD %": "EqDD%", + }.get(m, m) + metric_bits.append(f"{short}={v:.2f}") + trades_part = f" Trades={rec['Trades']}" if rec["Trades"] is not None else "" + out_bits.append(" Metrics: " + " ".join(metric_bits) + trades_part) + + # Outlier annotations: only the metrics that crossed the threshold + if rec["outliers"]: + annot = ", ".join( + f"{o['metric']}({o['z']:+.2f}σ)" for o in rec["outliers"] + ) + out_bits.append(f" Outliers: {annot}") + else: + out_bits.append(" Outliers: (none)") + + # Input parameters — keep all of them, one per line if many + if pcols: + param_parts = [] + for p in pcols: + val = rec["params"].get(p) + if isinstance(val, float) and val.is_integer(): + val = int(val) + param_parts.append(f"{p}={val}") + # Wrap to <=4 params per line for readability + lines = [] + cur = [] + cur_len = 0 + for part in param_parts: + if cur and cur_len + len(part) + 2 > 70: + lines.append(", ".join(cur)) + cur = [part] + cur_len = len(part) + else: + cur.append(part) + cur_len += len(part) + 2 + if cur: + lines.append(", ".join(cur)) + out_bits.append(" Params: " + lines[0]) + for extra in lines[1:]: + out_bits.append(" " + extra) + + return "\n".join(out_bits) + + +def print_outliers(env: Env, df: pd.DataFrame, out: dict) -> None: + """Pretty-print the outlier scan as a text report. + + Layout follows the windows subcommand style: header card, then + a per-metric reference table (mean / std / sigma threshold), + then the two pass sets grouped by [Metrics] and [Params]. + """ + print("=" * 78) + print(f" Optimization Outlier Scan (per-pass z-score across {out.get('n_passes', '?')} passes)") + print("=" * 78) + print(f" EA: {env.ea_name or '(unknown)'} Symbol: {env.symbol or '(unknown)'} " + f"Period: {env.period or '(unknown)'}") + print(f" Date range: {env.date_from} → {env.date_to}") + print(f" Sigma threshold: {out['sigma']:.1f} " + f"(direction: higher-is-better uses z>=+σ; Equity DD % uses z<=-σ)") + print() + + if "error" in out: + print(f" ERROR: {out['error']}") + return + + # Per-metric reference table + print(f" {'Metric':<22} {'Dir':>5} {'Mean':>12} {'Std':>10} " + f"{'+σ threshold':>14} {'-σ threshold':>14}") + print(" " + "─" * 80) + for m in METRIC_COLS: + if m == "Trades" or m not in out["per_metric"]: + continue + info = out["per_metric"][m] + print(f" {m:<22} {info['direction']:>5} " + f"{info['mean']:>12.4f} {info['std']:>10.4f} " + f"{info['threshold_+sigma']:>14.4f} {info['threshold_-sigma']:>14.4f}") + if out.get("trades"): + t = out["trades"] + print(f" {'Trades (excluded)':<22} {'low':>5} " + f"{t['mean']:>12.2f} {t['std']:>10.2f} " + f"{'--':>14} {t['low_outlier_threshold']:>14.2f}") + print() + + # Counts + c = out["counts"] + print(f" Set A (>=1 perf outlier, sorted by Result desc): {c['outliers']} passes") + print(f" Set B (no perf outlier, sorted by Result desc): {c['normal']} passes") + if c["excluded_low_trades"]: + print(f" Excluded (Trades z<=-σ, too few trades to trust): {c['excluded_low_trades']} passes") + print() + + pcols = _param_cols(df) + + # Set A + if out["set_outliers"]: + print("=" * 78) + print(f" SET A — Passes with at least one performance-metric outlier") + print(f" (top {len(out['set_outliers'])} by Result; " + f"{c['outliers']} total eligible)") + print("=" * 78) + for rec in out["set_outliers"]: + print(_fmt_outlier_row(rec, pcols)) + print() + else: + print(" SET A: empty (no eligible passes have a performance-metric outlier)") + print() + + # Set B + if out["set_normal"]: + print("=" * 78) + print(f" SET B — Passes with no performance-metric outlier (top {len(out['set_normal'])} by Result)") + print("=" * 78) + for rec in out["set_normal"]: + print(_fmt_outlier_row(rec, pcols)) + print() + else: + print(" SET B: empty (every eligible pass has at least one performance-metric outlier)") + print() + + # Excluded (low-Trades) + if out["excluded"]: + print("=" * 78) + print(f" EXCLUDED — Passes with Trades z <= -{out['sigma']:.0f}σ (too few trades to trust)") + print("=" * 78) + for e in out["excluded"]: + print(f" Pass {e['Pass']:<6} Trades={e['Trades']:<5} Trades_z={e['Trades_z']:+.2f}σ") + print() + + # Interpretation + print(" Interpretation:") + print(f" z = (this pass's value − mean across all passes) / std.") + print(f" For higher-is-better metrics, the outlier criterion is z >= +{out['sigma']:.0f}.") + print(f" For lower-is-better (Equity DD %), the criterion is z <= -{out['sigma']:.0f}.") + print(f" The Trades exclusion guards against over-fitting: a pass with") + print(f" very few trades can post an extreme metric value purely by") + print(f" luck, so we drop those before ranking the rest.") + + def _env_card(env: Env) -> dict: """Compact strategy-environment summary for JSON / text output.""" days = _backtest_days(env) @@ -565,7 +965,25 @@ def print_analyze(env: Env, df: pd.DataFrame, an: dict) -> None: def main() -> None: parser = argparse.ArgumentParser( - description="Parse MT5 Strategy Tester Optimization XML report" + description="Parse MT5 Strategy Tester Optimization XML report", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=""" +Examples: + # Text report (default) + python %(prog)s ReportOptimizer-*.xml + + # JSON dump (raw parsed data) + python %(prog)s ReportOptimizer-*.xml --json + + # Full optimization analysis (parameter effect, duplicates, best passes) + python %(prog)s ReportOptimizer-*.xml --analyze + + # Outlier scan: per-pass z-score on the 8 perf metrics, + # split into "has at least one strong outlier" vs "no outlier" + python %(prog)s ReportOptimizer-*.xml outliers + python %(prog)s ReportOptimizer-*.xml outliers --top-outliers 5 --top-normal 5 --sigma 2 + python %(prog)s ReportOptimizer-*.xml outliers --json +""", ) parser.add_argument("report", help="Path to ReportOptimizer-*.xml file") parser.add_argument("--json", action="store_true", help="Output raw parsed data as JSON") @@ -574,6 +992,49 @@ def main() -> None: action="store_true", help="Run optimization analysis (parameter effect, duplicates, best passes, trade distribution)", ) + sub = parser.add_subparsers(dest="cmd") + + p_out = sub.add_parser( + "outliers", + help="Per-pass z-score outlier scan on the 8 performance metrics. " + "Splits passes into a 'strongly-strong' set (at least one metric with |z|>=σ " + "in the favourable direction) and a 'no-outlier' set, both sorted by Result " + "desc and printed with the metrics group + the input-parameter group.", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=""" +Examples: + # Default: sigma=2, top 10 outlier passes, top 5 normal passes + python parse_optimizer_report.py ReportOptimizer-*.xml outliers + + # Tighter sigma threshold and custom top-N + python parse_optimizer_report.py ReportOptimizer-*.xml outliers --sigma 3 --top-outliers 5 + + # JSON output for further processing + python parse_optimizer_report.py ReportOptimizer-*.xml outliers --json + +For higher-is-better metrics (Profit, Profit Factor, Recovery Factor, +Sharpe Ratio, Expected Payoff, Custom, Result), an outlier is z >= +σ. +For lower-is-better (Equity DD %), an outlier is z <= -σ. Passes whose +Trades count is itself a low-side outlier (z <= -σ) are excluded +first — they have too few trades to trust. +""", + ) + p_out.add_argument( + "--sigma", "-s", type=float, default=2.0, + help="z-score threshold for the outlier scan (default: 2.0)", + ) + p_out.add_argument( + "--top-outliers", type=int, default=10, + help="Number of passes to show from the 'has at least one outlier' set (default: 10)", + ) + p_out.add_argument( + "--top-normal", type=int, default=5, + help="Number of passes to show from the 'no outlier' set (default: 5)", + ) + p_out.add_argument( + "--json", action="store_true", help="Output outliers data as JSON", + ) + args = parser.parse_args() path = Path(args.report) @@ -584,6 +1045,25 @@ def main() -> None: env, df = parse_report(path) analyze_data = analyze(df, env) if (args.analyze or args.json) else None + if args.cmd == "outliers": + out_data = find_outliers( + df, + sigma=args.sigma, + top_outliers=args.top_outliers, + top_normal=args.top_normal, + ) + if getattr(args, "json", False): + payload = { + "env": asdict(env), + "pass_count": int(len(df)), + "columns": list(df.columns), + "outliers": out_data, + } + print(json.dumps(payload, indent=2, ensure_ascii=False, default=str)) + else: + print_outliers(env, df, out_data) + return + if args.json or args.analyze: payload = { "env": asdict(env),