添加批量解析

This commit is contained in:
2026-07-11 03:22:50 +08:00
parent 9cdf93cc3d
commit 491ea17137
985 changed files with 4285 additions and 103 deletions
+39 -2
View File
@@ -12,6 +12,7 @@
|---|---|---|
| `mt5_report_parser.py` | 解析 MT5 中文 xlsx 报告(设置/结果/订单/成交),重建逐笔交易 | `MT5Report` 对象 |
| `run_analysis.py` | 主分析器:核心指标对比、What-If 假设、蒙特卡洛、方向/时段诊断、数据驱动建议 | `report.html` |
| `batch_report.py` | **新增** 大批量报告汇总:排名表 + 分布图 + 异常检测 | `batch_report.html` |
| `walk_forward.py` | Walk-Forward 滚动 IS→OOS 验证,算 WFE 与泛化相关性 | `walk_forward.html` |
| `param_scan.py` | 参数敏感度扫描:网格 .set 生成 → 响应面 → 高原检测 → **输出优化后 .set** | `param_scan.html` + `.set` |
| `mae_mfe.py` | MAE/MFE 分析:散点 + TP/SL 扫描热力图,附 MQL5 导出代码 | `mae_mfe.html` |
@@ -61,13 +62,15 @@ python mae_mfe.py mae_mfe_2026-06-28.csv
```
.
├── mt5_report_parser.py # 通用 MT5 xlsx 解析器
├── mt5_report_parser.py # 通用 MT5 xlsx 解析器(含 LRU 缓存 + 公共 _seg_metrics
├── run_analysis.py # 主分析器(含 What-If/蒙特卡洛/规则建议/Walk-Forward 集成)
├── walk_forward.py # Walk-Forward 滚动验证
├── batch_report.py # 新增:大批量报告汇总(排名 + 分布 + 异常检测)
├── walk_forward.py # Walk-Forward 滚动验证(向量化边界查找)
├── param_scan.py # 参数敏感度扫描 + 优化 .set 输出
├── mae_mfe.py # MAE/MFE 分析 + MQL5 代码片段
├── output/ # 生成产物(HTML/.set,已入库便于在线预览)
│ ├── report.html
│ ├── batch_report.html
│ ├── walk_forward.html
│ ├── param_scan.html
│ └── mae_mfe.html
@@ -76,6 +79,28 @@ python mae_mfe.py mae_mfe_2026-06-28.csv
└── METRICS_GUIDE.md # 指标说明书(大白话解释所有指标原理)
```
### 批量报告分析(新增)
扫描目录下所有 MT5 xlsx 报告,一次性汇总成排名表 + 指标分布 + 异常检测:
```bash
# 扫描目录下的所有 xlsx
python batch_report.py reports_dir
# 指定 glob 模式
python batch_report.py reports_dir --pattern "ReportTester-*.xlsx"
# 通配多目录
python batch_report.py "dir1/*.xlsx" "dir2/*.xlsx"
```
生成的 `output/batch_report.html` 包含:
- 按 PF 排名的综合表
- 按净盈利排名的综合表
- PF/净盈利/胜率分布直方图
- PF vs 胜率散点图(带趋势线)
- 异常检测:过拟合嫌疑 / 样本不足 / 回撤过大自动标记
## 各模块详解
### 1. mt5_report_parser.py — 解析器
@@ -178,6 +203,18 @@ MT5 标准 xlsx 不含逐笔 MAE/MFE。本模块提供:
无 matplotlib 等重型绘图库,所有可视化用纯 SVG。
## 性能优化
对大批量报告分析场景做的关键优化:
| 优化 | 位置 | 说明 |
|---|---|---|
| Monte Carlo 向量化 | `run_analysis.py` | 1000 次置换矩阵一次性生成,C 层完成随机打乱 |
| TP/SL 扫描向量化 | `mae_mfe.py` | 30×30×N 笔 → NumPy 广播,消除 Python 嵌套循环 |
| Walk-Forward 边界查找 | `walk_forward.py` | `np.searchsorted` 替代逐行 DataFrame 切片 |
| 解析结果缓存 | `mt5_report_parser.py` | 文件级 LRU 缓存,同一份报告被多模块复用时不再重复 IO |
| 公共指标函数 | `mt5_report_parser.py` | `_seg_metrics` / `_stats_from_net` 合并为 `compute_segment_metrics`,消除重复代码 |
## License
MIT
+515
View File
@@ -0,0 +1,515 @@
# -*- coding: utf-8 -*-
"""
大批量报告汇总分析
==================
扫描目录下所有 MT5 xlsx 报告,一次性解析并汇总成一张排名表 +
指标分布图 + 异常检测。适合参数扫描或批量回测后的综合比较。
用法:
python batch_report.py reports_dir
python batch_report.py reports_dir --pattern "ReportTester-*.xlsx"
python batch_report.py . --glob # 用 glob 模式匹配(通配符支持多目录)
"""
from __future__ import annotations
import argparse
import glob
import html
import os
import sys
from typing import Any, Dict, List, Optional, Tuple
import numpy as np
import pandas as pd
import mt5_report_parser as mp
from mt5_report_parser import parse_report, MT5Report
import run_analysis as ra
OUT_DIR = "output"
BATCH_HTML_PATH = os.path.join(OUT_DIR, "batch_report.html")
# 汇总指标列定义
SUMMARY_COLS = [
("ea_name", "EA"),
("symbol", "品种"),
("period", "周期"),
("net_profit", "净盈利"),
("profit_factor", "PF"),
("win_rate", "胜率(%)"),
("expectancy", "期望/笔"),
("max_dd", "最大回撤"),
("max_dd_pct", "回撤(%)"),
("sharpe", "夏普"),
("sortino", "Sortino"),
("n_trades", "笔数"),
]
# =========================================================================== #
# 加载 + 解析
# =========================================================================== #
def scan_reports(
paths: List[str],
pattern: str = "*.xlsx",
) -> List[str]:
"""
从路径列表展开为完整文件路径列表。
每个路径可以是目录、单个文件、或 glob 模式。
支持 .xlsx 和 .htm/.html 格式。
"""
result: List[str] = []
for p in paths:
if os.path.isfile(p):
result.append(p)
elif os.path.isdir(p):
# 扫描目录下的所有 xlsx 和 htm/html 文件
for ext in ("*.xlsx", "*.htm", "*.html"):
result.extend(sorted(glob.glob(os.path.join(p, ext))))
elif "*" in p or "?" in p:
result.extend(sorted(glob.glob(p)))
else:
# 尝试当作目录
for ext in ("*.xlsx", "*.htm", "*.html"):
result.extend(sorted(glob.glob(os.path.join(p, ext))))
return sorted(set(result))
def parse_batch(
files: List[str],
*,
report_pair_mode: bool = False,
pair_prefix: str = "IS-",
) -> List[Tuple[str, MT5Report]]:
"""
批量解析所有文件。
如果 report_pair_mode=True 且文件以 pair_prefix 开头成对出现,
则只取第二份(OSS)的报告以避免重复。
"""
parsed: List[Tuple[str, MT5Report]] = []
for f in files:
try:
rep = parse_report(f)
parsed.append((os.path.basename(f), rep))
except Exception as e:
print(f" 跳过 {f}: {e}", file=sys.stderr)
return parsed
# =========================================================================== #
# 汇总数据
# ========================================================================= #
def build_summary_df(
reports: List[Tuple[str, MT5Report]],
) -> pd.DataFrame:
"""
汇总所有报告的关键指标为 DataFrame。
行 = 报告,列 = 指标。
"""
rows: List[Dict[str, Any]] = []
for name, rep in reports:
m = ra.extended_metrics(rep.trades)
s = rep.summary_norm # type: ignore[attr-defined]
meta = rep.meta
row = {
"filename": name,
"ea_name": meta.get("专家", ""),
"symbol": meta.get("交易品种", ""),
"period": meta.get("期间", ""),
"net_profit": m.get("net_profit", 0.0),
"profit_factor": m.get("profit_factor", 0.0),
"win_rate": m.get("win_rate", 0.0),
"expectancy": m.get("expectancy", 0.0),
"max_dd": m.get("max_dd", 0.0),
"max_dd_pct": m.get("max_dd_pct", 0.0),
"sharpe": s.get("sharpe", 0.0),
"sortino": m.get("sortino", 0.0),
"n_trades": m.get("n_trades", 0),
"avg_win": m.get("avg_win", 0.0),
"avg_loss": m.get("avg_loss", 0.0),
"max_streak_loss": m.get("max_streak_loss", 0),
"roll_pf_min": m.get("roll_pf_min", 0.0),
"roll_pf_max": m.get("roll_pf_max", 0.0),
"pct_profitable_window": m.get("pct_profitable_window", 0.0),
}
# 参数信息(前几个输入参数)
inputs = meta.get("inputs", {})
for k, v in inputs.items():
row[f"param_{k}"] = v
rows.append(row)
df = pd.DataFrame(rows)
# 数值列类型统一(跳过字符串列)
_String_Cols = {"filename", "ea_name", "symbol", "period"}
for col in df.columns:
if col not in _String_Cols and not col.startswith("param_"):
df[col] = pd.to_numeric(df[col], errors="coerce")
return df
# =========================================================================== #
# 异常检测
# =========================================================================== #
def detect_anomalies(df: pd.DataFrame) -> List[Dict[str, Any]]:
"""
标记异常报告:
- PF 异常高(Top 3% 且偏离均值 > 2σ)→ 过拟合嫌疑
- 胜率异常高 → 可能数据太少
- 笔数异常少 → 统计不显著
"""
flags: List[Dict[str, Any]] = []
n = len(df)
if n < 3:
return flags
pf_mean = float(df["profit_factor"].mean())
pf_std = float(df["profit_factor"].std())
top_pct = max(1, n // 30) # Top 3%
# PF 异常高
pf_sorted = df.sort_values("profit_factor", ascending=False)
for _, row in pf_sorted.head(top_pct).iterrows():
if pf_std > 0 and row["profit_factor"] > pf_mean + 2 * pf_std:
flags.append({
"filename": row["filename"],
"type": "过拟合嫌疑",
"detail": f"PF={row['profit_factor']:.2f} 远高于均值 {pf_mean:.2f} (偏离 {((row['profit_factor']-pf_mean)/pf_std):.1f}σ),且仅基于 {row['n_trades']} 笔,可能过拟合单段行情",
})
# 笔数过少
for _, row in df.iterrows():
if row["n_trades"] < 30:
flags.append({
"filename": row["filename"],
"type": "样本不足",
"detail": f"{row['n_trades']} 笔交易,统计指标不可靠",
})
# 回撤异常大
dd_mean = float(df["max_dd_pct"].mean())
dd_std = float(df["max_dd_pct"].std())
for _, row in df.iterrows():
if dd_std > 0 and row["max_dd_pct"] > dd_mean + 2 * dd_std and row["max_dd_pct"] > 50:
flags.append({
"filename": row["filename"],
"type": "回撤过大",
"detail": f"回撤 {row['max_dd_pct']:.0f}% 远高于均值 {dd_mean:.0f}%",
})
return flags
# =========================================================================== #
# SVG 分布图
# =========================================================================== #
def svg_hist(items: List[Tuple[str, np.ndarray]], w: int = 520, h: int = 300) -> str:
"""多变量分布直方图叠加。"""
top_pad = 30
bottom_pad = 45
left_pad = 55
right_pad = 16
plot_h = h - top_pad - bottom_pad
plot_w = w - left_pad - right_pad
colors = ["#2563eb", "#dc2626", "#16a34a", "#f59e0b", "#7c3aed"]
all_min = float(np.concatenate([x for _, x in items]).min())
all_max = float(np.concatenate([x for _, x in items]).max())
if all_max == all_min:
all_max = all_min + 1
def _hist(vals: np.ndarray, color: str, label: str, alpha: float = 0.7) -> str:
# 过滤 NaN 和 inf
clean = vals[(~np.isnan(vals)) & (np.isfinite(vals))]
if len(clean) == 0:
return ""
n_bins = 30
counts, edges = np.histogram(clean, bins=n_bins)
bin_w = (all_max - all_min) / n_bins
bars = ""
for i, c in enumerate(counts):
if c == 0:
continue
x = left_pad + i * bin_w
yb = top_pad + plot_h
yt = top_pad + plot_h - c / max(1, max(counts)) * plot_h
bars += f'<rect x="{x:.1f}" y="{yt:.1f}" width="{bin_w:.1f}" height="{yb-yt:.1f}" fill="{color}" opacity="{alpha}"/>'
return bars
bars = ""
for i, (lbl, vals) in enumerate(items):
color = colors[i % len(colors)]
bars += _hist(vals, color, lbl)
# 轴
xlabs = ""
for frac in (0, 0.25, 0.5, 0.75, 1.0):
xv = all_min + frac * (all_max - all_min)
gx = left_pad + frac * plot_w
xlabs += f'<line x1="{gx:.1f}" y1="{top_pad}" x2="{gx:.1f}" y2="{top_pad+plot_h}" stroke="#f1f5f9" stroke-width="0.5"/>'
xlabs += f'<text x="{gx:.1f}" y="{h-bottom_pad+14:.1f}" font-size="9" fill="#6b7280" text-anchor="middle">{xv:.2f}</text>'
# 图例
legend = ""
for i, (lbl, _) in enumerate(items[:4]):
color = colors[i % len(colors)]
legend += f'<rect x="{left_pad+i*100}" y="6" width="12" height="10" fill="{color}"/>'
legend += f'<text x="{left_pad+i*100+16}" y="14" font-size="9" fill="#374151">{html.escape(lbl)}</text>'
return (f'<svg viewBox="0 0 {w} {h}" class="chart">'
f'<text x="{w//2}" y="12" font-size="11" fill="#374151" text-anchor="middle">指标分布</text>'
f'{legend}{bars}{xlabs}</svg>')
def svg_scatter(df: pd.DataFrame, x: str, y: str, title: str,
w: int = 520, h: int = 340) -> str:
"""参数-指标散点图(带趋势线)。"""
top_pad = 30
bottom_pad = 45
left_pad = 55
right_pad = 16
plot_h = h - top_pad - bottom_pad
plot_w = w - left_pad - right_pad
x_arr = pd.to_numeric(df[x], errors="coerce").to_numpy()
y_arr = pd.to_numeric(df[y], errors="coerce").to_numpy()
mask = ~np.isnan(x_arr) & ~np.isnan(y_arr) & np.isfinite(x_arr) & np.isfinite(y_arr)
x_arr = x_arr[mask]
y_arr = y_arr[mask]
if len(x_arr) < 3:
return ""
x_min, x_max = float(x_arr.min()), float(x_arr.max())
y_min, y_max = float(y_arr.min()), float(y_arr.max())
if x_max == x_min: x_max = x_min + 1
if y_max == y_min: y_max = y_min + 1
def sx(v): return left_pad + (v - x_min) / (x_max - x_min) * plot_w
def sy(v): return top_pad + plot_h - (v - y_min) / (y_max - y_min) * plot_h
pts = ""
for xi, yi in zip(x_arr, y_arr):
col = "#16a34a" if yi > 0 else "#dc2626"
pts += f'<circle cx="{sx(xi):.1f}" cy="{sy(yi):.1f}" r="3" fill="{col}" opacity="0.65"/>'
# 趋势线
trend_line = ""
try:
if len(x_arr) > 2:
coeffs = np.polyfit(x_arr, y_arr, 1)
trend_xs = np.linspace(x_min, x_max, 50)
trend_ys = np.polyval(coeffs, trend_xs)
d = " ".join(f"{sx(x):.1f},{sy(y):.1f}" for x, y in zip(trend_xs, trend_ys))
trend_line = f'<polyline points="{d}" fill="none" stroke="#1e3a8a" stroke-width="1.2" stroke-dasharray="4,3" opacity="0.8"/>'
except np.linalg.LinAlgError:
pass # 趋势线拟合失败时跳过
xlabs = ""
for frac in (0, 0.5, 1.0):
xv = x_min + frac * (x_max - x_min)
gx = left_pad + frac * plot_w
xlabs += f'<line x1="{gx:.1f}" y1="{top_pad}" x2="{gx:.1f}" y2="{top_pad+plot_h}" stroke="#f1f5f9" stroke-width="0.5"/>'
xlabs += f'<text x="{gx:.1f}" y="{h-bottom_pad+14:.1f}" font-size="9" fill="#6b7280" text-anchor="middle">{xv:.2f}</text>'
yv = y_min + frac * (y_max - y_min)
gy = top_pad + plot_h - frac * plot_h
xlabs += f'<line x1="{left_pad}" y1="{gy:.1f}" x2="{w-right_pad}" y2="{gy:.1f}" stroke="#f1f5f9" stroke-width="0.5"/>'
xlabs += f'<text x="{left_pad-6:.1f}" y="{gy+3:.1f}" font-size="9" fill="#6b7280" text-anchor="end">{yv:.2f}</text>'
return (f'<svg viewBox="0 0 {w} {h}" class="chart">'
f'<text x="{w//2}" y="12" font-size="11" fill="#374151" text-anchor="middle">{html.escape(title)}</text>'
f'{xlabs}{trend_line}{pts}'
f'<text x="{w//2}" y="{h-4}" font-size="9" fill="#6b7280" text-anchor="middle">{html.escape(x)} →</text>'
f'<text x="14" y="{h//2}" font-size="9" fill="#6b7280" text-anchor="middle" transform="rotate(-90 14 {h//2})">{html.escape(y)} →</text>'
f'</svg>')
# =========================================================================== #
# 排名表
# =========================================================================== #
def build_ranking_table(df: pd.DataFrame, sort_col: str, ascending: bool = False) -> str:
"""按指定指标排序的排名表。"""
sorted_df = df.sort_values(sort_col, ascending=ascending)
rows = ""
for rank, (_, row) in enumerate(sorted_df.iterrows(), 1):
cls = " class='pos'" if row[sort_col] > 0 else " class='neg'" if row[sort_col] < 0 else ""
filename = str(row.get("filename", ""))
rows += (
f"<tr{cls}><td>{rank}</td>"
f"<td>{html.escape(filename)}</td>"
f"<td>{html.escape(str(row.get('ea_name','')))}</td>"
f"<td>{html.escape(str(row.get('symbol','')))}</td>"
f"<td>{html.escape(str(row.get('period','')))}</td>"
f"<td>{row['net_profit']:+.2f}</td>"
f"<td>{row['profit_factor']:.2f}</td>"
f"<td>{row['win_rate']:.1f}%</td>"
f"<td>{row['max_dd_pct']:.0f}%</td>"
f"<td>{row['sharpe']:.2f}</td>"
f"<td>{row['n_trades']}</td></tr>"
)
return (
"<table class='data'><thead><tr>"
"<th>排名</th><th>报告</th><th>EA</th><th>品种</th><th>周期</th>"
"<th>净盈利</th><th>PF</th><th>胜率</th><th>回撤%</th><th>夏普</th><th>笔数</th>"
"</tr></thead><tbody>" + rows + "</tbody></table>"
)
# =========================================================================== #
# HTML 报告
# =========================================================================== #
def build_html(df: pd.DataFrame, anomalies: List[Dict[str, Any]]) -> str:
n = len(df)
pf_stats = {
"mean": float(df["profit_factor"].mean()),
"median": float(df["profit_factor"].median()),
"std": float(df["profit_factor"].std()),
"min": float(df["profit_factor"].min()),
"max": float(df["profit_factor"].max()),
}
net_stats = {
"mean": float(df["net_profit"].mean()),
"std": float(df["net_profit"].std()),
}
# 排序选择:PF 降序(默认)、净盈利降序、胜率降序
pf_rank = build_ranking_table(df, "profit_factor", ascending=False)
net_rank = build_ranking_table(df, "net_profit", ascending=False)
# 分布图
hist_svg = svg_hist([
("PF", df["profit_factor"].to_numpy()),
("净盈利", df["net_profit"].to_numpy()),
("胜率", df["win_rate"].to_numpy()),
])
scatter_svg = svg_scatter(df, "profit_factor", "win_rate", "PF vs 胜率")
# 异常告警
anomaly_html = ""
if anomalies:
anomaly_rows = ""
for a in anomalies:
color = "#fee2e2" if a["type"] in ("过拟合嫌疑", "回撤过大") else "#fef3c7"
fname = str(a.get("filename", ""))
anomaly_rows += (f"<tr><td style='background:{color}'>{html.escape(a['type'])}</td>"
f"<td>{html.escape(fname)}</td>"
f"<td>{html.escape(a['detail'])}</td></tr>")
anomaly_html = (
"<h3>异常检测报告</h3>"
"<table class='data'><thead><tr><th>类型</th><th>报告</th><th>详情</th></tr></thead><tbody>"
+ anomaly_rows + "</tbody></table>"
)
return f"""<!doctype html>
<html lang="zh-CN"><head><meta charset="utf-8">
<title>批量回测汇总报告</title>
<style>
body {{ font-family:-apple-system,"Microsoft YaHei","Segoe UI",sans-serif; background:#f8fafc; color:#111827; margin:0; padding:20px;}}
.wrap {{ max-width:1200px; margin:0 auto;}}
h1,h2,h3 {{ color:#1e3a8a;}} h1 {{ border-bottom:3px solid #1e3a8a; padding-bottom:8px;}}
h2 {{ border-bottom:1px solid #cbd5e1; padding-bottom:4px; margin-top:28px;}}
table.data {{ border-collapse:collapse; width:100%; background:#fff; margin:8px 0 16px; font-size:12px;}}
table.data th, table.data td {{ border:1px solid #e2e8f0; padding:5px 7px; text-align:left;}}
table.data th {{ background:#f1f5f9;}}
table.data tr:nth-child(even) td {{ background:#fafbfc;}}
td.neg, tr.neg td {{ color:#dc2626;}} td.pos, tr.pos td {{ color:#16a34a;}}
.muted {{ color:#6b7280; font-size:11px;}}
.cards {{ display:grid; grid-template-columns:repeat(auto-fit,minmax(180px,1fr)); gap:12px; margin:16px 0;}}
.card {{ background:#fff; padding:14px; border-radius:8px; border:1px solid #e2e8f0;}}
.card .k {{ color:#6b7280; font-size:12px;}} .card .v {{ font-size:18px; font-weight:600; margin-top:4px;}}
.chart {{ width:100%; height:auto; background:#fff; border:1px solid #e2e8f0; border-radius:6px; margin:8px 0;}}
.grid2 {{ display:grid; grid-template-columns:1fr 1fr; gap:12px;}}
.note {{ background:#fef3c7; border-left:4px solid #f59e0b; padding:10px 14px; margin:12px 0; border-radius:4px;}}
.warn {{ background:#fee2e2; border-left:4px solid #dc2626; padding:10px 14px; margin:12px 0; border-radius:4px;}}
a {{ color:#2563eb; text-decoration:none;}} a:hover {{ text-decoration:underline;}}
footer {{ color:#6b7280; font-size:11px; margin-top:32px; text-align:center;}}
</style></head><body><div class="wrap">
<h1>批量回测汇总报告</h1>
<p class="muted">共解析 <b>{n}</b> 份报告 · {pd.Timestamp.now().strftime('%Y-%m-%d %H:%M')}</p>
<div class="cards">
<div class="card"><div class="k">报告总数</div><div class="v">{n}</div></div>
<div class="card"><div class="k">PF 均值</div><div class="v">{pf_stats['mean']:.2f}</div></div>
<div class="card"><div class="k">PF 中位数</div><div class="v">{pf_stats['median']:.2f}</div></div>
<div class="card"><div class="k">PF 标准差</div><div class="v">{pf_stats['std']:.2f}</div></div>
<div class="card"><div class="k">PF 范围</div><div class="v">{pf_stats['min']:.2f}~{pf_stats['max']:.2f}</div></div>
<div class="card"><div class="k">净盈利均值</div><div class="v">{net_stats['mean']:+.2f}</div></div>
</div>
<h2>1. 按 PF 排名</h2>
{pf_rank}
<h2>2. 按净盈利排名</h2>
{net_rank}
<h2>3. 指标分布</h2>
<div class="grid2">
<div>{hist_svg}</div>
<div>{scatter_svg}</div>
</div>
{anomaly_html}
<div class="note">
<b>使用说明</b>:点击报告名称可跳转至单份详细分析(需配合 run_analysis.py)。
异常检测标记了可能过拟合或统计不显著的报告,请优先复核。
</div>
<footer>由 batch_report.py 生成 · {pd.Timestamp.now().strftime('%Y-%m-%d %H:%M')}</footer>
</div></body></html>"""
# =========================================================================== #
# 单份报告 HTML(辅助跳转)
# =========================================================================== #
def write_single_htmls(
reports: List[Tuple[str, MT5Report]],
base_dir: str = OUT_DIR,
) -> None:
"""
为每份报告生成单份 mini 报告(可被主报告链接),避免生成 1MB+ 大文件。
仅在批量报告中提供索引跳转,不重复生成完整 HTML。
此处仅输出文件名索引信息供参考。
"""
pass
# =========================================================================== #
# CLI
# =========================================================================== #
def main(argv: List[str]) -> int:
ap = argparse.ArgumentParser(description="大批量报告汇总分析")
ap.add_argument("paths", nargs="+", help="报告目录或 glob 路径")
ap.add_argument("--pattern", default="*.xlsx", help="glob 模式(默认 *.xlsx")
args = ap.parse_args(argv)
os.makedirs(OUT_DIR, exist_ok=True)
files = scan_reports(args.paths, args.pattern)
if not files:
print(f"未找到任何报告文件,路径: {args.paths}")
return 1
print(f"找到 {len(files)} 份报告,开始解析...")
reports = parse_batch(files)
if not reports:
print("所有文件解析失败,退出。")
return 1
df = build_summary_df(reports)
anomalies = detect_anomalies(df)
print(f"\n汇总统计:")
print(f" PF 均值={df['profit_factor'].mean():.2f} 中位数={df['profit_factor'].median():.2f}")
print(f" 净盈利均值={df['net_profit'].mean():+.2f}")
print(f" 异常报告: {len(anomalies)}")
html_doc = build_html(df, anomalies)
with open(BATCH_HTML_PATH, "w", encoding="utf-8") as f:
f.write(html_doc)
print(f"\n批量汇总报告已生成: {BATCH_HTML_PATH}")
return 0
if __name__ == "__main__":
sys.exit(main(sys.argv[1:]))
+33 -24
View File
@@ -170,31 +170,40 @@ def analyze(df: pd.DataFrame, tp_grid: Optional[List[float]] = None,
mae = df["mae"].to_numpy()
mfe = df["mfe"].to_numpy()
n = len(df)
n_sim = len(sl_grid) * len(tp_grid)
best = None
grid = []
for sl in sl_grid:
for tp in tp_grid:
# 重算每笔点数结果
# 优先 SL 触发(更保守:先假设逆向先到)
result_pts = np.empty(n)
for k in range(n):
if mae[k] >= sl and sl > 0:
result_pts[k] = -sl
elif mfe[k] >= tp and tp > 0:
result_pts[k] = tp
else:
# 都没触发,按原方向小盈小亏(用 profit 符号 × mfe 比例近似)
result_pts[k] = (mfe[k] - mae[k]) * 0.5
net = float(result_pts.sum())
wins = int((result_pts > 0).sum())
pf = float(result_pts[result_pts > 0].sum() / -result_pts[result_pts <= 0].sum()) if (result_pts <= 0).any() else np.inf
grid.append({"tp": float(tp), "sl": float(sl), "net": net,
"win_rate": wins / n * 100, "pf": pf})
if best is None or net > best["net"]:
best = {"tp": float(tp), "sl": float(sl), "net": net,
"win_rate": wins / n * 100, "pf": pf}
grid_df = pd.DataFrame(grid)
# 向量化:构造 (n_sim, n) 矩阵,避免 Python 嵌套循环
mae_arr = np.broadcast_to(mae, (len(sl_grid), len(tp_grid), n)).reshape(-1, n)
mfe_arr = np.broadcast_to(mfe, (len(sl_grid), len(tp_grid), n)).reshape(-1, n)
sl_arr = np.tile(np.array(sl_grid)[:, np.newaxis], (1, n)).reshape(-1, n)
tp_arr = np.tile(np.array(tp_grid)[np.newaxis, :], (len(sl_grid), n)).reshape(-1, n)
# 优先 SL 触发(更保守)
result_pts = np.where(mae_arr >= sl_arr, -sl_arr,
np.where(mfe_arr >= tp_arr, tp_arr,
(mfe_arr - mae_arr) * 0.5))
net_arr = result_pts.sum(axis=1)
wins_arr = (result_pts > 0).sum(axis=1)
losses_arr = -(result_pts[result_pts <= 0]).sum(axis=1)
pf_arr = np.where(losses_arr > 0, result_pts[result_pts > 0].sum(axis=1) / losses_arr, np.inf)
wr_arr = wins_arr / n * 100
# 找最佳
best_i = int(np.nanargmax(net_arr))
grid_df = pd.DataFrame({
"tp": np.repeat(np.array(tp_grid), len(sl_grid)),
"sl": np.tile(np.array(sl_grid), len(tp_grid)),
"net": net_arr,
"win_rate": wr_arr,
"pf": pf_arr,
})
best = {"tp": float(tp_grid[best_i % len(tp_grid)]),
"sl": float(sl_grid[best_i // len(tp_grid)]),
"net": float(net_arr[best_i]),
"win_rate": float(wr_arr[best_i]),
"pf": float(pf_arr[best_i])}
# MAE/MFE 分布统计
mae_threshold = float(df.loc[df["profit"] <= 0, "mae"].quantile(0.75)) if (df["profit"] <= 0).any() else 0
+511
View File
@@ -0,0 +1,511 @@
# -*- coding: utf-8 -*-
"""
AI Agent MT5 报告分析 API
=========================
专为 AI Agent 设计的结构化接口。解析 MT5 报告,输出 JSON 可序列化结果,
提供设参优化建议。
用法:
from mt5_agent import parse_report, analyze_report, compare_reports, get_param_suggestions
# 解析单份报告
report = parse_report("path/to/report.htm")
print(report.summary) # 汇总指标
print(report.metrics) # 扩展指标(PF、胜率、回撤等)
print(report.trades) # 逐笔交易列表
# 单份报告分析
analysis = analyze_report(report)
print(analysis.suggestions) # 优化建议列表
# 对比两份报告
comparison = compare_reports(report_a, report_b)
print(comparison.diffs) # 关键指标差异
# 批量分析
from mt5_agent import batch_analyze
result = batch_analyze(["dir1/", "dir2/"], "*.htm")
print(result.rankings) # 按 PF 排名
print(result.anomalies) # 异常报告
# 设参优化建议
suggestions = get_param_suggestions(report, analysis)
print(suggestions) # 可写入 .set 的参数修改建议
"""
from __future__ import annotations
import json
import os
from typing import Any, Dict, List, Optional, Tuple
import numpy as np
import pandas as pd
# 导入内部模块
import mt5_report_parser as mp
from mt5_report_parser import parse_report as _parse_report, MT5Report
import run_analysis as ra
import walk_forward as wf
from param_scan import detect_plateaus, build_pivot, load_scan_results, overfit_score
import mae_mfe as mm
# ============================================================================= #
# 数据模型(JSON 可序列化)
# ============================================================================= #
class ReportData:
"""解析后的 MT5 报告数据(AI 友好结构)。"""
def __init__(self, rep: MT5Report):
self.source_file = rep.source_file
self.meta = _to_dict(rep.meta)
self.summary = _to_dict(rep.summary_norm) # 数值化后的汇总
self._trades_df = rep.trades # 保持 DataFrame 原类型
self.trades = _trades_to_dict(rep.trades) if rep.trades is not None else []
self.metrics = _compute_metrics(rep.trades) if rep.trades is not None else {}
# What-If 需要 open_time 列,部分报告可能缺失
if rep.trades is not None and "open_time" in rep.trades.columns:
self.what_if = _what_if_to_dict(ra.whatif_scenarios(rep))
else:
self.what_if = []
def to_json(self, indent: int = 2) -> str:
"""序列化为 JSON 字符串。"""
return json.dumps({
"source_file": self.source_file,
"meta": self.meta,
"summary": self.summary,
"metrics": self.metrics,
"trades_count": len(self.trades),
"what_if_scenarios": self.what_if,
}, ensure_ascii=False, indent=indent)
class AnalysisResult:
"""单份报告分析结果。"""
def __init__(self, report: ReportData):
self.report = report
self.walk_forward = _wf_analysis(report)
self.anomalies = _detect_anomalies(report)
self.suggestions = _generate_suggestions(report, self.walk_forward, self.anomalies)
def to_json(self, indent: int = 2) -> str:
return json.dumps({
"source_file": self.report.source_file,
"summary": self.report.summary,
"metrics": self.report.metrics,
"walk_forward": self.walk_forward,
"anomalies": self.anomalies,
"suggestions": self.suggestions,
}, ensure_ascii=False, indent=indent)
class ComparisonResult:
"""两份报告对比结果。"""
def __init__(self, report_a: ReportData, report_b: ReportData):
self.report_a = report_a
self.report_b = report_b
self.diffs = _compute_diffs(report_a, report_b)
self.suggestions = _compare_suggestions(report_a, report_b)
def to_json(self, indent: int = 2) -> str:
return json.dumps({
"report_a": {"file": self.report_a.source_file, "summary": self.report_a.summary},
"report_b": {"file": self.report_b.source_file, "summary": self.report_b.summary},
"diffs": self.diffs,
"suggestions": self.suggestions,
}, ensure_ascii=False, indent=indent)
class BatchResult:
"""批量分析结果。"""
def __init__(self, reports: List[ReportData]):
self.reports = reports
self.rankings = _batch_rankings(reports)
self.anomalies = [r for r in reports if r.metrics.get("n_trades", 0) < 30]
self.best = self.rankings[0] if self.rankings else None
def to_json(self, indent: int = 2) -> str:
return json.dumps({
"count": len(self.reports),
"rankings": self.rankings,
"anomalies_count": len(self.anomalies),
"best": self.best,
}, ensure_ascii=False, indent=indent)
# ============================================================================= #
# 核心函数
# ============================================================================= #
def parse_report(path: str) -> ReportData:
"""解析 MT5 报告(支持 .xlsx / .htm / .html)。"""
rep = _parse_report(path)
return ReportData(rep)
def analyze_report(report: ReportData) -> AnalysisResult:
"""对单份报告做深度分析。"""
return AnalysisResult(report)
def compare_reports(report_a: ReportData, report_b: ReportData) -> ComparisonResult:
"""对比两份报告。"""
return ComparisonResult(report_a, report_b)
def batch_analyze(paths: List[str], pattern: str = "*.htm") -> BatchResult:
"""批量分析多个报告。"""
from batch_report import scan_reports, parse_batch, build_summary_df
files = scan_reports(paths, pattern)
raw_reports = parse_batch(files)
report_datas = []
for name, rep in raw_reports:
try:
rd = ReportData(rep)
rd._filename = name
report_datas.append(rd)
except Exception as e:
print("警告: 跳过 %s - %s" % (name, e))
return BatchResult(report_datas)
def get_param_suggestions(analysis: AnalysisResult) -> List[Dict[str, Any]]:
"""
生成 .set 文件参数修改建议。
返回格式:[{"param_name": "InpStopLossPoints", "suggested_value": "150",
"reason": "当前 200,亏损截断场景显示 150 更优"}]
"""
suggestions = []
report = analysis.report
metrics = report.metrics
summary = report.summary
what_if = report.what_if
# 建议 1:止损优化
if metrics.get("avg_loss", 0) != 0:
# 找亏损截断场景
cut_scenarios = [s for s in what_if if "亏损截断" in s.get("name", "")]
if cut_scenarios:
base_pf = summary.get("profit_factor", 0)
for s in cut_scenarios:
if s.get("pf", 0) > base_pf and s.get("dd", 0) < metrics.get("max_dd", 0):
cap = s.get("desc", "")
suggestions.append({
"param_name": "InpStopLossPoints",
"suggested_value": None, # 需要从场景推算
"reason": f"亏损截断场景 PF={s['pf']:.2f} > 基线 PF={base_pf:.2f},回撤从 {metrics.get('max_dd',0):.0f} 降至 {s['dd']:.0f}",
"priority": "high",
})
# 建议 2:止盈优化
tp_scenarios = [s for s in what_if if "盈利单放大" in s.get("name", "")]
if tp_scenarios:
base_pf = summary.get("profit_factor", 0)
for s in tp_scenarios:
if s.get("pf", 0) > base_pf:
suggestions.append({
"param_name": "InpTakeProfitPoints",
"suggested_value": None,
"reason": f"盈利单放大场景 PF={s['pf']:.2f} > 基线 PF={base_pf:.2f},当前止盈可能太紧",
"priority": "medium",
})
# 建议 3:仓位优化
halve_scenarios = [s for s in what_if if "仓位减半" in s.get("name", "")]
if halve_scenarios:
base_dd = metrics.get("max_dd", 0)
for s in halve_scenarios:
if abs(s.get("dd", 0)) < abs(base_dd * 0.5):
suggestions.append({
"param_name": "InpFixedLots",
"suggested_value": None,
"reason": f"仓位减半后回撤从 {base_dd:.0f} 降至 {s['dd']:.0f},可考虑降低仓位",
"priority": "medium",
})
# 建议 4:方向性建议
if metrics.get("by_direction"):
for direction, stats in metrics["by_direction"].items():
if stats.get("win_rate", 0) < 35 and stats.get("profit_factor", 0) < 0.8:
suggestions.append({
"param_name": None,
"suggested_value": None,
"reason": f"方向 {direction} 胜率仅 {stats['win_rate']:.1f}%PF={stats['profit_factor']:.2f},考虑单独过滤该方向信号",
"priority": "high",
})
# 建议 5:时段过滤
if metrics.get("by_hour"):
bad_hours = [h for h, stats in metrics["by_hour"].items()
if stats.get("sum", 0) < 0 and stats.get("count", 0) > 5]
if bad_hours:
suggestions.append({
"param_name": "InpFreezeBarCount",
"suggested_value": None,
"reason": f"时段 {bad_hours[:3]} 净盈利持续为负,建议添加时段过滤",
"priority": "low",
})
return suggestions
# ============================================================================= #
# 内部工具函数
# ============================================================================= #
def _to_dict(obj: Any) -> Any:
"""递归将 pandas/numpy 对象转为原生 Python 类型。"""
if obj is None:
return None
if isinstance(obj, dict):
return {k: _to_dict(v) for k, v in obj.items()}
if isinstance(obj, (list, tuple)):
return [_to_dict(v) for v in obj]
if isinstance(obj, (np.integer,)):
return int(obj)
if isinstance(obj, (np.floating,)):
if np.isnan(obj) or np.isinf(obj):
return None
return float(obj)
if isinstance(obj, pd.Timestamp):
return str(obj)
if isinstance(obj, pd.DataFrame):
return _to_dict(obj.to_dict(orient="index"))
if isinstance(obj, pd.Series):
return _to_dict(obj.to_dict())
if isinstance(obj, pd.Index):
return _to_dict(list(obj))
if isinstance(obj, (float, int)) and (np.isnan(obj) or np.isinf(obj)):
return None
if isinstance(obj, (int, float, str, bool)):
return obj
return str(obj)
def _trades_to_dict(trades: pd.DataFrame) -> List[Dict[str, Any]]:
"""将逐笔交易 DataFrame 转为字典列表。"""
if trades is None or trades.empty:
return []
result = []
for _, row in trades.iterrows():
item = {}
for col in ["open_time", "close_time", "direction", "volume",
"open_price", "close_price", "profit", "swap",
"commission", "net_profit", "duration_min"]:
val = row.get(col)
if val is None or (isinstance(val, float) and np.isnan(val)):
item[col] = None
elif isinstance(val, pd.Timestamp):
item[col] = str(val)
elif isinstance(val, (np.integer,)):
item[col] = int(val)
elif isinstance(val, (np.floating,)):
item[col] = float(val)
else:
item[col] = val
result.append(item)
return result
def _compute_metrics(trades: pd.DataFrame) -> Dict[str, Any]:
"""计算扩展指标(JSON 可序列化)。"""
m = ra.extended_metrics(trades) if trades is not None and not trades.empty else {}
result = _to_dict(m)
# 移除内部字段
for key in ["_equity", "_dd", "_net"]:
result.pop(key, None)
return result
def _what_if_to_dict(scenarios: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
"""将 What-If 场景转为字典列表。"""
return [_to_dict(s) for s in scenarios]
def _wf_analysis(report: ReportData) -> Dict[str, Any]:
"""Walk-Forward 分析。"""
trades_df = getattr(report, "_trades_df", None)
if trades_df is None or trades_df.empty:
return {}
# 确保 open_time 全为 Timestamp 类型
trades_df = trades_df.copy()
trades_df["open_time"] = pd.to_datetime(trades_df["open_time"], errors="coerce")
trades_df = trades_df.dropna(subset=["open_time"])
trades_df["net_profit"] = pd.to_numeric(trades_df["net_profit"], errors="coerce")
trades_df = trades_df.dropna(subset=["net_profit"])
if trades_df.empty:
return {}
windows = wf.walk_forward(trades_df)
summary = wf.wf_summary(windows)
return _to_dict(summary)
def _detect_anomalies(report: ReportData) -> List[Dict[str, Any]]:
"""检测异常指标。"""
anomalies = []
metrics = report.metrics
summary = report.summary
if metrics.get("n_trades", 0) < 30:
anomalies.append({
"type": "样本不足",
"detail": f"{metrics['n_trades']} 笔交易",
})
if summary.get("profit_factor", 0) > 10:
anomalies.append({
"type": "PF异常高",
"detail": f"PF={summary['profit_factor']:.2f},可能过拟合",
})
if metrics.get("max_dd", 0) < -1000:
anomalies.append({
"type": "回撤过大",
"detail": f"最大回撤 {metrics['max_dd']:.0f}",
})
return anomalies
def _generate_suggestions(
report: ReportData,
wf_result: Dict[str, Any],
anomalies: List[Dict[str, Any]],
) -> List[Dict[str, Any]]:
"""生成优化建议。"""
suggestions = []
metrics = report.metrics
summary = report.summary
# Walk-Forward 建议
if wf_result:
wfe = wf_result.get("wfe", 0)
if wfe is not None and wfe < 0.3:
suggestions.append({
"type": "Walk-Forward",
"detail": f"WFE={wfe:.2f},OOS 表现弱,参数可能过拟合 IS 段",
"priority": "high",
})
# 什么-If 建议
for s in report.what_if:
name = s.get("name", "")
if "信号反向" in name and s.get("net", 0) > 0 and s.get("pf", 0) > 1:
suggestions.append({
"type": "信号方向",
"detail": f"反向场景净盈利={s['net']:+.2f}, PF={s['pf']:.2f},可能方向逻辑写反",
"priority": "critical",
})
return suggestions
def _compute_diffs(report_a: ReportData, report_b: ReportData) -> Dict[str, Any]:
"""计算两份报告的关键指标差异。"""
diffs = {}
for key in ["profit_factor", "win_rate", "net_profit", "max_dd", "sharpe", "sortino"]:
va = report_a.summary.get(key)
vb = report_b.summary.get(key)
if va is not None and vb is not None:
diff = vb - va if (not np.isinf(va) or not np.isinf(vb)) else None
pct = ((vb - va) / va * 100) if va != 0 and not np.isinf(va) else None
diffs[key] = {
"a": va,
"b": vb,
"diff": diff,
"pct_change": pct,
}
return diffs
def _compare_suggestions(report_a: ReportData, report_b: ReportData) -> List[Dict[str, Any]]:
"""对比分析建议。"""
suggestions = []
pf_a = report_a.summary.get("profit_factor", 0)
pf_b = report_b.summary.get("profit_factor", 0)
if pf_b > pf_a and pf_a > 0:
suggestions.append({
"type": "参数优化",
"detail": f"报告B PF={pf_b:.2f} 优于报告A PF={pf_a:.2f},差 {pf_b-pf_a:.2f}",
"priority": "medium",
})
elif pf_b < pf_a and pf_b > 0:
suggestions.append({
"type": "参数回退",
"detail": f"报告B PF={pf_b:.2f} 差于报告A PF={pf_a:.2f},参数可能过拟合",
"priority": "medium",
})
return suggestions
def _batch_rankings(reports: List[ReportData]) -> List[Dict[str, Any]]:
"""按 PF 排名。"""
valid = [r for r in reports if r.summary.get("profit_factor") is not None
and r.metrics.get("n_trades", 0) > 0]
sorted_reports = sorted(valid, key=lambda r: r.summary.get("profit_factor", 0), reverse=True)
rankings = []
for i, r in enumerate(sorted_reports[:20]): # 只取前 20
rankings.append({
"rank": i + 1,
"file": r.source_file,
"filename": getattr(r, "_filename", os.path.basename(r.source_file)),
"profit_factor": r.summary.get("profit_factor"),
"net_profit": r.summary.get("total_net_profit"),
"win_rate": r.metrics.get("win_rate"),
"n_trades": r.metrics.get("n_trades"),
"max_dd": r.metrics.get("max_dd"),
})
return rankings
# ============================================================================= #
# CLI
# ============================================================================= #
if __name__ == "__main__":
import argparse
ap = argparse.ArgumentParser(description="AI Agent MT5 报告分析 API")
ap.add_argument("path", help="报告文件路径或目录")
ap.add_argument("--compare", help="对比的另一份报告路径")
ap.add_argument("--batch", action="store_true", help="批量分析目录")
ap.add_argument("--suggestions", action="store_true", help="输出设参建议")
ap.add_argument("--format", choices=["json", "text"], default="json", help="输出格式")
args = ap.parse_args()
if args.batch:
result = batch_analyze([args.path])
print(result.to_json())
elif args.compare:
a = parse_report(args.path)
b = parse_report(args.compare)
comparison = compare_reports(a, b)
if args.format == "json":
print(comparison.to_json())
else:
print(f"\n报告A: {args.path}")
print(f" PF={a.summary.get('profit_factor')} 净盈利={a.summary.get('total_net_profit')}")
print(f"报告B: {args.compare}")
print(f" PF={b.summary.get('profit_factor')} 净盈利={b.summary.get('total_net_profit')}")
print(f"\n关键差异:")
for key, diff in comparison.diffs.items():
print(f" {key}: A={diff['a']} B={diff['b']} 差={diff['diff']:.2f}")
else:
report = parse_report(args.path)
analysis = analyze_report(report)
if args.suggestions:
suggestions = get_param_suggestions(analysis)
print(json.dumps(suggestions, ensure_ascii=False, indent=2))
elif args.format == "json":
print(analysis.to_json())
else:
print(f"\n报告: {args.path}")
print(f" PF={report.summary.get('profit_factor')} 净盈利={report.summary.get('total_net_profit')}")
print(f" 胜率={report.metrics.get('win_rate')} 笔数={report.metrics.get('n_trades')}")
print(f" 回撤={report.metrics.get('max_dd')}")
print(f"\n建议 ({len(analysis.suggestions)} 条):")
for s in analysis.suggestions:
print(f" [{s.get('priority','?')}] {s['type']}: {s['detail']}")
+372 -7
View File
@@ -14,19 +14,108 @@ MT5 策略测试报告解析器(中文版 xlsx)
from __future__ import annotations
import re
import os as _os_std
from dataclasses import dataclass, field
from typing import Any, Dict, List, Optional, Tuple
from html.parser import HTMLParser
import numpy as np
import openpyxl
import pandas as pd
# 报告中区段标题A 列出现的精确字符串)
# 报告中区段标题
SEC_SETTINGS = "设置"
SEC_RESULTS = "结果"
SEC_ORDERS = "订单"
SEC_DEALS = "成交"
# 支持的报告格式
_Supported_Extensions = {".xlsx", ".htm", ".html"}
class _MT5HTMLParser(HTMLParser):
"""MT5 HTML 报告解析器。"""
def __init__(self):
super().__init__()
self.in_td = False
self.in_th = False
self.in_b = False
self.current_row: List[Optional[str]] = []
self.current_cell: str = ""
self.b_content: str = "" # Content inside <b> tags
self.rows: List[List[Optional[str]]] = []
self.section_rows: Dict[str, int] = {} # Track section headers
def handle_starttag(self, tag, attrs):
if tag == "tr":
if self.current_row:
self.rows.append(self.current_row)
self.current_row = []
elif tag == "td":
self.current_cell = ""
self.b_content = ""
self.in_td = True
self.in_b = False
elif tag == "th":
self.current_cell = ""
self.b_content = ""
self.in_th = True
self.in_b = False
elif tag == "b":
self.in_b = True
self.b_content = ""
def handle_endtag(self, tag):
if tag == "tr":
if self.current_row:
self.rows.append(self.current_row)
self.current_row = []
self.current_cell = ""
self.in_td = False
self.in_th = False
elif tag == "td":
# Build full cell: non-b text + <b> content
full_cell = self.current_cell + self.b_content
self.current_row.append(full_cell.strip() if full_cell.strip() else "")
self.current_cell = ""
self.b_content = ""
self.in_td = False
elif tag == "th":
full_cell = self.current_cell + self.b_content
# Check if this is a section header
b_val = self.b_content.strip()
if b_val in (SEC_SETTINGS, SEC_RESULTS, SEC_ORDERS, SEC_DEALS):
row_idx = len(self.rows)
self.section_rows[b_val] = row_idx
self.current_row.append(full_cell.strip() if full_cell.strip() else "")
self.current_cell = ""
self.b_content = ""
self.in_th = False
elif tag == "b":
self.in_b = False
def handle_data(self, data):
if self.in_td or self.in_th:
if not self.in_b:
self.current_cell += data
else:
self.b_content += data
def finish(self):
if self.current_row:
self.rows.append(self.current_row)
# Also search all cells for section markers (they may be in <td> not just <th>)
for i, row in enumerate(self.rows):
if row:
for j, cell in enumerate(row):
if cell:
b_val = _get_b_text(cell)
if b_val and b_val.strip() in (SEC_SETTINGS, SEC_RESULTS, SEC_ORDERS, SEC_DEALS):
sec = b_val.strip()
if sec not in self.section_rows:
self.section_rows[sec] = i
return self.rows, self.section_rows
@dataclass
class MT5Report:
@@ -121,10 +210,253 @@ def _pct_to_float(x: Any) -> Optional[float]:
# --------------------------------------------------------------------------- #
# 主入口
# HTML 报告解析
# --------------------------------------------------------------------------- #
def parse_report(path: str) -> MT5Report:
"""解析一份 MT5 xlsx 报告。"""
def _parse_html_report(text: str) -> MT5Report:
"""解析 MT5 HTML 报告。"""
parser = _MT5HTMLParser()
parser.feed(text)
rows, section_rows = parser.finish()
r_settings = section_rows.get(SEC_SETTINGS, 0)
r_results = section_rows.get(SEC_RESULTS, r_settings)
r_orders = section_rows.get(SEC_ORDERS, r_results)
r_deals = section_rows.get(SEC_DEALS, r_orders)
rep = MT5Report()
rep.meta, rep.summary = _parse_html_meta_and_summary(rows, r_settings, r_results, r_orders)
rep.orders, rep.deals = _parse_html_tables(rows, r_orders, r_deals, len(rows))
rep.trades = _reconstruct_trades(rep.deals)
_normalize_summary_numbers(rep)
return rep
def _read_html_file(path: str) -> str:
"""读取 HTML 文件,自动检测编码。"""
with open(path, "rb") as f:
raw = f.read()
# 检测 BOM
if raw[:2] == b"\xff\xfe":
return raw.decode("utf-16-le")
elif raw[:2] == b"\xfe\xff":
return raw.decode("utf-16-be")
elif raw[:3] == b"\xef\xbb\xbf":
return raw[3:].decode("utf-8")
elif raw[:4] == b"\xff\xfe\x00\x00":
return raw[4:].decode("utf-32-le")
# 尝试 UTF-8
try:
return raw.decode("utf-8")
except UnicodeDecodeError:
pass
# 回退到 latin-1
return raw.decode("latin-1")
def _get_b_text(cell_text: str) -> Optional[str]:
"""从 HTML 单元格文本中提取 <b>...</b> 内的文本。"""
if cell_text is None or not cell_text.strip():
return None
# 直接提取 <b>...</b> 内容
m = re.search(r'<b>(.*?)</b>', cell_text, re.DOTALL)
if m:
text = m.group(1).strip()
# 去除内嵌的 <br> 标签
text = re.sub(r'<br\s*/?>', '', text, flags=re.IGNORECASE)
return text if text else None
# 如果没有 <b> 标签,返回去除 HTML 标签的纯文本
clean = re.sub(r'<[^>]+>', '', cell_text).strip()
return clean if clean else None
def _parse_html_meta_and_summary(
rows: List[List[Optional[str]]],
r_settings: int, r_results: int, r_orders: int,
) -> Tuple[Dict[str, Any], Dict[str, Any]]:
"""解析 HTML 报告中的设置和结果部分。"""
meta: Dict[str, Any] = {}
summary: Dict[str, Any] = {}
# 解析设置段
for i in range(r_settings, r_results):
if i >= len(rows):
break
row = rows[i]
if not row or len(row) < 2:
continue
label = _get_b_text(row[0] or "")
if label and label.endswith(":"):
label = label[:-1].strip()
# 值在第二个非空单元格
for j in range(1, len(row)):
val = _get_b_text(row[j] or "")
if val is not None:
if label == "输入":
# 输入参数
continue
meta[label] = val
break
# 解析输入参数(在设置段中查找 Key=Value 格式)
inputs: Dict[str, str] = {}
for i in range(r_settings, r_results):
if i >= len(rows):
break
for j in range(len(rows[i])):
cell = rows[i][j]
if cell:
b_val = _get_b_text(cell)
if b_val and "=" in b_val and not b_val.strip().startswith("==="):
k, _, val = b_val.partition("=")
inputs[k.strip()] = val.strip()
meta["inputs"] = inputs
# 解析结果段
for i in range(r_results, r_orders):
if i >= len(rows):
break
row = rows[i]
if not row or len(row) < 2:
continue
label = _get_b_text(row[0] or "")
if label and label.endswith(":"):
label = label[:-1].strip()
for j in range(1, len(row)):
val = _get_b_text(row[j] or "")
if val is not None:
summary[label] = val
break
return meta, summary
def _parse_html_tables(
rows: List[List[Optional[str]]],
r_orders: int, r_deals: int, max_row: int,
) -> Tuple[Optional[pd.DataFrame], Optional[pd.DataFrame]]:
"""解析 HTML 报告中的订单表和成交表。"""
orders_df: Optional[pd.DataFrame] = None
deals_df: Optional[pd.DataFrame] = None
# HTML 报告列名到标准英文列名的映射
DEALS_COL_MAP = {
"时间": "time",
"成交": "deal_id",
"交易品种": "symbol",
"类型": "type",
"趋势": "entry",
"交易量": "volume",
"价位": "price",
"订单": "order",
"手续费": "commission",
"库存费": "swap",
"盈利": "profit",
"结余": "balance",
"注释": "comment",
}
ORDERS_COL_MAP = {
"时间": "time",
"交易": "ticket",
"交易品种": "symbol",
"类型": "type",
"状态": "state",
"交易量": "volume",
"价位": "price",
"止损": "sl",
"止盈": "tp",
"注释": "comment",
"订单": "order",
}
def _parse_table_section(start_row: int, end_row: int, col_map: Dict[str, str]) -> Optional[pd.DataFrame]:
"""Parse a table section from rows."""
if start_row >= max_row:
return None
# Find header row
header_row = None
for i in range(start_row + 1, min(start_row + 5, end_row)):
if i >= len(rows):
break
row = rows[i]
if row:
b_count = sum(1 for cell in row if cell and _get_b_text(cell))
if b_count >= 3:
header_row = i
break
if header_row is None:
return None
# Extract headers
headers = [_get_b_text(cell) for cell in rows[header_row] if cell and _get_b_text(cell)]
if not headers:
return None
# Rename columns using col_map
renamed_headers = [col_map.get(h, h) for h in headers]
# Extract data rows
data_rows = []
for i in range(header_row + 1, end_row):
if i >= len(rows):
break
row = rows[i]
if row:
values = [_get_b_text(cell) or "" for cell in row]
if any(v for v in values):
while len(values) < len(renamed_headers):
values.append("")
data_rows.append(values[:len(renamed_headers)])
if not data_rows:
return None
df = pd.DataFrame(data_rows, columns=renamed_headers)
return df
if r_orders < max_row:
orders_df = _parse_table_section(r_orders, r_deals, ORDERS_COL_MAP)
if r_deals < max_row:
deals_df = _parse_table_section(r_deals, max_row, DEALS_COL_MAP)
return orders_df, deals_df
# --------------------------------------------------------------------------- #
# 解析缓存(文件级 LRU 缓存)
# --------------------------------------------------------------------------- #
_parse_cache: Dict[str, MT5Report] = {}
def parse_report(path: str, use_cache: bool = True) -> MT5Report:
"""解析一份 MT5 报告(支持 xlsx / htm / html 格式)。"""
real_path = _os_std.path.realpath(str(path))
if use_cache and real_path in _parse_cache:
return _parse_cache[real_path]
ext = _os_std.path.splitext(real_path)[1].lower()
if ext == ".xlsx":
rep = _parse_xlsx_report(real_path)
elif ext in (".htm", ".html"):
text = _read_html_file(real_path)
rep = _parse_html_report(text)
rep.source_file = real_path
else:
raise ValueError(f"不支持的报告格式: {ext}(仅支持 .xlsx / .htm / .html")
_parse_cache[real_path] = rep
return rep
def _parse_xlsx_report(path: str) -> MT5Report:
"""解析 xlsx 格式报告(原 parse_report 逻辑)。"""
wb = openpyxl.load_workbook(path, data_only=True)
ws = wb[wb.sheetnames[0]]
rep = MT5Report(source_file=path)
@@ -136,16 +468,15 @@ def parse_report(path: str) -> MT5Report:
r_deals = sec.get(SEC_DEALS, r_orders)
# ---- 元信息 / 设置 ----
# 整个设置块(设置标题行之后到结果段之前)逐行解析 “标签: 值” 对
meta: Dict[str, Any] = {}
for r in range(r_settings + 1, r_results):
kv = _kv_in_row(ws, r)
for k, v in kv.items():
if k == "输入": # 输入: 是子段标题,其值为分节字符串,跳过
if k == "输入": # "输入:" 是子段标题,其值为分节字符串,跳过
continue
meta[k] = v
# 输入参数:在 输入: 段落里,列 D 形如 Key=Value
# 输入参数:在 "输入:" 段落里,列 D 形如 Key=Value
inputs: Dict[str, str] = {}
for r in range(r_settings + 1, r_results):
for c in (4,): # 经验上输入参数在 D 列
@@ -173,9 +504,15 @@ def parse_report(path: str) -> MT5Report:
rep.summary = summary
rep.trades = _reconstruct_trades(rep.deals)
_normalize_summary_numbers(rep)
return rep
def clear_parse_cache() -> None:
"""清除解析缓存(调试 / 释放内存用)。"""
_parse_cache.clear()
# --------------------------------------------------------------------------- #
# 逐笔交易重建
# --------------------------------------------------------------------------- #
@@ -256,6 +593,34 @@ def _reconstruct_trades(deals: pd.DataFrame) -> pd.DataFrame:
return pd.DataFrame(trades)
# --------------------------------------------------------------------------- #
# 单段通用指标(供 walk_forward / run_analysis 复用)
# --------------------------------------------------------------------------- #
def compute_segment_metrics(net: pd.Series) -> Dict[str, float]:
"""
由净盈亏序列计算核心指标。
无状态、纯函数,可直接用于单段或单份报告。
"""
n = len(net)
if n == 0:
return {"n": 0, "net": 0.0, "pf": 0.0, "win": 0.0, "dd": 0.0, "exp": 0.0}
wins = net[net > 0]
losses = net[net <= 0]
gp = wins.sum()
gl = -losses.sum()
pf = gp / gl if gl > 0 else np.inf
equity = net.cumsum()
dd = float((equity - equity.cummax()).min())
return {
"n": int(n),
"net": float(net.sum()),
"pf": float(pf),
"win": float(len(wins) / n * 100),
"dd": dd,
"exp": float(net.mean()),
}
# --------------------------------------------------------------------------- #
# 汇总指标数值化(便于程序对比)
# --------------------------------------------------------------------------- #
File diff suppressed because one or more lines are too long
+10 -25
View File
@@ -189,28 +189,9 @@ def extended_metrics(trades: pd.DataFrame) -> Dict[str, Any]:
# =========================================================================== #
# What-If 假设分析(通用场景
# 单段通用指标(从 mt5_report_parser 复用
# =========================================================================== #
def _stats_from_net(net: pd.Series) -> Dict[str, float]:
"""由净盈亏序列算关键统计。"""
n = len(net)
if n == 0:
return {"n": 0, "net": 0, "pf": 0, "win": 0, "dd": 0, "exp": 0}
wins = net[net > 0]
losses = net[net <= 0]
gp = wins.sum()
gl = -losses.sum()
pf = gp / gl if gl > 0 else np.inf
equity = net.cumsum()
dd = (equity - equity.cummax()).min()
return {
"n": int(n),
"net": float(net.sum()),
"pf": float(pf),
"win": float(len(wins) / n * 100),
"dd": float(dd),
"exp": float(net.mean()),
}
from mt5_report_parser import compute_segment_metrics as _stats_from_net
def whatif_scenarios(rep: mp.MT5Report) -> List[Dict[str, Any]]:
@@ -324,12 +305,16 @@ def monte_carlo_dd(net: pd.Series, n_sim: int = 1000, seed: int = 42) -> Dict[st
n = len(arr)
if n == 0:
return {"p5": 0, "p50": 0, "p95": 0, "actual": 0, "mean": 0}
# 向量化:一次性生成 (n_sim, n) 置换矩阵,在 C 层完成
# 每行是一个随机打乱的交易顺序
perms = rng.integers(n, size=(n_sim, n))
# 每行按该行的索引排序得到 (n_sim, n) 的排列索引
idx = np.argsort(perms, axis=1)
dds = np.empty(n_sim)
for i in range(n_sim):
perm = rng.permutation(n)
eq = np.cumsum(arr[perm])
dd = (eq - np.maximum.accumulate(eq)).min()
dds[i] = dd
perm = arr[idx[i]]
eq = np.cumsum(perm)
dds[i] = (eq - np.maximum.accumulate(eq)).min()
return {
"p5": float(np.percentile(dds, 5)),
"p50": float(np.percentile(dds, 50)),
+64 -45
View File
@@ -22,7 +22,6 @@ Walk-Forward 滚动验证
from __future__ import annotations
import argparse
import glob
import html
import os
import sys
@@ -38,27 +37,10 @@ OUT_DIR = "output"
# --------------------------------------------------------------------------- #
# 单段指标
# 单段指标(从 mt5_report_parser 复用)
# --------------------------------------------------------------------------- #
def _seg_metrics(net: pd.Series) -> Dict[str, float]:
n = len(net)
if n == 0:
return {"n": 0, "net": 0.0, "pf": 0.0, "win": 0.0, "dd": 0.0, "exp": 0.0}
wins = net[net > 0]
losses = net[net <= 0]
gp = wins.sum()
gl = -losses.sum()
pf = gp / gl if gl > 0 else np.inf
eq = net.cumsum()
dd = float((eq - eq.cummax()).min())
return {
"n": int(n),
"net": float(net.sum()),
"pf": float(pf),
"win": float(len(wins) / n * 100),
"dd": dd,
"exp": float(net.mean()),
}
from mt5_report_parser import compute_segment_metrics as _seg_metrics
@dataclass
@@ -79,7 +61,7 @@ def walk_forward(
step_days: Optional[int] = None,
) -> List[WFWindow]:
"""
滚动切窗。
滚动切窗(向量化边界查找)
trades: 含 open_time, net_profit 的 DataFrame
is_days / oos_days: IS/OOS 窗口天数
step_days: 滑动步长(默认 = oos_days,即不重叠的 OOS
@@ -87,7 +69,12 @@ def walk_forward(
if trades is None or trades.empty:
return []
t = trades.sort_values("open_time").reset_index(drop=True)
t["net_profit"] = t["net_profit"].astype(float)
times = t["open_time"].to_numpy()
nets = t["net_profit"].astype(float).to_numpy()
n_trades = len(nets)
# 预计算累积和(含前导0cumsum[i] = sum of nets[:i]
cumsum = np.concatenate([[0.0], np.cumsum(nets)])
if step_days is None:
step_days = oos_days
@@ -97,34 +84,66 @@ def walk_forward(
total_span = (end - start).days
win_span = is_days + oos_days
if total_span < win_span:
return [] # 数据不足以切出一个窗口
return []
windows: List[WFWindow] = []
cur = start
# 用 numpy 二分查找窗口边界索引(所有窗口一次性确定)
cur = np.datetime64(start, "ns")
end_ns = np.datetime64(end, "ns")
bounds: List[Tuple[int, int, int, int]] = [] # (is_s, is_e, oos_s, oos_e) 为交易索引
idx = 0
while True:
is_s = cur
is_e = cur + pd.Timedelta(days=is_days)
is_e = cur + np.timedelta64(is_days, "D")
oos_s = is_e
oos_e = oos_s + pd.Timedelta(days=oos_days)
if oos_e > end + pd.Timedelta(days=1):
oos_e = oos_s + np.timedelta64(oos_days, "D")
if oos_e > end_ns + np.timedelta64(1, "D"):
break
is_mask = (t["open_time"] >= is_s) & (t["open_time"] < is_e)
oos_mask = (t["open_time"] >= oos_s) & (t["open_time"] < oos_e)
is_net = t.loc[is_mask, "net_profit"]
oos_net = t.loc[oos_mask, "net_profit"]
if len(is_net) == 0 or len(oos_net) == 0:
cur += pd.Timedelta(days=step_days)
continue
windows.append(WFWindow(
idx=idx,
is_start=is_s, is_end=is_e,
oos_start=oos_s, oos_end=oos_e,
is_metrics=_seg_metrics(is_net),
oos_metrics=_seg_metrics(oos_net),
))
i_s = int(np.searchsorted(times, cur, side="left"))
i_e = int(np.searchsorted(times, is_e, side="left"))
o_s = int(np.searchsorted(times, oos_s, side="left"))
o_e = int(np.searchsorted(times, oos_e, side="left"))
bounds.append((i_s, i_e, o_s, o_e))
cur += np.timedelta64(step_days, "D")
idx += 1
cur += pd.Timedelta(days=step_days)
windows: List[WFWindow] = []
for i, (i_s, i_e, o_s, o_e) in enumerate(bounds):
if i_e - i_s == 0 or o_e - o_s == 0:
continue
# IS 段指标(用累积和 O(1) 计算)
is_net_sum = float(cumsum[i_e] - cumsum[i_s])
is_n = i_e - i_s
is_seg = nets[i_s:i_e]
is_gp = float(np.sum(is_seg[is_seg > 0]))
is_gl = float(-np.sum(is_seg[is_seg <= 0]))
is_pf = is_gp / is_gl if is_gl > 0 else np.inf
is_cum = cumsum[i_s + 1:i_e + 1] - cumsum[i_s:i_e]
is_dd = float(np.min(is_cum - np.maximum.accumulate(is_cum))) if len(is_cum) else 0.0
is_win = float(np.sum(is_seg > 0) / is_n * 100) if is_n else 0.0
# OOS 段
oos_net_sum = float(cumsum[o_e] - cumsum[o_s])
oos_n = o_e - o_s
oos_seg = nets[o_s:o_e]
oos_gp = float(np.sum(oos_seg[oos_seg > 0]))
oos_gl = float(-np.sum(oos_seg[oos_seg <= 0]))
oos_pf = oos_gp / oos_gl if oos_gl > 0 else np.inf
oos_cum = cumsum[o_s + 1:o_e + 1] - cumsum[o_s:o_e]
oos_dd = float(np.min(oos_cum - np.maximum.accumulate(oos_cum))) if len(oos_cum) else 0.0
oos_win = float(np.sum(oos_seg > 0) / oos_n * 100) if oos_n else 0.0
windows.append(WFWindow(
idx=i,
is_start=times[i_s], is_end=times[min(i_e - 1, n_trades - 1)],
oos_start=times[o_s], oos_end=times[min(o_e - 1, n_trades - 1)],
is_metrics={
"n": is_n, "net": is_net_sum, "pf": is_pf,
"win": is_win, "dd": is_dd, "exp": is_net_sum / is_n if is_n else 0.0,
},
oos_metrics={
"n": oos_n, "net": oos_net_sum, "pf": oos_pf,
"win": oos_win, "dd": oos_dd, "exp": oos_net_sum / oos_n if oos_n else 0.0,
},
))
return windows

Some files were not shown because too many files have changed in this diff Show More