From 51f9a6b98d314f71693e4ae40ed50df8e5678a67 Mon Sep 17 00:00:00 2001 From: gavindiaz Date: Sat, 27 Jun 2026 01:05:34 +0800 Subject: [PATCH] Add requirements.txt and rewrite README for handover --- COMMIT_MSG_2.txt | 10 -- README.md | 460 ++++++++++++++++++++++++++++++++++++++--------- _test_set_gen.py | 44 ----- requirements.txt | 23 +++ run.py | 153 ---------------- 5 files changed, 395 insertions(+), 295 deletions(-) delete mode 100644 COMMIT_MSG_2.txt delete mode 100644 _test_set_gen.py create mode 100644 requirements.txt delete mode 100644 run.py diff --git a/COMMIT_MSG_2.txt b/COMMIT_MSG_2.txt deleted file mode 100644 index 37a0275..0000000 --- a/COMMIT_MSG_2.txt +++ /dev/null @@ -1,10 +0,0 @@ -warmup 修复后重跑 Optuna study + 重生所有产物 - -- 重跑 500 trials Optuna study (warmup 修复版) -- 新 finalist 集合: #1 trial #491 / #2 trial #297 / #3 trial #492 -- finalist #1 IS net $263k (vs 旧 $29k, warmup 修复后搜索空间扩大) -- 重生 finalists JSON / features parquet / dashboard HTML -- 三个独立 registry 条目 (f1_t491 / f2_t297 / f3_t492) -- registry 文件名含 finalist index + trial number 避免覆盖 -- registry 状态: "已记录 (无 MT5 验证)" - 待 Phase 7 MT5 验证 -- 删除旧 registry 条目 (trial #324 已被新 study 取代) diff --git a/README.md b/README.md index 78d4f03..aca7f31 100644 --- a/README.md +++ b/README.md @@ -1,117 +1,401 @@ -# Backtesting + Optuna + MT5-Verification Stack — Knowledge Base +# Backtesting + Optuna + MT5 Stack -A blueprint for building a **personal trading-strategy research lab**: a fast Python -backtesting engine, a Bayesian parameter optimizer (Optuna), and an automated bridge to the -**MetaTrader 5 Strategy Tester** that cross-checks every result against the real terminal. +一个把 **MetaTrader 5 Strategy Tester 当作"金标准"**、用 **Python 镜像引擎**做高速贝叶斯 +搜索的个人量化策略研究实验室。当前已对 **GoldScalperPro**（XAUUSD M5 trailing/BE EA） +跑通完整闭环：**假设 → 搜索 → MT5 验证 → 入注册表**。 -This is a **knowledge base, not a code drop.** It describes the *architecture, algorithms, rules, -and tooling* so you can build your own version from scratch — with **your own** Expert Advisors, -**your own** strategies, and **your own** presets. There are no ready-made engines or strategy -files here on purpose: you supply those from your own MQL5 bots (ideally open-source EAs you own -or have the right to use). +> 接手者请按本文件的 §2 → §3 → §7 顺序读完即可上手。详细的架构 / 坑 / 扩展指南 +> 在 [PROJECT_GUIDE.md](PROJECT_GUIDE.md) 和 `01-08-*.md` 里。 --- -## What this stack does for you - -You have MQL5 Expert Advisors and a pile of ideas to test. The MetaTrader Strategy Tester is -accurate but **slow** — a single multi-year backtest with a real-tick model can take 10–30 minutes, -and an exhaustive optimization can run for days. That kills iteration speed. - -This stack solves it with a **two-tier model**: - -1. **Tier 1 — a fast Python "mirror engine"** that reproduces your EA's trade logic bar-by-bar over - pre-downloaded historical data. It is an *approximation* (more on fidelity below), but it runs a - full multi-year backtest in **seconds**, so you can rank thousands of parameter combinations with - Optuna in the time MT5 would run a handful. - -2. **Tier 2 — MetaTrader 5 as the gold standard.** Only the **finalists** from the Python search are - compiled and run in the real Strategy Tester. The Python number gets you the *shortlist*; the MT5 - number is what you trust for anything that goes live. +## 1. 项目一句话 ``` - Idea ─► Python mirror engine ─► Optuna search (thousands of trials, fast) - │ - ▼ - 2–3 diverse finalists - │ - ▼ - MetaTrader 5 Strategy Tester (gold standard) - │ - ▼ - Python-vs-MT5 comparison table ─► keep / discard / iterate +Idea ─► Python mirror engine ─► Optuna search (hundreds of trials, seconds each) + │ + ▼ + 2–3 diverse finalists + │ + ▼ + MetaTrader 5 Strategy Tester (gold standard) + │ + ▼ + Python-vs-MT5 gap table ─► registry/ (append-only) ``` -The whole point is **disciplined isolation**: the engine is frozen and trusted, instruments are -described by data not code, every hypothesis is an isolated experiment, and only triple-checked -results are promoted. That discipline is what keeps a research lab from rotting into a pile of -one-off scripts that nobody can reproduce. +- **Tier 1（Python 镜像引擎）**：把 EA 的填单 / 出场逻辑 bar-by-bar 重写，跑在 + Parquet 历史数据上，2 年回测几秒钟。 +- **Tier 2（MT5 Strategy Tester）**：只对 finalist 跑真实 tester。**MT5 数字才是 + live 决策依据**；Python 数字只负责排序。 +- **trailing/BE EA 必须 M1 tick-level 模拟**——bar-level 会产生 −40% 到 −50% + 的隐藏 gap（详见 [PROJECT_GUIDE.md §1.3](PROJECT_GUIDE.md)）。 --- -## Who this is for +## 2. 快速开始（接手者从这里读起） -- You run **MetaTrader 5** and write or use **MQL5** Expert Advisors. -- You want to test and optimize strategies **much faster** than the MT5 optimizer allows. -- You are comfortable with **Python** (intermediate) and the command line. -- You can run **MT5 on Windows** — either on the same machine (simplest) or on a separate - Windows box/VPS that your Python machine talks to. +### 2.1 环境要求 -You do **not** need to be a quant. The hard parts (engine fidelity, Bayesian search, robustness -testing) are explained from first principles. +- **Windows**（MetaTrader5 Python 包只在 Windows 上装得了） +- **Python 3.12+**（开发用 3.12.10） +- **MetaTrader 5 终端**已安装并登录一个 demo 账户 +- **Git**（commit 历史就是项目文档的一部分） + +### 2.2 一次性准备 + +```powershell +# 1. clone 后建 venv 装依赖 +python -m venv .venv +.\.venv\Scripts\activate +pip install -r requirements.txt + +# 2. 在根目录建 .env（gitignored，凭证不入库） +# MT5_DEMO_LOGIN=52845377 +# MT5_DEMO_PASSWORD=... +# MT5_DEMO_SERVER=ICMarketsSC-Demo +# MT5_TERMINAL_PATH=C:\Program Files\MetaTrader 5 IC Markets Global\terminal64.exe + +# 3. 打开 MT5 终端、登录 demo、确认 XAUUSD 历史已下载（M5 + M1） +python scripts/download_xauusd_history.py # → data/XAUUSD_M5_*.parquet +python scripts/download_xauusd_m1.py # → data/XAUUSD_M1_*.parquet（trailing/BE EA 必需） + +# 4. 查 symbol 规格（确认 tick_value / contract_size / lot_step） +python scripts/query_xauusd_spec.py +``` + +历史 parquet 在 `.gitignore` 里（`data/` 整个忽略），所以**接手后必须重跑下载脚本**， +否则后续 Optuna / 评估脚本会 `sys.exit("missing M5 data")`。 + +### 2.3 5 分钟跑通一遍 + +```powershell +# A. 烟雾测试：30 trials，IS H1 2025，<2 分钟验证全链路通 +python scripts/optimize.py --smoke + +# B. 全量搜索：500 trials，IS = 2025 全年，~10–20 分钟 +python scripts/optimize.py --trials 500 +# 断点续跑（load_if_exists=True），中途 Ctrl+C 不丢 +# 产物：studies/optuna/gold_scalper_pro_is2025.db + .log + +# C. 用 finalist 参数重跑 Python（warmup 修复版），落 JSON +python scripts/reeval_finalist_forward.py +# 产物：studies/finalists/gold_scalper_pro_is2025-2026.json + +# D. 生成中文 Optuna 交互式 dashboard（HTML） +python scripts/build_optuna_dashboard.py +# 产物：reports/optuna_dashboard_gold_scalper_pro_is2025.html + +# E. 生成 ML 特征数据集（trade-level + trial-level） +python scripts/build_feature_datasets.py +# 产物：studies/features/{trade,trial}_features_*.parquet + .csv + +# F. 自动生成 registry markdown 条目（每 finalist 一份） +python scripts/build_registry_entry.py +# 产物：registry/gold_scalper_pro_xauusd_*_f{idx}_t{trial}.md +``` + +**MT5 验证是手动步骤**（Strategy Tester 跑完导 HTML 到 `reports/`），跑完后用： + +```powershell +python scripts/build_registry_entry.py ` + --mt5-is-html reports\IS-ReportTester-52845377.html ` + --mt5-oos-html reports\OOS-ReportTester-52845377.html +``` + +补齐 registry 条目里的 MT5 指标和 gap 表。 --- -## How to read this KB +## 3. 目录结构 -Start with `CLAUDE.md` if you plan to use an AI coding assistant (Claude Code, Cursor, etc.) to -build this with you — it is an **adaptive setup playbook** that profiles *your* machine and OS and -walks the install from zero. Otherwise read the numbered docs in order: - -| # | Doc | What you get | -|---|-----|--------------| -| — | [`CLAUDE.md`](CLAUDE.md) | Adaptive AI-assistant playbook: profiles your device, drives install from scratch. Doubles as `AGENTS.md`. | -| 01 | [`01-stack-and-install.md`](01-stack-and-install.md) | The exact tech stack, every library, where to get it, and how to install — per OS. | -| 02 | [`02-architecture.md`](02-architecture.md) | The layered architecture and data flow. The mental model for everything else. | -| 03 | [`03-engine-design.md`](03-engine-design.md) | How to design your own bar-by-bar engine. Grid logic used as the worked example. Fidelity vs MT5. | -| 04 | [`04-isolation-rules.md`](04-isolation-rules.md) | The isolation discipline: frozen engines, forks, separated instruments/strategies, curated registry. | -| 05 | [`05-config-and-inputs.md`](05-config-and-inputs.md) | How test inputs are kept *separate*: instrument config, parameter space, the pre-run wizard, lot/money mode. | -| 06 | [`06-optimization-and-robustness.md`](06-optimization-and-robustness.md) | Optuna objective design, constraints, diverse top-N selection, and anti-overfit robustness layers. | -| 07 | [`07-mt5-bridge.md`](07-mt5-bridge.md) | Connecting to MT5: compiling EAs, auto-running the tester, parsing reports, comparing Python vs MT5. Local-Windows and remote variants. | -| 08 | [`08-workflow-cycle.md`](08-workflow-cycle.md) | The full repeatable cycle: hypothesis → scaffold → stats → optimize → verify → promote. | +``` +backtesting-optuna-mt5-stack/ +├── README.md ← 本文件，项目入口 +├── PROJECT_GUIDE.md ← 项目实例说明（架构+坑+使用+扩展，10 节） +├── CLAUDE.md ← 给 AI 助手的分阶段搭装 playbook +├── requirements.txt ← Python 依赖清单 +├── .gitignore ← data/ + .env 忽略；studies/*.db + reports/*.html 入库 +│ +├── 01-stack-and-install.md ← 知识库 §1：技术栈 + 每个系统的安装命令 +├── 02-architecture.md ← 知识库 §2：单向依赖分层 + 数据流 +├── 03-engine-design.md ← 知识库 §3：bar-by-bar engine + intra-bar 4-sub-tick +├── 04-isolation-rules.md ← 知识库 §4：8 条隔离铁律 +├── 05-config-and-inputs.md ← 知识库 §5：4 个声明输入源 +├── 06-optimization-and-robustness.md ← 知识库 §6：Optuna objective + 反过拟合 +├── 07-mt5-bridge.md ← 知识库 §7：编译 EA / .set / .ini / 解析报告 +├── 08-workflow-cycle.md ← 知识库 §8：8 步可重复循环 +│ +├── GoldScalperPro.mq5 ← EA 源码（MQL5） +├── GoldScalperPro.ex5 ← EA 编译产物 +│ +├── shared/ ← 9 个单向依赖层（低调高、高不知低） +│ ├── core/ ← engine（冻结）+ metrics +│ ├── data/ ← Parquet loaders + MT5 HTML parser +│ ├── gates/ ← 入场过滤 mask（regime / 时段 / exhaustion） +│ ├── indicators/ ← 纯函数：RSI / ATR / EMA / SMA +│ ├── instruments/ ← per-symbol config（tick_value / spread / swap） +│ ├── mt5_pipeline/ ← .set / .ini / compile / runner / compare +│ ├── optimizer/ ← objective / search_space / diverse top-N selector +│ ├── robustness/ ← 反过拟合分析（只读 over results） +│ ├── wizard/ ← 运行时 Q&A → wizard-answers.yaml +│ └── config.py ← .env 加载（含 plain fallback） +│ +├── strategies/ +│ └── gold_scalper_pro/ ← glue：data → signals → stops → engine → metrics +│ ├── instruments.py ← XAUUSD_REAL config 对象 +│ ├── params.py +│ ├── scalper_engine.py ← ScalperEngine（engine.run 的 wrapper） +│ ├── search_space.py ← 19 个可调参数 + 13 个冻结基线 + 约束 +│ ├── set_mappings.py ← Python 参数名 ↔ MQL5 input 名映射 +│ ├── signals.py ← build_signals（EMA cross + RSI + ATR） +│ └── wizard_questions.py +│ +├── scripts/ ← 一次性可执行脚本（按用途分组见 §5） +│ ├── optimize.py ← Phase 6 主入口（smoke / full） +│ ├── reeval_finalist_forward.py ← finalist 参数 + warmup 修复版重跑 +│ ├── build_optuna_dashboard.py ← 中文 plotly 交互式 HTML +│ ├── build_feature_datasets.py ← ML 特征 parquet +│ ├── build_registry_entry.py ← 自动生成 registry markdown（命令驱动，禁手写） +│ ├── walk_forward.py +│ ├── prepare_mt5_verify.py ← 生成 .set / .ini 给 MT5 tester +│ ├── verify_mt5.py +│ ├── compare_finalist.py ← Python vs MT5 差距表 +│ ├── diag_*.py ← 9 个诊断脚本（ATR / 时区 / 引擎 trace / 信号） +│ └── download_xauusd_*.py ← 历史 M5 / M1 数据下载 +│ +├── studies/ ← Optuna 研究产物（committed） +│ ├── optuna/ ← SQLite DB + 运行日志（resumable） +│ │ ├── gold_scalper_pro_is2025.db +│ │ ├── gold_scalper_pro_is2025.db.before-warmup-fix.bak ← 旧 study 备份 +│ │ └── gold_scalper_pro_is2025.log +│ ├── finalists/ ← 3 个 finalist 的 IS/OOS 指标 JSON +│ │ └── gold_scalper_pro_is2025-2026.json +│ └── features/ ← ML 特征数据集（parquet + csv 副本） +│ ├── trade_features_*.parquet ← trade-level 35 列 +│ └── trial_features_*.parquet ← trial-level 聚合 +│ +├── reports/ ← MT5 + dashboard HTML（committed） +│ ├── IS-ReportTester-52845377.html +│ ├── OOS-ReportTester-52845377.html +│ └── optuna_dashboard_gold_scalper_pro_is2025.html +│ +├── registry/ ← 已批准 finalist（append-only，真相源） +│ ├── gold_scalper_pro_xauusd_*_f1_t491.md ← finalist #1（trial #491） +│ ├── gold_scalper_pro_xauusd_*_f2_t297.md ← finalist #2（trial #297） +│ └── gold_scalper_pro_xauusd_*_f3_t492.md ← finalist #3（trial #492） +│ +└── data/ ← gitignored，历史 parquet（接手后需重下） + ├── XAUUSD_M5_2024-06-26_2026-06-26.parquet + └── XAUUSD_M1_2024-06-26_2026-06-26.parquet +``` --- -## The 30-second mental model +## 4. 核心数据流（一个 iteration 的闭环） -- **The engine knows nothing about your strategy.** It takes bars + entry signals + stop/target - prices and simulates fills. All strategy math lives in *caller* code. (Doc 02–03.) -- **The engine is frozen.** You never edit a validated engine to test an idea — you fork it, - prove the fork reproduces the original 1:1 with the change off, then test. (Doc 04.) -- **Instruments are data, not code branches.** Tick value, spread model, swap, lot steps — all in a - per-symbol config object. The engine reads everything from it. (Doc 05.) -- **Search inputs are declared, not scattered.** Every tunable parameter, its range, and its - constraints live in one declared search space; one wizard captures the run settings; one YAML - records the answers so any run is reproducible. (Doc 05–06.) -- **Python ranks, MT5 decides.** Fast Python search produces a shortlist; MT5 produces the trusted - number; a comparison table is saved for every finalist. (Doc 03, 06, 07.) +``` + ┌──────────────────────────────────────────────────────┐ + │ scripts/optimize.py │ + │ load_m5() ─► slice_window(IS_START, IS_END) │ + │ load_m1() ─► slice_window(IS_START, IS_END) │ + │ full_m5 ───────────────────┐ (warmup, signals_full_bars)│ + │ ▼ │ + │ ObjectiveConfig(bars_is, m1_is, signals_full_bars=full_m5)│ + │ │ │ + │ ▼ │ + │ build_objective(cfg) ─► trial loop │ + │ │ │ + │ ▼ │ + │ build_signals(full_m5) ─► slice to IS │ + │ engine_kwargs_from_params(params) │ + │ ScalperEngine.run(bars, signals, sl, tp, m1_bars) │ + │ │ │ + │ ▼ │ + │ compute_metrics + Constraints gate │ + │ study.optimize() ─► trials 0..499 │ + └──────────────────────────────────────────────────────┘ + │ + ▼ + ┌──────────────────────────────────────────────────────┐ + │ select_diverse_topn(study, n=3) │ + │ └─► 3 个 diverse finalist（greedy max-distance） │ + └──────────────────────────────────────────────────────┘ + │ + ▼ + ┌──────────────────────────────────────────────────────┐ + │ scripts/reeval_finalist_forward.py │ + │ 用 finalist params 重跑 Python（warmup 修复版） │ + │ ─► studies/finalists/gold_scalper_pro_is2025-2026.json│ + └──────────────────────────────────────────────────────┘ + │ + ▼ + ┌──────────────────────────────────────────────────────┐ + │ MT5 Strategy Tester（手动） │ + │ prepare_mt5_verify.py 生成 .set + .ini │ + │ Forward mode: IS=2025, OOS=2026 H1 │ + │ 导 HTML 报告到 reports/ │ + └──────────────────────────────────────────────────────┘ + │ + ▼ + ┌──────────────────────────────────────────────────────┐ + │ scripts/build_registry_entry.py │ + │ finalist JSON + Optuna study + MT5 HTML │ + │ ─► registry/gold_scalper_pro_xauusd_*_f{idx}.md │ + │ （9 节自文档化 markdown，含差距表） │ + └──────────────────────────────────────────────────────┘ +``` --- -## What you must bring yourself +## 5. 常用脚本速查（按场景） -This KB is deliberately empty of trading IP. To build a working lab you supply: - -- **Your MQL5 EA(s)** — compiled `.ex5` plus source `.mq5`, and any custom indicators they call. -- **Your strategy logic** — encoded once in Python (the caller) so the mirror engine can run it. -- **Your presets** — the `.set` files / input templates you want to test and optimize. -- **Historical data** — downloaded from your broker via MT5 (the stack includes a recipe). -- **A broker demo account** — for the MT5 Strategy Tester runs. - -Everything else — the architecture, the optimizer, the MT5 bridge, the robustness checks, and the -rules that hold it together — is described in the docs above. +| 场景 | 脚本 | 命令 | +|------|------|------| +| **跑 Optuna 搜索** | `optimize.py` | `--smoke`（30 trials，<2 分钟） / `--trials 500`（全量） | +| **重跑 finalist** | `reeval_finalist_forward.py` | 用 finalist 参数在 IS+OOS 重跑 Python，落 JSON | +| **走前验证（forward）** | `walk_forward.py` | finalist 在 OOS 窗口独立验证 | +| **生成 dashboard** | `build_optuna_dashboard.py` | 中文 plotly HTML（5 主图 + 18 单参数 + 15 等高线） | +| **生成 ML 特征** | `build_feature_datasets.py` | trade-level 35 列 + trial-level parquet | +| **生成 registry** | `build_registry_entry.py` | 命令驱动生成 finalist 条目（**禁手写**） | +| **准备 MT5 验证** | `prepare_mt5_verify.py` | 生成 `.set` + `.ini`，复制到 MT5 tester profiles | +| **比对 finalist** | `compare_finalist.py` | Python vs MT5 差距表 | +| **诊断（9 个）** | `diag_*.py` | ATR / 时区 / 引擎 trace / 信号触发 / 仓位 mismatch 等 | +| **查 study** | `inspect_study.py` | 打印 trials / best params | +| **查 MT5 报告** | `inspect_report.py` / `diag_mt5_summary.py` / `diag_mt5_trades.py` | 解析 HTML | +| **下历史数据** | `download_xauusd_history.py`（M5）/ `download_xauusd_m1.py`（M1） | 必须 MT5 终端已登录 | +| **查 symbol 规格** | `query_xauusd_spec.py` | tick_value / contract_size / lot_step | --- -*This KB is brand-free and self-contained. Drop the folder into a Git repository, open it with your -AI coding assistant, and build your own lab.* +## 6. 当前项目状态 + +### 6.1 已完成的 Phase + +| Phase | 内容 | 状态 | +|-------|------|------| +| 0–1 | 设备 profile + Python 栈安装 | ✅ | +| 2 | 仓库骨架（shared/ 9 子包 + strategies/） | ✅ | +| 3 | MT5 连接（分步 initialize+login）+ EA 资产导入 | ✅ | +| 4 | ScalperEngine + M1 tick-level 模拟路径 | ✅ | +| 5 | XAUUSD_REAL instrument + GoldScalperPro search space | ✅ | +| 6 | Optuna 500-trial 搜索 + 3 diverse finalist | ✅ | +| 7 | MT5 forward mode IS/OOS 验证（旧 finalist）+ 差距根因 | ✅ | +| 8 | APPROVAL + PROMOTE（registry 第一条） | ✅ | +| — | Optuna warmup bug 修复（2026-06-26 重跑） | ✅ | + +### 6.2 当前 finalist（warmup 修复后，2026-06-27 重跑） + +来自 [studies/finalists/gold_scalper_pro_is2025-2026.json](studies/finalists/gold_scalper_pro_is2025-2026.json)： + +| # | trial | score | IS net | OOS net | registry | +|---|-------|------:|-------:|--------:|----------| +| 1 | #491 | 186,075 | $263,527 | $10,632 | [f1_t491.md](registry/gold_scalper_pro_xauusd_2025-01-01_2026-06-26_f1_t491.md) | +| 2 | #297 | 68,772 | $94,197 | $11,153 | [f2_t297.md](registry/gold_scalper_pro_xauusd_2025-01-01_2026-06-26_f2_t297.md) | +| 3 | #492 | 129,575 | $165,740 | $6,969 | [f3_t492.md](registry/gold_scalper_pro_xauusd_2025-01-01_2026-06-26_f3_t492.md) | + +> 这三个 finalist **尚未通过 MT5 验证**——registry 条目标记为 +> "已记录（无 MT5 验证 — 待 Phase 7 MT5 验证）"。需要先在 MT5 Strategy Tester +> 跑出 IS+OOS HTML 报告，再用 `build_registry_entry.py --mt5-is-html ... --mt5-oos-html ...` +> 补齐 MT5 部分和差距表。 + +--- + +## 7. 接手者必读（续做指南） + +### 7.1 三条铁律（违反任一条等于把项目搞烂） + +1. **trailing/BE EA 必须传 `m1_bars=` 给 `engine.run()`**——否则 net profit + 会有 −40% 到 −50% 的隐藏 gap。这不是噪声，是 bug。详见 + [03-engine-design.md §7](03-engine-design.md)。 + +2. **`ObjectiveConfig` 必须传 `signals_full_bars=full_m5`**——否则 EMA/RSI/ATR + 在 IS_START 才开始预热，首笔交易会比 MT5 晚 14 小时，整个 finalist 集合都 + 是错的。详见 [scripts/optimize.py](scripts/optimize.py) 的 + `make_objective_config()` docstring。 + +3. **Engine 一旦验证就冻结，要试新想法就 fork**——直接改 validated engine + 可能让所有已信任的数字都失效。Fork + default-OFF 实验 hook + 证明 + fork-with-change-off == 原版 1:1 + 再 A/B。详见 + [04-isolation-rules.md Rule 2](04-isolation-rules.md)。 + +### 7.2 已知坑（接手时大概率还会遇到） + +| 坑 | 症状 | 修复 | +|----|------|------| +| MT5 连接 IPC 超时（-10005） | `mt5.initialize()` 直接传所有参数 | **分步**：先 `mt5.initialize(path=...)`，再 `mt5.login(login, password, server)` | +| 终端路径反斜杠 | IPC 超时 / 连不上 | 路径必须用**正斜杠** `/`，不是 `\` | +| MT5 进程残留 | 新连接失败 | 任务管理器清掉所有 `terminal64.exe` 进程后重试 | +| `.set` 文件加载失败 | MT5 tester 找不到参数 | 文件必须以 **BOM 开头**（`write_set_file` 已处理） | +| Plotly 图表看不见 | HTML 渲染 0 高度 | `fig.update_layout(height=<像素>)` + CDN + 单列 flex（见 `build_optuna_dashboard.py`） | +| 参数切片图标签重叠 | 5400px 宽复合图挤一起 | 拆成每参数独立 900×420 小图（`render_slice_grid`） | +| Optuna warmup bug | IS 起点附近 14h 没信号 | `signals_full_bars` 字段，详见 §7.1 第 2 条 | + +### 7.3 急需做的下一步（按优先级） + +1. **跑 MT5 验证新 finalist #1（trial #491）** —— 用 + `scripts/prepare_mt5_verify.py` 生成 `.set` + `.ini`，在 MT5 Strategy Tester + forward mode 跑 IS=2025 + OOS=2026 H1，导 HTML 到 `reports/`，然后用 + `build_registry_entry.py --mt5-is-html ... --mt5-oos-html ...` 补齐 registry。 + +2. **重新诊断 ATR 数据差异** —— 旧的 `diag_atr_check.py` / `diag_mt5_trades.py` + 是针对 trial #324 的，新 finalist 需重跑。 + +3. **实现自动化 MT5 pipeline** —— `shared/mt5_pipeline/runner.py` 现在只是骨架， + 把"打开 tester → 等跑完 → 拷 HTML"自动化掉。 + +4. **补全 robustness 层** —— `shared/robustness/layers.py` 只有 + `stability_region`，缺 doc 06 §5 描述的其它层（parameter stability、 + monte-carlo perturbation、walk-forward aggregate）。 + +5. **MT5 数据对齐** —— Python parquet M5 OHLC 与 MT5 tester 内部 history 在 + IS 起点附近有微差，复利下被指数放大。需要决定是接受差距（旧 finalist + #324 已接受）还是修对齐。 + +### 7.4 不要做的事 + +- **不要手写 registry 条目**——必须用 `scripts/build_registry_entry.py` 生成。 + 脚本会从 finalist JSON + Optuna study + MT5 HTML 三源自动提取，包含 + 9 节内容（标识 / 参数表含百分位 / 三个 finalist 对比 / Python 指标 / + MT5 指标 / 差距分析 / 搜索统计 / 复现命令 / 已知限制）。 + +- **不要编辑已批准的 registry 条目**——append-only。新发现是新条目， + 旧条目就算被超越也保留（负面结果也是数据）。 + +- **不要在 `engine.run()` 里加策略逻辑**——engine 只决定"价格是否触到 + 止损"，**从不决定止损放在哪**。换策略改 caller 和数组，engine 不动。 + +- **不要在 `mt5.initialize()` 里塞所有参数**——分步连接，详见 §7.2 第 1 条。 + +- **不要给单个 strategy import 另一个 strategy**——copy，不要 import。 + iteration 各自拥有 snapshot，几个月前的 iteration 仍要能跑。 + +### 7.5 改动约定 + +- 改了 engine / objective / search space → 跑 `--smoke` 先验证全链路通 +- 改了 finalist 挑选逻辑 → 重跑 `reeval_finalist_forward.py` + 重生 dashboard +- 改了 MT5 报告解析 → 用 `diag_mt5_summary.py` 对比新旧解析结果 +- commit 时分小步、写清楚 why（不只是 what），git log 就是项目文档的一部分 + +--- + +## 8. 文档地图 + +| 你想做的事 | 读这个 | +|------------|--------| +| 跑一遍流程 | 本文件 §2 + §5 | +| 理解架构为什么这样搭 | [PROJECT_GUIDE.md §1](PROJECT_GUIDE.md) + [02-architecture.md](02-architecture.md) | +| 看具体踩过的坑 | [PROJECT_GUIDE.md §5](PROJECT_GUIDE.md) + 本文件 §7.2 | +| 写新策略 | [02-architecture.md §4](02-architecture.md) + [08-workflow-cycle.md](08-workflow-cycle.md) | +| 改 engine | [03-engine-design.md](03-engine-design.md) + [04-isolation-rules.md Rule 2](04-isolation-rules.md) | +| 改 Optuna objective | [06-optimization-and-robustness.md](06-optimization-and-robustness.md) + [shared/optimizer/objective.py](shared/optimizer/objective.py) | +| 接 MT5 | [07-mt5-bridge.md](07-mt5-bridge.md) + [shared/mt5_pipeline/](shared/mt5_pipeline/) | +| 看 finalist 参数 | [studies/finalists/gold_scalper_pro_is2025-2026.json](studies/finalists/gold_scalper_pro_is2025-2026.json) + [registry/](registry/) | +| 看 Optuna trials | `python scripts/inspect_study.py` 或直接打开 `studies/optuna/gold_scalper_pro_is2025.db` | + +--- + +## 9. License & Disclaimer + +个人研究项目，不构成投资建议。EA 源码 `GoldScalperPro.mq5` 属于其原作者； +本项目仅用于参数优化方法论研究，不重新分发 EA 本体。 diff --git a/_test_set_gen.py b/_test_set_gen.py deleted file mode 100644 index a484a0a..0000000 --- a/_test_set_gen.py +++ /dev/null @@ -1,44 +0,0 @@ -"""Dry-run: generate .set + .ini with frozen baseline, verify format.""" -import sys -from pathlib import Path -PROJECT = Path(__file__).resolve().parent -sys.path.insert(0, str(PROJECT)) - -from shared.config import get_secret, load_env -from shared.mt5_pipeline.ini_gen import TesterConfig, write_tester_ini, MODEL_OHLC -from shared.mt5_pipeline.set_gen import write_set_file -from strategies.gold_scalper_pro.search_space import FROZEN_BASELINE -from strategies.gold_scalper_pro.set_mappings import GOLD_SCALPER_MAPPINGS - -load_env(PROJECT) -mt5_data = Path(get_secret("MT5_DATA_PATH")) -profiles = mt5_data / "MQL5" / "Profiles" / "Tester" -profiles.mkdir(parents=True, exist_ok=True) - -set_path = profiles / "GoldScalperPro_dryrun.set" -write_set_file(FROZEN_BASELINE, GOLD_SCALPER_MAPPINGS, set_path) -print(f"set written: {set_path}") -print(f" size: {set_path.stat().st_size} bytes") -print(f" first bytes: {set_path.read_bytes()[:40]}") - -ini_path = profiles / "GoldScalperPro_dryrun.ini" -tcfg = TesterConfig( - expert=r"Experts\GoldScalperPro.ex5", - symbol="XAUUSD", - period="M5", - model=MODEL_OHLC, - from_date="2024.06.26", - to_date="2026.06.26", - deposit=10000.0, - leverage=100, - report="report_dryrun", - shutdown_terminal=True, - set_file=str(set_path), - login=int(get_secret("MT5_DEMO_LOGIN")), - password=get_secret("MT5_DEMO_PASSWORD"), - server=get_secret("MT5_DEMO_SERVER"), -) -write_tester_ini(tcfg, ini_path) -print(f"\nini written: {ini_path}") -print(f"--- contents ---") -print(ini_path.read_text(encoding="utf-8")) diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..ac9ebae --- /dev/null +++ b/requirements.txt @@ -0,0 +1,23 @@ +# ── Core data stack ──────────────────────────────────────────────── +pandas>=2.0 # DataFrame everywhere; bars/trades/metrics +numpy>=1.24 # vectorized indicator math +pyarrow>=14.0 # Parquet backend for pandas (.parquet bars & features) + +# ── Optimizer ────────────────────────────────────────────────────── +optuna>=3.5 # TPE sampler, SQLite-persisted studies, visualization +sqlalchemy>=2.0 # Optuna's sqlite:// storage backend +plotly>=5.18 # optuna.visualization + dashboard HTML rendering +tqdm>=4.65 # progress bars (also used by optuna) + +# ── MT5 bridge (Windows-only) ───────────────────────────────────── +MetaTrader5>=5.0.45 # connect MT5 terminal, pull history, query symbol spec + # NOTE: only installs on Windows. On non-Windows dev + # boxes you can still edit shared/ + strategies/ code, + # but the data-download / verify scripts won't run. + +# ── Report parsing & config ──────────────────────────────────────── +lxml>=4.9 # parse MT5 Strategy Tester HTML reports (fast path) +html5lib>=1.1 # fallback HTML parser for malformed reports +PyYAML>=6.0 # shared/wizard/wizard.py — wizard-answers.yaml I/O +python-dotenv>=1.0 # optional .env loader (shared/config.py has a plain + # fallback, so this is convenience not a hard dep) diff --git a/run.py b/run.py deleted file mode 100644 index 123d7ad..0000000 --- a/run.py +++ /dev/null @@ -1,153 +0,0 @@ -"""Phase 5+6 entry point: optimize GoldScalperPro on real XAUUSD bars. - -Assembles ObjectiveConfig (engine + signals + search space + constraints), -runs Optuna, applies the diverse top-N selector, then runs the robustness -layers on the finalists and prints a report. This is the cycle that Phase 7 -will feed into MT5 for verification. - -Usage: - python run.py [--trials 200] [--top-n 3] [--deposit 10000] -""" -from __future__ import annotations - -import argparse -import sys -import time -from pathlib import Path - -PROJECT = Path(__file__).resolve().parent -sys.path.insert(0, str(PROJECT)) - -import optuna - -from shared.core.engine import SizingInputs -from shared.data.loaders import load_bars -from shared.optimizer.objective import ( - Constraints, - ObjectiveConfig, - build_objective, -) -from shared.optimizer.selector import select_diverse_topn -from shared.robustness.layers import stability_region -from strategies.gold_scalper_pro.instruments import XAUUSD_REAL -from strategies.gold_scalper_pro.scalper_engine import ( - ScalperConfig, - ScalperEngine, - config_from_params, - engine_kwargs_from_params, -) -from strategies.gold_scalper_pro.search_space import ( - FROZEN_BASELINE, - INT_PARAMS, - SEARCH_SPACE, -) -from strategies.gold_scalper_pro.signals import build_signals - - -def find_bars_file() -> Path: - """Auto-find the latest XAUUSD M5 parquet in data/.""" - data_dir = PROJECT / "data" - candidates = sorted(data_dir.glob("XAUUSD_M5_*.parquet")) - if not candidates: - raise FileNotFoundError(f"no XAUUSD M5 parquet in {data_dir}") - return candidates[-1] - - -def main() -> int: - ap = argparse.ArgumentParser(description="Optimize GoldScalperPro.") - ap.add_argument("--trials", type=int, default=100, - help="Optuna trials (default 100)") - ap.add_argument("--top-n", type=int, default=3, - help="diverse finalists to select (default 3)") - ap.add_argument("--deposit", type=float, default=10000.0, - help="initial deposit (default 10000)") - ap.add_argument("--bars", type=str, default="", - help="path to parquet bars (blank = auto-find)") - args = ap.parse_args() - - bars_path = Path(args.bars) if args.bars else find_bars_file() - print(f"=== GoldScalperPro optimization ===") - print(f"bars : {bars_path.name}") - print(f"trials : {args.trials}") - print(f"top-n : {args.top_n}") - print(f"deposit : {args.deposit:,.0f} USD") - print() - - bars = load_bars(bars_path) - print(f"loaded {len(bars):,} bars {bars['timestamp'].iloc[0]} → {bars['timestamp'].iloc[-1]}") - - # ── Assemble the objective ──────────────────────────────────────────── - constraints = Constraints( - min_trades=25, - min_profit_factor=1.2, # relaxed for first pass; tightened later - max_equity_dd_pct=0.40, # 40% hard cap - ) - obj_cfg = ObjectiveConfig( - engine=ScalperEngine(), - bars=bars, - instrument=XAUUSD_REAL, - sizing=SizingInputs(), - initial_deposit=args.deposit, - search_space=SEARCH_SPACE, - int_params=INT_PARAMS, - frozen_baseline=FROZEN_BASELINE, - constraints=constraints, - dd_weight=1.0, - build_signals=build_signals, - build_engine_kwargs=engine_kwargs_from_params, - ) - objective = build_objective(obj_cfg) - - # ── Run Optuna ───────────────────────────────────────────────────────── - optuna.logging.set_verbosity(optuna.logging.WARNING) - study = optuna.create_study(direction="maximize", - sampler=optuna.samplers.TPESampler(seed=42)) - print(f"\nrunning {args.trials} trials ...") - t0 = time.time() - study.optimize(objective, n_trials=args.trials, show_progress_bar=False) - elapsed = time.time() - t0 - print(f"done in {elapsed:.1f}s ({elapsed/args.trials:.2f}s/trial)") - - # ── Report ──────────────────────────────────────────────────────────── - best = study.best_trial - print(f"\n=== best trial #{best.number} ===") - print(f" score : {best.value:+,.2f}") - print(f" net profit : {best.user_attrs['net_profit']:+,.2f}") - print(f" profit factor : {best.user_attrs['profit_factor']:.2f}") - print(f" trades : {best.user_attrs['total_trades']}") - print(f" win rate : {best.user_attrs['win_rate']:.2%}") - print(f" equity DD : {best.user_attrs['max_equity_dd']:,.2f} " - f"({best.user_attrs['max_equity_dd_pct']:.2%})") - print(f" sharpe : {best.user_attrs['sharpe']:.2f}") - if best.user_attrs.get("violations"): - print(f" violations : {best.user_attrs['violations']}") - print(" params:") - for k, v in best.user_attrs["params"].items(): - if k in SEARCH_SPACE: - print(f" {k:24s} = {v}") - - # ── Diverse top-N ───────────────────────────────────────────────────── - print(f"\n=== diverse top-{args.top_n} finalists ===") - finalists = select_diverse_topn(study, args.top_n, SEARCH_SPACE) - for i, t in enumerate(finalists, 1): - print(f" #{i} trial {t.number}: score={t.value:+,.2f} " - f"net={t.user_attrs['net_profit']:+,.2f} " - f"PF={t.user_attrs['profit_factor']:.2f} " - f"trades={t.user_attrs['total_trades']}") - - # ── Stability region (is the best on a plateau?) ───────────────────── - print(f"\n=== stability region ===") - sr = stability_region(study, SEARCH_SPACE) - print(f" passed : {sr.get('passed')}") - print(f" cluster_size : {sr.get('cluster_size')}") - print(f" best_in_cluster : {sr.get('best_in_cluster')}") - if sr.get("reason"): - print(f" reason : {sr['reason']}") - - print("\n=== done ===") - print("Next: Phase 7 — generate .set/.ini for each finalist, run MT5, compare.") - return 0 - - -if __name__ == "__main__": - raise SystemExit(main())