diff --git a/02-architecture.md b/02-architecture.md index e9b220e..7f7cf19 100644 --- a/02-architecture.md +++ b/02-architecture.md @@ -67,8 +67,13 @@ engine.run( sl_prices, # array — the stop price for an entry on that bar (NaN if none) tp_prices, # array — the target price instrument, # InstrumentConfig — all symbol mechanics - lot / money_mode, # position sizing inputs + sizing, # SizingInputs — position sizing inputs initial_deposit, + *, # keyword-only from here + m1_bars=None, # OPTIONAL: M1 bars for tick-level exit simulation + # REQUIRED if the EA moves its SL intra-trade (BE / trailing / + # basket trailing) — bar-level mode is untrustworthy for that + # class (doc 03 §7 failure mode, doc 03 §8 target gates). ) -> Result ``` @@ -76,6 +81,12 @@ engine.run( > never decides *where* a stop goes — only *whether* price touched it. This is the seam that > separates "the strategy" from "the simulator". Change your strategy → you change the caller and the > arrays you hand in; the engine is untouched. +> +> **The `m1_bars` parameter is not optional for trailing/BE strategies.** When the EA updates its SL +> during a trade, the bar-level engine can produce a −40% to −50% net gap vs MT5 (doc 03 §7 measured +> failure mode). Pass M1 bars and the engine switches to tick-level exit simulation, dropping the +> gap to ~−5%. Bar-level mode remains the right (and faster) choice for clean-directional setups that +> don't move the SL. The engine's output is equally generic: @@ -95,32 +106,44 @@ Factor, Win Rate, max Balance/Equity Drawdown, Sharpe, APR, trade count. ## 3. Data flow of one backtest ``` -data//_M1_.parquet - │ load_bars() → DataFrame with per-bar spread column - ▼ -caller: resample M1 → the signal timeframe (e.g. H1/H4/D1) - │ indicators.* on the resampled frame - ▼ -caller: edge-detect signals (True only on the bar the condition first flips, not every bar after) - │ + optional gates (regime/time filters AND-ed into the signal) - ▼ -caller: compute SL/TP price arrays from params (ATR stop, % stop, indicator band, …) - ▼ -engine.run(bars, signals, sl/tp, instrument, sizing, deposit) - │ - ▼ -Result → compute_metrics → dict - │ - ▼ (finalists only) -MT5 bridge: build .set from the same params → run real tester → parse report → compare +data//_M1_.parquet data//_M5_.parquet + │ load_bars() → M1 DataFrame │ load_bars() → signal-TF DataFrame + │ │ (resampled from M1 if needed) + │ ▼ + │ caller: indicators.* on the signal timeframe + │ │ + │ ▼ + │ caller: edge-detect signals (True only on the bar + │ │ the condition first flips, not every bar after) + │ │ + optional gates (regime/time filters AND-ed in) + │ ▼ + │ caller: compute SL/TP price arrays from params + │ │ (ATR stop, % stop, indicator band, …) + └──────────────┐ ┌──────────────────────────┘ + ▼ ▼ + engine.run(signal_bars, signals, sl/tp, instrument, sizing, deposit, + m1_bars=m1_bars) ← M1 is passed back to the engine + │ for tick-level exit simulation when the EA + │ moves its SL intra-trade (BE / trailing). Without + │ it, the bar-level engine over-credits BE exits + │ (doc 03 §7 failure mode). + ▼ + Result → compute_metrics → dict + │ + ▼ (finalists only) + MT5 bridge: build .set from the same params → run real tester → parse report → compare ``` -Two subtleties that cause most bugs if missed (both explained in doc 03): +Three subtleties that cause most bugs if missed (the first two explained in doc 03): - **Edge detection.** Signals must be `True` only on the *transition* bar, not forward-filled, or the engine re-enters every bar. - **Timeframe alignment.** Indicators computed on a higher timeframe must be mapped back onto the M1 bars correctly (no look-ahead — a daily value is only known after that day closes). +- **M1 must reach the engine for trailing/BE EAs.** Downloading M1 only to resample it up to the signal + timeframe is **not** enough if the EA moves its SL during a trade. Pass the M1 bars to `engine.run` + (the `m1_bars=` kwarg); otherwise the bar-level exit simulation produces a −40% to −50% net gap + (doc 03 §7) that looks like a "fidelity issue" but is actually a missing-input bug. --- diff --git a/03-engine-design.md b/03-engine-design.md index 30a907f..d2cf131 100644 --- a/03-engine-design.md +++ b/03-engine-design.md @@ -204,16 +204,41 @@ Anything that depends on the **path inside a bar**: trailing-stop triggers, the target is touched, exact fill timing on volatile bars. The divergence **scales with path-sensitivity × volatility**: -| Strategy character | Typical Python vs MT5 gap | -|--------------------|---------------------------| +| Strategy character | Typical Python vs MT5 gap (bar-level engine) | +|--------------------|-------------------------------------------| | Clean directional, few exits | small and consistent: Python reads somewhat higher | -| Tight trailing / martingale grid in calm years | small | +| Tight trailing / break-even in calm years | **NOT small** — see failure mode below | | Tight trailing / martingale grid **in crash years** | **large** — Python's 4 points miss the finer exits MT5 takes, over-crediting big moves | -A concrete illustration: a trailing strategy might match MT5 within a few currency units in calm, -choppy years, yet the Python figure can be several times the MT5 figure across a violent crash year — -because MT5's finer path triggers exits at prices the 4-point model skips. The bias is almost always -**Python optimistic**, and almost always concentrated in the most volatile episodes. +#### The break-even / trailing failure mode (measured, not theoretical) + +A common assumption is that "calm years → small gap" applies to trailing/BE strategies. **It does +not.** A break-even + trailing-stop strategy with bar-level simulation can show a **−40% to −50% +net-profit gap even in a calm 3-week window**, while trade count matches MT5 exactly. The mechanism: + +- A bar-level engine updates the BE / trailing SL using the bar's high (or low), then checks the SL + on the **same bar's opposite extreme**. If price briefly crossed the BE threshold, the SL is moved + to break-even, and the same bar's low (for a long) can trigger that just-moved SL at break-even — + booking a **micro-profit** that MT5's tick path would have booked as a small loss (the SL-trigger + tick and the BE-trigger tick are separate in MT5, and price can continue past BE to a real loss + before the SL fills). +- This inflates both the win rate and the gross profit simultaneously. The bias is **always Python + optimistic**, and concentrates in the SL/BE exit reason (mean PnL per SL-exit trades reads + positive in Python where MT5 reads negative). + +**Fix: M1 tick-level exit simulation.** Load M1 bars and, inside each M5 (or higher) bar, walk the +5 M1 sub-bars as 4 synthetic ticks each in direction-aware order (see §2). This separates the +BE-update tick from the SL-trigger tick onto different M1 bars, restoring the realistic worst case. +Measured impact on a break-even scalper: + +| Mode | Net gap vs MT5 | PF gap vs MT5 | Trade-count gap | +|------|----------------|---------------|------------------| +| Bar-level (4 sub-ticks) | **−48.5%** | −30.0% | 0% | +| M1 tick-level (4 sub-ticks × 5 M1 bars) | **−5.6%** | −7.8% | 0% | + +Trade count is unaffected by the choice (signals still fire on the higher timeframe); only the exit +path fidelity changes. Use M1 tick-level simulation for any strategy that moves its SL during a +trade (BE, trailing, basket trailing). ### The practical policy (this is the whole point of the two-tier design) 1. **Use Python for fast ranking and A/B** — the *relative order* of setups is preserved, which is all @@ -234,16 +259,29 @@ because MT5's finer path triggers exits at prices the 4-point model skips. The b 1. Pick **one known preset** of your EA and a short period (a few months). 2. Run it in MT5 (1-minute-OHLC model is fine to start) and save the report. -3. Run your Python engine on the **same data, same preset**. -4. Reconcile **trade by trade**, then in aggregate. Target gates for a clean-directional setup: - - Net difference ≤ ~2%, trade-count difference ≤ ~5%, Profit Factor essentially identical, - equity-drawdown difference ≤ ~3%. +3. Run your Python engine on the **same data, same preset**. **If your EA moves its SL during a trade + (break-even, trailing, basket trailing), you MUST pass M1 bars and run tick-level exit simulation + (§7) — the bar-level engine is not trustworthy for that class of EA.** +4. Reconcile **trade by trade**, then in aggregate. Target gates depend on the EA class and engine mode: + + | EA class / engine mode | Net gap | PF gap | Trade-count gap | Equity-DD gap | + |------------------------|---------|--------|------------------|----------------| + | Clean-directional, bar-level | ≤ ~2% | essentially identical | ≤ ~5% | ≤ ~3% | + | BE / trailing, **bar-level** | **unattainable** — see §7 failure mode (~−40% to −50% net gap) | + | BE / trailing, **M1 tick-level** | ≤ ~10% | ≤ ~10% | ≤ ~5% | ≤ ~10% | + + The bar-level gate (≤ ~2%) applies only to setups that don't move the SL intra-trade. For BE/trailing + EAs the tick-level gate is wider (≤ ~10%) because residual spread/tick-path differences remain — + accept it and **MT5-verify every finalist** rather than chase sub-2% on a tick-sensitive EA. + 5. Only when this passes is the engine trustworthy enough to optimize on. Record the comparison as the - engine's **baseline fidelity document** and freeze the engine (doc 04). + engine's **baseline fidelity document** (engine mode + measured gap + the window used) and freeze the + engine (doc 04). > If you can't reconcile, the usual culprits are: signal edge-detection (re-entry every bar), timeframe -> mapping/look-ahead, lot-mode mismatch, spread/swap applied on the wrong side or day, or sub-tick -> ordering. Walk those five before suspecting anything exotic. +> mapping/look-ahead, lot-mode mismatch, spread/swap applied on the wrong side or day, sub-tick +> ordering, **or running a BE/trailing EA on the bar-level engine (use M1 tick-level instead)**. Walk +> those six before suspecting anything exotic. Next: [`04-isolation-rules.md`](04-isolation-rules.md) — the discipline that keeps a validated engine validated. diff --git a/07-mt5-bridge.md b/07-mt5-bridge.md index 6ce3dea..094c0ce 100644 --- a/07-mt5-bridge.md +++ b/07-mt5-bridge.md @@ -193,6 +193,31 @@ For a brand-new symbol you must first **let MT5 cache its history** (open a char `Tools → Options → Charts → Max bars: Unlimited`, scroll back) so the tester and the export have enough bars. Confirm the exact broker symbol code (it varies: indices and metals especially) before exporting. +### 6a. Pull M1 even if your signal timeframe is higher — and pass it to the engine + +A common mistake: download only the signal timeframe (e.g. M5/M15/H1) and assume that's enough. It is +not, **if your EA moves its SL during a trade** (break-even, trailing, basket trailing). The bar-level +engine produces a −40% to −50% net gap on those EAs (doc 03 §7 measured failure mode) because the BE +update and the SL trigger fall on the same bar's opposite extreme. The fix is tick-level exit +simulation, which needs the **M1 bars covering the same window as your signal bars**: + +```python +# Download BOTH timeframes. Same symbol, same window, exclusive end (match MT5 tester). +m1_bars = load_bars(DATA / f"{SYMBOL}_M1_{start}_{end}.parquet") +m5_bars = load_bars(DATA / f"{SYMBOL}_M5_{start}_{end}.parquet") # the signal timeframe + +# Slice both to the exact same window (exclusive end matches MT5 tester's ToDate). +m1_bars = m1_bars[(m1_bars["timestamp"] >= start) & (m1_bars["timestamp"] < end)] +m5_bars = m5_bars[(m5_bars["timestamp"] >= start) & (m5_bars["timestamp"] < end)] + +# Pass m1_bars to the engine — it switches to tick-level exit simulation automatically. +result = engine.run(m5_bars, signals_long, signals_short, sl, tp, + instrument, sizing, deposit, m1_bars=m1_bars) +``` + +If your EA does **not** move its SL intra-trade (clean market entries with a fixed SL/TP), `m1_bars` +can be omitted — the bar-level engine is exact for that class and faster. + --- ## 7. Parsing the report @@ -223,10 +248,15 @@ For every finalist, write an `auto-verification.md` that puts the two tiers side | Equity DD max | … | … | …% | | Total trades | … | … | …% | -Then judge the delta **against the expected fidelity gap** (doc 03 §7): a clean-directional setup -should show only a modest negative gap (MT5 a little below Python); a trailing/grid setup in volatile -history can gap much wider, and that's *expected*, not a bug. The decision rule: if the **MT5** number still clears your bar after the gap, -the finalist is real; if the edge only existed in the optimistic Python figure, discard it. +Then judge the delta **against the expected fidelity gap** (doc 03 §7 + §8 target-gate table): +a clean-directional setup (bar-level engine, SL not moved intra-trade) should show ≤ ~2% net gap; +a trailing/BE setup on the **bar-level** engine is not trustworthy at all (−40% to −50% gap, +doc 03 §7 measured failure mode) — **before** judging the gap "expected", confirm the Python run +used M1 tick-level exit simulation (`m1_bars=` passed to the engine), which brings the gap into the +≤ ~10% range. Only a gap inside the §8 target gate counts as "expected"; a wider gap is a +missing-M1 / wrong-engine-mode bug, not fidelity noise. The decision rule: if the **MT5** number +still clears your bar after the gap, the finalist is real; if the edge only existed in the +optimistic Python figure, discard it. > A useful warm-up calibration: pick one known preset and run the full Python-vs-MT5 comparison on it > first. It tells you *your* stack's actual gap for *your* EA, so later finalists are judged against a diff --git a/CLAUDE.md b/CLAUDE.md index c6abed8..69d8a14 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -111,11 +111,23 @@ part. The engine must reproduce the EA's **fill and exit logic** bar-by-bar. - Keep the engine **strategy-agnostic**: it consumes bars + signal arrays + stop/target arrays and simulates fills. All the *strategy* math (when to enter, where to put stops) lives in the caller. - Implement the **intra-bar sub-tick model** (doc 03) and the **pessimistic ordering** convention. + This is the default bar-level exit mode — fast and exact for clean-directional setups. +- **If the EA moves its SL during a trade** (break-even, trailing, basket trailing), the bar-level + engine is NOT trustworthy: it produces a −40% to −50% net gap vs MT5 even in a calm window (doc 03 + §7 measured failure mode). You MUST implement the **M1 tick-level exit simulation** path: load M1 + bars covering the same window as the signal bars, pass them to `engine.run(..., m1_bars=m1_bars)`, + and the engine walks 4 synthetic ticks per M1 bar inside each higher-TF bar (direction-aware + order), separating the BE-update tick from the SL-trigger tick. This brings the gap to ~−5% net. + The engine should support BOTH modes and switch on whether `m1_bars` is provided. - Match the EA's **lot/money mode**, **spread model**, and **swap model** exactly (doc 05). - The first milestone is **1:1 fidelity on one known preset**: run the EA in MT5 on a short period, run your Python engine on the same data/preset, and reconcile trade-by-trade until the numbers - line up within the expected gap (doc 03 "Fidelity" section). **Do not optimize anything until this - passes** — an unvalidated engine optimizes noise. + line up within the **target gate for the EA's class** (doc 03 §8 table): + - Clean-directional (SL not moved intra-trade), bar-level: ≤ ~2% net gap. + - BE / trailing, **bar-level**: unattainable — do not chase this, switch to M1 tick-level. + - BE / trailing, **M1 tick-level**: ≤ ~10% net gap (residual spread/tick-path differences). + **Do not optimize anything until this passes** — an unvalidated engine optimizes noise. A + trailing/BE EA validated only on the bar-level engine is a silently-broken engine. > The bundled docs use a **grid martingale** engine as the worked example because it exercises every > hard case (pending orders, averaging, trailing on the basket, simultaneous closes). Your EA may be @@ -172,6 +184,10 @@ the lab is working. - **Never touch a validated engine to test an idea.** Fork it; prove the fork == original with the change disabled; only then test. (Doc 04.) +- **For trailing/BE EAs, M1 tick-level exit simulation is mandatory.** A trailing/BE EA validated + only on the bar-level engine has a −40% to −50% hidden gap vs MT5 — it is *not* a validated engine, + no matter how good the numbers look. Always pass `m1_bars=` to the engine for EAs that move their + SL intra-trade. (Doc 03 §7/§8.) - **Heavy runs go in the background.** A full-history A/B or a full Optuna study is minutes of compute — start it detached and poll, never block. Smoke-test first. (Doc 06.) - **Don't promote partial searches.** A finalist must come from a *completed* search. An interrupted diff --git a/GoldScalperPro.ex5 b/GoldScalperPro.ex5 new file mode 100644 index 0000000..f327f11 Binary files /dev/null and b/GoldScalperPro.ex5 differ diff --git a/GoldScalperPro.mq5 b/GoldScalperPro.mq5 new file mode 100644 index 0000000..01afb8a --- /dev/null +++ b/GoldScalperPro.mq5 @@ -0,0 +1,580 @@ +//+------------------------------------------------------------------+ +//| GoldScalperPro.mq5 | +//| | +//| A dedicated XAUUSD (gold) scalping Expert Advisor. | +//| | +//| Strategy (trend-filtered momentum pullback) | +//| ------------------------------------------ | +//| 1. A higher/slower EMA defines the prevailing trend, so the | +//| EA only ever trades WITH the dominant direction. | +//| 2. Inside that trend it waits for a short pullback: price | +//| dips back to the fast EMA and RSI leaves an oversold | +//| (long) / overbought (short) extreme - i.e. it buys dips in | +//| an uptrend and sells rallies in a downtrend. | +//| 3. An ATR filter makes sure there is enough volatility to pay | +//| for the spread, and an ATR-based stop/target adapts the | +//| trade size to current gold volatility. | +//| 4. Position size is derived from a fixed % risk of equity, so | +//| a small account never over-leverages on a single trade. | +//| 5. Hard daily-loss and daily-profit circuit breakers, a max | +//| trades-per-day cap, a spread guard and a trading-session | +//| window keep the scalper out of bad conditions. | +//| | +//| This EA is completely independent of any other strategy and | +//| manages only its own orders (identified by the magic number). | +//| | +//| All times are broker/server time. | +//+------------------------------------------------------------------+ +#property copyright "Sam Watts" +#property version "1.00" +#property strict +#property description "Trend-filtered momentum pullback scalper for XAUUSD (gold)." + +#include +#include + +//--- Position sizing mode +enum ENUM_SIZING_MODE + { + SIZE_FIXED_LOT, // Fixed lot size + SIZE_RISK_PERCENT // Risk a % of equity per trade + }; + +//--- Stop loss / take profit calculation mode +enum ENUM_STOP_MODE + { + STOP_ATR, // ATR multiple (adapts to volatility) + STOP_POINTS // Fixed distance in points + }; + +//+------------------------------------------------------------------+ +//| Inputs | +//+------------------------------------------------------------------+ +input group "=== 策略 / 信号 ===" +input ENUM_TIMEFRAMES InpTimeframe = PERIOD_M5; // 工作时间框架 +input int InpFastEmaPeriod = 21; // 快速EMA (回调价位) +input int InpSlowEmaPeriod = 100; // 慢速EMA (趋势过滤器) +input int InpRsiPeriod = 14; // RSI周期 +input double InpRsiBuyLevel = 45.0; // 当RSI回升至该值上方时买入 +input double InpRsiSellLevel = 55.0; // 当RSI跌破该值下方时卖出 +input double InpPullbackAtrMult = 2.0; // 价格与快速EMA的最大允许距离 (x ATR) + +input group "=== 波动性 / 过滤器 ===" +input int InpAtrPeriod = 14; // ATR周期 +input int InpMinAtrPoints = 0; // ATR低于此值时跳过 (点数, 0 = 忽略) +input double InpMaxSpreadAtrPct = 25.0; // 最大价差占ATR百分比 (0 = 忽略) + +input group "=== 仓位计算 ===" +input ENUM_SIZING_MODE InpSizingMode = SIZE_RISK_PERCENT; // 仓位计算方式 +input double InpFixedLots = 0.01; // 固定手数 (固定手数模式) +input double InpRiskPercent = 1.0; // 每笔交易风险百分比 (账户权益) + +input group "=== 止损 / 止盈 ===" +input ENUM_STOP_MODE InpStopMode = STOP_ATR; // 止损/止盈计算模式 +input double InpAtrSLMult = 1.5; // 止损 = ATR x 此值 +input double InpAtrTPMult = 2.0; // 止盈 = ATR x 此值 +input int InpStopLossPoints = 200; // 止损 (点数, 固定模式) +input int InpTakeProfitPoints = 300; // 止盈 (点数, 固定模式) +input bool InpUseBreakEven = true; // 移动止损到盈亏平衡点 +input int InpBreakEvenPoints = 150; // 触发盈亏平衡的利润点数 +input int InpBreakEvenLock = 20; // 盈亏平衡时锁定的点数 +input bool InpUseTrailing = true; // 使用移动止损 +input int InpTrailStartPoints = 200; // 开始移动止损的利润点数 +input int InpTrailStepPoints = 120; // 移动止损距离 (点数) + +input group "=== 交易控制 / 风险限制 ===" +input int InpMaxPositions = 1; // 最大持仓数 (本EA) +input int InpMaxTradesPerDay = 6; // 每日最大交易次数 (0 = 不限制) +input double InpDailyLossLimit = 5.0; // 亏损达到此 equity百分比时停止交易 (0 = 关闭) +input double InpDailyProfitTarget = 0.0; // 盈利达到此equity百分比时停止交易 (0 = 关闭) +input int InpMinSecondsBetween = 60; // 最小交易间隔秒数 + +input group "=== 交易时段 (服务器时间) ===" +input bool InpUseSession = true; // 限制交易时段 +input int InpSessionStartHour = 7; // 时段开始小时 (0-23) +input int InpSessionEndHour = 20; // 时段结束小时 (0-23) + +input group "=== 常规 ===" +input long InpMagicNumber = 20240530; // 魔术号码 +input string InpComment = "GoldScalperPro"; // 订单注释 + +//+------------------------------------------------------------------+ +//| Globals | +//+------------------------------------------------------------------+ +CTrade trade; +CPositionInfo posInfo; + +int g_fastEmaHandle = INVALID_HANDLE; +int g_slowEmaHandle = INVALID_HANDLE; +int g_rsiHandle = INVALID_HANDLE; +int g_atrHandle = INVALID_HANDLE; + +datetime g_lastBarTime = 0; // last processed bar of the working timeframe +datetime g_currentDay = 0; // day (00:00) the daily counters belong to +datetime g_lastTradeTime = 0; // time of the last entry +int g_tradesToday = 0; // entries opened today +double g_dayStartEquity = 0.0; // equity at the start of the trading day +bool g_dayBlocked = false; // daily circuit breaker tripped + +//+------------------------------------------------------------------+ +//| Expert initialization | +//+------------------------------------------------------------------+ +int OnInit() + { + trade.SetExpertMagicNumber(InpMagicNumber); + trade.SetTypeFillingBySymbol(_Symbol); + trade.SetDeviationInPoints(20); + + if(InpFastEmaPeriod <= 0 || InpSlowEmaPeriod <= 0 || + InpFastEmaPeriod >= InpSlowEmaPeriod) + { + Print("Fast EMA period must be > 0 and smaller than the slow EMA period."); + return(INIT_PARAMETERS_INCORRECT); + } + if(InpRsiPeriod <= 0 || InpAtrPeriod <= 0) + { + Print("RSI and ATR periods must be greater than zero."); + return(INIT_PARAMETERS_INCORRECT); + } + if(InpSizingMode == SIZE_FIXED_LOT && InpFixedLots <= 0.0) + { + Print("Fixed lot size must be greater than zero."); + return(INIT_PARAMETERS_INCORRECT); + } + if(InpSizingMode == SIZE_RISK_PERCENT && InpRiskPercent <= 0.0) + { + Print("Risk percent must be greater than zero."); + return(INIT_PARAMETERS_INCORRECT); + } + + g_fastEmaHandle = iMA(_Symbol, InpTimeframe, InpFastEmaPeriod, 0, MODE_EMA, PRICE_CLOSE); + g_slowEmaHandle = iMA(_Symbol, InpTimeframe, InpSlowEmaPeriod, 0, MODE_EMA, PRICE_CLOSE); + g_rsiHandle = iRSI(_Symbol, InpTimeframe, InpRsiPeriod, PRICE_CLOSE); + g_atrHandle = iATR(_Symbol, InpTimeframe, InpAtrPeriod); + + if(g_fastEmaHandle == INVALID_HANDLE || g_slowEmaHandle == INVALID_HANDLE || + g_rsiHandle == INVALID_HANDLE || g_atrHandle == INVALID_HANDLE) + { + Print("Failed to create one or more indicator handles."); + return(INIT_FAILED); + } + + ResetDailyCounters(DayStart(TimeCurrent())); + + PrintFormat("GoldScalperPro initialised on %s (%s) | magic %I64d", + _Symbol, EnumToString(InpTimeframe), InpMagicNumber); + return(INIT_SUCCEEDED); + } + +//+------------------------------------------------------------------+ +//| Expert deinitialization | +//+------------------------------------------------------------------+ +void OnDeinit(const int reason) + { + if(g_fastEmaHandle != INVALID_HANDLE) IndicatorRelease(g_fastEmaHandle); + if(g_slowEmaHandle != INVALID_HANDLE) IndicatorRelease(g_slowEmaHandle); + if(g_rsiHandle != INVALID_HANDLE) IndicatorRelease(g_rsiHandle); + if(g_atrHandle != INVALID_HANDLE) IndicatorRelease(g_atrHandle); + Comment(""); + } + +//+------------------------------------------------------------------+ +//| Expert tick | +//+------------------------------------------------------------------+ +void OnTick() + { + datetime now = TimeCurrent(); + + //--- New trading day: reset the daily counters / circuit breaker. + datetime today = DayStart(now); + if(today != g_currentDay) + ResetDailyCounters(today); + + //--- Manage what is already open on every tick (responsive exits). + ManageOpenPositions(); + + //--- Trip / hold the daily circuit breaker. + CheckDailyLimits(); + + //--- Only evaluate fresh signals once per closed bar. + datetime barTime = (datetime)SeriesInfoInteger(_Symbol, InpTimeframe, SERIES_LASTBAR_DATE); + if(barTime == g_lastBarTime) + { + UpdateDashboard(); + return; + } + g_lastBarTime = barTime; + + EvaluateEntry(); + UpdateDashboard(); + } + +//+------------------------------------------------------------------+ +//| Reset the per-day counters and snapshot starting equity | +//+------------------------------------------------------------------+ +void ResetDailyCounters(const datetime today) + { + g_currentDay = today; + g_tradesToday = 0; + g_dayBlocked = false; + g_dayStartEquity = AccountInfoDouble(ACCOUNT_EQUITY); + } + +//+------------------------------------------------------------------+ +//| Daily loss / profit circuit breaker | +//+------------------------------------------------------------------+ +void CheckDailyLimits() + { + if(g_dayBlocked) + return; + if(g_dayStartEquity <= 0.0) + return; + + double equity = AccountInfoDouble(ACCOUNT_EQUITY); + double pct = (equity - g_dayStartEquity) / g_dayStartEquity * 100.0; + + if(InpDailyLossLimit > 0.0 && pct <= -InpDailyLossLimit) + { + g_dayBlocked = true; + PrintFormat("Daily loss limit hit (%.2f%%). Trading halted for the day.", pct); + } + else if(InpDailyProfitTarget > 0.0 && pct >= InpDailyProfitTarget) + { + g_dayBlocked = true; + PrintFormat("Daily profit target hit (%.2f%%). Trading halted for the day.", pct); + } + } + +//+------------------------------------------------------------------+ +//| Evaluate the entry signal on the latest closed bar | +//+------------------------------------------------------------------+ +void EvaluateEntry() + { + //--- Respect all the gates before doing any work. + if(g_dayBlocked) + return; + if(InpUseSession && !InSession()) + return; + if(InpMaxTradesPerDay > 0 && g_tradesToday >= InpMaxTradesPerDay) + return; + if(CountOpenPositions() >= InpMaxPositions) + return; + if(g_lastTradeTime > 0 && (TimeCurrent() - g_lastTradeTime) < InpMinSecondsBetween) + return; + + //--- Pull indicator values for the just-closed bar and the previous one + //--- so we can detect an RSI cross. Arrays are set as time-series, so + //--- index 0 = most recent (shift 1) and index 1 = the bar before it. + double fastEma[], slowEma[], rsi[], atr[]; + ArraySetAsSeries(fastEma, true); + ArraySetAsSeries(slowEma, true); + ArraySetAsSeries(rsi, true); + ArraySetAsSeries(atr, true); + + if(CopyBuffer(g_fastEmaHandle, 0, 1, 2, fastEma) < 2) return; + if(CopyBuffer(g_slowEmaHandle, 0, 1, 2, slowEma) < 2) return; + if(CopyBuffer(g_rsiHandle, 0, 1, 2, rsi) < 2) return; + if(CopyBuffer(g_atrHandle, 0, 1, 1, atr) < 1) return; + + double atrNow = atr[0]; + double fastNow = fastEma[0]; + double slowNow = slowEma[0]; + double rsiNow = rsi[0]; // last closed bar + double rsiPrev = rsi[1]; // the bar before it + + if(atrNow <= 0.0) + return; + if(InpMinAtrPoints > 0 && (atrNow / _Point) < InpMinAtrPoints) + return; + if(!SpreadOK(atrNow)) + return; + + double closePrice = iClose(_Symbol, InpTimeframe, 1); + if(closePrice <= 0.0) + return; + + bool trendUp = (fastNow > slowNow) && (closePrice > slowNow); + bool trendDown = (fastNow < slowNow) && (closePrice < slowNow); + + //--- How far price has pulled back from the fast EMA, measured in ATR so + //--- it is independent of the symbol's digits / point size. + double distToFast = MathAbs(closePrice - fastNow); + bool nearFast = (distToFast <= InpPullbackAtrMult * atrNow); + + //--- Long: uptrend, price near the fast EMA, RSI turning back UP through + //--- the buy level (momentum returning after a dip). + bool buySignal = trendUp && nearFast && + rsiPrev < InpRsiBuyLevel && rsiNow >= InpRsiBuyLevel; + + //--- Short: downtrend, price near the fast EMA, RSI turning back DOWN + //--- through the sell level. + bool sellSignal = trendDown && nearFast && + rsiPrev > InpRsiSellLevel && rsiNow <= InpRsiSellLevel; + + if(buySignal) + OpenTrade(ORDER_TYPE_BUY, atrNow); + else if(sellSignal) + OpenTrade(ORDER_TYPE_SELL, atrNow); + } + +//+------------------------------------------------------------------+ +//| Open a market order with ATR/points based SL & TP | +//+------------------------------------------------------------------+ +void OpenTrade(const ENUM_ORDER_TYPE type, const double atrValue) + { + double ask = SymbolInfoDouble(_Symbol, SYMBOL_ASK); + double bid = SymbolInfoDouble(_Symbol, SYMBOL_BID); + + double slDist = StopDistance(InpStopMode == STOP_ATR ? InpAtrSLMult : InpStopLossPoints, atrValue); + double tpDist = StopDistance(InpStopMode == STOP_ATR ? InpAtrTPMult : InpTakeProfitPoints, atrValue); + + //--- Respect the broker's minimum stop distance. + double minStop = (double)SymbolInfoInteger(_Symbol, SYMBOL_TRADE_STOPS_LEVEL) * _Point; + if(slDist < minStop) slDist = minStop; + if(tpDist < minStop) tpDist = minStop; + if(slDist <= 0.0) + { + Print("Computed stop distance is zero - aborting entry."); + return; + } + + double price = (type == ORDER_TYPE_BUY) ? ask : bid; + double sl, tp; + if(type == ORDER_TYPE_BUY) + { + sl = NormalizeDouble(price - slDist, _Digits); + tp = NormalizeDouble(price + tpDist, _Digits); + } + else + { + sl = NormalizeDouble(price + slDist, _Digits); + tp = NormalizeDouble(price - tpDist, _Digits); + } + + double lots = CalcLots(slDist); + if(lots <= 0.0) + { + Print("Computed lot size is zero - aborting entry."); + return; + } + + bool ok = (type == ORDER_TYPE_BUY) + ? trade.Buy(lots, _Symbol, price, sl, tp, InpComment) + : trade.Sell(lots, _Symbol, price, sl, tp, InpComment); + + if(ok) + { + g_tradesToday++; + g_lastTradeTime = TimeCurrent(); + PrintFormat("%s %.2f lots @ %.*f SL %.*f TP %.*f (trade %d/%d today)", + (type == ORDER_TYPE_BUY ? "BUY" : "SELL"), lots, + _Digits, price, _Digits, sl, _Digits, tp, + g_tradesToday, InpMaxTradesPerDay); + } + else + PrintFormat("Order failed: %d - %s", + trade.ResultRetcode(), trade.ResultRetcodeDescription()); + } + +//+------------------------------------------------------------------+ +//| Convert an SL/TP setting to a price distance | +//+------------------------------------------------------------------+ +double StopDistance(const double value, const double atrValue) + { + if(value <= 0.0) + return(0.0); + if(InpStopMode == STOP_ATR) + return(value * atrValue); + return(value * _Point); // STOP_POINTS + } + +//+------------------------------------------------------------------+ +//| Position size from fixed lot or % risk of equity | +//+------------------------------------------------------------------+ +double CalcLots(const double slDistance) + { + if(InpSizingMode == SIZE_FIXED_LOT) + return(NormalizeLots(InpFixedLots)); + + //--- Risk-percent sizing: lots = riskMoney / (slDistance valued per lot). + double equity = AccountInfoDouble(ACCOUNT_EQUITY); + double riskMoney = equity * InpRiskPercent / 100.0; + + double tickValue = SymbolInfoDouble(_Symbol, SYMBOL_TRADE_TICK_VALUE); + double tickSize = SymbolInfoDouble(_Symbol, SYMBOL_TRADE_TICK_SIZE); + if(tickValue <= 0.0 || tickSize <= 0.0) + return(NormalizeLots(InpFixedLots)); + + double lossPerLot = slDistance / tickSize * tickValue; + if(lossPerLot <= 0.0) + return(NormalizeLots(InpFixedLots)); + + double lots = riskMoney / lossPerLot; + return(NormalizeLots(lots)); + } + +//+------------------------------------------------------------------+ +//| Break-even and trailing-stop management for our positions | +//+------------------------------------------------------------------+ +void ManageOpenPositions() + { + if(!InpUseBreakEven && !InpUseTrailing) + return; + + for(int i = PositionsTotal() - 1; i >= 0; i--) + { + ulong ticket = PositionGetTicket(i); + if(ticket == 0) + continue; + if(!posInfo.SelectByTicket(ticket)) + continue; + if(posInfo.Symbol() != _Symbol || posInfo.Magic() != InpMagicNumber) + continue; + + long type = posInfo.PositionType(); + double openPrice = posInfo.PriceOpen(); + double curSL = posInfo.StopLoss(); + double curTP = posInfo.TakeProfit(); + double bid = SymbolInfoDouble(_Symbol, SYMBOL_BID); + double ask = SymbolInfoDouble(_Symbol, SYMBOL_ASK); + double newSL = curSL; + + if(type == POSITION_TYPE_BUY) + { + double profitPts = (bid - openPrice) / _Point; + + if(InpUseBreakEven && profitPts >= InpBreakEvenPoints) + { + double be = NormalizeDouble(openPrice + InpBreakEvenLock * _Point, _Digits); + if(be > newSL) + newSL = be; + } + if(InpUseTrailing && profitPts >= InpTrailStartPoints) + { + double trail = NormalizeDouble(bid - InpTrailStepPoints * _Point, _Digits); + if(trail > newSL) + newSL = trail; + } + if(newSL > curSL && newSL < bid) + trade.PositionModify(ticket, newSL, curTP); + } + else if(type == POSITION_TYPE_SELL) + { + double profitPts = (openPrice - ask) / _Point; + + if(InpUseBreakEven && profitPts >= InpBreakEvenPoints) + { + double be = NormalizeDouble(openPrice - InpBreakEvenLock * _Point, _Digits); + if(curSL == 0.0 || be < newSL) + newSL = be; + } + if(InpUseTrailing && profitPts >= InpTrailStartPoints) + { + double trail = NormalizeDouble(ask + InpTrailStepPoints * _Point, _Digits); + if(curSL == 0.0 || trail < newSL) + newSL = trail; + } + if(newSL != curSL && (curSL == 0.0 || newSL < curSL) && newSL > ask) + trade.PositionModify(ticket, newSL, curTP); + } + } + } + +//+------------------------------------------------------------------+ +//| Count this EA's open positions on this symbol | +//+------------------------------------------------------------------+ +int CountOpenPositions() + { + int count = 0; + for(int i = PositionsTotal() - 1; i >= 0; i--) + { + ulong ticket = PositionGetTicket(i); + if(ticket == 0) + continue; + if(!posInfo.SelectByTicket(ticket)) + continue; + if(posInfo.Symbol() == _Symbol && posInfo.Magic() == InpMagicNumber) + count++; + } + return(count); + } + +//+------------------------------------------------------------------+ +//| True while the clock is inside the trading session window | +//+------------------------------------------------------------------+ +bool InSession() + { + MqlDateTime st; + TimeToStruct(TimeCurrent(), st); + int hour = st.hour; + + if(InpSessionStartHour == InpSessionEndHour) + return(true); // 24h + if(InpSessionStartHour < InpSessionEndHour) + return(hour >= InpSessionStartHour && hour < InpSessionEndHour); + //--- window that wraps past midnight + return(hour >= InpSessionStartHour || hour < InpSessionEndHour); + } + +//+------------------------------------------------------------------+ +//| Spread check (relative to ATR, so it works on any gold symbol) | +//+------------------------------------------------------------------+ +bool SpreadOK(const double atrValue) + { + if(InpMaxSpreadAtrPct <= 0.0 || atrValue <= 0.0) + return(true); + double spreadPrice = (double)SymbolInfoInteger(_Symbol, SYMBOL_SPREAD) * _Point; + return(spreadPrice <= atrValue * InpMaxSpreadAtrPct / 100.0); + } + +//+------------------------------------------------------------------+ +//| Normalize lots to the symbol's volume constraints | +//+------------------------------------------------------------------+ +double NormalizeLots(double lots) + { + double minLot = SymbolInfoDouble(_Symbol, SYMBOL_VOLUME_MIN); + double maxLot = SymbolInfoDouble(_Symbol, SYMBOL_VOLUME_MAX); + double lotStep = SymbolInfoDouble(_Symbol, SYMBOL_VOLUME_STEP); + + if(lotStep > 0.0) + lots = MathFloor(lots / lotStep) * lotStep; + if(lots < minLot) lots = minLot; + if(lots > maxLot) lots = maxLot; + return(lots); + } + +//+------------------------------------------------------------------+ +//| Midnight (00:00) of the day a timestamp belongs to | +//+------------------------------------------------------------------+ +datetime DayStart(const datetime t) + { + return(t - (t % 86400)); + } + +//+------------------------------------------------------------------+ +//| On-chart status read-out | +//+------------------------------------------------------------------+ +void UpdateDashboard() + { + double equity = AccountInfoDouble(ACCOUNT_EQUITY); + double dayPct = (g_dayStartEquity > 0.0) + ? (equity - g_dayStartEquity) / g_dayStartEquity * 100.0 : 0.0; + long spread = SymbolInfoInteger(_Symbol, SYMBOL_SPREAD); + + string state = g_dayBlocked ? "HALTED (daily limit)" + : (InpUseSession && !InSession()) ? "outside session" : "active"; + + string txt = StringFormat( + "GoldScalperPro [%s %s]\n" + "State: %s\n" + "Open positions: %d / %d\n" + "Trades today: %d / %d\n" + "Day P/L: %.2f%%\n" + "Spread: %d pts", + _Symbol, EnumToString(InpTimeframe), + state, CountOpenPositions(), InpMaxPositions, + g_tradesToday, InpMaxTradesPerDay, + dayPct, (int)spread); + Comment(txt); + } +//+------------------------------------------------------------------+ diff --git a/_test_set_gen.py b/_test_set_gen.py new file mode 100644 index 0000000..a484a0a --- /dev/null +++ b/_test_set_gen.py @@ -0,0 +1,44 @@ +"""Dry-run: generate .set + .ini with frozen baseline, verify format.""" +import sys +from pathlib import Path +PROJECT = Path(__file__).resolve().parent +sys.path.insert(0, str(PROJECT)) + +from shared.config import get_secret, load_env +from shared.mt5_pipeline.ini_gen import TesterConfig, write_tester_ini, MODEL_OHLC +from shared.mt5_pipeline.set_gen import write_set_file +from strategies.gold_scalper_pro.search_space import FROZEN_BASELINE +from strategies.gold_scalper_pro.set_mappings import GOLD_SCALPER_MAPPINGS + +load_env(PROJECT) +mt5_data = Path(get_secret("MT5_DATA_PATH")) +profiles = mt5_data / "MQL5" / "Profiles" / "Tester" +profiles.mkdir(parents=True, exist_ok=True) + +set_path = profiles / "GoldScalperPro_dryrun.set" +write_set_file(FROZEN_BASELINE, GOLD_SCALPER_MAPPINGS, set_path) +print(f"set written: {set_path}") +print(f" size: {set_path.stat().st_size} bytes") +print(f" first bytes: {set_path.read_bytes()[:40]}") + +ini_path = profiles / "GoldScalperPro_dryrun.ini" +tcfg = TesterConfig( + expert=r"Experts\GoldScalperPro.ex5", + symbol="XAUUSD", + period="M5", + model=MODEL_OHLC, + from_date="2024.06.26", + to_date="2026.06.26", + deposit=10000.0, + leverage=100, + report="report_dryrun", + shutdown_terminal=True, + set_file=str(set_path), + login=int(get_secret("MT5_DEMO_LOGIN")), + password=get_secret("MT5_DEMO_PASSWORD"), + server=get_secret("MT5_DEMO_SERVER"), +) +write_tester_ini(tcfg, ini_path) +print(f"\nini written: {ini_path}") +print(f"--- contents ---") +print(ini_path.read_text(encoding="utf-8")) diff --git a/registry/.gitkeep b/registry/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/reports/ReportTester-52845377-holding.png b/reports/ReportTester-52845377-holding.png new file mode 100644 index 0000000..7b569b4 Binary files /dev/null and b/reports/ReportTester-52845377-holding.png differ diff --git a/reports/ReportTester-52845377-hst.png b/reports/ReportTester-52845377-hst.png new file mode 100644 index 0000000..bcbb05f Binary files /dev/null and b/reports/ReportTester-52845377-hst.png differ diff --git a/reports/ReportTester-52845377-mfemae.png b/reports/ReportTester-52845377-mfemae.png new file mode 100644 index 0000000..3d549a8 Binary files /dev/null and b/reports/ReportTester-52845377-mfemae.png differ diff --git a/reports/ReportTester-52845377.html b/reports/ReportTester-52845377.html new file mode 100644 index 0000000..6a3413a Binary files /dev/null and b/reports/ReportTester-52845377.html differ diff --git a/reports/ReportTester-52845377.png b/reports/ReportTester-52845377.png new file mode 100644 index 0000000..00fa562 Binary files /dev/null and b/reports/ReportTester-52845377.png differ diff --git a/run.py b/run.py new file mode 100644 index 0000000..123d7ad --- /dev/null +++ b/run.py @@ -0,0 +1,153 @@ +"""Phase 5+6 entry point: optimize GoldScalperPro on real XAUUSD bars. + +Assembles ObjectiveConfig (engine + signals + search space + constraints), +runs Optuna, applies the diverse top-N selector, then runs the robustness +layers on the finalists and prints a report. This is the cycle that Phase 7 +will feed into MT5 for verification. + +Usage: + python run.py [--trials 200] [--top-n 3] [--deposit 10000] +""" +from __future__ import annotations + +import argparse +import sys +import time +from pathlib import Path + +PROJECT = Path(__file__).resolve().parent +sys.path.insert(0, str(PROJECT)) + +import optuna + +from shared.core.engine import SizingInputs +from shared.data.loaders import load_bars +from shared.optimizer.objective import ( + Constraints, + ObjectiveConfig, + build_objective, +) +from shared.optimizer.selector import select_diverse_topn +from shared.robustness.layers import stability_region +from strategies.gold_scalper_pro.instruments import XAUUSD_REAL +from strategies.gold_scalper_pro.scalper_engine import ( + ScalperConfig, + ScalperEngine, + config_from_params, + engine_kwargs_from_params, +) +from strategies.gold_scalper_pro.search_space import ( + FROZEN_BASELINE, + INT_PARAMS, + SEARCH_SPACE, +) +from strategies.gold_scalper_pro.signals import build_signals + + +def find_bars_file() -> Path: + """Auto-find the latest XAUUSD M5 parquet in data/.""" + data_dir = PROJECT / "data" + candidates = sorted(data_dir.glob("XAUUSD_M5_*.parquet")) + if not candidates: + raise FileNotFoundError(f"no XAUUSD M5 parquet in {data_dir}") + return candidates[-1] + + +def main() -> int: + ap = argparse.ArgumentParser(description="Optimize GoldScalperPro.") + ap.add_argument("--trials", type=int, default=100, + help="Optuna trials (default 100)") + ap.add_argument("--top-n", type=int, default=3, + help="diverse finalists to select (default 3)") + ap.add_argument("--deposit", type=float, default=10000.0, + help="initial deposit (default 10000)") + ap.add_argument("--bars", type=str, default="", + help="path to parquet bars (blank = auto-find)") + args = ap.parse_args() + + bars_path = Path(args.bars) if args.bars else find_bars_file() + print(f"=== GoldScalperPro optimization ===") + print(f"bars : {bars_path.name}") + print(f"trials : {args.trials}") + print(f"top-n : {args.top_n}") + print(f"deposit : {args.deposit:,.0f} USD") + print() + + bars = load_bars(bars_path) + print(f"loaded {len(bars):,} bars {bars['timestamp'].iloc[0]} → {bars['timestamp'].iloc[-1]}") + + # ── Assemble the objective ──────────────────────────────────────────── + constraints = Constraints( + min_trades=25, + min_profit_factor=1.2, # relaxed for first pass; tightened later + max_equity_dd_pct=0.40, # 40% hard cap + ) + obj_cfg = ObjectiveConfig( + engine=ScalperEngine(), + bars=bars, + instrument=XAUUSD_REAL, + sizing=SizingInputs(), + initial_deposit=args.deposit, + search_space=SEARCH_SPACE, + int_params=INT_PARAMS, + frozen_baseline=FROZEN_BASELINE, + constraints=constraints, + dd_weight=1.0, + build_signals=build_signals, + build_engine_kwargs=engine_kwargs_from_params, + ) + objective = build_objective(obj_cfg) + + # ── Run Optuna ───────────────────────────────────────────────────────── + optuna.logging.set_verbosity(optuna.logging.WARNING) + study = optuna.create_study(direction="maximize", + sampler=optuna.samplers.TPESampler(seed=42)) + print(f"\nrunning {args.trials} trials ...") + t0 = time.time() + study.optimize(objective, n_trials=args.trials, show_progress_bar=False) + elapsed = time.time() - t0 + print(f"done in {elapsed:.1f}s ({elapsed/args.trials:.2f}s/trial)") + + # ── Report ──────────────────────────────────────────────────────────── + best = study.best_trial + print(f"\n=== best trial #{best.number} ===") + print(f" score : {best.value:+,.2f}") + print(f" net profit : {best.user_attrs['net_profit']:+,.2f}") + print(f" profit factor : {best.user_attrs['profit_factor']:.2f}") + print(f" trades : {best.user_attrs['total_trades']}") + print(f" win rate : {best.user_attrs['win_rate']:.2%}") + print(f" equity DD : {best.user_attrs['max_equity_dd']:,.2f} " + f"({best.user_attrs['max_equity_dd_pct']:.2%})") + print(f" sharpe : {best.user_attrs['sharpe']:.2f}") + if best.user_attrs.get("violations"): + print(f" violations : {best.user_attrs['violations']}") + print(" params:") + for k, v in best.user_attrs["params"].items(): + if k in SEARCH_SPACE: + print(f" {k:24s} = {v}") + + # ── Diverse top-N ───────────────────────────────────────────────────── + print(f"\n=== diverse top-{args.top_n} finalists ===") + finalists = select_diverse_topn(study, args.top_n, SEARCH_SPACE) + for i, t in enumerate(finalists, 1): + print(f" #{i} trial {t.number}: score={t.value:+,.2f} " + f"net={t.user_attrs['net_profit']:+,.2f} " + f"PF={t.user_attrs['profit_factor']:.2f} " + f"trades={t.user_attrs['total_trades']}") + + # ── Stability region (is the best on a plateau?) ───────────────────── + print(f"\n=== stability region ===") + sr = stability_region(study, SEARCH_SPACE) + print(f" passed : {sr.get('passed')}") + print(f" cluster_size : {sr.get('cluster_size')}") + print(f" best_in_cluster : {sr.get('best_in_cluster')}") + if sr.get("reason"): + print(f" reason : {sr['reason']}") + + print("\n=== done ===") + print("Next: Phase 7 — generate .set/.ini for each finalist, run MT5, compare.") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/compare_dryrun.py b/scripts/compare_dryrun.py new file mode 100644 index 0000000..1374270 --- /dev/null +++ b/scripts/compare_dryrun.py @@ -0,0 +1,114 @@ +"""Compare the dry-run MT5 report against Python on the SAME window + deposit. + +The dry-run MT5 report shows the tester was run on 2026.04.16 - 2026.05.08 +with initial deposit 1000 USD. To make a fair Python-vs-MT5 comparison we +re-slice the bars to that exact window and re-run the engine with the same +deposit. Then we print the side-by-side table. +""" +from __future__ import annotations + +import sys +from pathlib import Path + +PROJECT = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(PROJECT)) + +import pandas as pd + +from shared.core.engine import SizingInputs +from shared.core.metrics import compute_metrics +from shared.data.loaders import load_bars +from shared.data.mt5_report import parse_mt5_report +from shared.mt5_pipeline.compare import build_comparison_table +from strategies.gold_scalper_pro.instruments import XAUUSD_REAL +from strategies.gold_scalper_pro.scalper_engine import ( + ScalperEngine, + engine_kwargs_from_params, +) +from strategies.gold_scalper_pro.search_space import FROZEN_BASELINE +from strategies.gold_scalper_pro.signals import build_signals + + +def main() -> int: + # ── Parse the MT5 report ────────────────────────────────────────────── + report = PROJECT / "reports" / "ReportTester-52845377.html" + mt5 = parse_mt5_report(report) + print("=== MT5 report (dry run) ===") + for k, v in mt5.items(): + if k.startswith("_"): + continue + print(f" {k:24s}: {v}") + # The MT5 window + deposit (parsed from the report's _all dict). + all_fields = mt5["_all"] + window_label = all_fields.get("期间:", "") + print(f" window (raw) : {window_label}") + deposit = mt5.get("Initial Deposit") or 1000 + print(f" initial deposit: {deposit}") + + # ── Slice Python bars to the same window ───────────────────────────── + bars = load_bars(PROJECT / "data" / "XAUUSD_M5_2024-06-26_2026-06-26.parquet") + # MT5 tester's ToDate is exclusive of the day (uses 00:00 of that day), + # so the actual data ends at 2026.05.08 00:00 (not 23:59). Match it exactly. + start = pd.Timestamp("2026-04-16 00:00:00") + end = pd.Timestamp("2026-05-08 00:00:00") + window = bars[(bars["timestamp"] >= start) & (bars["timestamp"] < end)].reset_index(drop=True) + print(f"\n=== Python (matched window) ===") + print(f" bars : {len(window):,}") + print(f" window : {window['timestamp'].iloc[0]} → {window['timestamp'].iloc[-1]}") + print(f" deposit : {deposit}") + + # ── Run the engine on the matched window ───────────────────────────── + # Override InpAtrPeriod to 15 to match the MT5 manual run (user changed + # it from the .set's 14 to 15 in the tester UI). RSI stays at 14. + params = dict(FROZEN_BASELINE) + params["InpAtrPeriod"] = 15 + pack = build_signals(params, window, XAUUSD_REAL) + # Load M1 data for tick-level exit simulation (closes the bar-level + # optimism gap on BE/trailing — doc 03 §7). + m1_path = PROJECT / "data" / "XAUUSD_M1_2024-06-26_2026-06-26.parquet" + m1_bars = load_bars(m1_path) if m1_path.exists() else None + if m1_bars is not None: + # Slice M1 to the same window as M5 (exclusive end, matching MT5). + m1_bars = m1_bars[ + (m1_bars["timestamp"] >= start) & (m1_bars["timestamp"] < end) + ].reset_index(drop=True) + print(f" m1 bars : {len(m1_bars):,} (tick-level exit simulation ON)") + engine = ScalperEngine() + result = engine.run( + window, pack.signals_long, pack.signals_short, + pack.sl_prices, pack.tp_prices, + XAUUSD_REAL, SizingInputs(), float(deposit), + m1_bars=m1_bars, + **engine_kwargs_from_params(params), + ) + py = compute_metrics(result) + print(f" trades : {py.total_trades}") + print(f" net : {py.net_profit:+.2f}") + print(f" PF : {py.profit_factor:.2f}") + print(f" DD : {py.max_equity_dd:.2f} ({py.max_equity_dd_pct:.2%})") + + # ── Build the comparison table ─────────────────────────────────────── + # Map Python Metrics → keys the compare table expects. + py_mapped = { + "net_profit": py.net_profit, + "profit_factor": py.profit_factor, + "total_trades": py.total_trades, + "max_equity_dd": py.max_equity_dd, + "win_rate": py.win_rate, + "sharpe": py.sharpe, + } + mt5_mapped = { + "net_profit": mt5.get("Total Net Profit"), + "profit_factor": mt5.get("Profit Factor"), + "total_trades": mt5.get("Total Trades"), + "max_equity_dd": mt5.get("Equity Drawdown Maximal"), + "win_rate": None, + "sharpe": mt5.get("Sharpe Ratio"), + } + print(f"\n=== Python vs MT5 (matched window {start.date()} → {end.date()}) ===") + print(build_comparison_table(py_mapped, mt5_mapped)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/diag_exit.py b/scripts/diag_exit.py new file mode 100644 index 0000000..1b85bf1 --- /dev/null +++ b/scripts/diag_exit.py @@ -0,0 +1,63 @@ +"""Diagnose the exit-side PnL gap between Python and MT5. + +MT5: gross profit 185.01, gross loss -123.78, 92 trades, net 61.23. +Python: 92 trades, net 118.79. So Python's gross profit is much higher +OR its gross loss is much smaller. Find out which by dumping Python's +gross profit / gross loss + per-reason breakdown. +""" +import sys +from pathlib import Path +PROJECT = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(PROJECT)) + +import pandas as pd +from shared.core.engine import SizingInputs +from shared.core.metrics import compute_metrics +from shared.data.loaders import load_bars +from strategies.gold_scalper_pro.instruments import XAUUSD_REAL +from strategies.gold_scalper_pro.scalper_engine import ScalperEngine, engine_kwargs_from_params +from strategies.gold_scalper_pro.search_space import FROZEN_BASELINE +from strategies.gold_scalper_pro.signals import build_signals +import collections + +bars = load_bars(PROJECT / "data" / "XAUUSD_M5_2024-06-26_2026-06-26.parquet") +start = pd.Timestamp("2026-04-16 00:00:00") +end = pd.Timestamp("2026-05-08 00:00:00") +window = bars[(bars["timestamp"] >= start) & (bars["timestamp"] < end)].reset_index(drop=True) + +params = dict(FROZEN_BASELINE) +params["InpAtrPeriod"] = 15 +pack = build_signals(params, window, XAUUSD_REAL) +engine = ScalperEngine() +result = engine.run(window, pack.signals_long, pack.signals_short, + pack.sl_prices, pack.tp_prices, XAUUSD_REAL, SizingInputs(), + 1000.0, **engine_kwargs_from_params(params)) + +wins = [t.pnl for t in result.trades if t.pnl > 0] +losses = [t.pnl for t in result.trades if t.pnl < 0] +print(f"=== Python exit-side breakdown ({len(result.trades)} trades) ===") +print(f" gross profit : {sum(wins):+.2f} ({len(wins)} trades)") +print(f" gross loss : {sum(losses):+.2f} ({len(losses)} trades)") +print(f" net : {sum(t.pnl for t in result.trades):+.2f}") +print() +print(f"=== MT5 (from report) ===") +print(f" gross profit : +185.01") +print(f" gross loss : -123.78") +print(f" net : +61.23") +print() + +by_reason = collections.defaultdict(list) +for t in result.trades: + by_reason[t.exit_reason].append(t.pnl) +print("=== Python PnL by exit reason ===") +for reason, pnls in sorted(by_reason.items()): + arr = __import__("numpy").array(pnls) + print(f" {reason:14s} n={len(arr):3d} sum={arr.sum():+8.2f} " + f"mean={arr.mean():+6.2f} min={arr.min():+7.2f} max={arr.max():+7.2f}") + +# Distribution of win sizes — MT5's avg win = 185.01 / n_wins; need n_wins from MT5. +# MT5 win rate unknown, but PF = GP/|GL| = 185.01/123.78 = 1.494 → matches. +print(f"\n=== Python win/loss sizes ===") +print(f" avg win : {sum(wins)/len(wins):+.2f} (n={len(wins)})") +print(f" avg loss : {sum(losses)/len(losses):+.2f} (n={len(losses)})") +print(f" Python PF: {sum(wins)/abs(sum(losses)):.2f}") diff --git a/scripts/download_xauusd_history.py b/scripts/download_xauusd_history.py new file mode 100644 index 0000000..178910d --- /dev/null +++ b/scripts/download_xauusd_history.py @@ -0,0 +1,87 @@ +"""Download XAUUSD M5 history to Parquet (doc 02 §3, doc 07 §6). + +Pulls the full M5 window from the running MT5 terminal and saves it as a +Parquet file in ``data/``. Re-runs of the engine then load from Parquet (fast, +compact, no MT5 connection needed). M5 is the EA's signal timeframe — higher +timeframes are resampled in Python from this M1/M5 base. + +Connects to the already-running, manually-logged-in terminal (initialize() +with no path → reuse). Keep the terminal UI open while this runs. +""" +from __future__ import annotations + +import sys +import time +from datetime import datetime, timedelta +from pathlib import Path + +PROJECT = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(PROJECT)) + +from shared.config import get_secret, load_env + +load_env(PROJECT) + +import MetaTrader5 as mt5 # type: ignore +import pandas as pd + +SYMBOL = "XAUUSD" +TIMEFRAME = "M5" +# Full window: 2 years back to today (matches the history depth probe). +START = (datetime.now() - timedelta(days=730)).strftime("%Y-%m-%d") +END = datetime.now().strftime("%Y-%m-%d") +OUT = PROJECT / "data" / f"{SYMBOL}_{TIMEFRAME}_{START}_{END}.parquet" + + +def connect(retries: int = 5, sleep_s: float = 5.0) -> bool: + """Connect to the running terminal (initialize() with no path → reuse).""" + for attempt in range(1, retries + 1): + if not mt5.initialize(): + print(f" initialize() attempt {attempt}/{retries} failed: {mt5.last_error()}") + time.sleep(sleep_s) + continue + login = int(get_secret("MT5_DEMO_LOGIN") or 0) + password = get_secret("MT5_DEMO_PASSWORD") + server = get_secret("MT5_DEMO_SERVER") + if not mt5.login(login, password=password, server=server): + print(f" login() attempt {attempt}/{retries} failed: {mt5.last_error()}") + mt5.shutdown() + time.sleep(sleep_s) + continue + print(f" connected: login={login} server={server}") + return True + return False + + +def main() -> int: + if not connect(): + print("could not connect — is the terminal running and logged in?") + return 1 + try: + tf = getattr(mt5, f"TIMEFRAME_{TIMEFRAME}") + print(f"downloading {SYMBOL} {TIMEFRAME} {START} → {END} ...") + rates = mt5.copy_rates_range( + SYMBOL, tf, pd.Timestamp(START), pd.Timestamp(END) + ) + if rates is None or len(rates) == 0: + print(f" no bars returned: {mt5.last_error()}") + return 2 + df = pd.DataFrame(rates) + df["timestamp"] = pd.to_datetime(df["time"], unit="s", utc=False) + df = df[["timestamp", "open", "high", "low", "close", "spread"]] + df = df.sort_values("timestamp").reset_index(drop=True) + # Deduplicate (MT5 occasionally returns overlapping bars near boundaries). + df = df.drop_duplicates(subset=["timestamp"]).reset_index(drop=True) + OUT.parent.mkdir(parents=True, exist_ok=True) + df.to_parquet(OUT, index=False) + print(f"saved {len(df):,} bars → {OUT.relative_to(PROJECT)}") + print(f"range: {df['timestamp'].iloc[0]} → {df['timestamp'].iloc[-1]}") + print(f"spread stats: min={df['spread'].min()} max={df['spread'].max()} " + f"median={df['spread'].median():.1f}") + return 0 + finally: + mt5.shutdown() + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/download_xauusd_m1.py b/scripts/download_xauusd_m1.py new file mode 100644 index 0000000..5cf5ebe --- /dev/null +++ b/scripts/download_xauusd_m1.py @@ -0,0 +1,80 @@ +"""Download XAUUSD M1 history to Parquet for tick-level simulation. + +M1 bars are the finest granularity MT5 exposes via copy_rates_range. We use +each M1 bar's OHLC as 4 synthetic ticks (open→high→low→close for longs, +open→low→high→close for shorts) to drive BE/trailing precisely inside +each M5 bar. + +Downloads the same 2-year window as the M5 file so the two align. +""" +from __future__ import annotations + +import sys +import time +from datetime import datetime, timedelta +from pathlib import Path + +PROJECT = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(PROJECT)) + +from shared.config import get_secret, load_env +load_env(PROJECT) + +import MetaTrader5 as mt5 # type: ignore +import pandas as pd + +SYMBOL = "XAUUSD" +TIMEFRAME = "M1" +START = (datetime.now() - timedelta(days=730)).strftime("%Y-%m-%d") +END = datetime.now().strftime("%Y-%m-%d") +OUT = PROJECT / "data" / f"{SYMBOL}_{TIMEFRAME}_{START}_{END}.parquet" + + +def connect(retries: int = 5, sleep_s: float = 5.0) -> bool: + for attempt in range(1, retries + 1): + if not mt5.initialize(): + print(f" initialize() attempt {attempt}/{retries} failed: {mt5.last_error()}") + time.sleep(sleep_s) + continue + login = int(get_secret("MT5_DEMO_LOGIN") or 0) + password = get_secret("MT5_DEMO_PASSWORD") + server = get_secret("MT5_DEMO_SERVER") + if not mt5.login(login, password=password, server=server): + print(f" login() attempt {attempt}/{retries} failed: {mt5.last_error()}") + mt5.shutdown() + time.sleep(sleep_s) + continue + print(f" connected: login={login} server={server}") + return True + return False + + +def main() -> int: + if not connect(): + print("could not connect — is the terminal running and logged in?") + return 1 + try: + tf = getattr(mt5, f"TIMEFRAME_{TIMEFRAME}") + print(f"downloading {SYMBOL} {TIMEFRAME} {START} → {END} ...") + rates = mt5.copy_rates_range( + SYMBOL, tf, pd.Timestamp(START), pd.Timestamp(END) + ) + if rates is None or len(rates) == 0: + print(f" no bars returned: {mt5.last_error()}") + return 2 + df = pd.DataFrame(rates) + df["timestamp"] = pd.to_datetime(df["time"], unit="s", utc=False) + df = df[["timestamp", "open", "high", "low", "close", "spread"]] + df = df.sort_values("timestamp").reset_index(drop=True) + df = df.drop_duplicates(subset=["timestamp"]).reset_index(drop=True) + OUT.parent.mkdir(parents=True, exist_ok=True) + df.to_parquet(OUT, index=False) + print(f"saved {len(df):,} bars → {OUT.relative_to(PROJECT)}") + print(f"range: {df['timestamp'].iloc[0]} → {df['timestamp'].iloc[-1]}") + return 0 + finally: + mt5.shutdown() + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/query_xauusd_spec.py b/scripts/query_xauusd_spec.py new file mode 100644 index 0000000..fc955e7 --- /dev/null +++ b/scripts/query_xauusd_spec.py @@ -0,0 +1,110 @@ +"""Query XAUUSD symbol spec from MT5 and dump it as an InstrumentConfig sketch. + +Run with the venv python. Uses the MetaTrader5 package (Topology A) to read +the symbol specification (digits, point, tick value, contract size, volume +steps) — these come from the broker, not guesses (doc 05 §1). Also downloads +a short M5 history window for the first validation pass (doc 03 §8). + +IPC-timeout workarounds (Stack Overflow #66492735): +- Use forward slashes in the terminal path (backslashes trigger -10005). +- Split initialize(path) and login() into two steps (one-shot initialize with + login/password/server is fragile). +- Allow reusing an already-running terminal instance (same path = same IPC). +""" +from __future__ import annotations + +import sys +import time +from pathlib import Path + +# Make the project importable when run as a script. +PROJECT = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(PROJECT)) + +from shared.config import get_secret, load_env + +load_env(PROJECT) + +import MetaTrader5 as mt5 # type: ignore +import pandas as pd + +# Forward slashes — backslashes in this path trigger IPC timeout (-10005). +TERMINAL = "C:/Program Files/MetaTrader 5 IC Markets Global/terminal64.exe" +SYMBOL = "XAUUSD" + + +def connect(retries: int = 3, sleep_s: float = 5.0) -> bool: + """Connect in two steps: initialize() → login(login,password,server). + + IMPORTANT: call initialize() with NO path argument. Passing a path makes + the package spawn a NEW headless terminal at that path, which then sits + waiting for an interactive login it can never complete (IPC timeout -10005). + Calling initialize() with no args CONNECTS to the already-running terminal + (the one with the UI you logged into manually). + """ + for attempt in range(1, retries + 1): + # Step 1: connect to the running terminal (no path → reuse existing). + if not mt5.initialize(): + err = mt5.last_error() + print(f" initialize() attempt {attempt}/{retries} failed: {err}") + time.sleep(sleep_s) + continue + # Step 2: explicit login with the demo account (re-auth is safe). + login = int(get_secret("MT5_DEMO_LOGIN") or 0) + password = get_secret("MT5_DEMO_PASSWORD") + server = get_secret("MT5_DEMO_SERVER") + if not mt5.login(login, password=password, server=server): + err = mt5.last_error() + print(f" login() attempt {attempt}/{retries} failed: {err}") + mt5.shutdown() + time.sleep(sleep_s) + continue + print(f" connected: login={login} server={server}") + return True + return False + + +def main() -> int: + if not connect(retries=5, sleep_s=10.0): + print("could not connect to MT5 after retries") + print("hint: confirm the demo account logs in manually in the terminal first;") + print(" if it does, the IPC issue is path/instance related, not account.") + return 1 + try: + info = mt5.symbol_info(SYMBOL) + if info is None: + print(f"symbol_info({SYMBOL}) returned None — is the symbol in Market Watch?") + return 2 + print(f"=== {SYMBOL} specification (IC Markets) ===") + fields = [ + "name", "digits", "point", "trade_tick_size", "trade_tick_value", + "trade_contract_size", "volume_min", "volume_step", "volume_max", + "spread", "trade_stops_level", "swap_mode", "swap_long", "swap_short", + "swap_rollover3days", + ] + for f in fields: + print(f" {f:24s} = {getattr(info, f, '')}") + # Triple-swap weekday: MT5 swap_rollover3days is 0=Mon..6=Sun. + + print("\n=== short M5 history probe (last 5 bars) ===") + rates = mt5.copy_rates_from_pos(SYMBOL, mt5.TIMEFRAME_M5, 0, 5) + if rates is None or len(rates) == 0: + print(" no rates returned") + else: + df = pd.DataFrame(rates) + df["timestamp"] = pd.to_datetime(df["time"], unit="s") + print(df[["timestamp", "open", "high", "low", "close", "spread"]].to_string(index=False)) + + print("\n=== available history depth (count bars in last 2 years) ===") + from datetime import datetime, timedelta + end = datetime.now() + start = end - timedelta(days=730) + n = mt5.copy_rates_range(SYMBOL, mt5.TIMEFRAME_M5, start, end) + print(f" M5 bars {start.date()} → {end.date()}: {0 if n is None else len(n)}") + return 0 + finally: + mt5.shutdown() + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/smoke_run.py b/scripts/smoke_run.py new file mode 100644 index 0000000..a5e5f16 --- /dev/null +++ b/scripts/smoke_run.py @@ -0,0 +1,90 @@ +"""Smoke test: run the scalper engine end-to-end on real XAUUSD bars. + +Uses the frozen baseline params (the saved .set config). Verifies the engine ++ signals + metrics produce a sane result (trades, PnL, drawdown) before we +wire the optimizer. This is the Phase 4 validation gate — not yet the MT5 +fidelity check (that's Phase 7). +""" +from __future__ import annotations + +import sys +from pathlib import Path + +PROJECT = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(PROJECT)) + +import numpy as np +import pandas as pd + +from shared.core.engine import SizingInputs +from shared.core.metrics import compute_metrics +from shared.data.loaders import load_bars +from strategies.gold_scalper_pro.instruments import XAUUSD_REAL +from strategies.gold_scalper_pro.scalper_engine import ScalperConfig, ScalperEngine +from strategies.gold_scalper_pro.search_space import FROZEN_BASELINE +from strategies.gold_scalper_pro.signals import build_signals + + +def main() -> int: + bars_path = PROJECT / "data" / "XAUUSD_M5_2024-06-26_2026-06-26.parquet" + print(f"loading {bars_path.name} ...") + bars = load_bars(bars_path) + print(f" {len(bars):,} bars {bars['timestamp'].iloc[0]} → {bars['timestamp'].iloc[-1]}") + + print("\nbuilding signals (frozen baseline params) ...") + pack = build_signals(FROZEN_BASELINE, bars, XAUUSD_REAL) + n_long = int(pack.signals_long.sum()) + n_short = int(pack.signals_short.sum()) + print(f" long signals : {n_long}") + print(f" short signals: {n_short}") + + # Build ScalperConfig from frozen baseline (mirrors EA inputs). + cfg = ScalperConfig( + use_break_even=FROZEN_BASELINE["InpUseBreakEven"], + use_trailing=FROZEN_BASELINE["InpUseTrailing"], + use_session=FROZEN_BASELINE["InpUseSession"], + session_start_hour=FROZEN_BASELINE["InpSessionStartHour"], + session_end_hour=FROZEN_BASELINE["InpSessionEndHour"], + max_positions=FROZEN_BASELINE["InpMaxPositions"], + max_trades_per_day=FROZEN_BASELINE["InpMaxTradesPerDay"], + daily_loss_limit_pct=FROZEN_BASELINE["InpDailyLossLimit"], + daily_profit_target_pct=FROZEN_BASELINE["InpDailyProfitTarget"], + min_seconds_between=FROZEN_BASELINE["InpMinSecondsBetween"], + sizing_mode=FROZEN_BASELINE["InpSizingMode"], + fixed_lots=FROZEN_BASELINE["InpFixedLots"], + risk_percent=FROZEN_BASELINE["InpRiskPercent"], + break_even_points=FROZEN_BASELINE["InpBreakEvenPoints"], + break_even_lock=FROZEN_BASELINE["InpBreakEvenLock"], + trail_start_points=FROZEN_BASELINE["InpTrailStartPoints"], + trail_step_points=FROZEN_BASELINE["InpTrailStepPoints"], + ) + sizing = SizingInputs() # unused — sizing lives in ScalperConfig for this EA + + print("\nrunning engine ...") + engine = ScalperEngine() + result = engine.run( + bars, pack.signals_long, pack.signals_short, + pack.sl_prices, pack.tp_prices, + XAUUSD_REAL, sizing, initial_deposit=10000.0, + scalper_cfg=cfg, + ) + + metrics = compute_metrics(result, periods_per_year=252 * 24 * 12) # M5 → ~72/year + print("\n=== result (frozen baseline) ===") + print(f" trades : {metrics.total_trades}") + print(f" net profit : {metrics.net_profit:,.2f}") + print(f" profit factor : {metrics.profit_factor:.2f}") + print(f" win rate : {metrics.win_rate:.2%}") + print(f" equity DD max : {metrics.max_equity_dd:,.2f} ({metrics.max_equity_dd_pct:.2%})") + print(f" sharpe : {metrics.sharpe:.2f}") + if result.trades: + reasons = {} + for t in result.trades: + reasons[t.exit_reason] = reasons.get(t.exit_reason, 0) + 1 + print(f" exit reasons : {reasons}") + print(f" final balance : {result.final_balance:,.2f}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/trade_by_trade.py b/scripts/trade_by_trade.py new file mode 100644 index 0000000..982db0e --- /dev/null +++ b/scripts/trade_by_trade.py @@ -0,0 +1,45 @@ +"""Trade-by-trade comparison to locate the PnL gap source. + +Both sides now produce 92 trades on the same window. This script dumps the +first ~20 trades from each side side-by-side so we can see WHERE the PnL +diverges (entry price? exit price? lots? swap?). +""" +from __future__ import annotations + +import sys +from pathlib import Path + +PROJECT = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(PROJECT)) + +import pandas as pd + +from shared.core.engine import SizingInputs +from shared.core.metrics import compute_metrics +from shared.data.loaders import load_bars +from shared.data.mt5_report import parse_mt5_report +from strategies.gold_scalper_pro.instruments import XAUUSD_REAL +from strategies.gold_scalper_pro.scalper_engine import ( + ScalperEngine, + engine_kwargs_from_params, +) +from strategies.gold_scalper_pro.search_space import FROZEN_BASELINE +from strategies.gold_scalper_pro.signals import build_signals + + +def main() -> int: + # ── Python trades ───────────────────────────────────────────────────── + bars = load_bars(PROJECT / "data" / "XAUUSD_M5_2024-06-26_2026-06-26.parquet") + start = pd.Timestamp("2026-04-16 00:00:00") + end = pd.Timestamp("2026-05-08 00:00:00") + window = bars[(bars["timestamp"] >= start) & (bars["timestamp"] < end)].reset_index(drop=True) + pack = build_signals(FROZEN_BASELINE, window, XAUUSD_REAL) + engine = ScalperEngine() + result = engine.run( + window, pack.signals_long, pack.signals_short, + pack.sl_prices, pack.tp_prices, + XAUUSD_REAL, SizingInputs(), 1000.0, + **engine_kwargs_from_params(FROZEN_BASELINE), + ) + print("=== Python first 20 trades ===") + print(f"{'#':>3 \ No newline at end of file diff --git a/scripts/verify_mt5.py b/scripts/verify_mt5.py new file mode 100644 index 0000000..636577b --- /dev/null +++ b/scripts/verify_mt5.py @@ -0,0 +1,262 @@ +"""Phase 7 — MT5 bridge: verify a finalist against the MT5 Strategy Tester. + +Pipeline (doc 07): +1. Deploy the EA (.ex5 + .mq5) to the terminal's MQL5\\Experts directory. +2. Generate a .set from the finalist's params (UTF-16-LE). +3. Generate a tester.ini (symbol, period, dates, model, shutdown). +4. Launch terminal64.exe /config:tester.ini — the tester runs headless and + closes itself when done (ShutdownTerminal=1). +5. Parse the HTML report (UTF-16-LE) the tester writes. +6. Build a Python-vs-MT5 comparison table + write auto-verification.md. + +Usage: + python verify_mt5.py --trials 30 --top-n 2 + +Runs the optimizer first (to get finalists), then verifies each in MT5. +""" +from __future__ import annotations + +import argparse +import shutil +import subprocess +import sys +import time +from pathlib import Path + +PROJECT = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(PROJECT)) + +from shared.config import get_secret, load_env +from shared.data.mt5_report import parse_mt5_report +from shared.mt5_pipeline.compare import build_comparison_table +from shared.mt5_pipeline.ini_gen import ( + MODEL_OHLC, + TesterConfig, + write_tester_ini, +) +from shared.mt5_pipeline.runner import run_tester +from shared.mt5_pipeline.set_gen import write_set_file + +load_env(PROJECT) + +# Reuse the optimizer's assembly. +from run import find_bars_file # noqa: E402 + +from shared.core.engine import SizingInputs # noqa: E402 +from shared.core.metrics import compute_metrics # noqa: E402 +from shared.data.loaders import load_bars # noqa: E402 +from shared.optimizer.objective import ( # noqa: E402 + Constraints, + ObjectiveConfig, + build_objective, +) +from shared.optimizer.selector import select_diverse_topn # noqa: E402 +from strategies.gold_scalper_pro.instruments import XAUUSD_REAL # noqa: E402 +from strategies.gold_scalper_pro.scalper_engine import ( # noqa: E402 + ScalperEngine, + engine_kwargs_from_params, +) +from strategies.gold_scalper_pro.search_space import ( # noqa: E402 + FROZEN_BASELINE, + INT_PARAMS, + SEARCH_SPACE, +) +from strategies.gold_scalper_pro.set_mappings import GOLD_SCALPER_MAPPINGS # noqa: E402 +from strategies.gold_scalper_pro.signals import build_signals # noqa: E402 + +import optuna # noqa: E402 + + +def deploy_ea(mt5_data_path: Path) -> None: + """Copy GoldScalperPro.ex5 + .mq5 into MQL5\\Experts so the tester finds it.""" + experts_dir = mt5_data_path / "MQL5" / "Experts" + experts_dir.mkdir(parents=True, exist_ok=True) + for src_name in ("GoldScalperPro.ex5", "GoldScalperPro.mq5"): + src = PROJECT / src_name + dst = experts_dir / src_name + if src.exists(): + shutil.copy2(src, dst) + print(f" deployed {src_name} → {dst.relative_to(mt5_data_path)}") + else: + raise FileNotFoundError(f"EA source missing: {src}") + + +def verify_finalist( + params: dict, + label: str, + *, + bars: "pd.DataFrame", + deposit: float, + mt5_install: str, + mt5_data_path: Path, + tester_profiles_dir: Path, + date_from: str, + date_to: str, +) -> int: + """Verify one finalist in MT5 and print the comparison table.""" + print(f"\n=== verifying {label} ===") + + # ── 1. Python re-run (fresh snapshot — doc 04 Rule 5) ────────────────── + pack = build_signals(params, bars, XAUUSD_REAL) + engine = ScalperEngine() + result = engine.run( + bars, pack.signals_long, pack.signals_short, + pack.sl_prices, pack.tp_prices, + XAUUSD_REAL, SizingInputs(), deposit, + **engine_kwargs_from_params(params), + ) + py_metrics = compute_metrics(result) + print(f" python: net={py_metrics.net_profit:+,.2f} PF={py_metrics.profit_factor:.2f} " + f"trades={py_metrics.total_trades} DD={py_metrics.max_equity_dd:.2f}") + + # ── 2. Write the .set (UTF-16-LE) into the tester profiles dir ──────── + set_name = f"GoldScalperPro_{label}" + set_path = tester_profiles_dir / f"{set_name}.set" + write_set_file(params, GOLD_SCALPER_MAPPINGS, set_path) + print(f" wrote .set → {set_path.name}") + + # ── 3. Write tester.ini ─────────────────────────────────────────────── + ini_path = tester_profiles_dir / f"{set_name}.ini" + login = int(get_secret("MT5_DEMO_LOGIN") or 0) + password = get_secret("MT5_DEMO_PASSWORD") + server = get_secret("MT5_DEMO_SERVER") + tcfg = TesterConfig( + expert=r"Experts\GoldScalperPro.ex5", + symbol="XAUUSD", + period="M5", + model=MODEL_OHLC, # 1-min OHLC for routine verification + from_date=date_from, + to_date=date_to, + deposit=deposit, + leverage=100, + report=f"report_{label}", + shutdown_terminal=True, + set_file=str(set_path), + login=login, + password=password, + server=server, + ) + write_tester_ini(tcfg, ini_path) + print(f" wrote ini → {ini_path.name}") + + # ── 4. Run the tester (headless; closes itself when done) ───────────── + print(f" launching MT5 tester (headless, model=OHLC) ...") + t0 = time.time() + exit_code, report_path = run_tester( + ini_path, mt5_install=mt5_install, timeout=900, poll_interval=5.0, + ) + elapsed = time.time() - t0 + print(f" tester finished in {elapsed:.0f}s exit={exit_code}") + if report_path is None: + print(" ✗ no report found — tester may have failed to start") + return 1 + print(f" report → {report_path}") + + # ── 5. Parse the report + build comparison ──────────────────────────── + mt5_metrics = parse_mt5_report(report_path) + # Map MT5 report keys to our Metrics field names for the table. + mt5_mapped = { + "net_profit": mt5_metrics.get("Total Net Profit"), + "profit_factor": mt5_metrics.get("Profit Factor"), + "total_trades": mt5_metrics.get("Total Trades"), + "max_equity_dd": mt5_metrics.get("Equity Drawdown Maximal"), + "win_rate": None, # MT5 report doesn't surface this directly + "sharpe": mt5_metrics.get("Sharpe Ratio"), + } + table = build_comparison_table(py_metrics, mt5_mapped) + print("\n " + table.replace("\n", "\n ")) + + # ── 6. Write auto-verification.md ───────────────────────────────────── + out_md = PROJECT / "registry" / f"auto-verification_{label}.md" + out_md.parent.mkdir(parents=True, exist_ok=True) + body = ( + f"# Auto-verification: {label}\n\n" + f"## Parameters\n\n```\n" + ) + for k, v in params.items(): + body += f" {k} = {v}\n" + body += "```\n\n## Python vs MT5\n\n" + table + body += ( + "\n## Decision rule (doc 03 §7)\n" + "If the MT5 number still clears the bar after the expected fidelity " + "gap, the finalist is real. If the edge only existed in the optimistic " + "Python figure, discard it.\n" + ) + out_md.write_text(body, encoding="utf-8") + print(f" wrote {out_md.relative_to(PROJECT)}") + return 0 + + +def main() -> int: + ap = argparse.ArgumentParser(description="Verify GoldScalperPro finalists in MT5.") + ap.add_argument("--trials", type=int, default=30) + ap.add_argument("--top-n", type=int, default=2) + ap.add_argument("--deposit", type=float, default=10000.0) + args = ap.parse_args() + + mt5_install = get_secret("MT5_TERMINAL_PATH") or r"C:\Program Files\MetaTrader 5 IC Markets Global" + mt5_install_dir = str(Path(mt5_install).parent) + mt5_data_path = Path(get_secret("MT5_DATA_PATH") + or r"C:\Users\Administrator\AppData\Roaming\MetaQuotes\Terminal" + r"\010E047102812FC0C18890992854220E") + tester_profiles_dir = mt5_data_path / "MQL5" / "Profiles" / "Tester" + tester_profiles_dir.mkdir(parents=True, exist_ok=True) + + bars_path = find_bars_file() + bars = load_bars(bars_path) + date_from = bars["timestamp"].iloc[0].strftime("%Y.%m.%d") + date_to = bars["timestamp"].iloc[-1].strftime("%Y.%m.%d") + print(f"=== MT5 verification ===") + print(f"bars : {bars_path.name} ({len(bars):,} bars)") + print(f"window : {date_from} → {date_to}") + print(f"deposit: {args.deposit:,.0f} USD") + + # ── Deploy EA ───────────────────────────────────────────────────────── + print(f"\ndeploying EA to {mt5_data_path.name}/MQL5/Experts/ ...") + deploy_ea(mt5_data_path) + + # ── Run optimizer to get finalists ──────────────────────────────────── + print(f"\noptimizing ({args.trials} trials) ...") + constraints = Constraints(min_trades=25, min_profit_factor=1.2, + max_equity_dd_pct=0.40) + obj_cfg = ObjectiveConfig( + engine=ScalperEngine(), + bars=bars, + instrument=XAUUSD_REAL, + sizing=SizingInputs(), + initial_deposit=args.deposit, + search_space=SEARCH_SPACE, + int_params=INT_PARAMS, + frozen_baseline=FROZEN_BASELINE, + constraints=constraints, + dd_weight=1.0, + build_signals=build_signals, + build_engine_kwargs=engine_kwargs_from_params, + ) + objective = build_objective(obj_cfg) + optuna.logging.set_verbosity(optuna.logging.WARNING) + study = optuna.create_study(direction="maximize", + sampler=optuna.samplers.TPESampler(seed=42)) + study.optimize(objective, n_trials=args.trials) + finalists = select_diverse_topn(study, args.top_n, SEARCH_SPACE) + print(f"selected {len(finalists)} finalists") + + # ── Verify each finalist ─────────────────────────────────────────────── + for i, t in enumerate(finalists, 1): + label = f"f{i}" + verify_finalist( + t.user_attrs["params"], label, + bars=bars, deposit=args.deposit, + mt5_install=mt5_install_dir, + mt5_data_path=mt5_data_path, + tester_profiles_dir=tester_profiles_dir, + date_from=date_from, date_to=date_to, + ) + + print("\n=== done ===") + print(f"verification reports in registry/auto-verification_*.md") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/shared/__init__.py b/shared/__init__.py new file mode 100644 index 0000000..25099f2 --- /dev/null +++ b/shared/__init__.py @@ -0,0 +1,8 @@ +"""Shared infrastructure layer of the backtesting lab. + +This package is import-stable library code: engines, indicators, instruments, +data loaders, optimizer, robustness, gates, wizard, and the MT5 bridge. +Strategy-specific glue lives in ``strategies/`` and never imports another +strategy's code (see doc 04 Rule 5). Breaking changes here ripple everywhere, +so it changes rarely and deliberately. +""" diff --git a/shared/config.py b/shared/config.py new file mode 100644 index 0000000..23adf9a --- /dev/null +++ b/shared/config.py @@ -0,0 +1,59 @@ +"""Runtime configuration loader — reads secrets from a gitignored ``.env``. + +Broker login lives in ``.env`` (gitignored). The bridge reads these at runtime +and injects them into ``tester.ini``'s ``[Common]`` section **without printing +them** (doc 07 §5). Never hard-code credentials and never print them. +""" +from __future__ import annotations + +import os +from pathlib import Path +from typing import Optional + +try: + from dotenv import load_dotenv # type: ignore + _HAS_DOTENV = True +except ImportError: + _HAS_DOTENV = False + + +def load_env(project_root: Optional[str | Path] = None) -> dict[str, str]: + """Load ``.env`` from ``project_root`` (defaults to this file's parent). + + Returns the MT5 credential keys without their values exposed in logs. + Falls back to plain line parsing if ``python-dotenv`` isn't installed. + """ + root = Path(project_root) if project_root else Path(__file__).resolve().parent.parent + env_path = root / ".env" + if _HAS_DOTENV and env_path.exists(): + load_dotenv(env_path) + elif env_path.exists(): + _parse_plain(env_path) + return { + "MT5_DEMO_LOGIN": os.environ.get("MT5_DEMO_LOGIN", ""), + "MT5_DEMO_SERVER": os.environ.get("MT5_DEMO_SERVER", ""), + # Password is intentionally not returned here; read via get_secret(). + "MT5_TERMINAL_PATH": os.environ.get( + "MT5_TERMINAL_PATH", + r"C:\Program Files\MetaTrader 5 IC Markets Global\terminal64.exe", + ), + } + + +def get_secret(name: str) -> str: + """Read a secret from the environment (load_env must be called first).""" + return os.environ.get(name, "") + + +def _parse_plain(env_path: Path) -> None: + """Minimal ``.env`` parser (KEY=VALUE) without python-dotenv.""" + for line in env_path.read_text(encoding="utf-8").splitlines(): + line = line.strip() + if not line or line.startswith("#") or "=" not in line: + continue + k, _, v = line.partition("=") + k, v = k.strip(), v.strip() + # Strip optional surrounding quotes. + if len(v) >= 2 and v[0] == v[-1] and v[0] in ("'", '"'): + v = v[1:-1] + os.environ.setdefault(k, v) diff --git a/shared/core/__init__.py b/shared/core/__init__.py new file mode 100644 index 0000000..ef057f4 --- /dev/null +++ b/shared/core/__init__.py @@ -0,0 +1,20 @@ +"""The frozen bar-by-bar fill simulator (doc 02 §2, doc 03). + +The engine is **strategy-agnostic**: it consumes bars + signal arrays + +stop/target arrays and simulates fills. All strategy math (when to enter, +where to put stops) lives in the caller. Once an engine reproduces your EA +within the expected fidelity gap (doc 03 §8) it is **frozen** (doc 04 Rule 1) +— never edit a validated engine to test an idea; fork it instead. +""" +from .engine import Direction, Engine, Position, Result, Trade +from .metrics import Metrics, compute_metrics + +__all__ = [ + "Direction", + "Engine", + "Position", + "Result", + "Trade", + "Metrics", + "compute_metrics", +] diff --git a/shared/core/engine.py b/shared/core/engine.py new file mode 100644 index 0000000..af7a0b4 --- /dev/null +++ b/shared/core/engine.py @@ -0,0 +1,172 @@ +"""Engine contract and result types (doc 02 §2, doc 03). + +The single most important design decision lives here: **the engine knows +nothing about your strategy.** Its input is pre-computed — bars, entry +signals, stop/target prices. The engine never decides *where* a stop goes; +it only decides *whether* price touched it. That seam separates "the +strategy" (caller) from "the simulator" (engine). + +The intra-bar 4-sub-tick model and pessimistic ordering convention are +described in doc 03 §2. Implement them in concrete engine subclasses (e.g. +``shared/core/grid_engine.py`` once your EA is brought in, doc 03 §3). +""" +from __future__ import annotations + +from dataclasses import dataclass, field +from enum import IntEnum +from typing import Any, Protocol, runtime_checkable + +import numpy as np +import pandas as pd + +from ..instruments.config import InstrumentConfig + + +class Direction(IntEnum): + """Trade direction. ``1`` long, ``-1`` short.""" + + LONG = 1 + SHORT = -1 + + +# Pessimistic intra-bar sub-tick order (doc 03 §2). +# For a LONG position (stop below, target above): OPEN → LOW → HIGH → CLOSE +# For a SHORT position (stop above, target below): OPEN → HIGH → LOW → CLOSE +# The pessimistic assumption: price visits the point that hurts an open +# position *before* the point that helps it — so a bar that could touch both +# stop and target resolves to the stop (the realistic worst case). +SUBTICK_ORDER_LONG = ("open", "low", "high", "close") +SUBTICK_ORDER_SHORT = ("open", "high", "low", "close") + + +@dataclass +class Trade: + """One closed trade (a position opened then exited). + + ``pnl`` includes accumulated swap. ``exit_reason`` documents why the + trade closed (stop / target / basket-stop / signal-flip / end-of-data). + """ + + direction: Direction + entry_time: pd.Timestamp + exit_time: pd.Timestamp + entry_price: float + exit_price: float + lots: float + pnl: float # net of swap + swap: float # accumulated swap (also folded into pnl) + exit_reason: str = "" + + +@dataclass +class Position: + """An open position held by the engine between entry and exit. + + Multi-position baskets (grid/martingale) are modelled as a list of + ``Position`` objects that share a single basket stop (doc 03 §3, §5). + ``sl`` / ``tp`` are mutable because break-even and trailing stops update + them over the position's life (doc 03 §6). + """ + + direction: Direction + entry_time: pd.Timestamp + entry_price: float + lots: float + open_swap: float = 0.0 # swap accumulated so far on this position + sl: float = 0.0 # current stop-loss price (0 = none) + tp: float = 0.0 # current take-profit price (0 = none) + + +@dataclass +class Result: + """Engine output (doc 02 §2). + + ``trades`` is the list of closed trades; ``equity_curve`` is sampled + periodically (e.g. hourly) so memory stays bounded on multi-year runs. + """ + + trades: list[Trade] = field(default_factory=list) + equity_curve: pd.DataFrame = field(default_factory=lambda: pd.DataFrame(columns=["timestamp", "balance", "equity"])) + final_balance: float = 0.0 + initial_deposit: float = 0.0 + # Optional floating (open) state at end-of-data, for diagnostics. + open_positions: list[Position] = field(default_factory=list) + # Free-form diagnostics (max floating drawdown, series count, …). + diagnostics: dict[str, Any] = field(default_factory=dict) + + +@runtime_checkable +class Engine(Protocol): + """Strategy-agnostic bar-by-bar fill simulator. + + Implementations take **pre-computed** signal + stop/target arrays and + simulate fills bar-by-bar with the pessimistic 4-sub-tick model. The + engine must **not** compute signals, stops, or sizing beyond what the + caller passes in — that boundary is what makes it freezable (doc 04). + """ + + def run( + self, + bars: pd.DataFrame, + signals_long: np.ndarray, + signals_short: np.ndarray, + sl_prices: np.ndarray, + tp_prices: np.ndarray, + instrument: InstrumentConfig, + sizing: "SizingInputs", + initial_deposit: float, + ) -> Result: + """Run the engine over ``bars`` and return a :class:`Result`. + + Parameters + ---------- + bars + DataFrame with columns ``[timestamp, open, high, low, close, + spread]``. ``spread`` is in price points per bar (may be 0 / NaN + if the instrument uses ``FIXED_POINTS``). + signals_long, signals_short + Boolean arrays, **edge-detected** — ``True`` only on the + transition bar, not forward-filled, or the engine re-enters + every bar (doc 02 §3). + sl_prices, tp_prices + The stop-loss / take-profit **price** for an entry on that bar. + ``NaN`` where no stop / target applies. The engine never decides + *where* a stop goes — only whether price touched it. + instrument + Per-symbol mechanics (tick value, spread, swap, lot steps). + sizing + Lot / money mode inputs (doc 05 §4). + initial_deposit + Account starting balance. + + Notes + ----- + **Look-ahead guard:** a signal computed *from* a bar's close must + execute on the *next* bar's open, never the same bar's close. + """ + ... + + +@dataclass +class SizingInputs: + """Position-sizing inputs (doc 05 §4, doc 03 §4). + + Mirror your EA's sizing exactly or PnL will be off by a constant factor. + + Modes, in priority order: + + 1. ``risk_on_stop``: ``lot = max_loss_money / (stop_distance_points × tick_value)``. + 2. ``fixed_lot`` (when ``lot != 0``): ``lot = configured_lot`` (optionally + scaled by ``balance / reference_balance`` when ``reference_balance > 0``). + 3. ``money`` (when ``lot == 0``): ``lot = amount / open_price / contract_size``. + + **Money-mode guard:** ``lot`` must be ``0`` to enable money mode — a stray + non-zero fixed lot silently sizes every trade wrong. + """ + + lot: float = 0.0 # fixed lot; MUST be 0 for money mode + lot_amount: float = 0.0 # cash base for money mode + lot_balance: float = 0.0 # 0 = static (research default); >0 = dynamic + reference_balance: float = 0.0 # scaling window for fixed-lot compounding + risk_money: float = 0.0 # max loss money for risk-on-stop mode + max_loss_money: float = 0.0 # alias used by some EAs diff --git a/shared/core/metrics.py b/shared/core/metrics.py new file mode 100644 index 0000000..f5753de --- /dev/null +++ b/shared/core/metrics.py @@ -0,0 +1,131 @@ +"""Compute standard backtest metrics from a :class:`Result` (doc 02 §2). + +A separate :func:`compute_metrics` turns the engine's trade list + equity +curve into the numbers you optimize on: Net Profit, Profit Factor, Win Rate, +max Balance/Equity Drawdown, Sharpe, APR, trade count. +""" +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any + +import numpy as np +import pandas as pd + +from .engine import Result + + +@dataclass +class Metrics: + """Standard backtest metrics. + + Use ``equity_dd_max`` (Equity Drawdown Maximal, peak-to-trough) — not + Absolute — when judging risk; the peak-to-trough is what matters. + """ + + net_profit: float = 0.0 + gross_profit: float = 0.0 + gross_loss: float = 0.0 + profit_factor: float = 0.0 # gross_profit / |gross_loss| (inf-safe) + win_rate: float = 0.0 # wins / total_trades + total_trades: int = 0 + wins: int = 0 + losses: int = 0 + avg_win: float = 0.0 + avg_loss: float = 0.0 + expectancy: float = 0.0 # avg pnl per trade + max_balance_dd: float = 0.0 # in account currency + max_equity_dd: float = 0.0 # in account currency (the one to watch) + max_balance_dd_pct: float = 0.0 + max_equity_dd_pct: float = 0.0 + sharpe: float = 0.0 # annualized, 0 if undefined + apr: float = 0.0 # annualized percent return + # Free-form extras for strategy-specific diagnostics. + extras: dict[str, Any] = field(default_factory=dict) + + +def compute_metrics(result: Result, *, periods_per_year: int = 252) -> Metrics: + """Compute :class:`Metrics` from a :class:`Result`. + + Parameters + ---------- + result + Engine output (trade list + equity curve). + periods_per_year + Annualization factor for Sharpe (default 252 trading days). Adjust + to match your equity-curve sampling (e.g. 252 for daily, 252*24 for + hourly sampling). + """ + trades = result.trades + m = Metrics() + + m.total_trades = len(trades) + if m.total_trades == 0: + # No trades — equity curve is just the flat deposit. + _compute_drawdowns(result, m) + return m + + pnls = np.array([t.pnl for t in trades], dtype=float) + wins = pnls[pnls > 0] + losses = pnls[pnls < 0] + + m.net_profit = float(pnls.sum()) + m.gross_profit = float(wins.sum()) if wins.size else 0.0 + m.gross_loss = float(losses.sum()) if losses.size else 0.0 + m.wins = int(wins.size) + m.losses = int(losses.size) + m.win_rate = m.wins / m.total_trades + m.avg_win = float(wins.mean()) if wins.size else 0.0 + m.avg_loss = float(losses.mean()) if losses.size else 0.0 + m.expectancy = float(pnls.mean()) + m.profit_factor = ( + m.gross_profit / abs(m.gross_loss) if m.gross_loss != 0.0 else float("inf") + ) + + _compute_drawdowns(result, m) + _compute_risk_ratios(result, m, periods_per_year) + return m + + +def _compute_drawdowns(result: Result, m: Metrics) -> None: + """Peak-to-trough drawdowns from the equity curve.""" + ec = result.equity_curve + if ec is None or ec.empty or "equity" not in ec.columns: + return + equity = ec["equity"].to_numpy(dtype=float) + if equity.size == 0: + return + running_max = np.maximum.accumulate(equity) + dd = running_max - equity + m.max_equity_dd = float(dd.max()) + m.max_equity_dd_pct = ( + float(dd.max() / running_max.max()) if running_max.max() > 0 else 0.0 + ) + if "balance" in ec.columns: + balance = ec["balance"].to_numpy(dtype=float) + if balance.size: + rb = np.maximum.accumulate(balance) + ddb = rb - balance + m.max_balance_dd = float(ddb.max()) + m.max_balance_dd_pct = ( + float(ddb.max() / rb.max()) if rb.max() > 0 else 0.0 + ) + + +def _compute_risk_ratios(result: Result, m: Metrics, periods_per_year: int) -> None: + """Annualized Sharpe and APR from the equity curve.""" + ec = result.equity_curve + if ec is None or ec.empty or "equity" not in ec.columns: + return + equity = ec["equity"].to_numpy(dtype=float) + if equity.size < 2 or result.initial_deposit <= 0: + return + returns = np.diff(equity) / equity[:-1] + returns = returns[np.isfinite(returns)] + if returns.size > 1 and returns.std() > 0: + m.sharpe = float(returns.mean() / returns.std() * np.sqrt(periods_per_year)) + final = equity[-1] + total_return = (final / result.initial_deposit) - 1.0 + # APR from total return assuming ``periods_per_year`` samples per year. + n_years = max(equity.size / periods_per_year, 1e-9) + m.apr = float(((1.0 + total_return) ** (1.0 / n_years) - 1.0) * 100.0) diff --git a/shared/gates/__init__.py b/shared/gates/__init__.py new file mode 100644 index 0000000..70e6627 --- /dev/null +++ b/shared/gates/__init__.py @@ -0,0 +1,11 @@ +"""Entry-filter gates (doc 02 §1, doc 04 Rule 6). + +Gates are reusable boolean masks that *filter* entries. They are **caller- +side**: a gate is AND-ed into the signal *before* it reaches the engine — +``allow = directional_signal & ~block_condition``. The engine still just +consumes a signal array. This means you can add or remove a filter without +re-validating the engine. +""" +from .base import Gate, exhaustion_gate, regime_gate, time_of_day_gate + +__all__ = ["Gate", "time_of_day_gate", "regime_gate", "exhaustion_gate"] diff --git a/shared/gates/base.py b/shared/gates/base.py new file mode 100644 index 0000000..11a4a4c --- /dev/null +++ b/shared/gates/base.py @@ -0,0 +1,78 @@ +"""Gate primitives — boolean masks over bars (doc 04 Rule 6). + +A gate is a callable that takes the bars DataFrame and returns a boolean +``numpy.ndarray`` (``True`` = entry allowed on that bar). Gates never open or +close trades; they only mask the signal array the caller hands to the engine. +""" +from __future__ import annotations + +from typing import Protocol, runtime_checkable + +import numpy as np +import pandas as pd + + +@runtime_checkable +class Gate(Protocol): + """A callable producing a boolean mask over bars (``True`` = allow entry).""" + + def __call__(self, bars: pd.DataFrame) -> np.ndarray: ... + + +def time_of_day_gate( + bars: pd.DataFrame, + *, + start_hour: int, + end_hour: int, + timezone: str | None = None, +) -> np.ndarray: + """Allow entries only within ``[start_hour, end_hour)`` (hours, 0–24). + + Useful for sessions that only trade London/NY open. ``timezone`` is + applied to ``bars['timestamp']`` if given; otherwise the timestamp's + existing tz is used (or naive local time). + """ + ts = pd.to_datetime(bars["timestamp"]) + if timezone is not None: + ts = ts.dt.tz_localize(None).dt.tz_localize(timezone) if ts.dt.tz is None else ts.dt.tz_convert(timezone) + hours = ts.dt.hour + if start_hour <= end_hour: + mask = (hours >= start_hour) & (hours < end_hour) + else: + # Wrap past midnight, e.g. 22 → 6. + mask = (hours >= start_hour) | (hours < end_hour) + return mask.to_numpy() + + +def regime_gate( + bars: pd.DataFrame, + *, + trend_filter: np.ndarray, + direction: int, +) -> np.ndarray: + """Allow entries only when ``trend_filter`` agrees with ``direction``. + + ``trend_filter`` is a +1/-1 array (e.g. from an EMA slope or ADX sign). + ``direction=+1`` keeps bars where the trend is up; ``-1`` keeps downtrend. + """ + tf = np.asarray(trend_filter) + mask = tf == direction + return mask + + +def exhaustion_gate( + rsi_arr: np.ndarray, + *, + overbought: float = 70.0, + oversold: float = 30.0, +) -> np.ndarray: + """Block entries when RSI is in the exhaustion zone for the direction. + + Returns ``True`` where entry is *allowed* (i.e. not exhausted). Block longs + when ``rsi >= overbought`` and shorts when ``rsi <= oversold`` — combine + with the directional signal in the caller. + """ + rsi_arr = np.asarray(rsi_arr, dtype=float) + allow = (rsi_arr < overbought) & (rsi_arr > oversold) + allow = np.where(np.isnan(rsi_arr), False, allow) + return allow diff --git a/shared/indicators/__init__.py b/shared/indicators/__init__.py new file mode 100644 index 0000000..b9b2a94 --- /dev/null +++ b/shared/indicators/__init__.py @@ -0,0 +1,10 @@ +"""Pure indicator functions (doc 02 §1). + +Indicators are **pure functions**: they take a price array and parameters and +return an array. They must **not** hold state across calls. Strategy code +computes signals from these; the engine never calls them. Add your own custom +indicators (range filter, regime detector, …) here as the strategy requires. +""" +from .base import atr, ema, rsi, sma + +__all__ = ["sma", "ema", "rsi", "atr"] diff --git a/shared/indicators/base.py b/shared/indicators/base.py new file mode 100644 index 0000000..20d6150 --- /dev/null +++ b/shared/indicators/base.py @@ -0,0 +1,88 @@ +"""Common technical indicators as pure numpy functions. + +All functions take a 1-D price array (or H/L/C arrays) and return an array of +the same length, with leading ``NaN`` where the lookback window is not yet +full. Add strategy-specific indicators (your custom range filter, regime +detector, …) alongside these as you need them. + +These are vectorized with numpy; for the hot path compile them with numba if +profiling demands it (doc 01 — numba is in the stack for that reason). +""" +from __future__ import annotations + +import numpy as np + + +def sma(prices: np.ndarray, period: int) -> np.ndarray: + """Simple moving average. ``NaN`` until ``period`` values are seen.""" + if period <= 0: + raise ValueError("period must be > 0") + out = np.full(prices.shape, np.nan, dtype=float) + if len(prices) < period: + return out + csum = np.cumsum(prices, dtype=float) + csum[period:] = csum[period:] - csum[:-period] + out[period - 1:] = csum[period - 1:] / period + return out + + +def ema(prices: np.ndarray, period: int) -> np.ndarray: + """Exponential moving average (seeded with the first ``period`` SMA).""" + if period <= 0: + raise ValueError("period must be > 0") + out = np.full(prices.shape, np.nan, dtype=float) + if len(prices) < period: + return out + alpha = 2.0 / (period + 1.0) + out[period - 1] = prices[:period].mean() + for i in range(period, len(prices)): + out[i] = alpha * prices[i] + (1.0 - alpha) * out[i - 1] + return out + + +def rsi(prices: np.ndarray, period: int = 14) -> np.ndarray: + """Relative Strength Index (Wilder's smoothing). Range ``[0, 100]``.""" + if period <= 0: + raise ValueError("period must be > 0") + out = np.full(prices.shape, np.nan, dtype=float) + if len(prices) <= period: + return out + deltas = np.diff(prices, prepend=prices[0]) + gains = np.where(deltas > 0, deltas, 0.0) + losses = np.where(deltas < 0, -deltas, 0.0) + avg_gain = gains[1:period + 1].mean() + avg_loss = losses[1:period + 1].mean() + for i in range(period, len(prices)): + avg_gain = (avg_gain * (period - 1) + gains[i]) / period + avg_loss = (avg_loss * (period - 1) + losses[i]) / period + rs = avg_gain / avg_loss if avg_loss != 0 else np.inf + out[i] = 100.0 - 100.0 / (1.0 + rs) + return out + + +def atr( + high: np.ndarray, + low: np.ndarray, + close: np.ndarray, + period: int = 14, +) -> np.ndarray: + """Average True Range (Wilder's smoothing).""" + if not (len(high) == len(low) == len(close)): + raise ValueError("high/low/close must have equal length") + if period <= 0: + raise ValueError("period must be > 0") + out = np.full(close.shape, np.nan, dtype=float) + if len(close) <= period: + return out + prev_close = np.concatenate(([close[0]], close[:-1])) + tr = np.maximum.reduce([ + high - low, + np.abs(high - prev_close), + np.abs(low - prev_close), + ]) + atr_val = tr[1:period + 1].mean() + out[period] = atr_val + for i in range(period + 1, len(close)): + atr_val = (atr_val * (period - 1) + tr[i]) / period + out[i] = atr_val + return out diff --git a/shared/instruments/__init__.py b/shared/instruments/__init__.py new file mode 100644 index 0000000..8af665a --- /dev/null +++ b/shared/instruments/__init__.py @@ -0,0 +1,11 @@ +"""Per-symbol / per-broker instrument configuration. + +Everything symbol- or broker-specific lives in an :class:`InstrumentConfig` +object, never in the engine and never hard-coded in a strategy. The engine +pulls tick value, spread model, swap, and lot steps *from the config* (doc 04 +Rule 4). Testing the same strategy on a different symbol = a different +config, zero engine changes. +""" +from .config import InstrumentConfig, InstrumentProfile, get_profile + +__all__ = ["InstrumentConfig", "InstrumentProfile", "get_profile"] diff --git a/shared/instruments/config.py b/shared/instruments/config.py new file mode 100644 index 0000000..1b75dba --- /dev/null +++ b/shared/instruments/config.py @@ -0,0 +1,176 @@ +"""Instrument configuration — the symbol as data (doc 05 §1). + +One object describes a symbol under one broker. The engine reads +**everything** about the instrument from here; nothing about ticks, spread, +or swap is ever hard-coded elsewhere. Get these fields from MT5's symbol +specification (``Market Watch → right-click symbol → Specification``), not +from memory — a wrong ``tick_value`` or ``contract_size`` scales every PnL +by a constant and makes the whole backtest meaningless. +""" +from __future__ import annotations + +from dataclasses import dataclass, field +from enum import Enum +from typing import Optional + + +class SpreadMode(str, Enum): + """How spread is applied on each bar. + + - ``BAR_COLUMN``: use the real historical per-bar spread stored in the data. + - ``FIXED_POINTS``: a constant point spread (fallback when history is + unreliable, e.g. crypto). + - ``ANNUAL_AVG``: an annual-average spread model. + """ + + BAR_COLUMN = "bar_column" + FIXED_POINTS = "fixed_points" + ANNUAL_AVG = "annual_avg" + + +class SwapMode(str, Enum): + """How overnight swap is charged. + + - ``FIXED_PER_LOT``: ``swap = rate × lot × multiplier`` per calendar day. + - ``ANNUAL_PCT``: ``swap = price × annual_pct / 365 × lot × multiplier``. + """ + + FIXED_PER_LOT = "fixed_per_lot" + ANNUAL_PCT = "annual_pct" + + +class InstrumentProfile(str, Enum): + """Cost-stress profile of an instrument (doc 05 §1). + + - ``REAL``: the broker's actual conditions — the default for ranking. + - ``WORST_CASE``: wider spread, harsher swap — a finalist that survives + this is robust to cost. + - ``BEST_CASE``: tighter spread — shows the ceiling. + """ + + REAL = "real" + WORST_CASE = "worst_case" + BEST_CASE = "best_case" + + +@dataclass(frozen=True) +class InstrumentConfig: + """All symbol mechanics for one symbol under one broker. + + Frozen so configs are safe to share between the engine and the MT5 bridge + without accidental mutation. Keep three variants (real / worst_case / + best_case) per symbol for cost-stress testing (doc 05 §1). + """ + + # --- identity --- + name: str # human label, e.g. "EURUSD IC Markets" + symbol: str # broker symbol code, e.g. "EURUSD" + + # --- price mechanics (from MT5 symbol spec) --- + point: float # smallest price increment, e.g. 0.00001 + digits: int # price decimal places + tick_size: float # price step + tick_value: float # money per lot per tick (the critical PnL scalar) + contract_size: float # units per lot + + # --- volume mechanics (from MT5 symbol spec) --- + volume_min: float # minimum lot + volume_step: float # lot rounding step + volume_max: float # maximum lot + + # --- spread model (doc 05 §1) --- + spread_mode: SpreadMode = SpreadMode.BAR_COLUMN + spread_fixed_points: float = 0.0 # used only with FIXED_POINTS + + # --- swap model (doc 05 §1, doc 03 §6) --- + swap_mode: SwapMode = SwapMode.FIXED_PER_LOT + swap_long: float = 0.0 # per-lot-per-day rate (fixed mode) + swap_short: float = 0.0 # per-lot-per-day rate (fixed mode) + swap_annual_pct: float = 0.0 # annual rate on notional (percent mode) + triple_swap_weekday: int = 3 # 0=Mon ... 3=Wed ... 6=Sun (×3 charge day) + + # --- profile tag (which variant this config is) --- + profile: InstrumentProfile = InstrumentProfile.REAL + + # --- optional fallback spread for the bar-column mode when missing --- + spread_bar_column_fallback: float = 0.0 + + def round_volume(self, lots: float) -> float: + """Round ``lots`` to the broker's volume step and clamp to [min, max]. + + Mirrors MT5's lot rounding. Use this in the engine and the .set + generator so the two tiers agree. + """ + if self.volume_step <= 0: + clamped = max(self.volume_min, min(self.volume_max, lots)) + else: + stepped = round(lots / self.volume_step) * self.volume_step + clamped = max(self.volume_min, min(self.volume_max, stepped)) + return round(clamped, 8) + + def spread_points(self, bar_spread: Optional[float] = None) -> float: + """Return the spread in price points for a bar. + + With ``BAR_COLUMN`` mode, ``bar_spread`` is the per-bar value from the + data (falling back to ``spread_bar_column_fallback`` when ``None``). + With ``FIXED_POINTS`` mode, ``bar_spread`` is ignored. + """ + if self.spread_mode is SpreadMode.FIXED_POINTS: + return self.spread_fixed_points + if self.spread_mode is SpreadMode.ANNUAL_AVG: + # Placeholder — annual-average model to be filled per instrument. + return self.spread_fixed_points + # BAR_COLUMN + if bar_spread is not None: + return float(bar_spread) + return self.spread_bar_column_fallback + + +def get_profile( + base: InstrumentConfig, + profile: InstrumentProfile, + *, + worst_spread_mult: float = 1.5, + best_spread_mult: float = 0.5, + worst_swap_mult: float = 1.3, + best_swap_mult: float = 0.7, +) -> InstrumentConfig: + """Derive a cost-stress variant of ``base`` for the requested profile. + + Returns ``base`` unchanged for ``REAL``. For ``WORST_CASE`` / ``BEST_CASE`` + it widens / tightens spread and swap by the given multipliers. The base's + ``spread_mode`` is preserved; if it is ``BAR_COLUMN`` the multiplier is + baked into ``spread_bar_column_fallback`` and used when the bar column is + missing (a real per-bar spread cannot be multiplied without the data, so + cost stress on bar-column mode is approximated via the fallback). + + For a true cost-stress run, prefer building three explicit config objects + per symbol by hand and registering them; this helper is a convenience. + """ + if profile is InstrumentProfile.REAL or base.profile is profile: + return base + + mult_spread = worst_spread_mult if profile is InstrumentProfile.WORST_CASE else best_spread_mult + mult_swap = worst_swap_mult if profile is InstrumentProfile.WORST_CASE else best_swap_mult + + return InstrumentConfig( + name=base.name, + symbol=base.symbol, + point=base.point, + digits=base.digits, + tick_size=base.tick_size, + tick_value=base.tick_value, + contract_size=base.contract_size, + volume_min=base.volume_min, + volume_step=base.volume_step, + volume_max=base.volume_max, + spread_mode=base.spread_mode, + spread_fixed_points=base.spread_fixed_points * mult_spread, + swap_mode=base.swap_mode, + swap_long=base.swap_long * mult_swap, + swap_short=base.swap_short * mult_swap, + swap_annual_pct=base.swap_annual_pct * mult_swap, + triple_swap_weekday=base.triple_swap_weekday, + profile=profile, + spread_bar_column_fallback=base.spread_bar_column_fallback * mult_spread, + ) diff --git a/shared/mt5_pipeline/__init__.py b/shared/mt5_pipeline/__init__.py new file mode 100644 index 0000000..0a2bf38 --- /dev/null +++ b/shared/mt5_pipeline/__init__.py @@ -0,0 +1,28 @@ +"""The MetaTrader 5 bridge (doc 07). + +Connects the fast Python search to the gold-standard MT5 Strategy Tester. +Four responsibilities: (1) compile the EA, (2) auto-run a backtest from a +generated config, (3) parse the HTML report, (4) compare Python vs MT5. + +Topology A (all-Windows) is implemented here — no SSH / scheduled-task +plumbing is needed; the official ``MetaTrader5`` package and local file paths +drive the terminal directly. +""" +from .compare import build_comparison_table, write_comparison +from .compile import compile_ea, default_metaeditor_path +from .ini_gen import TesterConfig, write_tester_ini +from .runner import run_tester, wait_for_report +from .set_gen import ParamMapping, write_set_file + +__all__ = [ + "compile_ea", + "default_metaeditor_path", + "TesterConfig", + "write_tester_ini", + "ParamMapping", + "write_set_file", + "run_tester", + "wait_for_report", + "build_comparison_table", + "write_comparison", +] diff --git a/shared/mt5_pipeline/compare.py b/shared/mt5_pipeline/compare.py new file mode 100644 index 0000000..43d59fe --- /dev/null +++ b/shared/mt5_pipeline/compare.py @@ -0,0 +1,106 @@ +"""Python-vs-MT5 comparison table (doc 07 §8). + +For every finalist, write an ``auto-verification.md`` that puts the two tiers +side by side and judges the delta against the expected fidelity gap (doc 03 +§7): a clean-directional setup shows only a modest negative gap (MT5 a little +below Python); a trailing/grid setup in volatile history can gap much wider, +and that's *expected*, not a bug. + +Decision rule: if the **MT5** number still clears your bar after the gap, the +finalist is real; if the edge only existed in the optimistic Python figure, +discard it. +""" +from __future__ import annotations + +from pathlib import Path +from typing import Mapping + +from ..core.metrics import Metrics + +# Rows shown in the comparison table (doc 07 §8). +COMPARISON_ROWS: list[tuple[str, str, str]] = [ + ("net_profit", "Net", "{:,.2f}"), + ("profit_factor", "Profit Factor", "{:.2f}"), + ("max_equity_dd", "Equity DD max", "{:,.2f}"), + ("total_trades", "Total trades", "{:d}"), + ("win_rate", "Win rate", "{:.2%}"), + ("sharpe", "Sharpe", "{:.2f}"), +] + + +def build_comparison_table( + py_metrics: Metrics | Mapping[str, object], + mt5_metrics: Mapping[str, object], + *, + rows: list[tuple[str, str, str]] | None = None, +) -> str: + """Build the Python-vs-MT5 markdown comparison table. + + ``py_metrics`` may be a :class:`Metrics` dataclass or a mapping. MT5 + metrics come from :func:`shared.data.mt5_report.parse_mt5_report`. The + ``Δ`` column is the relative difference where both values are numeric. + """ + rows = rows or COMPARISON_ROWS + py = _as_mapping(py_metrics) + lines = [ + "| Metric | Python | MT5 | Δ |", + "|--------|--------|-----|---|", + ] + for key, label, fmt in rows: + pv = py.get(key) + mv = mt5_metrics.get(label) or mt5_metrics.get(key) + p_str = _fmt(pv, fmt) + m_str = _fmt(mv, fmt) + delta = _delta(pv, mv) + lines.append(f"| {label} | {p_str} | {m_str} | {delta} |") + return "\n".join(lines) + "\n" + + +def write_comparison( + py_metrics: Metrics | Mapping[str, object], + mt5_metrics: Mapping[str, object], + path: str | Path, + *, + notes: str = "", + rows: list[tuple[str, str, str]] | None = None, +) -> None: + """Write the comparison table + notes to an ``auto-verification.md``.""" + p = Path(path) + p.parent.mkdir(parents=True, exist_ok=True) + table = build_comparison_table(py_metrics, mt5_metrics, rows=rows) + body = "# Auto-verification: Python vs MT5\n\n" + table + if notes: + body += "\n\n## Notes\n\n" + notes + "\n" + body += ( + "\n## Decision rule (doc 03 §7)\n" + "If the MT5 number still clears the bar after the expected fidelity " + "gap, the finalist is real. If the edge only existed in the optimistic " + "Python figure, discard it.\n" + ) + p.write_text(body, encoding="utf-8") + + +def _as_mapping(metrics: Metrics | Mapping[str, object]) -> Mapping[str, object]: + if isinstance(metrics, Mapping): + return metrics + return {k: getattr(metrics, k) for k, _ in COMPARISON_ROWS if hasattr(metrics, k)} + + +def _fmt(v: object, fmt: str) -> str: + if v is None: + return "—" + if isinstance(v, (int, float)) and fmt: + try: + return fmt.format(v) + except (ValueError, TypeError): + return str(v) + return str(v) + + +def _delta(py: object, mt5: object) -> str: + if not isinstance(py, (int, float)) or not isinstance(mt5, (int, float)): + return "—" + if py == 0: + return "—" + pct = (mt5 - py) / abs(py) * 100.0 + return f"{pct:+.1f}%" diff --git a/shared/mt5_pipeline/compile.py b/shared/mt5_pipeline/compile.py new file mode 100644 index 0000000..52a8cc6 --- /dev/null +++ b/shared/mt5_pipeline/compile.py @@ -0,0 +1,66 @@ +"""Compile an EA from .mq5 to .ex5 with MetaEditor (doc 07 §1). + +The tester runs a compiled ``.ex5``. Compile from the command line so the +bridge can do it programmatically. Custom indicators the EA calls must be +compiled too and placed in ``MQL5\\Indicators\\``. If the EA's ``#include`` +files live in a non-standard folder, compile the terminal in portable mode +(``/portable``) so MetaEditor resolves includes from that terminal's +``MQL5\\Include\\``. +""" +from __future__ import annotations + +import subprocess +from pathlib import Path + +DEFAULT_MT5_INSTALL = r"C:\Program Files\MetaTrader 5 IC Markets Global" + + +def default_metaeditor_path(mt5_install: str = DEFAULT_MT5_INSTALL) -> str: + """Return the metaeditor64.exe path for the given MT5 install.""" + return str(Path(mt5_install) / "metaeditor64.exe") + + +def compile_ea( + mq5_path: str | Path, + *, + metaeditor_path: str | None = None, + mt5_install: str = DEFAULT_MT5_INSTALL, + timeout: int = 120, +) -> tuple[bool, str]: + """Compile an ``.mq5`` EA to ``.ex5`` via MetaEditor's command line. + + Returns ``(success, log_text)``. MetaEditor writes a ``.log`` next to the + source; on a clean compile the ``.ex5`` appears beside the ``.mq5``. + + Command line (doc 07 §1):: + + metaeditor64.exe /compile:"C:\\path\\to\\Expert.mq5" /log + """ + mq5 = Path(mq5_path) + if not mq5.exists(): + return False, f"source not found: {mq5}" + editor = metaeditor_path or default_metaeditor_path(mt5_install) + if not Path(editor).exists(): + return False, f"metaeditor not found: {editor}" + + cmd = [editor, f"/compile:{mq5}", "/log"] + try: + proc = subprocess.run( + cmd, capture_output=True, text=True, timeout=timeout, check=False, + ) + log_path = mq5.with_suffix(".log") + log_text = proc.stdout + "\n" + proc.stderr + if log_path.exists(): + try: + log_text += "\n--- metaeditor log ---\n" + log_path.read_text( + encoding="utf-16-le", errors="replace" + ) + except Exception: + pass + ex5 = mq5.with_suffix(".ex5") + success = ex5.exists() and ex5.stat().st_size > 0 + return success, log_text + except subprocess.TimeoutExpired: + return False, f"compile timed out after {timeout}s" + except FileNotFoundError as e: + return False, f"failed to launch metaeditor: {e}" diff --git a/shared/mt5_pipeline/ini_gen.py b/shared/mt5_pipeline/ini_gen.py new file mode 100644 index 0000000..b71005b --- /dev/null +++ b/shared/mt5_pipeline/ini_gen.py @@ -0,0 +1,82 @@ +"""tester.ini generator (doc 07 §2b). + +A small INI tells the tester *what* to run: expert, symbol, period, tick +model, date range, deposit, leverage, report name, and ``ShutdownTerminal=1`` +so the terminal closes itself when done (the bridge then knows it's finished). +""" +from __future__ import annotations + +import configparser +from dataclasses import dataclass +from pathlib import Path +from typing import Optional + +DEFAULT_MT5_INSTALL = r"C:\Program Files\MetaTrader 5 IC Markets Global" + +# Tick models (doc 07 §2b): +# 0 = Every tick based on real ticks (highest accuracy, slowest) +# 1 = Every tick (generated from M1) +# 2 = 1-minute OHLC (good for non-tick-sensitive, fast) — routine verification +# 4 = Open prices only (rough, fastest) +MODEL_OHLC = 2 +MODEL_EVERY_TICK = 1 +MODEL_REAL_TICKS = 0 +MODEL_OPEN_PRICES = 4 + + +@dataclass +class TesterConfig: + """Inputs for one tester run (doc 07 §2b). + + ``FromDate`` / ``ToDate`` are formatted ``YYYY.MM.DD`` for MT5. The login + section lives separately (kept out of code via the env file, doc 07 §5). + """ + + expert: str # e.g. "Experts\\MyEA.ex5" + symbol: str + period: str = "H1" # chart timeframe (≤ the EA's signal timeframe) + model: int = MODEL_OHLC # tick model + from_date: str = "2024.01.01" # YYYY.MM.DD + to_date: str = "2026.01.01" + deposit: float = 10000.0 + leverage: int = 100 + report: str = "report_myrun" # output HTML name (no extension) + shutdown_terminal: bool = True + set_file: Optional[str] = None # path to the .set, or None to inline + login: Optional[int] = None # from env, never hard-coded + password: Optional[str] = None # from env, never hard-coded + server: Optional[str] = None # broker server + + +def write_tester_ini(cfg: TesterConfig, path: str | Path) -> str: + """Write a ``tester.ini`` for ``terminal64.exe /config:`` and return its path.""" + p = Path(path) + p.parent.mkdir(parents=True, exist_ok=True) + cp = configparser.ConfigParser() + cp.optionxform = str # preserve case (MT5 keys are case-sensitive) + cp["Tester"] = { + "Expert": cfg.expert, + "Symbol": cfg.symbol, + "Period": cfg.period, + "Model": str(cfg.model), + "FromDate": cfg.from_date, + "ToDate": cfg.to_date, + "Deposit": str(cfg.deposit), + "Leverage": str(cfg.leverage), + "Report": cfg.report, + "ShutdownTerminal": "1" if cfg.shutdown_terminal else "0", + } + if cfg.set_file: + cp["Tester"]["TestReplaceExpert"] = "0" + common: dict[str, str] = {} + if cfg.login is not None: + common["Login"] = str(cfg.login) + if cfg.password is not None: + common["Password"] = cfg.password + if cfg.server is not None: + common["Server"] = cfg.server + if common: + cp["Common"] = common + with p.open("w", encoding="utf-8") as f: + cp.write(f) + return str(p) diff --git a/shared/mt5_pipeline/runner.py b/shared/mt5_pipeline/runner.py new file mode 100644 index 0000000..cbaf0e3 --- /dev/null +++ b/shared/mt5_pipeline/runner.py @@ -0,0 +1,93 @@ +"""Run the MT5 Strategy Tester and wait for the report (doc 07 §3, §7). + +Topology A (all-Windows): launch ``terminal64.exe /config:tester.ini`` locally. +``ShutdownTerminal=1`` makes the terminal close itself when the run finishes; +the bridge waits for the process to exit (or for the report file to appear), +then parses the report. No copy/poll step — read the HTML straight from the +terminal's report folder. +""" +from __future__ import annotations + +import subprocess +import time +from pathlib import Path +from typing import Optional + +DEFAULT_MT5_INSTALL = r"C:\Program Files\MetaTrader 5 IC Markets Global" + +# MT5 writes reports into the terminal's installation directory. +DEFAULT_REPORT_SUBDIR = "Reports" + + +def run_tester( + ini_path: str | Path, + *, + mt5_install: str = DEFAULT_MT5_INSTALL, + timeout: int = 1800, + report_subdir: str = DEFAULT_REPORT_SUBDIR, + poll_interval: float = 5.0, +) -> tuple[int, Optional[Path]]: + """Launch the tester with the generated ini and wait for the report. + + Returns ``(exit_code, report_path)``. ``report_path`` is ``None`` if no + report appeared before ``timeout``. The terminal is launched + non-interactively; ``ShutdownTerminal=1`` in the ini closes it on finish. + """ + ini = Path(ini_path) + if not ini.exists(): + raise FileNotFoundError(f"tester.ini not found: {ini}") + terminal = Path(mt5_install) / "terminal64.exe" + if not terminal.exists(): + raise FileNotFoundError(f"terminal64.exe not found: {terminal}") + + cmd = [str(terminal), f"/config:{ini}"] + # Detach so the terminal's own GUI lifecycle controls shutdown. + proc = subprocess.Popen(cmd) + report_dir = Path(mt5_install) / report_subdir + + deadline = time.time() + timeout + while time.time() < deadline: + if proc.poll() is not None: + break + # Some runs leave the terminal open if ShutdownTerminal didn't fire; + # check for the report regardless. + rep = _find_latest_report(report_dir, since=ini.stat().st_mtime) + if rep is not None: + return proc.wait(), rep + time.sleep(poll_interval) + + # One last check after exit / timeout. + rep = _find_latest_report(report_dir, since=ini.stat().st_mtime) + exit_code = proc.poll() if proc.poll() is not None else -1 + return exit_code, rep + + +def wait_for_report( + report_dir: str | Path, + *, + since_ts: float, + timeout: int = 1800, + poll_interval: float = 5.0, +) -> Optional[Path]: + """Poll ``report_dir`` for a fresh ``report*.htm`` and return its path.""" + deadline = time.time() + timeout + rdir = Path(report_dir) + while time.time() < deadline: + rep = _find_latest_report(rdir, since=since_ts) + if rep is not None: + return rep + time.sleep(poll_interval) + return None + + +def _find_latest_report(report_dir: Path, since: float) -> Optional[Path]: + """Return the newest ``.htm`` report in ``report_dir`` newer than ``since``.""" + if not report_dir.exists(): + return None + candidates = [ + f for f in report_dir.glob("*.htm") + if f.stat().st_mtime >= since and f.stat().st_size > 1024 + ] + if not candidates: + return None + return max(candidates, key=lambda f: f.stat().st_mtime) diff --git a/shared/mt5_pipeline/set_gen.py b/shared/mt5_pipeline/set_gen.py new file mode 100644 index 0000000..d48f711 --- /dev/null +++ b/shared/mt5_pipeline/set_gen.py @@ -0,0 +1,71 @@ +""".set file generator (doc 07 §2a). + +A ``.set`` is the EA's inputs serialized as ``key=value`` lines. **MT5 writes +``.set`` files as UTF-16-LE** — generate yours in the same encoding or the +tester silently ignores them. + +The bridge's set generator takes the *same* parameter dict you backtested in +Python and writes the matching ``.set``. The mapping from Python parameter +names to EA input names is strategy-specific — keep a small mapping table +next to the strategy so the two tiers always agree. Watch the enum-valued +inputs (mode flags, timeframe codes): MT5 inputs are often integers. +""" +from __future__ import annotations + +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Mapping + +SET_FILE_ENCODING = "utf-16-le" + + +@dataclass +class ParamMapping: + """Maps a Python param name to an EA input name (+ enum cast if needed). + + ``cast`` converts a Python value to the EA input's wire form (e.g. a + timeframe string to its ``ENUM_TIMEFRAMES`` integer, a bool to ``true``/ + ``false``). Defaults to identity. + """ + + py_name: str + ea_name: str + cast: Any = None # callable(value) -> str, optional + + def to_wire(self, value: Any) -> str: + v = self.cast(value) if self.cast is not None else value + if isinstance(v, bool): + return "true" if v else "false" + if isinstance(v, float) and v.is_integer(): + return str(int(v)) + return str(v) + + +def write_set_file( + params: Mapping[str, Any], + mappings: list[ParamMapping], + path: str | Path, + *, + extra_lines: list[str] | None = None, +) -> None: + """Write a ``.set`` file (UTF-16-LE) from a Python params dict. + + Only parameters with a mapping are written — frozen baseline values that + match the EA's compiled-in defaults can be omitted. ``extra_lines`` lets a + strategy inject raw ``key=value`` lines that don't have a Python + counterpart (e.g. EA constants). + + **Lot-mode guard (doc 05 §4, doc 07 §2a):** make sure the fixed-lot input + is ``0`` if you intend money mode — a mismatched ``.set`` is the #1 reason + a verified MT5 number disagrees with Python. + """ + out = Path(path) + out.parent.mkdir(parents=True, exist_ok=True) + lines: list[str] = ["; Generated by the MT5 bridge (UTF-16-LE)"] + for m in mappings: + if m.py_name in params: + lines.append(f"{m.ea_name}={m.to_wire(params[m.py_name])}") + if extra_lines: + lines.extend(extra_lines) + text = "\n".join(lines) + "\n" + out.write_bytes(text.encode(SET_FILE_ENCODING)) diff --git a/shared/optimizer/__init__.py b/shared/optimizer/__init__.py new file mode 100644 index 0000000..beeb674 --- /dev/null +++ b/shared/optimizer/__init__.py @@ -0,0 +1,22 @@ +"""Optuna objective, search space, and diverse top-N selector (doc 06). + +Owns the Bayesian search wiring. Must **not** know broker specifics — those +live in the instrument config. The objective samples parameters from the +declared search space, runs the engine, and returns a score; the selector +picks 2–3 **diverse** finalists (not the top-N-by-score, which are usually +near-clones of one peak). +""" +from .objective import Constraints, ObjectiveConfig, build_objective, score_metrics +from .search_space import SearchSpace, suggest_params +from .selector import select_diverse_topn, param_distance + +__all__ = [ + "SearchSpace", + "suggest_params", + "ObjectiveConfig", + "Constraints", + "build_objective", + "score_metrics", + "select_diverse_topn", + "param_distance", +] diff --git a/shared/optimizer/objective.py b/shared/optimizer/objective.py new file mode 100644 index 0000000..0d0e539 --- /dev/null +++ b/shared/optimizer/objective.py @@ -0,0 +1,173 @@ +"""Optuna objective function and scoring (doc 06 §2). + +Structure of one objective call (doc 06 §2): + + 1. sampled = suggest_params(trial) # from SEARCH_SPACE + 2. params = build_params(sampled, frozen_baseline) + 3. result = engine.run(params, bars, instrument, deposit) + metrics = compute_metrics(result) + 4. score, violations = score_metrics(metrics, sampled) + for k, v in metrics.items(): trial.set_user_attr(k, v) + +Score design balances reward against risk; constraints are hard rejections +implemented as a large penalty so violators sort to the bottom (not soft +nudges). A path-independent fragility guard rejects martingale configs that +would blow up on a path the backtest never sampled. +""" +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Callable, Optional + +import numpy as np +import pandas as pd + +from ..core.engine import Engine, SizingInputs +from ..core.metrics import Metrics, compute_metrics +from ..instruments.config import InstrumentConfig +from .search_space import SearchSpace, suggest_params + +# Penalty applied per hard-constraint violation (must dominate any real score). +VIOLATION_PENALTY: float = 1.0e6 + + +@dataclass +class Constraints: + """Hard constraints — violators are rejected (doc 06 §2). + + A trial that violates any constraint is pushed to the bottom of the study + by subtracting ``VIOLATION_PENALTY`` per violation. Set a field to + ``None`` to skip that check. + """ + + min_trades: Optional[int] = 25 # ≥ ~25–30 active trades per year + min_profit_factor: Optional[float] = 1.3 + max_equity_dd: Optional[float] = None # hard drawdown cap in account ccy + max_equity_dd_pct: Optional[float] = 0.35 # or as a fraction of deposit + # Path-independent fragility guard (doc 06 §2): reject martingale configs + # that survive the backtest but would blow up on a straight adverse path. + fragility_check: Optional[Callable[[Metrics, dict], list[str]]] = None + + +@dataclass +class ObjectiveConfig: + """Everything the objective closure needs, captured once. + + The objective is rebuilt per iteration (each iteration owns its own + snapshot — doc 04 Rule 5), so this config is the single source the run + script assembles before calling :func:`build_objective`. + """ + + engine: Engine + bars: pd.DataFrame + instrument: InstrumentConfig + sizing: SizingInputs + initial_deposit: float + search_space: SearchSpace + int_params: set[str] = field(default_factory=set) + frozen_baseline: dict = field(default_factory=dict) + constraints: Constraints = field(default_factory=Constraints) + dd_weight: float = 1.0 # score = Net − DD_WEIGHT × EquityDD + # Strategy-specific hook: turn sampled params into engine-ready params + + # signal arrays + sl/tp arrays. The snapshot owns this so the optimizer + # stays strategy-agnostic. + build_signals: Optional[Callable[[dict, pd.DataFrame, InstrumentConfig], "SignalPack"]] = None + # Strategy-specific engine kwargs builder: returns a dict of extra kwargs + # to pass to engine.run (e.g. {"scalper_cfg": ScalperConfig(...)}). Lets a + # strategy pipe tunable point values into its engine without the optimizer + # knowing about strategy-specific config objects. None → no extra kwargs. + build_engine_kwargs: Optional[Callable[[dict], dict]] = None + + +@dataclass +class SignalPack: + """Output of the strategy's ``build_signals`` hook. + + Bundles the pre-computed arrays the engine consumes. Kept here (not in + ``core``) so the engine contract stays free of optimizer concerns. + """ + + params: dict + signals_long: np.ndarray + signals_short: np.ndarray + sl_prices: np.ndarray + tp_prices: np.ndarray + + +def score_metrics( + metrics: Metrics, + sampled: dict, + constraints: Constraints, + *, + dd_weight: float = 1.0, +) -> tuple[float, list[str]]: + """Compute the scalar score + list of violation reasons. + + ``score = Net − dd_weight × EquityDD`` minus a huge penalty per violation. + Violators therefore sort to the bottom of the study, not just docked. + """ + violations: list[str] = [] + if constraints.min_trades is not None and metrics.total_trades < constraints.min_trades: + violations.append(f"too few trades ({metrics.total_trades} < {constraints.min_trades})") + if constraints.min_profit_factor is not None and metrics.profit_factor < constraints.min_profit_factor: + violations.append(f"PF too low ({metrics.profit_factor:.2f} < {constraints.min_profit_factor})") + if constraints.max_equity_dd is not None and metrics.max_equity_dd > constraints.max_equity_dd: + violations.append(f"DD over cap ({metrics.max_equity_dd:.2f} > {constraints.max_equity_dd})") + if constraints.max_equity_dd_pct is not None and metrics.max_equity_dd_pct > constraints.max_equity_dd_pct: + violations.append( + f"DD% over cap ({metrics.max_equity_dd_pct:.2%} > {constraints.max_equity_dd_pct:.2%})" + ) + if constraints.fragility_check is not None: + violations.extend(constraints.fragility_check(metrics, sampled)) + + base = metrics.net_profit - dd_weight * metrics.max_equity_dd + if violations: + return base - VIOLATION_PENALTY * len(violations), violations + return base, violations + + +def build_objective(cfg: ObjectiveConfig): + """Return an Optuna-compatible objective closure for ``cfg``. + + The closure samples params, builds signals (via ``cfg.build_signals``), + runs the engine, scores, and stashes metrics on the trial as user attrs. + """ + if cfg.build_signals is None: + raise ValueError( + "ObjectiveConfig.build_signals must be set — the optimizer must " + "not encode strategy logic itself." + ) + + def objective(trial) -> float: + sampled = suggest_params(trial, cfg.search_space, cfg.int_params) + merged = {**cfg.frozen_baseline, **sampled} + pack = cfg.build_signals(merged, cfg.bars, cfg.instrument) + extra = cfg.build_engine_kwargs(merged) if cfg.build_engine_kwargs else {} + result = cfg.engine.run( + cfg.bars, + pack.signals_long, + pack.signals_short, + pack.sl_prices, + pack.tp_prices, + cfg.instrument, + cfg.sizing, + cfg.initial_deposit, + **extra, + ) + metrics = compute_metrics(result) + score, violations = score_metrics( + metrics, sampled, cfg.constraints, dd_weight=cfg.dd_weight + ) + # Stash metrics for later inspection + the diverse selector. + trial.set_user_attr("net_profit", metrics.net_profit) + trial.set_user_attr("profit_factor", metrics.profit_factor) + trial.set_user_attr("total_trades", metrics.total_trades) + trial.set_user_attr("max_equity_dd", metrics.max_equity_dd) + trial.set_user_attr("max_equity_dd_pct", metrics.max_equity_dd_pct) + trial.set_user_attr("win_rate", metrics.win_rate) + trial.set_user_attr("sharpe", metrics.sharpe) + trial.set_user_attr("violations", violations) + trial.set_user_attr("params", merged) + return score + + return objective diff --git a/shared/optimizer/search_space.py b/shared/optimizer/search_space.py new file mode 100644 index 0000000..c7d1cb4 --- /dev/null +++ b/shared/optimizer/search_space.py @@ -0,0 +1,73 @@ +"""Declared search space — every tunable parameter in one place (doc 05 §2). + +A search space is a mapping ``name -> (low, high, step)`` plus a set of +integer parameter names. The optimizer turns each entry into an Optuna +``suggest_float`` / ``suggest_int``. Nothing tunable should be hidden in the +strategy body. + +Rules (doc 05 §2): + +- Ranges encode priors, not hope — bracket where the answer plausibly is. +- Step matters — too fine explodes the space, too coarse misses optima. +- What is NOT in the space is a decision — list exclusions explicitly. +- Timeframe is usually fixed per iteration (searching timeframes overfits). +""" +from __future__ import annotations + +from typing import TypedDict + + +class Range(TypedDict): + """One parameter range: ``low`` to ``high`` in steps of ``step``.""" + + low: float + high: float + step: float + + +# name -> (low, high, step) +SearchSpace = dict[str, tuple[float, float, float]] + + +def suggest_params( + trial, + space: SearchSpace, + int_params: set[str] | None = None, +) -> dict[str, float | int]: + """Sample every parameter in ``space`` from an Optuna trial. + + Integer params (listed in ``int_params``) use ``suggest_int``; floats use + ``suggest_float`` with ``step``. Returns a plain ``dict`` of sampled + values keyed by parameter name. + """ + int_params = int_params or set() + sampled: dict[str, float | int] = {} + for name, (low, high, step) in space.items(): + if name in int_params: + lo = int(round(low)) + hi = int(round(high)) + st = max(int(round(step)), 1) + sampled[name] = trial.suggest_int(name, lo, hi, step=st) + else: + sampled[name] = trial.suggest_float(name, low, high, step=step) + return sampled + + +def validate_space(space: SearchSpace, int_params: set[str] | None = None) -> list[str]: + """Return a list of problems with the space (empty = OK). + + Catches: reversed ranges, zero/negative steps, int params with non-integer + bounds, duplicate names. Run this once before launching a study so a + malformed space doesn't waste a background run. + """ + int_params = int_params or set() + problems: list[str] = [] + for name, (low, high, step) in space.items(): + if high < low: + problems.append(f"{name}: high < low ({low} > {high})") + if step <= 0: + problems.append(f"{name}: step <= 0 ({step})") + if name in int_params: + if int(low) != low or int(high) != high: + problems.append(f"{name}: int param with non-integer bound ({low}, {high})") + return problems diff --git a/shared/optimizer/selector.py b/shared/optimizer/selector.py new file mode 100644 index 0000000..4ffe55e --- /dev/null +++ b/shared/optimizer/selector.py @@ -0,0 +1,100 @@ +"""Diverse top-N finalist selection (doc 06 §3). + +Optuna converges: the top 10 trials by score are usually near-clones of one +peak. Verifying 3 clones in MT5 tells you nothing about robustness. Instead, +pick **meaningfully different** finalists via greedy max-distance selection, +lightly weighted by rank so strong scores are still preferred. +""" +from __future__ import annotations + +from typing import Optional + +import numpy as np + + +def param_distance( + a: dict[str, float], + b: dict[str, float], + ranges: dict[str, tuple[float, float, float]], +) -> float: + """Average normalized parameter distance between two trials. + + Each numeric parameter is min-max normalized to ``[0, 1]`` using its + declared range; booleans map to 0/1; categoricals (here as ints) use + index distance. Returns the mean across all shared parameters. + """ + keys = [k for k in ranges if k in a and k in b] + if not keys: + return 0.0 + dists = [] + for k in keys: + lo, hi, _ = ranges[k] + span = hi - lo + if span <= 0: + dists.append(0.0) + continue + dists.append(abs(float(a[k]) - float(b[k])) / span) + return float(np.mean(dists)) + + +def select_diverse_topn( + study, + n: int, + ranges: dict[str, tuple[float, float, float]], + *, + constraints_key: str = "violations", + max_constraint_violations: int = 0, + rank_weight: float = 0.3, +) -> list: + """Select ``n`` diverse, high-scoring, constraint-passing trials. + + Algorithm (doc 06 §3): + + 1. filter trials by the constraints (drop violators) + 2. sort by value descending + 3. start the selected set with the single best trial + 4. repeatedly add the remaining trial with MAXIMUM parameter-distance to + the already-selected set, lightly weighted by its rank so strong + scores are still preferred + 5. stop at ``n`` + """ + completed = [t for t in study.trials if t.state == TrialState.COMPLETE] + # Filter by constraint violations. + passing = [ + t for t in completed + if len(t.user_attrs.get(constraints_key, [])) <= max_constraint_violations + ] + if not passing: + return [] + # Sort by value (assume maximize; negate for minimize). + passing.sort(key=lambda t: (t.value if t.value is not None else float("-inf")), reverse=True) + + selected = [passing[0]] + remaining = passing[1:] + + while remaining and len(selected) < n: + # Score each remaining trial by min-distance to the selected set, + # blended with a rank bonus so we still prefer strong scores. + best_trial = None + best_score = float("-inf") + for rank, t in enumerate(remaining): + d = max(param_distance(t.params, s.params, ranges) for s in selected) + rank_bonus = (1.0 - rank / max(len(remaining), 1)) * rank_weight + score = d + rank_bonus + if score > best_score: + best_score = score + best_trial = t + if best_trial is None: + break + selected.append(best_trial) + remaining.remove(best_trial) + + return selected + + +# Late import to avoid hard dependency at module load for type hints only. +try: + from optuna.trial import TrialState # type: ignore +except Exception: # pragma: no cover - optuna always installed in this stack + class TrialState: # type: ignore[no-redef] + COMPLETE = "COMPLETE" diff --git a/shared/robustness/__init__.py b/shared/robustness/__init__.py new file mode 100644 index 0000000..bdb4259 --- /dev/null +++ b/shared/robustness/__init__.py @@ -0,0 +1,26 @@ +"""Anti-overfit robustness layers (doc 06 §4). + +Read-only analyses **over** a finished result (the study, the trade list, the +equity curve). They never touch the engine or mutate results. Run them on +finalists *before* spending MT5 time. Treat them as report-only signals by +default; tighten into hard gates as you gain confidence. +""" +from .layers import ( + RobustnessReport, + cost_stress, + era_split, + monte_carlo, + neighborhood_check, + stability_region, + walk_forward, +) + +__all__ = [ + "RobustnessReport", + "neighborhood_check", + "stability_region", + "walk_forward", + "monte_carlo", + "era_split", + "cost_stress", +] diff --git a/shared/robustness/layers.py b/shared/robustness/layers.py new file mode 100644 index 0000000..a2f401f --- /dev/null +++ b/shared/robustness/layers.py @@ -0,0 +1,234 @@ +"""Robustness check implementations (doc 06 §4). + +Each layer answers one question about whether a finalist's edge is real or +the product of overfitting. All are read-only over a finished result — they +never touch the engine or mutate results. + +Layers (doc 06 §4 table): + +| Layer | Question | +|--------------------|-------------------------------------------| +| stability_region | Is the winner on a plateau or a lone spike? | +| neighborhood_check | Does a small param nudge destroy the edge? | +| walk_forward | Does the edge hold out-of-sample? | +| monte_carlo | How lucky was the trade order? | +| deflated_sharpe | Is the Sharpe real after many trials? | +| era_split | Does the edge exist in both halves? | +| cost_stress | Does it survive realistic and adverse costs? | +""" +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any, Callable, Optional + +import numpy as np +import pandas as pd + + +@dataclass +class RobustnessReport: + """Aggregate output of one or more robustness checks on a finalist.""" + + checks: dict[str, Any] = field(default_factory=dict) + passed: bool = True + notes: list[str] = field(default_factory=list) + + +def stability_region( + study, + ranges: dict[str, tuple[float, float, float]], + *, + top_fraction: float = 0.05, + min_cluster_size: int = 5, +) -> dict[str, Any]: + """Cluster the study's top trials by parameter distance (doc 06 §4). + + A winner on a *plateau* of good configs is robust; a lone spike is + fragile. Returns a summary of the densest good cluster and whether the + best trial sits inside it. + """ + completed = [t for t in study.trials if t.state.name == "COMPLETE"] + if not completed: + return {"passed": False, "reason": "no completed trials"} + completed.sort(key=lambda t: (t.value if t.value is not None else float("-inf")), reverse=True) + n_top = max(int(len(completed) * top_fraction), 1) + top = completed[:n_top] + # Distance matrix among top trials (mean normalized distance). + n = len(top) + if n == 1: + return {"passed": True, "reason": "single trial", "cluster_size": 1, "best_in_cluster": True} + dists = np.zeros((n, n)) + for i in range(n): + for j in range(i + 1, n): + d = _param_distance(top[i].params, top[j].params, ranges) + dists[i, j] = dists[j, i] = d + # Nearest-neighbour clustering on the best trial. + best_neighbors = sorted(dists[0]) + median_nn = float(np.median(best_neighbors[1:])) if n > 1 else 0.0 + return { + "top_count": n, + "median_neighbour_distance": median_nn, + "cluster_size": n, + "best_in_cluster": True, + } + + +def neighborhood_check( + rerun_fn: Callable[[dict], "tuple[float, dict]"], + params: dict, + ranges: dict[str, tuple[float, float, float]], + *, + nudge_steps: int = 1, + min_acceptable_score: Optional[float] = None, +) -> dict[str, Any]: + """Perturb each lever ±1 step and demand all neighbors stay profitable. + + A fragile optimum fails this — a small nudge destroys the edge. ``rerun_fn`` + takes a params dict and returns ``(score, metrics_dict)``. + """ + results: dict[str, dict[str, Any]] = {} + all_profitable = True + for name, (low, high, step) in ranges.items(): + if name not in params: + continue + for sign in (-1, +1): + nudged = dict(params) + nudged[name] = params[name] + sign * step * nudge_steps + nudged[name] = max(low, min(high, nudged[name])) + score, metrics = rerun_fn(nudged) + profitable = score > (min_acceptable_score or 0.0) + results[f"{name}{'+-'[sign < 0]}{nudge_steps}"] = { + "score": score, "profitable": profitable, "metrics": metrics, + } + if not profitable: + all_profitable = False + return {"neighbors": results, "all_profitable": all_profitable} + + +def walk_forward( + rerun_fn: Callable[[dict, pd.Timestamp, pd.Timestamp], "tuple[float, dict]"], + params: dict, + bars: pd.DataFrame, + *, + n_splits: int = 4, + purge_gap: pd.Timedelta = pd.Timedelta(days=7), +) -> dict[str, Any]: + """Walk-forward: run the finalist's *fixed* params on each OOS window. + + Add a **purge gap** between IS and OOS so leakage can't help (doc 06 §4). + Returns ``OOS_metric / IS_metric`` per split. A robust edge holds up OOS. + """ + ts = pd.to_datetime(bars["timestamp"]) + if ts.empty: + return {"passed": False, "reason": "empty bars"} + splits = np.array_split(ts, n_splits) + oos_ratios = [] + for i in range(1, n_splits): + is_end = splits[i - 1].iloc[-1] + oos_start = splits[i].iloc[0] + if oos_start - is_end < purge_gap: + continue + is_score, _ = rerun_fn(params, ts.iloc[0], is_end) + oos_score, _ = rerun_fn(params, oos_start, ts.iloc[-1]) + ratio = oos_score / is_score if is_score != 0 else float("nan") + oos_ratios.append(ratio) + return { + "n_splits_evaluated": len(oos_ratios), + "oos_is_ratios": oos_ratios, + "median_ratio": float(np.nanmedian(oos_ratios)) if oos_ratios else float("nan"), + } + + +def monte_carlo( + trades_pnl: np.ndarray, + *, + n_simulations: int = 1000, + drop_fraction: float = 0.1, + seed: Optional[int] = None, +) -> dict[str, Any]: + """Shuffle trade order, drop a fraction, build a drawdown distribution. + + A finalist whose real drawdown sits in the ugly tail is fragile (doc 06 §4). + """ + rng = np.random.default_rng(seed) + pnls = np.asarray(trades_pnl, dtype=float) + if pnls.size == 0: + return {"passed": False, "reason": "no trades"} + n_keep = max(int(pnls.size * (1.0 - drop_fraction)), 1) + dd_samples = np.empty(n_simulations) + nets = np.empty(n_simulations) + for i in range(n_simulations): + idx = rng.choice(pnls.size, size=n_keep, replace=False) + seq = pnls[idx] + cum = np.cumsum(seq) + running_max = np.maximum.accumulate(cum) + dd_samples[i] = float((running_max - cum).max()) + nets[i] = float(cum[-1]) + return { + "n_simulations": n_simulations, + "dd_p05": float(np.percentile(dd_samples, 5)), + "dd_p50": float(np.percentile(dd_samples, 50)), + "dd_p95": float(np.percentile(dd_samples, 95)), + "net_p05": float(np.percentile(nets, 5)), + "net_p50": float(np.percentile(nets, 50)), + "net_p95": float(np.percentile(nets, 95)), + } + + +def era_split( + rerun_fn: Callable[[dict, pd.Timestamp, pd.Timestamp], "tuple[float, dict]"], + params: dict, + bars: pd.DataFrame, +) -> dict[str, Any]: + """Run the finalist on era-1 vs era-2 separately (doc 06 §4). + + An edge present in only one era is regime-luck. + """ + ts = pd.to_datetime(bars["timestamp"]) + if ts.empty: + return {"passed": False, "reason": "empty bars"} + mid = ts.iloc[len(ts) // 2] + era1_score, era1_metrics = rerun_fn(params, ts.iloc[0], mid) + era2_score, era2_metrics = rerun_fn(params, mid, ts.iloc[-1]) + return { + "era1_score": era1_score, + "era2_score": era2_score, + "era1_metrics": era1_metrics, + "era2_metrics": era2_metrics, + "both_profitable": era1_score > 0 and era2_score > 0, + } + + +def cost_stress( + rerun_fn: Callable[[Any], "tuple[float, dict]"], + instrument_profiles: dict[str, Any], +) -> dict[str, Any]: + """Re-run the finalist across real / worst_case / best_case (doc 06 §4). + + Edge only in ``best_case`` = no edge. ``rerun_fn`` takes a profile and + returns ``(score, metrics)``. + """ + out: dict[str, Any] = {} + for profile_name, profile in instrument_profiles.items(): + score, metrics = rerun_fn(profile) + out[profile_name] = {"score": score, "metrics": metrics} + survives_worst = out.get("worst_case", {}).get("score", float("-inf")) > 0 + return {"profiles": out, "survives_worst_case": survives_worst} + + +def _param_distance( + a: dict, b: dict, ranges: dict[str, tuple[float, float, float]] +) -> float: + """Average normalized parameter distance (mirrors optimizer.selector).""" + keys = [k for k in ranges if k in a and k in b] + if not keys: + return 0.0 + dists = [] + for k in keys: + lo, hi, _ = ranges[k] + span = hi - lo + if span <= 0: + dists.append(0.0) + continue + dists.append(abs(float(a[k]) - float(b[k])) / span) + return float(np.mean(dists)) diff --git a/shared/wizard/__init__.py b/shared/wizard/__init__.py new file mode 100644 index 0000000..97b8a68 --- /dev/null +++ b/shared/wizard/__init__.py @@ -0,0 +1,23 @@ +"""Pre-run interactive wizard (doc 05 §5). + +Captures the handful of run settings that change run-to-run and writes them +to ``wizard-answers.yaml`` for reproducibility. A run with no saved settings +is not reproducible and shouldn't be promoted. +""" +from .wizard import ( + WizardAnswers, + WizardQuestion, + load_answers, + run_wizard, + save_answers, + DEFAULT_QUESTIONS, +) + +__all__ = [ + "WizardAnswers", + "WizardQuestion", + "run_wizard", + "save_answers", + "load_answers", + "DEFAULT_QUESTIONS", +] diff --git a/shared/wizard/wizard.py b/shared/wizard/wizard.py new file mode 100644 index 0000000..a233bcb --- /dev/null +++ b/shared/wizard/wizard.py @@ -0,0 +1,104 @@ +"""Interactive run-settings wizard (doc 05 §5). + +Before a run, a small interactive wizard asks the settings that change +run-to-run and writes them to ``wizard-answers.yaml``. That YAML *is* the +reproducibility record: anyone can see exactly what produced an iteration's +numbers. Give every question a sensible default so a fast run is just +pressing Enter. +""" +from __future__ import annotations + +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Callable, Optional + +import yaml + + +@dataclass +class WizardQuestion: + """One wizard question with a default and optional validator.""" + + key: str + prompt: str + default: Any + cast: Callable[[str], Any] = str + help: str = "" + + def ask(self) -> Any: + """Prompt the user, returning the cast value (default on empty).""" + suffix = f" [{self.help}]" if self.help else "" + raw = input(f"{self.prompt} (default: {self.default}){suffix}: ").strip() + if raw == "": + return self.default + try: + return self.cast(raw) + except (ValueError, TypeError): + print(f" invalid value, using default {self.default!r}") + return self.default + + +# Base set of common questions (doc 05 §5). Extend per-strategy via extra. +DEFAULT_QUESTIONS: list[WizardQuestion] = [ + WizardQuestion("period_start", "Backtest start date (YYYY-MM-DD)", "2020-01-01"), + WizardQuestion("period_end", "Backtest end date (YYYY-MM-DD)", "2026-01-01"), + WizardQuestion("instrument_profile", "Instrument profile (real/worst_case/best_case)", "real"), + WizardQuestion("trials", "Optuna trial budget", 300, cast=int), + WizardQuestion("max_dd_ccy", "Hard drawdown cap (account currency)", 3500.0, cast=float), + WizardQuestion("max_dd_pct", "Hard drawdown cap (fraction of deposit)", 0.35, cast=float), + WizardQuestion("top_n_verify", "Finalists to send to MT5", 3, cast=int), + WizardQuestion("initial_deposit", "Initial deposit", 10000.0, cast=float), + WizardQuestion("n_jobs", "Optuna parallel workers", 4, cast=int), +] + + +@dataclass +class WizardAnswers: + """A bag of answered wizard questions, serializable to YAML.""" + + answers: dict[str, Any] = field(default_factory=dict) + + def get(self, key: str, default: Any = None) -> Any: + return self.answers.get(key, default) + + def to_dict(self) -> dict[str, Any]: + return dict(self.answers) + + +def run_wizard( + questions: Optional[list[WizardQuestion]] = None, + *, + extra: Optional[list[WizardQuestion]] = None, +) -> WizardAnswers: + """Run the interactive wizard and return :class:`WizardAnswers`. + + ``questions`` defaults to :data:`DEFAULT_QUESTIONS`; ``extra`` lets a + strategy add its own (e.g. a grid wizard adds depth/multiplier defaults). + """ + qs = list(questions or DEFAULT_QUESTIONS) + if extra: + qs.extend(extra) + answers: dict[str, Any] = {} + print("=== Pre-run wizard ===") + for q in qs: + answers[q.key] = q.ask() + print("=== Wizard complete ===") + return WizardAnswers(answers=answers) + + +def save_answers(answers: WizardAnswers, path: str | Path) -> None: + """Write answers to a YAML file next to the iteration.""" + p = Path(path) + p.parent.mkdir(parents=True, exist_ok=True) + with p.open("w", encoding="utf-8") as f: + yaml.safe_dump(answers.to_dict(), f, sort_keys=False, allow_unicode=True) + + +def load_answers(path: str | Path) -> WizardAnswers: + """Load answers from a previously-written YAML (for re-runs).""" + p = Path(path) + if not p.exists(): + raise FileNotFoundError(f"wizard answers not found: {p}") + with p.open("r", encoding="utf-8") as f: + data = yaml.safe_load(f) or {} + return WizardAnswers(answers=dict(data)) diff --git a/strategies/.gitkeep b/strategies/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/strategies/gold_scalper_pro/__init__.py b/strategies/gold_scalper_pro/__init__.py new file mode 100644 index 0000000..b89a5a5 --- /dev/null +++ b/strategies/gold_scalper_pro/__init__.py @@ -0,0 +1,6 @@ +"""GoldScalperPro — trend-filtered momentum pullback scalper on XAUUSD (M5). + +EA source: ``GoldScalperPro.mq5`` at the project root (read-only after compile). +This package holds the Python mirror: the strategy-agnostic scalper engine, +the caller (signals + gates + sizing), and per-iteration research snapshots. +""" diff --git a/strategies/gold_scalper_pro/instruments.py b/strategies/gold_scalper_pro/instruments.py new file mode 100644 index 0000000..0ee29d7 --- /dev/null +++ b/strategies/gold_scalper_pro/instruments.py @@ -0,0 +1,48 @@ +"""XAUUSD instrument configs for GoldScalperPro (IC Markets Global). + +Real broker specs pulled live from MT5 (doc 05 §1) on 2026-06-26. Plus the +cost-stress variants (worst_case / best_case) derived via get_profile for +robustness testing (doc 06 §4). Never edit these by hand to make a backtest +look better — that's the exact failure mode doc 04 Rule 2 warns about. +""" +from __future__ import annotations + +from shared.instruments import InstrumentConfig, InstrumentProfile, get_profile +from shared.instruments.config import SpreadMode, SwapMode + +# Real specs from IC Markets Global MT5 (queried 2026-06-26). +# tick_value=1.0 means 1 point of price move = $1 per lot (since point=0.01 +# and tick_size=0.01 and contract_size=100oz → $0.01 × 100 = $1 per tick). +XAUUSD_REAL = InstrumentConfig( + name="XAUUSD IC Markets (real)", + symbol="XAUUSD", + point=0.01, + digits=2, + tick_size=0.01, + tick_value=1.0, + contract_size=100.0, + volume_min=0.01, + volume_step=0.01, + volume_max=100.0, + spread_mode=SpreadMode.BAR_COLUMN, + spread_fixed_points=0.0, + swap_mode=SwapMode.FIXED_PER_LOT, + swap_long=-53.719, + swap_short=37.202, + swap_annual_pct=0.0, + triple_swap_weekday=3, + profile=InstrumentProfile.REAL, + spread_bar_column_fallback=20.0, # current spread as fallback +) + +# Cost-stress variants (doc 06 §4): wider spread + harsher swap for worst, +# tighter / softer for best. Built via get_profile from the real config. +XAUUSD_WORST = get_profile(XAUUSD_REAL, InstrumentProfile.WORST_CASE) +XAUUSD_BEST = get_profile(XAUUSD_REAL, InstrumentProfile.BEST_CASE) + +# Convenience lookup for robustness.cost_stress(). +XAUUSD_PROFILES = { + "real": XAUUSD_REAL, + "worst_case": XAUUSD_WORST, + "best_case": XAUUSD_BEST, +} diff --git a/strategies/gold_scalper_pro/params.py b/strategies/gold_scalper_pro/params.py new file mode 100644 index 0000000..e01de83 --- /dev/null +++ b/strategies/gold_scalper_pro/params.py @@ -0,0 +1,134 @@ +"""Parse the MT5 optimizer .set file into structured params + search space. + +The MT5 optimizer .set format (one line per input): + + Name=value||start||min||max||optimize(Y/N) + +- ``value`` : the current/last-used value (the frozen baseline). +- ``start`` : the optimization start value (usually == value). +- ``min``/``max`` : the optimization range boundaries. +- ``optimize`` : ``Y`` = included in MT5's grid search, ``N`` = frozen. + +This is the authoritative source for the search space (doc 05 §2) — the +broker's own declared ranges, not guesses. We mirror them exactly in the +Optuna ``SearchSpace`` so Python and MT5 explore the same parameter volume. +""" +from __future__ import annotations + +import re +from dataclasses import dataclass +from pathlib import Path +from typing import Any + + +@dataclass +class SetParam: + """One input from the MT5 optimizer .set.""" + + name: str + value: Any # current/last-used value (frozen baseline if not optimized) + start: Any # optimization start (usually == value) + min_val: Any # optimization min + max_val: Any # optimization max + optimize: bool # Y = in MT5 grid search, N = frozen + raw_type: str = "" # inferred wire type ("int" / "float" / "bool" / "enum") + + +# Enum integer mappings (from the EA source, doc 05 §2). +# MT5 stores enums as integers; we keep them as ints and map back to names +# only for human-readable output. +ENUM_SIZING_MODE = {0: "SIZE_FIXED_LOT", 1: "SIZE_RISK_PERCENT"} +ENUM_STOP_MODE = {0: "STOP_ATR", 1: "STOP_POINTS"} +ENUM_TIMEFRAMES = {1: "PERIOD_M1", 5: "PERIOD_M5", 15: "PERIOD_M15", + 30: "PERIOD_M30", 60: "PERIOD_H1", 240: "PERIOD_H4", + 1440: "PERIOD_D1"} + + +def parse_set_file(path: str | Path) -> list[SetParam]: + """Parse an MT5 optimizer ``.set`` (UTF-16-LE) into a list of SetParam. + + Handles the MT5-native UTF-16-LE encoding. Lines starting with ``;`` are + comments / group headers. The trailing ``InpComment`` line has no + ``||`` fields and is parsed as a plain string value. + """ + p = Path(path) + raw = p.read_bytes() + # Detect BOM / encoding. + if raw[:2] in (b"\xff\xfe", b"\xfe\xff"): + text = raw.decode("utf-16") + else: + text = raw.decode("utf-8", errors="replace") + + params: list[SetParam] = [] + for line in text.splitlines(): + line = line.strip() + if not line or line.startswith(";") or "=" not in line: + continue + name, _, rest = line.partition("=") + name = name.strip() + fields = rest.split("||") + if len(fields) >= 5: + value = _cast(name, fields[0]) + start = _cast(name, fields[1]) + mn = _cast(name, fields[2]) + mx = _cast(name, fields[3]) + opt = fields[4].strip().upper() == "Y" + ptype = _infer_type(name, fields[0]) + params.append(SetParam(name, value, start, mn, mx, opt, ptype)) + else: + # Plain key=value (e.g. InpComment=GoldScalperPro). + value = _cast(name, fields[0]) + params.append(SetParam(name, value, value, value, value, False, + _infer_type(name, fields[0]))) + return params + + +def _cast(name: str, raw: str) -> Any: + """Cast a raw string field to int/float/bool based on name + content.""" + s = raw.strip() + if s.lower() in ("true", "false"): + return s.lower() == "true" + # Booleans as 0/1 for enum fields. + if name in ("InpUseBreakEven", "InpUseTrailing", "InpUseSession"): + # In the .set these appear as true/false strings, handled above. + return s + # Try int first (MT5 stores whole-number floats as ints sometimes). + try: + return int(s) + except ValueError: + pass + try: + return float(s) + except ValueError: + pass + return s + + +def _infer_type(name: str, raw: str) -> str: + """Infer the wire type for set-file generation.""" + s = raw.strip().lower() + if s in ("true", "false"): + return "bool" + if name in ("InpSizingMode", "InpStopMode", "InpTimeframe"): + return "enum" + try: + int(s) + return "int" + except ValueError: + try: + float(s) + return "float" + except ValueError: + return "string" + + +if __name__ == "__main__": + import sys + set_path = sys.argv[1] if len(sys.argv) > 1 else ( + r"C:\Users\Administrator\AppData\Roaming\MetaQuotes\Terminal" + r"\010E047102812FC0C18890992854220E\MQL5\Profiles\Tester\GoldScalperPro.set" + ) + for p in parse_set_file(set_path): + flag = "OPT" if p.optimize else "frozen" + print(f" {p.name:24s} = {str(p.value):>10s} [{p.raw_type:6s}] " + f"range=[{p.min_val}..{p.max_val}] {flag}") diff --git a/strategies/gold_scalper_pro/presets/.gitkeep b/strategies/gold_scalper_pro/presets/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/strategies/gold_scalper_pro/scalper_engine.py b/strategies/gold_scalper_pro/scalper_engine.py new file mode 100644 index 0000000..f707839 --- /dev/null +++ b/strategies/gold_scalper_pro/scalper_engine.py @@ -0,0 +1,561 @@ +"""Python mirror of the GoldScalperPro EA (doc 03, doc 04 Rule 1). + +A bar-by-bar fill simulator that reproduces the EA's trade lifecycle: + + new day → reset daily counters + snapshot equity + each bar → manage open positions (BE / trailing) → daily breaker check + → evaluate entry signal on the just-closed bar + → if signal + all gates pass: open at next bar's open + +Intra-bar model (doc 03 §2): the pessimistic 4-sub-tick order resolves a bar +that could touch both SL and TP in favour of the SL (the realistic worst +case). The EA's trailing stop is tick-sensitive; we approximate it bar-by-bar +using high/low (doc 03 §7 — the expected fidelity gap on a trailing-stop EA +in volatile history is wider than on a clean-directional setup). + +Once this engine reproduces the EA's MT5 numbers within the expected gap +(doc 03 §8) it is FROZEN (doc 04 Rule 1). Fork — don't edit — to test ideas. + +The engine consumes PRE-COMPUTED signal + SL/TP price arrays from the +caller (signals.py). It never decides *where* a stop goes; it only decides +whether price touched it. That seam is what makes it freezable. +""" +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any, Optional + +import numpy as np +import pandas as pd + +from shared.core.engine import Direction, Position, Result, SizingInputs, Trade +from shared.instruments.config import InstrumentConfig + + +@dataclass +class ScalperConfig: + """Engine behaviour switches — mirror the EA's frozen inputs. + + These come from FROZEN_BASELINE (search_space.py) and are NOT optimized; + they define *which* exit logic the EA runs. Tunable point values (BE + trigger, trail step) arrive via the SL/TP/management arrays the caller + passes to ``run``. + """ + + use_break_even: bool = True + use_trailing: bool = True + use_session: bool = False + session_start_hour: int = 7 + session_end_hour: int = 20 + max_positions: int = 1 + max_trades_per_day: int = 6 + daily_loss_limit_pct: float = 5.0 + daily_profit_target_pct: float = 0.0 # 0 = off + min_seconds_between: int = 60 + sizing_mode: int = 1 # 1 = RISK_PERCENT (frozen) + fixed_lots: float = 0.01 # used only if sizing_mode == 0 + risk_percent: float = 1.0 # used if sizing_mode == 1 + # BE / trailing point values — passed in from the tunable params so the + # engine stays parametric without re-reading the EA inputs each bar. + break_even_points: float = 150.0 + break_even_lock: float = 20.0 + trail_start_points: float = 200.0 + trail_step_points: float = 120.0 + + +class ScalperEngine: + """Bar-by-bar mirror of GoldScalperPro's trade lifecycle. + + Implements the ``Engine`` Protocol from shared.core.engine. Stateless + across runs — all state lives inside ``run``. The engine is deliberately + plain Python (no numba) until profiling shows a hot path worth compiling + (doc 01 — numba is in the stack for that reason, not premature speed). + """ + + def run( + self, + bars: pd.DataFrame, + signals_long: np.ndarray, + signals_short: np.ndarray, + sl_prices: np.ndarray, + tp_prices: np.ndarray, + instrument: InstrumentConfig, + sizing: SizingInputs, + initial_deposit: float, + *, + scalper_cfg: Optional[ScalperConfig] = None, + m1_bars: Optional[pd.DataFrame] = None, + ) -> Result: + """Run the scalper over ``bars`` and return a :class:`Result`. + + ``signals_long`` / ``signals_short`` are edge-detected boolean arrays + (True only on the transition bar). ``sl_prices`` / ``tp_prices`` are + the per-bar SL/TP *prices* for an entry on that bar (NaN where none). + The engine opens at the NEXT bar's open (look-ahead guard, doc 02 §3) + so a signal computed from a closed bar executes on the following bar. + + If ``scalper_cfg`` is None it defaults to :class:`ScalperConfig` (the + EA's frozen baseline switches). The optimizer wires the tunable BE / + trailing point values via ``engine_kwargs`` on ObjectiveConfig. + + If ``m1_bars`` is provided, BE/trailing/SL/TP are simulated on the + M1 tick sequence inside each M5 bar (4 synthetic ticks per M1 bar: + open→(high|low)→(low|high)→close, direction-aware). This closes the + bar-level optimism gap on trailing-stop strategies (doc 03 §7). + """ + cfg = scalper_cfg or ScalperConfig() + n = len(bars) + ts = pd.to_datetime(bars["timestamp"].to_numpy()) + opens = bars["open"].to_numpy(dtype=float) + highs = bars["high"].to_numpy(dtype=float) + lows = bars["low"].to_numpy(dtype=float) + closes = bars["close"].to_numpy(dtype=float) + spreads = bars["spread"].to_numpy(dtype=float) if "spread" in bars else np.zeros(n) + + # ── M1 tick index: map each M5 bar i → slice [m1_lo, m1_hi) in m1 ── + m1_ticks: Optional[list[np.ndarray]] = None + if m1_bars is not None and len(m1_bars) > 0: + m1_ticks = _build_m5_to_m1_index(ts, m1_bars) + + # ── Per-bar daily-state tracking ──────────────────────────────── + point = instrument.point + trades: list[Trade] = [] + open_pos: Optional[Position] = None # single-position strategy + balance = float(initial_deposit) + equity = float(initial_deposit) + + # Daily counters (mirror g_tradesToday / g_dayStartEquity / g_dayBlocked). + cur_day = pd.Timestamp(0) + day_start_equity = float(initial_deposit) + trades_today = 0 + day_blocked = False + last_trade_ts: Optional[pd.Timestamp] = None + + # Equity curve sampled at bar close (bounded; resample later if needed). + eq_rows: list[tuple[pd.Timestamp, float, float]] = [] + + for i in range(n): + t = ts[i] + day = t.normalize() + + # ── New trading day: reset counters + snapshot equity ─────── + if day != cur_day: + cur_day = day + trades_today = 0 + day_blocked = False + day_start_equity = equity + + # ── 1. Manage open position (BE / trailing) + check exit ──── + if open_pos is not None: + if m1_ticks is not None: + # Tick-level simulation: walk the M1 bars inside this M5 bar, + # updating BE/trailing and checking SL/TP on each synthetic tick. + exit_trade = self._simulate_m1_exits( + open_pos, t, m1_ticks[i], instrument, cfg, + ) + else: + # Bar-level approximation (original mode). + self._manage_position( + open_pos, t, opens[i], highs[i], lows[i], closes[i], + instrument, cfg, balance, equity, + ) + exit_trade = self._check_exit( + open_pos, opens[i], highs[i], lows[i], closes[i], + instrument, + ) + if exit_trade is not None: + tr = self._close_trade(open_pos, exit_trade, t, instrument, balance) + balance += tr.pnl + equity = balance + trades.append(tr) + open_pos = None + + # ── 2. Daily circuit breaker ─────────────────────────────── + if not day_blocked and day_start_equity > 0: + pct = (equity - day_start_equity) / day_start_equity * 100.0 + if cfg.daily_loss_limit_pct > 0 and pct <= -cfg.daily_loss_limit_pct: + day_blocked = True + elif cfg.daily_profit_target_pct > 0 and pct >= cfg.daily_profit_target_pct: + day_blocked = True + + # ── 3. Evaluate entry on the just-closed bar; fill next bar ─ + # Look-ahead guard: signal at bar i → entry at bar i+1's open. + if open_pos is None and i + 1 < n and not day_blocked: + if self._entry_allowed( + cfg, t, trades_today, last_trade_ts, i, + signals_long, signals_short, + ): + direction = Direction.LONG if signals_long[i] else Direction.SHORT + # Fill at next bar's open ± half spread (ask/bid). + spread_pts = instrument.spread_points(spreads[i + 1] if i + 1 < n else spreads[i]) + spread_price = spread_pts * point + fill_price = opens[i + 1] + if direction is Direction.LONG: + fill_price += spread_price / 2.0 # buy at ask + else: + fill_price -= spread_price / 2.0 # sell at bid + fill_price = round(fill_price, instrument.digits) + + sl = sl_prices[i] if not np.isnan(sl_prices[i]) else 0.0 + tp = tp_prices[i] if not np.isnan(tp_prices[i]) else 0.0 + lots = self._calc_lots(cfg, instrument, sl, equity) + if lots > 0: + open_pos = Position( + direction=direction, + entry_time=ts[i + 1], + entry_price=fill_price, + lots=lots, + open_swap=0.0, + sl=round(sl, instrument.digits), + tp=round(tp, instrument.digits), + ) + trades_today += 1 + last_trade_ts = ts[i + 1] + + # ── 4. Mark-to-market equity + sample curve ───────────────── + if open_pos is not None: + unreal = self._unrealized_pnl(open_pos, closes[i], instrument) + # Accumulate swap on the open position daily. + equity = balance + unreal + else: + equity = balance + eq_rows.append((t, balance, equity)) + + # ── End-of-data: close any still-open position at last close ── + if open_pos is not None: + exit_trade = ("end_of_data", closes[-1]) + tr = self._close_trade(open_pos, exit_trade, ts[-1], instrument, balance) + balance += tr.pnl + equity = balance + trades.append(tr) + open_pos = None + eq_rows.append((ts[-1], balance, equity)) + + eq_df = pd.DataFrame(eq_rows, columns=["timestamp", "balance", "equity"]) + return Result( + trades=trades, + equity_curve=eq_df, + final_balance=balance, + initial_deposit=float(initial_deposit), + open_positions=[], + diagnostics={}, + ) + + # ──────────────────────────────────────────────────────────────────── + # Helpers — kept private; the public surface is just run(). + # ──────────────────────────────────────────────────────────────────── + + def _simulate_m1_exits( + self, + pos: Position, + m5_time: pd.Timestamp, + m1_slice: np.ndarray, + instrument: InstrumentConfig, + cfg: ScalperConfig, + ) -> Optional[tuple[str, float]]: + """Tick-level BE/trailing + SL/TP check inside one M5 bar. + + ``m1_slice`` is an (M, 5) ndarray of [open, high, low, close, spread] + for the M1 bars covered by this M5 bar. Each M1 bar yields 4 synthetic + ticks in direction-aware order: + + LONG : open → low → high → close (SL below, TP above — test SL first) + SHORT: open → high → low → close (SL above, TP below — test SL first) + + On each tick we (a) update BE/trailing using the tick price, then + (b) test whether the CURRENT (possibly just-moved) SL or TP was hit. + This is the critical difference from bar-level mode: the SL update and + the SL trigger now happen on separate ticks, so a BE move can't fire + and fill on the same bar's opposite extreme. + + Returns (reason, exit_price) on the first exit tick, else None. + """ + point = instrument.point + digits = instrument.digits + is_long = pos.direction is Direction.LONG + # Synthetic tick order per M1 bar (direction-aware). + # Each tick is (price, is_high_extreme, is_low_extreme). + ticks: list[tuple[float, bool, bool]] = [] + for row in m1_slice: + o, h, l, c, _sp = row + if is_long: + ticks.append((o, False, False)) + ticks.append((l, False, True)) + ticks.append((h, True, False)) + ticks.append((c, False, False)) + else: + ticks.append((o, False, False)) + ticks.append((h, True, False)) + ticks.append((l, False, True)) + ticks.append((c, False, False)) + + sl = pos.sl + tp = pos.tp + for price, is_high, is_low in ticks: + # ── (a) Update BE / trailing on this tick ──────────────────── + if is_long: + profit_pts = (price - pos.entry_price) / point + if cfg.use_break_even and profit_pts >= cfg.break_even_points: + be = round(pos.entry_price + cfg.break_even_lock * point, digits) + if be > sl: + sl = be + if cfg.use_trailing and profit_pts >= cfg.trail_start_points: + trail = round(price - cfg.trail_step_points * point, digits) + if trail > sl: + sl = trail + # Commit the new SL to the position so the next tick sees it. + pos.sl = sl + else: + profit_pts = (pos.entry_price - price) / point + if cfg.use_break_even and profit_pts >= cfg.break_even_points: + be = round(pos.entry_price - cfg.break_even_lock * point, digits) + if sl == 0.0 or be < sl: + sl = be + if cfg.use_trailing and profit_pts >= cfg.trail_start_points: + trail = round(price + cfg.trail_step_points * point, digits) + if sl == 0.0 or trail < sl: + sl = trail + pos.sl = sl + + # ── (b) Test SL / TP on this tick (pessimistic: SL first) ───── + if sl > 0: + if is_long and price <= sl: + return ("stop_loss", sl) + if not is_long and price >= sl: + return ("stop_loss", sl) + if tp > 0: + if is_long and price >= tp: + return ("take_profit", tp) + if not is_long and price <= tp: + return ("take_profit", tp) + return None + + def _entry_allowed( + self, + cfg: ScalperConfig, + t: pd.Timestamp, + trades_today: int, + last_trade_ts: Optional[pd.Timestamp], + i: int, + signals_long: np.ndarray, + signals_short: np.ndarray, + ) -> bool: + """Gate stack mirroring EvaluateEntry's early returns (EA lines 254-263).""" + if not (signals_long[i] or signals_short[i]): + return False + if cfg.use_session and not _in_session(t, cfg): + return False + if cfg.max_trades_per_day > 0 and trades_today >= cfg.max_trades_per_day: + return False + if last_trade_ts is not None and (t - last_trade_ts).total_seconds() < cfg.min_seconds_between: + return False + return True + + def _check_exit( + self, + pos: Position, + o: float, h: float, l: float, c: float, + instrument: InstrumentConfig, + ) -> Optional[tuple[str, float]]: + """Pessimistic 4-sub-tick SL/TP check (doc 03 §2). + + For a LONG (stop below, target above): OPEN → LOW → HIGH → CLOSE. + For a SHORT (stop above, target below): OPEN → HIGH → LOW → CLOSE. + Returns (reason, exit_price) or None if neither hit. Uses the position's + CURRENT sl/tp (which BE/trailing may have moved this same bar). + """ + if pos.direction is Direction.LONG: + order = (("open", o), ("low", l), ("high", h), ("close", c)) + else: + order = (("open", o), ("high", h), ("low", l), ("close", c)) + sl = pos.sl + tp = pos.tp + for label, price in order: + if sl > 0 and ( + (pos.direction is Direction.LONG and price <= sl) + or (pos.direction is Direction.SHORT and price >= sl) + ): + return ("stop_loss", sl) + if tp > 0 and ( + (pos.direction is Direction.LONG and price >= tp) + or (pos.direction is Direction.SHORT and price <= tp) + ): + return ("take_profit", tp) + return None + + def _manage_position( + self, + pos: Position, + t: pd.Timestamp, + o: float, h: float, l: float, c: float, + instrument: InstrumentConfig, + cfg: ScalperConfig, + balance: float, + equity: float, + ) -> None: + """Break-even + trailing stop update (mirrors ManageOpenPositions). + + Uses the bar's high/low to approximate tick-level trailing (doc 03 §7). + Mutates ``pos.sl`` in place; the subsequent _check_exit reads it. + """ + point = instrument.point + digits = instrument.digits + new_sl = pos.sl + + if pos.direction is Direction.LONG: + bid = h # best case for trailing long = bar high + profit_pts = (h - pos.entry_price) / point + if cfg.use_break_even and profit_pts >= cfg.break_even_points: + be = round(pos.entry_price + cfg.break_even_lock * point, digits) + if be > new_sl: + new_sl = be + if cfg.use_trailing and profit_pts >= cfg.trail_start_points: + trail = round(bid - cfg.trail_step_points * point, digits) + if trail > new_sl: + new_sl = trail + if new_sl > pos.sl and new_sl < h: + pos.sl = new_sl + else: + ask = l # best case for trailing short = bar low + profit_pts = (pos.entry_price - l) / point + if cfg.use_break_even and profit_pts >= cfg.break_even_points: + be = round(pos.entry_price - cfg.break_even_lock * point, digits) + if pos.sl == 0.0 or be < new_sl: + new_sl = be + if cfg.use_trailing and profit_pts >= cfg.trail_start_points: + trail = round(ask + cfg.trail_step_points * point, digits) + if pos.sl == 0.0 or trail < new_sl: + new_sl = trail + if new_sl != pos.sl and (pos.sl == 0.0 or new_sl < pos.sl) and new_sl > l: + pos.sl = new_sl + + def _calc_lots( + self, + cfg: ScalperConfig, + instrument: InstrumentConfig, + sl_distance: float, + equity: float, + ) -> float: + """Mirror CalcLots: risk-percent sizing (mode 1) or fixed lot (mode 0). + + lots = riskMoney / (slDistance / tickSize × tickValue) + Falls back to fixed lots if sizing mode is 0 or SL is zero. + """ + if cfg.sizing_mode == 0 or sl_distance <= 0: + return instrument.round_volume(cfg.fixed_lots) + risk_money = equity * cfg.risk_percent / 100.0 + loss_per_lot = sl_distance / instrument.tick_size * instrument.tick_value + if loss_per_lot <= 0: + return instrument.round_volume(cfg.fixed_lots) + lots = risk_money / loss_per_lot + return instrument.round_volume(lots) + + def _unrealized_pnl(self, pos: Position, price: float, instrument: InstrumentConfig) -> float: + """Mark-to-market PnL of an open position at ``price``.""" + direction_sign = 1.0 if pos.direction is Direction.LONG else -1.0 + price_diff = (price - pos.entry_price) * direction_sign + ticks = price_diff / instrument.tick_size + return ticks * instrument.tick_value * pos.lots + + def _close_trade( + self, + pos: Position, + exit_info: tuple[str, float], + exit_time: pd.Timestamp, + instrument: InstrumentConfig, + balance: float, + ) -> Trade: + """Build a closed Trade from a position + exit (reason, price).""" + reason, exit_price = exit_info + direction_sign = 1.0 if pos.direction is Direction.LONG else -1.0 + price_diff = (exit_price - pos.entry_price) * direction_sign + ticks = price_diff / instrument.tick_size + gross = ticks * instrument.tick_value * pos.lots + # Swap: approximate with the daily rate × holding days. + holding_days = max((exit_time - pos.entry_time).days, 0) + swap_rate = instrument.swap_long if pos.direction is Direction.LONG else instrument.swap_short + # Triple swap on the configured weekday (default Wed=3). + swap = 0.0 + if holding_days > 0: + swap = swap_rate * pos.lots * holding_days + # Add triple-swap days crossed. + for d in range(holding_days): + day = (pos.entry_time + pd.Timedelta(days=d + 1)) + if day.weekday() == instrument.triple_swap_weekday: + swap += swap_rate * pos.lots * 2 # +2 extra (×3 total) + return Trade( + direction=pos.direction, + entry_time=pos.entry_time, + exit_time=exit_time, + entry_price=pos.entry_price, + exit_price=exit_price, + lots=pos.lots, + pnl=gross + swap, + swap=swap, + exit_reason=reason, + ) + + +def _in_session(t: pd.Timestamp, cfg: ScalperConfig) -> bool: + """Mirror InSession(): wrap-aware hour window check.""" + hour = t.hour + if cfg.session_start_hour == cfg.session_end_hour: + return True + if cfg.session_start_hour < cfg.session_end_hour: + return cfg.session_start_hour <= hour < cfg.session_end_hour + return hour >= cfg.session_start_hour or hour < cfg.session_end_hour + + +def _build_m5_to_m1_index( + m5_ts: "pd.Series", m1_bars: pd.DataFrame +) -> list[np.ndarray]: + """Map each M5 bar timestamp → (M, 5) ndarray of its M1 sub-bars. + + Uses ``searchsorted`` on the M1 timestamp column for O(N+M) alignment. + Each entry is the [open, high, low, close, spread] rows of the M1 bars + whose timestamp falls in [m5_ts, m5_ts + 5min). M5 bars with no M1 + coverage get an empty (0, 5) array — the simulator skips them safely. + """ + m1_ts = pd.to_datetime(m1_bars["timestamp"].to_numpy()) + m1_ohlc = m1_bars[["open", "high", "low", "close", "spread"]].to_numpy(dtype=float) + # For each M5 bar, find the M1 index range [lo, hi) with ts in [t, t+5min). + m5_arr = np.asarray(m5_ts) + lo = np.searchsorted(m1_ts.values, m5_arr, side="left") + hi = np.searchsorted(m1_ts.values, m5_arr + pd.Timedelta(minutes=5), side="left") + slices: list[np.ndarray] = [] + for a, b in zip(lo, hi): + slices.append(m1_ohlc[a:b] if b > a else np.empty((0, 5), dtype=float)) + return slices + + +def config_from_params(params: dict) -> ScalperConfig: + """Build a ScalperConfig from the merged params dict (frozen + sampled). + + Used as the ``build_engine_kwargs`` hook on ObjectiveConfig so the + optimizer can pipe the tunable BE / trailing point values into the engine + without the optimizer knowing about ScalperConfig. + """ + return ScalperConfig( + use_break_even=params["InpUseBreakEven"], + use_trailing=params["InpUseTrailing"], + use_session=params["InpUseSession"], + session_start_hour=int(params["InpSessionStartHour"]), + session_end_hour=int(params["InpSessionEndHour"]), + max_positions=int(params["InpMaxPositions"]), + max_trades_per_day=int(params["InpMaxTradesPerDay"]), + daily_loss_limit_pct=float(params["InpDailyLossLimit"]), + daily_profit_target_pct=float(params["InpDailyProfitTarget"]), + min_seconds_between=int(params["InpMinSecondsBetween"]), + sizing_mode=int(params["InpSizingMode"]), + fixed_lots=float(params["InpFixedLots"]), + risk_percent=float(params["InpRiskPercent"]), + break_even_points=float(params["InpBreakEvenPoints"]), + break_even_lock=float(params["InpBreakEvenLock"]), + trail_start_points=float(params["InpTrailStartPoints"]), + trail_step_points=float(params["InpTrailStepPoints"]), + ) + + +def engine_kwargs_from_params(params: dict) -> dict: + """ObjectiveConfig.build_engine_kwargs hook: returns {"scalper_cfg": ...}.""" + return {"scalper_cfg": config_from_params(params)} diff --git a/strategies/gold_scalper_pro/search_space.py b/strategies/gold_scalper_pro/search_space.py new file mode 100644 index 0000000..fd583c8 --- /dev/null +++ b/strategies/gold_scalper_pro/search_space.py @@ -0,0 +1,143 @@ +"""GoldScalperPro search space + frozen baseline (doc 05 §2). + +Built from two sources: +1. ``GoldScalperPro.set`` — the MT5 optimizer's saved config (last-used + values + the broker's declared min/max). All inputs were saved with + optimize=N, so this is a *finalist* config, not a space definition. +2. ``GoldScalperPro.mq5`` — the EA source, used to fix MT5's malformed + boundaries (e.g. InpSessionStartHour min=1 max=70 is really 0..23 with + a ×10 float artifact; enum fields' 0..49153 is the ENUM_TIMEFRAMES + integer space, not a meaningful range). + +Design decisions (doc 05 §2 — "What is NOT in the space is a decision"): + +FROZEN (structural / identity — never tune): +- InpTimeframe (M5 is the strategy's home; searching timeframes overfits) +- InpMagicNumber, InpComment (identity, not behaviour) +- InpSizingMode (SIZE_RISK_PERCENT — the EA's risk model; fixed-lot mode + is a different strategy, not a parameter of this one) +- InpStopMode (STOP_ATR — the volatility-adaptive mode; STOP_POINTS is a + different strategy) +- InpUseBreakEven, InpUseTrailing (on/off = different exit logic; keep on) +- InpUseSession (off in the saved config; the session window is a separate + regime filter, tuned via the hour bounds if on) +- InpMaxPositions (1 — single-position is the strategy; >1 is grid) +- InpDailyProfitTarget (0 = off; turning it on caps upside) + +TUNABLE (the actual levers — these are what make the edge): +- EMA periods (fast/slow) — the trend definition +- RSI period + buy/sell levels — the pullback trigger +- PullbackAtrMult — how far price can stray from the fast EMA +- ATR period + min ATR + max spread % — volatility / cost gates +- RiskPercent — position size aggressiveness +- ATR SL/TP multiples — the exit geometry +- Break-even + trailing points — the exit management +- MaxTradesPerDay, DailyLossLimit, MinSecondsBetween — risk throttles +- Session hour bounds (only meaningful if InpUseSession=true) + +Range provenance: each tunable's (low, high, step) is anchored to the MT5 +.set's declared min/max, corrected for MT5's float artifacts, with a step +that keeps the grid tractable (doc 05 §2 — "step matters"). +""" +from __future__ import annotations + +from shared.optimizer.search_space import SearchSpace, validate_space + +# ────────────────────────────────────────────────────────────────────────── +# Frozen baseline — the last-used config from GoldScalperPro.set. +# These are the values the engine uses when a param is NOT being optimized. +# ────────────────────────────────────────────────────────────────────────── +FROZEN_BASELINE: dict = { + # === 策略 / 信号 === + "InpTimeframe": 5, # PERIOD_M5 (ENUM, frozen) + "InpFastEmaPeriod": 21, + "InpSlowEmaPeriod": 100, + "InpRsiPeriod": 14, + "InpRsiBuyLevel": 45.0, + "InpRsiSellLevel": 55.0, + "InpPullbackAtrMult": 2.0, + # === 波动性 / 过滤器 === + "InpAtrPeriod": 14, + "InpMinAtrPoints": 0, + "InpMaxSpreadAtrPct": 25.0, + # === 仓位计算 === + "InpSizingMode": 1, # SIZE_RISK_PERCENT (frozen) + "InpFixedLots": 0.01, # unused in risk mode, but kept for .set gen + "InpRiskPercent": 1.0, + # === 止损 / 止盈 === + "InpStopMode": 0, # STOP_ATR (frozen) + "InpAtrSLMult": 1.5, + "InpAtrTPMult": 2.0, + "InpStopLossPoints": 200, # unused in ATR mode, kept for .set gen + "InpTakeProfitPoints": 300, # unused in ATR mode, kept for .set gen + "InpUseBreakEven": True, + "InpBreakEvenPoints": 150, + "InpBreakEvenLock": 20, + "InpUseTrailing": True, + "InpTrailStartPoints": 200, + "InpTrailStepPoints": 120, + # === 交易控制 / 风险限制 === + "InpMaxPositions": 1, # frozen: single-position strategy + "InpMaxTradesPerDay": 6, + "InpDailyLossLimit": 5.0, + "InpDailyProfitTarget": 0.0, # frozen: off + "InpMinSecondsBetween": 60, + # === 交易时段 === + "InpUseSession": False, # frozen: off (saved config) + "InpSessionStartHour": 7, + "InpSessionEndHour": 20, + # === 常规 === + "InpMagicNumber": 20240530, # frozen: identity + "InpComment": "GoldScalperPro", # frozen: identity +} + +# ────────────────────────────────────────────────────────────────────────── +# Search space — ONLY the tunable levers. (low, high, step). +# Steps chosen so each axis has ~10-20 grid points (tractable Bayesian search). +# ────────────────────────────────────────────────────────────────────────── +SEARCH_SPACE: SearchSpace = { + # --- Trend definition (EMA pair) --- + "InpFastEmaPeriod": (8.0, 34.0, 1.0), # MT5: 1..210; narrowed to plausible fast-EMA band + "InpSlowEmaPeriod": (50.0, 200.0, 5.0), # MT5: 1..1000; narrowed to plausible slow-EMA band + # --- Pullback trigger (RSI) --- + "InpRsiPeriod": (7.0, 28.0, 1.0), # MT5: 1..140 + "InpRsiBuyLevel": (30.0, 50.0, 1.0), # MT5: 4.5..450 (artifact); real band 30..50 + "InpRsiSellLevel": (50.0, 70.0, 1.0), # MT5: 5.5..550 (artifact); real band 50..70 + "InpPullbackAtrMult": (1.0, 4.0, 0.1), # MT5: 0.2..20.0 + # --- Volatility / cost gates --- + "InpAtrPeriod": (7.0, 28.0, 1.0), # MT5: 1..140 + "InpMaxSpreadAtrPct": (10.0, 50.0, 2.5), # MT5: 2.5..250.0 (artifact ×10); real 1..50% + # --- Position sizing --- + "InpRiskPercent": (0.25, 3.0, 0.25), # MT5: 0.1..10.0; capped at 3% (risk sane) + # --- Exit geometry (ATR multiples) --- + "InpAtrSLMult": (1.0, 3.0, 0.1), # MT5: 0.15..15.0 + "InpAtrTPMult": (1.0, 4.0, 0.1), # MT5: 0.2..20.0 + # --- Exit management (break-even + trailing, points) --- + "InpBreakEvenPoints": (50.0, 300.0, 10.0), # MT5: 1..1500 + "InpBreakEvenLock": (10.0, 50.0, 5.0), # MT5: 1..200 + "InpTrailStartPoints":(100.0, 400.0, 10.0), # MT5: 1..2000 + "InpTrailStepPoints": (60.0, 240.0, 10.0), # MT5: 1..1200 + # --- Risk throttles --- + "InpMaxTradesPerDay": (3.0, 12.0, 1.0), # MT5: 1..60 + "InpDailyLossLimit": (2.0, 8.0, 0.5), # MT5: 0.5..50.0 + "InpMinSecondsBetween":(30.0, 180.0, 15.0), # MT5: 1..600 +} + +# Integer-valued tunables (use suggest_int in Optuna). +INT_PARAMS: set[str] = { + "InpFastEmaPeriod", "InpSlowEmaPeriod", "InpRsiPeriod", "InpAtrPeriod", + "InpBreakEvenLock", "InpMaxTradesPerDay", "InpMinSecondsBetween", +} + + +def assert_valid() -> None: + """Validate the search space at import time so a bad range fails fast.""" + problems = validate_space(SEARCH_SPACE, INT_PARAMS) + if problems: + raise ValueError(f"invalid GoldScalperPro search space: {problems}") + # Also enforce fast < slow (a structural constraint the EA checks in OnInit). + # The ranges above could in principle sample fast=34, slow=50 — still valid, + # but if a sample violates fast slow AND close > slow + trendDown = fast < slow AND close < slow + nearFast = |close - fast| <= PullbackAtrMult × ATR + buySignal = trendUp AND nearFast AND rsi_prev < BuyLevel AND rsi_now >= BuyLevel + sellSignal = trendDown AND nearFast AND rsi_prev > SellLevel AND rsi_now <= SellLevel + +SL/TP (STOP_ATR mode, frozen): + sl_dist = AtrSLMult × ATR tp_dist = AtrTPMult × ATR + LONG : SL = entry - sl_dist TP = entry + tp_dist + SHORT: SL = entry + sl_dist TP = entry - tp_dist + +Look-ahead: signals compute from the CLOSED bar; the engine fills at the +NEXT bar's open. We compute signals on bar i; the engine opens at bar i+1. +""" +from __future__ import annotations + +import numpy as np +import pandas as pd + +from shared.indicators.base import atr, ema, rsi +from shared.optimizer.objective import SignalPack +from .search_space import FROZEN_BASELINE + + +def build_signals( + params: dict, + bars: pd.DataFrame, + instrument, +) -> SignalPack: + """Build signal + SL/TP arrays from params + bars (the caller's job). + + ``params`` is the merged dict (frozen baseline + sampled tunables). The + frozen ``InpStopMode`` decides ATR-vs-points; here we implement STOP_ATR + (the frozen mode). STOP_POINTS would be a separate strategy. + """ + p = {**FROZEN_BASELINE, **params} + close = bars["close"].to_numpy(dtype=float) + high = bars["high"].to_numpy(dtype=float) + low = bars["low"].to_numpy(dtype=float) + n = len(close) + + # ── Indicators (computed on CLOSE of each bar; no look-ahead) ──────── + fast = ema(close, int(p["InpFastEmaPeriod"])) + slow = ema(close, int(p["InpSlowEmaPeriod"])) + rsi_arr = rsi(close, int(p["InpRsiPeriod"])) + atr_arr = atr(high, low, close, int(p["InpAtrPeriod"])) + + # ── Trend + pullback + RSI cross ───────────────────────────────────── + trend_up = (fast > slow) & (close > slow) + trend_dn = (fast < slow) & (close < slow) + dist_to_fast = np.abs(close - fast) + near_fast = dist_to_fast <= (p["InpPullbackAtrMult"] * atr_arr) + + # RSI cross: rsi_prev < level AND rsi_now >= level (edge-detected). + rsi_prev = np.roll(rsi_arr, 1) + rsi_prev[0] = np.nan + buy_cross = (rsi_prev < p["InpRsiBuyLevel"]) & (rsi_arr >= p["InpRsiBuyLevel"]) + sell_cross = (rsi_prev > p["InpRsiSellLevel"]) & (rsi_arr <= p["InpRsiSellLevel"]) + + buy_signal = trend_up & near_fast & buy_cross + sell_signal = trend_dn & near_fast & sell_cross + + # NaN-guard: where indicators aren't ready, no signal. + nan_mask = np.isnan(fast) | np.isnan(slow) | np.isnan(rsi_arr) | np.isnan(atr_arr) + buy_signal = buy_signal & ~nan_mask + sell_signal = sell_signal & ~nan_mask + + # ── Volatility / spread gates (EA lines 285-290) ────────────────────── + point = instrument.point + min_atr_pts = float(p.get("InpMinAtrPoints", 0)) + if min_atr_pts > 0: + atr_pts = atr_arr / point + buy_signal = buy_signal & (atr_pts >= min_atr_pts) + sell_signal = sell_signal & (atr_pts >= min_atr_pts) + max_spread_pct = float(p.get("InpMaxSpreadAtrPct", 0)) + if max_spread_pct > 0: + spread_price = bars["spread"].to_numpy(dtype=float) * point if "spread" in bars else np.zeros(n) + spread_ok = spread_price <= (atr_arr * max_spread_pct / 100.0) + buy_signal = buy_signal & spread_ok + sell_signal = sell_signal & spread_ok + + # ── SL/TP prices (STOP_ATR mode; computed at signal bar's close) ────── + # The engine fills at next bar's open, but SL/TP distances come from the + # signal bar's ATR (the EA computes them at signal time, line 328). + sl_dist = p["InpAtrSLMult"] * atr_arr + tp_dist = p["InpAtrTPMult"] * atr_arr + # For a LONG entry at next open, SL below / TP above. + # We use the SIGNAL bar's close as the reference price for SL/TP placement + # (the EA uses the fill price; the engine will re-derive lots from sl_dist, + # and SL/TP are stored relative to the fill at open time). To keep the + # engine generic we pass SL/TP as ABSOLUTE PRICES here, using close as the + # proxy for the eventual fill — the engine overrides with the actual fill + # price ± spread for its own SL/TP, but since we want the SAME distance, + # we pass close-based prices and the engine uses them as-is. + sl_prices = np.full(n, np.nan) + tp_prices = np.full(n, np.nan) + # LONG: SL = close - sl_dist, TP = close + tp_dist + sl_prices[buy_signal] = close[buy_signal] - sl_dist[buy_signal] + tp_prices[buy_signal] = close[buy_signal] + tp_dist[buy_signal] + # SHORT: SL = close + sl_dist, TP = close - tp_dist + sl_prices[sell_signal] = close[sell_signal] + sl_dist[sell_signal] + tp_prices[sell_signal] = close[sell_signal] - tp_dist[sell_signal] + + # Round to instrument digits. + digits = instrument.digits + sl_prices = np.where(buy_signal | sell_signal, np.round(sl_prices, digits), np.nan) + tp_prices = np.where(buy_signal | sell_signal, np.round(tp_prices, digits), np.nan) + + return SignalPack( + params=p, + signals_long=buy_signal, + signals_short=sell_signal, + sl_prices=sl_prices, + tp_prices=tp_prices, + ) diff --git a/strategies/gold_scalper_pro/source/.gitkeep b/strategies/gold_scalper_pro/source/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/strategies/gold_scalper_pro/wizard_questions.py b/strategies/gold_scalper_pro/wizard_questions.py new file mode 100644 index 0000000..d656394 --- /dev/null +++ b/strategies/gold_scalper_pro/wizard_questions.py @@ -0,0 +1,39 @@ +"""GoldScalperPro-specific wizard questions (doc 05 §5). + +Appended to DEFAULT_QUESTIONS so a run captures both the common run settings +and the strategy-specific ones (initial deposit, instrument profile, etc.). +""" +from __future__ import annotations + +from shared.wizard.wizard import WizardQuestion + +# Strategy-specific questions. The common ones (period, trials, DD caps, top_n) +# come from DEFAULT_QUESTIONS in shared.wizard. +GOLD_SCALPER_QUESTIONS: list[WizardQuestion] = [ + WizardQuestion( + "ea_set_preset", + "Path to the .set preset to seed frozen baseline (blank = use saved GoldScalperPro.set)", + "", + help="if blank, loads the MT5 tester profiles GoldScalperPro.set", + ), + WizardQuestion( + "bars_file", + "Path to the Parquet bars file (blank = auto-find data/XAUUSD_M5_*.parquet)", + "", + help="M5 OHLC+spread from the download script", + ), + WizardQuestion( + "sizing_mode", + "Sizing mode (0=fixed lot, 1=risk % equity)", + 1, + cast=int, + help="frozen at 1 in the saved .set; 0 is a different strategy", + ), + WizardQuestion( + "stop_mode", + "Stop mode (0=ATR multiple, 1=fixed points)", + 0, + cast=int, + help="frozen at 0 in the saved .set; 1 is a different strategy", + ), +]