diff --git a/Cargo.lock b/Cargo.lock index 3275d73..b6b51c5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -430,6 +430,7 @@ dependencies = [ "dirs", "encoding_rs", "regex", + "roxmltree", "serde", "serde_json", "serde_yaml", @@ -589,6 +590,15 @@ version = "0.8.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a" +[[package]] +name = "roxmltree" +version = "0.21.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1964b10c76125c36f8afe190065a4bf9a87bf324842c05701330bba9f1cacbb" +dependencies = [ + "memchr", +] + [[package]] name = "rustix" version = "1.1.4" diff --git a/Cargo.toml b/Cargo.toml index 14cb89e..082e44f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -25,3 +25,4 @@ walkdir = "2.0" chrono = { version = "0.4", features = ["serde"] } encoding_rs = "0.8" tempfile = "3.0" +roxmltree = "0.21.1" diff --git a/analytics/__init__.py b/analytics/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/analytics/analyze.py b/analytics/analyze.py deleted file mode 100644 index 4aea23d..0000000 --- a/analytics/analyze.py +++ /dev/null @@ -1,1000 +0,0 @@ -#!/usr/bin/env python3 -""" -analyze.py — Deal-level backtest analysis engine. - -Strategy-agnostic core + strategy-specific profiles. - -Usage: - # Generic (works for any EA — no hardcoded keyword assumptions) - python3 analytics/analyze.py generic deals.csv --output-dir DIR - - # Strategy-specific presets - python3 analytics/analyze.py grid deals.csv --output-dir DIR - python3 analytics/analyze.py scalper deals.csv --output-dir DIR - python3 analytics/analyze.py trend deals.csv --output-dir DIR - python3 analytics/analyze.py hedge deals.csv --output-dir DIR - - # Legacy (no subcommand) — defaults to 'grid' for backward compatibility - python3 analytics/analyze.py deals.csv --output-dir DIR - - # Flags - --deep Add hourly_pnl and volume_profile - --stdout Print JSON to stdout instead of writing analysis.json - -Entry points (after pip install -e .): - mt5-analyze generic analysis - mt5-analyze-grid grid / martingale - mt5-analyze-scalper scalper - mt5-analyze-trend trend following - mt5-analyze-hedge hedging -""" - -import argparse -import csv -import json -import os -import re -import sys -from collections import defaultdict -from datetime import datetime -from typing import Optional - - -# ── Strategy profiles ───────────────────────────────────────────────────────── -# -# Each profile is a plain dict with: -# name – human-readable label -# depth_re – regex to extract depth number from comment (None = no depth tracking) -# exit_keywords – {reason: [keywords]} for comment-based exit classification -# dd_cause_keywords– {cause: [keywords]} for comment-based DD cause classification -# cycle_group_by – 'magic' | 'magic+direction' | None -# cycle_gap_min – minutes between opens that marks a new cycle boundary -# -# When exit_keywords is empty, classification falls back to profit-sign (tp/sl). -# When dd_cause_keywords is empty, dd_events cause = 'unknown'. - -PROFILES: dict[str, dict] = { - 'generic': { - 'name': 'Generic', - 'depth_re': None, - 'exit_keywords': {}, - 'dd_cause_keywords': {}, - 'cycle_group_by': 'magic', - 'cycle_gap_min': 60, - }, - 'grid': { - 'name': 'Grid / Martingale', - 'depth_re': r'[Ll]ayer\s*#?(\d+)', - 'exit_keywords': { - 'locking': ['locking', 'lock'], - 'cutloss': ['cutloss', 'cut loss', 'cut_loss'], - 'zombie': ['zombie'], - 'timeout': ['timeout', 'time out', 'time_out'], - }, - 'dd_cause_keywords': { - 'locking_cascade': ['locking', 'lock'], - 'cutloss': ['cutloss', 'cut'], - 'zombie_exit': ['zombie'], - 'spike_entry': ['spike'], - }, - 'cycle_group_by': 'magic+direction', - 'cycle_gap_min': 60, - }, - 'scalper': { - 'name': 'Scalper', - 'depth_re': None, - 'exit_keywords': { - 'tp': ['tp', 'take profit', 'target'], - 'sl': ['sl', 'stop loss', 'stoploss'], - 'manual': ['manual', 'close'], - 'trailing': ['trailing', 'trail'], - }, - 'dd_cause_keywords': { - 'stop_loss': ['sl', 'stop'], - 'manual_close': ['manual', 'close'], - }, - 'cycle_group_by': 'magic', - 'cycle_gap_min': 10, - }, - 'trend': { - 'name': 'Trend Following', - 'depth_re': None, - 'exit_keywords': { - # More specific patterns must come before general ones to avoid substring - # false-positives (e.g. 'stop' inside 'breakeven stop' / 'trailing stop'). - 'breakeven': ['breakeven', 'break even', 'be stop'], - 'trailing': ['trailing', 'trail'], - 'partial': ['partial', 'scale out'], - 'tp': ['tp', 'target', 'take profit'], - 'sl': ['sl', 'stop loss', 'stoploss'], - }, - 'dd_cause_keywords': { - 'whipsaw': ['stop loss', 'stoploss'], - 'trailing_stop': ['trailing'], - 'breakeven_stop': ['breakeven', 'be stop'], - }, - 'cycle_group_by': 'magic', - 'cycle_gap_min': 240, - }, - 'hedge': { - 'name': 'Hedging', - 'depth_re': None, - 'exit_keywords': { - 'tp': ['tp'], - 'sl': ['sl'], - 'net_close': ['net', 'hedge close', 'hedge_close'], - 'partial': ['partial', 'reduce'], - }, - 'dd_cause_keywords': { - 'hedge_unwind': ['net', 'hedge'], - 'correlation_break': ['sl', 'stop'], - }, - 'cycle_group_by': 'magic+direction', - 'cycle_gap_min': 120, - }, -} - - -# ── Utility ─────────────────────────────────────────────────────────────────── - -def _parse_dt(time_str: str) -> Optional[datetime]: - """Parse MT5 time string ('2025.01.10 09:30:00') to datetime.""" - if not time_str: - return None - s = time_str.strip() - for length, fmt in [(19, '%Y.%m.%d %H:%M:%S'), (19, '%Y-%m-%d %H:%M:%S'), - (10, '%Y.%m.%d'), (10, '%Y-%m-%d')]: - try: - return datetime.strptime(s[:length], fmt) - except (ValueError, IndexError): - continue - return None - - -def _extract_depth(comment: str, depth_re: Optional[str]) -> int: - """Extract a numeric depth from comment using the given regex. Returns 0 if not matched.""" - if not depth_re or not comment: - return 0 - match = re.search(depth_re, comment) - return int(match.group(1)) if match else 0 - - -def _layer_from_comment(comment: str) -> int: - """Grid-specific: extract Layer #N number. Kept for backward compatibility.""" - return _extract_depth(comment, PROFILES['grid']['depth_re']) - - -def _classify_exit(comment: str, profit: float, - profile: Optional[dict] = None) -> str: - """ - Classify exit reason from deal comment. - - If profile is provided, uses its exit_keywords dict. - If profile is None, falls back to the grid profile keywords (backward compat). - If no keywords match, returns 'tp' (profit > 0) or 'sl' (profit <= 0). - """ - kw_map = (profile or PROFILES['grid'])['exit_keywords'] - c = comment.lower() - for reason, keywords in kw_map.items(): - if any(kw in c for kw in keywords): - return reason - return 'tp' if profit > 0 else 'sl' - - -def _classify_dd_cause(comment: str, profile: dict) -> str: - """Classify DD event cause from comment using profile's dd_cause_keywords.""" - c = comment.lower() - for cause, keywords in profile.get('dd_cause_keywords', {}).items(): - if any(kw in c for kw in keywords): - return cause - return 'unknown' - - -def _lot_tier(vol: float) -> str: - if vol <= 0.01: return '0.01' - if vol <= 0.04: return '0.02-0.04' - if vol <= 0.09: return '0.05-0.09' - if vol <= 0.49: return '0.10-0.49' - if vol <= 0.99: return '0.50-0.99' - return '1.00+' - - -def _session_for_hour(hour: int) -> str: - if 13 <= hour < 17: return 'london_ny_overlap' - if 8 <= hour < 17: return 'london' - if 17 <= hour < 22: return 'new_york' - if 0 <= hour < 8: return 'asian' - return 'off_hours' - - -# ── Loaders ─────────────────────────────────────────────────────────────────── - -def load_deals(csv_path: str) -> list[dict]: - deals = [] - with open(csv_path, newline='', encoding='utf-8') as f: - reader = csv.DictReader(f) - for row in reader: - row = {k: (v or '').strip() for k, v in row.items()} - for field in ('profit', 'volume', 'price', 'balance'): - try: - row[field] = float(row.get(field, 0).replace(',', '') or 0) - except ValueError: - row[field] = 0.0 - deals.append(row) - return deals - - -def load_metrics(metrics_path: str) -> dict: - if not os.path.exists(metrics_path): - return {} - with open(metrics_path) as f: - return json.load(f) - - -# ── Generic analytics (strategy-agnostic) ──────────────────────────────────── - -def monthly_pnl(deals: list[dict]) -> list[dict]: - monthly: dict[str, dict] = defaultdict(lambda: {'pnl': 0.0, 'trades': 0}) - for deal in deals: - time_str = deal.get('time', '') - profit = deal.get('profit', 0.0) - entry = deal.get('entry', '').lower() - if 'out' not in entry and entry != '': - continue - if not time_str or profit == 0.0: - continue - try: - dt = datetime.strptime(time_str[:10].replace('.', '-'), '%Y-%m-%d') - month = dt.strftime('%Y-%m') - except (ValueError, IndexError): - continue - monthly[month]['pnl'] += profit - monthly[month]['trades'] += 1 - - return [ - {'month': m, 'pnl': round(d['pnl'], 2), 'trades': d['trades'], 'green': d['pnl'] >= 0} - for m in sorted(monthly) - for d in [monthly[m]] - ] - - -def reconstruct_dd_events(deals: list[dict], metrics: dict, - profile: Optional[dict] = None) -> list[dict]: - """Walk deals chronologically, reconstruct drawdown events. Cause classified by profile.""" - if not deals: - return [] - - _profile = profile or PROFILES['grid'] - balance_curve = [] - peak_balance = 0.0 - initial_balance = None - - for deal in deals: - balance = deal.get('balance', 0.0) - if balance > 0: - if initial_balance is None: - initial_balance = balance - peak_balance = max(peak_balance, balance) - dd_pct = (peak_balance - balance) / peak_balance * 100 if peak_balance > 0 else 0 - balance_curve.append({ - 'time': deal.get('time', ''), - 'balance': balance, - 'dd_pct': round(dd_pct, 3), - 'profit': deal.get('profit', 0.0), - 'comment': deal.get('comment', ''), - }) - - if not balance_curve: - return [] - - events, in_dd, dd_start_idx = [], False, None - threshold = 1.0 - - for i, point in enumerate(balance_curve): - if not in_dd and point['dd_pct'] > threshold: - in_dd, dd_start_idx = True, i - elif in_dd and point['dd_pct'] < threshold: - peak_idx = max(range(dd_start_idx, i + 1), - key=lambda x: balance_curve[x]['dd_pct']) - event = _build_dd_event(balance_curve, dd_start_idx, peak_idx, i, _profile) - if event['peak_dd_pct'] > 1.0: - events.append(event) - in_dd, dd_start_idx = False, None - - if in_dd and dd_start_idx is not None: - peak_idx = max(range(dd_start_idx, len(balance_curve)), - key=lambda x: balance_curve[x]['dd_pct']) - event = _build_dd_event(balance_curve, dd_start_idx, peak_idx, None, _profile) - if event['peak_dd_pct'] > 1.0: - events.append(event) - - events.sort(key=lambda e: e['peak_dd_pct'], reverse=True) - return events[:10] - - -def _build_dd_event(curve: list[dict], start_idx: int, peak_idx: int, - recovery_idx: Optional[int], profile: dict) -> dict: - start = curve[start_idx] - peak = curve[peak_idx] - - event: dict = { - 'peak_dd_pct': round(peak['dd_pct'], 2), - 'start_date': start['time'][:10].replace('.', '-') if start['time'] else '', - 'end_date': peak['time'][:10].replace('.', '-') if peak['time'] else '', - 'cause': _classify_dd_cause(peak.get('comment', ''), profile), - } - - if recovery_idx is not None: - rec = curve[recovery_idx] - event['recovery_date'] = rec['time'][:10].replace('.', '-') if rec['time'] else None - try: - s_dt = datetime.strptime(event['start_date'], '%Y-%m-%d') - r_dt = datetime.strptime(event['recovery_date'], '%Y-%m-%d') - event['recovery_days'] = (r_dt - s_dt).days - except (ValueError, TypeError): - event['recovery_days'] = None - else: - event['recovery_date'] = None - event['recovery_days'] = None - - try: - s_dt = datetime.strptime(event['start_date'], '%Y-%m-%d') - e_dt = datetime.strptime(event['end_date'], '%Y-%m-%d') - event['duration_days'] = (e_dt - s_dt).days - except ValueError: - event['duration_days'] = 0 - - return event - - -def top_losses(deals: list[dict], n: int = 10) -> list[dict]: - losses = [ - { - 'date': deal.get('time', '')[:10].replace('.', '-'), - 'loss_usd': round(deal['profit'], 2), - 'comment': deal.get('comment', ''), - 'grid_depth_at_close': _layer_from_comment(deal.get('comment', '')), - 'volume': deal.get('volume', 0.0), - } - for deal in deals - if deal.get('profit', 0.0) < 0 - ] - losses.sort(key=lambda x: x['loss_usd']) - return losses[:n] - - -def loss_sequences(deals: list[dict]) -> list[dict]: - closed = [d for d in deals - if 'out' in d.get('entry', '').lower() and d.get('profit', 0) != 0] - if not closed: - return [] - - sequences, current_seq = [], [] - for deal in closed: - if deal['profit'] < 0: - current_seq.append(deal) - else: - if len(current_seq) >= 2: - total = sum(d['profit'] for d in current_seq) - sequences.append({ - 'length': len(current_seq), - 'total_loss': round(total, 2), - 'start': current_seq[0].get('time', '')[:10].replace('.', '-'), - 'end': current_seq[-1].get('time', '')[:10].replace('.', '-'), - }) - current_seq = [] - - if len(current_seq) >= 2: - total = sum(d['profit'] for d in current_seq) - sequences.append({ - 'length': len(current_seq), - 'total_loss': round(total, 2), - 'start': current_seq[0].get('time', '')[:10].replace('.', '-'), - 'end': current_seq[-1].get('time', '')[:10].replace('.', '-'), - }) - - sequences.sort(key=lambda x: x['total_loss']) - return sequences[:5] - - -def position_pairs(deals: list[dict]) -> list[dict]: - """Match in/out deals by order ticket → hold time + depth at close.""" - open_pos: dict[str, dict] = {} - pairs = [] - - for deal in deals: - order = deal.get('order', '') - entry = deal.get('entry', '').lower() - - if 'in' in entry and 'out' not in entry: - open_pos[order] = deal - - elif 'out' in entry: - profit = deal.get('profit', 0.0) - if profit == 0.0: - continue - - in_deal = open_pos.pop(order, None) - dt_out = _parse_dt(deal.get('time', '')) - comment = deal.get('comment', '') - - hold_minutes = None - if in_deal and dt_out: - dt_in = _parse_dt(in_deal.get('time', '')) - if dt_in: - hold_minutes = round((dt_out - dt_in).total_seconds() / 60, 1) - - pairs.append({ - 'time': deal.get('time', ''), - 'type': deal.get('type', ''), - 'profit': profit, - 'volume': deal.get('volume', 0.0), - 'layer': _layer_from_comment(comment), - 'hold_minutes': hold_minutes, - 'comment': comment, - 'magic': deal.get('magic', ''), - 'order': order, - }) - - return pairs - - -def direction_bias(deals: list[dict]) -> dict: - stats: dict[str, dict] = { - 'buy': {'trades': 0, 'wins': 0, 'total_pnl': 0.0}, - 'sell': {'trades': 0, 'wins': 0, 'total_pnl': 0.0}, - } - for deal in deals: - if 'out' not in deal.get('entry', '').lower(): - continue - profit = deal.get('profit', 0.0) - if profit == 0.0: - continue - d = deal.get('type', '').lower() - if d not in stats: - continue - stats[d]['trades'] += 1 - stats[d]['total_pnl'] += profit - if profit > 0: - stats[d]['wins'] += 1 - - return { - d: { - 'trades': s['trades'], - 'win_rate': round(s['wins'] / s['trades'] * 100, 1), - 'total_pnl': round(s['total_pnl'], 2), - 'avg_pnl': round(s['total_pnl'] / s['trades'], 2), - } - for d, s in stats.items() if s['trades'] > 0 - } - - -def streak_analysis(deals: list[dict]) -> dict: - closed = [d for d in deals - if 'out' in d.get('entry', '').lower() and d.get('profit', 0.0) != 0.0] - if not closed: - return {} - - max_win_streak = max_loss_streak = cur_win = cur_loss = 0 - max_win_start = max_win_end = max_loss_start = max_loss_end = '' - win_run_start = loss_run_start = '' - - for deal in closed: - profit = deal['profit'] - t = deal.get('time', '')[:10].replace('.', '-') - if profit > 0: - if cur_win == 0: - win_run_start = t - cur_win += 1 - cur_loss = 0 - if cur_win > max_win_streak: - max_win_streak = cur_win - max_win_start = win_run_start - max_win_end = t - else: - if cur_loss == 0: - loss_run_start = t - cur_loss += 1 - cur_win = 0 - if cur_loss > max_loss_streak: - max_loss_streak = cur_loss - max_loss_start = loss_run_start - max_loss_end = t - - last = closed[-1] - return { - 'max_win_streak': max_win_streak, - 'max_win_start': max_win_start, - 'max_win_end': max_win_end, - 'max_loss_streak': max_loss_streak, - 'max_loss_start': max_loss_start, - 'max_loss_end': max_loss_end, - 'current_streak': cur_win if last['profit'] > 0 else cur_loss, - 'current_streak_type': 'win' if last['profit'] > 0 else 'loss', - } - - -def session_breakdown(deals: list[dict]) -> dict: - """P/L by trading session (UTC hour-based).""" - sessions: dict = defaultdict(lambda: {'trades': 0, 'wins': 0, 'total_pnl': 0.0}) - for deal in deals: - if 'out' not in deal.get('entry', '').lower(): - continue - profit = deal.get('profit', 0.0) - if profit == 0.0: - continue - dt = _parse_dt(deal.get('time', '')) - if not dt: - continue - s = _session_for_hour(dt.hour) - sessions[s]['trades'] += 1 - sessions[s]['total_pnl'] += profit - if profit > 0: - sessions[s]['wins'] += 1 - - return { - s: { - 'trades': d['trades'], - 'win_rate': round(d['wins'] / d['trades'] * 100, 1) if d['trades'] > 0 else 0.0, - 'total_pnl': round(d['total_pnl'], 2), - } - for s, d in sessions.items() - } - - -def weekday_pnl(deals: list[dict]) -> list[dict]: - DAY_NAMES = ['Monday', 'Tuesday', 'Wednesday', 'Thursday', 'Friday', 'Saturday', 'Sunday'] - by_day: dict = defaultdict(lambda: {'pnl': 0.0, 'trades': 0, 'wins': 0}) - for deal in deals: - if 'out' not in deal.get('entry', '').lower(): - continue - profit = deal.get('profit', 0.0) - if profit == 0.0: - continue - dt = _parse_dt(deal.get('time', '')) - if not dt: - continue - by_day[dt.weekday()]['pnl'] += profit - by_day[dt.weekday()]['trades'] += 1 - if profit > 0: - by_day[dt.weekday()]['wins'] += 1 - - return [ - { - 'day': DAY_NAMES[day], - 'pnl': round(s['pnl'], 2), - 'trades': s['trades'], - 'win_rate': round(s['wins'] / s['trades'] * 100, 1) if s['trades'] > 0 else 0.0, - } - for day in sorted(by_day) - for s in [by_day[day]] - ] - - -def hourly_pnl(deals: list[dict]) -> list[dict]: - """P/L by close hour (0–23). Intended for --deep mode.""" - by_hour: dict = defaultdict(lambda: {'pnl': 0.0, 'trades': 0, 'wins': 0}) - for deal in deals: - if 'out' not in deal.get('entry', '').lower(): - continue - profit = deal.get('profit', 0.0) - if profit == 0.0: - continue - dt = _parse_dt(deal.get('time', '')) - if not dt: - continue - by_hour[dt.hour]['pnl'] += profit - by_hour[dt.hour]['trades'] += 1 - if profit > 0: - by_hour[dt.hour]['wins'] += 1 - - return [ - { - 'hour': h, - 'pnl': round(s['pnl'], 2), - 'trades': s['trades'], - 'win_rate': round(s['wins'] / s['trades'] * 100, 1) if s['trades'] > 0 else 0.0, - } - for h in sorted(by_hour) - for s in [by_hour[h]] - ] - - -def concurrent_peak(deals: list[dict]) -> dict: - """Peak number of simultaneously open positions.""" - events = [] - for deal in deals: - entry = deal.get('entry', '').lower() - dt = _parse_dt(deal.get('time', '')) - if not dt: - continue - if 'in' in entry and 'out' not in entry: - events.append((dt, 1, deal)) - elif 'out' in entry: - events.append((dt, -1, deal)) - - events.sort(key=lambda x: x[0]) - count = peak = 0 - peak_time = '' - for dt, delta, deal in events: - count = max(0, count + delta) - if count > peak: - peak = count - peak_time = deal.get('time', '') - - return {'peak_open': peak, 'peak_time': peak_time} - - -def volume_profile(deals: list[dict]) -> list[dict]: - """P/L breakdown by lot size tier. Intended for --deep mode.""" - TIER_ORDER = ['0.01', '0.02-0.04', '0.05-0.09', '0.10-0.49', '0.50-0.99', '1.00+'] - by_tier: dict = defaultdict(lambda: {'pnl': 0.0, 'trades': 0, 'wins': 0}) - for deal in deals: - if 'out' not in deal.get('entry', '').lower(): - continue - profit = deal.get('profit', 0.0) - if profit == 0.0: - continue - tier = _lot_tier(deal.get('volume', 0.0)) - by_tier[tier]['pnl'] += profit - by_tier[tier]['trades'] += 1 - if profit > 0: - by_tier[tier]['wins'] += 1 - - return [ - { - 'lot_tier': tier, - 'pnl': round(s['pnl'], 2), - 'trades': s['trades'], - 'win_rate': round(s['wins'] / s['trades'] * 100, 1) if s['trades'] > 0 else 0.0, - } - for tier in TIER_ORDER - if tier in by_tier - for s in [by_tier[tier]] - ] - - -# ── Strategy-aware analytics ────────────────────────────────────────────────── - -def depth_histogram(deals: list[dict], profile: dict) -> dict: - """ - Count how often each depth level was reached, using the profile's depth_re. - Returns an empty dict when the profile has no depth_re (e.g. generic, scalper). - For grid profiles, keys are L1 … L8+. - """ - depth_re = profile.get('depth_re') - if not depth_re: - return {} - - hist: dict[str, int] = {'L1': 0, 'L2': 0, 'L3': 0, 'L4': 0, - 'L5': 0, 'L6': 0, 'L7': 0, 'L8+': 0} - for deal in deals: - depth = _extract_depth(deal.get('comment', ''), depth_re) - if depth: - key = f'L{depth}' if depth <= 7 else 'L8+' - hist[key] = hist.get(key, 0) + 1 - - return hist - - -def grid_depth_histogram(deals: list[dict]) -> dict: - """Backward-compatible alias: depth_histogram using the grid profile.""" - return depth_histogram(deals, PROFILES['grid']) - - -def cycle_stats(deals: list[dict], profile: Optional[dict] = None) -> dict: - """ - Group deals into cycles using profile's cycle_group_by and cycle_gap_min. - Returns overall win rate and win_rate_by_depth. - - Default profile (None) uses grid settings for backward compatibility. - """ - _profile = profile or PROFILES['grid'] - group_by = _profile.get('cycle_group_by', 'magic+direction') - gap_minutes = _profile.get('cycle_gap_min', 60) - depth_re = _profile.get('depth_re') - - # active[key] = {'in_deals': [...], 'profit': float, 'max_depth': int, 'last_open': dt} - active: dict[tuple, dict] = {} - completed: list[dict] = [] - - def _key(deal: dict) -> tuple: - magic = deal.get('magic', '') - if group_by == 'magic+direction': - return (magic, deal.get('type', '').lower()) - return (magic,) - - def _flush(k: tuple) -> None: - if k in active and active[k]['in_deals']: - completed.append(active.pop(k)) - - for deal in sorted(deals, key=lambda d: _parse_dt(d.get('time', '')) or datetime.min): - entry = deal.get('entry', '').lower() - dt = _parse_dt(deal.get('time', '')) - if not dt or not deal.get('magic', ''): - continue - - key = _key(deal) - depth = _extract_depth(deal.get('comment', ''), depth_re) - - if 'in' in entry and 'out' not in entry: - if key in active: - gap = (dt - active[key]['last_open']).total_seconds() / 60 - if gap > gap_minutes: - _flush(key) - if key not in active: - active[key] = {'in_deals': [], 'profit': 0.0, - 'max_depth': 0, 'last_open': dt} - active[key]['in_deals'].append(deal) - active[key]['last_open'] = dt - active[key]['max_depth'] = max(active[key]['max_depth'], depth) - - elif 'out' in entry: - profit = deal.get('profit', 0.0) - if profit == 0.0: - continue - if key in active: - active[key]['profit'] += profit - active[key]['last_open'] = dt - - for k in list(active.keys()): - _flush(k) - - if not completed: - return {'total_cycles': 0, 'win_rate': 0.0, 'avg_profit': 0.0, 'win_rate_by_depth': {}} - - total = len(completed) - wins = sum(1 for c in completed if c['profit'] > 0) - total_profit = sum(c['profit'] for c in completed) - - by_depth: dict[str, dict] = defaultdict(lambda: {'wins': 0, 'total': 0}) - for c in completed: - d = c['max_depth'] - label = (f'L{d}' if 0 < d <= 7 else 'L8+') if d > 0 else 'L?' - by_depth[label]['total'] += 1 - if c['profit'] > 0: - by_depth[label]['wins'] += 1 - - return { - 'total_cycles': total, - 'win_rate': round(wins / total * 100, 1), - 'avg_profit': round(total_profit / total, 2), - 'win_rate_by_depth': { - d: {'total': s['total'], 'win_rate': round(s['wins'] / s['total'] * 100, 1)} - for d, s in sorted(by_depth.items()) - }, - } - - -def exit_reason_breakdown(deals: list[dict], - profile: Optional[dict] = None) -> dict: - """ - Classify closed deals by exit reason and aggregate P/L. - - Default profile (None) uses grid keywords for backward compatibility. - Generic profile returns only 'tp' / 'sl' (profit-sign classification). - """ - _profile = profile or PROFILES['grid'] - counts: dict[str, int] = defaultdict(int) - pnl: dict[str, float] = defaultdict(float) - - for deal in deals: - if 'out' not in deal.get('entry', '').lower(): - continue - profit = deal.get('profit', 0.0) - if profit == 0.0: - continue - reason = _classify_exit(deal.get('comment', ''), profit, _profile) - counts[reason] += 1 - pnl[reason] += profit - - return { - reason: { - 'count': counts[reason], - 'total_pnl': round(pnl[reason], 2), - 'avg_pnl': round(pnl[reason] / counts[reason], 2), - } - for reason in counts - } - - -# ── Summary ─────────────────────────────────────────────────────────────────── - -def build_summary(metrics: dict, monthly: list[dict], dd_events: list[dict], - *, - streak: Optional[dict] = None, - bias: Optional[dict] = None, - exits: Optional[dict] = None, - cycles: Optional[dict] = None, - strategy: Optional[str] = None) -> dict: - green = sum(1 for m in monthly if m['green']) - worst = min(monthly, key=lambda m: m['pnl'], default={}) - - summary: dict = { - 'net_profit': metrics.get('net_profit', 0), - 'profit_factor': metrics.get('profit_factor', 0), - 'max_dd_pct': metrics.get('max_dd_pct', 0), - 'sharpe_ratio': metrics.get('sharpe_ratio', 0), - 'total_trades': metrics.get('total_trades', 0), - 'recovery_factor': metrics.get('recovery_factor', 0), - 'green_months': green, - 'total_months': len(monthly), - 'worst_month': worst.get('month', ''), - 'worst_month_pnl': worst.get('pnl', 0), - } - - if strategy: - summary['strategy'] = strategy - - if streak: - summary['max_win_streak'] = streak.get('max_win_streak', 0) - summary['max_loss_streak'] = streak.get('max_loss_streak', 0) - summary['current_streak'] = streak.get('current_streak', 0) - summary['current_streak_type']= streak.get('current_streak_type', '') - - if bias: - for direction in ('buy', 'sell'): - if direction in bias: - summary[f'{direction}_win_rate'] = bias[direction].get('win_rate', 0) - summary[f'{direction}_total_pnl'] = bias[direction].get('total_pnl', 0) - - if exits: - dominant = max(exits, key=lambda k: exits[k]['count'], default='') - summary['dominant_exit'] = dominant - - if cycles and cycles.get('total_cycles', 0): - summary['cycle_win_rate'] = cycles.get('win_rate', 0) - summary['total_cycles'] = cycles.get('total_cycles', 0) - - return summary - - -# ── Main ────────────────────────────────────────────────────────────────────── - -def _make_parser(prog_suffix: str = '') -> argparse.ArgumentParser: - p = argparse.ArgumentParser( - prog=f'mt5-analyze{prog_suffix}' if prog_suffix else 'analyze.py', - description=f'MT5 deal analysis{(" — " + PROFILES[prog_suffix.lstrip("-")]["name"]) if prog_suffix else ""}', - ) - p.add_argument('deals_csv', help='Path to deals.csv') - p.add_argument('--output-dir', default='.', help='Output directory') - p.add_argument('--deep', action='store_true', help='Add hourly_pnl and volume_profile') - p.add_argument('--stdout', action='store_true', help='Print JSON to stdout') - return p - - -def _run(deals_csv: str, output_dir: str, deep: bool, stdout: bool, - strategy_name: str) -> None: - profile = PROFILES.get(strategy_name, PROFILES['grid']) - - os.makedirs(output_dir, exist_ok=True) - deals = load_deals(deals_csv) - metrics = load_metrics(os.path.join(output_dir, 'metrics.json')) - - if not deals: - print("WARNING: No deals found in CSV", file=sys.stderr) - - # Generic analytics (always run, strategy-agnostic) - monthly = monthly_pnl(deals) - dd_ev = reconstruct_dd_events(deals, metrics, profile) - top_loss = top_losses(deals) - loss_seq = loss_sequences(deals) - pairs = position_pairs(deals) - bias = direction_bias(deals) - streak = streak_analysis(deals) - sessions = session_breakdown(deals) - wday = weekday_pnl(deals) - peak = concurrent_peak(deals) - - # Strategy-aware analytics - depth_hist = depth_histogram(deals, profile) - cycles = cycle_stats(deals, profile) - exits = exit_reason_breakdown(deals, profile) - - summary = build_summary(metrics, monthly, dd_ev, - streak=streak, bias=bias, exits=exits, - cycles=cycles, strategy=strategy_name) - - analysis: dict = { - 'strategy': strategy_name, - 'summary': summary, - 'monthly_pnl': monthly, - 'dd_events': dd_ev, - 'top_losses': top_loss, - 'loss_sequences': loss_seq, - 'position_pairs': pairs, - 'direction_bias': bias, - 'streak_analysis': streak, - 'session_breakdown': sessions, - 'weekday_pnl': wday, - 'concurrent_peak': peak, - 'depth_histogram': depth_hist, - 'cycle_stats': cycles, - 'exit_reason_breakdown': exits, - } - - # Grid backward-compat alias - if strategy_name == 'grid' and depth_hist: - analysis['grid_depth_histogram'] = depth_hist - - if deep: - analysis['hourly_pnl'] = hourly_pnl(deals) - analysis['volume_profile'] = volume_profile(deals) - - if stdout: - json.dump(analysis, sys.stdout, indent=2) - print() - return - - out_path = os.path.join(output_dir, 'analysis.json') - with open(out_path, 'w') as f: - json.dump(analysis, f, indent=2) - - print(f"Analysis complete [{PROFILES[strategy_name]['name']}]: {out_path}") - print(f" {summary.get('green_months', 0)}/{summary.get('total_months', 0)} green months") - print(f" {len(dd_ev)} DD events reconstructed") - if depth_hist: - max_layer = max( - (k for k, v in depth_hist.items() if v > 0), - key=lambda x: int(x[1:].replace('+', '9')), - default='?' - ) - print(f" Grid depth: max {max_layer}") - if cycles.get('total_cycles', 0): - print(f" {cycles['total_cycles']} cycles — {cycles['win_rate']}% win rate") - if bias: - for d in ('buy', 'sell'): - if d in bias: - b = bias[d] - print(f" {d.capitalize()}: {b['trades']} trades, " - f"{b['win_rate']}% win rate, {b['total_pnl']:+.2f}") - - -def main() -> None: - """ - Entry point — auto-detects subcommand style vs legacy style. - - analyze.py grid deals.csv [options] → grid strategy - analyze.py deals.csv [options] → grid (backward compat default) - """ - strategy_name = 'grid' - argv = sys.argv[1:] - - if argv and argv[0] in PROFILES: - strategy_name = argv.pop(0) - - parser = _make_parser() - args = parser.parse_args(argv) - _run(args.deals_csv, args.output_dir, args.deep, args.stdout, strategy_name) - - -# ── Named entry points for each strategy ───────────────────────────────────── - -def main_generic() -> None: - """Entry point: mt5-analyze (generic, strategy-agnostic).""" - _entry('generic') - -def main_grid() -> None: - """Entry point: mt5-analyze-grid""" - _entry('grid') - -def main_scalper() -> None: - """Entry point: mt5-analyze-scalper""" - _entry('scalper') - -def main_trend() -> None: - """Entry point: mt5-analyze-trend""" - _entry('trend') - -def main_hedge() -> None: - """Entry point: mt5-analyze-hedge""" - _entry('hedge') - -def _entry(strategy_name: str) -> None: - parser = _make_parser(f'-{strategy_name}') - args = parser.parse_args() - _run(args.deals_csv, args.output_dir, args.deep, args.stdout, strategy_name) - - -if __name__ == '__main__': - main() diff --git a/analytics/extract.py b/analytics/extract.py deleted file mode 100644 index 30fa5b4..0000000 --- a/analytics/extract.py +++ /dev/null @@ -1,316 +0,0 @@ -#!/usr/bin/env python3 -""" -extract.py — Single-pass MT5 report parser. - -Reads MT5 backtest report (.htm or .htm.xml / SpreadsheetML) and produces: - - metrics.json (aggregate summary) - - deals.csv (all deals, 13 columns) - - deals.json (deals as JSON array) - -Usage: - python3 analytics/extract.py report.htm --output-dir reports/20250101_123456/ -""" - -import argparse -import csv -import json -import os -import re -import sys -import xml.etree.ElementTree as ET -from pathlib import Path -from typing import Optional - - -# MT5 backtest report deals table columns (actual order from HTML): -# Time, Deal, Symbol, Type, Direction, Volume, Price, Order, Commission, Swap, Profit, Balance, Comment -DEAL_COLUMNS = [ - "time", "deal", "symbol", "type", "entry", "volume", "price", - "order", "commission", "swap", "profit", "balance", "comment" -] - - -def detect_format(path: str) -> str: - """Return 'xml' for SpreadsheetML, 'html' for legacy HTML report.""" - if path.endswith('.xml') or path.endswith('.htm.xml'): - return 'xml' - # Peek at file header - with open(path, 'rb') as f: - header = f.read(512) - if b' str: - """Read file, handling UTF-16 (MT5 default) and latin-1 fallback.""" - with open(path, 'rb') as f: - raw = f.read() - for encoding in ('utf-16', 'utf-8', 'latin-1'): - try: - return raw.decode(encoding) - except (UnicodeDecodeError, LookupError): - continue - return raw.decode('latin-1', errors='replace') - - -def strip_tags(html: str) -> str: - return re.sub(r'<[^>]+>', '', html).strip() - - -# ── HTML parser ─────────────────────────────────────────────────────────────── - -def parse_html(path: str) -> tuple[dict, list[dict]]: - text = read_text(path) - - metrics = _parse_metrics_html(text) - deals = _parse_deals_html(text) - return metrics, deals - - -def _parse_metrics_html(text: str) -> dict: - """Extract aggregate metrics from the summary table.""" - m = {} - - # MT5 report HTML format: MetricLabel:\r\nVALUE - # Helper patterns — values always wrapped in ... - _b = r'[^<]*\s*]*>\s*([-\d\s.,]+)' # plain number - _b_pct = r'[^<]*\s*]*>\s*[^(]*\(([\d.,]+)%\)' # "abs (pct%)" — capture pct - patterns = { - 'net_profit': r'Net\s+Profit' + _b, - 'profit_factor': r'Profit\s+Factor' + _b, - 'max_dd_pct': r'Equity\s+Drawdown\s+Maximal' + _b_pct, - 'sharpe_ratio': r'Sharpe\s+Ratio' + _b, - 'total_trades': r'Total\s+Trades' + _b, - 'recovery_factor': r'Recovery\s+Factor' + _b, - 'win_rate_pct': r'Profit\s+Trades\s+\(%' + _b_pct, - 'gross_profit': r'Gross\s+Profit' + _b, - 'gross_loss': r'Gross\s+Loss' + _b, - } - - for key, pattern in patterns.items(): - match = re.search(pattern, text, re.IGNORECASE | re.DOTALL) - if match: - val = match.group(1).replace(' ', '').replace(',', '').strip() - try: - m[key] = float(val) - except ValueError: - pass - - # Trades needs int - if 'total_trades' in m: - m['total_trades'] = int(m['total_trades']) - - return m - - -def _parse_deals_html(text: str) -> list[dict]: - """Extract deal rows from the deals table.""" - # Find deals section (after "Deals" header) - deals_section = re.search( - r']*>.*?Deal.*?Time.*?Type.*?Direction.*?Volume.*?(.*)', - text, re.DOTALL | re.IGNORECASE - ) - if not deals_section: - return [] - - rows = re.findall( - r']*>(.*?)', - deals_section.group(1), - re.DOTALL | re.IGNORECASE - ) - - deals = [] - for row in rows: - cells = re.findall(r']*>(.*?)', row, re.DOTALL | re.IGNORECASE) - cells = [strip_tags(c).replace(',', '') for c in cells] - - if len(cells) < 3 or not cells[0]: - continue - # Skip balance/deposit/credit rows — 'balance' appears in the Type column (index 3) - # or sometimes in index 1; check first 5 cells - if any(c.strip().lower() in ('balance', 'credit') for c in cells[:5]): - continue - - deal = {} - for i, col in enumerate(DEAL_COLUMNS): - deal[col] = cells[i] if i < len(cells) else '' - deals.append(deal) - - return deals - - -# ── XML parser (SpreadsheetML) ───────────────────────────────────────────────── - -def parse_xml(path: str) -> tuple[dict, list[dict]]: - """Parse MT5 SpreadsheetML optimization/report XML.""" - tree = ET.parse(path) - root = tree.getroot() - - # Namespace handling — MT5 XML uses Excel namespace - ns = {} - ns_match = re.match(r'\{([^}]+)\}', root.tag) - if ns_match: - ns['ss'] = ns_match.group(1) - - def tag(name): - return f"{{{ns['ss']}}}{name}" if ns else name - - metrics = {} - deals = [] - in_deals_sheet = False - - for sheet in root.iter(tag('Worksheet')): - sheet_name = sheet.get(f"{{{ns['ss']}}}Name" if ns else 'Name', '') - - if 'result' in sheet_name.lower() or 'report' in sheet_name.lower(): - metrics = _parse_metrics_xml(sheet, tag) - elif 'deal' in sheet_name.lower() or 'trade' in sheet_name.lower(): - deals = _parse_deals_xml(sheet, tag) - elif sheet_name == '': - # Unnamed sheet — check if it has deal-like structure - rows = list(sheet.iter(tag('Row'))) - if len(rows) > 5: - # Try to parse as deals - candidate = _parse_deals_xml(sheet, tag) - if candidate: - deals = candidate - - return metrics, deals - - -def _cell_value(cell, tag) -> str: - data = cell.find(tag('Data')) - return data.text.strip() if data is not None and data.text else '' - - -def _parse_metrics_xml(sheet, tag) -> dict: - m = {} - for row in sheet.iter(tag('Row')): - cells = [_cell_value(c, tag) for c in row.iter(tag('Cell'))] - if len(cells) < 2: - continue - key = cells[0].lower() - val = cells[1].replace(',', '').strip() - try: - fval = float(val) - if 'net profit' in key or 'net_profit' in key: - m['net_profit'] = fval - elif 'profit factor' in key: - m['profit_factor'] = fval - elif 'drawdown' in key and '%' in cells[1]: - m['max_dd_pct'] = fval - elif 'sharpe' in key: - m['sharpe_ratio'] = fval - elif 'total trades' in key: - m['total_trades'] = int(fval) - except (ValueError, AttributeError): - pass - return m - - -def _parse_deals_xml(sheet, tag) -> list[dict]: - deals = [] - header_found = False - col_map = {} - - for row in sheet.iter(tag('Row')): - cells = [_cell_value(c, tag) for c in row.iter(tag('Cell'))] - - if not header_found: - # Detect header row - if any(h in str(cells).lower() for h in ('time', 'type', 'volume', 'profit')): - header_found = True - for i, h in enumerate(cells): - h_lower = h.lower().strip() - for col in DEAL_COLUMNS: - if col in h_lower or h_lower in col: - col_map[i] = col - break - continue - - if not cells or not cells[0]: - continue - - deal = {} - for i, val in enumerate(cells): - col = col_map.get(i) - if col: - deal[col] = val.replace(',', '') - - if deal: - deals.append(deal) - - return deals - - -# ── Writer ──────────────────────────────────────────────────────────────────── - -def write_outputs(metrics: dict, deals: list[dict], output_dir: str) -> dict: - os.makedirs(output_dir, exist_ok=True) - - metrics_path = os.path.join(output_dir, 'metrics.json') - deals_csv_path = os.path.join(output_dir, 'deals.csv') - deals_json_path = os.path.join(output_dir, 'deals.json') - - with open(metrics_path, 'w') as f: - json.dump(metrics, f, indent=2) - - with open(deals_json_path, 'w') as f: - json.dump(deals, f, indent=2) - - if deals: - all_keys = DEAL_COLUMNS - with open(deals_csv_path, 'w', newline='') as f: - writer = csv.DictWriter(f, fieldnames=all_keys, extrasaction='ignore') - writer.writeheader() - writer.writerows(deals) - else: - # Write empty CSV with headers - with open(deals_csv_path, 'w', newline='') as f: - writer = csv.writer(f) - writer.writerow(DEAL_COLUMNS) - - return { - 'metrics': metrics_path, - 'deals_csv': deals_csv_path, - 'deals_json': deals_json_path, - } - - -# ── Main ────────────────────────────────────────────────────────────────────── - -def main(): - parser = argparse.ArgumentParser(description='Extract MT5 backtest report') - parser.add_argument('report', help='Path to report.htm or report.htm.xml') - parser.add_argument('--output-dir', default='.', help='Output directory') - parser.add_argument('--stdout', action='store_true', - help='Print metrics JSON to stdout instead of writing files') - args = parser.parse_args() - - fmt = detect_format(args.report) - - if fmt == 'xml': - metrics, deals = parse_xml(args.report) - else: - metrics, deals = parse_html(args.report) - - if not metrics: - print(f"WARNING: No aggregate metrics found in report", file=sys.stderr) - if not deals: - print(f"WARNING: No deals found in report (check date range and symbol)", file=sys.stderr) - - if args.stdout: - json.dump({'metrics': metrics, 'deals_count': len(deals)}, sys.stdout, indent=2) - print() - return - - paths = write_outputs(metrics, deals, args.output_dir) - - print(f"Extracted: {len(deals)} deals, {len(metrics)} metrics") - for name, path in paths.items(): - print(f" {name}: {path}") - - -if __name__ == '__main__': - main() diff --git a/analytics/optimize_parser.py b/analytics/optimize_parser.py deleted file mode 100644 index fb28ce7..0000000 --- a/analytics/optimize_parser.py +++ /dev/null @@ -1,349 +0,0 @@ -#!/usr/bin/env python3 -""" -optimize_parser.py — Parse MT5 genetic optimization results. - -Handles both HTML (.htm) and SpreadsheetML XML (.htm.xml) formats. - -Usage: - python3 analytics/optimize_parser.py --job opt_20250619_143022 - python3 analytics/optimize_parser.py --file reports/opt_dir/optimization.htm - python3 analytics/optimize_parser.py --file report.htm.xml --top 30 --sort profit -""" - -import argparse -import json -import os -import re -import sys -import xml.etree.ElementTree as ET -from pathlib import Path - - -ROOT_DIR = Path(__file__).parent.parent - - -def find_report(job_id: str) -> str: - """Locate optimization report from job metadata.""" - jobs_dir = ROOT_DIR / '.mt5mcp_jobs' - meta_path = jobs_dir / f'{job_id}.json' - - if not meta_path.exists(): - raise FileNotFoundError(f"Job not found: {job_id}. Check .mt5mcp_jobs/") - - with open(meta_path) as f: - meta = json.load(f) - - wine_prefix = meta.get('wine_prefix', '') - base = os.path.join(wine_prefix, 'drive_c', 'mt5mcp_opt_report') - - for ext in ('.htm', '.htm.xml', '.html'): - candidate = base + ext - if os.path.exists(candidate): - return candidate - - raise FileNotFoundError( - f"Optimization report not found. Expected: {base}.htm or {base}.htm.xml\n" - f"Is MT5 optimization still running? Check log: {meta.get('log_file', '')}" - ) - - -def detect_format(path: str) -> str: - if path.endswith('.xml') or path.endswith('.htm.xml'): - return 'xml' - with open(path, 'rb') as f: - header = f.read(512) - if b' str: - with open(path, 'rb') as f: - raw = f.read() - for enc in ('utf-16', 'utf-8', 'latin-1'): - try: - return raw.decode(enc) - except (UnicodeDecodeError, LookupError): - continue - return raw.decode('latin-1', errors='replace') - - -# ── HTML parser ─────────────────────────────────────────────────────────────── - -def parse_html(path: str) -> list[dict]: - text = read_text(path) - - rows = re.findall(r']*>(.*?)', text, re.DOTALL | re.IGNORECASE) - results = [] - headers = [] - - for row in rows: - cells = re.findall(r']*>(.*?)', row, re.DOTALL | re.IGNORECASE) - cells = [re.sub(r'<[^>]+>', '', c).strip().replace(',', '') for c in cells] - - if not cells: - continue - - # Header row detection - if not headers and cells[0].lower() in ('pass', '#', 'result', 'run'): - headers = cells - continue - - # Data row: first cell is pass number (digit) - if headers and cells[0].isdigit(): - row_data = dict(zip(headers, cells)) - results.append(row_data) - elif not headers and cells[0].isdigit() and len(cells) > 5: - # No header — use positional mapping (common MT5 layout) - results.append(_positional_row(cells)) - - return results - - -def _positional_row(cells: list[str]) -> dict: - """Map cells by position for headerless optimization tables.""" - # MT5 optimization table columns (typical order): - # Pass | Profit | Expected Payoff | Profit Factor | Recovery Factor | Sharpe | Custom | DD% | Trades | ...params - pos_names = ['pass', 'profit', 'expected_payoff', 'profit_factor', - 'recovery_factor', 'sharpe_ratio', 'custom', 'max_dd_pct', 'total_trades'] - row = {} - for i, name in enumerate(pos_names): - if i < len(cells): - row[name] = cells[i] - # Remaining are parameters - row['_params_raw'] = cells[len(pos_names):] - return row - - -# ── XML parser ──────────────────────────────────────────────────────────────── - -def parse_xml(path: str) -> list[dict]: - tree = ET.parse(path) - root = tree.getroot() - - ns = {} - ns_match = re.match(r'\{([^}]+)\}', root.tag) - if ns_match: - ns['ss'] = ns_match.group(1) - - def tag(name): - return f"{{{ns['ss']}}}{name}" if ns else name - - def cell_val(cell): - data = cell.find(tag('Data')) - return data.text.strip() if data is not None and data.text else '' - - results = [] - headers = [] - - for sheet in root.iter(tag('Worksheet')): - for row in sheet.iter(tag('Row')): - cells = [cell_val(c) for c in row.iter(tag('Cell'))] - cells = [c.replace(',', '').strip() for c in cells] - - if not cells: - continue - - if not headers: - if any(h.lower() in ('pass', 'result', 'profit') for h in cells): - headers = cells - continue - - if cells[0].isdigit(): - if headers: - row_data = {} - for i, h in enumerate(headers): - row_data[h.lower().replace(' ', '_')] = cells[i] if i < len(cells) else '' - results.append(row_data) - else: - results.append(_positional_row(cells)) - - return results - - -# ── Normalizer ──────────────────────────────────────────────────────────────── - -def normalize(raw_results: list[dict]) -> list[dict]: - """Convert raw parsed rows to typed dicts with consistent keys.""" - normalized = [] - - for r in raw_results: - def fget(keys, default=0.0): - for k in keys: - for rk, rv in r.items(): - if k in rk.lower().replace(' ', '_'): - try: - return float(rv) - except (ValueError, TypeError): - pass - return default - - def iget(keys, default=0): - v = fget(keys, default) - return int(v) - - # Extract known fields - entry = { - 'pass': iget(['pass', '#']), - 'net_profit': fget(['profit', 'net_profit']), - 'profit_factor': fget(['profit_factor']), - 'max_dd_pct': fget(['dd', 'drawdown']), - 'total_trades': iget(['trades']), - 'sharpe_ratio': fget(['sharpe']), - 'recovery_factor': fget(['recovery']), - } - - # Remaining keys are parameters - known_keys = {'pass', 'profit', 'net_profit', 'profit_factor', 'expected_payoff', - 'dd', 'drawdown', 'max_dd_pct', 'trades', 'total_trades', - 'sharpe', 'sharpe_ratio', 'recovery', 'recovery_factor', - 'custom', '#', '_params_raw'} - - params = {} - for k, v in r.items(): - if not any(kw in k.lower() for kw in known_keys): - try: - params[k] = float(v) - except (ValueError, TypeError): - params[k] = v - - entry['params'] = params - normalized.append(entry) - - return normalized - - -# ── Convergence analysis ────────────────────────────────────────────────────── - -def convergence_analysis(results: list[dict], top_n: int = 10) -> dict: - top = results[:top_n] - if not top: - return {} - - all_param_keys = set() - for r in top: - all_param_keys.update(r.get('params', {}).keys()) - - strong = {} # Same value across all top-N - uncertain = [] # Varies - - for key in all_param_keys: - values = set() - for r in top: - v = r.get('params', {}).get(key) - if v is not None: - values.add(v) - if len(values) == 1: - strong[key] = list(values)[0] - else: - uncertain.append(key) - - return { - 'top_n_agreement': strong, - 'high_variance_params': uncertain, - } - - -# ── Display ─────────────────────────────────────────────────────────────────── - -def display_results(results: list[dict], top_n: int, dd_threshold: float, conv: dict): - print(f"\nTotal passes: {len(results)}") - print(f"Showing top {min(top_n, len(results))} by profit:\n") - - print(f"{'Rank':<5} {'Profit':>10} {'PF':>6} {'DD%':>6} {'Sharpe':>7} {'Trades':>7} Params") - print("─" * 80) - - for i, r in enumerate(results[:top_n], 1): - dd = r['max_dd_pct'] - risk_flag = ' ⚠' if dd > dd_threshold else '' - params_str = ' '.join(f"{k}={v}" for k, v in list(r.get('params', {}).items())[:4]) - print( - f"#{i:<4} ${r['net_profit']:>9,.2f} " - f"{r['profit_factor']:>5.2f} " - f"{dd:>5.2f}%" - f"{risk_flag} " - f"{r['sharpe_ratio']:>6.2f} " - f"{r['total_trades']:>7} " - f"{params_str}" - ) - - if conv: - print(f"\nConvergence (top-{min(top_n, len(results))} agreement):") - if conv.get('top_n_agreement'): - print(" Stable params:", ', '.join(f"{k}={v}" for k, v in conv['top_n_agreement'].items())) - if conv.get('high_variance_params'): - print(" Uncertain params:", ', '.join(conv['high_variance_params'])) - - -# ── Main ────────────────────────────────────────────────────────────────────── - -def main(): - parser = argparse.ArgumentParser(description='Parse MT5 optimization results') - parser.add_argument('--job', help='Job ID from optimize.sh output') - parser.add_argument('--file', help='Direct path to optimization.htm or .htm.xml') - parser.add_argument('--top', type=int, default=20, help='Show top N results') - parser.add_argument('--sort', choices=['profit', 'profit_factor', 'sharpe'], - default='profit', help='Sort metric') - parser.add_argument('--dd-threshold', type=float, default=20.0, - help='Flag DD above this % as high-risk') - parser.add_argument('--output', help='Save results as JSON') - args = parser.parse_args() - - # Locate report - if args.file: - report_path = args.file - elif args.job: - try: - report_path = find_report(args.job) - except FileNotFoundError as e: - print(f"ERROR: {e}", file=sys.stderr) - sys.exit(1) - else: - print("ERROR: Provide --job or --file", file=sys.stderr) - sys.exit(1) - - if not os.path.exists(report_path): - print(f"ERROR: Report not found: {report_path}", file=sys.stderr) - sys.exit(1) - - # Parse - fmt = detect_format(report_path) - if fmt == 'xml': - raw = parse_xml(report_path) - else: - raw = parse_html(report_path) - - results = normalize(raw) - - if not results: - print("ERROR: No optimization passes found in report.", file=sys.stderr) - sys.exit(1) - - # Sort - sort_key = { - 'profit': 'net_profit', - 'profit_factor': 'profit_factor', - 'sharpe': 'sharpe_ratio', - }[args.sort] - results.sort(key=lambda r: r.get(sort_key, 0), reverse=True) - - # Convergence analysis - conv = convergence_analysis(results, top_n=10) - - # Display - display_results(results, args.top, args.dd_threshold, conv) - - # Optional JSON output - if args.output: - output = { - 'total_passes': len(results), - 'results': results[:args.top], - 'convergence': conv, - } - with open(args.output, 'w') as f: - json.dump(output, f, indent=2) - print(f"\nSaved: {args.output}") - - -if __name__ == '__main__': - main() diff --git a/hooks/hook-mcp.py b/hooks/hook-mcp.py deleted file mode 100644 index 91b4cca..0000000 --- a/hooks/hook-mcp.py +++ /dev/null @@ -1,18 +0,0 @@ -# PyInstaller hook for mcp package -# Ensures all mcp submodules are included - -from PyInstaller.utils.hooks import collect_submodules, collect_data_files - -hiddenimports = collect_submodules('mcp') -datas = collect_data_files('mcp') - -# Also ensure anyio dependencies are included -hiddenimports += [ - 'anyio', - 'anyio.streams', - 'anyio.streams.memory', - 'anyio.streams.text', - 'anyio._backends', - 'anyio._backends._asyncio', - 'anyio._backends._trio', -] diff --git a/pyproject.toml b/pyproject.toml deleted file mode 100644 index 5ae33be..0000000 --- a/pyproject.toml +++ /dev/null @@ -1,32 +0,0 @@ -[build-system] -requires = ["hatchling"] -build-backend = "hatchling.build" - -[project] -name = "mt5-quant" -version = "0.1.0" -description = "MCP server for MetaTrader 5 backtesting and optimization" -readme = "README.md" -license = { text = "MIT" } -requires-python = ">=3.9" -dependencies = [ - "mcp>=1.0.0", - "pyyaml>=6.0", -] - -[project.optional-dependencies] -dev = [ - "pytest>=7.0", - "pytest-asyncio>=0.21", -] - -[project.scripts] -mt5-quant = "server.main:cli" -mt5-analyze = "analytics.analyze:main_generic" -mt5-analyze-grid = "analytics.analyze:main_grid" -mt5-analyze-scalper = "analytics.analyze:main_scalper" -mt5-analyze-trend = "analytics.analyze:main_trend" -mt5-analyze-hedge = "analytics.analyze:main_hedge" - -[tool.hatch.build.targets.wheel] -packages = ["server", "analytics"] diff --git a/scripts/optimize.sh b/scripts/optimize.sh deleted file mode 100755 index 888a934..0000000 --- a/scripts/optimize.sh +++ /dev/null @@ -1,226 +0,0 @@ -#!/usr/bin/env bash -# optimize.sh — Launch MT5 genetic optimization (always background + detached) -# -# Usage: -# ./scripts/optimize.sh [options] -# -# Options: -# --expert NAME EA name -# --set FILE Optimization .set file (with ||Y flags) -# --symbol SYMBOL Trading symbol -# --from YYYY.MM.DD Start date -# --to YYYY.MM.DD End date -# --deposit AMOUNT Initial deposit -# --model 0|1|2 Tick model (ALWAYS use 0 for grid/martingale EAs) -# --log FILE Log file path (default: /tmp/mt5opt_TIMESTAMP.log) -# -# IMPORTANT: This script launches MT5 as a detached background process. -# It returns immediately. Do NOT set a timeout on this script. -# Monitor /tmp/mt5opt_*.log and wait for user signal before parsing results. - -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -ROOT_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)" -source "${SCRIPT_DIR}/platform_detect.sh" - -# ── Defaults ────────────────────────────────────────────────────────────────── -DEFAULT_SYMBOL=$(_cfg "backtest_symbol" "XAUUSD") -DEFAULT_DEPOSIT=$(_cfg "backtest_deposit" "10000") -DEFAULT_CURRENCY=$(_cfg "backtest_currency" "USD") -DEFAULT_LEVERAGE=$(_cfg "backtest_leverage" "500") - -EXPERT="" -SET_FILE="" -SYMBOL="$DEFAULT_SYMBOL" -FROM_DATE="" -TO_DATE="" -DEPOSIT="$DEFAULT_DEPOSIT" -CURRENCY="$DEFAULT_CURRENCY" -LEVERAGE="$DEFAULT_LEVERAGE" -MODEL=0 # ALWAYS 0 for optimization — see below -LOG_FILE="" - -while [[ $# -gt 0 ]]; do - case "$1" in - --expert) EXPERT="$2"; shift 2 ;; - --set) SET_FILE="$2"; shift 2 ;; - --symbol) SYMBOL="$2"; shift 2 ;; - --from) FROM_DATE="$2"; shift 2 ;; - --to) TO_DATE="$2"; shift 2 ;; - --deposit) DEPOSIT="$2"; shift 2 ;; - --model) - # Warn if user tries to use model != 0 - if [[ "$2" != "0" ]]; then - echo "WARNING: --model $2 ignored. Optimization always uses model=0." >&2 - echo " Model 1/2 overfits martingale/grid EAs (intra-bar price not simulated)." >&2 - fi - shift 2 - ;; - --log) LOG_FILE="$2"; shift 2 ;; - *) echo "Unknown option: $1" >&2; exit 1 ;; - esac -done - -[[ -z "$EXPERT" ]] && { echo "ERROR: --expert is required" >&2; exit 1; } -[[ -z "$SET_FILE" ]] && { echo "ERROR: --set is required" >&2; exit 1; } -[[ -z "$FROM_DATE" ]] && { echo "ERROR: --from is required" >&2; exit 1; } -[[ -z "$TO_DATE" ]] && { echo "ERROR: --to is required" >&2; exit 1; } - -[[ ! -f "$SET_FILE" ]] && { echo "ERROR: Set file not found: $SET_FILE" >&2; exit 1; } - -TIMESTAMP=$(date +%Y%m%d_%H%M%S) -LOG_FILE="${LOG_FILE:-/tmp/mt5opt_${TIMESTAMP}.log}" -JOB_ID="opt_${TIMESTAMP}" - -# ── Resolve platform ────────────────────────────────────────────────────────── -resolve_platform - -# ── Write .set file as UTF-16LE with BOM (read-only) ───────────────────────── -# MT5 REQUIREMENT: optimization .set files must be UTF-16LE with BOM. -# If provided as UTF-8, MT5 strips the ||Y optimization flags silently — -# every pass runs with the fixed base value and optimization is useless. -python3 - << PYEOF -import sys, os, shutil - -src = "${SET_FILE}" -dst = "${MT5_TESTER_DIR}/${EXPERT}.set" -os.makedirs("${MT5_TESTER_DIR}", exist_ok=True) - -with open(src, 'r', encoding='utf-8', errors='replace') as f: - content = f.read() - -# Write UTF-16LE with BOM -with open(dst, 'w', encoding='utf-16-le') as f: - f.write('\ufeff') # BOM - f.write(content) - -# Make read-only — prevents MT5 from overwriting ||Y flags during optimization -os.chmod(dst, 0o444) -print(f" .set → {dst} (UTF-16LE, read-only)") -PYEOF - -# ── Reset OptMode in terminal.ini ───────────────────────────────────────────── -# After any optimization run (complete or aborted), MT5 writes OptMode=-1. -# On next launch, MT5 reads OptMode=-1 and exits immediately without running. -# Must reset to 0 before every optimization launch. -TERMINAL_INI="${MT5_DIR}/terminal.ini" -if [[ -f "$TERMINAL_INI" ]]; then - # Use Python for safe in-place edit (sed -i behaves differently on macOS vs Linux) - python3 - << PYEOF -import re - -ini_path = "${TERMINAL_INI}" -with open(ini_path, 'r', errors='replace') as f: - content = f.read() - -# Reset OptMode -content = re.sub(r'OptMode=.*', 'OptMode=0', content) -# Remove LastOptimization (causes MT5 to skip running) -content = re.sub(r'LastOptimization=.*\n?', '', content) - -with open(ini_path, 'w') as f: - f.write(content) -print(f" terminal.ini: OptMode reset to 0") -PYEOF -fi - -# ── Build optimization INI ──────────────────────────────────────────────────── -WINE_PREFIX_DIR=$(dirname "$(dirname "$MT5_DIR")") - -cat > "${WINE_PREFIX_DIR}/drive_c/mt5mcp_backtest.ini" << INI -[Tester] -Expert=${EXPERT} -Symbol=${SYMBOL} -Period=M5 -Deposit=${DEPOSIT} -Currency=${CURRENCY} -Leverage=${LEVERAGE} -Model=${MODEL} -FromDate=${FROM_DATE} -ToDate=${TO_DATE} -Report=C:\\mt5mcp_opt_report -Optimization=2 -ExpertParameters=${EXPERT}.set -ShutdownTerminal=1 -INI - -cat > "${WINE_PREFIX_DIR}/drive_c/mt5mcp_run.bat" << 'EOF' -@echo off -"C:\Program Files\MetaTrader 5\terminal64.exe" /config:C:\mt5mcp_backtest.ini -EOF - -# ── Count optimization combinations ────────────────────────────────────────── -COMBINATIONS=$(python3 - << PYEOF -import re, math - -with open("${SET_FILE}", 'r', errors='replace') as f: - lines = f.readlines() - -total = 1 -for line in lines: - line = line.strip() - if line.startswith(';') or '=' not in line: - continue - # Format: param=value||start||step||stop||Y - parts = line.split('||') - if len(parts) >= 5 and parts[-1].strip().upper() == 'Y': - try: - start = float(parts[1]) - step = float(parts[2]) - stop = float(parts[3]) - count = max(1, int((stop - start) / step) + 1) - total *= count - except (ValueError, ZeroDivisionError): - pass - -print(total) -PYEOF -) - -echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" -echo " MT5-Quant Genetic Optimization" -echo " Job ID: $JOB_ID" -echo " Expert: $EXPERT" -echo " Symbol: $SYMBOL Model: ${MODEL} (every tick)" -echo " Period: $FROM_DATE → $TO_DATE" -echo " Set file: $SET_FILE" -echo " Combos: $COMBINATIONS (genetic — converges in ~300-500 passes)" -echo " Log: $LOG_FILE" -echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" - -# ── Launch detached ─────────────────────────────────────────────────────────── -# nohup: prevents SIGHUP when parent (Claude task, SSH session) exits -# disown: removes from shell job table so shell exit doesn't kill it -# Both are required for true detachment. - -nohup bash -c "${MT5_ARCH} '${MT5_WINE}' cmd.exe /c 'C:\\mt5mcp_run.bat' 2>/dev/null || true" \ - > "$LOG_FILE" 2>&1 & -OPT_PID=$! -disown $OPT_PID - -# Write job metadata -JOBS_DIR="${ROOT_DIR}/.mt5mcp_jobs" -mkdir -p "$JOBS_DIR" -cat > "${JOBS_DIR}/${JOB_ID}.json" << JEOF -{ - "job_id": "${JOB_ID}", - "pid": ${OPT_PID}, - "expert": "${EXPERT}", - "symbol": "${SYMBOL}", - "from_date": "${FROM_DATE}", - "to_date": "${TO_DATE}", - "set_file": "${SET_FILE}", - "combinations": ${COMBINATIONS}, - "log_file": "${LOG_FILE}", - "wine_prefix": "${WINE_PREFIX_DIR}", - "started_at": "$(date -u +%Y-%m-%dT%H:%M:%SZ)" -} -JEOF - -echo "" -echo " Launched (pid: $OPT_PID)" -echo " Optimization runs for 2-6 hours. Do NOT kill this process." -echo " Signal when MT5 shows 'Optimization complete' and use:" -echo " python3 analytics/optimize_parser.py --job $JOB_ID" -echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" diff --git a/server/__init__.py b/server/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/server/main.py b/server/main.py deleted file mode 100644 index 006bcce..0000000 --- a/server/main.py +++ /dev/null @@ -1,2636 +0,0 @@ -#!/usr/bin/env python3 -""" -MT5-Quant MCP Server - -Exposes MT5 backtest and optimization tools via the Model Context Protocol. -Run with: python3 server/main.py - -Add to Claude Code: - claude mcp add MT5-Quant -- python3 /path/to/mt5-quant/server/main.py -""" - -import asyncio -import difflib -import json -import os -import shutil -import subprocess -import sys -from pathlib import Path -from typing import Any - -try: - import mcp.server.stdio - import mcp.types as types - from mcp.server import Server -except ImportError as e: - print(f"ERROR: Failed to import MCP package: {e}", file=sys.stderr) - import traceback - traceback.print_exc() - sys.exit(1) - -# Config/ROOT_DIR resolution (priority order): -# 1. MT5_MCP_HOME env var (explicit override) -# 2. ~/.config/mt5-quant/ (installed-package default) -# 3. parent of this file (development / run-from-repo) -def _resolve_root() -> Path: - env_home = os.environ.get('MT5_MCP_HOME') - if env_home: - return Path(env_home).expanduser().resolve() - user_cfg = Path.home() / '.config' / 'mt5-quant' - if (user_cfg / 'config' / 'mt5-quant.yaml').exists(): - return user_cfg - return Path(__file__).parent.parent - -ROOT_DIR = _resolve_root() - -# SCRIPTS_DIR always points to the package's scripts/ (adjacent to server/main.py), -# not the config dir. Scripts are never copied to ~/.config/mt5-quant. -SCRIPTS_DIR = Path(__file__).parent.parent / 'scripts' -if not SCRIPTS_DIR.exists(): - # Installed via pip — scripts landed in the package root via pyproject.toml include - SCRIPTS_DIR = ROOT_DIR / 'scripts' - -sys.path.insert(0, str(ROOT_DIR)) -# Also ensure analytics imports resolve from the package dir -_pkg_dir = str(Path(__file__).parent.parent) -if _pkg_dir not in sys.path: - sys.path.insert(0, _pkg_dir) - -from analytics.extract import detect_format, parse_html, parse_xml, write_outputs -from analytics.analyze import ( - load_deals, load_metrics, monthly_pnl, reconstruct_dd_events, - grid_depth_histogram, top_losses, loss_sequences, build_summary -) -from analytics.optimize_parser import ( - detect_format as opt_detect_format, - parse_html as opt_parse_html, - parse_xml as opt_parse_xml, - normalize, convergence_analysis -) - -app = Server("MT5-Quant") - -# ── Config ──────────────────────────────────────────────────────────────────── - -def load_config() -> dict: - config_path = ROOT_DIR / 'config' / 'mt5-quant.yaml' - if not config_path.exists(): - return {} - config = {} - with open(config_path) as f: - # Simple YAML key: value parser (no nested support needed for basic config) - for line in f: - line = line.strip() - if line.startswith('#') or ':' not in line: - continue - key, _, val = line.partition(':') - val = val.strip().strip('"').strip("'") - if val and val not in ('null', '~', ''): - config[key.strip()] = val - return config - - -CONFIG = load_config() - - -def cfg(key: str, default: str = '') -> str: - return CONFIG.get(key, default) or default - - -REPORTS_DIR = ROOT_DIR / cfg('reports_dir', 'reports') -HISTORY_FILE = ROOT_DIR / 'config' / 'backtest_history.json' -BASELINE_FILE = ROOT_DIR / 'config' / 'baseline.json' - - -def _validate_environment() -> dict | None: - """Fast pre-flight check — returns error dict if environment is broken, None if OK.""" - config_path = ROOT_DIR / 'config' / 'mt5-quant.yaml' - missing = [] - if not config_path.exists(): - missing.append('config/mt5-quant.yaml not found') - wine = cfg('wine_executable') - if not wine: - missing.append('wine_executable not set in config') - elif not os.access(wine, os.X_OK): - missing.append(f'wine_executable not found or not executable: {wine}') - terminal_dir = cfg('terminal_dir') - if not terminal_dir: - missing.append('terminal_dir not set in config') - elif not Path(terminal_dir).is_dir(): - missing.append(f'terminal_dir not found: {terminal_dir}') - if missing: - return { - 'success': False, - 'error': 'SETUP_REQUIRED', - 'missing': missing, - 'hint': 'Run: bash scripts/setup.sh', - } - return None - - -def _check_symbol(symbol: str) -> tuple[str | None, list[str]]: - """Check symbol against active server's history. Returns (warning, suggestions).""" - terminal_dir = cfg('terminal_dir') - if not terminal_dir: - return None, [] - - ini = _read_terminal_ini() - active_server = ini.get('LastScanServer', '') - bases_dir = Path(terminal_dir) / 'Bases' - - # Try active server first, then fall back to scanning all servers - history_dir = None - if active_server and (bases_dir / active_server / 'history').is_dir(): - history_dir = bases_dir / active_server / 'history' - elif bases_dir.is_dir(): - for srv in bases_dir.iterdir(): - if (srv / 'history').is_dir(): - history_dir = srv / 'history' - break - - if not history_dir: - return None, [] - - known = [d.name for d in history_dir.iterdir() if d.is_dir()] - if not known or symbol in known: - return None, [] - - suggestions = difflib.get_close_matches(symbol, known, n=3, cutoff=0.5) - - # Check if symbol exists in any other server - other_servers = [] - if bases_dir.is_dir(): - for srv in bases_dir.iterdir(): - if srv.name == active_server: - continue - if (srv / 'history' / symbol).is_dir(): - other_servers.append(srv.name) - - msg = f"Symbol '{symbol}' not found in active server '{active_server}'. Available: {', '.join(known[:6])}{'...' if len(known) > 6 else ''}" - if other_servers: - msg += f". Found in: {', '.join(other_servers)}" - return msg, suggestions - - -# ── History helpers ─────────────────────────────────────────────────────────── - -def load_history() -> list[dict]: - if not HISTORY_FILE.exists(): - return [] - with open(HISTORY_FILE) as f: - return json.load(f) - - -def save_history(entries: list[dict]) -> None: - HISTORY_FILE.parent.mkdir(exist_ok=True) - with open(HISTORY_FILE, 'w') as f: - json.dump(entries, f, indent=2) - - -def _build_history_entry(report_dir: str) -> dict | None: - """Build a compact, self-contained history entry from a report directory.""" - from datetime import datetime, timezone - d = Path(report_dir) - metrics = read_json(str(d / 'metrics.json')) - analysis = read_json(str(d / 'analysis.json')) - - if not metrics and not analysis: - return None - - entry: dict = { - 'id': d.name, - 'report_dir': str(d), - 'report_dir_deleted': False, - 'archived_at': datetime.now(timezone.utc).isoformat(), - 'ea': metrics.get('expert') or metrics.get('ea') or '', - 'symbol': metrics.get('symbol', ''), - 'timeframe': metrics.get('timeframe', ''), - 'from_date': metrics.get('from_date') or metrics.get('testing_from', ''), - 'to_date': metrics.get('to_date') or metrics.get('testing_to', ''), - 'metrics': { - 'net_profit': metrics.get('net_profit'), - 'profit_factor': metrics.get('profit_factor'), - 'max_dd_pct': metrics.get('max_dd_pct'), - 'sharpe_ratio': metrics.get('sharpe_ratio'), - 'total_trades': metrics.get('total_trades'), - 'recovery_factor': metrics.get('recovery_factor'), - 'win_rate_pct': metrics.get('win_rate_pct'), - 'expected_payoff': metrics.get('expected_payoff'), - }, - 'verdict': None, - 'notes': '', - 'tags': [], - 'promoted_to_baseline': False, - } - - if analysis: - summary = analysis.get('summary', {}) - entry['summary'] = {k: summary.get(k) for k in ( - 'green_months', 'total_months', 'worst_month', 'worst_month_pnl', - 'worst_dd_event_pct', 'max_grid_depth', 'l5_plus_count', - 'dominant_exit', 'max_win_streak', 'max_loss_streak', - 'current_streak', 'current_streak_type', - ) if summary.get(k) is not None} - monthly = analysis.get('monthly_pnl', []) - if monthly: - entry['monthly_pnl'] = monthly - dd_events = analysis.get('dd_events', []) - if dd_events: - entry['worst_dd_event'] = dd_events[0] - - return entry - - -# ── Tool helpers ────────────────────────────────────────────────────────────── - -def run_script(cmd: list[str], timeout: int = 900) -> tuple[bool, str]: - """Run a shell script synchronously and return (success, output).""" - try: - result = subprocess.run( - cmd, - capture_output=True, - text=True, - timeout=timeout, - cwd=str(ROOT_DIR), - ) - output = result.stdout + result.stderr - return result.returncode == 0, output - except subprocess.TimeoutExpired: - return False, f"Timeout after {timeout}s" - except Exception as e: - return False, str(e) - - -def latest_report_dir() -> str | None: - """Find most recently created report directory.""" - REPORTS_DIR.mkdir(exist_ok=True) - dirs = sorted(REPORTS_DIR.iterdir(), reverse=True) - for d in dirs: - if d.is_dir() and not d.name.endswith('_opt'): - return str(d) - return None - - -def read_json(path: str) -> dict: - if not os.path.exists(path): - return {} - with open(path) as f: - return json.load(f) - - -def format_result(data: dict) -> str: - return json.dumps(data, indent=2) - - -# ── Tool definitions ────────────────────────────────────────────────────────── - -@app.list_tools() -async def list_tools() -> list[types.Tool]: - return [ - types.Tool( - name="run_backtest", - description=( - "Run a complete MT5 backtest pipeline: compile → clean cache → " - "backtest → extract → analyze. Returns profit, DD%, Sharpe, monthly P/L, " - "and drawdown event reconstruction. Always compiles and clears cache unless " - "skip flags are set." - ), - inputSchema={ - "type": "object", - "required": ["expert"], - "properties": { - "expert": { - "type": "string", - "description": "EA name without path or extension. e.g. 'MyEA_v1.2'" - }, - "symbol": { - "type": "string", - "description": "Trading symbol. Use your broker's exact name. e.g. 'XAUUSD'" - }, - "from_date": { - "type": "string", - "description": "Start date in YYYY.MM.DD format" - }, - "to_date": { - "type": "string", - "description": "End date in YYYY.MM.DD format" - }, - "preset": { - "type": "string", - "enum": ["last_month", "last_3months", "ytd", "last_year"], - "description": "Date preset (alternative to from/to)" - }, - "timeframe": { - "type": "string", - "enum": ["M1", "M5", "M15", "M30", "H1", "H4", "D1"], - "description": "Chart timeframe (default: M5)" - }, - "deposit": { - "type": "number", - "description": "Initial deposit (default: from config)" - }, - "model": { - "type": "integer", - "enum": [0, 1, 2], - "description": "0=every tick (default), 1=1min OHLC, 2=open price" - }, - "set_file": { - "type": "string", - "description": "Path to .set parameter file" - }, - "skip_compile": { - "type": "boolean", - "description": "Skip compilation (use existing .ex5)" - }, - "skip_clean": { - "type": "boolean", - "description": "Skip cache clean (faster, risks stale results)" - }, - "skip_analyze": { - "type": "boolean", - "description": "Extract only, skip deal analysis" - }, - "deep": { - "type": "boolean", - "description": "Run deep analysis (grid + regime breakdown)" - }, - "gui": { - "type": "boolean", - "description": "Show MT5 visual mode (chart animation). Default: false (headless). Use for debugging or demo." - }, - "timeout": { - "type": "integer", - "description": "Backtest timeout in seconds (default: 900)" - }, - "shutdown": { - "type": "boolean", - "description": "Close MT5 after backtest completes. Default: false — MT5 stays open and report is detected via file watching. Set true for CI/headless environments." - }, - "kill_existing": { - "type": "boolean", - "description": "Kill a running MT5 instance before launching. REQUIRED when MT5 is already open — Wine does not support single-instance config passthrough. With shutdown=false (default), MT5 restarts, runs the backtest, then stays open so results are visible in the GUI." - }, - }, - }, - ), - types.Tool( - name="run_optimization", - description=( - "Launch MT5 genetic parameter optimization as a detached background process. " - "Returns immediately — MT5 runs for 2-6 hours. " - "Always uses model=0 (every tick). Call get_optimization_results only after " - "user confirms MT5 has finished." - ), - inputSchema={ - "type": "object", - "required": ["expert", "set_file", "from_date", "to_date"], - "properties": { - "expert": {"type": "string"}, - "set_file": { - "type": "string", - "description": "Path to optimization .set file with ||Y sweep flags" - }, - "from_date": {"type": "string"}, - "to_date": {"type": "string"}, - "symbol": {"type": "string"}, - "deposit": {"type": "number"}, - }, - }, - ), - types.Tool( - name="get_optimization_results", - description=( - "Parse completed MT5 optimization results. Call only after user signals " - "that MT5 optimization has finished. Returns top passes sorted by profit " - "with convergence analysis." - ), - inputSchema={ - "type": "object", - "properties": { - "job_id": { - "type": "string", - "description": "Job ID from run_optimization response" - }, - "report_file": { - "type": "string", - "description": "Direct path to optimization.htm or .htm.xml" - }, - "top_n": { - "type": "integer", - "description": "Number of top results to return (default: 20)" - }, - "dd_threshold": { - "type": "number", - "description": "Flag results above this DD% as high-risk (default: 20)" - }, - }, - }, - ), - types.Tool( - name="analyze_report", - description=( - "Read and summarize a completed backtest report. Does not re-run MT5. " - "Returns monthly P/L, drawdown events, grid depth histogram, and top losses." - ), - inputSchema={ - "type": "object", - "properties": { - "report_dir": { - "type": "string", - "description": "Path to report directory. If omitted, uses latest." - }, - }, - }, - ), - types.Tool( - name="compare_baseline", - description=( - "Compare a backtest report against a baseline. Returns winner/loser verdict " - "and delta metrics. Baseline must include net_profit and max_dd_pct." - ), - inputSchema={ - "type": "object", - "required": ["baseline"], - "properties": { - "report_dir": { - "type": "string", - "description": "Report to evaluate. If omitted, uses latest." - }, - "baseline": { - "type": "object", - "required": ["net_profit", "max_dd_pct"], - "properties": { - "net_profit": {"type": "number"}, - "max_dd_pct": {"type": "number"}, - "total_trades": {"type": "integer"}, - "label": {"type": "string"}, - }, - }, - "promote_dd_limit": { - "type": "number", - "description": "Auto-promote only if DD < this % (default: 20)" - }, - }, - }, - ), - types.Tool( - name="compile_ea", - description="Compile an MQL5 Expert Advisor via MetaEditor (Wine/CrossOver).", - inputSchema={ - "type": "object", - "required": ["expert_path"], - "properties": { - "expert_path": { - "type": "string", - "description": "Path to .mq5 source file" - }, - }, - }, - ), - types.Tool( - name="verify_setup", - description=( - "Verify the MT5-Quant environment without launching MT5. " - "Checks Wine executable, MT5 installation paths, and config file. " - "Run this first if other tools return SETUP_REQUIRED errors." - ), - inputSchema={"type": "object", "properties": {}}, - ), - types.Tool( - name="list_symbols", - description=( - "Detect the active MT5 broker session and list symbols that have local " - "tick history available for backtesting. Also shows all broker servers " - "found in the MT5 installation. Use this before run_backtest to confirm " - "the correct symbol name for the connected broker." - ), - inputSchema={ - "type": "object", - "properties": { - "server": { - "type": "string", - "description": "Filter to a specific server name. If omitted, shows active server and all servers.", - }, - }, - }, - ), - types.Tool( - name="list_experts", - description=( - "List all compiled Expert Advisors (.ex5 files) found in the MT5 Experts " - "directory, including those inside sub-folders. Returns the expert name to " - "use in run_backtest and the sub-folder path if applicable." - ), - inputSchema={ - "type": "object", - "properties": { - "filter": { - "type": "string", - "description": "Optional substring filter on EA name (case-insensitive).", - }, - }, - }, - ), - types.Tool( - name="get_backtest_status", - description=( - "Check progress of a running or recently completed backtest pipeline. " - "Returns current stage (COMPILE/CLEAN/BACKTEST/EXTRACT/ANALYZE/DONE), " - "elapsed time, and whether the pipeline has finished." - ), - inputSchema={ - "type": "object", - "properties": { - "report_dir": { - "type": "string", - "description": "Report directory path. If omitted, uses latest.", - }, - }, - }, - ), - types.Tool( - name="get_optimization_status", - description=( - "Check whether a background optimization job is still running. " - "Returns process alive status, elapsed time, last 20 log lines, " - "and whether the report file has appeared (definitive completion signal)." - ), - inputSchema={ - "type": "object", - "required": ["job_id"], - "properties": { - "job_id": { - "type": "string", - "description": "Job ID returned by run_optimization.", - }, - }, - }, - ), - types.Tool( - name="prune_reports", - description=( - "Delete old backtest report directories, keeping only the N most recent. " - "Optimization result directories (_opt suffix) are never deleted." - ), - inputSchema={ - "type": "object", - "properties": { - "keep_last": { - "type": "integer", - "description": "Number of most recent reports to keep (default: 20).", - }, - }, - }, - ), - types.Tool( - name="list_reports", - description=( - "List all backtest report directories with compact key metrics " - "(profit, DD%, trades, date). Much cheaper than calling analyze_report " - "repeatedly. Use this to survey what runs exist before drilling in." - ), - inputSchema={ - "type": "object", - "properties": { - "include_opt": { - "type": "boolean", - "description": "Include optimization result dirs (_opt suffix). Default: false.", - }, - "limit": { - "type": "integer", - "description": "Max reports to return, newest first (default: 30).", - }, - }, - }, - ), - types.Tool( - name="tail_log", - description=( - "Read the last N lines of a backtest progress log or optimization log. " - "Use filter='errors' to get only ERROR/WARN lines. Cheaper than " - "get_optimization_status when you only need log content." - ), - inputSchema={ - "type": "object", - "properties": { - "report_dir": { - "type": "string", - "description": "Backtest report dir (reads progress.log). Omit for latest.", - }, - "job_id": { - "type": "string", - "description": "Optimization job ID (reads its log file).", - }, - "log_file": { - "type": "string", - "description": "Absolute path to any log file.", - }, - "n": { - "type": "integer", - "description": "Number of lines to return (default: 50).", - }, - "filter": { - "type": "string", - "enum": ["all", "errors", "warnings"], - "description": "Line filter (default: all).", - }, - }, - }, - ), - types.Tool( - name="cache_status", - description=( - "Show MT5 tester cache size breakdown by symbol/timeframe directory. " - "Call before clean_cache to understand what will be deleted." - ), - inputSchema={"type": "object", "properties": {}}, - ), - types.Tool( - name="clean_cache", - description=( - "Delete MT5 tester cache files to force fresh price data on next backtest. " - "Optionally target a specific symbol. Returns bytes freed." - ), - inputSchema={ - "type": "object", - "properties": { - "symbol": { - "type": "string", - "description": "Delete only cache for this symbol. Omit to delete all.", - }, - "dry_run": { - "type": "boolean", - "description": "Report what would be deleted without deleting. Default: false.", - }, - }, - }, - ), - types.Tool( - name="read_set_file", - description=( - "Parse an MT5 .set parameter file (UTF-16LE or UTF-8) into structured JSON. " - "Returns each parameter with its value and optimization sweep config. " - "Use this instead of reading raw .set files." - ), - inputSchema={ - "type": "object", - "required": ["path"], - "properties": { - "path": { - "type": "string", - "description": "Path to .set file.", - }, - }, - }, - ), - types.Tool( - name="write_set_file", - description=( - "Write an MT5 .set parameter file in UTF-16LE encoding (required by MT5). " - "Accepts a dict of params. For optimization sweeps include from/to/step keys. " - "Existing file is overwritten and chmod 444 is applied." - ), - inputSchema={ - "type": "object", - "required": ["path", "params"], - "properties": { - "path": { - "type": "string", - "description": "Output path for .set file.", - }, - "params": { - "type": "object", - "description": ( - "Dict of param_name → value or dict with keys: " - "value, from, to, step, optimize (bool)." - ), - }, - }, - }, - ), - types.Tool( - name="patch_set_file", - description=( - "Modify specific parameters in an existing .set file in-place. " - "Preserves all other params, comments, and sweep config. " - "Returns a diff of what changed. " - "Use instead of read_set_file → edit → write_set_file (saves 2 round-trips)." - ), - inputSchema={ - "type": "object", - "required": ["path", "patches"], - "properties": { - "path": { - "type": "string", - "description": "Path to the .set file to modify.", - }, - "patches": { - "type": "object", - "description": ( - "Params to update. Each key is a param name. " - "Value can be a scalar (just updates value) or a dict " - "with keys: value, from, to, step, optimize." - ), - }, - }, - }, - ), - types.Tool( - name="clone_set_file", - description=( - "Copy a .set file to a new path, applying optional overrides. " - "One call instead of read → modify → write. " - "Useful for creating variant .set files from a base config." - ), - inputSchema={ - "type": "object", - "required": ["source", "destination"], - "properties": { - "source": { - "type": "string", - "description": "Path to source .set file.", - }, - "destination": { - "type": "string", - "description": "Output path for the cloned .set file.", - }, - "overrides": { - "type": "object", - "description": ( - "Optional param overrides to apply in the clone. " - "Same format as patch_set_file patches." - ), - }, - }, - }, - ), - types.Tool( - name="set_from_optimization", - description=( - "Generate a .set file directly from an optimization result's params dict. " - "Strips all sweep flags (||Y) to produce a clean backtest .set. " - "Optionally uses a template .set for params not in the optimization result. " - "Optionally re-adds sweep ranges to selected params for follow-on optimization. " - "Use immediately after get_optimization_results — params dict comes from results[0].params." - ), - inputSchema={ - "type": "object", - "required": ["path", "params"], - "properties": { - "path": { - "type": "string", - "description": "Output path for the generated .set file.", - }, - "params": { - "type": "object", - "description": ( - "Flat dict of param_name → value from optimization result. " - "e.g. {'TP_Pips': 400, 'Min_Confidence': 0.61}" - ), - }, - "template": { - "type": "string", - "description": ( - "Optional path to an existing .set file. " - "Params not in 'params' are filled from the template as fixed values." - ), - }, - "sweep": { - "type": "object", - "description": ( - "Optional: re-add sweep ranges to specific params after applying opt values. " - "Dict of param_name → {from, to, step, optimize: true}. " - "Use to create a narrowed follow-on optimization .set." - ), - }, - }, - }, - ), - types.Tool( - name="diff_set_files", - description=( - "Compare two .set files and return only the differences: " - "params added, removed, or changed (value or sweep flag). " - "Use instead of reading both files and comparing manually." - ), - inputSchema={ - "type": "object", - "required": ["path_a", "path_b"], - "properties": { - "path_a": {"type": "string", "description": "First .set file (baseline/old)."}, - "path_b": {"type": "string", "description": "Second .set file (candidate/new)."}, - }, - }, - ), - types.Tool( - name="describe_sweep", - description=( - "Show a .set file's sweep configuration: which params are swept, " - "their ranges, value counts, and total optimization combinations. " - "Use before run_optimization to verify scope." - ), - inputSchema={ - "type": "object", - "required": ["path"], - "properties": { - "path": {"type": "string", "description": "Path to .set file."}, - }, - }, - ), - types.Tool( - name="list_set_files", - description=( - "List all .set files in the MT5 tester profiles directory with " - "param counts, swept param counts, and total optimization combinations. " - "Use instead of reading each file individually to find the right .set." - ), - inputSchema={ - "type": "object", - "properties": { - "ea": { - "type": "string", - "description": "Filter by EA name substring (case-insensitive).", - }, - }, - }, - ), - types.Tool( - name="list_jobs", - description=( - "List all optimization jobs with compact status (alive/done/failed, elapsed). " - "Cheaper than calling get_optimization_status for each job individually." - ), - inputSchema={ - "type": "object", - "properties": { - "include_done": { - "type": "boolean", - "description": "Include completed jobs (default: true).", - }, - }, - }, - ), - types.Tool( - name="archive_report", - description=( - "Convert a backtest report directory into a compact JSON entry appended to " - "config/backtest_history.json. Captures all metrics, analysis summary, monthly P/L, " - "and worst DD event. Optionally deletes the source directory to reclaim disk space. " - "Skips if the report is already in history (idempotent)." - ), - inputSchema={ - "type": "object", - "properties": { - "report_dir": { - "type": "string", - "description": "Report directory to archive. If omitted, uses latest.", - }, - "delete_after": { - "type": "boolean", - "description": "Delete source directory after archiving (default: false).", - }, - "verdict": { - "type": "string", - "enum": ["winner", "loser", "marginal", "reference"], - "description": "Optional verdict to attach to the entry.", - }, - "notes": { - "type": "string", - "description": "Free-text notes to attach to the entry.", - }, - "tags": { - "type": "array", - "items": {"type": "string"}, - "description": "Tags to attach (e.g. ['tight-sl', 'new-entry-filter']).", - }, - }, - }, - ), - types.Tool( - name="archive_all_reports", - description=( - "Bulk-archive all backtest report directories into config/backtest_history.json, " - "then optionally delete the source directories. Skips dirs already in history. " - "Optimization dirs (_opt suffix) are never deleted. " - "Use this to clean up disk space while preserving all results as JSON." - ), - inputSchema={ - "type": "object", - "properties": { - "delete_after": { - "type": "boolean", - "description": "Delete source directories after archiving (default: false).", - }, - "keep_last": { - "type": "integer", - "description": "Keep this many newest dirs even if delete_after=true (default: 5).", - }, - "dry_run": { - "type": "boolean", - "description": "Report what would happen without making changes (default: false).", - }, - }, - }, - ), - types.Tool( - name="get_history", - description=( - "Query config/backtest_history.json with filters. Returns compact entries sorted " - "newest-first by default. Use this to compare past runs, find regressions, or " - "pick a candidate to promote to baseline." - ), - inputSchema={ - "type": "object", - "properties": { - "ea": { - "type": "string", - "description": "Filter by EA name (substring match).", - }, - "symbol": { - "type": "string", - "description": "Filter by symbol (exact match).", - }, - "verdict": { - "type": "string", - "enum": ["winner", "loser", "marginal", "reference"], - "description": "Filter by verdict.", - }, - "tag": { - "type": "string", - "description": "Filter entries that contain this tag.", - }, - "min_profit": { - "type": "number", - "description": "Filter entries with net_profit >= this value.", - }, - "max_dd_pct": { - "type": "number", - "description": "Filter entries with max_dd_pct <= this value.", - }, - "sort_by": { - "type": "string", - "enum": ["date", "profit", "dd", "sharpe"], - "description": "Sort order (default: date, newest first).", - }, - "limit": { - "type": "integer", - "description": "Max entries to return (default: 20).", - }, - "include_monthly": { - "type": "boolean", - "description": "Include monthly_pnl arrays (default: false, saves tokens).", - }, - }, - }, - ), - types.Tool( - name="promote_to_baseline", - description=( - "Promote a backtest result to config/baseline.json — the reference used by " - "compare_baseline and the Claude Code baseline hook. " - "Accepts a history entry id, a report_dir, or defaults to the latest report. " - "Also marks the history entry as promoted." - ), - inputSchema={ - "type": "object", - "properties": { - "history_id": { - "type": "string", - "description": "Entry id from get_history (report dir basename).", - }, - "report_dir": { - "type": "string", - "description": "Direct path to report directory (alternative to history_id).", - }, - "notes": { - "type": "string", - "description": "Notes written to baseline.json (e.g. 'v1.3 promoted after 3-month walk-forward').", - }, - }, - }, - ), - types.Tool( - name="annotate_history", - description=( - "Add or update notes, verdict, or tags on a history entry in " - "config/backtest_history.json. Use this after compare_baseline to record " - "the verdict, or to tag runs for later retrieval." - ), - inputSchema={ - "type": "object", - "required": ["history_id"], - "properties": { - "history_id": { - "type": "string", - "description": "Entry id (report dir basename) to update.", - }, - "verdict": { - "type": "string", - "enum": ["winner", "loser", "marginal", "reference"], - }, - "notes": { - "type": "string", - "description": "Free-text notes (replaces existing notes).", - }, - "tags": { - "type": "array", - "items": {"type": "string"}, - "description": "Tags to set (replaces existing tags).", - }, - "add_tags": { - "type": "array", - "items": {"type": "string"}, - "description": "Tags to append without replacing existing ones.", - }, - }, - }, - ), - ] - - -# ── Tool handlers ───────────────────────────────────────────────────────────── - -@app.call_tool() -async def call_tool(name: str, arguments: dict[str, Any]) -> list[types.TextContent]: - try: - if name == "run_backtest": - result = await handle_run_backtest(arguments) - elif name == "run_optimization": - result = await handle_run_optimization(arguments) - elif name == "get_optimization_results": - result = await handle_get_optimization_results(arguments) - elif name == "analyze_report": - result = await handle_analyze_report(arguments) - elif name == "compare_baseline": - result = await handle_compare_baseline(arguments) - elif name == "compile_ea": - result = await handle_compile_ea(arguments) - elif name == "list_symbols": - result = await handle_list_symbols(arguments) - elif name == "list_experts": - result = await handle_list_experts(arguments) - elif name == "verify_setup": - result = await handle_verify_setup(arguments) - elif name == "get_backtest_status": - result = await handle_get_backtest_status(arguments) - elif name == "get_optimization_status": - result = await handle_get_optimization_status(arguments) - elif name == "prune_reports": - result = await handle_prune_reports(arguments) - elif name == "list_reports": - result = await handle_list_reports(arguments) - elif name == "tail_log": - result = await handle_tail_log(arguments) - elif name == "cache_status": - result = await handle_cache_status(arguments) - elif name == "clean_cache": - result = await handle_clean_cache(arguments) - elif name == "read_set_file": - result = await handle_read_set_file(arguments) - elif name == "write_set_file": - result = await handle_write_set_file(arguments) - elif name == "patch_set_file": - result = await handle_patch_set_file(arguments) - elif name == "clone_set_file": - result = await handle_clone_set_file(arguments) - elif name == "set_from_optimization": - result = await handle_set_from_optimization(arguments) - elif name == "diff_set_files": - result = await handle_diff_set_files(arguments) - elif name == "describe_sweep": - result = await handle_describe_sweep(arguments) - elif name == "list_set_files": - result = await handle_list_set_files(arguments) - elif name == "list_jobs": - result = await handle_list_jobs(arguments) - elif name == "archive_report": - result = await handle_archive_report(arguments) - elif name == "archive_all_reports": - result = await handle_archive_all_reports(arguments) - elif name == "get_history": - result = await handle_get_history(arguments) - elif name == "promote_to_baseline": - result = await handle_promote_to_baseline(arguments) - elif name == "annotate_history": - result = await handle_annotate_history(arguments) - else: - result = {"error": f"Unknown tool: {name}"} - except Exception as e: - result = {"error": str(e), "success": False} - - return [types.TextContent(type="text", text=format_result(result))] - - -async def handle_run_backtest(args: dict) -> dict: - env_error = _validate_environment() - if env_error: - return env_error - - symbol = args.get('symbol') or cfg('backtest_symbol', 'XAUUSD') - symbol_warning, symbol_suggestions = _check_symbol(symbol) - - cmd = [str(SCRIPTS_DIR / 'backtest_pipeline.sh')] - cmd += ['--expert', args['expert']] - project_dir = cfg('project_dir', '') - if project_dir: - cmd += ['--project-dir', project_dir] - - if 'symbol' in args: - cmd += ['--symbol', args['symbol']] - if 'preset' in args: - cmd += ['--preset', args['preset']] - if 'from_date' in args: - cmd += ['--from', args['from_date']] - if 'to_date' in args: - cmd += ['--to', args['to_date']] - if 'timeframe' in args: - cmd += ['--timeframe', args['timeframe']] - if 'deposit' in args: - cmd += ['--deposit', str(args['deposit'])] - if 'model' in args: - cmd += ['--model', str(args['model'])] - if 'set_file' in args: - set_file = args['set_file'] - # Resolve relative paths against project_dir (where the EA repo lives) - if project_dir and not os.path.isabs(set_file): - set_file = os.path.join(project_dir, set_file) - cmd += ['--set', set_file] - if args.get('skip_compile'): - cmd.append('--skip-compile') - if args.get('skip_clean'): - cmd.append('--skip-clean') - if args.get('skip_analyze'): - cmd.append('--skip-analyze') - if args.get('deep'): - cmd.append('--deep') - if args.get('gui'): - cmd.append('--gui') - if args.get('shutdown'): - cmd.append('--shutdown') - if args.get('kill_existing'): - cmd.append('--kill-existing') - - timeout = args.get('timeout', 900) - success, output = run_script(cmd, timeout=timeout) - - if not success: - return {'success': False, 'error': output[-2000:]} # last 2k chars - - # Parse report dir from pipeline output (reliable, avoids stale REPORTS_DIR at startup) - report_dir = None - for line in output.splitlines(): - if line.strip().startswith('Report:') or ' Report: ' in line: - parts = line.split('Report:', 1) - if len(parts) == 2: - candidate = parts[1].strip() - if os.path.isdir(candidate): - report_dir = candidate - break - if not report_dir: - report_dir = latest_report_dir() - if not report_dir: - return {'success': False, 'error': 'Pipeline completed but no report directory found'} - - metrics = read_json(os.path.join(report_dir, 'metrics.json')) - analysis = read_json(os.path.join(report_dir, 'analysis.json')) - - result = { - 'success': True, - 'report_dir': report_dir, - 'metrics': metrics, - 'analysis_summary': analysis.get('summary', {}), - 'worst_dd_event': analysis.get('dd_events', [{}])[0] if analysis.get('dd_events') else None, - 'monthly_pnl': analysis.get('monthly_pnl', []), - 'grid_depth_histogram': analysis.get('grid_depth_histogram', {}), - 'output': output[-1000:], - } - if symbol_warning: - result['symbol_warning'] = symbol_warning - result['symbol_suggestions'] = symbol_suggestions - return result - - -async def handle_run_optimization(args: dict) -> dict: - env_error = _validate_environment() - if env_error: - return env_error - - cmd = [str(SCRIPTS_DIR / 'optimize.sh')] - cmd += ['--expert', args['expert']] - cmd += ['--set', args['set_file']] - cmd += ['--from', args['from_date']] - cmd += ['--to', args['to_date']] - - if 'symbol' in args: - cmd += ['--symbol', args['symbol']] - if 'deposit' in args: - cmd += ['--deposit', str(args['deposit'])] - - success, output = run_script(cmd, timeout=60) # script returns quickly (nohup) - - # Extract job ID from output - import re - job_match = re.search(r'opt_\d{8}_\d{6}', output) - job_id = job_match.group(0) if job_match else None - - return { - 'success': success, - 'job_id': job_id, - 'message': 'Optimization launched in background. Do NOT poll. Signal me when MT5 completes.', - 'output': output[-500:], - } - - -async def handle_get_optimization_results(args: dict) -> dict: - from analytics.optimize_parser import find_report as find_opt_report - - # Locate report - report_path = None - if 'report_file' in args: - report_path = args['report_file'] - elif 'job_id' in args: - try: - report_path = find_opt_report(args['job_id']) - except FileNotFoundError as e: - return {'success': False, 'error': str(e)} - - if not report_path or not os.path.exists(report_path): - return {'success': False, 'error': 'Report not found. Is optimization still running?'} - - fmt = opt_detect_format(report_path) - if fmt == 'xml': - raw = opt_parse_xml(report_path) - else: - raw = opt_parse_html(report_path) - - results = normalize(raw) - results.sort(key=lambda r: r.get('net_profit', 0), reverse=True) - - top_n = args.get('top_n', 20) - dd_threshold = args.get('dd_threshold', 20.0) - conv = convergence_analysis(results, top_n=10) - - # Flag high-risk results - for r in results: - r['high_risk'] = r.get('max_dd_pct', 0) > dd_threshold - - return { - 'success': True, - 'total_passes': len(results), - 'results': results[:top_n], - 'convergence': conv, - 'recommendation': _opt_recommendation(results, dd_threshold), - } - - -def _opt_recommendation(results: list[dict], dd_threshold: float) -> dict: - safe = [r for r in results if r.get('max_dd_pct', 999) < dd_threshold] - if not safe: - return { - 'verdict': 'all_high_risk', - 'message': f'All top results exceed DD threshold ({dd_threshold}%). Widen parameter ranges or increase DD threshold.', - } - best = safe[0] - return { - 'verdict': 'verify_model0' if best.get('model', 0) != 0 else 'promote_candidate', - 'best_params': best.get('params', {}), - 'net_profit': best.get('net_profit', 0), - 'max_dd_pct': best.get('max_dd_pct', 0), - 'message': f"Run verification backtest with these params before promoting.", - } - - -async def handle_analyze_report(args: dict) -> dict: - report_dir = args.get('report_dir') or latest_report_dir() - if not report_dir: - return {'success': False, 'error': 'No report directory found'} - - metrics = read_json(os.path.join(report_dir, 'metrics.json')) - analysis = read_json(os.path.join(report_dir, 'analysis.json')) - - if not metrics and not analysis: - # Try re-running analysis on deals.csv - deals_csv = os.path.join(report_dir, 'deals.csv') - if os.path.exists(deals_csv): - deals = load_deals(deals_csv) - monthly = monthly_pnl(deals) - dd_events = reconstruct_dd_events(deals, metrics) - analysis = { - 'summary': build_summary(metrics, monthly, dd_events), - 'monthly_pnl': monthly, - 'dd_events': dd_events, - 'grid_depth_histogram': grid_depth_histogram(deals), - 'top_losses': top_losses(deals), - 'loss_sequences': loss_sequences(deals), - } - else: - return {'success': False, 'error': f'No data found in {report_dir}'} - - return { - 'success': True, - 'report_dir': report_dir, - 'metrics': metrics, - **analysis, - } - - -async def handle_compare_baseline(args: dict) -> dict: - report_dir = args.get('report_dir') or latest_report_dir() - if not report_dir: - return {'success': False, 'error': 'No report directory found'} - - metrics = read_json(os.path.join(report_dir, 'metrics.json')) - baseline = args['baseline'] - dd_limit = args.get('promote_dd_limit', 20.0) - - candidate_profit = metrics.get('net_profit', 0) - candidate_dd = metrics.get('max_dd_pct', 999) - baseline_profit = baseline['net_profit'] - baseline_dd = baseline['max_dd_pct'] - - profit_delta = candidate_profit - baseline_profit - dd_delta = candidate_dd - baseline_dd - profit_pct = (profit_delta / baseline_profit * 100) if baseline_profit else 0 - - is_winner = candidate_profit > baseline_profit and candidate_dd < dd_limit - - if is_winner: - verdict = 'winner' - elif candidate_profit > baseline_profit: - verdict = 'marginal' # Better profit but DD too high - else: - verdict = 'loser' - - sign = '+' if profit_delta >= 0 else '' - dd_sign = '+' if dd_delta >= 0 else '' - - return { - 'success': True, - 'verdict': verdict, - 'auto_promote': is_winner, - 'delta': { - 'profit_usd': round(profit_delta, 2), - 'profit_pct': round(profit_pct, 1), - 'dd_pp': round(dd_delta, 2), - }, - 'summary': ( - f"{sign}${profit_delta:,.2f} ({sign}{profit_pct:.1f}%) profit vs {baseline.get('label', 'baseline')}. " - f"DD: {candidate_dd:.2f}% vs {baseline_dd:.2f}% ({dd_sign}{dd_delta:.2f}pp). " - f"{'Auto-promoting.' if is_winner else 'Not promoting (DD too high).' if verdict == 'marginal' else 'Regression.'}" - ), - 'candidate': { - 'net_profit': candidate_profit, - 'max_dd_pct': candidate_dd, - 'total_trades': metrics.get('total_trades', 0), - }, - 'baseline': baseline, - } - - -async def handle_compile_ea(args: dict) -> dict: - env_error = _validate_environment() - if env_error: - return env_error - - expert_path = args['expert_path'] - cmd = [str(SCRIPTS_DIR / 'mqlcompile.sh'), expert_path] - success, output = run_script(cmd, timeout=120) - - return { - 'success': success, - 'output': output, - 'expert_path': expert_path, - } - - -def _read_terminal_ini() -> dict: - """Parse terminal.ini (UTF-16LE or UTF-8) into a flat key→value dict.""" - terminal_dir = cfg('terminal_dir') - if not terminal_dir: - return {} - ini_path = Path(terminal_dir) / 'config' / 'terminal.ini' - if not ini_path.exists(): - return {} - try: - raw = ini_path.read_bytes() - text = raw.decode('utf-16') if raw[:2] in (b'\xff\xfe', b'\xfe\xff') else raw.decode('utf-8', errors='replace') - except Exception: - return {} - result: dict = {} - for line in text.splitlines(): - line = line.strip() - if '=' in line and not line.startswith(';') and not line.startswith('['): - k, _, v = line.partition('=') - result[k.strip()] = v.strip() - return result - - -async def handle_list_symbols(args: dict) -> dict: - env_error = _validate_environment() - if env_error: - return env_error - - terminal_dir = cfg('terminal_dir') - bases_dir = Path(terminal_dir) / 'Bases' - - if not bases_dir.is_dir(): - return {'success': False, 'error': f'Bases directory not found: {bases_dir}'} - - # Detect active server from terminal.ini - ini = _read_terminal_ini() - active_server = ini.get('LastScanServer', '') - opt_mode = ini.get('OptMode', '0') - - # Collect all servers and their symbols - filter_server = args.get('server', '').lower() - servers: list[dict] = [] - for server_dir in sorted(bases_dir.iterdir()): - if not server_dir.is_dir(): - continue - name = server_dir.name - if filter_server and filter_server not in name.lower(): - continue - history_dir = server_dir / 'history' - symbols = sorted(d.name for d in history_dir.iterdir() if d.is_dir()) if history_dir.is_dir() else [] - servers.append({ - 'server': name, - 'active': name == active_server, - 'symbol_count': len(symbols), - 'symbols': symbols, - }) - - # Put active server first - servers.sort(key=lambda s: (0 if s['active'] else 1, s['server'])) - - warnings = [] - if opt_mode == '-1': - warnings.append('OptMode=-1 detected in terminal.ini — the CLEAN stage will reset this before backtest.') - - return { - 'success': True, - 'active_server': active_server or '(unknown — open MT5 to connect)', - 'servers': servers, - 'warnings': warnings, - 'hint': 'Use the symbol name exactly as shown (e.g. "XAUUSD.cent" not "XAUUSD") in run_backtest.', - } - - -async def handle_list_experts(args: dict) -> dict: - env_error = _validate_environment() - if env_error: - return env_error - - terminal_dir = cfg('terminal_dir') - experts_root = Path(cfg('experts_dir') or os.path.join(terminal_dir, 'MQL5', 'Experts')) - - if not experts_root.is_dir(): - return {'success': False, 'error': f'Experts directory not found: {experts_root}'} - - name_filter = args.get('filter', '').lower() - experts: list[dict] = [] - - for ex5 in sorted(experts_root.rglob('*.ex5')): - rel = ex5.relative_to(experts_root) - expert_name = ex5.stem # filename without .ex5 - subfolder = str(rel.parent) if rel.parent != Path('.') else '' - if name_filter and name_filter not in expert_name.lower(): - continue - experts.append({ - 'name': expert_name, - 'subfolder': subfolder, - 'run_backtest_expert': f'{subfolder}/{expert_name}' if subfolder else expert_name, - 'path': str(ex5), - }) - - return { - 'success': True, - 'count': len(experts), - 'experts_root': str(experts_root), - 'experts': experts, - 'hint': 'Use the "run_backtest_expert" value as the expert parameter in run_backtest.', - } - - -async def handle_verify_setup(args: dict) -> dict: - checks: dict = {} - all_ok = True - - # Config file - config_path = ROOT_DIR / 'config' / 'mt5-quant.yaml' - checks['config_file'] = { - 'ok': config_path.exists(), - 'detail': str(config_path) if config_path.exists() else 'Not found — run: bash scripts/setup.sh', - } - if not config_path.exists(): - all_ok = False - - # Wine executable - wine = cfg('wine_executable') - if not wine: - checks['wine_executable'] = {'ok': False, 'detail': 'Not configured in mt5-quant.yaml'} - all_ok = False - else: - executable = os.access(wine, os.X_OK) - version = '' - if executable: - try: - r = subprocess.run([wine, '--version'], capture_output=True, text=True, timeout=5) - version = ((r.stdout or '') + (r.stderr or '')).strip().splitlines()[0] - except Exception as e: - version = f'error: {e}' - checks['wine_executable'] = { - 'ok': executable, - 'version': version, - 'detail': wine if executable else f'Not executable: {wine}', - } - if not executable: - all_ok = False - - # terminal_dir and derived paths - terminal_dir = cfg('terminal_dir') - if not terminal_dir: - checks['terminal_dir'] = {'ok': False, 'detail': 'Not configured in mt5-quant.yaml'} - all_ok = False - else: - td_ok = Path(terminal_dir).is_dir() - checks['terminal_dir'] = { - 'ok': td_ok, - 'detail': terminal_dir if td_ok else f'Directory not found: {terminal_dir}', - } - if not td_ok: - all_ok = False - - terminal_exe = Path(terminal_dir) / 'terminal64.exe' - checks['terminal64_exe'] = { - 'ok': terminal_exe.exists(), - 'detail': str(terminal_exe) if terminal_exe.exists() - else 'Not found — launch MT5 once to unpack it', - } - - experts_dir = Path(cfg('experts_dir') or os.path.join(terminal_dir, 'MQL5', 'Experts')) - ea_count = len(list(experts_dir.glob('*.ex5'))) if experts_dir.is_dir() else 0 - checks['experts_dir'] = { - 'ok': experts_dir.is_dir(), - 'detail': f'{ea_count} .ex5 file(s)' if experts_dir.is_dir() - else f'Not found (will be created on first EA compile): {experts_dir}', - } - - tester_dir = Path(cfg('tester_profiles_dir') or os.path.join(terminal_dir, 'MQL5', 'Profiles', 'Tester')) - set_count = len(list(tester_dir.glob('*.set'))) if tester_dir.is_dir() else 0 - checks['tester_profiles_dir'] = { - 'ok': tester_dir.is_dir(), - 'detail': f'{set_count} .set file(s)' if tester_dir.is_dir() - else f'Not found (will be created on first backtest): {tester_dir}', - } - - cache_dir = Path(cfg('tester_cache_dir') or os.path.join(terminal_dir, 'Tester')) - checks['tester_cache_dir'] = { - 'ok': cache_dir.is_dir(), - 'detail': str(cache_dir) if cache_dir.is_dir() else f'Not found: {cache_dir}', - } - - return { - 'all_ok': all_ok, - 'checks': checks, - 'hint': 'Run: bash scripts/setup.sh' if not all_ok else 'Environment looks good.', - } - - -async def handle_get_backtest_status(args: dict) -> dict: - report_dir = args.get('report_dir') or latest_report_dir() - if not report_dir: - return {'success': False, 'error': 'No report directory found'} - - progress_log = Path(report_dir) / 'progress.log' - pipeline_meta = Path(report_dir) / 'pipeline_metadata.json' - - stages = [] - if progress_log.exists(): - for line in progress_log.read_text().splitlines(): - parts = line.split() - if len(parts) >= 3: - stages.append({'stage': parts[0], 'timestamp': parts[1], 'elapsed': parts[2]}) - - current_stage = stages[-1]['stage'] if stages else 'UNKNOWN' - finished = pipeline_meta.exists() or current_stage == 'DONE' - elapsed = None - if stages: - try: - elapsed = int(stages[-1]['elapsed'].replace('elapsed=', '').rstrip('s')) - except (ValueError, AttributeError): - pass - - return { - 'success': True, - 'report_dir': report_dir, - 'current_stage': current_stage, - 'elapsed_seconds': elapsed, - 'finished': finished, - 'stages': stages, - } - - -async def handle_get_optimization_status(args: dict) -> dict: - job_id = args['job_id'] - meta_path = ROOT_DIR / '.mt5mcp_jobs' / f'{job_id}.json' - - if not meta_path.exists(): - return {'success': False, 'error': f'Job not found: {job_id}. Check .mt5mcp_jobs/'} - - with open(meta_path) as f: - meta = json.load(f) - - pid = meta.get('pid') - log_file = meta.get('log_file', '') - wine_prefix = meta.get('wine_prefix', '') - started_at = meta.get('started_at', '') - - # Check process alive via kill -0 - alive = False - if pid: - try: - os.kill(int(pid), 0) - alive = True - except (OSError, ProcessLookupError): - alive = False - - # Report file existence = definitive completion signal - report_found = False - report_path = None - if wine_prefix: - base = os.path.join(wine_prefix, 'drive_c', 'mt5mcp_opt_report') - for ext in ('.htm', '.htm.xml', '.html'): - candidate = base + ext - if os.path.exists(candidate): - report_found = True - report_path = candidate - break - - # Tail log - log_tail: list[str] = [] - if log_file and os.path.exists(log_file): - try: - log_tail = Path(log_file).read_text(errors='replace').splitlines()[-20:] - except Exception: - pass - - # Elapsed time - elapsed_seconds = None - if started_at: - try: - from datetime import datetime, timezone - start_dt = datetime.fromisoformat(started_at.replace('Z', '+00:00')) - elapsed_seconds = int((datetime.now(timezone.utc) - start_dt).total_seconds()) - except Exception: - pass - - if report_found: - hint = f'Optimization complete. Call get_optimization_results with job_id="{job_id}".' - elif alive: - hint = f'Still running. Monitor: tail -f {log_file}' - else: - hint = f'Process not running and no report found. Check log: {log_file}' - - return { - 'success': True, - 'job_id': job_id, - 'alive': alive, - 'finished': report_found, - 'elapsed_seconds': elapsed_seconds, - 'report_found': report_found, - 'report_path': report_path, - 'log_file': log_file, - 'log_tail': log_tail, - 'hint': hint, - } - - -async def handle_prune_reports(args: dict) -> dict: - keep_last = int(args.get('keep_last') or cfg('keep_last', '20') or 20) - REPORTS_DIR.mkdir(exist_ok=True) - - all_dirs = sorted( - [d for d in REPORTS_DIR.iterdir() if d.is_dir() and not d.name.endswith('_opt')], - key=lambda d: d.stat().st_mtime, - ) - to_delete = all_dirs[:-keep_last] if len(all_dirs) > keep_last else [] - kept = all_dirs[-keep_last:] if len(all_dirs) > keep_last else all_dirs - - deleted_names = [] - for d in to_delete: - try: - shutil.rmtree(str(d)) - deleted_names.append(d.name) - except Exception: - pass - - return { - 'success': True, - 'deleted_count': len(deleted_names), - 'kept_count': len(kept), - 'deleted_dirs': deleted_names, - 'kept_dirs': [d.name for d in kept], - } - - -async def handle_list_reports(args: dict) -> dict: - REPORTS_DIR.mkdir(exist_ok=True) - include_opt = args.get('include_opt', False) - limit = int(args.get('limit') or 30) - - dirs = sorted( - [d for d in REPORTS_DIR.iterdir() if d.is_dir()], - key=lambda d: d.stat().st_mtime, - reverse=True, - ) - if not include_opt: - dirs = [d for d in dirs if not d.name.endswith('_opt')] - dirs = dirs[:limit] - - rows = [] - for d in dirs: - m = read_json(str(d / 'metrics.json')) - row: dict = {'name': d.name, 'is_opt': d.name.endswith('_opt')} - if m: - row['net_profit'] = m.get('net_profit') - row['max_dd_pct'] = m.get('max_dd_pct') - row['total_trades'] = m.get('total_trades') - row['symbol'] = m.get('symbol') - row['timeframe'] = m.get('timeframe') - row['from_date'] = m.get('from_date') or m.get('testing_from') - row['to_date'] = m.get('to_date') or m.get('testing_to') - else: - row['metrics'] = 'missing' - rows.append(row) - - return {'success': True, 'count': len(rows), 'reports': rows} - - -async def handle_tail_log(args: dict) -> dict: - n = int(args.get('n') or 50) - filt = args.get('filter', 'all') - - log_path: str | None = args.get('log_file') - - if not log_path and 'job_id' in args: - job_id = args['job_id'] - meta_path = ROOT_DIR / '.mt5mcp_jobs' / f'{job_id}.json' - if not meta_path.exists(): - return {'success': False, 'error': f'Job not found: {job_id}'} - with open(meta_path) as f: - meta = json.load(f) - log_path = meta.get('log_file', '') - - if not log_path: - report_dir = args.get('report_dir') or latest_report_dir() - if report_dir: - log_path = str(Path(report_dir) / 'progress.log') - - if not log_path or not os.path.exists(log_path): - return {'success': False, 'error': f'Log file not found: {log_path}'} - - try: - lines = Path(log_path).read_text(errors='replace').splitlines() - except Exception as e: - return {'success': False, 'error': str(e)} - - if filt == 'errors': - lines = [l for l in lines if 'error' in l.lower() or 'fail' in l.lower() or 'err:' in l.lower()] - elif filt == 'warnings': - lines = [l for l in lines if 'warn' in l.lower() or 'error' in l.lower()] - - return { - 'success': True, - 'log_file': log_path, - 'total_lines': len(lines), - 'lines': lines[-n:], - } - - -def _dir_size(path: Path) -> int: - return sum(f.stat().st_size for f in path.rglob('*') if f.is_file()) - - -async def handle_cache_status(args: dict) -> dict: - terminal_dir = cfg('terminal_dir') - if not terminal_dir: - return {'success': False, 'error': 'terminal_dir not configured'} - - cache_dir = Path(cfg('tester_cache_dir') or os.path.join(terminal_dir, 'Tester')) - if not cache_dir.is_dir(): - return {'success': False, 'error': f'Cache dir not found: {cache_dir}'} - - total_bytes = 0 - breakdown: list[dict] = [] - - for item in sorted(cache_dir.iterdir()): - if item.is_dir(): - sz = _dir_size(item) - total_bytes += sz - breakdown.append({'symbol': item.name, 'size_mb': round(sz / 1024 / 1024, 2)}) - elif item.is_file(): - sz = item.stat().st_size - total_bytes += sz - - return { - 'success': True, - 'cache_dir': str(cache_dir), - 'total_size_mb': round(total_bytes / 1024 / 1024, 2), - 'symbols': breakdown, - } - - -async def handle_clean_cache(args: dict) -> dict: - terminal_dir = cfg('terminal_dir') - if not terminal_dir: - return {'success': False, 'error': 'terminal_dir not configured'} - - cache_dir = Path(cfg('tester_cache_dir') or os.path.join(terminal_dir, 'Tester')) - if not cache_dir.is_dir(): - return {'success': False, 'error': f'Cache dir not found: {cache_dir}'} - - symbol = args.get('symbol', '').strip() - dry_run = bool(args.get('dry_run', False)) - - targets: list[Path] = [] - if symbol: - target = cache_dir / symbol - if target.is_dir(): - targets.append(target) - else: - return {'success': False, 'error': f'No cache found for symbol: {symbol}'} - else: - targets = [d for d in cache_dir.iterdir() if d.is_dir()] - - freed_bytes = sum(_dir_size(t) for t in targets) - names = [t.name for t in targets] - - if not dry_run: - for t in targets: - shutil.rmtree(str(t)) - - return { - 'success': True, - 'dry_run': dry_run, - 'deleted_symbols': names, - 'freed_mb': round(freed_bytes / 1024 / 1024, 2), - 'hint': 'Next backtest will regenerate tick data (slower first run).', - } - - -# ── .set file helpers ───────────────────────────────────────────────────────── - -def _parse_set_line(line: str) -> tuple[str, dict] | None: - """Parse one .set file line → (name, param_dict) or None.""" - line = line.strip() - if not line or line.startswith(';') or '=' not in line: - return None - name, _, raw = line.partition('=') - name = name.strip() - parts = raw.split('||') - value = parts[0].strip() - param: dict = {'value': value} - if len(parts) >= 4: - param['from'] = parts[1].strip() - param['to'] = parts[2].strip() - param['step'] = parts[3].strip() if len(parts) > 3 else '' - param['optimize'] = parts[4].strip() == 'Y' if len(parts) > 4 else False - return name, param - - -def _decode_set(path: str) -> tuple[dict, list[str]]: - """Load a .set file → (params, comments). Raises ValueError on decode failure.""" - content = None - raw = Path(path).read_bytes() - for enc in ('utf-16-le', 'utf-16', 'utf-8-sig', 'utf-8'): - try: - if enc in ('utf-16-le', 'utf-16') and raw[:2] in (b'\xff\xfe', b'\xfe\xff'): - content = raw.decode('utf-16') - else: - content = raw.decode(enc) - break - except (UnicodeDecodeError, LookupError): - continue - if content is None: - raise ValueError(f'Cannot decode {path} — unknown encoding') - - params: dict = {} - comments: list[str] = [] - for line in content.splitlines(): - if line.strip().startswith(';'): - comments.append(line.strip().lstrip(';').strip()) - continue - result = _parse_set_line(line) - if result: - name, param = result - params[name] = param - return params, comments - - -def _encode_set(params: dict, comments: list[str] | None = None) -> bytes: - """Serialize params (and optional header comments) to UTF-16LE bytes.""" - lines: list[str] = [] - if comments: - for c in comments: - lines.append(f'; {c}') - for name, spec in params.items(): - if isinstance(spec, dict): - value = str(spec.get('value', '')) - if 'from' in spec: - flag = 'Y' if spec.get('optimize', False) else 'N' - lines.append(f"{name}={value}||{spec['from']}||{spec.get('to', value)}||{spec.get('step', '1')}||{flag}") - else: - lines.append(f"{name}={value}") - else: - lines.append(f"{name}={spec}") - return ('\r\n'.join(lines) + '\r\n').encode('utf-16-le') - - -def _write_set(path: str, data: bytes) -> None: - """Write bytes to path and apply chmod 444 (required by MT5).""" - p = Path(path) - p.parent.mkdir(parents=True, exist_ok=True) - # chmod 644 first in case file already exists as 444 - if p.exists(): - os.chmod(path, 0o644) - p.write_bytes(data) - os.chmod(path, 0o444) - - -def _sweep_combinations(params: dict) -> tuple[list[dict], int]: - """Return (swept_param_details, total_combinations) for a parsed params dict.""" - import math - swept = [] - total = 1 - for name, spec in params.items(): - if not isinstance(spec, dict) or not spec.get('optimize'): - continue - try: - f = float(spec['from']) - t = float(spec['to']) - s = float(spec['step']) - count = max(1, math.floor(abs(t - f) / s) + 1) if s else 1 - except (KeyError, ValueError, ZeroDivisionError): - count = 1 - swept.append({ - 'name': name, - 'from': spec.get('from'), - 'to': spec.get('to'), - 'step': spec.get('step'), - 'count': count, - }) - total *= count - return swept, total - - -async def handle_read_set_file(args: dict) -> dict: - path = args['path'] - if not os.path.exists(path): - return {'success': False, 'error': f'File not found: {path}'} - try: - params, comments = _decode_set(path) - except Exception as e: - return {'success': False, 'error': str(e)} - return { - 'success': True, - 'path': path, - 'param_count': len(params), - 'comments': comments, - 'params': params, - } - - -async def handle_write_set_file(args: dict) -> dict: - path = args['path'] - params: dict = args['params'] - try: - _write_set(path, _encode_set(params)) - except Exception as e: - return {'success': False, 'error': str(e)} - return { - 'success': True, - 'path': path, - 'param_count': len(params), - 'encoding': 'utf-16-le', - 'permissions': '444 (read-only, required by MT5)', - } - - -async def handle_patch_set_file(args: dict) -> dict: - path = args['path'] - patches: dict = args['patches'] - if not os.path.exists(path): - return {'success': False, 'error': f'File not found: {path}'} - try: - params, comments = _decode_set(path) - except Exception as e: - return {'success': False, 'error': str(e)} - - changed: list[dict] = [] - for name, new_spec in patches.items(): - old = params.get(name, {}) - old_value = old.get('value') if isinstance(old, dict) else str(old) - if isinstance(new_spec, dict): - # Merge: keep existing sweep config unless overridden - merged = dict(old) if isinstance(old, dict) else {'value': old_value} - merged.update(new_spec) - params[name] = merged - new_value = str(merged.get('value', '')) - else: - new_value = str(new_spec) - if isinstance(params.get(name), dict): - params[name] = dict(params[name]) - params[name]['value'] = new_value - else: - params[name] = {'value': new_value} - if old_value != new_value: - changed.append({'name': name, 'old': old_value, 'new': new_value}) - - try: - _write_set(path, _encode_set(params, comments)) - except Exception as e: - return {'success': False, 'error': str(e)} - - return { - 'success': True, - 'path': path, - 'changed_count': len(changed), - 'changed': changed, - 'param_count': len(params), - } - - -async def handle_clone_set_file(args: dict) -> dict: - source = args['source'] - destination = args['destination'] - overrides: dict = args.get('overrides', {}) - if not os.path.exists(source): - return {'success': False, 'error': f'Source not found: {source}'} - try: - params, comments = _decode_set(source) - except Exception as e: - return {'success': False, 'error': str(e)} - - changed: list[dict] = [] - for name, new_spec in overrides.items(): - old = params.get(name, {}) - old_value = old.get('value') if isinstance(old, dict) else str(old) if old else None - if isinstance(new_spec, dict): - merged = dict(old) if isinstance(old, dict) else {} - merged.update(new_spec) - params[name] = merged - new_value = str(merged.get('value', '')) - else: - new_value = str(new_spec) - if isinstance(params.get(name), dict): - params[name] = dict(params[name]) - params[name]['value'] = new_value - else: - params[name] = {'value': new_value} - if old_value != new_value: - changed.append({'name': name, 'old': old_value, 'new': new_value}) - - try: - _write_set(destination, _encode_set(params, comments)) - except Exception as e: - return {'success': False, 'error': str(e)} - - return { - 'success': True, - 'source': source, - 'destination': destination, - 'param_count': len(params), - 'overridden_count': len(changed), - 'overridden': changed, - } - - -async def handle_set_from_optimization(args: dict) -> dict: - path = args['path'] - opt_params: dict = args['params'] # {name: value} from optimization result - template: str | None = args.get('template') - sweep: dict = args.get('sweep', {}) # {name: {from, to, step}} to add sweep flags - - base_params: dict = {} - base_comments: list[str] = [] - - if template: - if not os.path.exists(template): - return {'success': False, 'error': f'Template not found: {template}'} - try: - base_params, base_comments = _decode_set(template) - except Exception as e: - return {'success': False, 'error': str(e)} - - # Start from template (or empty), apply opt values, strip all sweep flags - merged: dict = {} - for name, spec in base_params.items(): - # Copy as fixed value (no sweep) - value = spec.get('value') if isinstance(spec, dict) else str(spec) - merged[name] = {'value': value} - - # Apply optimization result values (overwrite template values, add new params) - for name, value in opt_params.items(): - merged[name] = {'value': str(value)} - - # Optionally re-add sweep ranges for a subset of params - for name, sweep_spec in sweep.items(): - if name in merged: - merged[name].update({ - 'from': str(sweep_spec.get('from', '')), - 'to': str(sweep_spec.get('to', '')), - 'step': str(sweep_spec.get('step', '1')), - 'optimize': bool(sweep_spec.get('optimize', True)), - }) - - try: - _write_set(path, _encode_set(merged, base_comments)) - except Exception as e: - return {'success': False, 'error': str(e)} - - swept, total = _sweep_combinations(merged) - return { - 'success': True, - 'path': path, - 'param_count': len(merged), - 'from_template': bool(template), - 'opt_params_applied': len(opt_params), - 'swept_params': len(swept), - 'total_combinations': total if swept else 0, - } - - -async def handle_diff_set_files(args: dict) -> dict: - path_a = args['path_a'] - path_b = args['path_b'] - - for p in (path_a, path_b): - if not os.path.exists(p): - return {'success': False, 'error': f'File not found: {p}'} - try: - params_a, _ = _decode_set(path_a) - params_b, _ = _decode_set(path_b) - except Exception as e: - return {'success': False, 'error': str(e)} - - keys_a = set(params_a) - keys_b = set(params_b) - - added = [] - for k in sorted(keys_b - keys_a): - spec = params_b[k] - added.append({'name': k, 'value': spec.get('value') if isinstance(spec, dict) else str(spec)}) - - removed = [] - for k in sorted(keys_a - keys_b): - spec = params_a[k] - removed.append({'name': k, 'value': spec.get('value') if isinstance(spec, dict) else str(spec)}) - - changed = [] - for k in sorted(keys_a & keys_b): - sa = params_a[k] - sb = params_b[k] - va = sa.get('value') if isinstance(sa, dict) else str(sa) - vb = sb.get('value') if isinstance(sb, dict) else str(sb) - opt_a = sa.get('optimize', False) if isinstance(sa, dict) else False - opt_b = sb.get('optimize', False) if isinstance(sb, dict) else False - if va != vb or opt_a != opt_b: - entry: dict = {'name': k, 'a': va, 'b': vb} - if opt_a != opt_b: - entry['sweep_a'] = opt_a - entry['sweep_b'] = opt_b - changed.append(entry) - - identical = not added and not removed and not changed - return { - 'success': True, - 'path_a': path_a, - 'path_b': path_b, - 'identical': identical, - 'added_count': len(added), - 'removed_count': len(removed), - 'changed_count': len(changed), - 'added': added, - 'removed': removed, - 'changed': changed, - } - - -async def handle_describe_sweep(args: dict) -> dict: - path = args['path'] - if not os.path.exists(path): - return {'success': False, 'error': f'File not found: {path}'} - try: - params, comments = _decode_set(path) - except Exception as e: - return {'success': False, 'error': str(e)} - - swept, total = _sweep_combinations(params) - fixed_count = len(params) - len(swept) - - return { - 'success': True, - 'path': path, - 'total_params': len(params), - 'swept_count': len(swept), - 'fixed_count': fixed_count, - 'total_combinations': total, - 'swept_params': swept, - 'hint': ( - 'No swept params — this is a backtest .set, not an optimization .set.' - if not swept else - f'{total:,} combinations. Typical range: 1–8h depending on EA tick speed.' - ), - } - - -async def handle_list_set_files(args: dict) -> dict: - terminal_dir = cfg('terminal_dir') - if not terminal_dir: - return {'success': False, 'error': 'terminal_dir not configured'} - - profiles_dir = Path(cfg('tester_profiles_dir') or - os.path.join(terminal_dir, 'MQL5', 'Profiles', 'Tester')) - if not profiles_dir.is_dir(): - return {'success': False, 'error': f'Tester profiles dir not found: {profiles_dir}'} - - ea_filter = args.get('ea', '').lower() - rows: list[dict] = [] - - for f in sorted(profiles_dir.glob('*.set'), key=lambda x: x.stat().st_mtime, reverse=True): - if ea_filter and ea_filter not in f.stem.lower(): - continue - try: - params, _ = _decode_set(str(f)) - swept, total = _sweep_combinations(params) - rows.append({ - 'name': f.name, - 'param_count': len(params), - 'swept_count': len(swept), - 'total_combinations': total if swept else 0, - 'modified': f.stat().st_mtime, - }) - except Exception: - rows.append({'name': f.name, 'error': 'unreadable'}) - - # Convert mtime to ISO for readability - from datetime import datetime - for r in rows: - if 'modified' in r: - r['modified'] = datetime.fromtimestamp(r['modified']).strftime('%Y-%m-%d %H:%M') - - return { - 'success': True, - 'profiles_dir': str(profiles_dir), - 'count': len(rows), - 'files': rows, - } - - -async def handle_archive_report(args: dict) -> dict: - report_dir = args.get('report_dir') or latest_report_dir() - if not report_dir: - return {'success': False, 'error': 'No report directory found'} - - entry = _build_history_entry(report_dir) - if not entry: - return {'success': False, 'error': f'No metrics.json or analysis.json in {report_dir}'} - - if args.get('verdict'): - entry['verdict'] = args['verdict'] - if args.get('notes'): - entry['notes'] = args['notes'] - if args.get('tags'): - entry['tags'] = args['tags'] - - history = load_history() - existing_ids = {e['id'] for e in history} - already_exists = entry['id'] in existing_ids - - if not already_exists: - history.append(entry) - save_history(history) - - deleted = False - if args.get('delete_after') and not already_exists: - try: - shutil.rmtree(report_dir) - deleted = True - # Update the entry in history to reflect deletion - for e in history: - if e['id'] == entry['id']: - e['report_dir_deleted'] = True - break - save_history(history) - except Exception as exc: - return {'success': False, 'error': f'Archive succeeded but delete failed: {exc}'} - - return { - 'success': True, - 'id': entry['id'], - 'already_existed': already_exists, - 'deleted_source': deleted, - 'history_file': str(HISTORY_FILE), - 'entry_summary': { - 'ea': entry['ea'], - 'symbol': entry['symbol'], - 'metrics': entry['metrics'], - 'verdict': entry['verdict'], - }, - } - - -async def handle_archive_all_reports(args: dict) -> dict: - REPORTS_DIR.mkdir(exist_ok=True) - delete_after = bool(args.get('delete_after', False)) - keep_last = int(args.get('keep_last', 5)) - dry_run = bool(args.get('dry_run', False)) - - all_dirs = sorted( - [d for d in REPORTS_DIR.iterdir() if d.is_dir() and not d.name.endswith('_opt')], - key=lambda d: d.stat().st_mtime, - ) - - history = load_history() - existing_ids = {e['id'] for e in history} - - # Dirs protected from deletion regardless of keep_last - protected = {d.name for d in all_dirs[-keep_last:]} if keep_last > 0 else set() - - results = {'archived': [], 'skipped': [], 'deleted': [], 'failed': []} - - for d in all_dirs: - if d.name in existing_ids: - results['skipped'].append(d.name) - continue - - entry = _build_history_entry(str(d)) - if not entry: - results['failed'].append(d.name) - continue - - if not dry_run: - history.append(entry) - results['archived'].append(d.name) - - should_delete = delete_after and d.name not in protected - if should_delete and not dry_run: - try: - shutil.rmtree(str(d)) - entry['report_dir_deleted'] = True - results['deleted'].append(d.name) - except Exception: - results['failed'].append(d.name) - - if not dry_run and results['archived']: - save_history(history) - - return { - 'success': True, - 'dry_run': dry_run, - 'archived_count': len(results['archived']), - 'skipped_count': len(results['skipped']), - 'deleted_count': len(results['deleted']), - 'failed_count': len(results['failed']), - 'history_file': str(HISTORY_FILE), - **results, - } - - -async def handle_get_history(args: dict) -> dict: - history = load_history() - if not history: - return {'success': True, 'count': 0, 'entries': []} - - ea_filter = args.get('ea', '').lower() - symbol_filter = args.get('symbol', '').upper() - verdict_filter = args.get('verdict') - tag_filter = args.get('tag', '') - min_profit = args.get('min_profit') - max_dd = args.get('max_dd_pct') - sort_by = args.get('sort_by', 'date') - limit = int(args.get('limit') or 20) - include_monthly = bool(args.get('include_monthly', False)) - - filtered = [] - for e in history: - if ea_filter and ea_filter not in e.get('ea', '').lower(): - continue - if symbol_filter and e.get('symbol', '').upper() != symbol_filter: - continue - if verdict_filter and e.get('verdict') != verdict_filter: - continue - if tag_filter and tag_filter not in e.get('tags', []): - continue - m = e.get('metrics', {}) - if min_profit is not None and (m.get('net_profit') or 0) < min_profit: - continue - if max_dd is not None and (m.get('max_dd_pct') or 999) > max_dd: - continue - filtered.append(e) - - key_map = { - 'date': lambda e: e.get('archived_at', ''), - 'profit': lambda e: (e.get('metrics') or {}).get('net_profit') or 0, - 'dd': lambda e: (e.get('metrics') or {}).get('max_dd_pct') or 999, - 'sharpe': lambda e: (e.get('metrics') or {}).get('sharpe_ratio') or 0, - } - reverse = sort_by != 'dd' - filtered.sort(key=key_map.get(sort_by, key_map['date']), reverse=reverse) - filtered = filtered[:limit] - - if not include_monthly: - for e in filtered: - e.pop('monthly_pnl', None) - - return {'success': True, 'count': len(filtered), 'entries': filtered} - - -async def handle_promote_to_baseline(args: dict) -> dict: - from datetime import datetime, timezone - - # Resolve source: history entry, explicit report_dir, or latest - entry: dict | None = None - report_dir: str | None = None - - if 'history_id' in args: - history = load_history() - matches = [e for e in history if e['id'] == args['history_id']] - if not matches: - return {'success': False, 'error': f"History entry not found: {args['history_id']}"} - entry = matches[0] - report_dir = entry.get('report_dir') if not entry.get('report_dir_deleted') else None - else: - report_dir = args.get('report_dir') or latest_report_dir() - if not report_dir: - return {'success': False, 'error': 'No report directory found'} - - # Load metrics — prefer live report dir, fall back to history entry - if report_dir and Path(report_dir).is_dir(): - metrics = read_json(os.path.join(report_dir, 'metrics.json')) - elif entry: - metrics = entry.get('metrics', {}) - else: - return {'success': False, 'error': 'Source not found (report dir missing and no history entry)'} - - if not metrics: - return {'success': False, 'error': 'No metrics found in source'} - - now = datetime.now(timezone.utc).strftime('%Y-%m-%d') - ea = (entry or {}).get('ea') or metrics.get('expert') or metrics.get('ea') or '' - symbol = (entry or {}).get('symbol') or metrics.get('symbol') or '' - from_date = (entry or {}).get('from_date') or '' - to_date = (entry or {}).get('to_date') or '' - period = f"{from_date}/{to_date}" if from_date and to_date else '' - - baseline = { - 'ea': ea, - 'symbol': symbol, - 'period': period, - 'net_profit': metrics.get('net_profit'), - 'profit_factor': metrics.get('profit_factor'), - 'max_drawdown_pct': metrics.get('max_dd_pct'), - 'sharpe_ratio': metrics.get('sharpe_ratio'), - 'total_trades': metrics.get('total_trades'), - 'recovery_factor': metrics.get('recovery_factor'), - 'promoted_from': (entry or {}).get('id') or Path(report_dir or '').name, - 'promoted_at': now, - 'notes': args.get('notes', f'Promoted {now}'), - } - - BASELINE_FILE.parent.mkdir(exist_ok=True) - with open(BASELINE_FILE, 'w') as f: - json.dump(baseline, f, indent=2) - - # Mark in history - if entry: - history = load_history() - for e in history: - if e['id'] == entry['id']: - e['promoted_to_baseline'] = True - e['verdict'] = e.get('verdict') or 'reference' - break - save_history(history) - - return { - 'success': True, - 'baseline_file': str(BASELINE_FILE), - 'baseline': baseline, - } - - -async def handle_annotate_history(args: dict) -> dict: - history_id = args['history_id'] - history = load_history() - - target = next((e for e in history if e['id'] == history_id), None) - if not target: - return {'success': False, 'error': f'Entry not found: {history_id}'} - - if 'verdict' in args: - target['verdict'] = args['verdict'] - if 'notes' in args: - target['notes'] = args['notes'] - if 'tags' in args: - target['tags'] = args['tags'] - if 'add_tags' in args: - existing = target.get('tags') or [] - for t in args['add_tags']: - if t not in existing: - existing.append(t) - target['tags'] = existing - - save_history(history) - - return { - 'success': True, - 'id': history_id, - 'verdict': target.get('verdict'), - 'notes': target.get('notes'), - 'tags': target.get('tags'), - } - - -async def handle_list_jobs(args: dict) -> dict: - jobs_dir = ROOT_DIR / '.mt5mcp_jobs' - if not jobs_dir.is_dir(): - return {'success': True, 'jobs': [], 'count': 0} - - include_done = args.get('include_done', True) - rows: list[dict] = [] - - from datetime import datetime, timezone - - for meta_file in sorted(jobs_dir.glob('*.json'), reverse=True): - try: - with open(meta_file) as f: - meta = json.load(f) - except Exception: - continue - - job_id = meta_file.stem - pid = meta.get('pid') - started_at = meta.get('started_at', '') - log_file = meta.get('log_file', '') - wine_prefix = meta.get('wine_prefix', '') - - alive = False - if pid: - try: - os.kill(int(pid), 0) - alive = True - except (OSError, ProcessLookupError): - pass - - report_found = False - if wine_prefix: - base = os.path.join(wine_prefix, 'drive_c', 'mt5mcp_opt_report') - for ext in ('.htm', '.htm.xml', '.html'): - if os.path.exists(base + ext): - report_found = True - break - - status = 'running' if alive else ('done' if report_found else 'failed') - - elapsed_seconds = None - if started_at: - try: - start_dt = datetime.fromisoformat(started_at.replace('Z', '+00:00')) - elapsed_seconds = int((datetime.now(timezone.utc) - start_dt).total_seconds()) - except Exception: - pass - - if not include_done and status != 'running': - continue - - rows.append({ - 'job_id': job_id, - 'status': status, - 'elapsed_seconds': elapsed_seconds, - 'expert': meta.get('expert', ''), - 'started_at': started_at, - 'log_file': log_file, - }) - - return {'success': True, 'count': len(rows), 'jobs': rows} - - -# ── Entry point ─────────────────────────────────────────────────────────────── - -async def main(): - async with mcp.server.stdio.stdio_server() as (read_stream, write_stream): - await app.run( - read_stream, - write_stream, - app.create_initialization_options(), - ) - - -def cli(): - """Sync entry point for [project.scripts] — pyproject.toml requires a sync callable.""" - asyncio.run(main()) - - -if __name__ == '__main__': - asyncio.run(main()) diff --git a/src/analytics/analyze.rs b/src/analytics/analyze.rs index d44f859..633f4e1 100644 --- a/src/analytics/analyze.rs +++ b/src/analytics/analyze.rs @@ -1,4 +1,4 @@ -use chrono::{DateTime, Datelike, NaiveDateTime}; +use chrono::{DateTime, NaiveDateTime}; use serde::{Deserialize, Serialize}; use std::collections::HashMap; @@ -392,7 +392,7 @@ impl DealAnalyzer { let mut peak = 0; let mut peak_time = String::new(); - for (dt, delta, deal) in events { + for (_dt, delta, deal) in events { count = (count + delta).max(0); if count > peak { peak = count; diff --git a/src/analytics/extract.rs b/src/analytics/extract.rs index 49bd5ef..2172023 100644 --- a/src/analytics/extract.rs +++ b/src/analytics/extract.rs @@ -277,8 +277,11 @@ impl ReportExtractor { pub struct ExtractionResult { pub metrics: Metrics, pub deals: Vec, + #[allow(dead_code)] pub metrics_path: PathBuf, + #[allow(dead_code)] pub deals_csv_path: PathBuf, + #[allow(dead_code)] pub deals_json_path: PathBuf, } diff --git a/src/compile/mql_compiler.rs b/src/compile/mql_compiler.rs index 2e5b13e..35553a2 100644 --- a/src/compile/mql_compiler.rs +++ b/src/compile/mql_compiler.rs @@ -58,7 +58,7 @@ impl MqlCompiler { let wine_src_path = Self::host_to_wine_path(&dest_path)?; let wine_log_path = Self::host_to_wine_path(&log_file)?; - let output = Command::new(wine_exe) + let _output = Command::new(wine_exe) .arg(&metaeditor) .arg(format!("/compile:{}", wine_src_path)) .arg(format!("/log:{}", wine_log_path)) diff --git a/src/config.rs b/src/config.rs index d02f075..e28bbce 100644 --- a/src/config.rs +++ b/src/config.rs @@ -4,6 +4,7 @@ use std::collections::HashMap; use std::fs; use std::path::Path; +#[allow(dead_code)] #[derive(Debug, Clone, Serialize, Deserialize)] pub struct Config { pub wine_executable: Option, @@ -47,6 +48,7 @@ impl Default for Config { } } +#[allow(dead_code)] impl Config { pub fn load() -> Result { let config_path = Self::get_config_path(); diff --git a/src/main.rs b/src/main.rs index bf80d0a..86a6a2e 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,6 +1,7 @@ mod analytics; mod compile; mod models; +mod optimization; mod pipeline; mod tools; diff --git a/src/models/deals.rs b/src/models/deals.rs index de8c849..9b59d39 100644 --- a/src/models/deals.rs +++ b/src/models/deals.rs @@ -20,6 +20,7 @@ pub struct Deal { pub magic: Option, } +#[allow(dead_code)] #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(rename_all = "lowercase")] pub enum DealType { @@ -69,6 +70,7 @@ pub struct LossSequence { pub end: String, } +#[allow(dead_code)] #[derive(Debug, Clone, Serialize, Deserialize)] pub struct CycleStats { pub total_cycles: i32, @@ -77,6 +79,7 @@ pub struct CycleStats { pub win_rate_by_depth: HashMap, } +#[allow(dead_code)] #[derive(Debug, Clone, Serialize, Deserialize)] pub struct WinRateByDepth { pub total: i32, diff --git a/src/models/report.rs b/src/models/report.rs index 88d2094..c2cc1f9 100644 --- a/src/models/report.rs +++ b/src/models/report.rs @@ -1,6 +1,7 @@ use serde::{Deserialize, Serialize}; use std::path::PathBuf; +#[allow(dead_code)] #[derive(Debug, Clone, Serialize, Deserialize)] pub struct Report { pub report_dir: PathBuf, @@ -40,6 +41,7 @@ pub struct FilePaths { pub deals_json: String, } +#[allow(dead_code)] #[derive(Debug, Clone, Serialize, Deserialize)] pub struct BacktestStatus { pub stage: PipelineStage, @@ -48,6 +50,7 @@ pub struct BacktestStatus { pub message: String, } +#[allow(dead_code)] #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(rename_all = "UPPERCASE")] pub enum PipelineStage { diff --git a/src/optimization/mod.rs b/src/optimization/mod.rs new file mode 100644 index 0000000..d9ce823 --- /dev/null +++ b/src/optimization/mod.rs @@ -0,0 +1,5 @@ +pub mod optimizer; +pub mod parser; + +pub use optimizer::{OptimizationParams, OptimizationRunner}; +pub use parser::OptimizationParser; diff --git a/src/optimization/optimizer.rs b/src/optimization/optimizer.rs new file mode 100644 index 0000000..8122b25 --- /dev/null +++ b/src/optimization/optimizer.rs @@ -0,0 +1,375 @@ +use anyhow::{anyhow, Result}; +use chrono::Utc; +use std::fs; +use std::path::{Path, PathBuf}; +use std::process::{Command, Stdio}; + +use crate::models::Config; + +pub struct OptimizationParams { + pub expert: String, + pub set_file: String, + pub symbol: String, + pub from_date: String, + pub to_date: String, + pub deposit: u32, + pub model: u8, + pub leverage: u32, + pub currency: String, +} + +impl Default for OptimizationParams { + fn default() -> Self { + Self { + expert: String::new(), + set_file: String::new(), + symbol: "XAUUSD".to_string(), + from_date: String::new(), + to_date: String::new(), + deposit: 10000, + model: 0, + leverage: 500, + currency: "USD".to_string(), + } + } +} + +pub struct OptimizationResult { + pub success: bool, + pub job_id: String, + pub pid: u32, + pub log_file: PathBuf, + pub combinations: u64, + pub message: String, +} + +pub struct OptimizationRunner { + config: Config, +} + +impl OptimizationRunner { + pub fn new(config: Config) -> Self { + Self { config } + } + + pub async fn run(&self, params: OptimizationParams) -> Result { + // Validate required fields + if params.expert.is_empty() { + return Err(anyhow!("expert is required")); + } + if params.set_file.is_empty() { + return Err(anyhow!("set_file is required")); + } + if params.from_date.is_empty() { + return Err(anyhow!("from_date is required")); + } + if params.to_date.is_empty() { + return Err(anyhow!("to_date is required")); + } + + let set_path = Path::new(¶ms.set_file); + if !set_path.exists() { + return Err(anyhow!("Set file not found: {}", params.set_file)); + } + + // Generate job ID and log file + let timestamp = Utc::now().format("%Y%m%d_%H%M%S").to_string(); + let job_id = format!("opt_{}", timestamp); + let log_file = PathBuf::from(format!("/tmp/mt5opt_{}.log", timestamp)); + + // Count combinations + let combinations = self.count_combinations(¶ms.set_file)?; + + // Get paths + let mt5_dir = self.config.terminal_dir.as_ref() + .ok_or_else(|| anyhow!("terminal_dir not configured"))?; + let wine_exe = self.config.wine_executable.as_ref() + .ok_or_else(|| anyhow!("wine_executable not configured"))?; + + // Write .set file as UTF-16LE with BOM directly to MT5 tester directory + let wine_prefix_dir = self.get_wine_prefix_dir(mt5_dir)?; + let tester_dir = wine_prefix_dir.join("drive_c/Program Files/MetaTrader 5/MQL5/Profiles/Tester"); + fs::create_dir_all(&tester_dir)?; + let dst_set_file = tester_dir.join(format!("{}.set", params.expert)); + self.write_utf16le_set(¶ms.set_file, &dst_set_file)?; + + // Reset OptMode in terminal.ini + self.reset_optmode(mt5_dir)?; + + // Get Wine prefix directory + let wine_prefix_dir = self.get_wine_prefix_dir(mt5_dir)?; + + // Build optimization INI + let ini_path = wine_prefix_dir.join("drive_c/mt5mcp_backtest.ini"); + let ini_content = format!(r#"[Tester] +Expert={} +Symbol={} +Period=M5 +Deposit={} +Currency={} +Leverage={} +Model={} +FromDate={} +ToDate={} +Report=C:\mt5mcp_opt_report +Optimization=2 +ExpertParameters={}.set +ShutdownTerminal=1 +"#, params.expert, params.symbol, params.deposit, params.currency, + params.leverage, params.model, params.from_date, params.to_date, params.expert); + fs::write(&ini_path, ini_content)?; + + // Build batch file + let batch_path = wine_prefix_dir.join("drive_c/mt5mcp_run.bat"); + let batch_content = format!(r#"@echo off +"C:\Program Files\MetaTrader 5\terminal64.exe" /config:C:\mt5mcp_backtest.ini +"#); + fs::write(&batch_path, batch_content)?; + + // Launch detached process + let cmd = format!("cmd.exe /c 'C:\\mt5mcp_run.bat'"); + let child = Command::new(wine_exe) + .arg(&cmd) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn()?; + + let pid = child.id(); + + // Write job metadata + self.write_job_metadata(&job_id, pid, ¶ms, &log_file, combinations, &wine_prefix_dir)?; + + Ok(OptimizationResult { + success: true, + job_id, + pid, + log_file, + combinations, + message: format!("Optimization launched (pid: {}). Runs for 2-6 hours. Do NOT kill this process.", pid), + }) + } + + fn count_combinations(&self, set_file: &str) -> Result { + let content = fs::read_to_string(set_file)?; + let mut total: u64 = 1; + + for line in content.lines() { + let line = line.trim(); + if line.starts_with(';') || !line.contains('=') { + continue; + } + + // Format: param=value||start||step||stop||Y + let parts: Vec<&str> = line.split("||").collect(); + if parts.len() >= 5 && parts.last().unwrap().trim().to_uppercase() == "Y" { + if let (Ok(start), Ok(step), Ok(stop)) = ( + parts[1].trim().parse::(), + parts[2].trim().parse::(), + parts[3].trim().parse::(), + ) { + if step > 0.0 { + let count = ((stop - start) / step).max(0.0) as u64 + 1; + total = total.saturating_mul(count); + } + } + } + } + + Ok(total.max(1)) + } + + fn write_utf16le_set(&self, src: &str, dst: &Path) -> Result<()> { + let content = fs::read_to_string(src)?; + + // Create parent directory if needed + if let Some(parent) = dst.parent() { + fs::create_dir_all(parent)?; + } + + // Write UTF-16LE with BOM + let mut utf16_content: Vec = vec![0xFEFF]; // BOM + utf16_content.extend(content.encode_utf16()); + + let bytes: Vec = utf16_content.iter() + .flat_map(|&c| vec![(c & 0xFF) as u8, ((c >> 8) & 0xFF) as u8]) + .collect(); + + fs::write(dst, bytes)?; + + // Make read-only + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + fs::set_permissions(dst, fs::Permissions::from_mode(0o444))?; + } + + Ok(()) + } + + fn reset_optmode(&self, mt5_dir: &str) -> Result<()> { + let terminal_ini = Path::new(mt5_dir).join("terminal.ini"); + + if !terminal_ini.exists() { + return Ok(()); + } + + let content = fs::read_to_string(&terminal_ini)?; + let updated = content + .lines() + .map(|line| { + if line.starts_with("OptMode=") { + "OptMode=0".to_string() + } else if line.starts_with("LastOptimization=") { + String::new() + } else { + line.to_string() + } + }) + .filter(|l| !l.is_empty()) + .collect::>() + .join("\n"); + + fs::write(&terminal_ini, updated)?; + Ok(()) + } + + fn get_wine_prefix_dir(&self, mt5_dir: &str) -> Result { + let path = Path::new(mt5_dir); + // Go up two levels: .../drive_c/Program Files/MetaTrader 5 -> .../drive_c + let prefix_dir = path + .parent() + .and_then(|p| p.parent()) + .ok_or_else(|| anyhow!("Cannot determine Wine prefix from terminal_dir"))?; + Ok(prefix_dir.to_path_buf()) + } + + fn write_job_metadata( + &self, + job_id: &str, + pid: u32, + params: &OptimizationParams, + log_file: &Path, + combinations: u64, + wine_prefix: &Path, + ) -> Result<()> { + let jobs_dir = Path::new(".mt5mcp_jobs"); + fs::create_dir_all(jobs_dir)?; + + let meta_path = jobs_dir.join(format!("{}.json", job_id)); + let started_at = Utc::now().to_rfc3339(); + + let metadata = serde_json::json!({ + "job_id": job_id, + "pid": pid, + "expert": params.expert, + "symbol": params.symbol, + "from_date": params.from_date, + "to_date": params.to_date, + "set_file": params.set_file, + "combinations": combinations, + "log_file": log_file.to_string_lossy(), + "wine_prefix": wine_prefix.to_string_lossy(), + "started_at": started_at, + }); + + fs::write(&meta_path, serde_json::to_string_pretty(&metadata)?)?; + Ok(()) + } + + pub fn get_job_status(&self, job_id: &str) -> Result { + let jobs_dir = Path::new(".mt5mcp_jobs"); + let meta_path = jobs_dir.join(format!("{}.json", job_id)); + + if !meta_path.exists() { + return Ok(serde_json::json!({ + "status": "not_found", + "message": format!("Job {} not found", job_id) + })); + } + + let meta: serde_json::Value = serde_json::from_str(&fs::read_to_string(&meta_path)?)?; + let pid = meta.get("pid").and_then(|v| v.as_u64()).unwrap_or(0) as u32; + + // Check if process is still running + let is_running = self.is_process_running(pid); + + // Check for completion marker in log + let log_file = meta.get("log_file").and_then(|v| v.as_str()).unwrap_or(""); + let is_complete = if !log_file.is_empty() && Path::new(log_file).exists() { + fs::read_to_string(log_file) + .map(|content| content.contains("Optimization complete")) + .unwrap_or(false) + } else { + false + }; + + let status = if is_complete { + "completed" + } else if is_running { + "running" + } else { + "stopped" + }; + + Ok(serde_json::json!({ + "status": status, + "job_id": job_id, + "pid": pid, + "expert": meta.get("expert"), + "symbol": meta.get("symbol"), + "started_at": meta.get("started_at"), + "log_file": log_file, + })) + } + + fn is_process_running(&self, pid: u32) -> bool { + #[cfg(unix)] + { + Command::new("kill") + .args(["-0", &pid.to_string()]) + .output() + .map(|output| output.status.success()) + .unwrap_or(false) + } + #[cfg(windows)] + { + // Windows implementation would use different method + false + } + } + + pub fn list_jobs(&self) -> Result> { + let jobs_dir = Path::new(".mt5mcp_jobs"); + let mut jobs = Vec::new(); + + if jobs_dir.exists() { + for entry in fs::read_dir(jobs_dir)? { + if let Ok(entry) = entry { + let path = entry.path(); + if path.extension().map(|e| e == "json").unwrap_or(false) { + if let Ok(content) = fs::read_to_string(&path) { + if let Ok(meta) = serde_json::from_str::(&content) { + let job_id = path.file_stem() + .and_then(|s| s.to_str()) + .unwrap_or("unknown") + .to_string(); + + let pid = meta.get("pid").and_then(|v| v.as_u64()).unwrap_or(0) as u32; + let is_running = self.is_process_running(pid); + + jobs.push(serde_json::json!({ + "job_id": job_id, + "expert": meta.get("expert"), + "status": if is_running { "running" } else { "stopped" }, + "started_at": meta.get("started_at"), + })); + } + } + } + } + } + } + + Ok(jobs) + } +} diff --git a/src/optimization/parser.rs b/src/optimization/parser.rs new file mode 100644 index 0000000..4d6a360 --- /dev/null +++ b/src/optimization/parser.rs @@ -0,0 +1,270 @@ +use anyhow::{anyhow, Result}; +use serde::{Deserialize, Serialize}; +use std::collections::HashMap; +use std::fs; +use std::path::Path; + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct OptimizationPass { + pub pass: u32, + pub profit: f64, + pub total_trades: u32, + pub profit_factor: f64, + pub expected_payoff: f64, + pub drawdown_pct: f64, + pub params: HashMap, +} + +pub struct OptimizationParser; + +impl OptimizationParser { + pub fn new() -> Self { + Self + } + + pub fn parse_job(&self, job_id: &str) -> Result> { + let jobs_dir = Path::new(".mt5mcp_jobs"); + let meta_path = jobs_dir.join(format!("{}.json", job_id)); + + if !meta_path.exists() { + return Err(anyhow!("Job not found: {}. Check .mt5mcp_jobs/", job_id)); + } + + let meta: serde_json::Value = serde_json::from_str(&fs::read_to_string(&meta_path)?)?; + let wine_prefix = meta.get("wine_prefix") + .and_then(|v| v.as_str()) + .ok_or_else(|| anyhow!("wine_prefix not in job metadata"))?; + + let base_path = Path::new(wine_prefix).join("drive_c/mt5mcp_opt_report"); + + // Try different extensions + for ext in &[".htm", ".htm.xml", ".html"] { + let candidate = base_path.with_extension(ext.trim_start_matches('.')); + if candidate.exists() { + return self.parse_file(&candidate); + } + } + + Err(anyhow!( + "Optimization report not found. Expected: {}.htm or {}.htm.xml\nIs MT5 optimization still running?", + base_path.display(), + base_path.display() + )) + } + + pub fn parse_file(&self, path: &Path) -> Result> { + let format = self.detect_format(path); + let text = self.read_text(path)?; + + match format { + "xml" => self.parse_xml(&text), + _ => self.parse_html(&text), + } + } + + fn detect_format(&self, path: &Path) -> &str { + let path_str = path.to_string_lossy(); + if path_str.ends_with(".xml") || path_str.ends_with(".htm.xml") { + return "xml"; + } + + if let Ok(header) = fs::read(path) { + let header = &header[..header.len().min(512)]; + if header.windows(5).any(|w| w == b" Result { + let raw = fs::read(path)?; + + // Try UTF-16 first (common for MT5 reports) + if raw.len() >= 2 { + if raw[0] == 0xFF && raw[1] == 0xFE { + // UTF-16 LE with BOM + let u16_vec: Vec = raw[2..].chunks_exact(2) + .map(|c| u16::from_le_bytes([c[0], c[1]])) + .collect(); + return Ok(String::from_utf16_lossy(&u16_vec)); + } else if raw[0] == 0xFE && raw[1] == 0xFF { + // UTF-16 BE with BOM + let u16_vec: Vec = raw[2..].chunks_exact(2) + .map(|c| u16::from_be_bytes([c[0], c[1]])) + .collect(); + return Ok(String::from_utf16_lossy(&u16_vec)); + } + } + + // Try UTF-8, then fallback to lossy + if let Ok(text) = String::from_utf8(raw.clone()) { + return Ok(text); + } + + // Try UTF-16 without BOM + if raw.len() % 2 == 0 { + let u16_vec: Vec = raw.chunks_exact(2) + .map(|c| u16::from_le_bytes([c[0], c[1]])) + .collect(); + let text = String::from_utf16_lossy(&u16_vec); + if text.chars().any(|c| c.is_ascii_alphanumeric()) { + return Ok(text); + } + } + + Ok(String::from_utf8_lossy(&raw).to_string()) + } + + fn parse_html(&self, text: &str) -> Result> { + let mut results = Vec::new(); + let mut headers: Vec = Vec::new(); + + // Find all table rows + let row_regex = regex::Regex::new(r"]*>(.*?)")?; + let cell_regex = regex::Regex::new(r"]*>(.*?)")?; + let tag_regex = regex::Regex::new(r"<[^>]+>")?; + + for row_caps in row_regex.captures_iter(text) { + let row = &row_caps[1]; + let cells: Vec = cell_regex.captures_iter(row) + .map(|c| { + let cell = &c[1]; + tag_regex.replace_all(cell, "").trim().to_string().replace(',', "") + }) + .collect(); + + if cells.is_empty() { + continue; + } + + // Header row detection + if headers.is_empty() && cells[0].to_lowercase().contains("pass") { + headers = cells; + continue; + } + + // Data row + if !headers.is_empty() && cells[0].parse::().is_ok() { + let row_map: HashMap = headers.iter() + .zip(cells.iter()) + .map(|(h, c)| (h.to_lowercase().replace(' ', "_"), c.clone())) + .collect(); + + if let Some(pass) = self.row_to_pass(&row_map) { + results.push(pass); + } + } + } + + Ok(results) + } + + fn parse_xml(&self, text: &str) -> Result> { + let mut results = Vec::new(); + + // Parse SpreadsheetML XML + let doc = roxmltree::Document::parse(text)?; + + // Find all rows in Worksheet/Table + for node in doc.descendants() { + if node.has_tag_name(("http://schemas.microsoft.com/office/excel/2003/xml", "Row")) || + node.has_tag_name("Row") { + let cells: Vec = node.children() + .filter(|n: &roxmltree::Node<'_, '_>| { + n.has_tag_name(("http://schemas.microsoft.com/office/excel/2003/xml", "Cell")) || + n.has_tag_name("Cell") || + n.has_tag_name(("http://schemas.microsoft.com/office/excel/2003/xml", "Data")) || + n.has_tag_name("Data") + }) + .map(|n| n.text().unwrap_or("").trim().to_string().replace(',', "")) + .collect(); + + if cells.is_empty() { + continue; + } + + // Check if first cell is a pass number + if let Ok(pass_num) = cells[0].parse::() { + if pass_num > 0 { + let mut row_map = HashMap::new(); + + // Standard MT5 optimization report columns + let headers = vec![ + "pass", "result", "profit", "total_trades", "profit_factor", + "expected_payoff", "drawdown_pct", "recovery_factor", "sharpe_ratio", + "custom", "consecutive_wins", "consecutive_losses", + ]; + + for (i, cell) in cells.iter().enumerate() { + if let Some(header) = headers.get(i) { + row_map.insert(header.to_string(), cell.clone()); + } + } + + if let Some(pass) = self.row_to_pass(&row_map) { + results.push(pass); + } + } + } + } + } + + Ok(results) + } + + fn row_to_pass(&self, row: &HashMap) -> Option { + let pass = row.get("pass").or_else(|| row.get("#")) + .and_then(|v| v.parse().ok())?; + + let profit = row.get("profit").or_else(|| row.get("total_net_profit")) + .and_then(|v| v.replace(' ', "").parse().ok())?; + + let total_trades = row.get("total_trades").or_else(|| row.get("trades")) + .and_then(|v| v.parse().ok())?; + + let profit_factor = row.get("profit_factor") + .and_then(|v| v.parse().ok())?; + + let expected_payoff = row.get("expected_payoff") + .and_then(|v| v.parse().ok())?; + + let drawdown_pct = row.get("drawdown_pct").or_else(|| row.get("max_drawdown")) + .and_then(|v| v.trim_end_matches('%').trim().parse().ok())?; + + // Extract parameter values from row + let params: HashMap = row.iter() + .filter(|(k, _)| ![ + "pass", "result", "profit", "total_trades", "profit_factor", + "expected_payoff", "drawdown_pct", "max_drawdown", "recovery_factor", + "sharpe_ratio", "custom", "consecutive_wins", "consecutive_losses" + ].contains(&k.as_str())) + .map(|(k, v)| (k.clone(), v.clone())) + .collect(); + + Some(OptimizationPass { + pass, + profit, + total_trades, + profit_factor, + expected_payoff, + drawdown_pct, + params, + }) + } + + pub fn find_best_pass<'a>(&self, passes: &'a [OptimizationPass], criteria: &str) -> Option<&'a OptimizationPass> { + match criteria { + "profit" => passes.iter().max_by(|a, b| a.profit.partial_cmp(&b.profit).unwrap()), + "profit_factor" => passes.iter().max_by(|a, b| a.profit_factor.partial_cmp(&b.profit_factor).unwrap()), + "sharpe" => passes.iter().max_by(|a, b| { + let a_sharpe = a.params.get("sharpe_ratio").and_then(|v| v.parse::().ok()).unwrap_or(0.0); + let b_sharpe = b.params.get("sharpe_ratio").and_then(|v| v.parse::().ok()).unwrap_or(0.0); + a_sharpe.partial_cmp(&b_sharpe).unwrap() + }), + "drawdown" => passes.iter().min_by(|a, b| a.drawdown_pct.partial_cmp(&b.drawdown_pct).unwrap()), + _ => passes.iter().max_by(|a, b| a.profit.partial_cmp(&b.profit).unwrap()), + } + } +} diff --git a/src/pipeline/backtest.rs b/src/pipeline/backtest.rs index 3a72d28..2a692a3 100644 --- a/src/pipeline/backtest.rs +++ b/src/pipeline/backtest.rs @@ -30,6 +30,7 @@ pub struct BacktestParams { pub skip_compile: bool, pub skip_clean: bool, pub skip_analyze: bool, + #[allow(dead_code)] pub deep_analyze: bool, pub shutdown: bool, pub kill_existing: bool, @@ -250,7 +251,7 @@ start terminal64.exe /config:"C:\Program Files\MetaTrader 5\backtest_config.ini" let bat_path = wine_prefix.join("drive_c").join("_mt5mcp_run.bat"); fs::write(&bat_path, bat_content)?; - let cmd = format!("cmd.exe /c 'C:\\_mt5mcp_run.bat'"); + let _cmd = format!("cmd.exe /c 'C:\\_mt5mcp_run.bat'"); if params.shutdown { let output = Command::new("timeout") @@ -346,7 +347,7 @@ start terminal64.exe /config:"C:\Program Files\MetaTrader 5\backtest_config.ini" } async fn kill_mt5(&self) -> Result<()> { - let output = Command::new("pkill") + let _output = Command::new("pkill") .args(&["-TERM", "-f", "terminal64\\.exe"]) .output()?; diff --git a/src/pipeline/stages.rs b/src/pipeline/stages.rs index deacd53..4eb0faa 100644 --- a/src/pipeline/stages.rs +++ b/src/pipeline/stages.rs @@ -1,6 +1,7 @@ use anyhow::Result; use std::path::PathBuf; +#[allow(dead_code)] #[derive(Debug, Clone, Copy, PartialEq)] pub enum Stage { Compile, @@ -11,6 +12,7 @@ pub enum Stage { Done, } +#[allow(dead_code)] impl Stage { pub fn as_str(&self) -> &'static str { match self { @@ -35,8 +37,10 @@ impl Stage { } } +#[allow(dead_code)] pub struct StageExecutor; +#[allow(dead_code)] impl StageExecutor { pub fn new() -> Self { Self @@ -74,12 +78,14 @@ impl StageExecutor { } } +#[allow(dead_code)] pub struct StageResult { pub success: bool, pub message: String, pub output: Option, } +#[allow(dead_code)] impl StageResult { pub fn success() -> Self { Self { diff --git a/src/tools/handlers.rs b/src/tools/handlers.rs index ffd2c97..f960e2d 100644 --- a/src/tools/handlers.rs +++ b/src/tools/handlers.rs @@ -3,8 +3,12 @@ use serde_json::{json, Value}; use std::collections::HashMap; use std::fs; use std::path::Path; +use crate::analytics::DealAnalyzer; use crate::compile::MqlCompiler; use crate::models::Config; +use crate::models::deals::Deal; +use crate::models::metrics::Metrics; +use crate::optimization::{OptimizationParams, OptimizationParser, OptimizationRunner}; use crate::pipeline::backtest::{BacktestParams, BacktestPipeline}; #[derive(Debug)] @@ -31,6 +35,28 @@ impl ToolHandler { "prune_reports" => self.handle_prune_reports(args).await, "list_set_files" => self.handle_list_set_files().await, "describe_sweep" => self.handle_describe_sweep(args).await, + // Optimization tools + "run_optimization" => self.handle_run_optimization(args).await, + "get_optimization_status" => self.handle_get_optimization_status(args).await, + "get_optimization_results" => self.handle_get_optimization_results(args).await, + "list_jobs" => self.handle_list_jobs().await, + // Analysis tools + "analyze_report" => self.handle_analyze_report(args).await, + "compare_baseline" => self.handle_compare_baseline(args).await, + // Set file tools + "read_set_file" => self.handle_read_set_file(args).await, + "write_set_file" => self.handle_write_set_file(args).await, + "patch_set_file" => self.handle_patch_set_file(args).await, + "clone_set_file" => self.handle_clone_set_file(args).await, + "diff_set_files" => self.handle_diff_set_files(args).await, + "set_from_optimization" => self.handle_set_from_optimization(args).await, + // Utility tools + "tail_log" => self.handle_tail_log(args).await, + "archive_report" => self.handle_archive_report(args).await, + "archive_all_reports" => self.handle_archive_all_reports(args).await, + "promote_to_baseline" => self.handle_promote_to_baseline(args).await, + "get_history" => self.handle_get_history(args).await, + "annotate_history" => self.handle_annotate_history(args).await, _ => Ok(json!({ "content": [{ "type": "text", "text": format!("Tool '{}' not implemented", name) }], "isError": true @@ -461,4 +487,641 @@ impl ToolHandler { "isError": false })) } + + // Optimization handlers + async fn handle_run_optimization(&self, args: &Value) -> Result { + let expert = args.get("expert") + .and_then(|v| v.as_str()) + .ok_or_else(|| anyhow::anyhow!("expert is required"))?; + + let set_file = args.get("set_file") + .and_then(|v| v.as_str()) + .ok_or_else(|| anyhow::anyhow!("set_file is required"))?; + + let from_date = args.get("from_date") + .and_then(|v| v.as_str()) + .ok_or_else(|| anyhow::anyhow!("from_date is required"))?; + + let to_date = args.get("to_date") + .and_then(|v| v.as_str()) + .ok_or_else(|| anyhow::anyhow!("to_date is required"))?; + + let params = OptimizationParams { + expert: expert.to_string(), + set_file: set_file.to_string(), + symbol: args.get("symbol").and_then(|v| v.as_str()).unwrap_or("XAUUSD").to_string(), + from_date: from_date.to_string(), + to_date: to_date.to_string(), + deposit: args.get("deposit").and_then(|v| v.as_u64()).unwrap_or(10000) as u32, + model: 0, // Always 0 for optimization + leverage: args.get("leverage").and_then(|v| v.as_u64()).unwrap_or(500) as u32, + currency: args.get("currency").and_then(|v| v.as_str()).unwrap_or("USD").to_string(), + }; + + let runner = OptimizationRunner::new(self.config.clone()); + let result = runner.run(params).await?; + + Ok(json!({ + "content": [{ "type": "text", "text": json!({ + "success": result.success, + "job_id": result.job_id, + "pid": result.pid, + "log_file": result.log_file.to_string_lossy(), + "combinations": result.combinations, + "message": result.message, + }).to_string() }], + "isError": false + })) + } + + async fn handle_get_optimization_status(&self, args: &Value) -> Result { + let job_id = args.get("job_id") + .and_then(|v| v.as_str()) + .ok_or_else(|| anyhow::anyhow!("job_id is required"))?; + + let runner = OptimizationRunner::new(self.config.clone()); + let status = runner.get_job_status(job_id)?; + + Ok(json!({ + "content": [{ "type": "text", "text": status.to_string() }], + "isError": false + })) + } + + async fn handle_get_optimization_results(&self, args: &Value) -> Result { + let job_id = args.get("job_id") + .and_then(|v| v.as_str()); + + let file = args.get("file") + .and_then(|v| v.as_str()); + + let parser = OptimizationParser::new(); + + let passes = if let Some(jid) = job_id { + parser.parse_job(jid)? + } else if let Some(f) = file { + parser.parse_file(std::path::Path::new(f))? + } else { + return Err(anyhow::anyhow!("Either job_id or file is required")); + }; + + let sort_by = args.get("sort").and_then(|v| v.as_str()).unwrap_or("profit"); + let top_n = args.get("top").and_then(|v| v.as_u64()).unwrap_or(30) as usize; + + // Find best pass + let best = parser.find_best_pass(&passes, sort_by); + + let mut sorted_passes = passes.clone(); + sorted_passes.sort_by(|a, b| b.profit.partial_cmp(&a.profit).unwrap()); + sorted_passes.truncate(top_n); + + Ok(json!({ + "content": [{ "type": "text", "text": json!({ + "total_passes": passes.len(), + "top_passes": sorted_passes, + "best": best, + "sort_by": sort_by, + }).to_string() }], + "isError": false + })) + } + + async fn handle_list_jobs(&self) -> Result { + let runner = OptimizationRunner::new(self.config.clone()); + let jobs = runner.list_jobs()?; + + Ok(json!({ + "content": [{ "type": "text", "text": json!({ "jobs": jobs }).to_string() }], + "isError": false + })) + } + + // Analysis handlers + async fn handle_analyze_report(&self, args: &Value) -> Result { + let report_dir = args.get("report_dir") + .and_then(|v| v.as_str()) + .ok_or_else(|| anyhow::anyhow!("report_dir is required"))?; + + let deals_csv = std::path::Path::new(report_dir).join("deals.csv"); + let metrics_json = std::path::Path::new(report_dir).join("metrics.json"); + + if !deals_csv.exists() { + return Err(anyhow::anyhow!("deals.csv not found in {}", report_dir)); + } + + // Read deals + let deals = self.read_deals_from_csv(&deals_csv)?; + + // Read metrics + let metrics = if metrics_json.exists() { + let content = fs::read_to_string(&metrics_json)?; + serde_json::from_str(&content)? + } else { + Metrics::default() + }; + + let _strategy = args.get("strategy").and_then(|v| v.as_str()).unwrap_or("grid"); + let _deep = args.get("deep").and_then(|v| v.as_bool()).unwrap_or(false); + + let analyzer = DealAnalyzer::new(); + let result = analyzer.analyze(&deals, &metrics); + + // Write analysis.json + let analysis_path = std::path::Path::new(report_dir).join("analysis.json"); + fs::write(&analysis_path, serde_json::to_string_pretty(&result)?)?; + + Ok(json!({ + "content": [{ "type": "text", "text": json!({ + "success": true, + "analysis_file": analysis_path.to_string_lossy(), + "summary": result, + }).to_string() }], + "isError": false + })) + } + + fn read_deals_from_csv(&self, path: &std::path::Path) -> Result> { + let content = fs::read_to_string(path)?; + let mut deals = Vec::new(); + + let mut lines = content.lines(); + let _header = lines.next(); // Skip header + + for line in lines { + let parts: Vec<&str> = line.split(',').collect(); + if parts.len() >= 12 { + deals.push(Deal { + time: parts[0].to_string(), + deal: parts[1].to_string(), + symbol: parts[2].to_string(), + deal_type: parts[3].to_string(), + entry: parts[4].to_string(), + volume: parts[5].parse().unwrap_or(0.0), + price: parts[6].parse().unwrap_or(0.0), + order: parts[7].to_string(), + commission: parts[8].parse().unwrap_or(0.0), + swap: parts[9].parse().unwrap_or(0.0), + profit: parts[10].parse().unwrap_or(0.0), + balance: parts[11].parse().unwrap_or(0.0), + comment: parts.get(12).unwrap_or(&"").to_string(), + magic: parts.get(13).map(|s| s.to_string()), + }); + } + } + + Ok(deals) + } + + async fn handle_compare_baseline(&self, args: &Value) -> Result { + let report_dir = args.get("report_dir") + .and_then(|v| v.as_str()) + .ok_or_else(|| anyhow::anyhow!("report_dir is required"))?; + + let baseline_path = std::path::Path::new("config/baseline.json"); + let metrics_path = std::path::Path::new(report_dir).join("metrics.json"); + + if !baseline_path.exists() { + return Ok(json!({ + "content": [{ "type": "text", "text": "No baseline.json found in config/" }], + "isError": false + })); + } + + let baseline: Value = serde_json::from_str(&fs::read_to_string(baseline_path)?)?; + let current: Value = serde_json::from_str(&fs::read_to_string(metrics_path)?)?; + + let comparison = json!({ + "baseline": baseline, + "current": current, + "improvements": { + "profit": current.get("net_profit").and_then(|v| v.as_f64()).unwrap_or(0.0) + - baseline.get("net_profit").and_then(|v| v.as_f64()).unwrap_or(0.0), + "drawdown": current.get("max_dd_pct").and_then(|v| v.as_f64()).unwrap_or(0.0) + - baseline.get("max_dd_pct").and_then(|v| v.as_f64()).unwrap_or(0.0), + } + }); + + Ok(json!({ + "content": [{ "type": "text", "text": comparison.to_string() }], + "isError": false + })) + } + + // Set file handlers + async fn handle_read_set_file(&self, args: &Value) -> Result { + let path = args.get("path") + .and_then(|v| v.as_str()) + .ok_or_else(|| anyhow::anyhow!("path is required"))?; + + let content = fs::read_to_string(path)?; + let mut params = serde_json::Map::new(); + + for line in content.lines() { + if let Some((key, value)) = line.split_once(':') { + let key = key.trim(); + let value = value.trim(); + + if value.contains("||Y") { + let parts: Vec<&str> = value.split("||").collect(); + if parts.len() >= 5 { + params.insert(key.to_string(), json!({ + "value": parts[0], + "from": parts[1], + "step": parts[2], + "to": parts[3], + "optimize": true, + })); + } + } else { + params.insert(key.to_string(), json!({ "value": value, "optimize": false })); + } + } + } + + Ok(json!({ + "content": [{ "type": "text", "text": json!({ + "path": path, + "parameters": params, + }).to_string() }], + "isError": false + })) + } + + async fn handle_write_set_file(&self, args: &Value) -> Result { + let path = args.get("path") + .and_then(|v| v.as_str()) + .ok_or_else(|| anyhow::anyhow!("path is required"))?; + + let params = args.get("parameters") + .and_then(|v| v.as_object()) + .ok_or_else(|| anyhow::anyhow!("parameters object is required"))?; + + let mut lines = Vec::new(); + for (key, value) in params { + if let Some(obj) = value.as_object() { + if obj.get("optimize").and_then(|v| v.as_bool()).unwrap_or(false) { + let from_val = obj.get("from").and_then(|v| v.as_str()).unwrap_or("0"); + let step = obj.get("step").and_then(|v| v.as_str()).unwrap_or("1"); + let to_val = obj.get("to").and_then(|v| v.as_str()).unwrap_or("0"); + lines.push(format!("{}={}||{}||{}||{}||Y", key, obj.get("value").and_then(|v| v.as_str()).unwrap_or("0"), from_val, step, to_val)); + } else { + lines.push(format!("{}={}", key, obj.get("value").and_then(|v| v.as_str()).unwrap_or("0"))); + } + } + } + + fs::write(path, lines.join("\n"))?; + + Ok(json!({ + "content": [{ "type": "text", "text": json!({ + "success": true, + "path": path, + "parameters_written": lines.len(), + }).to_string() }], + "isError": false + })) + } + + async fn handle_patch_set_file(&self, args: &Value) -> Result { + let path = args.get("path") + .and_then(|v| v.as_str()) + .ok_or_else(|| anyhow::anyhow!("path is required"))?; + + let patches = args.get("patches") + .and_then(|v| v.as_object()) + .ok_or_else(|| anyhow::anyhow!("patches object is required"))?; + + // Read existing file + let content = fs::read_to_string(path)?; + let mut lines: Vec = content.lines().map(|s| s.to_string()).collect(); + let mut patched_count = 0; + + for (key, value) in patches { + let new_value = if let Some(s) = value.as_str() { + s.to_string() + } else if let Some(n) = value.as_f64() { + n.to_string() + } else if let Some(b) = value.as_bool() { + if b { "true".to_string() } else { "false".to_string() } + } else { + value.to_string() + }; + + // Find and patch the parameter + let mut found = false; + for line in &mut lines { + if line.starts_with(&format!("{}:", key)) { + *line = format!("{}: {}", key, new_value); + found = true; + patched_count += 1; + break; + } else if line.starts_with(&format!("{}=", key)) { + *line = format!("{}={}", key, new_value); + found = true; + patched_count += 1; + break; + } + } + + // If not found, add it + if !found { + lines.push(format!("{}: {}", key, new_value)); + patched_count += 1; + } + } + + fs::write(path, lines.join("\n"))?; + + Ok(json!({ + "content": [{ "type": "text", "text": json!({ + "success": true, + "path": path, + "parameters_patched": patched_count, + }).to_string() }], + "isError": false + })) + } + + async fn handle_clone_set_file(&self, args: &Value) -> Result { + let source = args.get("source") + .and_then(|v| v.as_str()) + .ok_or_else(|| anyhow::anyhow!("source is required"))?; + + let destination = args.get("destination") + .and_then(|v| v.as_str()) + .ok_or_else(|| anyhow::anyhow!("destination is required"))?; + + fs::copy(source, destination)?; + + Ok(json!({ + "content": [{ "type": "text", "text": json!({ + "success": true, + "source": source, + "destination": destination, + }).to_string() }], + "isError": false + })) + } + + async fn handle_diff_set_files(&self, args: &Value) -> Result { + let file_a = args.get("file_a") + .and_then(|v| v.as_str()) + .ok_or_else(|| anyhow::anyhow!("file_a is required"))?; + + let file_b = args.get("file_b") + .and_then(|v| v.as_str()) + .ok_or_else(|| anyhow::anyhow!("file_b is required"))?; + + let content_a = fs::read_to_string(file_a)?; + let content_b = fs::read_to_string(file_b)?; + + let mut differences = Vec::new(); + + for (i, (line_a, line_b)) in content_a.lines().zip(content_b.lines()).enumerate() { + if line_a != line_b { + differences.push(json!({ + "line": i + 1, + "file_a": line_a, + "file_b": line_b, + })); + } + } + + Ok(json!({ + "content": [{ "type": "text", "text": json!({ + "file_a": file_a, + "file_b": file_b, + "differences": differences, + "total_differences": differences.len(), + }).to_string() }], + "isError": false + })) + } + + async fn handle_set_from_optimization(&self, args: &Value) -> Result { + let path = args.get("path") + .and_then(|v| v.as_str()) + .ok_or_else(|| anyhow::anyhow!("path is required"))?; + + let params = args.get("params") + .and_then(|v| v.as_object()) + .ok_or_else(|| anyhow::anyhow!("params is required"))?; + + let mut lines = Vec::new(); + for (key, value) in params { + if let Some(val_str) = value.as_str() { + lines.push(format!("{}={}", key, val_str)); + } + } + + fs::write(path, lines.join("\n"))?; + + Ok(json!({ + "content": [{ "type": "text", "text": json!({ + "success": true, + "path": path, + "parameters_written": lines.len(), + }).to_string() }], + "isError": false + })) + } + + // Utility handlers + async fn handle_tail_log(&self, args: &Value) -> Result { + let job_id = args.get("job_id") + .and_then(|v| v.as_str()); + + let lines = args.get("lines").and_then(|v| v.as_u64()).unwrap_or(50) as usize; + + let log_path = if let Some(jid) = job_id { + let jobs_dir = std::path::Path::new(".mt5mcp_jobs"); + let meta_path = jobs_dir.join(format!("{}.json", jid)); + let meta: Value = serde_json::from_str(&fs::read_to_string(meta_path)?)?; + meta.get("log_file").and_then(|v| v.as_str()).map(|s| s.to_string()) + } else { + args.get("file").and_then(|v| v.as_str()).map(|s| s.to_string()) + }; + + let log_path = log_path.ok_or_else(|| anyhow::anyhow!("Could not determine log file"))?; + + let content = fs::read_to_string(&log_path)?; + let all_lines: Vec<&str> = content.lines().collect(); + let start = all_lines.len().saturating_sub(lines); + let last_lines = &all_lines[start..]; + + Ok(json!({ + "content": [{ "type": "text", "text": last_lines.join("\n") }], + "isError": false + })) + } + + async fn handle_archive_report(&self, args: &Value) -> Result { + let report_dir = args.get("report_dir") + .and_then(|v| v.as_str()) + .ok_or_else(|| anyhow::anyhow!("report_dir is required"))?; + + let delete_after = args.get("delete_after").and_then(|v| v.as_bool()).unwrap_or(false); + + let history_dir = std::path::Path::new(".mt5mcp_history"); + fs::create_dir_all(history_dir)?; + + let report_name = std::path::Path::new(report_dir).file_name() + .and_then(|s| s.to_str()) + .unwrap_or("unknown"); + + let archive_path = history_dir.join(format!("{}.tar.gz", report_name)); + + // Create tarball + let status = std::process::Command::new("tar") + .args(["-czf", &archive_path.to_string_lossy(), "-C", + std::path::Path::new(report_dir).parent().unwrap().to_str().unwrap(), + report_name]) + .status()?; + + if delete_after && status.success() { + fs::remove_dir_all(report_dir)?; + } + + Ok(json!({ + "content": [{ "type": "text", "text": json!({ + "success": status.success(), + "archive_path": archive_path.to_string_lossy(), + "deleted": delete_after && status.success(), + }).to_string() }], + "isError": false + })) + } + + async fn handle_archive_all_reports(&self, args: &Value) -> Result { + let keep_last = args.get("keep_last").and_then(|v| v.as_u64()).unwrap_or(10) as usize; + + let reports_dir = self.config.reports_dir(); + let history_dir = std::path::Path::new(".mt5mcp_history"); + fs::create_dir_all(history_dir)?; + + let mut archived = 0; + + if let Ok(entries) = fs::read_dir(&reports_dir) { + let mut entries: Vec<_> = entries.flatten().collect(); + entries.sort_by(|a, b| { + b.metadata().and_then(|m| m.modified()).unwrap_or(std::time::UNIX_EPOCH) + .cmp(&a.metadata().and_then(|m| m.modified()).unwrap_or(std::time::UNIX_EPOCH)) + }); + + for entry in entries.into_iter().skip(keep_last) { + let path = entry.path(); + if path.is_dir() && !path.to_string_lossy().ends_with("_opt") { + let report_name = path.file_name().and_then(|s| s.to_str()).unwrap_or("unknown"); + let archive_path = history_dir.join(format!("{}.tar.gz", report_name)); + + let _ = std::process::Command::new("tar") + .args(["-czf", &archive_path.to_string_lossy(), "-C", + path.parent().unwrap().to_str().unwrap(), report_name]) + .status(); + + let _ = fs::remove_dir_all(&path); + archived += 1; + } + } + } + + Ok(json!({ + "content": [{ "type": "text", "text": json!({ + "success": true, + "archived": archived, + "kept": keep_last, + }).to_string() }], + "isError": false + })) + } + + async fn handle_promote_to_baseline(&self, args: &Value) -> Result { + let report_dir = args.get("report_dir") + .and_then(|v| v.as_str()) + .ok_or_else(|| anyhow::anyhow!("report_dir is required"))?; + + let metrics_path = std::path::Path::new(report_dir).join("metrics.json"); + let baseline_path = std::path::Path::new("config/baseline.json"); + + fs::copy(&metrics_path, &baseline_path)?; + + Ok(json!({ + "content": [{ "type": "text", "text": json!({ + "success": true, + "baseline_file": baseline_path.to_string_lossy(), + "source": metrics_path.to_string_lossy(), + }).to_string() }], + "isError": false + })) + } + + async fn handle_get_history(&self, args: &Value) -> Result { + let _limit = args.get("limit").and_then(|v| v.as_u64()).unwrap_or(50) as usize; + + let history_dir = std::path::Path::new(".mt5mcp_history"); + let mut history = Vec::new(); + + if history_dir.exists() { + for entry in fs::read_dir(history_dir)? { + if let Ok(entry) = entry { + let path = entry.path(); + if path.extension().map(|e| e == "tar.gz").unwrap_or(false) { + let name = path.file_stem() + .and_then(|s| s.to_str()) + .unwrap_or("unknown") + .to_string(); + + let metadata = entry.metadata()?; + let modified = metadata.modified()?; + let size = metadata.len(); + + history.push(json!({ + "name": name, + "path": path.to_string_lossy(), + "size": size, + "archived_at": modified.elapsed().map(|e| e.as_secs()).unwrap_or(0), + })); + } + } + } + } + + Ok(json!({ + "content": [{ "type": "text", "text": json!({ + "total_archived": history.len(), + "history": history, + }).to_string() }], + "isError": false + })) + } + + async fn handle_annotate_history(&self, args: &Value) -> Result { + let report_name = args.get("report_name") + .and_then(|v| v.as_str()) + .ok_or_else(|| anyhow::anyhow!("report_name is required"))?; + + let note = args.get("note") + .and_then(|v| v.as_str()) + .unwrap_or(""); + + let notes_path = std::path::Path::new(".mt5mcp_history").join("notes.json"); + + let mut notes: serde_json::Map = if notes_path.exists() { + serde_json::from_str(&fs::read_to_string(¬es_path)?)? + } else { + serde_json::Map::new() + }; + + notes.insert(report_name.to_string(), json!(note)); + fs::write(¬es_path, serde_json::to_string_pretty(¬es)?)?; + + Ok(json!({ + "content": [{ "type": "text", "text": json!({ + "success": true, + "report": report_name, + "note": note, + }).to_string() }], + "isError": false + })) + } } diff --git a/tests/__init__.py b/tests/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/test_rcp_server.sh b/tests/integration_test.sh similarity index 100% rename from test_rcp_server.sh rename to tests/integration_test.sh diff --git a/tests/integration_tests.rs b/tests/integration_tests.rs new file mode 100644 index 0000000..cac951e --- /dev/null +++ b/tests/integration_tests.rs @@ -0,0 +1,57 @@ +use std::fs; +use std::path::PathBuf; + +fn get_fixture_path(name: &str) -> PathBuf { + let mut path = PathBuf::from(env!("CARGO_MANIFEST_DIR")); + path.push("tests/fixtures"); + path.push(name); + path +} + +#[test] +fn test_fixtures_exist() { + let fixtures = vec![ + "sample_deals.csv", + "sample_report.htm", + "sample_report.htm.xml", + ]; + + for fixture in fixtures { + let path = get_fixture_path(fixture); + assert!(path.exists(), "Fixture {} should exist", fixture); + } +} + +#[test] +fn test_sample_deals_csv_format() { + let path = get_fixture_path("sample_deals.csv"); + let content = fs::read_to_string(path).expect("Should read sample_deals.csv"); + + // Check CSV has header and data rows + let lines: Vec<&str> = content.lines().collect(); + assert!(!lines.is_empty(), "CSV should have at least a header"); + + // Check for expected columns in header + let header = lines[0]; + assert!(header.contains("Time") || header.contains("time"), "Header should contain Time column"); +} + +#[test] +fn test_sample_report_html_format() { + let path = get_fixture_path("sample_report.htm"); + let content = fs::read_to_string(path).expect("Should read sample_report.htm"); + + // Check HTML structure + assert!(content.contains(" 0 - - -def test_load_deals_numeric_fields(deals): - for deal in deals: - assert isinstance(deal['profit'], float) - assert isinstance(deal['balance'], float) - assert isinstance(deal['volume'], float) - - -def test_monthly_pnl_groups_correctly(deals): - result = monthly_pnl(deals) - assert isinstance(result, list) - assert len(result) >= 1 - for entry in result: - assert 'month' in entry - assert 'pnl' in entry - assert 'trades' in entry - assert 'green' in entry - assert isinstance(entry['green'], bool) - - -def test_monthly_pnl_only_out_entries(deals): - """Only 'out' entries should be counted.""" - result = monthly_pnl(deals) - # All trades in fixture are closed, so at least one month should have trades - total_trades = sum(m['trades'] for m in result) - assert total_trades > 0 - - -def test_monthly_pnl_has_jan_and_feb(deals): - result = monthly_pnl(deals) - months = [m['month'] for m in result] - assert '2025-01' in months - assert '2025-02' in months - - -def test_reconstruct_dd_events_returns_list(deals): - metrics = {'net_profit': 38.0, 'max_dd_pct': 1.5} - result = reconstruct_dd_events(deals, metrics) - assert isinstance(result, list) - - -def test_reconstruct_dd_events_empty_on_no_deals(): - result = reconstruct_dd_events([], {}) - assert result == [] - - -def test_grid_depth_histogram_keys(deals): - hist = grid_depth_histogram(deals) - assert isinstance(hist, dict) - assert 'L1' in hist - assert 'L2' in hist - assert 'L3' in hist - assert 'L8+' in hist - - -def test_grid_depth_histogram_counts_layers(deals): - hist = grid_depth_histogram(deals) - # Fixture has Layer #1, #2, #3 comments - assert hist['L1'] > 0 - assert hist['L3'] > 0 - - -def test_top_losses_are_negative(deals): - losses = top_losses(deals) - assert isinstance(losses, list) - for loss in losses: - assert loss['loss_usd'] < 0 - - -def test_top_losses_sorted_ascending(deals): - losses = top_losses(deals) - if len(losses) >= 2: - assert losses[0]['loss_usd'] <= losses[1]['loss_usd'] - - -def test_loss_sequences_structure(deals): - seqs = loss_sequences(deals) - assert isinstance(seqs, list) - for seq in seqs: - assert 'length' in seq - assert 'total_loss' in seq - assert seq['total_loss'] < 0 - - -def test_build_summary_keys(deals): - metrics = {'net_profit': 38.0, 'max_dd_pct': 1.5, 'total_trades': 11, - 'profit_factor': 1.2, 'sharpe_ratio': 0.5, 'recovery_factor': 2.0} - monthly = monthly_pnl(deals) - dd = reconstruct_dd_events(deals, metrics) - summary = build_summary(metrics, monthly, dd) - - expected_keys = ['net_profit', 'profit_factor', 'max_dd_pct', 'sharpe_ratio', - 'total_trades', 'green_months', 'total_months', - 'worst_month', 'worst_month_pnl'] - for k in expected_keys: - assert k in summary, f"Missing key: {k}" - - -def test_build_summary_green_months(deals): - metrics = {'net_profit': 38.0, 'max_dd_pct': 1.5} - monthly = monthly_pnl(deals) - dd = reconstruct_dd_events(deals, metrics) - summary = build_summary(metrics, monthly, dd) - assert summary['green_months'] >= 0 - assert summary['total_months'] >= summary['green_months'] - - -# ── Utility helpers ──────────────────────────────────────────────────────────── - -def test_parse_dt_standard_format(): - dt = _parse_dt('2025.01.10 09:30:00') - assert dt is not None - assert dt.year == 2025 - assert dt.month == 1 - assert dt.day == 10 - assert dt.hour == 9 - - -def test_parse_dt_iso_format(): - dt = _parse_dt('2025-02-05 14:00:00') - assert dt is not None - assert dt.month == 2 - - -def test_parse_dt_invalid_returns_none(): - assert _parse_dt('') is None - assert _parse_dt('not-a-date') is None - - -def test_classify_exit_locking(): - assert _classify_exit('locking hedge', -50.0) == 'locking' - - -def test_classify_exit_cutloss(): - assert _classify_exit('cutloss fired', -20.0) == 'cutloss' - assert _classify_exit('cut loss', -20.0) == 'cutloss' - - -def test_classify_exit_tp_sl_by_profit(): - assert _classify_exit('Layer #1', 15.0) == 'tp' - assert _classify_exit('Layer #1', -10.0) == 'sl' - - -def test_lot_tier(): - assert _lot_tier(0.01) == '0.01' - assert _lot_tier(0.02) == '0.02-0.04' - assert _lot_tier(0.04) == '0.02-0.04' - assert _lot_tier(0.06) == '0.05-0.09' - assert _lot_tier(0.10) == '0.10-0.49' - assert _lot_tier(1.0) == '1.00+' - - -def test_session_for_hour(): - assert _session_for_hour(3) == 'asian' - assert _session_for_hour(9) == 'london' - assert _session_for_hour(14) == 'london_ny_overlap' - assert _session_for_hour(18) == 'new_york' - assert _session_for_hour(23) == 'off_hours' - - -# ── Position pairs ───────────────────────────────────────────────────────────── - -def test_position_pairs_count(deals): - pairs = position_pairs(deals) - assert isinstance(pairs, list) - assert len(pairs) > 0 - - -def test_position_pairs_hold_minutes(deals): - pairs = position_pairs(deals) - for p in pairs: - if p['hold_minutes'] is not None: - assert p['hold_minutes'] > 0 - - -def test_position_pairs_has_layer(deals): - pairs = position_pairs(deals) - layers = [p['layer'] for p in pairs if p['layer'] > 0] - assert len(layers) > 0 - - -def test_position_pairs_profit_nonzero(deals): - pairs = position_pairs(deals) - for p in pairs: - assert p['profit'] != 0.0 - - -# ── Cycle stats ──────────────────────────────────────────────────────────────── - -def test_cycle_stats_structure(deals): - result = cycle_stats(deals) - assert 'total_cycles' in result - assert 'win_rate' in result - assert 'avg_profit' in result - assert 'win_rate_by_depth' in result - - -def test_cycle_stats_total_cycles(deals): - result = cycle_stats(deals) - assert result['total_cycles'] > 0 - - -def test_cycle_stats_win_rate_range(deals): - result = cycle_stats(deals) - assert 0.0 <= result['win_rate'] <= 100.0 - - -def test_cycle_stats_empty(): - result = cycle_stats([]) - assert result['total_cycles'] == 0 - - -# ── Exit reason breakdown ────────────────────────────────────────────────────── - -def test_exit_reason_breakdown_structure(deals): - result = exit_reason_breakdown(deals) - assert isinstance(result, dict) - for reason, data in result.items(): - assert 'count' in data - assert 'total_pnl' in data - assert 'avg_pnl' in data - assert data['count'] > 0 - - -def test_exit_reason_breakdown_has_cutloss(deals): - result = exit_reason_breakdown(deals) - # fixture has 'cutloss' in comment for some deals - assert 'cutloss' in result - - -def test_exit_reason_breakdown_counts_match(deals): - result = exit_reason_breakdown(deals) - total_counted = sum(r['count'] for r in result.values()) - closed_with_pnl = [d for d in deals - if 'out' in d.get('entry', '').lower() and d.get('profit', 0.0) != 0.0] - assert total_counted == len(closed_with_pnl) - - -# ── Direction bias ───────────────────────────────────────────────────────────── - -def test_direction_bias_keys(deals): - result = direction_bias(deals) - assert isinstance(result, dict) - # fixture has both buy and sell - assert 'buy' in result - assert 'sell' in result - - -def test_direction_bias_win_rate_range(deals): - result = direction_bias(deals) - for d, s in result.items(): - assert 0.0 <= s['win_rate'] <= 100.0 - assert s['trades'] > 0 - - -def test_direction_bias_buy_profitable(deals): - result = direction_bias(deals) - # fixture: buy deals net positive - assert result['buy']['total_pnl'] > 0 - - -# ── Streak analysis ──────────────────────────────────────────────────────────── - -def test_streak_analysis_structure(deals): - result = streak_analysis(deals) - assert isinstance(result, dict) - for key in ('max_win_streak', 'max_loss_streak', 'current_streak', 'current_streak_type'): - assert key in result - - -def test_streak_analysis_nonnegative(deals): - result = streak_analysis(deals) - assert result['max_win_streak'] >= 0 - assert result['max_loss_streak'] >= 0 - assert result['current_streak'] >= 1 - - -def test_streak_analysis_type_valid(deals): - result = streak_analysis(deals) - assert result['current_streak_type'] in ('win', 'loss') - - -def test_streak_analysis_empty(): - assert streak_analysis([]) == {} - - -# ── Session breakdown ────────────────────────────────────────────────────────── - -def test_session_breakdown_structure(deals): - result = session_breakdown(deals) - assert isinstance(result, dict) - for session, data in result.items(): - assert 'trades' in data - assert 'win_rate' in data - assert 'total_pnl' in data - - -def test_session_breakdown_has_sessions(deals): - result = session_breakdown(deals) - # fixture has deals at 09:00, 10:00, 14:00, 15:00, 16:00 (London + London/NY) - # and 02:30, 03:15 (Asian), 20:00-21:30 (NY) - known_sessions = {'london', 'london_ny_overlap', 'asian', 'new_york'} - assert len(set(result.keys()) & known_sessions) >= 2 - - -def test_session_breakdown_win_rate_range(deals): - result = session_breakdown(deals) - for session, data in result.items(): - assert 0.0 <= data['win_rate'] <= 100.0 - - -# ── Weekday P/L ──────────────────────────────────────────────────────────────── - -def test_weekday_pnl_structure(deals): - result = weekday_pnl(deals) - assert isinstance(result, list) - for entry in result: - assert 'day' in entry - assert 'pnl' in entry - assert 'trades' in entry - assert 'win_rate' in entry - - -def test_weekday_pnl_day_names(deals): - result = weekday_pnl(deals) - valid_days = {'Monday', 'Tuesday', 'Wednesday', 'Thursday', 'Friday', 'Saturday', 'Sunday'} - for entry in result: - assert entry['day'] in valid_days - - -def test_weekday_pnl_has_results(deals): - result = weekday_pnl(deals) - assert len(result) >= 1 - - -# ── Hourly P/L ───────────────────────────────────────────────────────────────── - -def test_hourly_pnl_structure(deals): - result = hourly_pnl(deals) - assert isinstance(result, list) - for entry in result: - assert 'hour' in entry - assert 0 <= entry['hour'] <= 23 - assert 'pnl' in entry - assert 'trades' in entry - - -def test_hourly_pnl_has_results(deals): - result = hourly_pnl(deals) - assert len(result) >= 1 - - -# ── Concurrent peak ──────────────────────────────────────────────────────────── - -def test_concurrent_peak_structure(deals): - result = concurrent_peak(deals) - assert 'peak_open' in result - assert 'peak_time' in result - - -def test_concurrent_peak_at_least_one(deals): - result = concurrent_peak(deals) - assert result['peak_open'] >= 1 - - -def test_concurrent_peak_multi_layer(deals): - # fixture has a cycle where L2 and L3 open before close → peak >= 2 - result = concurrent_peak(deals) - assert result['peak_open'] >= 2 - - -# ── Volume profile ───────────────────────────────────────────────────────────── - -def test_volume_profile_structure(deals): - result = volume_profile(deals) - assert isinstance(result, list) - for entry in result: - assert 'lot_tier' in entry - assert 'pnl' in entry - assert 'trades' in entry - assert 'win_rate' in entry - - -def test_volume_profile_has_micro_lots(deals): - result = volume_profile(deals) - tiers = [e['lot_tier'] for e in result] - assert '0.01' in tiers - - -def test_volume_profile_win_rate_range(deals): - result = volume_profile(deals) - for entry in result: - assert 0.0 <= entry['win_rate'] <= 100.0 - - -# ── build_summary with new stats ─────────────────────────────────────────────── - -def test_build_summary_with_streak(deals): - metrics = {'net_profit': 38.0, 'max_dd_pct': 1.5} - monthly = monthly_pnl(deals) - dd = reconstruct_dd_events(deals, metrics) - streak = streak_analysis(deals) - summary = build_summary(metrics, monthly, dd, streak=streak) - assert 'max_win_streak' in summary - assert 'max_loss_streak' in summary - assert 'current_streak_type' in summary - - -def test_build_summary_with_bias(deals): - metrics = {'net_profit': 38.0, 'max_dd_pct': 1.5} - monthly = monthly_pnl(deals) - dd = reconstruct_dd_events(deals, metrics) - bias = direction_bias(deals) - summary = build_summary(metrics, monthly, dd, bias=bias) - assert 'buy_win_rate' in summary or 'sell_win_rate' in summary - - -def test_build_summary_with_cycles(deals): - metrics = {'net_profit': 38.0, 'max_dd_pct': 1.5} - monthly = monthly_pnl(deals) - dd = reconstruct_dd_events(deals, metrics) - cycles = cycle_stats(deals) - summary = build_summary(metrics, monthly, dd, cycles=cycles) - assert 'cycle_win_rate' in summary - assert 'total_cycles' in summary - - -# ── Strategy profiles ────────────────────────────────────────────────────────── - -def test_profiles_registry(): - """All expected strategy names are registered.""" - for name in ('generic', 'grid', 'scalper', 'trend', 'hedge'): - assert name in PROFILES - p = PROFILES[name] - assert 'name' in p - assert 'exit_keywords' in p - assert 'dd_cause_keywords' in p - assert 'cycle_group_by' in p - assert 'cycle_gap_min' in p - - -def test_profiles_depth_re(): - assert PROFILES['grid']['depth_re'] is not None - assert PROFILES['generic']['depth_re'] is None - assert PROFILES['scalper']['depth_re'] is None - assert PROFILES['trend']['depth_re'] is None - assert PROFILES['hedge']['depth_re'] is None - - -# ── _extract_depth ───────────────────────────────────────────────────────────── - -def test_extract_depth_grid_pattern(): - depth_re = PROFILES['grid']['depth_re'] - assert _extract_depth('Layer #3', depth_re) == 3 - assert _extract_depth('Layer #1', depth_re) == 1 - assert _extract_depth('layer 7', depth_re) == 7 - - -def test_extract_depth_no_pattern(): - assert _extract_depth('Layer #3', None) == 0 - assert _extract_depth('', None) == 0 - - -def test_extract_depth_no_match(): - assert _extract_depth('TP hit', PROFILES['grid']['depth_re']) == 0 - - -# ── _classify_exit with profiles ────────────────────────────────────────────── - -def test_classify_exit_grid_locking(): - assert _classify_exit('locking hedge', -50.0, PROFILES['grid']) == 'locking' - - -def test_classify_exit_grid_cutloss(): - assert _classify_exit('cutloss fired', -20.0, PROFILES['grid']) == 'cutloss' - - -def test_classify_exit_scalper_manual(): - assert _classify_exit('manual close', -5.0, PROFILES['scalper']) == 'manual' - - -def test_classify_exit_scalper_trailing(): - assert _classify_exit('trailing stop', 10.0, PROFILES['scalper']) == 'trailing' - - -def test_classify_exit_trend_breakeven(): - assert _classify_exit('breakeven stop', 0.5, PROFILES['trend']) == 'breakeven' - - -def test_classify_exit_trend_partial(): - assert _classify_exit('partial scale out', 15.0, PROFILES['trend']) == 'partial' - - -def test_classify_exit_hedge_net_close(): - assert _classify_exit('net close', -30.0, PROFILES['hedge']) == 'net_close' - - -def test_classify_exit_generic_fallback(): - """Generic profile has no keywords — falls back to profit sign.""" - assert _classify_exit('Layer #3 locking', -50.0, PROFILES['generic']) == 'sl' - assert _classify_exit('Layer #1', 15.0, PROFILES['generic']) == 'tp' - - -# ── _classify_dd_cause ──────────────────────────────────────────────────────── - -def test_classify_dd_cause_grid(): - assert _classify_dd_cause('locking total', PROFILES['grid']) == 'locking_cascade' - assert _classify_dd_cause('cutloss fired', PROFILES['grid']) == 'cutloss' - assert _classify_dd_cause('zombie exit', PROFILES['grid']) == 'zombie_exit' - - -def test_classify_dd_cause_generic_unknown(): - assert _classify_dd_cause('locking total', PROFILES['generic']) == 'unknown' - assert _classify_dd_cause('', PROFILES['generic']) == 'unknown' - - -def test_classify_dd_cause_scalper_stop(): - assert _classify_dd_cause('sl hit', PROFILES['scalper']) == 'stop_loss' - - -def test_classify_dd_cause_trend_whipsaw(): - assert _classify_dd_cause('stop loss', PROFILES['trend']) == 'whipsaw' - - -# ── depth_histogram with profiles ───────────────────────────────────────────── - -def test_depth_histogram_grid_returns_layers(deals): - result = depth_histogram(deals, PROFILES['grid']) - assert isinstance(result, dict) - assert 'L1' in result - assert result['L1'] > 0 - - -def test_depth_histogram_generic_returns_empty(deals): - """Generic profile has no depth_re → empty dict.""" - result = depth_histogram(deals, PROFILES['generic']) - assert result == {} - - -def test_depth_histogram_scalper_returns_empty(deals): - result = depth_histogram(deals, PROFILES['scalper']) - assert result == {} - - -def test_grid_depth_histogram_is_alias(deals): - """grid_depth_histogram must equal depth_histogram with grid profile.""" - assert grid_depth_histogram(deals) == depth_histogram(deals, PROFILES['grid']) - - -# ── cycle_stats with profiles ───────────────────────────────────────────────── - -def test_cycle_stats_grid_profile(deals): - result = cycle_stats(deals, PROFILES['grid']) - assert result['total_cycles'] > 0 - assert 0.0 <= result['win_rate'] <= 100.0 - - -def test_cycle_stats_scalper_profile(deals): - """Scalper uses magic-only grouping and 10-min gap.""" - result = cycle_stats(deals, PROFILES['scalper']) - assert 'total_cycles' in result - assert result['total_cycles'] > 0 - - -def test_cycle_stats_generic_profile(deals): - result = cycle_stats(deals, PROFILES['generic']) - assert 'total_cycles' in result - - -def test_cycle_stats_scalper_vs_grid_differ(deals): - """Different grouping rules can produce different cycle counts.""" - grid_result = cycle_stats(deals, PROFILES['grid']) - scalper_result = cycle_stats(deals, PROFILES['scalper']) - # Both must be valid; counts may differ due to grouping - assert grid_result['total_cycles'] >= 0 - assert scalper_result['total_cycles'] >= 0 - - -# ── exit_reason_breakdown with profiles ─────────────────────────────────────── - -def test_exit_reason_breakdown_grid(deals): - result = exit_reason_breakdown(deals, PROFILES['grid']) - assert 'cutloss' in result # fixture has "cutloss" in comments - - -def test_exit_reason_breakdown_generic_only_tp_sl(deals): - """Generic profile has no keywords → only 'tp' and 'sl' keys.""" - result = exit_reason_breakdown(deals, PROFILES['generic']) - for reason in result: - assert reason in ('tp', 'sl'), f"Unexpected reason '{reason}' from generic profile" - - -def test_exit_reason_breakdown_scalper_keywords(deals): - """Scalper profile recognises 'cutloss' comment as 'manual' (not 'cutloss').""" - result = exit_reason_breakdown(deals, PROFILES['scalper']) - # 'cutloss' is not a scalper keyword → falls back to profit-sign → 'sl' - assert 'cutloss' not in result - - -def test_exit_reason_breakdown_counts_sum(deals): - """Total count must equal number of non-zero closed deals, regardless of profile.""" - closed = [d for d in deals - if 'out' in d.get('entry', '').lower() and d.get('profit', 0.0) != 0.0] - for profile in PROFILES.values(): - result = exit_reason_breakdown(deals, profile) - assert sum(r['count'] for r in result.values()) == len(closed) - - -# ── reconstruct_dd_events with profiles ─────────────────────────────────────── - -def test_dd_events_cause_generic_unknown(deals): - """Generic profile → all causes must be 'unknown'.""" - metrics = {'net_profit': 38.0, 'max_dd_pct': 1.5} - events = reconstruct_dd_events(deals, metrics, PROFILES['generic']) - for ev in events: - assert ev['cause'] == 'unknown' - - -def test_dd_events_cause_grid_classified(deals): - """Grid profile → cause is classified from comment keywords.""" - metrics = {'net_profit': 38.0, 'max_dd_pct': 5.0} - events = reconstruct_dd_events(deals, metrics, PROFILES['grid']) - valid = {'locking_cascade', 'cutloss', 'zombie_exit', 'spike_entry', 'unknown'} - for ev in events: - assert ev['cause'] in valid - - -# ── build_summary strategy field ────────────────────────────────────────────── - -def test_build_summary_strategy_field(deals): - metrics = {'net_profit': 38.0, 'max_dd_pct': 1.5} - monthly = monthly_pnl(deals) - dd = reconstruct_dd_events(deals, metrics) - summary = build_summary(metrics, monthly, dd, strategy='scalper') - assert summary['strategy'] == 'scalper' - - -def test_build_summary_no_strategy_field(deals): - metrics = {'net_profit': 38.0, 'max_dd_pct': 1.5} - monthly = monthly_pnl(deals) - dd = reconstruct_dd_events(deals, metrics) - summary = build_summary(metrics, monthly, dd) - assert 'strategy' not in summary diff --git a/tests/test_extract.py b/tests/test_extract.py deleted file mode 100644 index dd7375b..0000000 --- a/tests/test_extract.py +++ /dev/null @@ -1,146 +0,0 @@ -"""Tests for analytics/extract.py — runs without MT5 or Wine.""" - -import json -import os -import sys -import tempfile -from pathlib import Path - -import pytest - -FIXTURES = Path(__file__).parent / 'fixtures' -sys.path.insert(0, str(Path(__file__).parent.parent)) - -from analytics.extract import ( - detect_format, parse_html, parse_xml, write_outputs, - _parse_metrics_html, _parse_deals_html, -) - - -def test_detect_format_html(): - assert detect_format(str(FIXTURES / 'sample_report.htm')) == 'html' - - -def test_detect_format_xml(): - assert detect_format(str(FIXTURES / 'sample_report.htm.xml')) == 'xml' - - -# HTML parsing is tested via internal functions to avoid the UTF-16 decode dance -# (read_text tries UTF-16 first, which silently garbles plain UTF-8/ASCII files). -# The fixture is used for format-detection only. - -HTML_TEXT = """ - - - - - - - - - - -
Net profit1234.56
Profit factor1.25
Maximal drawdown500.00 (5.00%)
Sharpe Ratio0.75
Total trades150
Recovery factor2.50
Profit trades (% of total)90 (60.00%)
Gross profit2000.00
Gross loss-765.44
- - - - -
Deal TimeTypeDirectionVolumePriceS/LT/PProfitBalanceCommentOrderMagicEntry
2025.01.10 09:30:00buyout0.011915.000015.0010015.00Layer #1100112345out
2025.02.05 14:00:00sellout0.011945.0000-15.0010020.00Layer #1100512345out
-""" - - -@pytest.fixture -def html_report_path(tmp_path): - """Write HTML fixture as UTF-16 LE with BOM so read_text() decodes it correctly.""" - p = tmp_path / 'report.htm' - p.write_bytes(b'\xff\xfe' + HTML_TEXT.encode('utf-16-le')) - return str(p) - - -def test_parse_html_returns_metrics(html_report_path): - metrics, _ = parse_html(html_report_path) - assert isinstance(metrics, dict) - assert 'net_profit' in metrics - assert metrics['net_profit'] == pytest.approx(1234.56) - assert metrics['total_trades'] == 150 - - -def test_parse_html_returns_deals(html_report_path): - _, deals = parse_html(html_report_path) - assert isinstance(deals, list) - assert len(deals) >= 1 - deal = deals[0] - assert 'profit' in deal - assert 'balance' in deal - - -def test_parse_metrics_html_directly(): - """Test HTML metric extraction without encoding layer.""" - metrics = _parse_metrics_html(HTML_TEXT) - assert metrics['net_profit'] == pytest.approx(1234.56) - assert metrics['profit_factor'] == pytest.approx(1.25) - assert metrics['max_dd_pct'] == pytest.approx(5.00) - assert metrics['total_trades'] == 150 - - -def test_parse_deals_html_directly(): - """Test HTML deal extraction without encoding layer.""" - deals = _parse_deals_html(HTML_TEXT) - assert len(deals) == 2 - assert float(deals[0]['profit']) == pytest.approx(15.00) - assert float(deals[1]['profit']) == pytest.approx(-15.00) - - -def test_parse_xml_returns_metrics(): - metrics, deals = parse_xml(str(FIXTURES / 'sample_report.htm.xml')) - assert isinstance(metrics, dict) - assert 'net_profit' in metrics - assert metrics['net_profit'] == pytest.approx(1234.56) - assert metrics['total_trades'] == 150 - - -def test_parse_xml_returns_deals(): - metrics, deals = parse_xml(str(FIXTURES / 'sample_report.htm.xml')) - assert isinstance(deals, list) - assert len(deals) >= 1 - deal = deals[0] - assert deal.get('profit') is not None - - -def test_write_outputs_creates_files(): - metrics = {'net_profit': 100.0, 'total_trades': 5} - deals = [ - {'time': '2025.01.10', 'type': 'buy', 'direction': 'out', 'volume': '0.01', - 'price': '1900', 'sl': '0', 'tp': '0', 'profit': '10.00', - 'balance': '10010', 'comment': '', 'order': '1', 'magic': '1', 'entry': 'out'}, - ] - with tempfile.TemporaryDirectory() as tmp: - paths = write_outputs(metrics, deals, tmp) - assert Path(paths['metrics']).exists() - assert Path(paths['deals_csv']).exists() - assert Path(paths['deals_json']).exists() - # Verify metrics.json content - with open(paths['metrics']) as f: - saved = json.load(f) - assert saved['net_profit'] == 100.0 - - -def test_parse_html_skips_balance_rows(): - """Rows with type='balance' should be filtered out.""" - html = """ - - - - -
Deal TimeTypeDirectionVolumePriceS/LT/PProfitBalanceCommentOrderMagicEntry
2025.01.10 09:30:00balance000001000000
2025.01.10 10:00:00buyout0.011910005.0010005Layer #111out
- """ - import tempfile, os - with tempfile.NamedTemporaryFile(mode='w', suffix='.htm', delete=False) as f: - f.write(html) - path = f.name - try: - _, deals = parse_html(path) - types = [d.get('type', '').lower() for d in deals] - assert 'balance' not in types - finally: - os.unlink(path)