LEGSTECH Optimizer
51a73c953b
- AIReasoner default model now claude-opus-4-7 (was sonnet-4-6); pipeline reads ai.model from config and passes to constructor. The reasoner class attribute is also Opus 4.7 so any caller without explicit model defaults to the hackathon model - README hero gains a "Built with Claude Opus 4.7" badge + a line linking to the Cerebral Valley × Anthropic hackathon page - New SUBMISSION.md — copy-paste blocks for every field of the submission form: name, taglines, short / long description, "how Claude is used" detail, GitHub URL, demo instructions, tags, tech stack, team, license, and a final pre-submission checklist
1231 lines
55 KiB
Python
1231 lines
55 KiB
Python
"""
|
||
optimizer/pipeline.py
|
||
The Smart 3-Phase Optimization Pipeline.
|
||
|
||
Replaces optimizer_loop.py as the core orchestration engine.
|
||
Receives a SessionConfig → runs Phase 1, 2, 3 → emits SocketIO events.
|
||
|
||
Architecture:
|
||
Phase 1: Broad Discovery (LHS samples, 20–30 runs)
|
||
Phase 2: Deep Refinement (neighbor search around top 3)
|
||
Phase 3: Validation (OOS backtest + sensitivity)
|
||
Decision: RECOMMENDED / RISKY / NOT_RELIABLE + .set file download
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import hashlib
|
||
import os
|
||
import random
|
||
import time
|
||
from datetime import datetime
|
||
from pathlib import Path
|
||
from typing import Optional
|
||
|
||
import yaml
|
||
from loguru import logger
|
||
|
||
from ea.registry import EARegistry
|
||
from ea.schema import ParameterSchema
|
||
from mt5.ini_builder import IniBuilder
|
||
from mt5.runner import MT5Runner
|
||
from mt5.report_parser import ReportParser
|
||
from data.models import Run
|
||
from data.store import DataStore
|
||
from reports.writer import ReportWriter
|
||
|
||
from optimizer.session_config import SessionConfig
|
||
from optimizer.lhs_sampler import LatinHypercubeSampler
|
||
from optimizer.result_ranker import ResultRanker, RankedResult
|
||
from optimizer.budget import BudgetManager
|
||
|
||
import pandas as pd
|
||
|
||
from analysis.equity_curve import EquityCurveAnalyzer
|
||
from analysis.time_performance import TimePerformanceAnalyzer
|
||
from analysis.ai_reasoner import AIReasoner
|
||
from analysis.ai_reasoner_config import load_api_key
|
||
|
||
BASE_DIR = Path(__file__).parent.parent
|
||
RUNS_DIR = BASE_DIR / "runs"
|
||
DB_PATH = BASE_DIR / "optimizer.db"
|
||
|
||
|
||
class OptimizationPipeline:
|
||
"""
|
||
3-phase autonomous optimization pipeline. Run in a background thread.
|
||
|
||
Usage:
|
||
pipeline = OptimizationPipeline("config.yaml", socketio, reports_dir)
|
||
pipeline.configure(session_config)
|
||
thread = threading.Thread(target=pipeline.run, daemon=True)
|
||
thread.start()
|
||
"""
|
||
|
||
def __init__(self, config_path: str, socketio, reports_dir: Path):
|
||
self.config_path = config_path
|
||
self.socketio = socketio
|
||
self.reports_dir = reports_dir
|
||
|
||
with open(config_path) as f:
|
||
self.cfg = yaml.safe_load(f)
|
||
|
||
# Runtime state
|
||
self.session: Optional[SessionConfig] = None
|
||
self.running = False
|
||
self._stop_flag = False
|
||
self._phase = "idle"
|
||
self._run_count = 0
|
||
self._total_runs = 0
|
||
self.best_result: Optional[RankedResult] = None
|
||
self.phase1_results: list[RankedResult] = []
|
||
self.phase2_results: list[RankedResult] = []
|
||
self.final_result: Optional[RankedResult] = None
|
||
self.verdict: Optional[str] = None
|
||
self.best_set_path: Optional[Path] = None
|
||
self.run_start_ts: Optional[float] = None
|
||
|
||
# AI & analysis state
|
||
self._ai_reasoner: Optional[AIReasoner] = None
|
||
self._ai_insights: list[dict] = []
|
||
self._run_findings: dict[str, list] = {}
|
||
self._run_insights: dict[str, dict] = {}
|
||
self._baseline_metrics: Optional[dict] = None
|
||
|
||
# Running best (updated each run so status can serve it live)
|
||
self._live_best: Optional[RankedResult] = None
|
||
# All completed runs (for history restoration)
|
||
self._completed_runs: list[dict] = []
|
||
|
||
# Live activity logs — persisted in-memory so a dashboard refresh during
|
||
# a run can replay them via /api/thinking, /api/param_changes, /api/validation.
|
||
self._thinking_log: list[dict] = []
|
||
self._param_changes: list[dict] = []
|
||
self._validation_log: list[dict] = []
|
||
self._early_term: Optional[dict] = None
|
||
|
||
# ── Public API ────────────────────────────────────────────────────────────
|
||
|
||
def configure(self, session: SessionConfig) -> None:
|
||
self.session = session
|
||
|
||
def reload_config(self) -> dict:
|
||
"""
|
||
Re-read config.yaml and apply runtime-mutable changes to the live
|
||
pipeline. Called from the Settings save handler so users can tune
|
||
AI model / timeout / thresholds mid-run without restarting.
|
||
|
||
Returns a dict of what actually changed, for the API response.
|
||
"""
|
||
try:
|
||
with open(self.config_path, encoding="utf-8") as f:
|
||
new_cfg = yaml.safe_load(f) or {}
|
||
except Exception as e:
|
||
return {"ok": False, "error": str(e)}
|
||
|
||
changed = {}
|
||
# AI settings — model, timeout, enabled — applied to the live reasoner
|
||
old_ai = self.cfg.get("ai", {})
|
||
new_ai = new_cfg.get("ai", {})
|
||
if self._ai_reasoner is not None:
|
||
for fld in ("model", "timeout_seconds", "enabled"):
|
||
if old_ai.get(fld) != new_ai.get(fld):
|
||
changed[f"ai.{fld}"] = new_ai.get(fld)
|
||
# Apply to live reasoner
|
||
try:
|
||
if "model" in new_ai and new_ai["model"]:
|
||
self._ai_reasoner.MODEL = new_ai["model"]
|
||
if "timeout_seconds" in new_ai:
|
||
self._ai_reasoner.TIMEOUT = int(new_ai["timeout_seconds"])
|
||
# API key swap: if a new full key was saved, rebuild reasoner
|
||
old_key = (old_ai.get("anthropic_api_key") or "").strip()
|
||
new_key = (new_ai.get("anthropic_api_key") or "").strip()
|
||
if new_key and new_key != old_key and not new_key.startswith("${") and len(new_key) >= 30:
|
||
self._ai_reasoner = AIReasoner(api_key=new_key)
|
||
changed["ai.api_key"] = "rotated"
|
||
except Exception as e:
|
||
logger.warning(f"reload_config: AI hot-reload failed: {e}")
|
||
|
||
# Threshold changes — applied to ResultRanker on next make_result.
|
||
# The thresholds module reads from config at import; we surface the
|
||
# change so the UI can confirm.
|
||
old_thr = self.cfg.get("thresholds", {})
|
||
new_thr = new_cfg.get("thresholds", {})
|
||
for fld in ("min_trades", "min_profit_factor", "min_calmar"):
|
||
if old_thr.get(fld) != new_thr.get(fld):
|
||
changed[f"thresholds.{fld}"] = new_thr.get(fld)
|
||
|
||
self.cfg = new_cfg
|
||
if changed:
|
||
self._log("info", f"⚙ Settings hot-reloaded: {', '.join(changed.keys())}")
|
||
self._emit("settings_reloaded", {"changed": changed})
|
||
return {"ok": True, "changed": changed}
|
||
|
||
def stop(self) -> None:
|
||
self._stop_flag = True
|
||
self._emit("status_change", {"state": "stopping"})
|
||
self._emit_early_termination(
|
||
reason_code="user_stop",
|
||
message="User requested stop. Finishing current run and exiting.",
|
||
details={"phase": self._phase},
|
||
)
|
||
|
||
def get_status(self) -> dict:
|
||
elapsed = int(time.time() - self.run_start_ts) if self.run_start_ts else 0
|
||
# Use final_result if set, otherwise best seen so far during the run
|
||
best = self.final_result or self._live_best
|
||
return {
|
||
"state": "running" if self.running else "idle",
|
||
"phase": self._phase,
|
||
"run_count": self._run_count,
|
||
"total_runs": self._total_runs,
|
||
"best_score": round(best.score, 4) if best else 0.0,
|
||
"verdict": self.verdict,
|
||
"elapsed_s": elapsed,
|
||
"ea_name": self.session.ea_name if self.session else "",
|
||
"symbol": self.session.symbol if self.session else "",
|
||
"timeframe": self.session.timeframe if self.session else "",
|
||
"has_insight": bool(self._ai_insights),
|
||
"insight_count": len(self._ai_insights),
|
||
"latest_insight": self._ai_insights[-1] if self._ai_insights else None,
|
||
"best_net_profit": round(best.net_profit, 2) if best else None,
|
||
"best_calmar": round(best.calmar, 3) if best else None,
|
||
"best_pf": round(best.profit_factor, 3) if best else None,
|
||
"best_win_rate": round(best.win_rate * 100, 1) if best else None,
|
||
"best_max_drawdown": round(best.max_drawdown * 100, 2) if best else None,
|
||
"best_total_trades": best.total_trades if best else None,
|
||
}
|
||
|
||
# ── Main entry point ──────────────────────────────────────────────────────
|
||
|
||
def run(self) -> None:
|
||
self.running = True
|
||
self._stop_flag = False
|
||
self.run_start_ts = time.time()
|
||
|
||
try:
|
||
self._run_pipeline()
|
||
except Exception as e:
|
||
logger.exception(f"Pipeline crashed: {e}")
|
||
self._emit("error", {"msg": f"Pipeline error: {e}"})
|
||
finally:
|
||
self.running = False
|
||
self._phase = "idle"
|
||
self._emit("status_change", {"state": "idle"})
|
||
|
||
def _run_pipeline(self) -> None:
|
||
cfg = self.session
|
||
self._total_runs = cfg.total_budget_runs
|
||
|
||
self._emit("status_change", {"state": "running", "phase": "setup"})
|
||
self._log("info", f"🚀 Smart Optimizer started — {cfg.ea_name} | {cfg.symbol} | {cfg.timeframe}")
|
||
self._log("info", f"📋 Budget: {cfg.budget_minutes} min | Samples: {cfg.phase1_samples} Phase1 + {cfg.phase2_samples} Phase2 + {cfg.phase3_samples} Phase3")
|
||
|
||
# ── Build components ─────────────────────────────────────────────────
|
||
reg = EARegistry(self.config_path)
|
||
profile = reg.get(cfg.ea_name)
|
||
schema = reg.get_schema(profile)
|
||
|
||
# Apply user's param selection if specified
|
||
if cfg.selected_params:
|
||
for p in schema.parameters.values():
|
||
if p.type != "fixed":
|
||
p.optimize = p.name in cfg.selected_params
|
||
|
||
builder = IniBuilder(self.config_path, schema=schema)
|
||
runner = MT5Runner(self.config_path)
|
||
parser = ReportParser()
|
||
store = DataStore(DB_PATH, RUNS_DIR)
|
||
writer = ReportWriter(self.reports_dir)
|
||
sampler = LatinHypercubeSampler(seed=int(time.time()))
|
||
ranker = ResultRanker(weights=cfg.scoring_weights)
|
||
budget = BudgetManager(cfg.budget_minutes)
|
||
|
||
# Initialize AI reasoning layer (model + timeout from config)
|
||
api_key = load_api_key(self.config_path)
|
||
ai_cfg = (self.cfg.get("ai") or {})
|
||
model = ai_cfg.get("model") or "claude-opus-4-7"
|
||
self._ai_reasoner = AIReasoner(api_key=api_key, model=model)
|
||
if "timeout_seconds" in ai_cfg:
|
||
try: self._ai_reasoner.TIMEOUT = int(ai_cfg["timeout_seconds"])
|
||
except Exception: pass
|
||
# Stream Claude's reasoning tokens to the dashboard as they arrive.
|
||
# The Live AI Thinking Feed listens for `ai_thinking_chunk` events.
|
||
try:
|
||
self._ai_reasoner.set_stream_callback(self._stream_ai_chunk)
|
||
except Exception:
|
||
pass
|
||
self._ai_insights.clear()
|
||
self._run_findings.clear()
|
||
self._run_insights.clear()
|
||
# Reset live-activity logs for the new run
|
||
self._thinking_log = []
|
||
self._param_changes = []
|
||
self._validation_log = []
|
||
self._early_term = None
|
||
|
||
budget.start()
|
||
|
||
# ── Phase 1: Broad Discovery ─────────────────────────────────────────
|
||
if self._stop_flag:
|
||
return
|
||
|
||
self._phase = "phase1"
|
||
self._emit("phase_start", {"phase": "phase1", "total": cfg.phase1_samples})
|
||
self._emit_thinking(
|
||
f"Starting broad discovery with {cfg.phase1_samples} Latin-Hypercube samples. "
|
||
f"Scanning parameter space to find profitable regions before focused refinement.",
|
||
kind="reasoning",
|
||
)
|
||
self._log("info", f"━━ Phase 1: Broad Discovery ({cfg.phase1_samples} configurations) ━━")
|
||
|
||
samples = sampler.sample(schema, cfg.phase1_samples)
|
||
phase1_raw: list[RankedResult] = []
|
||
|
||
for i, params in enumerate(samples):
|
||
if self._stop_flag:
|
||
break
|
||
|
||
run_id = f"p1_{i+1:02d}_{datetime.utcnow().strftime('%H%M%S')}"
|
||
self._log("info", f"[{i+1}/{cfg.phase1_samples}] Testing configuration {i+1}...")
|
||
|
||
t0 = time.time()
|
||
result = self._execute_run(
|
||
run_id, params, cfg.train_start, cfg.train_end,
|
||
"phase1", builder, runner, parser, store, writer, ranker, profile
|
||
)
|
||
budget.record_run(time.time() - t0)
|
||
phase1_raw.append(result)
|
||
self._run_count += 1
|
||
|
||
# Track live best and completed runs for status/history APIs
|
||
if self._live_best is None or result.score > self._live_best.score:
|
||
self._live_best = result
|
||
run_dict = self._make_run_dict(run_id, result, "phase1")
|
||
self._completed_runs.append(run_dict)
|
||
|
||
# Emit progress after each run
|
||
self._emit("run_complete", {
|
||
**run_dict,
|
||
"run_number": i + 1,
|
||
"total": cfg.phase1_samples,
|
||
"progress_pct": round((i + 1) / cfg.phase1_samples * 100),
|
||
"budget_summary": budget.summary(),
|
||
})
|
||
|
||
if budget.is_exhausted():
|
||
self._log("warning", "⏱ Time budget reached during Phase 1")
|
||
break
|
||
|
||
# Rank Phase 1 results
|
||
self.phase1_results = ranker.rank(phase1_raw)
|
||
top5 = ranker.top_n(self.phase1_results, 5)
|
||
|
||
n_passing = sum(1 for r in self.phase1_results if r.passing)
|
||
self._log("info" if n_passing > 0 else "warning",
|
||
f"Phase 1 complete: {n_passing}/{len(self.phase1_results)} profitable configurations found"
|
||
)
|
||
|
||
# Emit Phase 1 summary for checkpoint UI
|
||
self._emit("phase1_complete", {
|
||
"total_tested": len(self.phase1_results),
|
||
"n_passing": n_passing,
|
||
"top_results": [self._result_to_dict(r) for r in top5],
|
||
})
|
||
|
||
if not top5:
|
||
self._emit("no_profitable_config", {
|
||
"msg": (
|
||
"Phase 1 found no profitable configuration. "
|
||
"Suggestions: try a different date range, check EA settings, "
|
||
"or try a different timeframe."
|
||
)
|
||
})
|
||
self._emit_early_termination(
|
||
reason_code="no_profit",
|
||
message="Optimization stopped early: Phase 1 found no profitable configuration.",
|
||
details={
|
||
"phase": "phase1",
|
||
"total_tested": len(self.phase1_results),
|
||
"suggestions": [
|
||
"Try a different date range",
|
||
"Check EA settings for issues",
|
||
"Try a different timeframe",
|
||
],
|
||
},
|
||
)
|
||
self._emit_thinking(
|
||
"No profitable configuration found across the broad scan. "
|
||
"Strategy is unstable under current EA settings — stopping optimization.",
|
||
kind="warning",
|
||
)
|
||
self._log("error", "❌ No profitable configuration found in Phase 1. Stopping.")
|
||
return
|
||
|
||
if self._stop_flag:
|
||
return
|
||
|
||
# ── Phase 2: Refinement (AI-Guided or Random Neighbor) ───────────────
|
||
if not budget.can_fit(3):
|
||
self._log("warning", "⏱ Not enough budget for Phase 2 — using Phase 1 winner directly")
|
||
self.final_result = top5[0]
|
||
|
||
elif cfg.autonomous_mode and self._ai_reasoner and self._ai_reasoner.enabled:
|
||
# ── AI-Guided Autonomous Loop ─────────────────────────────────────
|
||
self._phase = "phase2"
|
||
self._emit("phase_start", {
|
||
"phase": "phase2",
|
||
"total": cfg.autonomous_max_iterations,
|
||
"mode": "autonomous",
|
||
})
|
||
self._emit_thinking(
|
||
f"Phase 1 found {len(top5)} promising candidates. Best Calmar is "
|
||
f"{top5[0].calmar:.2f}. Switching to autonomous AI loop — I'll read each "
|
||
f"result, decide what parameters to change, and iterate toward the targets.",
|
||
kind="reasoning",
|
||
)
|
||
self._log("info",
|
||
f"━━ Phase 2: Autonomous AI Loop "
|
||
f"(up to {cfg.autonomous_max_iterations} iterations) ━━"
|
||
)
|
||
|
||
from optimizer.ai_guided_loop import AIGuidedLoop
|
||
loop = AIGuidedLoop(
|
||
pipeline=self, schema=schema, cfg=cfg,
|
||
builder=builder, runner=runner, parser=parser,
|
||
store=store, writer=writer, ranker=ranker,
|
||
profile=profile, budget=budget,
|
||
)
|
||
targets = {
|
||
"min_profit_factor": cfg.target_profit_factor,
|
||
"max_drawdown_pct": cfg.target_max_drawdown_pct,
|
||
"min_calmar": cfg.target_min_calmar,
|
||
}
|
||
loop.run(
|
||
seed_results=self.phase1_results,
|
||
max_iterations=cfg.autonomous_max_iterations,
|
||
targets=targets,
|
||
)
|
||
|
||
self.phase2_results = loop.all_results
|
||
all_candidates = [r for r in (self.phase1_results + loop.all_results) if r.passing]
|
||
all_ranked = ranker.rank(all_candidates) if all_candidates else ranker.rank(self.phase1_results)
|
||
self.final_result = all_ranked[0] if all_ranked else top5[0]
|
||
|
||
self._emit("phase2_complete", {
|
||
"best_run_id": self.final_result.run_id,
|
||
"best_score": round(self.final_result.score, 4),
|
||
"best_profit": round(self.final_result.net_profit, 2),
|
||
"best_calmar": round(self.final_result.calmar, 3),
|
||
"mode": "autonomous",
|
||
"iterations": len(loop.all_results),
|
||
})
|
||
self._log("info",
|
||
f"AI Loop complete. Best: {self.final_result.run_id} "
|
||
f"(PF={self.final_result.profit_factor:.2f}, "
|
||
f"profit=${self.final_result.net_profit:.0f}, "
|
||
f"calmar={self.final_result.calmar:.2f})"
|
||
)
|
||
|
||
else:
|
||
# ── Original Random-Neighbor Phase 2 ─────────────────────────────
|
||
self._phase = "phase2"
|
||
self._emit("phase_start", {"phase": "phase2", "total": cfg.phase2_samples})
|
||
self._emit_thinking(
|
||
f"Phase 2 starting in classic mode: sampling {cfg.phase2_samples} neighbors "
|
||
f"around the top {min(3, len(top5))} Phase-1 winners (no AI loop).",
|
||
kind="info",
|
||
)
|
||
self._log("info", f"━━ Phase 2: Deep Refinement (refining top {min(3, len(top5))} configs) ━━")
|
||
|
||
top3 = top5[:3]
|
||
phase2_raw = []
|
||
neighbors_per = cfg.phase2_samples // max(1, len(top3))
|
||
|
||
for rank_i, base_result in enumerate(top3):
|
||
if self._stop_flag:
|
||
break
|
||
|
||
self._log("info", f" Refining config #{rank_i+1}: {base_result.run_id}")
|
||
neighbors = sampler.sample_neighbors(
|
||
base_result.params, schema,
|
||
n_neighbors=neighbors_per, step_pct=0.20
|
||
)
|
||
|
||
for j, params in enumerate(neighbors):
|
||
if self._stop_flag or budget.is_exhausted():
|
||
break
|
||
|
||
run_id = f"p2_{rank_i+1}_{j+1:02d}_{datetime.utcnow().strftime('%H%M%S')}"
|
||
t0 = time.time()
|
||
result = self._execute_run(
|
||
run_id, params, cfg.train_start, cfg.train_end,
|
||
"phase2", builder, runner, parser, store, writer, ranker, profile
|
||
)
|
||
budget.record_run(time.time() - t0)
|
||
phase2_raw.append(result)
|
||
self._run_count += 1
|
||
|
||
run_dict_p2 = self._make_run_dict(run_id, result, "phase2")
|
||
self._completed_runs.append(run_dict_p2)
|
||
if self._live_best is None or result.score > self._live_best.score:
|
||
self._live_best = result
|
||
self._emit("run_complete", {
|
||
**run_dict_p2,
|
||
"progress_pct": round(self._run_count / self._total_runs * 100),
|
||
})
|
||
|
||
all_results = list(self.phase1_results) + ranker.rank(phase2_raw)
|
||
all_ranked = ranker.rank(
|
||
[r for r in all_results if r.passing]
|
||
or list(self.phase1_results)
|
||
)
|
||
self.phase2_results = ranker.rank(phase2_raw)
|
||
self.final_result = all_ranked[0] if all_ranked else top5[0]
|
||
|
||
self._emit("phase2_complete", {
|
||
"best_run_id": self.final_result.run_id,
|
||
"best_score": round(self.final_result.score, 4),
|
||
"best_profit": round(self.final_result.net_profit, 2),
|
||
"best_calmar": round(self.final_result.calmar, 3),
|
||
})
|
||
self._log("info",
|
||
f"Phase 2 complete. Best config: {self.final_result.run_id} "
|
||
f"(profit=${self.final_result.net_profit:.0f}, calmar={self.final_result.calmar:.2f})"
|
||
)
|
||
|
||
if self._stop_flag:
|
||
return
|
||
|
||
# ── Phase 3: Validation ───────────────────────────────────────────────
|
||
if not budget.can_fit(2):
|
||
self._log("warning", "⏱ Not enough budget for Phase 3 validation — skipping OOS test")
|
||
self._emit_early_termination(
|
||
reason_code="budget_exhausted",
|
||
message="Skipping validation: not enough time budget remaining.",
|
||
details={"phase": "phase3"},
|
||
)
|
||
self.verdict = "RISKY"
|
||
oos_result = None
|
||
else:
|
||
self._phase = "phase3"
|
||
self._emit("phase_start", {"phase": "phase3", "total": cfg.phase3_samples})
|
||
|
||
# Count the validation runs we intend to run so progress makes sense
|
||
opts = schema.optimizable()
|
||
plan_sens = bool(opts) and budget.can_fit(2)
|
||
planned_runs = 1 + (2 if plan_sens else 0) # 1 OOS + 2 sens
|
||
self._emit("validation_start", {
|
||
"best_run_id": self.final_result.run_id,
|
||
"planned_runs": planned_runs,
|
||
"oos_start": cfg.val_start,
|
||
"oos_end": cfg.val_end,
|
||
"train_start": cfg.train_start,
|
||
"train_end": cfg.train_end,
|
||
"sensitivity": plan_sens,
|
||
})
|
||
self._emit_thinking(
|
||
f"Entering validation phase. Testing best config {self.final_result.run_id} "
|
||
f"on out-of-sample data ({cfg.val_start} → {cfg.val_end}) + sensitivity checks.",
|
||
kind="reasoning",
|
||
)
|
||
self._log("info", "━━ Phase 3: Validation (out-of-sample + sensitivity) ━━")
|
||
|
||
# ── OOS test ──
|
||
oos_id = f"oos_{datetime.utcnow().strftime('%Y%m%d_%H%M%S')}"
|
||
self._log("info", f" OOS test: {cfg.val_start} → {cfg.val_end}")
|
||
self._emit("validation_run_start", {
|
||
"run_id": oos_id,
|
||
"kind": "oos",
|
||
"label": "Out-of-Sample",
|
||
"description": f"Re-testing best config on unseen data: {cfg.val_start} → {cfg.val_end}",
|
||
"index": 1,
|
||
"total": planned_runs,
|
||
"period_start": cfg.val_start,
|
||
"period_end": cfg.val_end,
|
||
"params": self.final_result.params,
|
||
})
|
||
self._emit_thinking(
|
||
"Running out-of-sample test: if the strategy holds up on data it wasn't "
|
||
"optimized on, the edge is real — not curve-fit noise.",
|
||
kind="hypothesis",
|
||
)
|
||
t0 = time.time()
|
||
oos_result = self._execute_run(
|
||
oos_id, self.final_result.params,
|
||
cfg.val_start, cfg.val_end,
|
||
"phase3_oos", builder, runner, parser, store, writer, ranker, profile
|
||
)
|
||
budget.record_run(time.time() - t0)
|
||
self._run_count += 1
|
||
|
||
oos_dict = self._make_run_dict(oos_id, oos_result, "phase3_oos")
|
||
self._completed_runs.append(oos_dict)
|
||
self._emit("run_complete", {**oos_dict, "progress_pct": round(self._run_count / self._total_runs * 100)})
|
||
self._emit("validation_run_complete", {
|
||
"run_id": oos_id,
|
||
"kind": "oos",
|
||
"label": "Out-of-Sample",
|
||
"index": 1,
|
||
"total": planned_runs,
|
||
"net_profit": round(oos_result.net_profit, 2),
|
||
"profit_factor": round(oos_result.profit_factor, 3),
|
||
"calmar": round(oos_result.calmar, 3),
|
||
"max_drawdown": round(oos_result.max_drawdown * 100, 2),
|
||
"total_trades": oos_result.total_trades,
|
||
"passing": oos_result.passing,
|
||
})
|
||
# Thinking narration on OOS outcome
|
||
oos_ratio = (oos_result.calmar / max(self.final_result.calmar, 0.001)) if oos_result.calmar else 0
|
||
if oos_result.net_profit > 0 and oos_ratio >= 0.50:
|
||
self._emit_thinking(
|
||
f"OOS profitable: ${oos_result.net_profit:.0f} with Calmar {oos_result.calmar:.2f} "
|
||
f"(≈{oos_ratio*100:.0f}% of training performance). The edge generalizes.",
|
||
kind="success",
|
||
)
|
||
else:
|
||
self._emit_thinking(
|
||
f"OOS weak: profit ${oos_result.net_profit:.0f}, Calmar {oos_result.calmar:.2f} — "
|
||
f"strategy likely overfit to the training period.",
|
||
kind="warning",
|
||
)
|
||
|
||
# ── Sensitivity test (2 runs: nudge top param up and down) ──
|
||
sens_results = []
|
||
if opts and budget.can_fit(2):
|
||
top_param = opts[0] # first optimizable param
|
||
self._emit_thinking(
|
||
f"Sensitivity check: nudging `{top_param.name}` ±20% to see if performance "
|
||
f"survives small parameter drift.",
|
||
kind="reasoning",
|
||
)
|
||
for i, direction in enumerate([1, -1], start=2):
|
||
if budget.is_exhausted() or self._stop_flag:
|
||
break
|
||
nudged = dict(self.final_result.params)
|
||
current = float(nudged.get(top_param.name, top_param.default))
|
||
span = float(top_param.max) - float(top_param.min)
|
||
nudged[top_param.name] = top_param.clamp(current + direction * span * 0.20)
|
||
|
||
sens_id = f"sens_{direction}_{datetime.utcnow().strftime('%H%M%S')}"
|
||
label = f"Sensitivity ({top_param.name} {'+' if direction > 0 else '-'}20%)"
|
||
self._emit("validation_run_start", {
|
||
"run_id": sens_id,
|
||
"kind": "sensitivity",
|
||
"label": label,
|
||
"description": f"Nudging `{top_param.name}` to {nudged[top_param.name]} "
|
||
f"(from {current}) to test parameter stability.",
|
||
"index": i,
|
||
"total": planned_runs,
|
||
"period_start": cfg.train_start,
|
||
"period_end": cfg.train_end,
|
||
"params": nudged,
|
||
})
|
||
t0 = time.time()
|
||
sr = self._execute_run(
|
||
sens_id, nudged, cfg.train_start, cfg.train_end,
|
||
"phase3_sens", builder, runner, parser, store, writer, ranker, profile
|
||
)
|
||
budget.record_run(time.time() - t0)
|
||
sens_results.append(sr)
|
||
self._run_count += 1
|
||
|
||
sens_dict = self._make_run_dict(sens_id, sr, "phase3_sens")
|
||
self._completed_runs.append(sens_dict)
|
||
self._emit("run_complete", {**sens_dict, "progress_pct": round(self._run_count / max(self._total_runs, 1) * 100)})
|
||
self._emit("validation_run_complete", {
|
||
"run_id": sens_id,
|
||
"kind": "sensitivity",
|
||
"label": label,
|
||
"index": i,
|
||
"total": planned_runs,
|
||
"net_profit": round(sr.net_profit, 2),
|
||
"profit_factor": round(sr.profit_factor, 3),
|
||
"calmar": round(sr.calmar, 3),
|
||
"max_drawdown": round(sr.max_drawdown * 100, 2),
|
||
"total_trades": sr.total_trades,
|
||
"passing": bool(sr.passing),
|
||
})
|
||
|
||
# Determine verdict
|
||
self.verdict = self._determine_verdict(self.final_result, oos_result, sens_results)
|
||
|
||
# Validation done — narrate the conclusion
|
||
self._emit("validation_done", {
|
||
"verdict": self.verdict,
|
||
"oos_passing": bool(oos_result and oos_result.passing),
|
||
"sens_passing": sum(1 for s in sens_results if s.passing),
|
||
"sens_total": len(sens_results),
|
||
})
|
||
verdict_narration = {
|
||
"RECOMMENDED": "Validation passed on all fronts. Strategy is robust and ready for deployment.",
|
||
"RISKY": "Validation mixed. Strategy may work but has stability concerns — proceed with care.",
|
||
"NOT_RELIABLE": "Validation failed. Strategy is not reliable — likely overfit or unstable.",
|
||
}.get(self.verdict, "Validation complete.")
|
||
self._emit_thinking(
|
||
verdict_narration,
|
||
kind=("success" if self.verdict == "RECOMMENDED"
|
||
else "warning" if self.verdict == "RISKY" else "warning"),
|
||
)
|
||
|
||
# ── Generate .set output ─────────────────────────────────────────────
|
||
self.best_set_path = self._write_set_file(self.final_result, schema, cfg)
|
||
|
||
# ── Final emit ───────────────────────────────────────────────────────
|
||
completion_payload = {
|
||
"verdict": self.verdict,
|
||
"best_run_id": self.final_result.run_id,
|
||
"score": round(self.final_result.score, 4),
|
||
"net_profit": round(self.final_result.net_profit, 2),
|
||
"calmar": round(self.final_result.calmar, 3),
|
||
"profit_factor": round(self.final_result.profit_factor, 3),
|
||
"win_rate": round(self.final_result.win_rate * 100, 1),
|
||
"max_drawdown": round(self.final_result.max_drawdown * 100, 2),
|
||
"total_trades": self.final_result.total_trades,
|
||
"oos_profit": round(oos_result.net_profit, 2) if oos_result else None,
|
||
"oos_calmar": round(oos_result.calmar, 3) if oos_result else None,
|
||
"set_file_url": f"/download_set/{self.final_result.run_id}" if self.best_set_path else None,
|
||
"total_runs": self._run_count,
|
||
"elapsed_min": round((time.time() - self.run_start_ts) / 60, 1),
|
||
}
|
||
self._emit("optimization_complete", completion_payload)
|
||
|
||
verdict_icon = {"RECOMMENDED": "✅", "RISKY": "⚠️", "NOT_RELIABLE": "❌"}.get(self.verdict, "?")
|
||
self._log("success" if self.verdict == "RECOMMENDED" else "warning",
|
||
f"{verdict_icon} VERDICT: {self.verdict} | "
|
||
f"Profit: ${self.final_result.net_profit:.0f} | "
|
||
f"Calmar: {self.final_result.calmar:.2f}"
|
||
)
|
||
|
||
# ── Webhook notification (Discord / Slack / generic) ─────────────────
|
||
try:
|
||
self._send_completion_webhook(completion_payload)
|
||
except Exception as e:
|
||
logger.debug(f"Webhook notification failed: {e}")
|
||
|
||
# ── Single run executor ───────────────────────────────────────────────────
|
||
|
||
def _execute_run(
|
||
self, run_id, params, period_start, period_end, phase,
|
||
builder, runner, parser, store, writer, ranker, profile
|
||
) -> RankedResult:
|
||
"""Execute one MT5 backtest and return a RankedResult."""
|
||
# Demo mode short-circuit — generate synthetic metrics so judges can see the
|
||
# AI loop without an MT5 install. Toggle with APEX_DEMO_MODE=1.
|
||
if os.environ.get("APEX_DEMO_MODE", "").strip() in ("1", "true", "yes"):
|
||
return self._execute_demo_run(run_id, params, phase, ranker)
|
||
|
||
run_dir = RUNS_DIR / run_id
|
||
run_dir.mkdir(parents=True, exist_ok=True)
|
||
|
||
try:
|
||
ini_path = builder.build(
|
||
run_id=run_id, params=params,
|
||
period_start=period_start, period_end=period_end,
|
||
output_dir=run_dir, phase=phase,
|
||
ea_file=profile.ex5_file,
|
||
ea_symbol=profile.symbol,
|
||
ea_timeframe=profile.timeframe,
|
||
)
|
||
|
||
run = Run(
|
||
run_id=run_id,
|
||
ea_name=profile.name, symbol=profile.symbol,
|
||
timeframe=profile.timeframe,
|
||
period_start=period_start, period_end=period_end,
|
||
params=params, phase=phase,
|
||
tester_model=self.cfg["mt5"]["tester_model"],
|
||
ini_snapshot=ini_path.read_text(),
|
||
)
|
||
store.save_run(run)
|
||
|
||
result = runner.run(
|
||
run_id, ini_path, run_dir / "report",
|
||
log_csv_search_dir=Path(self.cfg["mt5"]["mql5_files_path"]),
|
||
profile=profile,
|
||
)
|
||
|
||
if not result.success:
|
||
return ranker.make_result(run_id, params, phase, None,
|
||
error=result.error_message)
|
||
|
||
metrics, trades = parser.parse(result.report_xml, result.report_html)
|
||
if metrics is None:
|
||
return ranker.make_result(run_id, params, phase, None,
|
||
error="parse_failed")
|
||
|
||
metrics.run_id = run_id
|
||
|
||
# Run analysis & AI reasoning
|
||
findings = self._analyze_run(run_id, metrics, trades or [], params)
|
||
self._reason_about_run(run_id, metrics, findings, params)
|
||
|
||
trades_df = pd.DataFrame()
|
||
if trades:
|
||
try:
|
||
trades_df = pd.DataFrame([t.model_dump() for t in trades])
|
||
except Exception:
|
||
try:
|
||
trades_df = pd.DataFrame([vars(t) for t in trades])
|
||
except Exception:
|
||
pass
|
||
|
||
# Build RankedResult first so we have the ranked_score for summary.json
|
||
ranked = ranker.make_result(run_id, params, phase, metrics)
|
||
|
||
ai_insight = self._run_insights.get(run_id)
|
||
writer.write(
|
||
run_id, metrics, trades_df, findings, params,
|
||
phase=phase,
|
||
ranked_score=ranked.score,
|
||
ai_insight=ai_insight,
|
||
)
|
||
|
||
return ranked
|
||
|
||
except Exception as e:
|
||
logger.warning(f"[{run_id}] Run error: {e}")
|
||
return ranker.make_result(run_id, params, phase, None, error=str(e))
|
||
|
||
# ── Demo mode (synthetic backtest) ────────────────────────────────────────
|
||
|
||
def _execute_demo_run(self, run_id: str, params: dict, phase: str, ranker) -> RankedResult:
|
||
"""
|
||
Generate a synthetic, deterministic-but-realistic RankedResult from the
|
||
params hash. Lets the AI loop run end-to-end without MT5 installed.
|
||
|
||
The metrics improve as parameters approach a hidden "sweet spot" so the
|
||
AI can hill-climb in a way judges can observe. We also add small jitter
|
||
to look organic.
|
||
"""
|
||
from data.models import RunMetrics # local import to avoid cycle
|
||
|
||
# Hash params → stable seed per config
|
||
key = ",".join(f"{k}={v}" for k, v in sorted(params.items()))
|
||
seed = int(hashlib.md5(key.encode()).hexdigest()[:12], 16)
|
||
rng = random.Random(seed)
|
||
|
||
# Hidden sweet-spot hash drift: a deterministic "true score" between 0–1
|
||
# based on a smooth function of parameter values. Small param changes
|
||
# produce small score changes — that's what lets the AI hill-climb.
|
||
true_score = 0.0
|
||
for k, v in sorted(params.items()):
|
||
try:
|
||
fv = float(v)
|
||
# Map value into [-1,1] using a stable hash, multiply by a gentle
|
||
# bias toward "moderate" values (sweet spot is mid-range).
|
||
slot = (int(hashlib.md5(k.encode()).hexdigest()[:8], 16) % 100) / 100.0
|
||
normalized = (fv % 100) / 100.0
|
||
true_score += 1.0 - abs(normalized - slot)
|
||
except Exception:
|
||
true_score += 0.5
|
||
if params:
|
||
true_score /= len(params)
|
||
true_score = max(0.05, min(0.98, true_score))
|
||
|
||
# Add jitter for realism (±10%)
|
||
true_score *= rng.uniform(0.90, 1.10)
|
||
true_score = max(0.05, min(0.99, true_score))
|
||
|
||
# ── Regime failure ────────────────────────────────────────────────
|
||
# Real backtests produce occasional duds — bad luck on a market regime
|
||
# the parameters can't handle, or curve-fit overfit blowing up. Inject
|
||
# those so the verdict isn't always RECOMMENDED. Probability is higher
|
||
# in Phase 1 (pure exploration) and lower in Phase 2 (AI is improving).
|
||
regime_fail_prob = {
|
||
"phase1": 0.35, # 1 in 3 LHS samples lands in a bad spot
|
||
"phase2_ai": 0.10, # AI is improving, occasional misstep
|
||
"phase2": 0.20, # random neighbor search
|
||
"phase3_oos": 0.25, # the market the params never saw — sometimes brutal
|
||
"phase3_sens": 0.15, # nudged param → small chance of falling off the cliff
|
||
}.get(phase, 0.15)
|
||
|
||
regime_failed = rng.random() < regime_fail_prob
|
||
if regime_failed:
|
||
# Crush true_score so PF dives below 1, DD blows out, profit goes negative
|
||
true_score *= rng.uniform(0.10, 0.35)
|
||
|
||
# Project onto realistic metric ranges
|
||
profit_factor = round(0.7 + true_score * 1.8, 3) # 0.7–2.5
|
||
calmar = round(true_score * 1.4, 3) # 0.0–1.4
|
||
max_drawdown = round(28 - true_score * 22, 2) # 28%→6%
|
||
win_rate = round(40 + true_score * 25, 1) # 40%→65%
|
||
total_trades = int(80 + rng.random() * 220) # 80–300
|
||
net_profit = round((profit_factor - 1) * 5000 * (1 + rng.uniform(-0.2, 0.2)), 2)
|
||
|
||
# Out-of-sample is ALWAYS at least somewhat worse (reality bias).
|
||
# Roughly 30% of OOS runs degrade significantly (>40%) — that's what
|
||
# produces RISKY / NOT_RELIABLE verdicts in production.
|
||
if phase.startswith("phase3_oos"):
|
||
severe = rng.random() < 0.30
|
||
shrink = rng.uniform(0.35, 0.55) if severe else rng.uniform(0.75, 0.92)
|
||
profit_factor *= shrink
|
||
calmar *= shrink
|
||
net_profit *= shrink
|
||
if severe:
|
||
# Severe OOS failures also widen drawdown
|
||
max_drawdown = min(45.0, max_drawdown * rng.uniform(1.3, 1.8))
|
||
|
||
avg_trade = net_profit / max(total_trades, 1)
|
||
winners = int(total_trades * (win_rate / 100.0))
|
||
losers = max(1, total_trades - winners)
|
||
# Derive avg win/loss consistent with profit_factor: PF = (winners*avg_win)/(losers*|avg_loss|)
|
||
avg_loss = -abs(avg_trade) * (1 + 1.5 / max(profit_factor, 0.5))
|
||
avg_win = (profit_factor * losers * abs(avg_loss)) / max(winners, 1)
|
||
|
||
metrics = RunMetrics(
|
||
run_id=run_id,
|
||
net_profit=net_profit,
|
||
profit_factor=profit_factor,
|
||
calmar_ratio=calmar,
|
||
max_drawdown_pct=max_drawdown / 100.0,
|
||
max_drawdown_abs=round(net_profit * (max_drawdown / 100.0) * 1.2 + 200, 2),
|
||
win_rate=win_rate / 100.0,
|
||
total_trades=total_trades,
|
||
recovery_factor=round(profit_factor * 1.5, 2),
|
||
sharpe_ratio=round(true_score * 1.8, 2),
|
||
avg_win=round(avg_win, 2),
|
||
avg_loss=round(avg_loss, 2),
|
||
largest_loss=round(avg_loss * 3.5, 2),
|
||
expected_payoff=round(avg_trade, 2),
|
||
)
|
||
|
||
# Simulate per-run latency so the dashboard feels alive — not instant.
|
||
# Each run takes ~1.5s so a 10-iteration AI loop is ~15s total.
|
||
delay = float(os.environ.get("APEX_DEMO_RUN_SECONDS", "1.5"))
|
||
time.sleep(max(0.05, delay))
|
||
|
||
# Run AI reasoning if enabled — same flow as live mode
|
||
try:
|
||
self._reason_about_run(run_id, metrics, [], params)
|
||
except Exception:
|
||
pass
|
||
|
||
return ranker.make_result(run_id, params, phase, metrics)
|
||
|
||
# ── Verdict logic ─────────────────────────────────────────────────────────
|
||
|
||
def _determine_verdict(
|
||
self,
|
||
best: RankedResult,
|
||
oos: Optional[RankedResult],
|
||
sens: list[RankedResult],
|
||
) -> str:
|
||
"""
|
||
RECOMMENDED: IS profitable + OOS profitable + not fragile
|
||
RISKY: IS profitable but OOS weak OR fragile
|
||
NOT_RELIABLE: IS marginal or OOS loss
|
||
"""
|
||
if best.calmar < 0.1 or best.net_profit <= 0:
|
||
return "NOT_RELIABLE"
|
||
|
||
oos_ok = False
|
||
if oos and oos.net_profit > 0:
|
||
# OOS degradation: acceptable if OOS calmar ≥ 50% of IS calmar
|
||
oos_ratio = oos.calmar / max(best.calmar, 0.001)
|
||
oos_ok = oos_ratio >= 0.50
|
||
elif oos is None:
|
||
oos_ok = True # No OOS test — can't penalize
|
||
|
||
# Sensitivity: fragile if any nudge drops Calmar by >50%
|
||
fragile = any(
|
||
s.passing and s.calmar < best.calmar * 0.50
|
||
for s in sens
|
||
) if sens else False
|
||
|
||
if oos_ok and not fragile and best.calmar >= 0.30:
|
||
return "RECOMMENDED"
|
||
if oos_ok or (not fragile and best.calmar >= 0.20):
|
||
return "RISKY"
|
||
return "NOT_RELIABLE"
|
||
|
||
# ── .set file output ──────────────────────────────────────────────────────
|
||
|
||
def _write_set_file(
|
||
self, result: RankedResult, schema: ParameterSchema, cfg: SessionConfig
|
||
) -> Optional[Path]:
|
||
"""Write a clean .set file for MT5 import."""
|
||
try:
|
||
out_dir = self.reports_dir / result.run_id
|
||
out_dir.mkdir(parents=True, exist_ok=True)
|
||
out_path = out_dir / f"{cfg.ea_name}_optimized_{cfg.symbol}_{cfg.timeframe}.set"
|
||
|
||
header = (
|
||
f"Optimized by MT5 Smart Optimizer\n"
|
||
f"EA: {cfg.ea_name} | Symbol: {cfg.symbol} | TF: {cfg.timeframe}\n"
|
||
f"Training: {cfg.train_start} – {cfg.train_end}\n"
|
||
f"Verdict: {self.verdict}\n"
|
||
f"Net Profit: ${result.net_profit:.2f} | Calmar: {result.calmar:.2f} | "
|
||
f"Win Rate: {result.win_rate:.1f}%\n"
|
||
f"Generated: {datetime.utcnow().strftime('%Y-%m-%d %H:%M UTC')}"
|
||
)
|
||
|
||
content = schema.to_set_file(result.params, header_comment=header)
|
||
out_path.write_text(content, encoding="utf-8")
|
||
logger.info(f"Optimized .set file written: {out_path}")
|
||
return out_path
|
||
except Exception as e:
|
||
logger.warning(f"Could not write .set file: {e}")
|
||
return None
|
||
|
||
# ── Helpers ───────────────────────────────────────────────────────────────
|
||
|
||
def _make_run_dict(self, run_id: str, result: RankedResult, phase: str) -> dict:
|
||
"""Canonical run dict used for both _completed_runs and run_complete emits.
|
||
|
||
IMPORTANT — unit convention: win_rate and max_drawdown are emitted as
|
||
PERCENTAGES (0–100), not fractions, so the dashboard can render them
|
||
directly with `${val}%`. RankedResult stores these as fractions so we
|
||
scale here at the API boundary.
|
||
"""
|
||
d = {
|
||
"run_id": run_id,
|
||
"phase": phase,
|
||
"ts": datetime.utcnow().isoformat(),
|
||
"net_profit": round(result.net_profit, 2),
|
||
"calmar": round(result.calmar, 3),
|
||
"profit_factor": round(result.profit_factor, 3),
|
||
"win_rate": round(result.win_rate * 100, 1),
|
||
"max_drawdown": round(result.max_drawdown * 100, 2),
|
||
"total_trades": result.total_trades,
|
||
"passing": bool(result.passing),
|
||
"score": round(result.score, 4),
|
||
"params": result.params,
|
||
}
|
||
insight = self._run_insights.get(run_id)
|
||
if insight:
|
||
d["ai_insight"] = insight
|
||
return d
|
||
|
||
def _result_to_dict(self, r: RankedResult) -> dict:
|
||
d = {
|
||
"run_id": r.run_id,
|
||
"rank": r.rank,
|
||
"score": round(r.score, 4),
|
||
"net_profit": round(r.net_profit, 2),
|
||
"calmar": round(r.calmar, 3),
|
||
"profit_factor": round(r.profit_factor, 3),
|
||
"win_rate": round(r.win_rate * 100, 1), # fraction → %
|
||
"max_drawdown": round(r.max_drawdown * 100, 2), # fraction → %
|
||
"total_trades": r.total_trades,
|
||
"passing": bool(r.passing),
|
||
"params": r.params, # full param dict
|
||
"params_summary": self._params_summary(r.params),
|
||
}
|
||
# Attach AI insight if available
|
||
insight = self._run_insights.get(r.run_id)
|
||
if insight:
|
||
d["ai_insight"] = insight
|
||
return d
|
||
|
||
@staticmethod
|
||
def _params_summary(params: dict) -> str:
|
||
"""Show a few key params for display — works with any EA."""
|
||
parts = []
|
||
for k, v in list(params.items())[:8]:
|
||
short = k.replace("Inp", "").replace("inp", "")[:12]
|
||
parts.append(f"{short}={v}")
|
||
return " | ".join(parts[:4])
|
||
|
||
def _analyze_run(self, run_id: str, metrics, trades: list, params: dict) -> list:
|
||
"""Run Tier 1 analyzers on trade data. Always available from HTML report."""
|
||
findings = []
|
||
if not trades:
|
||
return findings
|
||
|
||
try:
|
||
trades_df = pd.DataFrame([t.model_dump() for t in trades])
|
||
except Exception:
|
||
try:
|
||
trades_df = pd.DataFrame([vars(t) for t in trades])
|
||
except Exception:
|
||
return findings
|
||
|
||
if trades_df.empty:
|
||
return findings
|
||
|
||
for analyzer_cls in (EquityCurveAnalyzer, TimePerformanceAnalyzer):
|
||
try:
|
||
az = analyzer_cls()
|
||
result = az.analyze(trades_df)
|
||
if isinstance(result, list):
|
||
findings.extend(result)
|
||
elif result is not None:
|
||
findings.append(result)
|
||
except Exception as e:
|
||
logger.debug(f"[{run_id}] {analyzer_cls.__name__} skipped: {e}")
|
||
|
||
self._run_findings[run_id] = findings
|
||
return findings
|
||
|
||
def _reason_about_run(
|
||
self, run_id: str, metrics, findings: list, params: dict
|
||
) -> Optional[dict]:
|
||
"""Call AI reasoner and emit insight via SocketIO."""
|
||
if not self._ai_reasoner or not self._ai_reasoner.enabled:
|
||
return None
|
||
try:
|
||
history = [
|
||
{
|
||
"run_id": r.run_id, "score": r.score,
|
||
"calmar": r.calmar, "pf": r.profit_factor, "phase": r.phase,
|
||
}
|
||
for r in (self.phase1_results + self.phase2_results)[-5:]
|
||
]
|
||
insight = self._ai_reasoner.analyze(
|
||
findings=findings,
|
||
metrics=metrics,
|
||
run_history=history,
|
||
current_params=params,
|
||
)
|
||
d = insight.to_dict()
|
||
d["run_id"] = run_id
|
||
self._ai_insights.append(d)
|
||
self._run_insights[run_id] = d
|
||
self._emit("ai_insight", d)
|
||
return d
|
||
except Exception as e:
|
||
logger.warning(f"[{run_id}] AI reasoning failed: {e}")
|
||
return None
|
||
|
||
def get_latest_insight(self) -> Optional[dict]:
|
||
"""Return the most recent AI insight."""
|
||
return self._ai_insights[-1] if self._ai_insights else None
|
||
|
||
def get_all_insights(self) -> list[dict]:
|
||
return list(self._ai_insights)
|
||
|
||
def _log(self, level: str, msg: str) -> None:
|
||
getattr(logger, level, logger.info)(msg)
|
||
self._emit("log", {"level": level, "msg": msg})
|
||
|
||
def _emit_thinking(
|
||
self,
|
||
msg: str,
|
||
kind: str = "info",
|
||
iteration: Optional[int] = None,
|
||
meta: Optional[dict] = None,
|
||
) -> None:
|
||
"""
|
||
Emit an AI-thinking stream event — distinct from system logs.
|
||
Shows up in the dashboard's "Live AI Thinking Feed".
|
||
|
||
kind: 'info' | 'reasoning' | 'decision' | 'warning' | 'success' | 'hypothesis'
|
||
"""
|
||
payload = {
|
||
"msg": msg,
|
||
"kind": kind,
|
||
"iteration": iteration,
|
||
"phase": self._phase,
|
||
"ts": datetime.utcnow().isoformat(),
|
||
}
|
||
if meta:
|
||
payload["meta"] = meta
|
||
# Persist for dashboard refresh — keep last 200 entries
|
||
self._thinking_log.append(payload)
|
||
if len(self._thinking_log) > 200:
|
||
self._thinking_log = self._thinking_log[-200:]
|
||
self._emit("ai_thinking", payload)
|
||
|
||
def _emit_early_termination(self, reason_code: str, message: str, details: dict = None) -> None:
|
||
"""
|
||
Surface an early stop to the user.
|
||
reason_code examples:
|
||
- 'no_profit' (Phase 1 found nothing)
|
||
- 'targets_met' (AI loop hit all quality targets)
|
||
- 'budget_exhausted'(time budget used up)
|
||
- 'user_stop' (user clicked Stop)
|
||
- 'stuck_escape' (optimizer stuck, bailing)
|
||
"""
|
||
payload = {
|
||
"reason": reason_code,
|
||
"message": message,
|
||
"phase": self._phase,
|
||
"ts": datetime.utcnow().isoformat(),
|
||
}
|
||
if details:
|
||
payload["details"] = details
|
||
self._early_term = payload # remember for refresh
|
||
self._emit("early_termination", payload)
|
||
|
||
def _stream_ai_chunk(self, evt: dict) -> None:
|
||
"""
|
||
Receive an event from AIReasoner streaming and forward to the
|
||
dashboard. Events: {event: 'start'|'delta'|'end'|'error', text?, error?}.
|
||
The frontend appends deltas to a single growing thinking-feed bubble.
|
||
"""
|
||
try:
|
||
self._emit("ai_thinking_chunk", {
|
||
"event": evt.get("event"),
|
||
"text": evt.get("text", ""),
|
||
"error": evt.get("error", ""),
|
||
"phase": self._phase,
|
||
"ts": datetime.utcnow().isoformat(),
|
||
})
|
||
except Exception:
|
||
pass
|
||
|
||
def _send_completion_webhook(self, payload: dict) -> None:
|
||
"""
|
||
POST a completion summary to a user-configured webhook URL. Supports
|
||
Discord and Slack styles natively; everything else gets the raw JSON.
|
||
Silent no-op if no URL is configured.
|
||
"""
|
||
notif = (self.cfg or {}).get("notifications", {}) or {}
|
||
url = (notif.get("webhook_url") or "").strip()
|
||
if not url:
|
||
return
|
||
|
||
verdict_emoji = {"RECOMMENDED": "✅", "RISKY": "⚠️", "NOT_RELIABLE": "❌"}.get(payload.get("verdict"), "📊")
|
||
title = f"{verdict_emoji} APEX optimization {payload.get('verdict','complete')}"
|
||
line = (
|
||
f"Best run **{payload.get('best_run_id','?')}** "
|
||
f"— PF {payload.get('profit_factor', 0):.2f} · "
|
||
f"Calmar {payload.get('calmar', 0):.2f} · "
|
||
f"DD {payload.get('max_drawdown', 0):.1f}% · "
|
||
f"Profit ${payload.get('net_profit', 0):,.0f} · "
|
||
f"{payload.get('total_trades', 0)} trades · "
|
||
f"{payload.get('elapsed_min', 0)} min"
|
||
)
|
||
|
||
# Detect style
|
||
style = (notif.get("webhook_style") or "auto").lower()
|
||
if style == "auto":
|
||
if "discord.com" in url or "discordapp.com" in url:
|
||
style = "discord"
|
||
elif "slack.com" in url or "hooks.slack" in url:
|
||
style = "slack"
|
||
else:
|
||
style = "generic"
|
||
|
||
try:
|
||
import requests
|
||
if style == "discord":
|
||
body = {"content": f"**{title}**\n{line}"}
|
||
elif style == "slack":
|
||
body = {"text": f"*{title}*\n{line}"}
|
||
else:
|
||
body = {"title": title, "summary": line, **payload}
|
||
requests.post(url, json=body, timeout=5)
|
||
self._log("info", f"📨 Webhook ({style}) notified.")
|
||
except Exception as e:
|
||
logger.warning(f"Webhook POST failed: {e}")
|
||
|
||
def _emit(self, event: str, data: dict = {}) -> None:
|
||
# Tee select live-activity events into in-memory logs so a dashboard
|
||
# refresh during the run can replay them via REST.
|
||
try:
|
||
if event == "param_changes":
|
||
self._param_changes.append(data)
|
||
if len(self._param_changes) > 50:
|
||
self._param_changes = self._param_changes[-50:]
|
||
elif event in ("validation_start", "validation_run_start", "validation_run_complete", "validation_done"):
|
||
self._validation_log.append({"event": event, **data})
|
||
if len(self._validation_log) > 40:
|
||
self._validation_log = self._validation_log[-40:]
|
||
except Exception:
|
||
pass
|
||
try:
|
||
self.socketio.emit(event, data)
|
||
except Exception as e:
|
||
logger.debug(f"Emit error ({event}): {e}")
|