feat: open-source release — AI-driven autonomous optimization with live visibility
Major upgrade making the AI loop visible and the project ready for public release. UI / UX - Live AI Thinking Feed: streams reasoning, decisions, and outcomes per iteration - Parameter Changes panel: prev → new + reason for every AI-driven edit - Validation Activity panel: out-of-sample + sensitivity runs with live metrics - Early Termination banner: surfaces why optimization stopped (targets met, no profit, budget, stuck, user stop) - 3-phase tracker renamed Exploration / Iteration / Validation with live N/total - Best Result modal exposes Evolution Path showing how the AI arrived at the winner - Run-detail modal accessible from every recent run row - Setup form validation (dates, walk-forward order, params selection, AI targets) - Pause button removed; misleading sidebar nav consolidated to Dashboard / New Run / Reports / Source Backend - AIGuidedLoop streams ai_thinking, param_changes, ai_targets_met, ai_stuck - Pipeline emits validation_start / validation_run_start / validation_run_complete / validation_done - Pipeline emits early_termination on every early-stop path - /api/best_result returns best run + full evolution chain - /api/run/<id> + /api/runs sorted by ts - AIReasoner falls back to ANTHROPIC_API_KEY env var when config is a placeholder - Demo mode (APEX_DEMO_MODE=1) generates deterministic synthetic backtests so judges can run end-to-end without MT5 Open-source readiness - README.md with pitch, demo flow, architecture diagram, quickstart, event reference - LICENSE (MIT) - config.example.yaml template (config.yaml now git-ignored) - requirements.txt: added anthropic / requests / psutil / beautifulsoup4, capped majors - .gitignore: secrets, *.set, scratch screenshots, ea_registry.yaml - demo/run_demo.py: one-command offline demo runner - 10 polished screenshots for README + judge review
This commit is contained in:
LEGSTECH Optimizer
@@ -0,0 +1,576 @@
|
||||
"""
|
||||
optimizer/ai_guided_loop.py
|
||||
AI-Guided Autonomous Optimization Loop.
|
||||
|
||||
Replaces Phase 2's blind random neighbor search with directed,
|
||||
AI-driven parameter evolution. Each iteration:
|
||||
|
||||
1. Build rich context: parameter schema + full history
|
||||
2. Ask AI: "what parameter values should I try next?"
|
||||
3. Apply changes with bounds checking
|
||||
4. Deduplicate (don't re-test seen param sets)
|
||||
5. Run backtest via existing pipeline._execute_run()
|
||||
6. Check stop conditions (targets met OR max iterations)
|
||||
7. Emit progress to frontend, update pipeline state
|
||||
8. Loop
|
||||
|
||||
The loop terminates when:
|
||||
- All quality targets are met by the current best result
|
||||
- Max iterations reached
|
||||
- Stop flag set externally (user clicked Stop)
|
||||
- Budget exhausted
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import random
|
||||
import time
|
||||
from datetime import datetime
|
||||
from typing import Optional
|
||||
|
||||
from loguru import logger
|
||||
|
||||
from ea.schema import ParameterSchema
|
||||
from optimizer.result_ranker import RankedResult, ResultRanker
|
||||
from optimizer.session_config import SessionConfig
|
||||
from analysis.ai_reasoner import AIReasoner, AIParamSuggestion
|
||||
|
||||
|
||||
class AIGuidedLoop:
|
||||
"""
|
||||
Autonomous AI-driven parameter search.
|
||||
|
||||
Usage (from inside OptimizationPipeline._run_pipeline):
|
||||
loop = AIGuidedLoop(pipeline, schema, cfg, builder, runner,
|
||||
parser, store, writer, ranker, profile, budget)
|
||||
loop.run(seed_results, max_iterations, targets)
|
||||
# Results available in loop.all_results, loop.best_result
|
||||
"""
|
||||
|
||||
# If last N iterations show less than this score improvement → escape
|
||||
STUCK_WINDOW = 3
|
||||
STUCK_THRESHOLD = 0.005
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
pipeline, # OptimizationPipeline — for _execute_run / _emit / _log
|
||||
schema: ParameterSchema,
|
||||
cfg: SessionConfig,
|
||||
builder, runner, parser, store, writer,
|
||||
ranker: ResultRanker,
|
||||
profile,
|
||||
budget,
|
||||
):
|
||||
self.pipeline = pipeline
|
||||
self.schema = schema
|
||||
self.cfg = cfg
|
||||
self.builder = builder
|
||||
self.runner = runner
|
||||
self.parser = parser
|
||||
self.store = store
|
||||
self.writer = writer
|
||||
self.ranker = ranker
|
||||
self.profile = profile
|
||||
self.budget = budget
|
||||
|
||||
# Public results — populated during run()
|
||||
self.all_results: list[RankedResult] = []
|
||||
self.best_result: Optional[RankedResult] = None
|
||||
|
||||
# Internal state
|
||||
self._iteration_history: list[dict] = [] # rich history for AI prompt
|
||||
self._seen_hashes: set[str] = set()
|
||||
self._rng = random.Random(int(time.time()))
|
||||
|
||||
# ── Public entry point ────────────────────────────────────────────────────
|
||||
|
||||
def run(
|
||||
self,
|
||||
seed_results: list[RankedResult],
|
||||
max_iterations: int,
|
||||
targets: dict,
|
||||
) -> RankedResult:
|
||||
"""
|
||||
Run the autonomous loop. Returns the best result found.
|
||||
|
||||
seed_results: Phase 1 ranked results (provides initial best + seen params)
|
||||
max_iterations: hard cap on AI-directed iterations
|
||||
targets: {min_profit_factor, max_drawdown_pct, min_calmar}
|
||||
"""
|
||||
self._initialize_from_seeds(seed_results)
|
||||
|
||||
self._log("info", f"━━ AI-Guided Loop: up to {max_iterations} iterations ━━")
|
||||
self._log("info",
|
||||
f" Targets → PF≥{targets.get('min_profit_factor',1.5)} | "
|
||||
f"DD≤{targets.get('max_drawdown_pct',20)}% | "
|
||||
f"Calmar≥{targets.get('min_calmar',0.5)}"
|
||||
)
|
||||
self._think(
|
||||
f"Targets set — PF≥{targets.get('min_profit_factor',1.5)}, "
|
||||
f"DD≤{targets.get('max_drawdown_pct',20)}%, Calmar≥{targets.get('min_calmar',0.5)}. "
|
||||
f"I'll stop as soon as I hit them, or after {max_iterations} iterations.",
|
||||
kind="reasoning",
|
||||
)
|
||||
|
||||
schema_info = self._build_schema_info()
|
||||
|
||||
for iteration in range(1, max_iterations + 1):
|
||||
if self.pipeline._stop_flag:
|
||||
self._log("info", "Loop stopped by user.")
|
||||
break
|
||||
|
||||
if self.budget.is_exhausted():
|
||||
self._log("warning", "⏱ Time budget exhausted — stopping AI loop.")
|
||||
break
|
||||
|
||||
# Check if current best already satisfies all targets
|
||||
if self.best_result and self._targets_met(self.best_result, targets):
|
||||
self._log("info",
|
||||
f"✅ All targets met after iteration {iteration - 1}! "
|
||||
f"PF={self.best_result.profit_factor:.2f}, "
|
||||
f"DD={self.best_result.max_drawdown:.1f}%, "
|
||||
f"Calmar={self.best_result.calmar:.2f}"
|
||||
)
|
||||
self._think(
|
||||
f"All quality targets reached after iteration {iteration - 1}. "
|
||||
f"Best config: PF={self.best_result.profit_factor:.2f}, "
|
||||
f"DD={self.best_result.max_drawdown:.1f}%, "
|
||||
f"Calmar={self.best_result.calmar:.2f}. Stopping early — no need to keep iterating.",
|
||||
kind="success",
|
||||
)
|
||||
self._emit("ai_targets_met", {
|
||||
"iteration": iteration - 1,
|
||||
"profit_factor": round(self.best_result.profit_factor, 3),
|
||||
"max_drawdown": round(self.best_result.max_drawdown, 2),
|
||||
"calmar": round(self.best_result.calmar, 3),
|
||||
})
|
||||
self.pipeline._emit_early_termination(
|
||||
reason_code="targets_met",
|
||||
message=f"All targets met at iteration {iteration - 1}. Optimization complete.",
|
||||
details={
|
||||
"iteration": iteration - 1,
|
||||
"profit_factor": round(self.best_result.profit_factor, 3),
|
||||
"max_drawdown": round(self.best_result.max_drawdown, 2),
|
||||
"calmar": round(self.best_result.calmar, 3),
|
||||
},
|
||||
)
|
||||
break
|
||||
|
||||
self._log("info",
|
||||
f"[AI Loop {iteration}/{max_iterations}] "
|
||||
f"Best so far: PF={self.best_result.profit_factor:.2f}, "
|
||||
f"Calmar={self.best_result.calmar:.2f}, "
|
||||
f"DD={self.best_result.max_drawdown:.1f}%"
|
||||
if self.best_result else f"[AI Loop {iteration}/{max_iterations}] Starting..."
|
||||
)
|
||||
self._think(
|
||||
f"Iteration {iteration}: reviewing history and deciding what to change next...",
|
||||
kind="info", iteration=iteration,
|
||||
)
|
||||
|
||||
# Get AI suggestion for next params
|
||||
suggestion = self._get_suggestion(schema_info, targets)
|
||||
|
||||
# Surface the AI's reasoning as its own thinking message
|
||||
if suggestion.analysis:
|
||||
self._think(suggestion.analysis, kind="reasoning", iteration=iteration)
|
||||
|
||||
# Apply changes to best params → candidate param set
|
||||
base_params = self.best_result.params if self.best_result else self.schema.defaults()
|
||||
next_params = self._apply_changes(
|
||||
base=base_params,
|
||||
changes=suggestion.changes,
|
||||
)
|
||||
|
||||
# Escape if stuck or AI returned no changes
|
||||
is_stuck = self._check_stuck()
|
||||
if is_stuck or not suggestion.changes:
|
||||
if is_stuck:
|
||||
self._log("warning", f" ⚠ Stuck detected — applying random escape at iteration {iteration}")
|
||||
self._think(
|
||||
f"Recent scores are flat — the AI is stuck in a local optimum. "
|
||||
f"Applying a random ±30% perturbation to escape and explore a new region.",
|
||||
kind="warning", iteration=iteration,
|
||||
)
|
||||
else:
|
||||
self._log("warning", f" ⚠ AI returned no changes — applying random escape")
|
||||
self._think(
|
||||
"AI returned no changes — falling back to a random perturbation so we keep exploring.",
|
||||
kind="warning", iteration=iteration,
|
||||
)
|
||||
next_params = self._random_escape(next_params)
|
||||
self._emit("ai_stuck", {"iteration": iteration})
|
||||
|
||||
# Deduplicate — ensure we're not re-testing an identical config
|
||||
next_params = self._ensure_unique(next_params, max_attempts=5)
|
||||
|
||||
# Build rich param change records (prev → new + reason) for the UI
|
||||
change_records = self._build_change_records(base_params, next_params, suggestion.changes)
|
||||
|
||||
# Narrate the actual parameter changes
|
||||
for c in change_records[:4]: # cap noise
|
||||
self._think(
|
||||
f"{c['param']}: {c['from']} → {c['to']} — {c['reason']}",
|
||||
kind="decision",
|
||||
iteration=iteration,
|
||||
meta=c,
|
||||
)
|
||||
|
||||
# Emit iteration start
|
||||
self._emit("ai_iteration_start", {
|
||||
"iteration": iteration,
|
||||
"max_iterations": max_iterations,
|
||||
"analysis": suggestion.analysis,
|
||||
"changes": suggestion.changes,
|
||||
"change_records": change_records,
|
||||
"confidence": round(suggestion.confidence, 2),
|
||||
"is_stuck_escape": is_stuck or not suggestion.changes,
|
||||
})
|
||||
|
||||
# Dedicated richer event that the dashboard subscribes to
|
||||
self._emit("param_changes", {
|
||||
"iteration": iteration,
|
||||
"run_id": None, # filled in after run below via complete event
|
||||
"analysis": suggestion.analysis,
|
||||
"changes": change_records,
|
||||
"confidence": round(suggestion.confidence, 2),
|
||||
})
|
||||
|
||||
# Run the backtest
|
||||
run_id = f"ai_{iteration:02d}_{datetime.utcnow().strftime('%H%M%S')}"
|
||||
t0 = time.time()
|
||||
result = self.pipeline._execute_run(
|
||||
run_id, next_params,
|
||||
self.cfg.train_start, self.cfg.train_end,
|
||||
"phase2_ai",
|
||||
self.builder, self.runner, self.parser,
|
||||
self.store, self.writer, self.ranker, self.profile,
|
||||
)
|
||||
elapsed = time.time() - t0
|
||||
self.budget.record_run(elapsed)
|
||||
|
||||
# Register result
|
||||
self.all_results.append(result)
|
||||
self._mark_seen(next_params)
|
||||
self._update_best(result)
|
||||
self.pipeline._run_count += 1
|
||||
|
||||
# Update pipeline live state
|
||||
if (self.pipeline._live_best is None
|
||||
or result.score > self.pipeline._live_best.score):
|
||||
self.pipeline._live_best = result
|
||||
|
||||
run_dict = self.pipeline._make_run_dict(run_id, result, "phase2_ai")
|
||||
self.pipeline._completed_runs.append(run_dict)
|
||||
|
||||
# Record iteration for AI history
|
||||
targets_met = self.best_result and self._targets_met(self.best_result, targets)
|
||||
self._record_iteration(iteration, run_id, result, suggestion)
|
||||
|
||||
goal_status = {
|
||||
"profit_factor_met": result.profit_factor >= targets.get("min_profit_factor", 1.5),
|
||||
"drawdown_ok": result.max_drawdown <= targets.get("max_drawdown_pct", 20.0),
|
||||
"calmar_met": result.calmar >= targets.get("min_calmar", 0.5),
|
||||
}
|
||||
|
||||
# Narrate the outcome of this iteration
|
||||
improved = (self.best_result is self.all_results[-1]) if self.all_results else False
|
||||
if result.passing and improved:
|
||||
self._think(
|
||||
f"✓ Iteration {iteration} improved the best score to {result.score:.3f} "
|
||||
f"(PF={result.profit_factor:.2f}, Calmar={result.calmar:.2f}, "
|
||||
f"DD={result.max_drawdown:.1f}%). Keeping these params as the new baseline.",
|
||||
kind="success", iteration=iteration,
|
||||
)
|
||||
elif result.passing:
|
||||
self._think(
|
||||
f"Iteration {iteration} passed thresholds but didn't beat the best — "
|
||||
f"score {result.score:.3f} vs best {self.best_result.score:.3f}.",
|
||||
kind="info", iteration=iteration,
|
||||
)
|
||||
else:
|
||||
diagnosis = self._diagnose_failure(result, targets)
|
||||
self._think(
|
||||
f"✗ Iteration {iteration} failed: {diagnosis} "
|
||||
f"(PF={result.profit_factor:.2f}, DD={result.max_drawdown:.1f}%). "
|
||||
f"Will adjust in the next step.",
|
||||
kind="warning", iteration=iteration,
|
||||
)
|
||||
|
||||
# Emit iteration complete
|
||||
self._emit("ai_iteration_complete", {
|
||||
"iteration": iteration,
|
||||
"max_iterations": max_iterations,
|
||||
"run_id": run_id,
|
||||
"score": round(result.score, 4),
|
||||
"profit_factor": round(result.profit_factor, 3),
|
||||
"calmar": round(result.calmar, 3),
|
||||
"max_drawdown": round(result.max_drawdown, 2),
|
||||
"net_profit": round(result.net_profit, 2),
|
||||
"total_trades": result.total_trades,
|
||||
"passing": result.passing,
|
||||
"best_score": round(self.best_result.score, 4) if self.best_result else 0,
|
||||
"best_pf": round(self.best_result.profit_factor, 3) if self.best_result else 0,
|
||||
"best_calmar": round(self.best_result.calmar, 3) if self.best_result else 0,
|
||||
"goal_status": goal_status,
|
||||
"targets_met": bool(targets_met),
|
||||
"improved": bool(improved),
|
||||
"confidence": round(suggestion.confidence, 2),
|
||||
"analysis": suggestion.analysis,
|
||||
"change_records": change_records,
|
||||
})
|
||||
|
||||
self._emit("run_complete", {
|
||||
"run_id": run_id,
|
||||
"phase": "phase2_ai",
|
||||
"net_profit": round(result.net_profit, 2),
|
||||
"calmar": round(result.calmar, 3),
|
||||
"profit_factor": round(result.profit_factor, 3),
|
||||
"win_rate": round(result.win_rate, 1),
|
||||
"max_drawdown": round(result.max_drawdown, 2),
|
||||
"total_trades": result.total_trades,
|
||||
"passing": result.passing,
|
||||
"score": round(result.score, 4),
|
||||
"progress_pct": round(self.pipeline._run_count / max(self.pipeline._total_runs, 1) * 100),
|
||||
})
|
||||
|
||||
status = "✅" if result.passing else "❌"
|
||||
self._log(
|
||||
"info" if result.passing else "warning",
|
||||
f" {status} iter={iteration} | PF={result.profit_factor:.2f} | "
|
||||
f"Calmar={result.calmar:.2f} | DD={result.max_drawdown:.1f}% | "
|
||||
f"trades={result.total_trades} | confidence={suggestion.confidence:.2f}"
|
||||
)
|
||||
|
||||
return self.best_result
|
||||
|
||||
# ── Initialization ────────────────────────────────────────────────────────
|
||||
|
||||
def _initialize_from_seeds(self, seed_results: list[RankedResult]) -> None:
|
||||
"""Register Phase 1 results as seen and find initial best."""
|
||||
for r in seed_results:
|
||||
self._mark_seen(r.params)
|
||||
|
||||
passing = [r for r in seed_results if r.passing]
|
||||
if passing:
|
||||
self.best_result = max(passing, key=lambda r: r.score)
|
||||
self._log("info",
|
||||
f"AI loop seed: best Phase 1 result is {self.best_result.run_id} "
|
||||
f"(PF={self.best_result.profit_factor:.2f}, score={self.best_result.score:.4f})"
|
||||
)
|
||||
|
||||
# Populate initial iteration history from Phase 1 top results
|
||||
top_seeds = sorted(passing, key=lambda r: r.score, reverse=True)[:5]
|
||||
for i, r in enumerate(top_seeds):
|
||||
self._iteration_history.append({
|
||||
"iteration": f"p1_top{i+1}",
|
||||
"run_id": r.run_id,
|
||||
"score": round(r.score, 4),
|
||||
"pf": round(r.profit_factor, 3),
|
||||
"calmar": round(r.calmar, 3),
|
||||
"dd": round(r.max_drawdown, 2),
|
||||
"trades": r.total_trades,
|
||||
"changes": [], # LHS seeds have no "changes"
|
||||
"params": r.params,
|
||||
})
|
||||
|
||||
# ── AI interaction ────────────────────────────────────────────────────────
|
||||
|
||||
def _get_suggestion(
|
||||
self, schema_info: list[dict], targets: dict
|
||||
) -> AIParamSuggestion:
|
||||
"""Ask AIReasoner for the next parameter set."""
|
||||
reasoner: AIReasoner = self.pipeline._ai_reasoner
|
||||
if not reasoner or not reasoner.enabled:
|
||||
return AIParamSuggestion(
|
||||
analysis="AI unavailable — using random escape.",
|
||||
changes=[], confidence=0.0, goal_status={}, error="no_ai",
|
||||
)
|
||||
|
||||
current_params = self.best_result.params if self.best_result else self.schema.defaults()
|
||||
|
||||
return reasoner.suggest_next_params(
|
||||
current_best_params=current_params,
|
||||
schema_info=schema_info,
|
||||
iteration_history=self._iteration_history,
|
||||
targets=targets,
|
||||
)
|
||||
|
||||
# ── Parameter manipulation ────────────────────────────────────────────────
|
||||
|
||||
def _build_schema_info(self) -> list[dict]:
|
||||
"""Convert schema optimizable params to serializable dicts for the AI prompt."""
|
||||
return [
|
||||
{
|
||||
"name": p.name,
|
||||
"type": p.type,
|
||||
"min": p.min,
|
||||
"max": p.max,
|
||||
"step": p.step,
|
||||
"default": p.default,
|
||||
"enum_values": p.enum_values if p.type == "enum" else [],
|
||||
}
|
||||
for p in self.schema.optimizable()
|
||||
]
|
||||
|
||||
def _apply_changes(self, base: dict, changes: list[dict]) -> dict:
|
||||
"""
|
||||
Apply AI-suggested changes to base params.
|
||||
Uses ParameterDef.clamp() to enforce valid ranges and types.
|
||||
"""
|
||||
result = dict(base)
|
||||
param_map = {p.name: p for p in self.schema.optimizable()}
|
||||
|
||||
for change in changes:
|
||||
name = change.get("param", "")
|
||||
value = change.get("value")
|
||||
if name not in param_map or value is None:
|
||||
continue
|
||||
pdef = param_map[name]
|
||||
try:
|
||||
result[name] = pdef.clamp(float(value) if pdef.type in ("float", "int") else value)
|
||||
except Exception as e:
|
||||
logger.debug(f"AIGuidedLoop: skipping change {name}={value}: {e}")
|
||||
|
||||
return result
|
||||
|
||||
def _build_change_records(
|
||||
self, before: dict, after: dict, ai_changes: list[dict]
|
||||
) -> list[dict]:
|
||||
"""
|
||||
Produce a list of {param, from, to, reason} records for the UI.
|
||||
|
||||
The AI's suggested changes may include a `reason` per change; we match
|
||||
those by name. Parameters that differ without a matching reason still
|
||||
get recorded (labelled "random perturbation").
|
||||
"""
|
||||
reason_by_param = {
|
||||
c.get("param"): c.get("reason", "").strip()
|
||||
for c in (ai_changes or [])
|
||||
if c.get("param")
|
||||
}
|
||||
records = []
|
||||
for name, new_val in after.items():
|
||||
old_val = before.get(name)
|
||||
if old_val == new_val:
|
||||
continue
|
||||
records.append({
|
||||
"param": name,
|
||||
"from": old_val,
|
||||
"to": new_val,
|
||||
"reason": reason_by_param.get(name) or "random perturbation (escape from stuck region)",
|
||||
})
|
||||
return records
|
||||
|
||||
def _random_escape(self, base: dict) -> dict:
|
||||
"""Random perturbation when stuck — perturbs 2-4 random optimizable params by ±30% of range."""
|
||||
opts = self.schema.optimizable()
|
||||
if not opts:
|
||||
return dict(base)
|
||||
|
||||
candidate = dict(base)
|
||||
n_perturb = min(len(opts), self._rng.randint(2, 4))
|
||||
to_perturb = self._rng.sample(opts, n_perturb)
|
||||
|
||||
for p in to_perturb:
|
||||
if p.type == "bool":
|
||||
candidate[p.name] = not candidate.get(p.name, p.default)
|
||||
elif p.type == "enum":
|
||||
candidate[p.name] = self._rng.choice(p.enum_values)
|
||||
else:
|
||||
span = float(p.max) - float(p.min)
|
||||
delta = span * 0.30 * self._rng.choice([-1, 1])
|
||||
candidate[p.name] = p.clamp(float(candidate.get(p.name, p.default)) + delta)
|
||||
|
||||
return candidate
|
||||
|
||||
def _ensure_unique(self, params: dict, max_attempts: int = 5) -> dict:
|
||||
"""If params already seen, perturb until unique (or give up)."""
|
||||
for _ in range(max_attempts):
|
||||
if self._hash(params) not in self._seen_hashes:
|
||||
return params
|
||||
params = self._random_escape(params)
|
||||
return params # best effort
|
||||
|
||||
def _mark_seen(self, params: dict) -> None:
|
||||
self._seen_hashes.add(self._hash(params))
|
||||
|
||||
@staticmethod
|
||||
def _hash(params: dict) -> str:
|
||||
key = json.dumps(params, sort_keys=True, default=str)
|
||||
return hashlib.md5(key.encode()).hexdigest()
|
||||
|
||||
# ── Best tracking ─────────────────────────────────────────────────────────
|
||||
|
||||
def _update_best(self, result: RankedResult) -> None:
|
||||
if result.passing:
|
||||
if self.best_result is None or result.score > self.best_result.score:
|
||||
self.best_result = result
|
||||
|
||||
# ── Stop conditions ───────────────────────────────────────────────────────
|
||||
|
||||
def _diagnose_failure(self, result: RankedResult, targets: dict) -> str:
|
||||
"""Human-readable reason this iteration didn't pass quality gates."""
|
||||
reasons = []
|
||||
if result.max_drawdown > targets.get("max_drawdown_pct", 20.0):
|
||||
reasons.append(f"drawdown too high ({result.max_drawdown:.1f}%)")
|
||||
if result.profit_factor < targets.get("min_profit_factor", 1.5):
|
||||
reasons.append(f"profit factor too low ({result.profit_factor:.2f})")
|
||||
if result.calmar < targets.get("min_calmar", 0.5):
|
||||
reasons.append(f"Calmar too low ({result.calmar:.2f})")
|
||||
if result.net_profit <= 0:
|
||||
reasons.append(f"unprofitable (${result.net_profit:.0f})")
|
||||
if result.total_trades < 10:
|
||||
reasons.append(f"too few trades ({result.total_trades})")
|
||||
return ", ".join(reasons) or "result below quality threshold"
|
||||
|
||||
def _targets_met(self, result: RankedResult, targets: dict) -> bool:
|
||||
if not result or not result.passing:
|
||||
return False
|
||||
return (
|
||||
result.profit_factor >= targets.get("min_profit_factor", 1.5)
|
||||
and result.max_drawdown <= targets.get("max_drawdown_pct", 20.0)
|
||||
and result.calmar >= targets.get("min_calmar", 0.5)
|
||||
)
|
||||
|
||||
def _check_stuck(self) -> bool:
|
||||
"""Return True if last STUCK_WINDOW iterations improved less than STUCK_THRESHOLD."""
|
||||
ai_iters = [h for h in self._iteration_history if str(h.get("iteration", "")).startswith(("1","2","3","4","5","6","7","8","9"))]
|
||||
if len(ai_iters) < self.STUCK_WINDOW:
|
||||
return False
|
||||
recent_scores = [h["score"] for h in ai_iters[-self.STUCK_WINDOW:]]
|
||||
return (max(recent_scores) - min(recent_scores)) < self.STUCK_THRESHOLD
|
||||
|
||||
# ── History tracking ──────────────────────────────────────────────────────
|
||||
|
||||
def _record_iteration(
|
||||
self,
|
||||
iteration: int,
|
||||
run_id: str,
|
||||
result: RankedResult,
|
||||
suggestion: AIParamSuggestion,
|
||||
) -> None:
|
||||
self._iteration_history.append({
|
||||
"iteration": iteration,
|
||||
"run_id": run_id,
|
||||
"score": round(result.score, 4),
|
||||
"pf": round(result.profit_factor, 3),
|
||||
"calmar": round(result.calmar, 3),
|
||||
"dd": round(result.max_drawdown, 2),
|
||||
"trades": result.total_trades,
|
||||
"changes": suggestion.changes,
|
||||
"params": result.params,
|
||||
})
|
||||
|
||||
# ── Pipeline helpers ──────────────────────────────────────────────────────
|
||||
|
||||
def _emit(self, event: str, data: dict = {}) -> None:
|
||||
self.pipeline._emit(event, data)
|
||||
|
||||
def _log(self, level: str, msg: str) -> None:
|
||||
self.pipeline._log(level, msg)
|
||||
|
||||
def _think(self, msg: str, kind: str = "info", iteration: Optional[int] = None, meta: Optional[dict] = None) -> None:
|
||||
"""Stream an AI-thinking message to the dashboard."""
|
||||
self.pipeline._emit_thinking(msg, kind=kind, iteration=iteration, meta=meta)
|
||||
+549
-52
@@ -13,17 +13,18 @@ Architecture:
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import os
|
||||
import random
|
||||
import time
|
||||
import uuid
|
||||
import threading
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Any, Optional, Callable
|
||||
from typing import Optional
|
||||
|
||||
import yaml
|
||||
from loguru import logger
|
||||
|
||||
from ea.registry import EARegistry, EAProfile
|
||||
from ea.registry import EARegistry
|
||||
from ea.schema import ParameterSchema
|
||||
from mt5.ini_builder import IniBuilder
|
||||
from mt5.runner import MT5Runner
|
||||
@@ -39,6 +40,11 @@ from optimizer.budget import BudgetManager
|
||||
|
||||
import pandas as pd
|
||||
|
||||
from analysis.equity_curve import EquityCurveAnalyzer
|
||||
from analysis.time_performance import TimePerformanceAnalyzer
|
||||
from analysis.ai_reasoner import AIReasoner
|
||||
from analysis.ai_reasoner_config import load_api_key
|
||||
|
||||
BASE_DIR = Path(__file__).parent.parent
|
||||
RUNS_DIR = BASE_DIR / "runs"
|
||||
DB_PATH = BASE_DIR / "optimizer.db"
|
||||
@@ -78,6 +84,18 @@ class OptimizationPipeline:
|
||||
self.best_set_path: Optional[Path] = None
|
||||
self.run_start_ts: Optional[float] = None
|
||||
|
||||
# AI & analysis state
|
||||
self._ai_reasoner: Optional[AIReasoner] = None
|
||||
self._ai_insights: list[dict] = []
|
||||
self._run_findings: dict[str, list] = {}
|
||||
self._run_insights: dict[str, dict] = {}
|
||||
self._baseline_metrics: Optional[dict] = None
|
||||
|
||||
# Running best (updated each run so status can serve it live)
|
||||
self._live_best: Optional[RankedResult] = None
|
||||
# All completed runs (for history restoration)
|
||||
self._completed_runs: list[dict] = []
|
||||
|
||||
# ── Public API ────────────────────────────────────────────────────────────
|
||||
|
||||
def configure(self, session: SessionConfig) -> None:
|
||||
@@ -86,20 +104,36 @@ class OptimizationPipeline:
|
||||
def stop(self) -> None:
|
||||
self._stop_flag = True
|
||||
self._emit("status_change", {"state": "stopping"})
|
||||
self._emit_early_termination(
|
||||
reason_code="user_stop",
|
||||
message="User requested stop. Finishing current run and exiting.",
|
||||
details={"phase": self._phase},
|
||||
)
|
||||
|
||||
def get_status(self) -> dict:
|
||||
elapsed = int(time.time() - self.run_start_ts) if self.run_start_ts else 0
|
||||
# Use final_result if set, otherwise best seen so far during the run
|
||||
best = self.final_result or self._live_best
|
||||
return {
|
||||
"state": "running" if self.running else "idle",
|
||||
"phase": self._phase,
|
||||
"run_count": self._run_count,
|
||||
"total_runs": self._total_runs,
|
||||
"best_score": round(self.best_result.score, 4) if self.best_result else 0.0,
|
||||
"best_score": round(best.score, 4) if best else 0.0,
|
||||
"verdict": self.verdict,
|
||||
"elapsed_s": elapsed,
|
||||
"ea_name": self.session.ea_name if self.session else "",
|
||||
"symbol": self.session.symbol if self.session else "",
|
||||
"timeframe": self.session.timeframe if self.session else "",
|
||||
"has_insight": bool(self._ai_insights),
|
||||
"insight_count": len(self._ai_insights),
|
||||
"latest_insight": self._ai_insights[-1] if self._ai_insights else None,
|
||||
"best_net_profit": round(best.net_profit, 2) if best else None,
|
||||
"best_calmar": round(best.calmar, 3) if best else None,
|
||||
"best_pf": round(best.profit_factor, 3) if best else None,
|
||||
"best_win_rate": round(best.win_rate, 1) if best else None,
|
||||
"best_max_drawdown": round(best.max_drawdown, 2) if best else None,
|
||||
"best_total_trades": best.total_trades if best else None,
|
||||
}
|
||||
|
||||
# ── Main entry point ──────────────────────────────────────────────────────
|
||||
@@ -147,6 +181,13 @@ class OptimizationPipeline:
|
||||
ranker = ResultRanker(weights=cfg.scoring_weights)
|
||||
budget = BudgetManager(cfg.budget_minutes)
|
||||
|
||||
# Initialize AI reasoning layer
|
||||
api_key = load_api_key(self.config_path)
|
||||
self._ai_reasoner = AIReasoner(api_key=api_key)
|
||||
self._ai_insights.clear()
|
||||
self._run_findings.clear()
|
||||
self._run_insights.clear()
|
||||
|
||||
budget.start()
|
||||
|
||||
# ── Phase 1: Broad Discovery ─────────────────────────────────────────
|
||||
@@ -155,6 +196,11 @@ class OptimizationPipeline:
|
||||
|
||||
self._phase = "phase1"
|
||||
self._emit("phase_start", {"phase": "phase1", "total": cfg.phase1_samples})
|
||||
self._emit_thinking(
|
||||
f"Starting broad discovery with {cfg.phase1_samples} Latin-Hypercube samples. "
|
||||
f"Scanning parameter space to find profitable regions before focused refinement.",
|
||||
kind="reasoning",
|
||||
)
|
||||
self._log("info", f"━━ Phase 1: Broad Discovery ({cfg.phase1_samples} configurations) ━━")
|
||||
|
||||
samples = sampler.sample(schema, cfg.phase1_samples)
|
||||
@@ -176,19 +222,17 @@ class OptimizationPipeline:
|
||||
phase1_raw.append(result)
|
||||
self._run_count += 1
|
||||
|
||||
# Track live best and completed runs for status/history APIs
|
||||
if self._live_best is None or result.score > self._live_best.score:
|
||||
self._live_best = result
|
||||
run_dict = self._make_run_dict(run_id, result, "phase1")
|
||||
self._completed_runs.append(run_dict)
|
||||
|
||||
# Emit progress after each run
|
||||
self._emit("run_complete", {
|
||||
"run_id": run_id,
|
||||
"phase": "phase1",
|
||||
**run_dict,
|
||||
"run_number": i + 1,
|
||||
"total": cfg.phase1_samples,
|
||||
"net_profit": round(result.net_profit, 2),
|
||||
"calmar": round(result.calmar, 3),
|
||||
"profit_factor": round(result.profit_factor, 3),
|
||||
"win_rate": round(result.win_rate, 1),
|
||||
"max_drawdown": round(result.max_drawdown, 2),
|
||||
"total_trades": result.total_trades,
|
||||
"passing": result.passing,
|
||||
"progress_pct": round((i + 1) / cfg.phase1_samples * 100),
|
||||
"budget_summary": budget.summary(),
|
||||
})
|
||||
@@ -221,19 +265,101 @@ class OptimizationPipeline:
|
||||
"or try a different timeframe."
|
||||
)
|
||||
})
|
||||
self._emit_early_termination(
|
||||
reason_code="no_profit",
|
||||
message="Optimization stopped early: Phase 1 found no profitable configuration.",
|
||||
details={
|
||||
"phase": "phase1",
|
||||
"total_tested": len(self.phase1_results),
|
||||
"suggestions": [
|
||||
"Try a different date range",
|
||||
"Check EA settings for issues",
|
||||
"Try a different timeframe",
|
||||
],
|
||||
},
|
||||
)
|
||||
self._emit_thinking(
|
||||
"No profitable configuration found across the broad scan. "
|
||||
"Strategy is unstable under current EA settings — stopping optimization.",
|
||||
kind="warning",
|
||||
)
|
||||
self._log("error", "❌ No profitable configuration found in Phase 1. Stopping.")
|
||||
return
|
||||
|
||||
if self._stop_flag:
|
||||
return
|
||||
|
||||
# ── Phase 2: Deep Refinement ─────────────────────────────────────────
|
||||
# ── Phase 2: Refinement (AI-Guided or Random Neighbor) ───────────────
|
||||
if not budget.can_fit(3):
|
||||
self._log("warning", "⏱ Not enough budget for Phase 2 — using Phase 1 winner directly")
|
||||
self.final_result = top5[0]
|
||||
|
||||
elif cfg.autonomous_mode and self._ai_reasoner and self._ai_reasoner.enabled:
|
||||
# ── AI-Guided Autonomous Loop ─────────────────────────────────────
|
||||
self._phase = "phase2"
|
||||
self._emit("phase_start", {
|
||||
"phase": "phase2",
|
||||
"total": cfg.autonomous_max_iterations,
|
||||
"mode": "autonomous",
|
||||
})
|
||||
self._emit_thinking(
|
||||
f"Phase 1 found {len(top5)} promising candidates. Best Calmar is "
|
||||
f"{top5[0].calmar:.2f}. Switching to autonomous AI loop — I'll read each "
|
||||
f"result, decide what parameters to change, and iterate toward the targets.",
|
||||
kind="reasoning",
|
||||
)
|
||||
self._log("info",
|
||||
f"━━ Phase 2: Autonomous AI Loop "
|
||||
f"(up to {cfg.autonomous_max_iterations} iterations) ━━"
|
||||
)
|
||||
|
||||
from optimizer.ai_guided_loop import AIGuidedLoop
|
||||
loop = AIGuidedLoop(
|
||||
pipeline=self, schema=schema, cfg=cfg,
|
||||
builder=builder, runner=runner, parser=parser,
|
||||
store=store, writer=writer, ranker=ranker,
|
||||
profile=profile, budget=budget,
|
||||
)
|
||||
targets = {
|
||||
"min_profit_factor": cfg.target_profit_factor,
|
||||
"max_drawdown_pct": cfg.target_max_drawdown_pct,
|
||||
"min_calmar": cfg.target_min_calmar,
|
||||
}
|
||||
loop.run(
|
||||
seed_results=self.phase1_results,
|
||||
max_iterations=cfg.autonomous_max_iterations,
|
||||
targets=targets,
|
||||
)
|
||||
|
||||
self.phase2_results = loop.all_results
|
||||
all_candidates = [r for r in (self.phase1_results + loop.all_results) if r.passing]
|
||||
all_ranked = ranker.rank(all_candidates) if all_candidates else ranker.rank(self.phase1_results)
|
||||
self.final_result = all_ranked[0] if all_ranked else top5[0]
|
||||
|
||||
self._emit("phase2_complete", {
|
||||
"best_run_id": self.final_result.run_id,
|
||||
"best_score": round(self.final_result.score, 4),
|
||||
"best_profit": round(self.final_result.net_profit, 2),
|
||||
"best_calmar": round(self.final_result.calmar, 3),
|
||||
"mode": "autonomous",
|
||||
"iterations": len(loop.all_results),
|
||||
})
|
||||
self._log("info",
|
||||
f"AI Loop complete. Best: {self.final_result.run_id} "
|
||||
f"(PF={self.final_result.profit_factor:.2f}, "
|
||||
f"profit=${self.final_result.net_profit:.0f}, "
|
||||
f"calmar={self.final_result.calmar:.2f})"
|
||||
)
|
||||
|
||||
else:
|
||||
# ── Original Random-Neighbor Phase 2 ─────────────────────────────
|
||||
self._phase = "phase2"
|
||||
self._emit("phase_start", {"phase": "phase2", "total": cfg.phase2_samples})
|
||||
self._emit_thinking(
|
||||
f"Phase 2 starting in classic mode: sampling {cfg.phase2_samples} neighbors "
|
||||
f"around the top {min(3, len(top5))} Phase-1 winners (no AI loop).",
|
||||
kind="info",
|
||||
)
|
||||
self._log("info", f"━━ Phase 2: Deep Refinement (refining top {min(3, len(top5))} configs) ━━")
|
||||
|
||||
top3 = top5[:3]
|
||||
@@ -264,20 +390,19 @@ class OptimizationPipeline:
|
||||
phase2_raw.append(result)
|
||||
self._run_count += 1
|
||||
|
||||
run_dict_p2 = self._make_run_dict(run_id, result, "phase2")
|
||||
self._completed_runs.append(run_dict_p2)
|
||||
if self._live_best is None or result.score > self._live_best.score:
|
||||
self._live_best = result
|
||||
self._emit("run_complete", {
|
||||
"run_id": run_id,
|
||||
"phase": "phase2",
|
||||
"net_profit": round(result.net_profit, 2),
|
||||
"calmar": round(result.calmar, 3),
|
||||
"passing": result.passing,
|
||||
**run_dict_p2,
|
||||
"progress_pct": round(self._run_count / self._total_runs * 100),
|
||||
})
|
||||
|
||||
# Best from Phase 1 + Phase 2 combined
|
||||
all_results = list(self.phase1_results) + ranker.rank(phase2_raw)
|
||||
all_ranked = ranker.rank(
|
||||
[r for r in all_results if r.passing]
|
||||
or list(self.phase1_results) # fallback to Phase 1 if P2 all fail
|
||||
or list(self.phase1_results)
|
||||
)
|
||||
self.phase2_results = ranker.rank(phase2_raw)
|
||||
self.final_result = all_ranked[0] if all_ranked else top5[0]
|
||||
@@ -299,16 +424,56 @@ class OptimizationPipeline:
|
||||
# ── Phase 3: Validation ───────────────────────────────────────────────
|
||||
if not budget.can_fit(2):
|
||||
self._log("warning", "⏱ Not enough budget for Phase 3 validation — skipping OOS test")
|
||||
self._emit_early_termination(
|
||||
reason_code="budget_exhausted",
|
||||
message="Skipping validation: not enough time budget remaining.",
|
||||
details={"phase": "phase3"},
|
||||
)
|
||||
self.verdict = "RISKY"
|
||||
oos_result = None
|
||||
else:
|
||||
self._phase = "phase3"
|
||||
self._emit("phase_start", {"phase": "phase3", "total": cfg.phase3_samples})
|
||||
|
||||
# Count the validation runs we intend to run so progress makes sense
|
||||
opts = schema.optimizable()
|
||||
plan_sens = bool(opts) and budget.can_fit(2)
|
||||
planned_runs = 1 + (2 if plan_sens else 0) # 1 OOS + 2 sens
|
||||
self._emit("validation_start", {
|
||||
"best_run_id": self.final_result.run_id,
|
||||
"planned_runs": planned_runs,
|
||||
"oos_start": cfg.val_start,
|
||||
"oos_end": cfg.val_end,
|
||||
"train_start": cfg.train_start,
|
||||
"train_end": cfg.train_end,
|
||||
"sensitivity": plan_sens,
|
||||
})
|
||||
self._emit_thinking(
|
||||
f"Entering validation phase. Testing best config {self.final_result.run_id} "
|
||||
f"on out-of-sample data ({cfg.val_start} → {cfg.val_end}) + sensitivity checks.",
|
||||
kind="reasoning",
|
||||
)
|
||||
self._log("info", "━━ Phase 3: Validation (out-of-sample + sensitivity) ━━")
|
||||
|
||||
# OOS test
|
||||
# ── OOS test ──
|
||||
oos_id = f"oos_{datetime.utcnow().strftime('%Y%m%d_%H%M%S')}"
|
||||
self._log("info", f" OOS test: {cfg.val_start} → {cfg.val_end}")
|
||||
self._emit("validation_run_start", {
|
||||
"run_id": oos_id,
|
||||
"kind": "oos",
|
||||
"label": "Out-of-Sample",
|
||||
"description": f"Re-testing best config on unseen data: {cfg.val_start} → {cfg.val_end}",
|
||||
"index": 1,
|
||||
"total": planned_runs,
|
||||
"period_start": cfg.val_start,
|
||||
"period_end": cfg.val_end,
|
||||
"params": self.final_result.params,
|
||||
})
|
||||
self._emit_thinking(
|
||||
"Running out-of-sample test: if the strategy holds up on data it wasn't "
|
||||
"optimized on, the edge is real — not curve-fit noise.",
|
||||
kind="hypothesis",
|
||||
)
|
||||
t0 = time.time()
|
||||
oos_result = self._execute_run(
|
||||
oos_id, self.final_result.params,
|
||||
@@ -318,20 +483,47 @@ class OptimizationPipeline:
|
||||
budget.record_run(time.time() - t0)
|
||||
self._run_count += 1
|
||||
|
||||
self._emit("run_complete", {
|
||||
"run_id": oos_id,
|
||||
"phase": "phase3_oos",
|
||||
"net_profit": round(oos_result.net_profit, 2),
|
||||
"calmar": round(oos_result.calmar, 3),
|
||||
"passing": oos_result.passing,
|
||||
oos_dict = self._make_run_dict(oos_id, oos_result, "phase3_oos")
|
||||
self._completed_runs.append(oos_dict)
|
||||
self._emit("run_complete", {**oos_dict, "progress_pct": round(self._run_count / self._total_runs * 100)})
|
||||
self._emit("validation_run_complete", {
|
||||
"run_id": oos_id,
|
||||
"kind": "oos",
|
||||
"label": "Out-of-Sample",
|
||||
"index": 1,
|
||||
"total": planned_runs,
|
||||
"net_profit": round(oos_result.net_profit, 2),
|
||||
"profit_factor": round(oos_result.profit_factor, 3),
|
||||
"calmar": round(oos_result.calmar, 3),
|
||||
"max_drawdown": round(oos_result.max_drawdown, 2),
|
||||
"total_trades": oos_result.total_trades,
|
||||
"passing": oos_result.passing,
|
||||
})
|
||||
# Thinking narration on OOS outcome
|
||||
oos_ratio = (oos_result.calmar / max(self.final_result.calmar, 0.001)) if oos_result.calmar else 0
|
||||
if oos_result.net_profit > 0 and oos_ratio >= 0.50:
|
||||
self._emit_thinking(
|
||||
f"OOS profitable: ${oos_result.net_profit:.0f} with Calmar {oos_result.calmar:.2f} "
|
||||
f"(≈{oos_ratio*100:.0f}% of training performance). The edge generalizes.",
|
||||
kind="success",
|
||||
)
|
||||
else:
|
||||
self._emit_thinking(
|
||||
f"OOS weak: profit ${oos_result.net_profit:.0f}, Calmar {oos_result.calmar:.2f} — "
|
||||
f"strategy likely overfit to the training period.",
|
||||
kind="warning",
|
||||
)
|
||||
|
||||
# Sensitivity test (2 runs: nudge top param up and down)
|
||||
# ── Sensitivity test (2 runs: nudge top param up and down) ──
|
||||
sens_results = []
|
||||
opts = schema.optimizable()
|
||||
if opts and budget.can_fit(2):
|
||||
top_param = opts[0] # first optimizable param
|
||||
for direction in [1, -1]:
|
||||
self._emit_thinking(
|
||||
f"Sensitivity check: nudging `{top_param.name}` ±20% to see if performance "
|
||||
f"survives small parameter drift.",
|
||||
kind="reasoning",
|
||||
)
|
||||
for i, direction in enumerate([1, -1], start=2):
|
||||
if budget.is_exhausted() or self._stop_flag:
|
||||
break
|
||||
nudged = dict(self.final_result.params)
|
||||
@@ -340,6 +532,19 @@ class OptimizationPipeline:
|
||||
nudged[top_param.name] = top_param.clamp(current + direction * span * 0.20)
|
||||
|
||||
sens_id = f"sens_{direction}_{datetime.utcnow().strftime('%H%M%S')}"
|
||||
label = f"Sensitivity ({top_param.name} {'+' if direction > 0 else '-'}20%)"
|
||||
self._emit("validation_run_start", {
|
||||
"run_id": sens_id,
|
||||
"kind": "sensitivity",
|
||||
"label": label,
|
||||
"description": f"Nudging `{top_param.name}` to {nudged[top_param.name]} "
|
||||
f"(from {current}) to test parameter stability.",
|
||||
"index": i,
|
||||
"total": planned_runs,
|
||||
"period_start": cfg.train_start,
|
||||
"period_end": cfg.train_end,
|
||||
"params": nudged,
|
||||
})
|
||||
t0 = time.time()
|
||||
sr = self._execute_run(
|
||||
sens_id, nudged, cfg.train_start, cfg.train_end,
|
||||
@@ -349,9 +554,44 @@ class OptimizationPipeline:
|
||||
sens_results.append(sr)
|
||||
self._run_count += 1
|
||||
|
||||
sens_dict = self._make_run_dict(sens_id, sr, "phase3_sens")
|
||||
self._completed_runs.append(sens_dict)
|
||||
self._emit("run_complete", {**sens_dict, "progress_pct": round(self._run_count / max(self._total_runs, 1) * 100)})
|
||||
self._emit("validation_run_complete", {
|
||||
"run_id": sens_id,
|
||||
"kind": "sensitivity",
|
||||
"label": label,
|
||||
"index": i,
|
||||
"total": planned_runs,
|
||||
"net_profit": round(sr.net_profit, 2),
|
||||
"profit_factor": round(sr.profit_factor, 3),
|
||||
"calmar": round(sr.calmar, 3),
|
||||
"max_drawdown": round(sr.max_drawdown, 2),
|
||||
"total_trades": sr.total_trades,
|
||||
"passing": sr.passing,
|
||||
})
|
||||
|
||||
# Determine verdict
|
||||
self.verdict = self._determine_verdict(self.final_result, oos_result, sens_results)
|
||||
|
||||
# Validation done — narrate the conclusion
|
||||
self._emit("validation_done", {
|
||||
"verdict": self.verdict,
|
||||
"oos_passing": bool(oos_result and oos_result.passing),
|
||||
"sens_passing": sum(1 for s in sens_results if s.passing),
|
||||
"sens_total": len(sens_results),
|
||||
})
|
||||
verdict_narration = {
|
||||
"RECOMMENDED": "Validation passed on all fronts. Strategy is robust and ready for deployment.",
|
||||
"RISKY": "Validation mixed. Strategy may work but has stability concerns — proceed with care.",
|
||||
"NOT_RELIABLE": "Validation failed. Strategy is not reliable — likely overfit or unstable.",
|
||||
}.get(self.verdict, "Validation complete.")
|
||||
self._emit_thinking(
|
||||
verdict_narration,
|
||||
kind=("success" if self.verdict == "RECOMMENDED"
|
||||
else "warning" if self.verdict == "RISKY" else "warning"),
|
||||
)
|
||||
|
||||
# ── Generate .set output ─────────────────────────────────────────────
|
||||
self.best_set_path = self._write_set_file(self.final_result, schema, cfg)
|
||||
|
||||
@@ -359,6 +599,7 @@ class OptimizationPipeline:
|
||||
self._emit("optimization_complete", {
|
||||
"verdict": self.verdict,
|
||||
"best_run_id": self.final_result.run_id,
|
||||
"score": round(self.final_result.score, 4),
|
||||
"net_profit": round(self.final_result.net_profit, 2),
|
||||
"calmar": round(self.final_result.calmar, 3),
|
||||
"profit_factor": round(self.final_result.profit_factor, 3),
|
||||
@@ -386,6 +627,11 @@ class OptimizationPipeline:
|
||||
builder, runner, parser, store, writer, ranker, profile
|
||||
) -> RankedResult:
|
||||
"""Execute one MT5 backtest and return a RankedResult."""
|
||||
# Demo mode short-circuit — generate synthetic metrics so judges can see the
|
||||
# AI loop without an MT5 install. Toggle with APEX_DEMO_MODE=1.
|
||||
if os.environ.get("APEX_DEMO_MODE", "").strip() in ("1", "true", "yes"):
|
||||
return self._execute_demo_run(run_id, params, phase, ranker)
|
||||
|
||||
run_dir = RUNS_DIR / run_id
|
||||
run_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
@@ -420,20 +666,135 @@ class OptimizationPipeline:
|
||||
return ranker.make_result(run_id, params, phase, None,
|
||||
error=result.error_message)
|
||||
|
||||
metrics, _ = parser.parse(result.report_xml, result.report_html)
|
||||
metrics, trades = parser.parse(result.report_xml, result.report_html)
|
||||
if metrics is None:
|
||||
return ranker.make_result(run_id, params, phase, None,
|
||||
error="parse_failed")
|
||||
|
||||
metrics.run_id = run_id
|
||||
writer.write(run_id, metrics, pd.DataFrame(), [], params)
|
||||
|
||||
return ranker.make_result(run_id, params, phase, metrics)
|
||||
# Run analysis & AI reasoning
|
||||
findings = self._analyze_run(run_id, metrics, trades or [], params)
|
||||
self._reason_about_run(run_id, metrics, findings, params)
|
||||
|
||||
trades_df = pd.DataFrame()
|
||||
if trades:
|
||||
try:
|
||||
trades_df = pd.DataFrame([t.model_dump() for t in trades])
|
||||
except Exception:
|
||||
try:
|
||||
trades_df = pd.DataFrame([vars(t) for t in trades])
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Build RankedResult first so we have the ranked_score for summary.json
|
||||
ranked = ranker.make_result(run_id, params, phase, metrics)
|
||||
|
||||
ai_insight = self._run_insights.get(run_id)
|
||||
writer.write(
|
||||
run_id, metrics, trades_df, findings, params,
|
||||
phase=phase,
|
||||
ranked_score=ranked.score,
|
||||
ai_insight=ai_insight,
|
||||
)
|
||||
|
||||
return ranked
|
||||
|
||||
except Exception as e:
|
||||
logger.warning(f"[{run_id}] Run error: {e}")
|
||||
return ranker.make_result(run_id, params, phase, None, error=str(e))
|
||||
|
||||
# ── Demo mode (synthetic backtest) ────────────────────────────────────────
|
||||
|
||||
def _execute_demo_run(self, run_id: str, params: dict, phase: str, ranker) -> RankedResult:
|
||||
"""
|
||||
Generate a synthetic, deterministic-but-realistic RankedResult from the
|
||||
params hash. Lets the AI loop run end-to-end without MT5 installed.
|
||||
|
||||
The metrics improve as parameters approach a hidden "sweet spot" so the
|
||||
AI can hill-climb in a way judges can observe. We also add small jitter
|
||||
to look organic.
|
||||
"""
|
||||
from data.models import RunMetrics # local import to avoid cycle
|
||||
|
||||
# Hash params → stable seed per config
|
||||
key = ",".join(f"{k}={v}" for k, v in sorted(params.items()))
|
||||
seed = int(hashlib.md5(key.encode()).hexdigest()[:12], 16)
|
||||
rng = random.Random(seed)
|
||||
|
||||
# Hidden sweet-spot hash drift: a deterministic "true score" between 0–1
|
||||
# based on a smooth function of parameter values. Small param changes
|
||||
# produce small score changes — that's what lets the AI hill-climb.
|
||||
true_score = 0.0
|
||||
for k, v in sorted(params.items()):
|
||||
try:
|
||||
fv = float(v)
|
||||
# Map value into [-1,1] using a stable hash, multiply by a gentle
|
||||
# bias toward "moderate" values (sweet spot is mid-range).
|
||||
slot = (int(hashlib.md5(k.encode()).hexdigest()[:8], 16) % 100) / 100.0
|
||||
normalized = (fv % 100) / 100.0
|
||||
true_score += 1.0 - abs(normalized - slot)
|
||||
except Exception:
|
||||
true_score += 0.5
|
||||
if params:
|
||||
true_score /= len(params)
|
||||
true_score = max(0.05, min(0.98, true_score))
|
||||
|
||||
# Add jitter for realism (±10%)
|
||||
true_score *= rng.uniform(0.90, 1.10)
|
||||
true_score = max(0.05, min(0.99, true_score))
|
||||
|
||||
# Project onto realistic metric ranges
|
||||
profit_factor = round(0.7 + true_score * 1.8, 3) # 0.7–2.5
|
||||
calmar = round(true_score * 1.4, 3) # 0.0–1.4
|
||||
max_drawdown = round(28 - true_score * 22, 2) # 28%→6%
|
||||
win_rate = round(40 + true_score * 25, 1) # 40%→65%
|
||||
total_trades = int(80 + rng.random() * 220) # 80–300
|
||||
net_profit = round((profit_factor - 1) * 5000 * (1 + rng.uniform(-0.2, 0.2)), 2)
|
||||
|
||||
# Out-of-sample tends to be slightly worse (more realistic)
|
||||
if phase.startswith("phase3_oos"):
|
||||
profit_factor *= 0.85
|
||||
calmar *= 0.80
|
||||
net_profit *= 0.75
|
||||
|
||||
avg_trade = net_profit / max(total_trades, 1)
|
||||
winners = int(total_trades * (win_rate / 100.0))
|
||||
losers = max(1, total_trades - winners)
|
||||
# Derive avg win/loss consistent with profit_factor: PF = (winners*avg_win)/(losers*|avg_loss|)
|
||||
avg_loss = -abs(avg_trade) * (1 + 1.5 / max(profit_factor, 0.5))
|
||||
avg_win = (profit_factor * losers * abs(avg_loss)) / max(winners, 1)
|
||||
|
||||
metrics = RunMetrics(
|
||||
run_id=run_id,
|
||||
net_profit=net_profit,
|
||||
profit_factor=profit_factor,
|
||||
calmar_ratio=calmar,
|
||||
max_drawdown_pct=max_drawdown / 100.0,
|
||||
max_drawdown_abs=round(net_profit * (max_drawdown / 100.0) * 1.2 + 200, 2),
|
||||
win_rate=win_rate / 100.0,
|
||||
total_trades=total_trades,
|
||||
recovery_factor=round(profit_factor * 1.5, 2),
|
||||
sharpe_ratio=round(true_score * 1.8, 2),
|
||||
avg_win=round(avg_win, 2),
|
||||
avg_loss=round(avg_loss, 2),
|
||||
largest_loss=round(avg_loss * 3.5, 2),
|
||||
expected_payoff=round(avg_trade, 2),
|
||||
)
|
||||
|
||||
# Simulate per-run latency so the dashboard feels alive — not instant.
|
||||
# Each run takes ~1.5s so a 10-iteration AI loop is ~15s total.
|
||||
delay = float(os.environ.get("APEX_DEMO_RUN_SECONDS", "1.5"))
|
||||
time.sleep(max(0.05, delay))
|
||||
|
||||
# Run AI reasoning if enabled — same flow as live mode
|
||||
try:
|
||||
self._reason_about_run(run_id, metrics, [], params)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return ranker.make_result(run_id, params, phase, metrics)
|
||||
|
||||
# ── Verdict logic ─────────────────────────────────────────────────────────
|
||||
|
||||
def _determine_verdict(
|
||||
@@ -501,37 +862,173 @@ class OptimizationPipeline:
|
||||
|
||||
# ── Helpers ───────────────────────────────────────────────────────────────
|
||||
|
||||
def _make_run_dict(self, run_id: str, result: RankedResult, phase: str) -> dict:
|
||||
"""Canonical run dict used for both _completed_runs and run_complete emits."""
|
||||
d = {
|
||||
"run_id": run_id,
|
||||
"phase": phase,
|
||||
"ts": datetime.utcnow().isoformat(),
|
||||
"net_profit": round(result.net_profit, 2),
|
||||
"calmar": round(result.calmar, 3),
|
||||
"profit_factor": round(result.profit_factor, 3),
|
||||
"win_rate": round(result.win_rate, 1),
|
||||
"max_drawdown": round(result.max_drawdown, 2),
|
||||
"total_trades": result.total_trades,
|
||||
"passing": result.passing,
|
||||
"score": round(result.score, 4),
|
||||
"params": result.params,
|
||||
}
|
||||
insight = self._run_insights.get(run_id)
|
||||
if insight:
|
||||
d["ai_insight"] = insight
|
||||
return d
|
||||
|
||||
def _result_to_dict(self, r: RankedResult) -> dict:
|
||||
return {
|
||||
"run_id": r.run_id,
|
||||
"rank": r.rank,
|
||||
"score": round(r.score, 4),
|
||||
"net_profit": round(r.net_profit, 2),
|
||||
"calmar": round(r.calmar, 3),
|
||||
d = {
|
||||
"run_id": r.run_id,
|
||||
"rank": r.rank,
|
||||
"score": round(r.score, 4),
|
||||
"net_profit": round(r.net_profit, 2),
|
||||
"calmar": round(r.calmar, 3),
|
||||
"profit_factor": round(r.profit_factor, 3),
|
||||
"win_rate": round(r.win_rate, 1),
|
||||
"max_drawdown": round(r.max_drawdown, 2),
|
||||
"total_trades": r.total_trades,
|
||||
"passing": r.passing,
|
||||
"win_rate": round(r.win_rate, 1),
|
||||
"max_drawdown": round(r.max_drawdown, 2),
|
||||
"total_trades": r.total_trades,
|
||||
"passing": r.passing,
|
||||
"params": r.params, # full param dict
|
||||
"params_summary": self._params_summary(r.params),
|
||||
}
|
||||
# Attach AI insight if available
|
||||
insight = self._run_insights.get(r.run_id)
|
||||
if insight:
|
||||
d["ai_insight"] = insight
|
||||
return d
|
||||
|
||||
@staticmethod
|
||||
def _params_summary(params: dict) -> str:
|
||||
"""Show a few key params for display."""
|
||||
keys = ["InpRiskPercent", "InpRRRatio", "InpMaxDailyLossPct",
|
||||
"InpTrailStartPips", "InpMinScore", "InpSessionStart", "InpSessionEnd"]
|
||||
"""Show a few key params for display — works with any EA."""
|
||||
parts = []
|
||||
for k in keys:
|
||||
if k in params:
|
||||
short = k.replace("Inp", "")
|
||||
parts.append(f"{short}={params[k]}")
|
||||
for k, v in list(params.items())[:8]:
|
||||
short = k.replace("Inp", "").replace("inp", "")[:12]
|
||||
parts.append(f"{short}={v}")
|
||||
return " | ".join(parts[:4])
|
||||
|
||||
def _analyze_run(self, run_id: str, metrics, trades: list, params: dict) -> list:
|
||||
"""Run Tier 1 analyzers on trade data. Always available from HTML report."""
|
||||
findings = []
|
||||
if not trades:
|
||||
return findings
|
||||
|
||||
try:
|
||||
trades_df = pd.DataFrame([t.model_dump() for t in trades])
|
||||
except Exception:
|
||||
try:
|
||||
trades_df = pd.DataFrame([vars(t) for t in trades])
|
||||
except Exception:
|
||||
return findings
|
||||
|
||||
if trades_df.empty:
|
||||
return findings
|
||||
|
||||
for analyzer_cls in (EquityCurveAnalyzer, TimePerformanceAnalyzer):
|
||||
try:
|
||||
az = analyzer_cls()
|
||||
result = az.analyze(trades_df)
|
||||
if isinstance(result, list):
|
||||
findings.extend(result)
|
||||
elif result is not None:
|
||||
findings.append(result)
|
||||
except Exception as e:
|
||||
logger.debug(f"[{run_id}] {analyzer_cls.__name__} skipped: {e}")
|
||||
|
||||
self._run_findings[run_id] = findings
|
||||
return findings
|
||||
|
||||
def _reason_about_run(
|
||||
self, run_id: str, metrics, findings: list, params: dict
|
||||
) -> Optional[dict]:
|
||||
"""Call AI reasoner and emit insight via SocketIO."""
|
||||
if not self._ai_reasoner or not self._ai_reasoner.enabled:
|
||||
return None
|
||||
try:
|
||||
history = [
|
||||
{
|
||||
"run_id": r.run_id, "score": r.score,
|
||||
"calmar": r.calmar, "pf": r.profit_factor, "phase": r.phase,
|
||||
}
|
||||
for r in (self.phase1_results + self.phase2_results)[-5:]
|
||||
]
|
||||
insight = self._ai_reasoner.analyze(
|
||||
findings=findings,
|
||||
metrics=metrics,
|
||||
run_history=history,
|
||||
current_params=params,
|
||||
)
|
||||
d = insight.to_dict()
|
||||
d["run_id"] = run_id
|
||||
self._ai_insights.append(d)
|
||||
self._run_insights[run_id] = d
|
||||
self._emit("ai_insight", d)
|
||||
return d
|
||||
except Exception as e:
|
||||
logger.warning(f"[{run_id}] AI reasoning failed: {e}")
|
||||
return None
|
||||
|
||||
def get_latest_insight(self) -> Optional[dict]:
|
||||
"""Return the most recent AI insight."""
|
||||
return self._ai_insights[-1] if self._ai_insights else None
|
||||
|
||||
def get_all_insights(self) -> list[dict]:
|
||||
return list(self._ai_insights)
|
||||
|
||||
def _log(self, level: str, msg: str) -> None:
|
||||
getattr(logger, level, logger.info)(msg)
|
||||
self._emit("log", {"level": level, "msg": msg})
|
||||
|
||||
def _emit_thinking(
|
||||
self,
|
||||
msg: str,
|
||||
kind: str = "info",
|
||||
iteration: Optional[int] = None,
|
||||
meta: Optional[dict] = None,
|
||||
) -> None:
|
||||
"""
|
||||
Emit an AI-thinking stream event — distinct from system logs.
|
||||
Shows up in the dashboard's "Live AI Thinking Feed".
|
||||
|
||||
kind: 'info' | 'reasoning' | 'decision' | 'warning' | 'success' | 'hypothesis'
|
||||
"""
|
||||
payload = {
|
||||
"msg": msg,
|
||||
"kind": kind,
|
||||
"iteration": iteration,
|
||||
"phase": self._phase,
|
||||
"ts": datetime.utcnow().isoformat(),
|
||||
}
|
||||
if meta:
|
||||
payload["meta"] = meta
|
||||
self._emit("ai_thinking", payload)
|
||||
|
||||
def _emit_early_termination(self, reason_code: str, message: str, details: dict = None) -> None:
|
||||
"""
|
||||
Surface an early stop to the user.
|
||||
reason_code examples:
|
||||
- 'no_profit' (Phase 1 found nothing)
|
||||
- 'targets_met' (AI loop hit all quality targets)
|
||||
- 'budget_exhausted'(time budget used up)
|
||||
- 'user_stop' (user clicked Stop)
|
||||
- 'stuck_escape' (optimizer stuck, bailing)
|
||||
"""
|
||||
payload = {
|
||||
"reason": reason_code,
|
||||
"message": message,
|
||||
"phase": self._phase,
|
||||
"ts": datetime.utcnow().isoformat(),
|
||||
}
|
||||
if details:
|
||||
payload["details"] = details
|
||||
self._emit("early_termination", payload)
|
||||
|
||||
def _emit(self, event: str, data: dict = {}) -> None:
|
||||
try:
|
||||
self.socketio.emit(event, data)
|
||||
|
||||
@@ -5,7 +5,7 @@ Passed from the /setup form → /api/start → pipeline.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
from dataclasses import dataclass, field, asdict
|
||||
from typing import Literal, Optional
|
||||
from typing import Literal
|
||||
|
||||
|
||||
ObjectiveType = Literal["balanced", "max_profit", "min_drawdown"]
|
||||
@@ -42,6 +42,13 @@ class SessionConfig:
|
||||
# Phase 1 sample count (derived from budget, not user-set directly)
|
||||
phase1_samples: int = 20
|
||||
|
||||
# ── Autonomous AI Loop settings ───────────────────────────────────────────
|
||||
autonomous_mode: bool = False # Replace Phase 2 with AI-guided loop
|
||||
autonomous_max_iterations: int = 10 # Max AI-directed iterations
|
||||
target_profit_factor: float = 1.5 # Stop when PF ≥ this
|
||||
target_max_drawdown_pct: float = 20.0 # Stop when DD ≤ this %
|
||||
target_min_calmar: float = 0.5 # Stop when Calmar ≥ this
|
||||
|
||||
# ── Derived helpers ───────────────────────────────────────────────────────
|
||||
|
||||
def derive_samples(self, seconds_per_run: float = 75.0) -> None:
|
||||
@@ -66,7 +73,8 @@ class SessionConfig:
|
||||
|
||||
@property
|
||||
def total_budget_runs(self) -> int:
|
||||
return self.phase1_samples + self.phase2_samples + self.phase3_samples
|
||||
phase2 = self.autonomous_max_iterations if self.autonomous_mode else self.phase2_samples
|
||||
return self.phase1_samples + phase2 + self.phase3_samples
|
||||
|
||||
# ── Scoring weights based on objective ───────────────────────────────────
|
||||
|
||||
@@ -99,6 +107,13 @@ class SessionConfig:
|
||||
def i(key, default=0):
|
||||
try: return int(form.get(key, default))
|
||||
except (ValueError, TypeError): return default
|
||||
def f(key, default=0.0):
|
||||
try: return float(form.get(key, default))
|
||||
except (ValueError, TypeError): return default
|
||||
def b(key):
|
||||
v = form.get(key, False)
|
||||
if isinstance(v, bool): return v
|
||||
return str(v).lower() in ("true", "1", "yes", "on")
|
||||
|
||||
cfg = cls(
|
||||
ea_name = s("ea_name", "LEGSTECH_EA_V2"),
|
||||
@@ -111,6 +126,12 @@ class SessionConfig:
|
||||
objective = s("objective", "balanced"),
|
||||
budget_minutes = i("budget_minutes", 60),
|
||||
selected_params = form.get("selected_params", []),
|
||||
# Autonomous loop
|
||||
autonomous_mode = b("autonomous_mode"),
|
||||
autonomous_max_iterations = i("autonomous_max_iterations", 10),
|
||||
target_profit_factor = f("target_profit_factor", 1.5),
|
||||
target_max_drawdown_pct = f("target_max_drawdown_pct", 20.0),
|
||||
target_min_calmar = f("target_min_calmar", 0.5),
|
||||
)
|
||||
cfg.derive_samples()
|
||||
return cfg
|
||||
|
||||
Reference in New Issue
Block a user