"""
================================================================================
Quantum Terminal — Account & Calibration API Routes
================================================================================
FastAPI router for Layer 3 (Account Management + Risk Governor) and
Layer 2 (Calibration Store) endpoints.

Wired into data_server.py via:
    from account_routes import create_account_router
    app.include_router(create_account_router(broadcast_event))

Endpoints:
    # ── Account & Risk ──
    GET    /api/account/status           — equity, P&L, circuit breakers, halt status
    GET    /api/account/settings         — current trading settings (prop firm rules)
    PATCH  /api/account/settings         — update trading settings
    POST   /api/account/sync             — force MT5 equity sync
    POST   /api/account/reset-baseline   — reset baseline (initial balance + clear breakers)
    GET    /api/account/trades           — recent trade log (last 50)

    # ── Calibration ──
    GET    /api/calibration/summary      — universe calibration badge summary
    GET    /api/calibration/management   — unified management dashboard (freshness + calibration + flow + anchor)
    GET    /api/calibration/{ticker}     — full calibration report for one asset
    GET    /api/calibration/badges/{ticker} — badge lookup for all setups on one asset
    GET    /api/calibration/detail/{ticker}/{signal_type} — IS/OOS drill-down for one setup
    GET    /api/calibration/outcomes/{ticker}/{signal_type} — raw trade records for chart viz
    POST   /api/calibration/run/{ticker} — trigger backtest calibration for one asset
    POST   /api/calibration/run          — trigger universe calibration (background)
    GET    /api/calibration/status       — calibration job status

    # ── Governor ──
    GET    /api/governor/status          — risk governor status
    POST   /api/governor/check           — pre-trade check (dry run)

    # ── Walk-Forward (Phase 3I) ──
    POST   /api/wf/run/{ticker}          — trigger WF engine for one asset
    GET    /api/wf/status                — WF job status

    # ── Flow Calibration (Phase 3I) ──
    POST   /api/flow/calibrate/{ticker}  — trigger flow calibrator + validator
    GET    /api/flow/status              — flow calibration job status
================================================================================
"""

import json
import asyncio
import logging
import threading
from debug_subprocess import debug_popen as _debug_popen
from typing import Optional, Callable, Awaitable
from pathlib import Path
from datetime import datetime, timezone, timedelta
from fastapi import APIRouter, HTTPException
from pydantic import BaseModel

log = logging.getLogger("account_routes")

PROJECT_ROOT = Path(__file__).resolve().parent


# ── Request models ──

class DebugConfigRequest(BaseModel):
    """Debug configuration update."""
    subprocess_debug: Optional[bool] = None


class SettingsUpdateRequest(BaseModel):
    """Partial settings update — only include fields to change."""
    account_size: Optional[float] = None
    max_daily_loss_pct: Optional[float] = None
    max_daily_profit_pct: Optional[float] = None
    max_total_drawdown_pct: Optional[float] = None
    max_trailing_drawdown_pct: Optional[float] = None
    max_open_positions: Optional[int] = None
    max_correlated_positions: Optional[int] = None
    risk_per_trade_pct: Optional[float] = None
    use_kelly: Optional[bool] = None
    kelly_fraction: Optional[float] = None
    min_calibration_grade: Optional[str] = None
    require_calibration: Optional[bool] = None
    trading_enabled: Optional[bool] = None
    # Auto Trading (Phase 3G)
    auto_trading_enabled: Optional[bool] = None
    auto_min_confidence: Optional[float] = None
    auto_min_grade: Optional[str] = None
    auto_max_daily_trades: Optional[int] = None
    auto_allowed_types: Optional[list] = None
    auto_allowed_tickers: Optional[list] = None
    auto_scan_interval: Optional[float] = None
    auto_log_only: Optional[bool] = None
    # Scheduler
    scheduler_enabled: Optional[bool] = None
    scheduler_weekly_enabled: Optional[bool] = None
    scheduler_weekly_day: Optional[int] = None
    scheduler_weekly_time: Optional[str] = None
    scheduler_daily_enabled: Optional[bool] = None
    scheduler_daily_time: Optional[str] = None
    scheduler_daily_skip_weekends: Optional[bool] = None
    scheduler_calc_before_weekly: Optional[bool] = None
    # Calibration
    calibration_workers: Optional[int] = None


class PreTradeCheckRequest(BaseModel):
    """Dry-run pre-trade check."""
    ticker: str
    direction: str         # "long" or "short"
    signal_type: str
    entry_price: float
    stop_loss: float
    target_price: float
    proposed_lots: float = 0.1


class CalibrationRunRequest(BaseModel):
    """Calibration run parameters."""
    lookback_months: int = 6
    mc_sims: int = 20000
    parallel_workers: int = 1  # CPU cores for sweep (1 = sequential)


# ── Router factory ──

def create_account_router(
    broadcast_event: Optional[Callable[[dict], Awaitable[None]]] = None,
) -> APIRouter:
    """
    Create account/calibration API router.

    Lazy-loads AccountManager, RiskGovernor, CalibrationStore on first call
    to avoid import errors if modules aren't deployed yet.
    """
    router = APIRouter(tags=["account"])

    # ── Lazy singletons ──
    _cache = {}

    def _get_account_manager():
        if 'acct' not in _cache:
            try:
                from account_manager import get_account_manager
                _cache['acct'] = get_account_manager()
            except ImportError:
                raise HTTPException(503, "account_manager module not available")
        return _cache['acct']

    def _get_governor():
        if 'gov' not in _cache:
            try:
                from risk_governor import get_governor
                _cache['gov'] = get_governor()
            except ImportError:
                raise HTTPException(503, "risk_governor module not available")
        return _cache['gov']

    def _get_cal_store():
        if 'cal' not in _cache:
            try:
                from calibration_store import get_store
                _cache['cal'] = get_store()
            except ImportError:
                raise HTTPException(503, "calibration_store module not available")
        return _cache['cal']

    async def _notify(event_type: str, detail: str = ""):
        if broadcast_event:
            await broadcast_event({
                "type": "account_update",
                "detail": event_type,
                "message": detail,
                "timestamp": datetime.now(timezone.utc).isoformat(),
            })

    # ════════════════════════════════════════════════════════
    # DEBUG CONFIG — subprocess_debug flag
    # ════════════════════════════════════════════════════════

    _DEBUG_CONFIG_PATH = PROJECT_ROOT / "debug_config.json"

    def _load_debug_config() -> dict:
        """Load debug_config.json — returns defaults if missing."""
        try:
            if _DEBUG_CONFIG_PATH.exists():
                return json.loads(_DEBUG_CONFIG_PATH.read_text())
        except Exception:
            pass
        return {"subprocess_debug": False}

    def _save_debug_config(cfg: dict):
        try:
            _DEBUG_CONFIG_PATH.write_text(json.dumps(cfg, indent=2))
        except Exception as e:
            log.warning(f"Failed to save debug_config.json: {e}")

    @router.get("/api/dev/debug-config")
    async def get_debug_config():
        """Get current debug configuration."""
        return _load_debug_config()

    @router.patch("/api/dev/debug-config")
    async def update_debug_config(payload: DebugConfigRequest):
        """
        Update debug configuration.
          subprocess_debug (bool) — open visible console windows for subprocesses
        """
        cfg = _load_debug_config()
        updates = {k: v for k, v in payload.dict().items() if v is not None}
        cfg.update(updates)
        _save_debug_config(cfg)
        mode = "ON" if cfg.get("subprocess_debug") else "OFF"
        await _notify("debug_config_changed",
                      f"Subprocess debug mode: {mode} — takes effect on next calibration/WF run")
        return cfg

    # ════════════════════════════════════════════════════════
    # ACCOUNT & RISK ENDPOINTS
    # ════════════════════════════════════════════════════════

    @router.get("/api/account/status")
    async def get_account_status():
        """Full account status — equity, P&L, circuit breakers."""
        mgr = _get_account_manager()
        return mgr.get_status_dict()

    @router.get("/api/account/settings")
    async def get_account_settings():
        """Current trading settings."""
        mgr = _get_account_manager()
        return mgr.get_settings_dict()

    @router.patch("/api/account/settings")
    async def update_account_settings(req: SettingsUpdateRequest):
        """Update trading settings (partial — only include changed fields)."""
        mgr = _get_account_manager()
        # Build dict of non-None fields only
        updates = {k: v for k, v in req.dict().items() if v is not None}
        if not updates:
            raise HTTPException(400, "No fields to update")
        mgr.update_settings(updates)
        await _notify("settings_changed", f"Updated: {', '.join(updates.keys())}")
        return {"status": "ok", "updated": list(updates.keys()), "settings": mgr.get_settings_dict()}

    @router.post("/api/account/sync")
    async def force_mt5_sync():
        """Force MT5 equity sync."""
        mgr = _get_account_manager()
        success = await asyncio.to_thread(mgr.sync_from_mt5)
        if success:
            await _notify("mt5_sync", "Equity synced from MT5")
            return {"status": "ok", "equity": mgr.state.current_equity}
        else:
            return {"status": "failed", "message": "MT5 sync failed — check connection"}

    @router.post("/api/account/reset-baseline")
    async def reset_baseline():
        """
        Reset account baseline to current account_size.
        Clears all circuit breaker halts, resets initial_balance,
        peak_equity, and daily/weekly tracking.

        Call this after changing account_size or starting fresh.
        """
        mgr = _get_account_manager()
        mgr.reset_baseline()
        await _notify("baseline_reset", f"Baseline reset to ${mgr.state.initial_balance:,.2f}")
        return {
            "status": "ok",
            "initial_balance": mgr.state.initial_balance,
            "current_equity": mgr.state.current_equity,
            "peak_equity": mgr.state.peak_equity,
            "circuit_breakers_cleared": True,
        }

    @router.get("/api/account/trades")
    async def get_recent_trades():
        """Recent trade log (last 50)."""
        mgr = _get_account_manager()
        return {"trades": mgr.state.recent_trades}

    @router.get("/api/account/period-pnl")
    async def get_period_pnl():
        """
        Real closed-trade P&L from MT5 deal history.
        Periods: today, yesterday, this_week, last_week, this_month, last_month.
        Each period: {pnl, trades, pct}.
        """
        mgr = _get_account_manager()
        return await asyncio.to_thread(mgr.get_period_pnl)

    # ════════════════════════════════════════════════════════
    # GOVERNOR ENDPOINTS
    # ════════════════════════════════════════════════════════

    @router.get("/api/governor/status")
    async def get_governor_status():
        """Full risk governor status."""
        gov = _get_governor()
        return gov.get_status()

    @router.post("/api/governor/check")
    async def pre_trade_check(req: PreTradeCheckRequest):
        """
        Dry-run pre-trade check — tests if a trade would be allowed
        without actually placing it. Useful for UI preview.
        """
        gov = _get_governor()

        # Build a mock signal object for the check
        class _MockSignal:
            pass
        sig = _MockSignal()
        sig.ticker = req.ticker.upper()

        class _Dir:
            value = req.direction.lower()
        class _Type:
            value = req.signal_type
        sig.direction = _Dir()
        sig.signal_type = _Type()
        sig.entry_price = req.entry_price
        sig.stop_loss = req.stop_loss
        sig.target_price = req.target_price

        allowed, reason = gov.pre_trade_check(sig, proposed_lots=req.proposed_lots)
        adjusted_lots = req.proposed_lots
        adjust_notes = ""
        if allowed:
            adjusted_lots, adjust_notes = gov.adjust_position_size(sig, req.proposed_lots)

        return {
            "allowed": allowed,
            "reason": reason,
            "adjusted_lots": round(adjusted_lots, 4),
            "adjust_notes": adjust_notes,
        }

    # ════════════════════════════════════════════════════════
    # CALIBRATION ENDPOINTS
    # ════════════════════════════════════════════════════════

    @router.get("/api/calibration/summary")
    async def get_calibration_summary():
        """Universe calibration badge summary — for dashboard display."""
        store = _get_cal_store()
        return store.get_universe_summary()

    @router.get("/api/calibration/management")
    async def get_calibration_management():
        """
        Unified management dashboard data — aggregates freshness, calibration,
        flow params, and entry confirmation status across all assets.

        Used by CalibrationDashboard.jsx (Phase 3I).
        """
        import asyncio as _aio

        # ── Universe ──
        try:
            from config_manager import ConfigManager
            cm = ConfigManager()
            universe = cm.get_active_universe()
        except Exception:
            try:
                from server_config import ASSET_UNIVERSE
                universe = list(ASSET_UNIVERSE)
            except Exception:
                universe = []

        # ── Freshness (all 9 data types × universe) ──
        freshness = {}
        try:
            from data_freshness import DataFreshnessMonitor
            monitor = DataFreshnessMonitor()
            for ticker in universe:
                tf = monitor.get_ticker_freshness(ticker)
                freshness[ticker] = tf.to_dict()
        except ImportError:
            log.warning("data_freshness not available for management endpoint")
        except Exception as e:
            log.warning(f"Freshness fetch failed: {e}")

        # ── Calibration badges ──
        calibration = {}
        try:
            store = _get_cal_store()
            calibration = store.get_universe_summary()
        except Exception as e:
            log.warning(f"Calibration summary failed: {e}")

        # ── Flow params status ──
        flow = {"calibrated": [], "missing": [], "details": {}}
        flow_path = PROJECT_ROOT / "flow_params.json"
        if flow_path.exists():
            try:
                with open(flow_path, "r") as f:
                    flow_data = json.load(f)
                for ticker in universe:
                    if ticker in flow_data:
                        flow["calibrated"].append(ticker)
                        fd = flow_data[ticker]
                        flow["details"][ticker] = {
                            "calibrated_at": fd.get("calibrated_at", ""),
                            "wf_score": fd.get("wf_score", 0),
                            "wf_variance": fd.get("wf_variance", 0),
                            "classify_method": fd.get("classify_method", ""),
                        }
                    else:
                        flow["missing"].append(ticker)
            except Exception as e:
                log.warning(f"Flow params read failed: {e}")
                flow["missing"] = universe[:]
        else:
            flow["missing"] = universe[:]

        # ── Anchor params status ──
        anchor = {"calibrated": [], "missing": [], "details": {}}
        anchor_path = PROJECT_ROOT / "anchor_params.json"
        if anchor_path.exists():
            try:
                with open(anchor_path, "r") as f:
                    anchor_data = json.load(f)
                for ticker in universe:
                    if ticker in anchor_data:
                        anchor["calibrated"].append(ticker)
                        ad = anchor_data[ticker]
                        anchor["details"][ticker] = {
                            "threshold": ad.get("anchor_distance_max_sd", ""),
                            "calibrated_at": ad.get("calibrated_at", ""),
                        }
                    else:
                        anchor["missing"].append(ticker)
            except Exception as e:
                log.warning(f"Anchor params read failed: {e}")
                anchor["missing"] = universe[:]
        else:
            anchor["missing"] = universe[:]

        # ── Entry confirmation edge (from calibration reports) ──
        entry_conf = {}
        for ticker in universe:
            ticker_cal = calibration.get(ticker, {})
            setups = ticker_cal.get("setups", {})
            edges = {}
            for sig_type, badge in setups.items():
                edge = badge.get("confirmation_edge", "unknown")
                edges[sig_type] = edge
            has_any = any(e != "unknown" for e in edges.values())
            entry_conf[ticker] = {
                "calibrated": has_any,
                "edges": edges,
            }

        # ── Source-of-truth reference ──
        sources = {
            "freshness": "data_freshness.py — 9 data types per asset",
            "calibration": "calibration_reports/*.json — backtest_engine output",
            "flow": "flow_params.json — flow_calibrator.py output",
            "wf_settings": "best_wf_settings.json — walk_forward_engine output",
            "anchor_params": "anchor_params.json — anchor_signal_backtester output",
        }

        return {
            "universe": universe,
            "freshness": freshness,
            "calibration": calibration,
            "flow": flow,
            "anchor": anchor,
            "entry_confirmation": entry_conf,
            "sources": sources,
        }

    # ── Calibration job tracking (must be before {ticker} routes) ──
    _running_jobs = {}  # ticker → {"status": "running"/"done"/"error", ...}
    _cancel_tokens = {}  # ticker → threading.Event (set = cancel)

    @router.get("/api/calibration/status")
    async def get_calibration_status():
        """Get calibration job status for all tickers."""
        return _running_jobs

    @router.post("/api/calibration/cancel/{ticker}")
    async def cancel_calibration_ticker(ticker: str):
        """Cancel a running calibration for one asset."""
        canonical = ticker.upper()
        token = _cancel_tokens.get(canonical)
        if token:
            token.set()
            if canonical in _running_jobs:
                _running_jobs[canonical]["status"] = "cancelled"
            log.info(f"Calibration cancel requested for {canonical}")
            return {"cancelled": True, "ticker": canonical}
        return {"cancelled": False, "reason": f"No running job for {canonical}"}

    @router.post("/api/calibration/cancel")
    async def cancel_calibration_all():
        """Cancel all running calibrations."""
        cancelled = []
        for ticker, token in _cancel_tokens.items():
            if not token.is_set():
                token.set()
                if ticker in _running_jobs:
                    _running_jobs[ticker]["status"] = "cancelled"
                cancelled.append(ticker)
        log.info(f"Calibration cancel all: {cancelled}")
        return {"cancelled": cancelled}

    @router.get("/api/calibration/badges/{ticker}")
    async def get_calibration_badges(ticker: str):
        """Badge lookup for all setups on one asset."""
        store = _get_cal_store()
        badges = store.get_all_badges(ticker.upper())
        if not badges:
            return {"ticker": ticker.upper(), "calibrated": False, "badges": {}}
        return {"ticker": ticker.upper(), "calibrated": True, "badges": badges}

    @router.get("/api/calibration/detail/{ticker}/{signal_type}")
    async def get_calibration_detail(ticker: str, signal_type: str):
        """
        Detailed IS/OOS breakdown for one signal type on one asset.
        Used by the calibration page drill-down.

        Returns: {
            ticker, signal_type,
            overall: { grade, score, win_rate, expectancy, ... },
            in_sample: { grade, score, win_rate, total, ... },
            out_of_sample: { grade, score, win_rate, total, ... },
            overfitting_risk, config, date_range, computed_at,
        }
        """
        store = _get_cal_store()
        detail = store.get_detail(ticker.upper(), signal_type)
        if detail is None:
            raise HTTPException(
                404,
                f"No calibration detail for {ticker.upper()} / {signal_type}"
            )
        return detail

    @router.get("/api/calibration/outcomes/{ticker}/{signal_type}")
    async def get_calibration_outcomes(ticker: str, signal_type: str):
        """
        Raw trade outcome records for one signal type on one asset.
        Used by the trade chart visualization in the calibration page.

        Returns: [
            { eval_date, direction, entry_price, stop_loss, target_price,
              exit_price, exit_date, outcome, r_multiple, is_oos,
              entry_confirmed, confirmation_pattern, ... },
            ...
        ]
        """
        store = _get_cal_store()
        outcomes = store.get_outcomes(ticker.upper(), signal_type)
        if outcomes is None:
            raise HTTPException(
                404,
                f"No outcome data for {ticker.upper()} / {signal_type}. "
                f"Re-run calibration to generate outcomes."
            )
        return outcomes

    @router.get("/api/calibration/{ticker}")
    async def get_calibration_report(ticker: str):
        """Full calibration report for one asset."""
        store = _get_cal_store()
        report = store.load_report(ticker.upper())
        if report is None:
            raise HTTPException(404, f"No calibration report for {ticker.upper()}")
        return report

    @router.post("/api/calibration/run/{ticker}")
    async def run_calibration_ticker(ticker: str, req: CalibrationRunRequest = None):
        """
        Trigger backtest calibration for one asset (runs in background thread).
        Returns immediately — poll /api/calibration/status for progress.
        """
        canonical = ticker.upper()

        if canonical in _running_jobs and _running_jobs[canonical].get("status") == "running":
            raise HTTPException(409, f"Calibration already running for {canonical}")

        if req is None:
            req = CalibrationRunRequest()

        _running_jobs[canonical] = {
            "status": "running",
            "ticker": canonical,
            "started_at": datetime.now(timezone.utc).isoformat(),
            "pct": 0,
            "eval_date": "loading data...",
            "eval_num": 0,
            "eval_total": 0,
            "signals": 0,
            "wins": 0,
            "losses": 0,
            "elapsed_s": 0,
            "eta_s": 0,
        }

        # Create cancel token for this job
        cancel_token = threading.Event()
        _cancel_tokens[canonical] = cancel_token

        async def _run():
            try:
                from backtest_engine import BacktestEngine, BacktestConfig

                # Resolve parallel workers: request → account settings → default 1
                pw = req.parallel_workers
                if pw <= 1:
                    try:
                        mgr = _get_account_manager()
                        pw = mgr.get_settings_dict().get("calibration_workers", 1) or 1
                    except Exception:
                        pw = 1

                config = BacktestConfig(
                    mc_sims=req.mc_sims,
                    lookback_months=req.lookback_months,
                    parallel_workers=pw,
                )
                engine = BacktestEngine(config)

                def _on_progress(info):
                    """Update job status with live progress from engine."""
                    _running_jobs[canonical] = {
                        "status": "running",
                        "ticker": canonical,
                        "started_at": _running_jobs.get(canonical, {}).get("started_at", ""),
                        "pct": info.get("pct", 0),
                        "eval_date": info.get("eval_date", ""),
                        "eval_num": info.get("eval_num", 0),
                        "eval_total": info.get("eval_total", 0),
                        "signals": info.get("signals_so_far", 0),
                        "wins": info.get("wins_so_far", 0),
                        "losses": info.get("losses_so_far", 0),
                        "elapsed_s": info.get("elapsed_s", 0),
                        "eta_s": info.get("eta_s", 0),
                    }

                report = await asyncio.to_thread(
                    engine.run_with_atr_sweep, canonical,
                    lookback_months=req.lookback_months,
                    progress_callback=_on_progress,
                    cancel_token=cancel_token,
                )

                # Check if cancelled
                if cancel_token.is_set():
                    _running_jobs[canonical] = {
                        "status": "cancelled", "ticker": canonical,
                    }
                    log.info(f"Calibration cancelled for {canonical}")
                else:
                    # Save to store
                    store = _get_cal_store()
                    store.save_report(report)

                    _running_jobs[canonical] = {
                        "status": "done",
                        "ticker": canonical,
                        "overall_score": report.overall_score,
                        "overall_grade": report.overall_grade,
                        "completed_at": datetime.now(timezone.utc).isoformat(),
                    }

                    await _notify("calibration_complete",
                                  f"{canonical}: {report.overall_grade} ({report.overall_score:.2f})")

            except Exception as e:
                log.error(f"Calibration failed for {canonical}: {e}", exc_info=True)
                _running_jobs[canonical] = {
                    "status": "error",
                    "ticker": canonical,
                    "error": str(e),
                }
            finally:
                _cancel_tokens.pop(canonical, None)

        asyncio.create_task(_run())
        return {"status": "started", "ticker": canonical}

    @router.post("/api/calibration/run")
    async def run_calibration_universe(req: CalibrationRunRequest = None):
        """
        Trigger calibration for the full universe (background).
        Returns immediately — poll /api/calibration/status.
        """
        if req is None:
            req = CalibrationRunRequest()

        # Get universe from config_manager
        try:
            from config_manager import ConfigManager
            cm = ConfigManager()
            tickers = cm.get_active_universe()
        except Exception:
            try:
                from server_config import ASSET_UNIVERSE
                tickers = list(ASSET_UNIVERSE)
            except Exception:
                raise HTTPException(500, "Cannot determine asset universe")

        # Mark all as running
        for t in tickers:
            _running_jobs[t] = {
                "status": "queued",
                "ticker": t,
                "started_at": datetime.now(timezone.utc).isoformat(),
                "pct": 0,
                "eval_date": "",
                "eval_num": 0,
                "eval_total": 0,
                "signals": 0,
                "wins": 0,
                "losses": 0,
                "elapsed_s": 0,
                "eta_s": 0,
            }

        # Create a shared cancel token for the universe run
        universe_cancel = threading.Event()
        for t in tickers:
            _cancel_tokens[t] = universe_cancel

        async def _run_all():
            try:
                from backtest_engine import BacktestEngine, BacktestConfig

                # Resolve parallel workers: request → account settings → default 1
                pw = req.parallel_workers
                if pw <= 1:
                    try:
                        mgr = _get_account_manager()
                        pw = mgr.get_settings_dict().get("calibration_workers", 1) or 1
                    except Exception:
                        pw = 1

                config = BacktestConfig(
                    mc_sims=req.mc_sims,
                    lookback_months=req.lookback_months,
                    parallel_workers=pw,
                )
                engine = BacktestEngine(config)
                store = _get_cal_store()

                for i, ticker in enumerate(tickers):
                    # ── Cancel check between tickers ──
                    if universe_cancel.is_set():
                        for remaining in tickers[i:]:
                            _running_jobs[remaining] = {
                                "status": "cancelled", "ticker": remaining,
                            }
                        log.info(f"Universe calibration cancelled at ticker {i}/{len(tickers)}")
                        break

                    _running_jobs[ticker] = {
                        "status": "running",
                        "ticker": ticker,
                        "started_at": _running_jobs.get(ticker, {}).get("started_at", ""),
                        "pct": 0,
                        "eval_date": "loading data...",
                        "eval_num": 0,
                        "eval_total": 0,
                        "signals": 0,
                        "wins": 0,
                        "losses": 0,
                        "elapsed_s": 0,
                        "eta_s": 0,
                    }

                    def _make_cb(t):
                        """Create a closure-safe callback for this ticker."""
                        def _on_progress(info):
                            _running_jobs[t] = {
                                "status": "running",
                                "ticker": t,
                                "started_at": _running_jobs.get(t, {}).get("started_at", ""),
                                "pct": info.get("pct", 0),
                                "eval_date": info.get("eval_date", ""),
                                "eval_num": info.get("eval_num", 0),
                                "eval_total": info.get("eval_total", 0),
                                "signals": info.get("signals_so_far", 0),
                                "wins": info.get("wins_so_far", 0),
                                "losses": info.get("losses_so_far", 0),
                                "elapsed_s": info.get("elapsed_s", 0),
                                "eta_s": info.get("eta_s", 0),
                            }
                        return _on_progress

                    try:
                        report = await asyncio.to_thread(
                            engine.run_with_atr_sweep, ticker,
                            lookback_months=req.lookback_months,
                            progress_callback=_make_cb(ticker),
                            cancel_token=universe_cancel,
                        )

                        if universe_cancel.is_set():
                            _running_jobs[ticker] = {
                                "status": "cancelled", "ticker": ticker,
                            }
                        else:
                            store.save_report(report)
                            _running_jobs[ticker] = {
                                "status": "done",
                                "ticker": ticker,
                                "overall_score": report.overall_score,
                                "overall_grade": report.overall_grade,
                            }
                    except Exception as e:
                        _running_jobs[ticker] = {
                            "status": "error",
                            "ticker": ticker,
                            "error": str(e),
                        }

                if not universe_cancel.is_set():
                    await _notify("calibration_universe_complete",
                                  f"{len(tickers)} assets calibrated")

            except Exception as e:
                log.error(f"Universe calibration failed: {e}", exc_info=True)
            finally:
                for t in tickers:
                    _cancel_tokens.pop(t, None)

        asyncio.create_task(_run_all())
        return {"status": "started", "tickers": tickers}

    # ════════════════════════════════════════════════════════
    # AUTO EXECUTOR ENDPOINTS (Phase 3G)
    # ════════════════════════════════════════════════════════

    @router.get("/api/auto/status")
    async def get_auto_status():
        """Auto executor status — settings + today's activity."""
        mgr = _get_account_manager()
        settings = mgr.get_settings_dict()

        # Read today's trade count from auto log
        trades_today = 0
        signals_processed = 0
        auto_log_path = PROJECT_ROOT / "auto_executor_log.json"
        if auto_log_path.exists():
            try:
                with open(auto_log_path, "r") as f:
                    log_data = json.load(f)
                today_str = datetime.now(timezone.utc).strftime("%Y-%m-%d")
                for entry in reversed(log_data):
                    ts = entry.get("timestamp", "")
                    if not ts.startswith(today_str):
                        break
                    signals_processed += 1
                    if entry.get("action") in ("executed", "dry_run"):
                        trades_today += 1
            except Exception:
                pass

        return {
            "enabled": settings.get("auto_trading_enabled", False),
            "log_only_mode": settings.get("auto_log_only", True),
            "live_trading_on": mgr.settings.trading_enabled,
            "trades_today": trades_today,
            "signals_processed": signals_processed,
            "max_daily_trades": settings.get("auto_max_daily_trades", 5),
            "min_confidence": settings.get("auto_min_confidence", 0.60),
            "min_grade": settings.get("auto_min_grade", "C"),
            "allowed_types": settings.get("auto_allowed_types", []),
            "allowed_tickers": settings.get("auto_allowed_tickers", []) or "all",
            "scan_interval": settings.get("auto_scan_interval", 30.0),
        }

    @router.get("/api/auto/log")
    async def get_auto_log(limit: int = 50):
        """Recent auto executor actions (audit trail)."""
        auto_log_path = PROJECT_ROOT / "auto_executor_log.json"
        if not auto_log_path.exists():
            return {"actions": [], "count": 0}
        try:
            with open(auto_log_path, "r") as f:
                data = json.load(f)
            return {"actions": data[-limit:], "count": len(data)}
        except Exception:
            return {"actions": [], "count": 0}

    @router.get("/api/auto/log/summary")
    async def get_auto_log_summary(days: int = 30):
        """
        Performance summary from auto executor log.
        Aggregates by overall, per-ticker, per-signal-type, and per-ticker-type.
        """
        auto_log_path = PROJECT_ROOT / "auto_executor_log.json"
        if not auto_log_path.exists():
            return {"trades": 0, "summary": {}}
        try:
            with open(auto_log_path, "r") as f:
                all_data = json.load(f)
        except Exception:
            return {"trades": 0, "summary": {}}

        # Filter to requested time window
        cutoff = (datetime.now(timezone.utc) - timedelta(days=days)).isoformat()
        entries = [e for e in all_data
                   if e.get("action") in ("executed", "dry_run")
                   and e.get("timestamp", "") >= cutoff]

        if not entries:
            return {"trades": 0, "days": days, "summary": {}}

        # Build aggregations
        def _agg(subset):
            count = len(subset)
            tickers = list(set(e.get("ticker", "") for e in subset))
            types = list(set(e.get("signal_type", "") for e in subset))
            avg_conf = sum(e.get("confidence", 0) for e in subset) / count if count else 0
            avg_rr = sum(e.get("risk_reward", 0) for e in subset) / count if count else 0
            total_risk = sum(e.get("risk_usd", 0) for e in subset)
            avg_risk_pct = sum(e.get("risk_pct", 0) for e in subset) / count if count else 0
            grades = {}
            for e in subset:
                g = e.get("calibration_grade", "?")
                grades[g] = grades.get(g, 0) + 1
            regimes = {}
            for e in subset:
                r = e.get("regime", "?")
                regimes[r] = regimes.get(r, 0) + 1
            directions = {"long": 0, "short": 0}
            for e in subset:
                d = e.get("direction", "")
                if d in directions:
                    directions[d] += 1
            return {
                "count": count,
                "avg_confidence": round(avg_conf, 4),
                "avg_risk_reward": round(avg_rr, 2),
                "total_risk_usd": round(total_risk, 2),
                "avg_risk_pct": round(avg_risk_pct, 4),
                "grade_distribution": grades,
                "regime_distribution": regimes,
                "direction_split": directions,
                "tickers": tickers,
                "signal_types": types,
            }

        # Overall
        summary = {"overall": _agg(entries)}

        # Per ticker
        by_ticker = {}
        for e in entries:
            t = e.get("ticker", "?")
            by_ticker.setdefault(t, []).append(e)
        summary["by_ticker"] = {t: _agg(v) for t, v in by_ticker.items()}

        # Per signal type
        by_type = {}
        for e in entries:
            st = e.get("signal_type", "?")
            by_type.setdefault(st, []).append(e)
        summary["by_signal_type"] = {st: _agg(v) for st, v in by_type.items()}

        # Per ticker × signal type
        by_ticker_type = {}
        for e in entries:
            key = f"{e.get('ticker', '?')}|{e.get('signal_type', '?')}"
            by_ticker_type.setdefault(key, []).append(e)
        summary["by_ticker_type"] = {k: _agg(v) for k, v in by_ticker_type.items()}

        return {
            "trades": len(entries),
            "days": days,
            "summary": summary,
        }

    # ════════════════════════════════════════════════════════
    # TRADE MANAGER ENDPOINTS
    # ════════════════════════════════════════════════════════

    @router.get("/api/trade-manager/status")
    async def get_trade_manager_status():
        """Trade manager status — managed positions and modes."""
        try:
            from trade_manager import get_trade_manager
            tm = get_trade_manager()
            return tm.get_status()
        except ImportError:
            raise HTTPException(503, "trade_manager module not available")

    # ════════════════════════════════════════════════════════
    # WALK-FORWARD CALIBRATION (Phase 3I)
    # ════════════════════════════════════════════════════════

    _wf_jobs = {}  # ticker → {"status": "running"/"done"/"error", ...}

    @router.post("/api/wf/run/{ticker}")
    async def run_wf_ticker(ticker: str):
        """
        Trigger walk-forward engine for one asset (background subprocess).
        Calls: python walk_forward_engine.py --assets TICKER --sims 100000
        Writes to backtest_results/best_wf_settings.json (merge-on-save).
        Returns immediately — poll /api/wf/status for progress.
        """
        import subprocess
        canonical = ticker.upper()

        if canonical in _wf_jobs and _wf_jobs[canonical].get("status") == "running":
            raise HTTPException(409, f"WF already running for {canonical}")

        _wf_jobs[canonical] = {
            "status": "running",
            "ticker": canonical,
            "started_at": datetime.now(timezone.utc).isoformat(),
        }

        async def _run():
            try:
                # Walk-forward engine is verbose — redirect to log file, not PIPE.
                cmd = [
                    "python", str(PROJECT_ROOT / "walk_forward_engine.py"),
                    "--assets", canonical,
                    "--sims", "100000",
                    "--step", "5",
                ]
                wf_log_path = PROJECT_ROOT / "flow_output" / f"wf_{canonical}.log"
                await _notify("wf_started",
                              f"Walk-forward engine started for {canonical}")

                def _wf_runner():
                    p = _debug_popen(cmd, label=f"wf_{canonical}")
                    p.wait(timeout=900)
                    return p
                proc = await asyncio.to_thread(_wf_runner)

                if proc.returncode == 0:
                    _wf_jobs[canonical] = {
                        "status": "done",
                        "ticker": canonical,
                        "completed_at": datetime.now(timezone.utc).isoformat(),
                        "wf_log": str(wf_log_path),
                    }
                    await _notify("wf_done",
                                  f"Walk-forward complete for {canonical} — best_wf_settings.json updated")
                    log.info(f"WF completed for {canonical}")
                else:
                    tail = f"See subprocess_logs/wf_{canonical}.log" 
                    _wf_jobs[canonical] = {
                        "status": "error",
                        "ticker": canonical,
                        "error": tail,
                    }
                    await _notify("wf_error",
                                  f"Walk-forward FAILED for {canonical} — check flow_output/wf_{canonical}.log")
                    log.error(f"WF failed for {canonical} (log: {wf_log_path})")
            except subprocess.TimeoutExpired:
                _wf_jobs[canonical] = {
                    "status": "error", "ticker": canonical,
                    "error": "Timed out after 900s",
                }
            except Exception as e:
                _wf_jobs[canonical] = {
                    "status": "error", "ticker": canonical,
                    "error": str(e),
                }

        asyncio.create_task(_run())
        return {"status": "started", "ticker": canonical}

    @router.get("/api/wf/status")
    async def get_wf_status():
        """Walk-forward job status for all tickers."""
        return _wf_jobs

    # ════════════════════════════════════════════════════════
    # FLOW CALIBRATION (Phase 3I)
    # ════════════════════════════════════════════════════════

    _flow_jobs = {}  # ticker → {"status": "running"/"done"/"error", ...}

    @router.post("/api/flow/calibrate/{ticker}")
    async def run_flow_calibrate_ticker(ticker: str):
        """
        Trigger flow_calibrator.py for one asset (background subprocess).
        Updates flow_params.json for this ticker, then validates via
        flow_confirmation.py CLI diagnostic.
        Returns immediately — poll /api/flow/status for progress.
        """
        import subprocess
        canonical = ticker.upper()

        if canonical in _flow_jobs and _flow_jobs[canonical].get("status") == "running":
            raise HTTPException(409, f"Flow calibration already running for {canonical}")

        _flow_jobs[canonical] = {
            "status": "running",
            "ticker": canonical,
            "started_at": datetime.now(timezone.utc).isoformat(),
            "phase": "calibrating",
        }

        async def _run():
            try:
                # Phase 1: Run flow_calibrator.py for this ticker
                # IMPORTANT: flow_calibrator produces massive output (1944-combo grid search +
                # GPU progress bars). capture_output=True / PIPE deadlocks when the pipe buffer
                # fills (~64 KB on Windows). Redirect stdout+stderr to a log file instead.
                # Rule: Use DEVNULL/file not PIPE for verbose subprocesses.
                cal_cmd = [
                    "python", str(PROJECT_ROOT / "flow_calibrator.py"),
                    canonical,
                ]
                _flow_jobs[canonical]["phase"] = "calibrating"
                await _notify("flow_calibration_started",
                              f"Flow calibration started for {canonical} — GPU grid search running")

                def _cal_runner():
                    p = _debug_popen(cal_cmd, label=f"flow_calibrator_{canonical}")
                    p.wait(timeout=900)
                    return p
                proc = await asyncio.to_thread(_cal_runner)

                if proc.returncode != 0:
                    # Read tail of log for error context
                    tail = f"See subprocess_logs/flow_calibrator_{canonical}.log" 
                    _flow_jobs[canonical] = {
                        "status": "error", "ticker": canonical,
                        "phase": "calibration_failed",
                        "error": tail,
                    }
                    await _notify("flow_calibration_error",
                                  f"Flow calibration FAILED for {canonical} — check flow_output/calibrate_{canonical}.log")
                    log.error(f"Flow calibration failed for {canonical} (log: {cal_log_path})")
                    return

                # Phase 2: Run flow_confirmation.py CLI diagnostic to validate.
                # Validator output is small (~50 lines) — PIPE is safe here.
                _flow_jobs[canonical]["phase"] = "validating"
                await _notify("flow_validation_started",
                              f"Flow calibration complete for {canonical} — running validator")
                val_cmd = [
                    "python", str(PROJECT_ROOT / "flow_confirmation.py"),
                    canonical,
                ]
                val_proc = await asyncio.to_thread(
                    subprocess.run, val_cmd,
                    cwd=str(PROJECT_ROOT),
                    capture_output=True, text=True, timeout=120,
                    env={**__import__("os").environ, "PYTHONUTF8": "1"},
                )

                _flow_jobs[canonical] = {
                    "status": "done",
                    "ticker": canonical,
                    "completed_at": datetime.now(timezone.utc).isoformat(),
                    "validation_output": val_proc.stdout[-1000:] if val_proc.stdout else "",
                }
                await _notify("flow_calibration_done",
                              f"Flow calibration + validation complete for {canonical} — flow_params.json updated")
                log.info(f"Flow calibration + validation complete for {canonical}")

            except subprocess.TimeoutExpired:
                _flow_jobs[canonical] = {
                    "status": "error", "ticker": canonical,
                    "error": "Timed out after 900s",
                }
                await _notify("flow_calibration_error",
                              f"Flow calibration TIMED OUT for {canonical} (>900s)")
            except Exception as e:
                _flow_jobs[canonical] = {
                    "status": "error", "ticker": canonical,
                    "error": str(e),
                }
                await _notify("flow_calibration_error",
                              f"Flow calibration ERROR for {canonical}: {e}")

        asyncio.create_task(_run())
        return {"status": "started", "ticker": canonical}

    @router.get("/api/flow/status")
    async def get_flow_status():
        """Flow calibration job status for all tickers."""
        return _flow_jobs

    return router