From b7ea4186784a896f9c3e4cc87d4c747978cd5978 Mon Sep 17 00:00:00 2001 From: Quantum Terminal Date: Thu, 21 May 2026 01:44:50 +0300 Subject: [PATCH] =?UTF-8?q?Initial=20commit:=20Quantum=20Terminal=20?= =?UTF-8?q?=E2=80=94=20Free=20&=20Open=20Source=20Trading=20Platform?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 51 + LICENSE | 21 + README.md | 108 + backend/account_manager.py | 805 +++++++ backend/account_routes.py | 1186 ++++++++++ backend/cache_watcher.py | 323 +++ backend/chart_presets_routes.py | 82 + backend/chart_templates_routes.py | 84 + backend/config_manager.py | 1413 +++++++++++ backend/config_routes.py | 203 ++ backend/consumer_startup.py | 251 ++ backend/data_manager/__init__.py | 50 + backend/data_manager/cli.py | 134 ++ backend/data_manager/data_catalog.py | 253 ++ backend/data_manager/data_manager.py | 332 +++ backend/data_manager/data_store.py | 278 +++ backend/data_manager/fetchers/base_fetcher.py | 54 + backend/data_manager/fetchers/mt5_fetcher.py | 272 +++ backend/data_manager/forcaster_dmm_patch.py | 207 ++ backend/data_manager/test_data_manager.py | 318 +++ backend/data_server.py | 2097 +++++++++++++++++ backend/data_sync_client.py | 1378 +++++++++++ backend/execution_routes.py | 312 +++ backend/freshness_routes.py | 10 + backend/launcher.py | 323 +++ backend/logging_setup.py | 91 + backend/macro_data.py | 304 +++ backend/models.py | 178 ++ backend/mt5_routes.py | 346 +++ backend/options_routes.py | 142 ++ backend/periodic_sync.py | 172 ++ backend/providers/__init__.py | 68 + backend/providers/base_provider.py | 276 +++ backend/providers/mt5_provider.py | 1039 ++++++++ backend/providers/rithmic_provider.py | 758 ++++++ backend/providers/tradovate_provider.py | 454 ++++ backend/pulse_room/__init__.py | 10 + backend/pulse_room/builder.py | 463 ++++ backend/pulse_room/routes.py | 123 + backend/pulse_room/state.py | 118 + backend/pulse_room/sync_hook.py | 43 + backend/regime_routes.py | 108 + backend/requirements.txt | 9 + backend/server_config.py | 173 ++ backend/system_routes.py | 10 + backend/trade_mapper_routes.py | 10 + backend/tradovate_routes.py | 144 ++ backend/version_info.py | 81 + electron_shell/main.js | 698 ++++++ electron_shell/preload.js | 101 + frontend_build/asset-manifest.json | 13 + frontend_build/index.html | 1 + frontend_build/public/index.html | 10 + frontend_build/sounds/band_extreem.wav | Bin 0 -> 194506 bytes frontend_build/sounds/connected.wav | Bin 0 -> 156242 bytes frontend_build/sounds/disconnected.wav | Bin 0 -> 176186 bytes frontend_build/sounds/order_filled.wav | Bin 0 -> 179954 bytes frontend_build/sounds/order_placed.wav | Bin 0 -> 185344 bytes frontend_build/sounds/regime_change.wav | Bin 0 -> 196936 bytes frontend_build/sounds/stoploss_hit.wav | Bin 0 -> 211658 bytes frontend_build/sounds/target_hit.wav | Bin 0 -> 172226 bytes frontend_build/sounds/you_have_a_msg.wav | Bin 0 -> 205024 bytes frontend_build/static/css/main.64fab706.css | 2 + .../static/css/main.64fab706.css.map | 1 + frontend_build/static/js/main.be9bfbf6.js | 3 + .../static/js/main.be9bfbf6.js.LICENSE.txt | 87 + frontend_build/static/js/main.be9bfbf6.js.map | 1 + frontend_build/static/mk_debug_telemetry.js | 244 ++ 68 files changed, 16826 insertions(+) create mode 100644 .gitignore create mode 100644 LICENSE create mode 100644 README.md create mode 100644 backend/account_manager.py create mode 100644 backend/account_routes.py create mode 100644 backend/cache_watcher.py create mode 100644 backend/chart_presets_routes.py create mode 100644 backend/chart_templates_routes.py create mode 100644 backend/config_manager.py create mode 100644 backend/config_routes.py create mode 100644 backend/consumer_startup.py create mode 100644 backend/data_manager/__init__.py create mode 100644 backend/data_manager/cli.py create mode 100644 backend/data_manager/data_catalog.py create mode 100644 backend/data_manager/data_manager.py create mode 100644 backend/data_manager/data_store.py create mode 100644 backend/data_manager/fetchers/base_fetcher.py create mode 100644 backend/data_manager/fetchers/mt5_fetcher.py create mode 100644 backend/data_manager/forcaster_dmm_patch.py create mode 100644 backend/data_manager/test_data_manager.py create mode 100644 backend/data_server.py create mode 100644 backend/data_sync_client.py create mode 100644 backend/execution_routes.py create mode 100644 backend/freshness_routes.py create mode 100644 backend/launcher.py create mode 100644 backend/logging_setup.py create mode 100644 backend/macro_data.py create mode 100644 backend/models.py create mode 100644 backend/mt5_routes.py create mode 100644 backend/options_routes.py create mode 100644 backend/periodic_sync.py create mode 100644 backend/providers/__init__.py create mode 100644 backend/providers/base_provider.py create mode 100644 backend/providers/mt5_provider.py create mode 100644 backend/providers/rithmic_provider.py create mode 100644 backend/providers/tradovate_provider.py create mode 100644 backend/pulse_room/__init__.py create mode 100644 backend/pulse_room/builder.py create mode 100644 backend/pulse_room/routes.py create mode 100644 backend/pulse_room/state.py create mode 100644 backend/pulse_room/sync_hook.py create mode 100644 backend/regime_routes.py create mode 100644 backend/requirements.txt create mode 100644 backend/server_config.py create mode 100644 backend/system_routes.py create mode 100644 backend/trade_mapper_routes.py create mode 100644 backend/tradovate_routes.py create mode 100644 backend/version_info.py create mode 100644 electron_shell/main.js create mode 100644 electron_shell/preload.js create mode 100644 frontend_build/asset-manifest.json create mode 100644 frontend_build/index.html create mode 100644 frontend_build/public/index.html create mode 100644 frontend_build/sounds/band_extreem.wav create mode 100644 frontend_build/sounds/connected.wav create mode 100644 frontend_build/sounds/disconnected.wav create mode 100644 frontend_build/sounds/order_filled.wav create mode 100644 frontend_build/sounds/order_placed.wav create mode 100644 frontend_build/sounds/regime_change.wav create mode 100644 frontend_build/sounds/stoploss_hit.wav create mode 100644 frontend_build/sounds/target_hit.wav create mode 100644 frontend_build/sounds/you_have_a_msg.wav create mode 100644 frontend_build/static/css/main.64fab706.css create mode 100644 frontend_build/static/css/main.64fab706.css.map create mode 100644 frontend_build/static/js/main.be9bfbf6.js create mode 100644 frontend_build/static/js/main.be9bfbf6.js.LICENSE.txt create mode 100644 frontend_build/static/js/main.be9bfbf6.js.map create mode 100644 frontend_build/static/mk_debug_telemetry.js diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..4c2df79 --- /dev/null +++ b/.gitignore @@ -0,0 +1,51 @@ +# === Python === +__pycache__/ +*.py[cod] +*$py.class +*.pyo +*.egg-info/ +*.egg +dist/ +*.whl + +# === Virtual Environments === +.env +.venv +env/ +venv/ +ENV/ + +# === IDE / Editors === +.vscode/ +.idea/ +*.swp +*.swo +*~ + +# === Node.js === +node_modules/ + +# === Build Artifacts === +build/ +*.spec + +# === OS Files === +.DS_Store +Thumbs.db +desktop.ini + +# === Logs & Databases === +*.log +*.sqlite3 +*.db +logs/ + +# === Local Config (generated at runtime) === +consumer_config.ini +cache/ + +# === Data Library (local market data โ€” large files) === +data_library/ + +# === Test Caches === +.pytest_cache/ diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..10bc02e --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 Quantum Terminal Contributors + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 0000000..32c6ee7 --- /dev/null +++ b/README.md @@ -0,0 +1,108 @@ +# ๐ŸŒŒ Quantum Terminal + +
+

A Next-Generation Open Source Trading & Quantitative Analysis Platform

+ Python Version + FastAPI + License + Architecture +
+ +--- + +## ๐Ÿ“– About The Project (ู†ุธุฑุฉ ุนุงู…ุฉ ุนู„ู‰ ุงู„ู…ุดุฑูˆุน) + +**Quantum Terminal** is an advanced, fully free, and open-source trading terminal designed to bridge the gap between institutional-grade quantitative analysis and retail day trading. + +Unlike traditional platforms that rely on lagging indicators (like RSI or MACD), Quantum Terminal processes **real-time tick data** from professional brokerages and overlays complex mathematical probability models directly onto your charts. It acts as a localized display hub, separating heavy mathematical computations from real-time charting to ensure zero-latency trade monitoring. + +--- + +## โœจ Core Features (ุงู„ู…ู…ูŠุฒุงุช ุงู„ุฃุณุงุณูŠุฉ) + +### 1. ๐ŸŒ Multi-Provider Live Feeds (ุฑุจุท ู…ุจุงุดุฑ ู…ุน ุงู„ุฃุณูˆุงู‚) +Quantum Terminal is built to handle ultra-fast data streams from the most reliable data providers in the industry: +- **MetaTrader 5 (MT5):** Direct integration via Python API for Forex, Crypto, and CFD markets. +- **Rithmic:** High-frequency, sub-millisecond tick processing for CME Futures (NQ, ES). +- **Tradovate:** Seamless integration for retail futures traders. +*(The platform features a dedicated `focus_tick_loop` ensuring the chart you are actively watching refreshes at maximum FPS without bottlenecking the system).* + +### 2. ๐Ÿ“ Advanced Quantitative Models (ุงู„ู†ู…ุงุฐุฌ ุงู„ูƒู…ูŠุฉ ุงู„ู…ุชู‚ุฏู…ุฉ) +Move beyond simple support and resistance. The platform integrates institutional volatility models to forecast price action probabilities: +- **Volatility Cones:** Visualizing expected price ranges using Geometric Brownian Motion (**GBM**), Merton Jump Diffusion (**MJD**), and **Bates** models. +- **Scalp Bands & Micro Volatility:** Real-time bands designed specifically for high-frequency scalpers. +- **Probability Fields:** Heatmaps mapping the statistical likelihood of price reverting or trending. + +### 3. ๐ŸŒŠ Options Flow & Market Maker Positioning (ุชุญู„ูŠู„ ุณูŠูˆู„ุฉ ุงู„ุฎูŠุงุฑุงุช) +Understanding where Market Makers are hedging is critical. Quantum Terminal visualizes: +- **Gamma Exposure (GEX):** Identifies sticky price levels and volatility triggers. +- **Max Pain:** Highlights the strike price with the most open options contracts. +- **Option Walls:** Visualizes heavy Call/Put resistance and support barriers. + +### 4. ๐ŸŒ Macro Regime & Sector Rotation (ุชุญู„ูŠู„ ุงู„ู…ุงูƒุฑูˆ ูˆุงู„ุณูŠูˆู„ุฉ) +Using data from `yfinance`, the terminal analyzes global market regimes: +- Tracks ETF Sector Rotation to see where smart money is flowing. +- Identifies the current market regime (Risk-On, Risk-Off, Inflationary, etc.) to align your trades with the broader macroeconomic trend. + +--- + +## ๐Ÿ—๏ธ Architecture & Repository Structure (ู‡ูŠูƒู„ูŠุฉ ุงู„ู…ุดุฑูˆุน) + +The project follows a decoupled architecture, ensuring the UI remains buttery smooth while the backend handles heavy lifting. + +```text +QuantumTerminal/ +โ”œโ”€โ”€ backend/ # The Core Engine (Python/FastAPI) +โ”‚ โ”œโ”€โ”€ providers/ # MT5, Rithmic, and Tradovate API wrappers +โ”‚ โ”œโ”€โ”€ data_server.py # Main FastAPI server and WebSocket broadcaster +โ”‚ โ””โ”€โ”€ launcher.py # Entry point and config manager +โ”‚ +โ”œโ”€โ”€ electron_shell/ # Desktop Application Wrapper +โ”‚ โ”œโ”€โ”€ main.js # Electron main process +โ”‚ โ””โ”€โ”€ preload.js # Security and IPC bridging +โ”‚ +โ””โ”€โ”€ frontend_build/ # Compiled React UI + โ”œโ”€โ”€ index.html # Main entry point + โ””โ”€โ”€ static/ # Minified JS/CSS for the charting interface +``` + +> **Note on Open Source:** The Python backend and Electron shell are fully open source. The React frontend is provided as a compiled production build (`frontend_build/`), meaning it is fully functional and ready to run, though the raw `.jsx` source files are not included in this repository. + +--- + +## ๐Ÿš€ Getting Started (ุทุฑูŠู‚ุฉ ุงู„ุชุดุบูŠู„) + +### Prerequisites +- **Python 3.10** or higher. +- **MetaTrader 5** (If you intend to use the MT5 data feed). +- `Node.js` (Optional, if you wish to re-package the Electron shell). + +### Installation +1. Clone the repository: + ```bash + git clone https://github.com/YourUsername/QuantumTerminal.git + cd QuantumTerminal/backend + ``` +2. Install Python dependencies: + ```bash + pip install fastapi uvicorn websockets watchfiles yfinance MetaTrader5 + ``` +3. Run the Backend Server: + ```bash + python launcher.py + ``` +4. The terminal will automatically serve the UI on `http://127.0.0.1:8502` and launch your default browser. + +*(To run as a standalone desktop app, you can package the `electron_shell` directory using Electron Forge or Electron Builder).* + +--- + +## ๐Ÿ› ๏ธ Contributing (ุงู„ู…ุณุงู‡ู…ุฉ) +This is a community-driven project! If you are a Python developer, Quant, or React engineer, your pull requests are welcome. +- **Areas needing help:** Enhancing thread safety for MT5 in `data_server.py`, adding new broker providers (like Interactive Brokers), and rebuilding an open-source React UI. + +## โš–๏ธ Disclaimer +*Quantum Terminal is for educational and analytical purposes only. Trading financial markets carries a high level of risk and may not be suitable for all investors. The mathematical models provided by this software are probabilistic, not predictive.* + +## ๐Ÿ“„ License +This project is licensed under the **MIT License**. See the [LICENSE](LICENSE) file for more details. diff --git a/backend/account_manager.py b/backend/account_manager.py new file mode 100644 index 0000000..58569dc --- /dev/null +++ b/backend/account_manager.py @@ -0,0 +1,805 @@ +""" +================================================================================ +ACCOUNT MANAGER โ€” Real Account State & Persistent P&L Tracking +================================================================================ +Quantum Terminal | Layer 3 โ€” Account & Risk Management + +Bridges the signal engine to real account data. Reads MT5 account balance/equity, +tracks daily and weekly P&L with persistence across restarts, and provides +configurable trading rules for prop firm compliance. + +Architecture: + MT5 Account โ”€โ”€โ–บ AccountManager โ”€โ”€โ–บ PortfolioState (signal engine) + โ”‚ + โ–ผ + account_state.json (persistent) + โ”‚ + โ–ผ + RiskGovernor (pre-trade checks) + +Settings are stored in account_settings.json (user-editable from terminal UI). +Account state (daily P&L, peak equity, etc.) is stored in account_state.json. + +Usage: + from account_manager import AccountManager + mgr = AccountManager() + mgr.sync_from_mt5() + portfolio = mgr.get_portfolio_state() # feeds into signal engine + mgr.record_trade_result(pnl_usd=150.0) + +Dependencies: MetaTrader5 (optional), json, pathlib +================================================================================ +""" + +import json +import logging +import os +from dataclasses import dataclass, field, asdict +from typing import Dict, List, Optional +from datetime import datetime, date, timedelta +from pathlib import Path + +# version: v4 (broker-TZ-aware period-pnl bucketing) +# v4 โ€” get_period_pnl now corrects for the broker TZ before bucketing +# deals into today/yesterday/this_week/etc. The MetaTrader5 library +# returns deal.time as broker server time treated as a Unix epoch +# (i.e. broker_local seconds, not real UTC). Our boundaries +# (datetime.now() + datetime(year, month, day)) are in real UTC +# via .timestamp(). Without correction, the comparison was off by +# the broker's TZ offset โ€” for an operator + broker both at GMT+3, +# ~3h of deals spilled across day boundaries, e.g. yesterday-late +# trades counted as today and vice versa. v4 detects the broker +# offset from a recent tick (round to whole hours) and converts +# d.time โ†’ real-UTC before comparison. +log = logging.getLogger("mk.account_manager") + +PROJECT_ROOT = Path(__file__).resolve().parent + +# v3: AppData dir name from env var so v1 / v2 builds get separate dirs. +APP_DIR = os.environ.get("MK_APP_DIR_NAME", "QuantumTerminal") + + +def _get_user_data_dir() -> Path: + """Per-user writable dir for account_settings.json + account_state.json. + Fixes Errno 13 Permission denied under Program Files on installed consumer. + v3: AppData dir name comes from MK_APP_DIR_NAME (default "QuantumTerminal").""" + import platform as _platform + if _platform.system() == "Windows": + appdata = os.environ.get("APPDATA", "") + base = Path(appdata) / APP_DIR if appdata else \ + Path.home() / "AppData" / "Roaming" / APP_DIR + else: + base = Path.home() / f".{APP_DIR.lower()}" + base.mkdir(parents=True, exist_ok=True) + return base + + +def _resolve_default_settings_path() -> Path: + """v2: Prefer the AppData path. If a legacy file exists next to the module + (dev-mode / old install), migrate it on first use.""" + user_dir = _get_user_data_dir() + user_path = user_dir / "account_settings.json" + legacy = PROJECT_ROOT / "account_settings.json" + if legacy.exists() and not user_path.exists(): + try: + user_path.write_bytes(legacy.read_bytes()) + log.info(f"Migrated {legacy} โ†’ {user_path}") + except Exception as e: + log.warning(f"Settings migration failed: {e}") + return user_path + + +def _resolve_default_state_path() -> Path: + user_dir = _get_user_data_dir() + user_path = user_dir / "account_state.json" + legacy = PROJECT_ROOT / "account_state.json" + if legacy.exists() and not user_path.exists(): + try: + user_path.write_bytes(legacy.read_bytes()) + log.info(f"Migrated {legacy} โ†’ {user_path}") + except Exception as e: + log.warning(f"State migration failed: {e}") + return user_path + + +# ============================================================ +# 1. TRADING SETTINGS (user-configurable) +# ============================================================ + +@dataclass +class TradingSettings: + """ + User-configurable trading rules. Saved to account_settings.json. + Editable from the terminal settings panel. + """ + # -- Account -- + account_size: float = 100_000.0 # Initial account balance (or prop firm start) + currency: str = "USD" + + # -- Prop Firm Rules -- + max_daily_loss_pct: float = 2.0 # Max daily loss as % of initial balance + max_daily_profit_pct: float = 0.0 # Max daily profit lock (0 = disabled) + max_total_drawdown_pct: float = 10.0 # Max drawdown from initial balance + max_trailing_drawdown_pct: float = 0.0 # Max drawdown from peak equity (0 = disabled) + + # -- Position Limits -- + max_open_positions: int = 4 + max_correlated_positions: int = 2 # Max same-direction in correlated group + risk_per_trade_pct: float = 0.5 # Fixed fractional risk per trade + + # -- Kelly -- + use_kelly: bool = True + kelly_fraction: float = 0.25 # Quarter-Kelly + + # -- Calibration Gating -- + min_calibration_grade: str = "C" # Filter signals below this grade + require_calibration: bool = False # If True, only trade calibrated setups + + # -- Session Rules -- + trading_enabled: bool = True # Master switch + auto_sync_mt5: bool = True # Auto-sync equity from MT5 on startup + + # -- Auto Trading (Phase 3G) -- + auto_trading_enabled: bool = False # Auto-execute signals (separate from live_trading) + auto_min_confidence: float = 0.60 # Min signal confidence to auto-trade + auto_min_grade: str = "C" # Min calibration grade to auto-trade + auto_max_daily_trades: int = 5 # Max auto-trades per day + auto_allowed_types: list = field(default_factory=lambda: [ + "cone_boundary_fade", "drift_momentum", "institutional_anchor", + "cone_convergence", "regime_transition", "cone_breakout", + ]) + auto_allowed_tickers: list = field(default_factory=list) # Empty = all universe + auto_scan_interval: float = 30.0 # Seconds between scans + auto_log_only: bool = True # True = dry run (log but don't execute) + + # -- Scheduler (Phase 3G) -- + scheduler_enabled: bool = True + scheduler_weekly_enabled: bool = True + scheduler_weekly_day: int = 6 # 0=Mon, 6=Sun + scheduler_weekly_time: str = "21:30" # HH:MM UTC + scheduler_daily_enabled: bool = True + scheduler_daily_time: str = "21:30" # HH:MM UTC + scheduler_daily_skip_weekends: bool = True + scheduler_calc_before_weekly: bool = True # Run cones before weekly orch + # Calibration + calibration_workers: int = 1 # CPU threads for ATR/management sweep + scheduler_calc_modules: str = "anchors,cones" + + def to_dict(self) -> dict: + return asdict(self) + + @classmethod + def from_dict(cls, d: dict) -> 'TradingSettings': + # Only take keys that exist in the dataclass + valid_keys = {f.name for f in cls.__dataclass_fields__.values()} + filtered = {k: v for k, v in d.items() if k in valid_keys} + return cls(**filtered) + + +# ============================================================ +# 2. ACCOUNT STATE (persistent) +# ============================================================ + +@dataclass +class AccountState: + """ + Persistent account state โ€” survives server restarts. + Saved to account_state.json. + """ + # -- Equity tracking -- + initial_balance: float = 100_000.0 # Set once at start (prop firm funded amount) + current_equity: float = 100_000.0 # Last known equity + peak_equity: float = 100_000.0 # Highest equity ever (for trailing DD) + + # -- Daily tracking -- + daily_start_equity: float = 100_000.0 # Equity at start of current day + daily_pnl: float = 0.0 # P&L since daily reset + daily_trades: int = 0 # Trade count today + current_date: str = "" # ISO date of current tracking day + + # -- Weekly tracking -- + weekly_start_equity: float = 100_000.0 + weekly_pnl: float = 0.0 + weekly_trades: int = 0 + current_week: str = "" # ISO week string (e.g., "2026-W12") + + # -- Circuit breaker states -- + daily_loss_halt: bool = False # True = daily loss limit hit + daily_profit_lock: bool = False # True = daily profit target hit + total_drawdown_halt: bool = False # True = max drawdown breached + trailing_drawdown_halt: bool = False # True = trailing DD breached + + # -- Trade log (lightweight) -- + recent_trades: List[Dict] = field(default_factory=list) # Last 50 trades + + # -- Timestamps -- + last_mt5_sync: str = "" + last_updated: str = "" + + def to_dict(self) -> dict: + return asdict(self) + + @classmethod + def from_dict(cls, d: dict) -> 'AccountState': + valid_keys = {f.name for f in cls.__dataclass_fields__.values()} + filtered = {k: v for k, v in d.items() if k in valid_keys} + # Handle list fields that may come as None + if 'recent_trades' not in filtered or filtered['recent_trades'] is None: + filtered['recent_trades'] = [] + return cls(**filtered) + + +# ============================================================ +# 3. ACCOUNT MANAGER +# ============================================================ + +class AccountManager: + """ + Manages account state, MT5 sync, and trading settings. + """ + + def __init__(self, settings_path: Optional[Path] = None, + state_path: Optional[Path] = None): + # v2: default to %APPDATA%\QuantumTerminal\ (writable) instead of the module + # directory, which lives under Program Files and is read-only for + # non-admin users. Legacy files are migrated on first use. + self._settings_path = settings_path or _resolve_default_settings_path() + self._state_path = state_path or _resolve_default_state_path() + + self.settings = self._load_settings() + self.state = self._load_state() + + # Check for day/week rollover + self._check_daily_reset() + self._check_weekly_reset() + + # โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + # PERSISTENCE + # โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + + def _load_settings(self) -> TradingSettings: + """Load or create settings file.""" + if self._settings_path.exists(): + try: + with open(self._settings_path) as f: + data = json.load(f) + log.info(f"Loaded trading settings from {self._settings_path}") + return TradingSettings.from_dict(data) + except Exception as e: + log.warning(f"Failed to load settings: {e} โ€” using defaults") + + settings = TradingSettings() + self._save_settings(settings) + return settings + + def _save_settings(self, settings: TradingSettings = None): + """Save settings to JSON.""" + if settings is None: + settings = self.settings + try: + with open(self._settings_path, "w") as f: + json.dump(settings.to_dict(), f, indent=2) + except Exception as e: + log.warning(f"Failed to save settings: {e}") + + def _load_state(self) -> AccountState: + """Load or create state file.""" + if self._state_path.exists(): + try: + with open(self._state_path) as f: + data = json.load(f) + log.info(f"Loaded account state from {self._state_path}") + return AccountState.from_dict(data) + except Exception as e: + log.warning(f"Failed to load state: {e} โ€” using defaults") + + state = AccountState( + initial_balance=self.settings.account_size, + current_equity=self.settings.account_size, + peak_equity=self.settings.account_size, + daily_start_equity=self.settings.account_size, + weekly_start_equity=self.settings.account_size, + ) + self._save_state(state) + return state + + def _save_state(self, state: AccountState = None): + """Save state to JSON.""" + if state is None: + state = self.state + state.last_updated = datetime.now().isoformat() + try: + with open(self._state_path, "w") as f: + json.dump(state.to_dict(), f, indent=2, default=str) + except Exception as e: + log.warning(f"Failed to save state: {e}") + + def update_settings(self, new_settings: dict): + """Update settings from terminal UI (partial update).""" + old_account_size = self.settings.account_size + current = self.settings.to_dict() + current.update(new_settings) + self.settings = TradingSettings.from_dict(current) + self._save_settings() + log.info(f"Trading settings updated: {list(new_settings.keys())}") + + # Auto-reset baseline when account_size changes + if "account_size" in new_settings and new_settings["account_size"] != old_account_size: + self.reset_baseline(self.settings.account_size) + + def reset_baseline(self, new_balance: float = None): + """ + Reset account state baseline. Call when: + - Setting up a new funded account + - Account size changes in settings + - Starting fresh after a reset + + Sets initial_balance, peak_equity, daily/weekly start to the new value. + Clears all circuit breaker halts. Preserves current_equity if MT5-synced. + """ + bal = new_balance or self.settings.account_size + equity = self.state.current_equity if self.state.current_equity > 0 else bal + + self.state.initial_balance = bal + self.state.peak_equity = max(bal, equity) + self.state.daily_start_equity = equity + self.state.weekly_start_equity = equity + self.state.daily_pnl = 0.0 + self.state.weekly_pnl = 0.0 + self.state.daily_trades = 0 + self.state.weekly_trades = 0 + + # Clear all circuit breaker halts + self.state.daily_loss_halt = False + self.state.daily_profit_lock = False + self.state.total_drawdown_halt = False + self.state.trailing_drawdown_halt = False + + self._save_state() + log.info(f"Account baseline reset: initial_balance=${bal:,.2f}, " + f"equity=${equity:,.2f}, all circuit breakers cleared") + + # โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + # MT5 SYNC + # โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + + def sync_from_mt5(self) -> bool: + """ + Read real account equity from MT5. + Returns True if sync successful. + """ + try: + import MetaTrader5 as mt5 + + if not mt5.terminal_info(): + log.warning("MT5 not connected โ€” cannot sync account") + return False + + account = mt5.account_info() + if account is None: + log.warning("MT5 account_info() returned None") + return False + + equity = account.equity + balance = account.balance + + self.state.current_equity = equity + self.state.peak_equity = max(self.state.peak_equity, equity) + self.state.last_mt5_sync = datetime.now().isoformat() + + # Update daily P&L + self.state.daily_pnl = equity - self.state.daily_start_equity + self.state.weekly_pnl = equity - self.state.weekly_start_equity + + # Check circuit breakers + self._check_circuit_breakers() + + self._save_state() + log.info(f"MT5 sync: equity=${equity:,.2f}, balance=${balance:,.2f}, " + f"daily P&L=${self.state.daily_pnl:+,.2f}") + return True + + except ImportError: + log.info("MetaTrader5 package not available โ€” manual equity mode") + return False + except Exception as e: + log.warning(f"MT5 sync failed: {e}") + return False + + def sync_positions_from_mt5(self) -> List[Dict]: + """Read open positions from MT5.""" + try: + import MetaTrader5 as mt5 + + if not mt5.terminal_info(): + return [] + + positions = mt5.positions_get() + if positions is None: + return [] + + result = [] + for pos in positions: + result.append({ + 'ticket': pos.ticket, + 'ticker': pos.symbol, + 'direction': 'long' if pos.type == 0 else 'short', + 'size': pos.volume, + 'entry': pos.price_open, + 'current_price': pos.price_current, + 'current_pnl': pos.profit, + 'sl': pos.sl, + 'tp': pos.tp, + 'magic': pos.magic, + 'comment': pos.comment, + }) + + return result + + except (ImportError, Exception): + return [] + + # โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + # DAILY/WEEKLY RESETS + # โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + + def _check_daily_reset(self): + """Reset daily tracking if it's a new day.""" + today = date.today().isoformat() + if self.state.current_date != today: + if self.state.current_date: + log.info(f"Daily reset: {self.state.current_date} โ†’ {today} " + f"(yesterday P&L: ${self.state.daily_pnl:+,.2f})") + self.state.daily_start_equity = self.state.current_equity + self.state.daily_pnl = 0.0 + self.state.daily_trades = 0 + self.state.daily_loss_halt = False + self.state.daily_profit_lock = False + self.state.current_date = today + self._save_state() + + def _check_weekly_reset(self): + """Reset weekly tracking if it's a new week.""" + current_week = date.today().isocalendar() + week_str = f"{current_week[0]}-W{current_week[1]:02d}" + if self.state.current_week != week_str: + if self.state.current_week: + log.info(f"Weekly reset: {self.state.current_week} โ†’ {week_str} " + f"(last week P&L: ${self.state.weekly_pnl:+,.2f})") + self.state.weekly_start_equity = self.state.current_equity + self.state.weekly_pnl = 0.0 + self.state.weekly_trades = 0 + self.state.current_week = week_str + self._save_state() + + # โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + # CIRCUIT BREAKERS + # โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + + def _check_circuit_breakers(self): + """Check all prop firm circuit breakers.""" + s = self.settings + st = self.state + + # Daily loss limit + if s.max_daily_loss_pct > 0: + daily_loss_limit = st.initial_balance * (s.max_daily_loss_pct / 100) + if st.daily_pnl <= -daily_loss_limit: + if not st.daily_loss_halt: + log.warning(f"๐Ÿ›‘ DAILY LOSS LIMIT HIT: ${st.daily_pnl:+,.2f} " + f"exceeds -{s.max_daily_loss_pct}% of ${st.initial_balance:,.0f}") + st.daily_loss_halt = True + + # Daily profit lock + if s.max_daily_profit_pct > 0: + daily_profit_limit = st.initial_balance * (s.max_daily_profit_pct / 100) + if st.daily_pnl >= daily_profit_limit: + if not st.daily_profit_lock: + log.info(f"๐Ÿ”’ DAILY PROFIT LOCK: ${st.daily_pnl:+,.2f} " + f"exceeds +{s.max_daily_profit_pct}% target") + st.daily_profit_lock = True + + # Total drawdown from initial + if s.max_total_drawdown_pct > 0: + total_dd = (st.initial_balance - st.current_equity) / st.initial_balance * 100 + if total_dd >= s.max_total_drawdown_pct: + if not st.total_drawdown_halt: + log.warning(f"๐Ÿ›‘ MAX DRAWDOWN BREACHED: {total_dd:.1f}% from initial " + f"(limit: {s.max_total_drawdown_pct}%)") + st.total_drawdown_halt = True + + # Trailing drawdown from peak + if s.max_trailing_drawdown_pct > 0 and st.peak_equity > 0: + trailing_dd = (st.peak_equity - st.current_equity) / st.peak_equity * 100 + if trailing_dd >= s.max_trailing_drawdown_pct: + if not st.trailing_drawdown_halt: + log.warning(f"๐Ÿ›‘ TRAILING DRAWDOWN BREACHED: {trailing_dd:.1f}% from peak " + f"${st.peak_equity:,.2f} (limit: {s.max_trailing_drawdown_pct}%)") + st.trailing_drawdown_halt = True + + @property + def is_trading_allowed(self) -> bool: + """Check if trading is allowed based on all circuit breakers.""" + if not self.settings.trading_enabled: + return False + st = self.state + return not (st.daily_loss_halt or st.daily_profit_lock or + st.total_drawdown_halt or st.trailing_drawdown_halt) + + @property + def halt_reason(self) -> str: + """Get the reason trading is halted, or empty string.""" + if not self.settings.trading_enabled: + return "Trading disabled in settings" + st = self.state + reasons = [] + if st.daily_loss_halt: + reasons.append(f"Daily loss limit ({self.settings.max_daily_loss_pct}%)") + if st.daily_profit_lock: + reasons.append(f"Daily profit locked ({self.settings.max_daily_profit_pct}%)") + if st.total_drawdown_halt: + reasons.append(f"Max drawdown ({self.settings.max_total_drawdown_pct}%)") + if st.trailing_drawdown_halt: + reasons.append(f"Trailing drawdown ({self.settings.max_trailing_drawdown_pct}%)") + return " | ".join(reasons) + + # โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + # TRADE RECORDING + # โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + + def record_trade_result(self, pnl_usd: float, ticker: str = "", + direction: str = "", signal_type: str = "", + lots: float = 0.0, magic: int = 0): + """ + Record a trade result and update P&L tracking. + Called when a trade closes (from lifecycle manager or MT5 bridge). + """ + self.state.daily_pnl += pnl_usd + self.state.weekly_pnl += pnl_usd + self.state.daily_trades += 1 + self.state.weekly_trades += 1 + self.state.current_equity += pnl_usd + self.state.peak_equity = max(self.state.peak_equity, self.state.current_equity) + + # Append to recent trades (keep last 50) + self.state.recent_trades.append({ + 'timestamp': datetime.now().isoformat(), + 'ticker': ticker, + 'direction': direction, + 'signal_type': signal_type, + 'lots': lots, + 'pnl_usd': round(pnl_usd, 2), + 'magic': magic, + 'equity_after': round(self.state.current_equity, 2), + }) + if len(self.state.recent_trades) > 50: + self.state.recent_trades = self.state.recent_trades[-50:] + + self._check_circuit_breakers() + self._save_state() + + log.info(f"Trade recorded: {ticker} {direction} {signal_type} โ†’ " + f"PnL=${pnl_usd:+,.2f} | Daily=${self.state.daily_pnl:+,.2f} | " + f"Equity=${self.state.current_equity:,.2f}") + + # โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + # BRIDGE TO SIGNAL ENGINE + # โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + + def get_portfolio_state(self): + """ + Build a PortfolioState for the signal engine from current account data. + """ + # Import here to avoid circular dependency + from signal_engine_v2 import PortfolioState + + positions = self.sync_positions_from_mt5() + + return PortfolioState( + equity=self.state.current_equity, + open_positions=[ + { + 'ticker': p['ticker'], + 'direction': p['direction'], + 'size': p['size'], + 'entry': p['entry'], + 'current_pnl': p['current_pnl'], + 'risk_usd': abs(p['entry'] - p.get('sl', p['entry'])) * p['size'], + } + for p in positions + ], + daily_pnl=self.state.daily_pnl, + weekly_pnl=self.state.weekly_pnl, + peak_equity=self.state.peak_equity, + ) + + # โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + # REST API HELPERS (for data_server.py endpoints) + # โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + + def get_status_dict(self) -> dict: + """Get full account status for the terminal dashboard.""" + s = self.settings + st = self.state + + daily_loss_limit = st.initial_balance * (s.max_daily_loss_pct / 100) if s.max_daily_loss_pct > 0 else 0 + daily_profit_limit = st.initial_balance * (s.max_daily_profit_pct / 100) if s.max_daily_profit_pct > 0 else 0 + total_dd = (st.initial_balance - st.current_equity) / st.initial_balance * 100 if st.initial_balance > 0 else 0 + trailing_dd = (st.peak_equity - st.current_equity) / st.peak_equity * 100 if st.peak_equity > 0 else 0 + + return { + "trading_allowed": self.is_trading_allowed, + "halt_reason": self.halt_reason, + "equity": round(st.current_equity, 2), + "initial_balance": round(st.initial_balance, 2), + "peak_equity": round(st.peak_equity, 2), + "daily_pnl": round(st.daily_pnl, 2), + "daily_pnl_pct": round(st.daily_pnl / st.initial_balance * 100, 2) if st.initial_balance > 0 else 0, + "daily_loss_limit": round(daily_loss_limit, 2), + "daily_profit_limit": round(daily_profit_limit, 2), + "daily_loss_halt": st.daily_loss_halt, + "daily_profit_lock": st.daily_profit_lock, + "weekly_pnl": round(st.weekly_pnl, 2), + "total_drawdown_pct": round(total_dd, 2), + "trailing_drawdown_pct": round(trailing_dd, 2), + "total_drawdown_halt": st.total_drawdown_halt, + "trailing_drawdown_halt": st.trailing_drawdown_halt, + "daily_trades": st.daily_trades, + "weekly_trades": st.weekly_trades, + "open_positions": len(self.sync_positions_from_mt5()), + "max_positions": s.max_open_positions, + "last_mt5_sync": st.last_mt5_sync, + "last_updated": st.last_updated, + } + + def get_settings_dict(self) -> dict: + """Get current settings for the terminal UI.""" + return self.settings.to_dict() + + # โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + # MT5 DEAL HISTORY โ€” REAL PERIOD P&L + # โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + + def get_period_pnl(self) -> dict: + """ + Query MT5 closed deal history for accurate period P&L. + + Returns real closed-trade P&L (profit + swap + commission) for: + - today, yesterday + - this week, last week + - this month, last month + + This is the source of truth โ€” NOT the equity-delta tracking + which drifts when weekly_start_equity is stale. + """ + try: + import MetaTrader5 as mt5 + import time as _time + + if not mt5.terminal_info(): + return self._empty_period_pnl("MT5 not connected") + + # v4: detect broker TZ offset (broker_time treated as UTC - + # real_UTC). Round to whole hours since broker TZs are integer + # offsets. Falls back to 0 (treat as UTC broker) on failure. + broker_offset_sec = 0 + try: + for sym in ("EURUSD", "XAUUSD", "USDJPY", "BTCUSD"): + tick = mt5.symbol_info_tick(sym) + if tick and getattr(tick, "time", 0): + raw = int(tick.time) - int(_time.time()) + broker_offset_sec = round(raw / 3600) * 3600 + # Sanity: clamp to ยฑ14h (no real broker outside that) + if abs(broker_offset_sec) > 14 * 3600: + broker_offset_sec = 0 + break + except Exception as e: + log.debug(f"broker offset detection failed: {e}") + + now = datetime.now() + today_start = datetime(now.year, now.month, now.day) + yesterday_start = today_start - timedelta(days=1) + + # Week boundaries (Monday 00:00) + weekday = now.weekday() # 0=Mon + this_week_start = today_start - timedelta(days=weekday) + last_week_start = this_week_start - timedelta(days=7) + last_week_end = this_week_start + + # Month boundaries + this_month_start = datetime(now.year, now.month, 1) + if now.month == 1: + last_month_start = datetime(now.year - 1, 12, 1) + else: + last_month_start = datetime(now.year, now.month - 1, 1) + last_month_end = this_month_start + + # v4: fetch a wider window than strictly needed so deals near the + # boundaries don't get clipped by MT5's own pseudo-UTC interpretation + # of the input range. We filter precisely in Python below. + query_from = last_month_start - timedelta(days=2) + query_to = now + timedelta(days=2) + all_deals = mt5.history_deals_get(query_from, query_to) + if all_deals is None: + all_deals = () + + # Filter to closing deals only (entry: 1=out, 2=inout, 3=out_by) + # Deal type 6 = balance operations โ€” skip those + close_deals = [ + d for d in all_deals + if d.entry in (1, 2, 3) and d.type not in (6,) + ] + + def _sum_pnl(deals, dt_from, dt_to): + """Sum profit + swap + commission for deals in [dt_from, dt_to). + v4: convert d.time (broker pseudo-UTC) to real UTC before + comparing against the boundaries (which are real UTC via + .timestamp()).""" + ts_from = dt_from.timestamp() + ts_to = dt_to.timestamp() + total = 0.0 + count = 0 + for d in deals: + d_real_utc = d.time - broker_offset_sec + if ts_from <= d_real_utc < ts_to: + total += d.profit + d.swap + d.commission + count += 1 + return round(total, 2), count + + today_pnl, today_trades = _sum_pnl(close_deals, today_start, now + timedelta(hours=1)) + yesterday_pnl, yesterday_trades = _sum_pnl(close_deals, yesterday_start, today_start) + this_week_pnl, this_week_trades = _sum_pnl(close_deals, this_week_start, now + timedelta(hours=1)) + last_week_pnl, last_week_trades = _sum_pnl(close_deals, last_week_start, last_week_end) + this_month_pnl, this_month_trades = _sum_pnl(close_deals, this_month_start, now + timedelta(hours=1)) + last_month_pnl, last_month_trades = _sum_pnl(close_deals, last_month_start, last_month_end) + + equity = self.state.current_equity or self.state.initial_balance + + return { + "today": {"pnl": today_pnl, "trades": today_trades, + "pct": round(today_pnl / equity * 100, 2) if equity > 0 else 0}, + "yesterday": {"pnl": yesterday_pnl, "trades": yesterday_trades, + "pct": round(yesterday_pnl / equity * 100, 2) if equity > 0 else 0}, + "this_week": {"pnl": this_week_pnl, "trades": this_week_trades, + "pct": round(this_week_pnl / equity * 100, 2) if equity > 0 else 0}, + "last_week": {"pnl": last_week_pnl, "trades": last_week_trades, + "pct": round(last_week_pnl / equity * 100, 2) if equity > 0 else 0}, + "this_month": {"pnl": this_month_pnl, "trades": this_month_trades, + "pct": round(this_month_pnl / equity * 100, 2) if equity > 0 else 0}, + "last_month": {"pnl": last_month_pnl, "trades": last_month_trades, + "pct": round(last_month_pnl / equity * 100, 2) if equity > 0 else 0}, + "source": "mt5_deals", + "deals_scanned": len(close_deals), + } + + except ImportError: + return self._empty_period_pnl("MetaTrader5 not available") + except Exception as e: + log.warning(f"Period P&L fetch failed: {e}") + return self._empty_period_pnl(str(e)) + + @staticmethod + def _empty_period_pnl(reason: str = "") -> dict: + """Return empty period P&L structure.""" + empty = {"pnl": 0, "trades": 0, "pct": 0} + return { + "today": empty.copy(), "yesterday": empty.copy(), + "this_week": empty.copy(), "last_week": empty.copy(), + "this_month": empty.copy(), "last_month": empty.copy(), + "source": "unavailable", "reason": reason, + "deals_scanned": 0, + } + + +# ============================================================ +# 4. SINGLETON +# ============================================================ + +_manager_instance = None + +def get_account_manager() -> AccountManager: + """Get the singleton AccountManager instance.""" + global _manager_instance + if _manager_instance is None: + _manager_instance = AccountManager() + return _manager_instance \ No newline at end of file diff --git a/backend/account_routes.py b/backend/account_routes.py new file mode 100644 index 0000000..a2d1fa9 --- /dev/null +++ b/backend/account_routes.py @@ -0,0 +1,1186 @@ +""" +================================================================================ +Quantum Terminal โ€” Account & Calibration API Routes +================================================================================ +FastAPI router for Layer 3 (Account Management + Risk Governor) and +Layer 2 (Calibration Store) endpoints. + +Wired into data_server.py via: + from account_routes import create_account_router + app.include_router(create_account_router(broadcast_event)) + +Endpoints: + # โ”€โ”€ Account & Risk โ”€โ”€ + GET /api/account/status โ€” equity, P&L, circuit breakers, halt status + GET /api/account/settings โ€” current trading settings (prop firm rules) + PATCH /api/account/settings โ€” update trading settings + POST /api/account/sync โ€” force MT5 equity sync + POST /api/account/reset-baseline โ€” reset baseline (initial balance + clear breakers) + GET /api/account/trades โ€” recent trade log (last 50) + + # โ”€โ”€ Calibration โ”€โ”€ + GET /api/calibration/summary โ€” universe calibration badge summary + GET /api/calibration/management โ€” unified management dashboard (freshness + calibration + flow + anchor) + GET /api/calibration/{ticker} โ€” full calibration report for one asset + GET /api/calibration/badges/{ticker} โ€” badge lookup for all setups on one asset + GET /api/calibration/detail/{ticker}/{signal_type} โ€” IS/OOS drill-down for one setup + GET /api/calibration/outcomes/{ticker}/{signal_type} โ€” raw trade records for chart viz + POST /api/calibration/run/{ticker} โ€” trigger backtest calibration for one asset + POST /api/calibration/run โ€” trigger universe calibration (background) + GET /api/calibration/status โ€” calibration job status + + # โ”€โ”€ Governor โ”€โ”€ + GET /api/governor/status โ€” risk governor status + POST /api/governor/check โ€” pre-trade check (dry run) + + # โ”€โ”€ Walk-Forward (Phase 3I) โ”€โ”€ + POST /api/wf/run/{ticker} โ€” trigger WF engine for one asset + GET /api/wf/status โ€” WF job status + + # โ”€โ”€ Flow Calibration (Phase 3I) โ”€โ”€ + POST /api/flow/calibrate/{ticker} โ€” trigger flow calibrator + validator + GET /api/flow/status โ€” flow calibration job status +================================================================================ +""" + +import json +import asyncio +import logging +import threading +from debug_subprocess import debug_popen as _debug_popen +from typing import Optional, Callable, Awaitable +from pathlib import Path +from datetime import datetime, timezone, timedelta +from fastapi import APIRouter, HTTPException +from pydantic import BaseModel + +log = logging.getLogger("account_routes") + +PROJECT_ROOT = Path(__file__).resolve().parent + + +# โ”€โ”€ Request models โ”€โ”€ + +class DebugConfigRequest(BaseModel): + """Debug configuration update.""" + subprocess_debug: Optional[bool] = None + + +class SettingsUpdateRequest(BaseModel): + """Partial settings update โ€” only include fields to change.""" + account_size: Optional[float] = None + max_daily_loss_pct: Optional[float] = None + max_daily_profit_pct: Optional[float] = None + max_total_drawdown_pct: Optional[float] = None + max_trailing_drawdown_pct: Optional[float] = None + max_open_positions: Optional[int] = None + max_correlated_positions: Optional[int] = None + risk_per_trade_pct: Optional[float] = None + use_kelly: Optional[bool] = None + kelly_fraction: Optional[float] = None + min_calibration_grade: Optional[str] = None + require_calibration: Optional[bool] = None + trading_enabled: Optional[bool] = None + # Auto Trading (Phase 3G) + auto_trading_enabled: Optional[bool] = None + auto_min_confidence: Optional[float] = None + auto_min_grade: Optional[str] = None + auto_max_daily_trades: Optional[int] = None + auto_allowed_types: Optional[list] = None + auto_allowed_tickers: Optional[list] = None + auto_scan_interval: Optional[float] = None + auto_log_only: Optional[bool] = None + # Scheduler + scheduler_enabled: Optional[bool] = None + scheduler_weekly_enabled: Optional[bool] = None + scheduler_weekly_day: Optional[int] = None + scheduler_weekly_time: Optional[str] = None + scheduler_daily_enabled: Optional[bool] = None + scheduler_daily_time: Optional[str] = None + scheduler_daily_skip_weekends: Optional[bool] = None + scheduler_calc_before_weekly: Optional[bool] = None + # Calibration + calibration_workers: Optional[int] = None + + +class PreTradeCheckRequest(BaseModel): + """Dry-run pre-trade check.""" + ticker: str + direction: str # "long" or "short" + signal_type: str + entry_price: float + stop_loss: float + target_price: float + proposed_lots: float = 0.1 + + +class CalibrationRunRequest(BaseModel): + """Calibration run parameters.""" + lookback_months: int = 6 + mc_sims: int = 20000 + parallel_workers: int = 1 # CPU cores for sweep (1 = sequential) + + +# โ”€โ”€ Router factory โ”€โ”€ + +def create_account_router( + broadcast_event: Optional[Callable[[dict], Awaitable[None]]] = None, +) -> APIRouter: + """ + Create account/calibration API router. + + Lazy-loads AccountManager, RiskGovernor, CalibrationStore on first call + to avoid import errors if modules aren't deployed yet. + """ + router = APIRouter(tags=["account"]) + + # โ”€โ”€ Lazy singletons โ”€โ”€ + _cache = {} + + def _get_account_manager(): + if 'acct' not in _cache: + try: + from account_manager import get_account_manager + _cache['acct'] = get_account_manager() + except ImportError: + raise HTTPException(503, "account_manager module not available") + return _cache['acct'] + + def _get_governor(): + if 'gov' not in _cache: + try: + from risk_governor import get_governor + _cache['gov'] = get_governor() + except ImportError: + raise HTTPException(503, "risk_governor module not available") + return _cache['gov'] + + def _get_cal_store(): + if 'cal' not in _cache: + try: + from calibration_store import get_store + _cache['cal'] = get_store() + except ImportError: + raise HTTPException(503, "calibration_store module not available") + return _cache['cal'] + + async def _notify(event_type: str, detail: str = ""): + if broadcast_event: + await broadcast_event({ + "type": "account_update", + "detail": event_type, + "message": detail, + "timestamp": datetime.now(timezone.utc).isoformat(), + }) + + # โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ• + # DEBUG CONFIG โ€” subprocess_debug flag + # โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ• + + _DEBUG_CONFIG_PATH = PROJECT_ROOT / "debug_config.json" + + def _load_debug_config() -> dict: + """Load debug_config.json โ€” returns defaults if missing.""" + try: + if _DEBUG_CONFIG_PATH.exists(): + return json.loads(_DEBUG_CONFIG_PATH.read_text()) + except Exception: + pass + return {"subprocess_debug": False} + + def _save_debug_config(cfg: dict): + try: + _DEBUG_CONFIG_PATH.write_text(json.dumps(cfg, indent=2)) + except Exception as e: + log.warning(f"Failed to save debug_config.json: {e}") + + @router.get("/api/dev/debug-config") + async def get_debug_config(): + """Get current debug configuration.""" + return _load_debug_config() + + @router.patch("/api/dev/debug-config") + async def update_debug_config(payload: DebugConfigRequest): + """ + Update debug configuration. + subprocess_debug (bool) โ€” open visible console windows for subprocesses + """ + cfg = _load_debug_config() + updates = {k: v for k, v in payload.dict().items() if v is not None} + cfg.update(updates) + _save_debug_config(cfg) + mode = "ON" if cfg.get("subprocess_debug") else "OFF" + await _notify("debug_config_changed", + f"Subprocess debug mode: {mode} โ€” takes effect on next calibration/WF run") + return cfg + + # โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ• + # ACCOUNT & RISK ENDPOINTS + # โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ• + + @router.get("/api/account/status") + async def get_account_status(): + """Full account status โ€” equity, P&L, circuit breakers.""" + mgr = _get_account_manager() + return mgr.get_status_dict() + + @router.get("/api/account/settings") + async def get_account_settings(): + """Current trading settings.""" + mgr = _get_account_manager() + return mgr.get_settings_dict() + + @router.patch("/api/account/settings") + async def update_account_settings(req: SettingsUpdateRequest): + """Update trading settings (partial โ€” only include changed fields).""" + mgr = _get_account_manager() + # Build dict of non-None fields only + updates = {k: v for k, v in req.dict().items() if v is not None} + if not updates: + raise HTTPException(400, "No fields to update") + mgr.update_settings(updates) + await _notify("settings_changed", f"Updated: {', '.join(updates.keys())}") + return {"status": "ok", "updated": list(updates.keys()), "settings": mgr.get_settings_dict()} + + @router.post("/api/account/sync") + async def force_mt5_sync(): + """Force MT5 equity sync.""" + mgr = _get_account_manager() + success = await asyncio.to_thread(mgr.sync_from_mt5) + if success: + await _notify("mt5_sync", "Equity synced from MT5") + return {"status": "ok", "equity": mgr.state.current_equity} + else: + return {"status": "failed", "message": "MT5 sync failed โ€” check connection"} + + @router.post("/api/account/reset-baseline") + async def reset_baseline(): + """ + Reset account baseline to current account_size. + Clears all circuit breaker halts, resets initial_balance, + peak_equity, and daily/weekly tracking. + + Call this after changing account_size or starting fresh. + """ + mgr = _get_account_manager() + mgr.reset_baseline() + await _notify("baseline_reset", f"Baseline reset to ${mgr.state.initial_balance:,.2f}") + return { + "status": "ok", + "initial_balance": mgr.state.initial_balance, + "current_equity": mgr.state.current_equity, + "peak_equity": mgr.state.peak_equity, + "circuit_breakers_cleared": True, + } + + @router.get("/api/account/trades") + async def get_recent_trades(): + """Recent trade log (last 50).""" + mgr = _get_account_manager() + return {"trades": mgr.state.recent_trades} + + @router.get("/api/account/period-pnl") + async def get_period_pnl(): + """ + Real closed-trade P&L from MT5 deal history. + Periods: today, yesterday, this_week, last_week, this_month, last_month. + Each period: {pnl, trades, pct}. + """ + mgr = _get_account_manager() + return await asyncio.to_thread(mgr.get_period_pnl) + + # โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ• + # GOVERNOR ENDPOINTS + # โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ• + + @router.get("/api/governor/status") + async def get_governor_status(): + """Full risk governor status.""" + gov = _get_governor() + return gov.get_status() + + @router.post("/api/governor/check") + async def pre_trade_check(req: PreTradeCheckRequest): + """ + Dry-run pre-trade check โ€” tests if a trade would be allowed + without actually placing it. Useful for UI preview. + """ + gov = _get_governor() + + # Build a mock signal object for the check + class _MockSignal: + pass + sig = _MockSignal() + sig.ticker = req.ticker.upper() + + class _Dir: + value = req.direction.lower() + class _Type: + value = req.signal_type + sig.direction = _Dir() + sig.signal_type = _Type() + sig.entry_price = req.entry_price + sig.stop_loss = req.stop_loss + sig.target_price = req.target_price + + allowed, reason = gov.pre_trade_check(sig, proposed_lots=req.proposed_lots) + adjusted_lots = req.proposed_lots + adjust_notes = "" + if allowed: + adjusted_lots, adjust_notes = gov.adjust_position_size(sig, req.proposed_lots) + + return { + "allowed": allowed, + "reason": reason, + "adjusted_lots": round(adjusted_lots, 4), + "adjust_notes": adjust_notes, + } + + # โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ• + # CALIBRATION ENDPOINTS + # โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ• + + @router.get("/api/calibration/summary") + async def get_calibration_summary(): + """Universe calibration badge summary โ€” for dashboard display.""" + store = _get_cal_store() + return store.get_universe_summary() + + @router.get("/api/calibration/management") + async def get_calibration_management(): + """ + Unified management dashboard data โ€” aggregates freshness, calibration, + flow params, and entry confirmation status across all assets. + + Used by CalibrationDashboard.jsx (Phase 3I). + """ + import asyncio as _aio + + # โ”€โ”€ Universe โ”€โ”€ + try: + from config_manager import ConfigManager + cm = ConfigManager() + universe = cm.get_active_universe() + except Exception: + try: + from server_config import ASSET_UNIVERSE + universe = list(ASSET_UNIVERSE) + except Exception: + universe = [] + + # โ”€โ”€ Freshness (all 9 data types ร— universe) โ”€โ”€ + freshness = {} + try: + from data_freshness import DataFreshnessMonitor + monitor = DataFreshnessMonitor() + for ticker in universe: + tf = monitor.get_ticker_freshness(ticker) + freshness[ticker] = tf.to_dict() + except ImportError: + log.warning("data_freshness not available for management endpoint") + except Exception as e: + log.warning(f"Freshness fetch failed: {e}") + + # โ”€โ”€ Calibration badges โ”€โ”€ + calibration = {} + try: + store = _get_cal_store() + calibration = store.get_universe_summary() + except Exception as e: + log.warning(f"Calibration summary failed: {e}") + + # โ”€โ”€ Flow params status โ”€โ”€ + flow = {"calibrated": [], "missing": [], "details": {}} + flow_path = PROJECT_ROOT / "flow_params.json" + if flow_path.exists(): + try: + with open(flow_path, "r") as f: + flow_data = json.load(f) + for ticker in universe: + if ticker in flow_data: + flow["calibrated"].append(ticker) + fd = flow_data[ticker] + flow["details"][ticker] = { + "calibrated_at": fd.get("calibrated_at", ""), + "wf_score": fd.get("wf_score", 0), + "wf_variance": fd.get("wf_variance", 0), + "classify_method": fd.get("classify_method", ""), + } + else: + flow["missing"].append(ticker) + except Exception as e: + log.warning(f"Flow params read failed: {e}") + flow["missing"] = universe[:] + else: + flow["missing"] = universe[:] + + # โ”€โ”€ Anchor params status โ”€โ”€ + anchor = {"calibrated": [], "missing": [], "details": {}} + anchor_path = PROJECT_ROOT / "anchor_params.json" + if anchor_path.exists(): + try: + with open(anchor_path, "r") as f: + anchor_data = json.load(f) + for ticker in universe: + if ticker in anchor_data: + anchor["calibrated"].append(ticker) + ad = anchor_data[ticker] + anchor["details"][ticker] = { + "threshold": ad.get("anchor_distance_max_sd", ""), + "calibrated_at": ad.get("calibrated_at", ""), + } + else: + anchor["missing"].append(ticker) + except Exception as e: + log.warning(f"Anchor params read failed: {e}") + anchor["missing"] = universe[:] + else: + anchor["missing"] = universe[:] + + # โ”€โ”€ Entry confirmation edge (from calibration reports) โ”€โ”€ + entry_conf = {} + for ticker in universe: + ticker_cal = calibration.get(ticker, {}) + setups = ticker_cal.get("setups", {}) + edges = {} + for sig_type, badge in setups.items(): + edge = badge.get("confirmation_edge", "unknown") + edges[sig_type] = edge + has_any = any(e != "unknown" for e in edges.values()) + entry_conf[ticker] = { + "calibrated": has_any, + "edges": edges, + } + + # โ”€โ”€ Source-of-truth reference โ”€โ”€ + sources = { + "freshness": "data_freshness.py โ€” 9 data types per asset", + "calibration": "calibration_reports/*.json โ€” backtest_engine output", + "flow": "flow_params.json โ€” flow_calibrator.py output", + "wf_settings": "best_wf_settings.json โ€” walk_forward_engine output", + "anchor_params": "anchor_params.json โ€” anchor_signal_backtester output", + } + + return { + "universe": universe, + "freshness": freshness, + "calibration": calibration, + "flow": flow, + "anchor": anchor, + "entry_confirmation": entry_conf, + "sources": sources, + } + + # โ”€โ”€ Calibration job tracking (must be before {ticker} routes) โ”€โ”€ + _running_jobs = {} # ticker โ†’ {"status": "running"/"done"/"error", ...} + _cancel_tokens = {} # ticker โ†’ threading.Event (set = cancel) + + @router.get("/api/calibration/status") + async def get_calibration_status(): + """Get calibration job status for all tickers.""" + return _running_jobs + + @router.post("/api/calibration/cancel/{ticker}") + async def cancel_calibration_ticker(ticker: str): + """Cancel a running calibration for one asset.""" + canonical = ticker.upper() + token = _cancel_tokens.get(canonical) + if token: + token.set() + if canonical in _running_jobs: + _running_jobs[canonical]["status"] = "cancelled" + log.info(f"Calibration cancel requested for {canonical}") + return {"cancelled": True, "ticker": canonical} + return {"cancelled": False, "reason": f"No running job for {canonical}"} + + @router.post("/api/calibration/cancel") + async def cancel_calibration_all(): + """Cancel all running calibrations.""" + cancelled = [] + for ticker, token in _cancel_tokens.items(): + if not token.is_set(): + token.set() + if ticker in _running_jobs: + _running_jobs[ticker]["status"] = "cancelled" + cancelled.append(ticker) + log.info(f"Calibration cancel all: {cancelled}") + return {"cancelled": cancelled} + + @router.get("/api/calibration/badges/{ticker}") + async def get_calibration_badges(ticker: str): + """Badge lookup for all setups on one asset.""" + store = _get_cal_store() + badges = store.get_all_badges(ticker.upper()) + if not badges: + return {"ticker": ticker.upper(), "calibrated": False, "badges": {}} + return {"ticker": ticker.upper(), "calibrated": True, "badges": badges} + + @router.get("/api/calibration/detail/{ticker}/{signal_type}") + async def get_calibration_detail(ticker: str, signal_type: str): + """ + Detailed IS/OOS breakdown for one signal type on one asset. + Used by the calibration page drill-down. + + Returns: { + ticker, signal_type, + overall: { grade, score, win_rate, expectancy, ... }, + in_sample: { grade, score, win_rate, total, ... }, + out_of_sample: { grade, score, win_rate, total, ... }, + overfitting_risk, config, date_range, computed_at, + } + """ + store = _get_cal_store() + detail = store.get_detail(ticker.upper(), signal_type) + if detail is None: + raise HTTPException( + 404, + f"No calibration detail for {ticker.upper()} / {signal_type}" + ) + return detail + + @router.get("/api/calibration/outcomes/{ticker}/{signal_type}") + async def get_calibration_outcomes(ticker: str, signal_type: str): + """ + Raw trade outcome records for one signal type on one asset. + Used by the trade chart visualization in the calibration page. + + Returns: [ + { eval_date, direction, entry_price, stop_loss, target_price, + exit_price, exit_date, outcome, r_multiple, is_oos, + entry_confirmed, confirmation_pattern, ... }, + ... + ] + """ + store = _get_cal_store() + outcomes = store.get_outcomes(ticker.upper(), signal_type) + if outcomes is None: + raise HTTPException( + 404, + f"No outcome data for {ticker.upper()} / {signal_type}. " + f"Re-run calibration to generate outcomes." + ) + return outcomes + + @router.get("/api/calibration/{ticker}") + async def get_calibration_report(ticker: str): + """Full calibration report for one asset.""" + store = _get_cal_store() + report = store.load_report(ticker.upper()) + if report is None: + raise HTTPException(404, f"No calibration report for {ticker.upper()}") + return report + + @router.post("/api/calibration/run/{ticker}") + async def run_calibration_ticker(ticker: str, req: CalibrationRunRequest = None): + """ + Trigger backtest calibration for one asset (runs in background thread). + Returns immediately โ€” poll /api/calibration/status for progress. + """ + canonical = ticker.upper() + + if canonical in _running_jobs and _running_jobs[canonical].get("status") == "running": + raise HTTPException(409, f"Calibration already running for {canonical}") + + if req is None: + req = CalibrationRunRequest() + + _running_jobs[canonical] = { + "status": "running", + "ticker": canonical, + "started_at": datetime.now(timezone.utc).isoformat(), + "pct": 0, + "eval_date": "loading data...", + "eval_num": 0, + "eval_total": 0, + "signals": 0, + "wins": 0, + "losses": 0, + "elapsed_s": 0, + "eta_s": 0, + } + + # Create cancel token for this job + cancel_token = threading.Event() + _cancel_tokens[canonical] = cancel_token + + async def _run(): + try: + from backtest_engine import BacktestEngine, BacktestConfig + + # Resolve parallel workers: request โ†’ account settings โ†’ default 1 + pw = req.parallel_workers + if pw <= 1: + try: + mgr = _get_account_manager() + pw = mgr.get_settings_dict().get("calibration_workers", 1) or 1 + except Exception: + pw = 1 + + config = BacktestConfig( + mc_sims=req.mc_sims, + lookback_months=req.lookback_months, + parallel_workers=pw, + ) + engine = BacktestEngine(config) + + def _on_progress(info): + """Update job status with live progress from engine.""" + _running_jobs[canonical] = { + "status": "running", + "ticker": canonical, + "started_at": _running_jobs.get(canonical, {}).get("started_at", ""), + "pct": info.get("pct", 0), + "eval_date": info.get("eval_date", ""), + "eval_num": info.get("eval_num", 0), + "eval_total": info.get("eval_total", 0), + "signals": info.get("signals_so_far", 0), + "wins": info.get("wins_so_far", 0), + "losses": info.get("losses_so_far", 0), + "elapsed_s": info.get("elapsed_s", 0), + "eta_s": info.get("eta_s", 0), + } + + report = await asyncio.to_thread( + engine.run_with_atr_sweep, canonical, + lookback_months=req.lookback_months, + progress_callback=_on_progress, + cancel_token=cancel_token, + ) + + # Check if cancelled + if cancel_token.is_set(): + _running_jobs[canonical] = { + "status": "cancelled", "ticker": canonical, + } + log.info(f"Calibration cancelled for {canonical}") + else: + # Save to store + store = _get_cal_store() + store.save_report(report) + + _running_jobs[canonical] = { + "status": "done", + "ticker": canonical, + "overall_score": report.overall_score, + "overall_grade": report.overall_grade, + "completed_at": datetime.now(timezone.utc).isoformat(), + } + + await _notify("calibration_complete", + f"{canonical}: {report.overall_grade} ({report.overall_score:.2f})") + + except Exception as e: + log.error(f"Calibration failed for {canonical}: {e}", exc_info=True) + _running_jobs[canonical] = { + "status": "error", + "ticker": canonical, + "error": str(e), + } + finally: + _cancel_tokens.pop(canonical, None) + + asyncio.create_task(_run()) + return {"status": "started", "ticker": canonical} + + @router.post("/api/calibration/run") + async def run_calibration_universe(req: CalibrationRunRequest = None): + """ + Trigger calibration for the full universe (background). + Returns immediately โ€” poll /api/calibration/status. + """ + if req is None: + req = CalibrationRunRequest() + + # Get universe from config_manager + try: + from config_manager import ConfigManager + cm = ConfigManager() + tickers = cm.get_active_universe() + except Exception: + try: + from server_config import ASSET_UNIVERSE + tickers = list(ASSET_UNIVERSE) + except Exception: + raise HTTPException(500, "Cannot determine asset universe") + + # Mark all as running + for t in tickers: + _running_jobs[t] = { + "status": "queued", + "ticker": t, + "started_at": datetime.now(timezone.utc).isoformat(), + "pct": 0, + "eval_date": "", + "eval_num": 0, + "eval_total": 0, + "signals": 0, + "wins": 0, + "losses": 0, + "elapsed_s": 0, + "eta_s": 0, + } + + # Create a shared cancel token for the universe run + universe_cancel = threading.Event() + for t in tickers: + _cancel_tokens[t] = universe_cancel + + async def _run_all(): + try: + from backtest_engine import BacktestEngine, BacktestConfig + + # Resolve parallel workers: request โ†’ account settings โ†’ default 1 + pw = req.parallel_workers + if pw <= 1: + try: + mgr = _get_account_manager() + pw = mgr.get_settings_dict().get("calibration_workers", 1) or 1 + except Exception: + pw = 1 + + config = BacktestConfig( + mc_sims=req.mc_sims, + lookback_months=req.lookback_months, + parallel_workers=pw, + ) + engine = BacktestEngine(config) + store = _get_cal_store() + + for i, ticker in enumerate(tickers): + # โ”€โ”€ Cancel check between tickers โ”€โ”€ + if universe_cancel.is_set(): + for remaining in tickers[i:]: + _running_jobs[remaining] = { + "status": "cancelled", "ticker": remaining, + } + log.info(f"Universe calibration cancelled at ticker {i}/{len(tickers)}") + break + + _running_jobs[ticker] = { + "status": "running", + "ticker": ticker, + "started_at": _running_jobs.get(ticker, {}).get("started_at", ""), + "pct": 0, + "eval_date": "loading data...", + "eval_num": 0, + "eval_total": 0, + "signals": 0, + "wins": 0, + "losses": 0, + "elapsed_s": 0, + "eta_s": 0, + } + + def _make_cb(t): + """Create a closure-safe callback for this ticker.""" + def _on_progress(info): + _running_jobs[t] = { + "status": "running", + "ticker": t, + "started_at": _running_jobs.get(t, {}).get("started_at", ""), + "pct": info.get("pct", 0), + "eval_date": info.get("eval_date", ""), + "eval_num": info.get("eval_num", 0), + "eval_total": info.get("eval_total", 0), + "signals": info.get("signals_so_far", 0), + "wins": info.get("wins_so_far", 0), + "losses": info.get("losses_so_far", 0), + "elapsed_s": info.get("elapsed_s", 0), + "eta_s": info.get("eta_s", 0), + } + return _on_progress + + try: + report = await asyncio.to_thread( + engine.run_with_atr_sweep, ticker, + lookback_months=req.lookback_months, + progress_callback=_make_cb(ticker), + cancel_token=universe_cancel, + ) + + if universe_cancel.is_set(): + _running_jobs[ticker] = { + "status": "cancelled", "ticker": ticker, + } + else: + store.save_report(report) + _running_jobs[ticker] = { + "status": "done", + "ticker": ticker, + "overall_score": report.overall_score, + "overall_grade": report.overall_grade, + } + except Exception as e: + _running_jobs[ticker] = { + "status": "error", + "ticker": ticker, + "error": str(e), + } + + if not universe_cancel.is_set(): + await _notify("calibration_universe_complete", + f"{len(tickers)} assets calibrated") + + except Exception as e: + log.error(f"Universe calibration failed: {e}", exc_info=True) + finally: + for t in tickers: + _cancel_tokens.pop(t, None) + + asyncio.create_task(_run_all()) + return {"status": "started", "tickers": tickers} + + # โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ• + # AUTO EXECUTOR ENDPOINTS (Phase 3G) + # โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ• + + @router.get("/api/auto/status") + async def get_auto_status(): + """Auto executor status โ€” settings + today's activity.""" + mgr = _get_account_manager() + settings = mgr.get_settings_dict() + + # Read today's trade count from auto log + trades_today = 0 + signals_processed = 0 + auto_log_path = PROJECT_ROOT / "auto_executor_log.json" + if auto_log_path.exists(): + try: + with open(auto_log_path, "r") as f: + log_data = json.load(f) + today_str = datetime.now(timezone.utc).strftime("%Y-%m-%d") + for entry in reversed(log_data): + ts = entry.get("timestamp", "") + if not ts.startswith(today_str): + break + signals_processed += 1 + if entry.get("action") in ("executed", "dry_run"): + trades_today += 1 + except Exception: + pass + + return { + "enabled": settings.get("auto_trading_enabled", False), + "log_only_mode": settings.get("auto_log_only", True), + "live_trading_on": mgr.settings.trading_enabled, + "trades_today": trades_today, + "signals_processed": signals_processed, + "max_daily_trades": settings.get("auto_max_daily_trades", 5), + "min_confidence": settings.get("auto_min_confidence", 0.60), + "min_grade": settings.get("auto_min_grade", "C"), + "allowed_types": settings.get("auto_allowed_types", []), + "allowed_tickers": settings.get("auto_allowed_tickers", []) or "all", + "scan_interval": settings.get("auto_scan_interval", 30.0), + } + + @router.get("/api/auto/log") + async def get_auto_log(limit: int = 50): + """Recent auto executor actions (audit trail).""" + auto_log_path = PROJECT_ROOT / "auto_executor_log.json" + if not auto_log_path.exists(): + return {"actions": [], "count": 0} + try: + with open(auto_log_path, "r") as f: + data = json.load(f) + return {"actions": data[-limit:], "count": len(data)} + except Exception: + return {"actions": [], "count": 0} + + @router.get("/api/auto/log/summary") + async def get_auto_log_summary(days: int = 30): + """ + Performance summary from auto executor log. + Aggregates by overall, per-ticker, per-signal-type, and per-ticker-type. + """ + auto_log_path = PROJECT_ROOT / "auto_executor_log.json" + if not auto_log_path.exists(): + return {"trades": 0, "summary": {}} + try: + with open(auto_log_path, "r") as f: + all_data = json.load(f) + except Exception: + return {"trades": 0, "summary": {}} + + # Filter to requested time window + cutoff = (datetime.now(timezone.utc) - timedelta(days=days)).isoformat() + entries = [e for e in all_data + if e.get("action") in ("executed", "dry_run") + and e.get("timestamp", "") >= cutoff] + + if not entries: + return {"trades": 0, "days": days, "summary": {}} + + # Build aggregations + def _agg(subset): + count = len(subset) + tickers = list(set(e.get("ticker", "") for e in subset)) + types = list(set(e.get("signal_type", "") for e in subset)) + avg_conf = sum(e.get("confidence", 0) for e in subset) / count if count else 0 + avg_rr = sum(e.get("risk_reward", 0) for e in subset) / count if count else 0 + total_risk = sum(e.get("risk_usd", 0) for e in subset) + avg_risk_pct = sum(e.get("risk_pct", 0) for e in subset) / count if count else 0 + grades = {} + for e in subset: + g = e.get("calibration_grade", "?") + grades[g] = grades.get(g, 0) + 1 + regimes = {} + for e in subset: + r = e.get("regime", "?") + regimes[r] = regimes.get(r, 0) + 1 + directions = {"long": 0, "short": 0} + for e in subset: + d = e.get("direction", "") + if d in directions: + directions[d] += 1 + return { + "count": count, + "avg_confidence": round(avg_conf, 4), + "avg_risk_reward": round(avg_rr, 2), + "total_risk_usd": round(total_risk, 2), + "avg_risk_pct": round(avg_risk_pct, 4), + "grade_distribution": grades, + "regime_distribution": regimes, + "direction_split": directions, + "tickers": tickers, + "signal_types": types, + } + + # Overall + summary = {"overall": _agg(entries)} + + # Per ticker + by_ticker = {} + for e in entries: + t = e.get("ticker", "?") + by_ticker.setdefault(t, []).append(e) + summary["by_ticker"] = {t: _agg(v) for t, v in by_ticker.items()} + + # Per signal type + by_type = {} + for e in entries: + st = e.get("signal_type", "?") + by_type.setdefault(st, []).append(e) + summary["by_signal_type"] = {st: _agg(v) for st, v in by_type.items()} + + # Per ticker ร— signal type + by_ticker_type = {} + for e in entries: + key = f"{e.get('ticker', '?')}|{e.get('signal_type', '?')}" + by_ticker_type.setdefault(key, []).append(e) + summary["by_ticker_type"] = {k: _agg(v) for k, v in by_ticker_type.items()} + + return { + "trades": len(entries), + "days": days, + "summary": summary, + } + + # โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ• + # TRADE MANAGER ENDPOINTS + # โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ• + + @router.get("/api/trade-manager/status") + async def get_trade_manager_status(): + """Trade manager status โ€” managed positions and modes.""" + try: + from trade_manager import get_trade_manager + tm = get_trade_manager() + return tm.get_status() + except ImportError: + raise HTTPException(503, "trade_manager module not available") + + # โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ• + # WALK-FORWARD CALIBRATION (Phase 3I) + # โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ• + + _wf_jobs = {} # ticker โ†’ {"status": "running"/"done"/"error", ...} + + @router.post("/api/wf/run/{ticker}") + async def run_wf_ticker(ticker: str): + """ + Trigger walk-forward engine for one asset (background subprocess). + Calls: python walk_forward_engine.py --assets TICKER --sims 100000 + Writes to backtest_results/best_wf_settings.json (merge-on-save). + Returns immediately โ€” poll /api/wf/status for progress. + """ + import subprocess + canonical = ticker.upper() + + if canonical in _wf_jobs and _wf_jobs[canonical].get("status") == "running": + raise HTTPException(409, f"WF already running for {canonical}") + + _wf_jobs[canonical] = { + "status": "running", + "ticker": canonical, + "started_at": datetime.now(timezone.utc).isoformat(), + } + + async def _run(): + try: + # Walk-forward engine is verbose โ€” redirect to log file, not PIPE. + cmd = [ + "python", str(PROJECT_ROOT / "walk_forward_engine.py"), + "--assets", canonical, + "--sims", "100000", + "--step", "5", + ] + wf_log_path = PROJECT_ROOT / "flow_output" / f"wf_{canonical}.log" + await _notify("wf_started", + f"Walk-forward engine started for {canonical}") + + def _wf_runner(): + p = _debug_popen(cmd, label=f"wf_{canonical}") + p.wait(timeout=900) + return p + proc = await asyncio.to_thread(_wf_runner) + + if proc.returncode == 0: + _wf_jobs[canonical] = { + "status": "done", + "ticker": canonical, + "completed_at": datetime.now(timezone.utc).isoformat(), + "wf_log": str(wf_log_path), + } + await _notify("wf_done", + f"Walk-forward complete for {canonical} โ€” best_wf_settings.json updated") + log.info(f"WF completed for {canonical}") + else: + tail = f"See subprocess_logs/wf_{canonical}.log" + _wf_jobs[canonical] = { + "status": "error", + "ticker": canonical, + "error": tail, + } + await _notify("wf_error", + f"Walk-forward FAILED for {canonical} โ€” check flow_output/wf_{canonical}.log") + log.error(f"WF failed for {canonical} (log: {wf_log_path})") + except subprocess.TimeoutExpired: + _wf_jobs[canonical] = { + "status": "error", "ticker": canonical, + "error": "Timed out after 900s", + } + except Exception as e: + _wf_jobs[canonical] = { + "status": "error", "ticker": canonical, + "error": str(e), + } + + asyncio.create_task(_run()) + return {"status": "started", "ticker": canonical} + + @router.get("/api/wf/status") + async def get_wf_status(): + """Walk-forward job status for all tickers.""" + return _wf_jobs + + # โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ• + # FLOW CALIBRATION (Phase 3I) + # โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ• + + _flow_jobs = {} # ticker โ†’ {"status": "running"/"done"/"error", ...} + + @router.post("/api/flow/calibrate/{ticker}") + async def run_flow_calibrate_ticker(ticker: str): + """ + Trigger flow_calibrator.py for one asset (background subprocess). + Updates flow_params.json for this ticker, then validates via + flow_confirmation.py CLI diagnostic. + Returns immediately โ€” poll /api/flow/status for progress. + """ + import subprocess + canonical = ticker.upper() + + if canonical in _flow_jobs and _flow_jobs[canonical].get("status") == "running": + raise HTTPException(409, f"Flow calibration already running for {canonical}") + + _flow_jobs[canonical] = { + "status": "running", + "ticker": canonical, + "started_at": datetime.now(timezone.utc).isoformat(), + "phase": "calibrating", + } + + async def _run(): + try: + # Phase 1: Run flow_calibrator.py for this ticker + # IMPORTANT: flow_calibrator produces massive output (1944-combo grid search + + # GPU progress bars). capture_output=True / PIPE deadlocks when the pipe buffer + # fills (~64 KB on Windows). Redirect stdout+stderr to a log file instead. + # Rule: Use DEVNULL/file not PIPE for verbose subprocesses. + cal_cmd = [ + "python", str(PROJECT_ROOT / "flow_calibrator.py"), + canonical, + ] + _flow_jobs[canonical]["phase"] = "calibrating" + await _notify("flow_calibration_started", + f"Flow calibration started for {canonical} โ€” GPU grid search running") + + def _cal_runner(): + p = _debug_popen(cal_cmd, label=f"flow_calibrator_{canonical}") + p.wait(timeout=900) + return p + proc = await asyncio.to_thread(_cal_runner) + + if proc.returncode != 0: + # Read tail of log for error context + tail = f"See subprocess_logs/flow_calibrator_{canonical}.log" + _flow_jobs[canonical] = { + "status": "error", "ticker": canonical, + "phase": "calibration_failed", + "error": tail, + } + await _notify("flow_calibration_error", + f"Flow calibration FAILED for {canonical} โ€” check flow_output/calibrate_{canonical}.log") + log.error(f"Flow calibration failed for {canonical} (log: {cal_log_path})") + return + + # Phase 2: Run flow_confirmation.py CLI diagnostic to validate. + # Validator output is small (~50 lines) โ€” PIPE is safe here. + _flow_jobs[canonical]["phase"] = "validating" + await _notify("flow_validation_started", + f"Flow calibration complete for {canonical} โ€” running validator") + val_cmd = [ + "python", str(PROJECT_ROOT / "flow_confirmation.py"), + canonical, + ] + val_proc = await asyncio.to_thread( + subprocess.run, val_cmd, + cwd=str(PROJECT_ROOT), + capture_output=True, text=True, timeout=120, + env={**__import__("os").environ, "PYTHONUTF8": "1"}, + ) + + _flow_jobs[canonical] = { + "status": "done", + "ticker": canonical, + "completed_at": datetime.now(timezone.utc).isoformat(), + "validation_output": val_proc.stdout[-1000:] if val_proc.stdout else "", + } + await _notify("flow_calibration_done", + f"Flow calibration + validation complete for {canonical} โ€” flow_params.json updated") + log.info(f"Flow calibration + validation complete for {canonical}") + + except subprocess.TimeoutExpired: + _flow_jobs[canonical] = { + "status": "error", "ticker": canonical, + "error": "Timed out after 900s", + } + await _notify("flow_calibration_error", + f"Flow calibration TIMED OUT for {canonical} (>900s)") + except Exception as e: + _flow_jobs[canonical] = { + "status": "error", "ticker": canonical, + "error": str(e), + } + await _notify("flow_calibration_error", + f"Flow calibration ERROR for {canonical}: {e}") + + asyncio.create_task(_run()) + return {"status": "started", "ticker": canonical} + + @router.get("/api/flow/status") + async def get_flow_status(): + """Flow calibration job status for all tickers.""" + return _flow_jobs + + return router \ No newline at end of file diff --git a/backend/cache_watcher.py b/backend/cache_watcher.py new file mode 100644 index 0000000..31d1901 --- /dev/null +++ b/backend/cache_watcher.py @@ -0,0 +1,323 @@ +# version: v3 +""" +================================================================================ +Quantum Terminal Consumer โ€” Cache Directory Watcher +================================================================================ +Async task that watches %APPDATA%\\QuantumTerminal\\cache\\ for JSON file changes and: + 1. Loads the changed file into the sync client's in-memory store. + 2. Broadcasts a 'data_sync_complete' WS event so the frontend re-fetches. + +Purpose +------- +During the Data Center transition (and for local debugging) the user may drop +files directly into the cache dir. Without this watcher the running server +never re-reads disk after startup, so those drops are invisible until a full +restart. This watcher closes the gap. + +It also gives live updates when sync_all() writes new cache files โ€” the usual +periodic_sync broadcast happens too, but this provides a second, finer-grained +trigger (per-file, immediate) and covers the case where sync_all is not the +writer (manual drop, future DC auto-uploader, etc.). + +Mechanism +--------- +Polls os.stat() mtime on *.json in the cache dir (no new dependency). A file +must have a *stable* mtime for two consecutive ticks before we act on it โ€” +this debounces partial-write races. + +Poll interval defaults to 1 second (configurable). Zero compute, Rule C1 safe. + +Config (consumer_config.ini) +---------------------------- + [sync] + cache_watcher_enabled = true ; set false to disable entirely + cache_watcher_interval_seconds = 1 ; floor enforced at 0.25s + +Integration +----------- +Wired from data_server.py lifespan, same pattern as periodic_sync. Takes the +WS broadcast_event callable. Cancellation honored via asyncio.CancelledError. + +v2 โ€” Deletions detected and propagated. When a file vanishes from the cache + directory, the corresponding entry in the sync client's in-memory store is + purged and the broadcast payload reports it via a "removed_files" list. + Frontend treats this like any other data_sync_complete event โ†’ refetches + and gets a 404/null โ†’ clears the overlay. + +v3 โ€” Filename parsing now delegates to data_sync_client.resolve_cache_filename + so bare globals ("money_flow.json"), prefixed globals with multi-word + categories ("GLOBAL_probability_state.json"), and the legacy + GLOBAL__.json misnaming all route to the + correct in-memory slot. Same resolver used by _load_all_from_cache at + startup, ensuring watcher and loader stay consistent. + +Rule C1 compliance: this module performs ZERO calculations โ€” it only reads +files already on disk and pushes WS events. +================================================================================ +""" + +import asyncio +import configparser +import json +import logging +import os + +# v2: AppData dir name (env-var driven; default "QuantumTerminal" preserves v1 path). +APP_DIR = os.environ.get("MK_APP_DIR_NAME", "QuantumTerminal") +import re +from datetime import datetime, timezone +from pathlib import Path +from typing import Awaitable, Callable, Optional + +log = logging.getLogger("mk.cache_watcher") + +DEFAULT_INTERVAL_SECONDS = 1.0 +MIN_INTERVAL_SECONDS = 0.25 + +# Same filename pattern as data_sync_client._load_all_from_cache(). +# Greedy first group + underscore-delimited category tail. +_FILE_PATTERN = re.compile(r'^(.+)_([a-z_]+)\.json$', re.IGNORECASE) + + +def _read_config() -> tuple[bool, float]: + """Read cache_watcher_enabled + interval from consumer_config.ini. + Returns (enabled, interval_seconds). Defaults: enabled=True, interval=1.0.""" + project_root = Path(__file__).resolve().parent + + candidates = [ + project_root / "consumer_config.ini", + project_root.parent / "consumer_config.ini", + ] + + import sys as _sys + if getattr(_sys, "frozen", False): + candidates.append(Path(_sys.executable).parent / "consumer_config.ini") + + appdata = os.environ.get("APPDATA", "") + if appdata: + candidates.append(Path(appdata) / APP_DIR / "consumer_config.ini") + + enabled = True + interval = DEFAULT_INTERVAL_SECONDS + + for p in candidates: + if not p.exists(): + continue + try: + cp = configparser.ConfigParser() + cp.read(str(p), encoding="utf-8-sig") + if cp.has_option("sync", "cache_watcher_enabled"): + enabled = cp.getboolean("sync", "cache_watcher_enabled") + if cp.has_option("sync", "cache_watcher_interval_seconds"): + try: + interval = max(MIN_INTERVAL_SECONDS, + cp.getfloat("sync", "cache_watcher_interval_seconds")) + except (ValueError, TypeError): + pass + break + except Exception as e: + log.warning(f"Could not parse cache_watcher config from {p.name}: {e}") + break + + return enabled, interval + + +def _get_cache_dir() -> Optional[Path]: + """Resolve %APPDATA%\\QuantumTerminal\\cache via the same helper data_sync_client uses.""" + try: + from data_sync_client import _get_cache_dir as _dsc_cache_dir + return _dsc_cache_dir() + except Exception: + appdata = os.environ.get("APPDATA", "") + if not appdata: + return None + return Path(appdata) / APP_DIR / "cache" + + +def create_cache_watcher_task( + broadcast_event: Callable[[dict], Awaitable[None]], +) -> Optional[Callable]: + """ + Returns an async coroutine factory that watches the cache dir. + Returns None if disabled via config, so the caller can skip task creation. + + Usage in data_server.py lifespan: + from cache_watcher import create_cache_watcher_task + watcher = create_cache_watcher_task(manager.broadcast_event) + if watcher: + tasks.append(asyncio.create_task(watcher())) + """ + enabled, interval = _read_config() + if not enabled: + log.info("Cache watcher disabled via config") + return None + + cache_dir = _get_cache_dir() + if cache_dir is None: + log.warning("Cache watcher: cannot resolve cache dir โ€” disabled") + return None + + log.info(f"Cache watcher configured: dir={cache_dir}, interval={interval}s") + + async def _task(): + # Track stable mtimes (the "known good" state we've already processed). + stable_mtimes: dict[str, int] = {} + # Track the most-recent-seen mtime so we can require two consecutive + # equal readings before acting (debounce against partial writes). + last_seen: dict[str, int] = {} + + # Seed stable_mtimes with everything currently on disk so we don't + # fire a flood of "data_sync_complete" events on startup. + try: + if cache_dir.exists(): + for f in cache_dir.glob("*.json"): + if f.name.startswith("_"): + continue + try: + stable_mtimes[f.name] = f.stat().st_mtime_ns + except OSError: + continue + except Exception as e: + log.warning(f"Cache watcher seed failed: {e}") + + while True: + try: + if not cache_dir.exists(): + await asyncio.sleep(interval) + continue + + changed: list[Path] = [] + current_names: set[str] = set() + + for f in cache_dir.glob("*.json"): + if f.name.startswith("_"): + continue + current_names.add(f.name) + try: + mt = f.stat().st_mtime_ns + except OSError: + continue + + prev_seen = last_seen.get(f.name) + prev_stable = stable_mtimes.get(f.name) + + if mt == prev_stable: + # Already processed at this mtime โ€” nothing to do. + last_seen[f.name] = mt + continue + + if prev_seen == mt: + # Stable for 2 ticks at a new mtime โ†’ accept. + changed.append(f) + stable_mtimes[f.name] = mt + last_seen[f.name] = mt + + # Detect deletions: files that were known-stable last tick but + # are no longer in the directory. Drop bookkeeping AND purge + # the sync client's in-memory store so /api endpoints stop + # serving the stale data. + deleted: list[str] = [] + for gone in list(stable_mtimes.keys()): + if gone not in current_names: + deleted.append(gone) + stable_mtimes.pop(gone, None) + last_seen.pop(gone, None) + + if changed or deleted: + await _handle_changes(changed, deleted, broadcast_event) + + except asyncio.CancelledError: + log.info("Cache watcher task cancelled") + break + except Exception as e: + log.error(f"Cache watcher tick error: {e}", exc_info=True) + + try: + await asyncio.sleep(interval) + except asyncio.CancelledError: + break + + return _task + + +async def _handle_changes( + changed: list[Path], + deleted: list[str], + broadcast_event: Callable[[dict], Awaitable[None]], +) -> None: + """Reconcile the sync client's in-memory store with disk, then broadcast.""" + try: + from data_sync_client import get_sync_client + client = get_sync_client() + except Exception as e: + log.warning(f"Cache watcher: sync client unavailable: {e}") + return + + # v3: use data_sync_client's shared resolver for consistent naming rules. + try: + from data_sync_client import resolve_cache_filename + except Exception: + resolve_cache_filename = None + + def _resolve(fname: str): + if not fname.lower().endswith(".json"): + return None + stem = fname[:-5] + if resolve_cache_filename: + return resolve_cache_filename(stem) + # Fallback โ€” legacy regex if sync client unavailable for any reason. + m = _FILE_PATTERN.match(fname) + return (m.group(1).upper(), m.group(2).lower()) if m else None + + loaded: list[dict] = [] + for path in changed: + resolved = _resolve(path.name) + if not resolved: + continue + ticker, category = resolved + + try: + with path.open("r", encoding="utf-8") as fh: + data = json.load(fh) + except Exception as e: + log.warning(f"Cache watcher: failed to read {path.name}: {e}") + continue + + try: + client._get_store(category)[ticker] = data + client._known_categories.add(category) + loaded.append({"file": path.name, "ticker": ticker, "category": category}) + log.info(f"Cache watcher: reloaded {path.name} โ†’ ({ticker}, {category})") + except Exception as e: + log.warning(f"Cache watcher: store update failed for {path.name}: {e}") + + removed: list[dict] = [] + for fname in deleted: + resolved = _resolve(fname) + if not resolved: + continue + ticker, category = resolved + try: + store = client._get_store(category) + if ticker in store: + store.pop(ticker, None) + removed.append({"file": fname, "ticker": ticker, "category": category}) + log.info(f"Cache watcher: purged {fname} (deleted from disk)") + except Exception as e: + log.warning(f"Cache watcher: store purge failed for {fname}: {e}") + + if not loaded and not removed: + return + + try: + await broadcast_event({ + "type": "data_sync_complete", + "synced": len(loaded), + "deleted": len(removed), + "source": "cache_watcher", + "files": [x["file"] for x in loaded], + "removed_files": [x["file"] for x in removed], + "timestamp": datetime.now(timezone.utc).isoformat(), + }) + except Exception as e: + log.warning(f"Cache watcher: broadcast failed: {e}") diff --git a/backend/chart_presets_routes.py b/backend/chart_presets_routes.py new file mode 100644 index 0000000..4583d33 --- /dev/null +++ b/backend/chart_presets_routes.py @@ -0,0 +1,82 @@ +# version: v1 +"""Chart-preset REST endpoints. + +Wire into data_server.py: + from chart_presets_routes import create_chart_presets_router + app.include_router(create_chart_presets_router()) + +Endpoints: + GET /api/chart-presets โ€” returns dict of all named presets + PUT /api/chart-presets/{name} โ€” body is the style dict; creates or replaces + DELETE /api/chart-presets/{name} โ€” removes the named preset (idempotent) +""" + +import logging +from typing import Any +from urllib.parse import unquote + +from fastapi import APIRouter, HTTPException +from pydantic import BaseModel + +import config_manager + + +log = logging.getLogger("mk.chart_presets_routes") + +# Pydantic body model โ€” accepts the full chart-style shape but is permissive +# on field presence so future shape changes don't break the API. +class ChartStyleIn(BaseModel): + chartType: str | None = None + background: str | None = None + bullishColor: str | None = None + bearishColor: str | None = None + gridLines: bool | None = None + marketCloseSeparator: dict | None = None + rthSeparator: dict | None = None + + class Config: + extra = "allow" # forward-compat: ignore unknown keys gracefully + + +_RESERVED_NAMES = {"defaults", "default"} +_MAX_NAME_LEN = 60 + + +def _validate_name(name: str) -> None: + if not name or not name.strip(): + raise HTTPException(400, detail="Preset name cannot be empty") + if len(name) > _MAX_NAME_LEN: + raise HTTPException(400, detail=f"Preset name exceeds {_MAX_NAME_LEN} characters") + if name.strip().lower() in _RESERVED_NAMES: + raise HTTPException(400, detail=f"'{name}' is a reserved name") + + +def create_chart_presets_router() -> APIRouter: + router = APIRouter(prefix="/api/chart-presets", tags=["chart-presets"]) + + @router.get("") + async def list_presets() -> dict[str, Any]: + return config_manager.get_chart_presets() + + @router.put("/{name}") + async def upsert_preset(name: str, body: ChartStyleIn) -> dict[str, Any]: + decoded = unquote(name) + _validate_name(decoded) + try: + config_manager.save_chart_preset(decoded, body.model_dump(exclude_none=False)) + except Exception as e: + log.exception("save_chart_preset failed") + raise HTTPException(500, detail=f"Save failed: {e}") from e + return config_manager.get_chart_presets() + + @router.delete("/{name}") + async def delete_preset(name: str) -> dict[str, Any]: + decoded = unquote(name) + try: + config_manager.delete_chart_preset(decoded) + except Exception as e: + log.exception("delete_chart_preset failed") + raise HTTPException(500, detail=f"Delete failed: {e}") from e + return config_manager.get_chart_presets() + + return router diff --git a/backend/chart_templates_routes.py b/backend/chart_templates_routes.py new file mode 100644 index 0000000..ac0f8ea --- /dev/null +++ b/backend/chart_templates_routes.py @@ -0,0 +1,84 @@ +# version: v1 +"""Chart-template REST endpoints. + +A "chart template" is a named full-chart-config snapshot the operator can +save and re-apply to any chart ("XAUUSD swing", "XAUUSD scalp", ...). +Distinct from /api/chart-presets, which holds chart-STYLE-only fragments. + +Wire into data_server.py: + from chart_templates_routes import create_chart_templates_router + app.include_router(create_chart_templates_router()) + +Endpoints: + GET /api/chart-templates โ€” returns dict of all named templates {name: {sourceTicker, config, savedAt}} + PUT /api/chart-templates/{name} โ€” body is {sourceTicker, config, savedAt}; creates or replaces + DELETE /api/chart-templates/{name} โ€” removes the named template (idempotent) +""" + +import logging +from typing import Any +from urllib.parse import unquote + +from fastapi import APIRouter, HTTPException +from pydantic import BaseModel + +import config_manager + + +log = logging.getLogger("mk.chart_templates_routes") + + +# Permissive body model: a template carries a sourceTicker, a config dict +# (the full chart-config bundle), and a savedAt timestamp. `extra = "allow"` +# keeps the API forward-compatible if the shape grows. +class ChartTemplateIn(BaseModel): + sourceTicker: str | None = None + config: dict | None = None + savedAt: str | None = None + + class Config: + extra = "allow" + + +_RESERVED_NAMES = {"defaults", "default"} +_MAX_NAME_LEN = 60 + + +def _validate_name(name: str) -> None: + if not name or not name.strip(): + raise HTTPException(400, detail="Template name cannot be empty") + if len(name) > _MAX_NAME_LEN: + raise HTTPException(400, detail=f"Template name exceeds {_MAX_NAME_LEN} characters") + if name.strip().lower() in _RESERVED_NAMES: + raise HTTPException(400, detail=f"'{name}' is a reserved name") + + +def create_chart_templates_router() -> APIRouter: + router = APIRouter(prefix="/api/chart-templates", tags=["chart-templates"]) + + @router.get("") + async def list_templates() -> dict[str, Any]: + return config_manager.get_chart_templates() + + @router.put("/{name}") + async def upsert_template(name: str, body: ChartTemplateIn) -> dict[str, Any]: + decoded = unquote(name) + _validate_name(decoded) + try: + config_manager.save_chart_template(decoded, body.model_dump(exclude_none=False)) + except Exception as e: + log.exception("save_chart_template failed") + raise HTTPException(500, detail=f"Save failed: {e}") from e + return config_manager.get_chart_templates() + + @router.delete("/{name}") + async def delete_template(name: str) -> dict[str, Any]: + decoded = unquote(name) + try: + config_manager.delete_chart_template(decoded) + except Exception as e: + log.exception("delete_chart_template failed") + raise HTTPException(500, detail=f"Delete failed: {e}") from e + return config_manager.get_chart_templates() + + return router diff --git a/backend/config_manager.py b/backend/config_manager.py new file mode 100644 index 0000000..95265e4 --- /dev/null +++ b/backend/config_manager.py @@ -0,0 +1,1413 @@ +# version: v11 +# v11 โ€” User-owned saves move to %USERPROFILE%\Documents\Quantum Terminal\ as +# per-file stores (2026-05-13). Bug: a 4-key trimmed user_config.json +# was being written when _load_user_config silently fell back to {} +# on read errors; profiles/templates/presets were getting wiped on +# every rebuild/update. Fix: (a) new per-file store with one JSON +# per saved item under Documents/Quantum Terminal/{profiles,chart_templates, +# chart_presets}/; (b) one-time migration from AppData-v2 (and +# QuantumTerminal-X + v1 QuantumTerminal as best-effort fallback sources, read-only) +# gated on a marker file; (c) _load_user_config renames a corrupt +# file to .corrupt. instead of silently wiping it. Existing REST +# routes /api/chart-templates, /api/chart-presets, /api/config/profiles +# are unchanged on the wire โ€” only the backing storage changes. +# Spec: docs/specs/2026-05-13-user-data-store-rework-design.md. +# +# v10 โ€” Module-level chart_templates API (get_chart_templates, +# save_chart_template, delete_chart_template), mirroring the v7 +# chart_presets layer. Default config skeleton gains a +# "chart_templates": {} key; _save_user_config preserves it from +# on-disk like chart_presets; _load_config defaults + backfills it. +# Consumed via /api/chart-templates only โ€” doesn't flow through the +# merged config served to the frontend. (2026-05-11) +# +# v9 โ€” _save_user_config now preserves chart_presets from on-disk state +# before writing. This closes the residual race where ConfigManager +# could overwrite a preset just saved via the module path (because the +# instance's self._user_config is loaded once at init and doesn't +# track chart_presets writes from the module path). Together with v8, +# both write directions for chart_presets are now race-free. +# +# v8 โ€” removed the v7 module-level `_cached_config` layer. The dual-layer +# design (ConfigManager instance cache + module-level cache, both +# writing to user_config.json) was a lost-write hazard: a chart-preset +# PUT would update disk + module cache, but the ConfigManager instance +# held a stale `self._user_config`; the next ConfigManager._save_user_config +# (e.g. from a profile save or any /api/config PATCH) would overwrite +# the file with its stale snapshot, silently dropping the just-saved +# chart preset. v8 fixes this by making the chart-preset path +# always-read-fresh-from-disk + atomic-write-via-tempfile, so the two +# layers no longer race over a shared cache. Tests hook the path +# resolver, not the cache, so dropping the cache is test-transparent. +# v7 โ€” added module-level chart_presets API (get_chart_presets, +# save_chart_preset, delete_chart_preset) plus a module-level +# load/save layer (_user_config_path, _load_config, _save_config, +# _cached_config) that operates directly on user_config.json. This +# is independent of the ConfigManager class instance โ€” chart presets +# are consumed via /api/chart-presets only and don't flow through +# the merged config served to the frontend, so the two layers can +# coexist without a stale-cache hazard. Default config skeleton now +# includes a "chart_presets": {} key so first-write of a preset slots +# cleanly into existing user configs. +# v6 โ€” user_config.json (profiles + universe + per-ticker preferences) +# now lives in %APPDATA%\QuantumTerminal-v2\ instead of the install directory. +# Reason: NSIS overwrites the install directory on every update, +# which was wiping operator profiles on every reinstall. AppData +# survives reinstalls. First-launch migration copies the legacy +# PROJECT_ROOT/user_config.json into AppData if AppData is empty, +# so existing users keep their profiles transparently. +""" +================================================================================ +Quantum Terminal โ€” Configuration Manager + +v2 โ€” Added get_broker_symbol() helper and auto-wires broker-symbol override + callback into each provider after creation. Lets users override a + broker's symbol naming (e.g. EURUSD โ†’ EURUSD_VV2) from the Settings + panel and have the MT5 provider use that mapping at resolve time. +================================================================================ +================================================================================ +Central config system for the terminal. Single source of truth. + +Architecture: + server_config.py (static defaults, versioned in Git) + โ†“ merged with + user_config.json (user runtime preferences, gitignored) + โ†“ produces + ConfigManager.get_config() โ†’ unified config dict served to frontend + +What it manages: + - Universe: which tickers are active, their metadata, display order + - Providers: which data/execution sources are configured and active + - Display: theme, chart height, visible panels, default timeframe + - Modules: which quant modules are enabled (cones, bands, signals, etc.) + - Profiles: save/load named terminal configurations + +Mutation: + - PATCH /api/config โ†’ ConfigManager.update(patch_dict) โ†’ writes user_config.json + - Frontend reads via GET /api/config on mount (useConfig hook) + - Changes broadcast via WS event so all clients stay in sync + +Portability (Rule 3): + - user_config.json lives at PROJECT_ROOT/user_config.json + - No absolute paths in config โ€” terminal_path comes from local_config.ini + - Machine-specific values (credentials, paths) stay in local_config.ini + +Usage: + from config_manager import ConfigManager + cfg = ConfigManager() + full_config = cfg.get_config() # Merged config dict + cfg.update({"display": {"chart_height": 500}}) # Partial update +================================================================================ +""" + +import json +import copy +import logging +import os +import tempfile +import hashlib +import re +from pathlib import Path +from typing import Dict, List, Optional, Any +from datetime import datetime, timezone + +log = logging.getLogger("config_manager") + +# โ”€โ”€ Project root (Rule 3) โ”€โ”€ +PROJECT_ROOT = Path(__file__).resolve().parent + +# v6: AppData dir โ€” same env-var convention used by license_client.py so v1 +# and v2 can run side by side without sharing user state. Default +# "QuantumTerminal" preserves v1 behavior. +APP_DIR = os.environ.get("MK_APP_DIR_NAME", "QuantumTerminal") + + +def _appdata_user_config_path() -> Path: + """Resolve %APPDATA%\\\\user_config.json. Used as the canonical + storage path for user preferences + saved profiles. Survives reinstalls + of the app (NSIS overwrites the install dir but leaves AppData alone).""" + appdata = os.environ.get("APPDATA") + if appdata: + base = Path(appdata) / APP_DIR + else: + base = Path.home() / "AppData" / "Roaming" / APP_DIR + base.mkdir(parents=True, exist_ok=True) + return base / "user_config.json" + + +# โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ• +# v11: per-file user-data store (Documents\Quantum Terminal\) +# โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ• +# +# Three classes of user-owned saves โ€” profiles, chart_templates, +# chart_presets โ€” live as one JSON file per saved item under +# %USERPROFILE%\Documents\Quantum Terminal\\. Decouples them from the +# user_config.json trimming-on-save bug, and puts them in a folder users +# can see/back up/copy. + +_USER_DATA_SUBDIRS = ("profiles", "chart_templates", "chart_presets") +_MIGRATION_MARKER = ".migrated_v1" +_README_TEXT = ( + "Quantum Terminal โ€” your saved settings live here.\n" + "\n" + " profiles/ โ€” saved workspace profiles\n" + " chart_templates/ โ€” saved chart templates\n" + " chart_presets/ โ€” saved chart style presets\n" + "\n" + "Each .json file is one saved item. Safe to back up this whole folder.\n" + "Copying these files to another machine restores your settings there.\n" +) + + +def _user_data_dir() -> Path: + """Resolve the root of the per-file user-data store. Defaults to + %USERPROFILE%\\Documents\\Quantum Terminal\\. Overridable via MK_USER_DATA_DIR + env var (used by tests + power users).""" + override = os.environ.get("MK_USER_DATA_DIR", "").strip() + if override: + return Path(override) + # Path.home() returns USERPROFILE on Windows. "Documents" is the + # conventional name; if a user has redirected (OneDrive), Path.home() + # still resolves under the OneDrive-managed Documents in practice. + return Path.home() / "Documents" / "Quantum Terminal" + + +def _user_data_subdir(kind: str) -> Path: + """Resolve and create one subdir of the user-data store.""" + if kind not in _USER_DATA_SUBDIRS: + raise ValueError(f"unknown user-data subdir: {kind!r}") + d = _user_data_dir() / kind + d.mkdir(parents=True, exist_ok=True) + return d + + +def _ensure_user_data_root() -> Path: + """Create Documents\\Quantum Terminal\\ + all subdirs + README. Idempotent.""" + root = _user_data_dir() + root.mkdir(parents=True, exist_ok=True) + for sub in _USER_DATA_SUBDIRS: + (root / sub).mkdir(parents=True, exist_ok=True) + readme = root / "README.txt" + if not readme.exists(): + try: + readme.write_text(_README_TEXT, encoding="utf-8") + except OSError: + pass + return root + + +_FILENAME_INVALID_RE = re.compile(r"[^A-Za-z0-9_\- ]") + + +def _slugify_name(name: str) -> str: + """Make a filename-safe slug from a user-supplied name. Replaces + unsafe chars with `_`, collapses runs, trims, caps at 60 chars.""" + s = _FILENAME_INVALID_RE.sub("_", (name or "").strip()) + s = re.sub(r"_+", "_", s) + s = s.strip("_ ") or "untitled" + return s[:60] + + +def _name_hash8(name: str) -> str: + return hashlib.sha256((name or "").encode("utf-8")).hexdigest()[:8] + + +def _user_data_filename(name: str) -> str: + """`__.json` โ€” deterministic, collision-safe.""" + return f"{_slugify_name(name)}__{_name_hash8(name)}.json" + + +def _user_data_atomic_write(path: Path, payload: Dict[str, Any]) -> None: + """Atomic write via tempfile + os.replace.""" + path.parent.mkdir(parents=True, exist_ok=True) + fd, tmp_path = tempfile.mkstemp( + prefix=path.stem + ".", suffix=".tmp", dir=str(path.parent) + ) + try: + with os.fdopen(fd, "w", encoding="utf-8") as f: + json.dump(payload, f, indent=2, ensure_ascii=False, sort_keys=True) + os.replace(tmp_path, path) + except Exception: + if os.path.exists(tmp_path): + try: + os.unlink(tmp_path) + except OSError: + pass + raise + + +def _user_data_list(kind: str) -> Dict[str, Dict[str, Any]]: + """Walk a subdir, parse each .json file, return {name: payload}. + Each file's `name` field is the authoritative key โ€” filename slug is + cosmetic. Skips files that don't parse or don't carry a `name`.""" + out: Dict[str, Dict[str, Any]] = {} + d = _user_data_subdir(kind) + for p in sorted(d.glob("*.json")): + try: + with open(p, "r", encoding="utf-8") as f: + data = json.load(f) + name = data.get("name") + if not isinstance(name, str) or not name: + # Fallback: try to recover the name from filename slug. + # Shouldn't normally happen, but be lenient on read. + continue + payload = {k: v for k, v in data.items() if k != "name"} + out[name] = payload + except (OSError, json.JSONDecodeError): + log.warning(f"Skipping unreadable user-data file: {p}") + continue + return out + + +def _user_data_load(kind: str, name: str) -> Optional[Dict[str, Any]]: + """Load one item by exact name. Returns None if not present.""" + fn = _user_data_filename(name) + p = _user_data_subdir(kind) / fn + if not p.exists(): + # Fallback: scan in case filename was hand-edited. + all_items = _user_data_list(kind) + return all_items.get(name) + try: + with open(p, "r", encoding="utf-8") as f: + data = json.load(f) + if data.get("name") == name: + return {k: v for k, v in data.items() if k != "name"} + # Hash collision (extremely unlikely) โ€” fall back to scan + all_items = _user_data_list(kind) + return all_items.get(name) + except (OSError, json.JSONDecodeError): + return None + + +def _user_data_save(kind: str, name: str, payload: Dict[str, Any]) -> None: + """Write one item. Atomic. Overwrites if a file with the same + slug+hash already exists (same name โ†’ same filename).""" + if not name or not name.strip(): + raise ValueError("name cannot be empty") + # Function-arg `name` MUST win over any stray `name` key inside payload; + # spread payload first, then set name last. + body = {**payload, "name": name} + if "saved_at" not in body: + body["saved_at"] = datetime.now(timezone.utc).isoformat() + p = _user_data_subdir(kind) / _user_data_filename(name) + _user_data_atomic_write(p, body) + + +def _user_data_delete(kind: str, name: str) -> bool: + """Delete one item. Returns True if a file was removed.""" + fn = _user_data_filename(name) + p = _user_data_subdir(kind) / fn + if p.exists(): + try: + p.unlink() + return True + except OSError: + return False + # Fallback: scan filenames in case slug rules changed. + for cand in _user_data_subdir(kind).glob("*.json"): + try: + with open(cand, "r", encoding="utf-8") as f: + if json.load(f).get("name") == name: + cand.unlink() + return True + except (OSError, json.JSONDecodeError): + continue + return False + + +# โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ• +# v11: one-time migration from user_config.json โ†’ per-file store +# โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ• +# +# Reads up to three legacy sources (live AppData v2, orphaned QuantumTerminal-X, +# v1 QuantumTerminal โ€” read-only) and writes each non-empty profiles[]/ +# chart_presets[]/chart_templates[] entry to the new per-file store. +# Existing per-file entries are NEVER overwritten โ€” first-source-wins. +# Gated on a marker file in the user-data root so this fires exactly once. + +def _migration_marker_path() -> Path: + return _user_data_dir() / _MIGRATION_MARKER + + +def _legacy_user_config_paths() -> List[Path]: + """Return up to three paths to legacy user_config.json files, in + priority order: live AppData v2 first, then any sibling dirs.""" + appdata = os.environ.get("APPDATA") + paths: List[Path] = [] + if appdata: + base = Path(appdata) + # Live v2 first (most authoritative). + paths.append(base / APP_DIR / "user_config.json") + # QuantumTerminal-X โ€” orphaned attempt with a typo'd env var. + if APP_DIR != "QuantumTerminal-X": + paths.append(base / "QuantumTerminal-X" / "user_config.json") + # v1 โ€” read-only fallback (Rule 8: never write). + if APP_DIR != "QuantumTerminal": + paths.append(base / "QuantumTerminal" / "user_config.json") + return [p for p in paths if p.exists()] + + +def _migrate_user_data_once() -> None: + """Run the one-time migration. Idempotent โ€” gated on marker file.""" + _ensure_user_data_root() + marker = _migration_marker_path() + if marker.exists(): + return # already migrated + + sources = _legacy_user_config_paths() + migrated_counts = {"profiles": 0, "chart_templates": 0, "chart_presets": 0} + + for src in sources: + try: + with open(src, "r", encoding="utf-8") as f: + legacy = json.load(f) + except (OSError, json.JSONDecodeError) as e: + log.warning(f"Migration skipping unreadable {src}: {e}") + continue + + for kind in ("profiles", "chart_templates", "chart_presets"): + section = legacy.get(kind) or {} + if not isinstance(section, dict): + continue + existing = _user_data_list(kind) + for name, payload in section.items(): + if not isinstance(name, str) or not name: + continue + if name in existing: + continue # never overwrite existing per-file entries + if not isinstance(payload, dict): + continue + try: + _user_data_save(kind, name, payload) + migrated_counts[kind] += 1 + log.info(f"Migrated {kind}[{name!r}] from {src}") + except Exception as e: + log.warning(f"Migration failed to write {kind}[{name!r}]: {e}") + + # Strip the migrated keys from the LIVE AppData v2 user_config.json + # ONLY. Never touch QuantumTerminal-X or v1. + live = _appdata_user_config_path() + if live.exists(): + try: + with open(live, "r", encoding="utf-8") as f: + cfg = json.load(f) + stripped = False + for kind in ("profiles", "chart_templates", "chart_presets"): + if kind in cfg: + del cfg[kind] + stripped = True + if stripped: + _save_config(cfg) # module-level atomic writer (existing) + log.info(f"Stripped legacy save dicts from {live}") + except (OSError, json.JSONDecodeError) as e: + log.warning(f"Could not strip legacy keys from {live}: {e}") + + try: + marker.write_text("migrated 2026-05-13\n", encoding="utf-8") + log.info( + f"User-data migration complete: profiles={migrated_counts['profiles']}, " + f"chart_templates={migrated_counts['chart_templates']}, " + f"chart_presets={migrated_counts['chart_presets']}" + ) + except OSError as e: + log.warning(f"Could not write migration marker: {e}") + + +# โ”€โ”€ Import static defaults โ”€โ”€ +from server_config import ( + ASSET_UNIVERSE, ASSET_DECIMALS, ASSET_CLASSES, SYMBOL_ALIASES, + MT5_TIMEFRAMES, ServerConfig, +) + + +def _read_rithmic_config() -> Optional[Dict[str, str]]: + """ + Read Rithmic credentials from local_config.ini [rithmic] section. + Returns dict with user, password, system_name, url โ€” or None if missing. + Machine-specific credentials stay in local_config.ini (Rule 3, gitignored). + """ + import configparser + ini_path = PROJECT_ROOT / "local_config.ini" + if not ini_path.exists(): + return None + try: + parser = configparser.ConfigParser() + parser.read(str(ini_path)) + if "rithmic" not in parser: + return None + section = parser["rithmic"] + cfg = {} + for key in ("user", "password", "system_name", "url"): + val = section.get(key, "").strip() + if val and not val.startswith("YOUR_"): + cfg[key] = val + # Only return if we have at least user + password + url + if cfg.get("user") and cfg.get("password") and cfg.get("url"): + return cfg + return None + except Exception as e: + log.warning(f"Failed to read [rithmic] from local_config.ini: {e}") + return None + + +# โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ• +# Default user config schema +# โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ• + +DEFAULT_USER_CONFIG = { + "version": 1, + + # โ”€โ”€ Universe โ”€โ”€ + "universe": { + # Active tickers โ€” derived from ASSET_UNIVERSE on first run + # User can add/remove/reorder via settings panel + "active_tickers": list(ASSET_UNIVERSE), + + # Custom tickers added by user (not in default universe) + # Key = canonical name, value = metadata overrides + "custom_tickers": {}, + + # Tickers to hide from display (still tracked, just not shown) + "hidden_tickers": [], + + # Display order โ€” if empty, uses active_tickers order + "display_order": [], + }, + + # โ”€โ”€ Providers โ”€โ”€ + "providers": { + # Which provider instance handles price feed + "active_data": "mt5_default", + + # Which provider instance handles execution + "active_execution": "mt5_default", + + # Configured accounts / data sources + "accounts": { + "mt5_default": { + "id": "mt5_default", + "type": "mt5", + "label": "MetaTrader 5", + "enabled": True, + "terminal_path": None, # None = auto-detect, or read from local_config.ini + "aliases": {}, # Per-account alias overrides (merged with defaults) + # When True, the tick loop polls MT5 every 0.2s instead of 1.0s + # so the chart's last candle flickers at MT5-style rates. + # Default OFF โ€” opt-in for users with capable hardware. + "use_tick_data": False, + }, + "rithmic_default": { + "id": "rithmic_default", + "type": "rithmic", + "label": "Rithmic โ€” Paper Trading", + "enabled": False, # User enables after entering credentials + "user": "", + "password": "", + "system_name": "Rithmic Paper Trading", + "url": "", + "app_name": "Quantum Terminal", + "app_version": "1.0", + }, + }, + }, + + # โ”€โ”€ Display โ”€โ”€ + "display": { + "theme": "dark", + "default_timeframe": "M15", + "default_lookback_bars": 200, + "chart_height": 420, + "visible_panels": { + "regime_bar": True, + "signal_panel": True, + "quant_panel": True, + "asset_sidebar": True, + }, + "chart_layers": { + "cones_gbm": True, + "cones_mjd": True, + "bands": True, + "signals": True, + "anchors": True, + "volume": True, + }, + }, + + # โ”€โ”€ Execution โ”€โ”€ + # ALL OFF by default โ€” user must explicitly enable live trading. + "execution": { + "live_trading": False, # Master kill switch โ€” no orders sent unless True + "show_positions": False, # Show current open positions from broker + "show_trade_history": False, # Show closed trades on chart / panel + "confirm_orders": True, # Require confirmation dialog before sending + "default_risk_pct": 1.0, # Default risk per trade (% of equity) + "max_lots": 10.0, # Hard cap on lot size + "magic_number": 202603, # MT5 magic number for Quantum Terminal orders + }, + + # โ”€โ”€ Modules โ”€โ”€ + "modules": { + "enabled": [ + "cones", "bands", "signals", "anchors", "regime", + "garch", "ou", "wf", "kelly", + ], + }, + + # โ”€โ”€ Saved profiles โ”€โ”€ + "profiles": { + # "profile_name": { full config snapshot minus profiles } + }, + + # โ”€โ”€ Chart presets (named chart-style fragments, managed via /api/chart-presets) โ”€โ”€ + "chart_presets": { + # "preset_name": { chart style fragment โ€” see chartStyleDefaults.js for shape } + }, + + # โ”€โ”€ Chart templates (named full-chart-config snapshots, managed via /api/chart-templates) โ”€โ”€ + "chart_templates": { + # "name": { "sourceTicker": str, "config": {...full chart config...}, "savedAt": ISO str } + }, + + # โ”€โ”€ Chart state (managed by MainChart, saved/restored with profiles) โ”€โ”€ + "_chart_state": {}, + + # โ”€โ”€ Last used profile name (auto-loaded on startup) โ”€โ”€ + "_last_profile": None, +} + + +# โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ• +# Asset metadata builder +# โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ• + +# v4: pattern-based asset-class inference for dynamically-discovered tickers +# that don't have a server_config.ASSET_CLASSES entry. Used by the settings +# UI to group new assets correctly (FX / INDEX / COMMODITY / CRYPTO / +# STOCK) instead of dumping them all under OTHER. +_CRYPTO_PREFIXES = ("BTC", "ETH", "SOL", "XRP", "ADA", "LTC", "DOGE", + "DOT", "AVAX", "LINK", "MATIC", "BNB", "TRX", "SHIB") +_COMMODITY_PREFIXES = ("XAU", "XAG", "XPT", "XPD", "XTI") +_COMMODITY_NAMES = frozenset({"BRENT", "WTI", "NGAS", "HG", "CL"}) +_INDEX_NAMES = frozenset({ + "US500", "USTEC", "US30", "GER40", "UK100", "FR40", "AUS200", + "JP225", "HK50", "SPX", "NDX", "DJI", "DAX", "FTSE", "NI225", + "ESTX50", "EU50", +}) +_CURRENCIES = frozenset({ + "USD", "EUR", "GBP", "JPY", "CHF", "CAD", "AUD", "NZD", + "SEK", "NOK", "DKK", "CNH", "MXN", "TRY", "ZAR", "SGD", "HKD", +}) + + +def _infer_asset_class(ticker: str) -> str: + t = (ticker or "").upper() + if not t: + return "OTHER" + # Crypto โ€” exact prefix or known ticker + if any(t.startswith(p) for p in _CRYPTO_PREFIXES): + return "CRYPTO" + # Commodity โ€” metals/energy + if any(t.startswith(p) for p in _COMMODITY_PREFIXES) or t in _COMMODITY_NAMES: + return "COMMODITY" + # Index โ€” explicit list + if t in _INDEX_NAMES: + return "INDEX" + # FX โ€” 6-letter pair of known currencies + if len(t) == 6 and t[:3] in _CURRENCIES and t[3:] in _CURRENCIES: + return "FX" + # Stock โ€” 1-5 letters, pure alpha (AAPL, TSLA, MSFT, NVDA, โ€ฆ) + if 1 <= len(t) <= 5 and t.isalpha(): + return "STOCK" + return "OTHER" + + +def _build_asset_metadata() -> Dict[str, dict]: + """ + Build asset metadata dict from server_config static data. + This is the server-side source of truth โ€” replaces the hardcoded + ASSET_META dict that was duplicated in the frontend. + + Returns: + { + "XAUUSD": { + "class": "COMMODITY", + "decimals": 2, + "aliases": ["XAUUSD", "GOLD", "XAUUSD.cash"], + }, + ... + } + """ + meta = {} + for ticker in ASSET_UNIVERSE: + meta[ticker] = { + "class": ASSET_CLASSES.get(ticker, "OTHER"), + "decimals": ASSET_DECIMALS.get(ticker, 2), + "aliases": SYMBOL_ALIASES.get(ticker, [ticker]), + } + return meta + + +# โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ• +# ConfigManager +# โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ• + +class ConfigManager: + """ + Central configuration manager. + + Lifecycle: + 1. Load server_config.py defaults (static, versioned) + 2. Load user_config.json if it exists (user overrides, gitignored) + 3. Merge: user config wins where specified + 4. Expose merged config via get_config() + 5. Accept mutations via update() โ†’ writes back to user_config.json + + Provider management: + - Creates provider instances from accounts config + - Tracks which provider is active for data vs execution + - Provides universe to active provider for symbol resolution + """ + + def __init__(self, config_path: Optional[Path] = None): + # v6: default storage moved from PROJECT_ROOT (install dir, wiped + # on update) to AppData (survives reinstalls). Tests can still + # override via the config_path argument. + self._config_path = config_path or _appdata_user_config_path() + self._user_config: Dict[str, Any] = {} + self._asset_metadata: Dict[str, dict] = _build_asset_metadata() + self._providers: Dict[str, Any] = {} # id โ†’ BaseProvider instance + + # v6: one-time migration โ€” if AppData has no file yet but the legacy + # install-dir file exists, copy it over so existing users keep + # their profiles. Idempotent: only fires when AppData is empty. + try: + legacy_path = PROJECT_ROOT / "user_config.json" + if ( + config_path is None + and not self._config_path.exists() + and legacy_path.exists() + and legacy_path.resolve() != self._config_path.resolve() + ): + self._config_path.parent.mkdir(parents=True, exist_ok=True) + self._config_path.write_text( + legacy_path.read_text(encoding="utf-8"), encoding="utf-8" + ) + log.info( + f"Migrated legacy user_config from {legacy_path} โ†’ {self._config_path}" + ) + except Exception as e: + log.warning(f"Legacy user_config migration failed (non-fatal): {e}") + + # Load persisted user config + self._load_user_config() + + # v11: one-time per-file user-data migration. Idempotent (gated on + # a marker file in Documents\Quantum Terminal\). Safe to call always. + try: + _migrate_user_data_once() + # v11: in-memory mirror of the disk strip. _user_config was + # loaded BEFORE migration and still carries any legacy + # profiles/chart_templates/chart_presets dicts. If we leave + # them in the cache, the next workspace auto-save would + # write them back to disk and silently undo the strip. + # Idempotent โ€” pop is a no-op if the key isn't there. + for _k in ("profiles", "chart_templates", "chart_presets"): + self._user_config.pop(_k, None) + except Exception as e: + log.warning(f"User-data migration error (non-fatal): {e}") + + # โ”€โ”€ Persistence โ”€โ”€ + + def _load_user_config(self) -> None: + """Load user_config.json. On parse/IO failure, rename the corrupt + file to user_config.json.corrupt. so a transient + error can't trigger the workspace-save wipe sequence, then start + with an empty config. Per-file user-data store (profiles / + chart_templates / chart_presets) is unaffected โ€” those live in + Documents\\Quantum Terminal\\ and are not read here.""" + path = self._config_path + if not path.exists(): + log.info("No user_config.json found โ€” using defaults (will create on first save)") + self._user_config = {} + return + try: + with open(path, "r", encoding="utf-8") as f: + self._user_config = json.load(f) + log.info(f"Loaded user config from {path.name}") + return + except (json.JSONDecodeError, IOError, OSError) as e: + ts = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ") + backup = path.with_suffix(path.suffix + f".corrupt.{ts}") + try: + os.replace(str(path), str(backup)) + log.error( + f"user_config.json unreadable ({e}); preserved at {backup.name}. " + f"Starting with empty config โ€” saved profiles / templates / presets " + f"in Documents\\Quantum Terminal\\ are unaffected." + ) + except OSError as e2: + log.error(f"user_config.json unreadable and could not be backed up: {e2}") + self._user_config = {} + + def _save_user_config(self) -> None: + """Write current user config to disk. + + v9: Before writing, preserve `chart_presets` from on-disk state. + The chart-presets module-level layer (see _load_config / _save_config + below) writes presets directly to disk, bypassing this instance's + in-memory cache. Without this preservation, the next instance write + would silently clobber any preset added since the last instance load. + """ + try: + preserved_chart_presets = None + preserved_chart_templates = None + if self._config_path.exists(): + try: + with open(self._config_path, "r", encoding="utf-8") as f: + on_disk = json.load(f) + preserved_chart_presets = on_disk.get("chart_presets") + preserved_chart_templates = on_disk.get("chart_templates") + except (OSError, json.JSONDecodeError): + preserved_chart_presets = None + preserved_chart_templates = None + if preserved_chart_presets is not None: + self._user_config["chart_presets"] = preserved_chart_presets + if preserved_chart_templates is not None: + self._user_config["chart_templates"] = preserved_chart_templates + with open(self._config_path, "w", encoding="utf-8") as f: + json.dump(self._user_config, f, indent=2, ensure_ascii=False) + log.info(f"Saved user config to {self._config_path.name}") + except IOError as e: + log.error(f"Failed to save user config: {e}") + + # โ”€โ”€ Config Access โ”€โ”€ + + def get_config(self) -> Dict[str, Any]: + """ + Return the full merged config dict. + This is what GET /api/config serves to the frontend. + + Shape: + { + "version": 1, + "universe": { ... }, + "providers": { ... }, + "display": { ... }, + "modules": { ... }, + "profiles": { ... }, + "asset_metadata": { ticker: {class, decimals, aliases} }, + "available_timeframes": ["M1", "M5", ...], + "available_provider_types": ["mt5", ...], + "server": { host, port, uptime_seconds }, + } + """ + merged = self._deep_merge( + copy.deepcopy(DEFAULT_USER_CONFIG), + copy.deepcopy(self._user_config), + ) + + # v11: overlay user-owned saves from the per-file store. These keys + # are no longer in self._user_config (post-migration); the new + # store is authoritative. + try: + merged["profiles"] = _user_data_list("profiles") + merged["chart_templates"] = _user_data_list("chart_templates") + merged["chart_presets"] = _user_data_list("chart_presets") + except Exception as e: + log.warning(f"Failed to load user-data store entries: {e}") + # leave whatever was already in merged (could be {} from defaults) + + # Inject computed / read-only fields + merged["asset_metadata"] = self._get_full_asset_metadata(merged) + + # v3: the Settings Universe UI reads `universe.active_tickers` straight + # out of this dict, so overwrite it with the dynamic union of + # configured + cache-discovered tickers. Raw user_config on disk + # stays untouched โ€” the override is applied at read time only. + try: + dyn = self.get_active_universe() + if dyn: + merged.setdefault("universe", {})["active_tickers"] = dyn + except Exception: + log.warning("universe auto-discovery failed; falling back to configured list", exc_info=True) + + # v5: surface per-ticker data-type availability so the Settings UI + # can paint the dot color (green = full, purple = quarterly-only, + # grey = nothing). Empty dict if cache discovery failed. + try: + merged["quant_data_types"] = self._discover_tickers_data_types() + except Exception: + merged["quant_data_types"] = {} + # Tickers that actually have synced quant data in the consumer's local cache. + # Source of truth = data_sync_client.synced_tickers (what the VPS has shipped), + # NOT the hardcoded ASSET_UNIVERSE constant (which can lag behind producer changes). + # Falls back to ASSET_UNIVERSE only if the sync client is unavailable (PRO mode + # or pre-first-sync). Custom user tickers are never in this list. + try: + from data_sync_client import get_sync_client + synced = get_sync_client().synced_tickers + merged["quant_supported_tickers"] = list(synced) if synced else list(ASSET_UNIVERSE) + except Exception: + merged["quant_supported_tickers"] = list(ASSET_UNIVERSE) + merged["available_timeframes"] = list(MT5_TIMEFRAMES.keys()) + + from providers import list_provider_types + merged["available_provider_types"] = list_provider_types() + + # Provider status + merged["provider_status"] = {} + for pid, prov in self._providers.items(): + merged["provider_status"][pid] = { + "connected": prov.connected, + "type": prov.provider_type, + "label": prov.label, + "can_execute": prov.can_execute, + } + + return merged + + # v3: discover tickers from the sync cache so new DC-shipped assets + # auto-appear in the universe without user intervention. + # Matches only per-ticker filenames ending in one of the known + # data-type suffixes โ€” this prevents GLOBAL-scope files like + # `GLOBAL_macro_sector_flows.json` from being mistaken for a + # "MACRO" ticker. Add suffixes here when DC ships new per-ticker + # data types. + _PER_TICKER_SUFFIXES = ( + "bands", "cones", "historical_cones", "quarterly_cones", + "scalp_bands", "options", "probfield", "bands_replay", + ) + + # v4: aggregate / composite files produced by DC that match the per-ticker + # filename pattern but aren't actually tickers (their content is a + # cross-asset rollup). Ask DC to rename these to GLOBAL_* prefixes + # so we don't need to maintain this list. + _NON_TICKER_NAMES = frozenset({"SCORE", "COMPOSITE", "AGGREGATE"}) + + def _discover_tickers_data_types(self) -> Dict[str, List[str]]: + """v5: return {TICKER: [data_types]} from cache scan. Single source + of truth for (a) get_active_universe discovery and (b) per-ticker + quant-data-availability diagnostics surfaced in /api/config. + """ + try: + from data_sync_client import _get_cache_dir + cache_dir = _get_cache_dir() + except Exception: + return {} + if not cache_dir or not Path(cache_dir).is_dir(): + return {} + # v4: longest-first so `_quarterly_cones` beats `_cones` on + # `GLOBAL_aapl_quarterly_cones.json` and AAPL survives. + ordered = sorted(self._PER_TICKER_SUFFIXES, key=len, reverse=True) + suffixes = tuple((f"_{s}", s) for s in ordered) + result: Dict[str, set] = {} + try: + for f in Path(cache_dir).iterdir(): + if not f.is_file() or f.suffix.lower() != ".json": + continue + name = f.stem + if name.startswith("_"): + continue + match_sfx = match_type = None + for sfx_full, sfx_bare in suffixes: + if name.endswith(sfx_full): + match_sfx, match_type = sfx_full, sfx_bare + break + if not match_sfx: + continue + prefix = name[:-len(match_sfx)] + if prefix.startswith("GLOBAL_"): + prefix = prefix[len("GLOBAL_"):] + prefix = prefix.strip("_") + if prefix and prefix.replace("-", "").isalnum(): + up = prefix.upper() + if up in self._NON_TICKER_NAMES: + continue + result.setdefault(up, set()).add(match_type) + except Exception: + log.warning("cache discovery failed", exc_info=True) + return {} + return {t: sorted(types) for t, types in sorted(result.items())} + + def _discover_tickers_from_cache(self) -> List[str]: + # Kept for back-compat โ€” thin wrapper over the richer discovery. + return list(self._discover_tickers_data_types().keys()) + + def get_active_universe(self) -> List[str]: + """ + Return the current active ticker list. + v3: union of (a) tickers the user explicitly added to their universe + and (b) tickers discovered in the local sync cache. Discovery alone + is usually enough, but the union keeps user-pinned tickers visible + even when their data hasn't arrived yet. + """ + merged = self._deep_merge( + copy.deepcopy(DEFAULT_USER_CONFIG.get("universe", {})), + copy.deepcopy(self._user_config.get("universe", {})), + ) + configured = merged.get("active_tickers", list(ASSET_UNIVERSE)) + discovered = self._discover_tickers_from_cache() + if not discovered: + return list(configured) + # Union preserving order: configured first, then new discoveries. + seen = set() + out = [] + for t in list(configured) + discovered: + tu = t.upper() + if tu not in seen: + seen.add(tu) + out.append(tu) + return out + + def get_display_order(self) -> List[str]: + """ + Return tickers in display order, excluding hidden ones. + Falls back to active_tickers order if no custom order set. + v5: pulls the active list from get_active_universe() so dynamically + discovered tickers (e.g. AAPL) flow through to /api/health and the + chart's + menu. Previously it read the raw disk config, which + bypassed discovery. + """ + merged_uni = self._deep_merge( + copy.deepcopy(DEFAULT_USER_CONFIG.get("universe", {})), + copy.deepcopy(self._user_config.get("universe", {})), + ) + active = self.get_active_universe() + hidden = set(merged_uni.get("hidden_tickers", [])) + order = merged_uni.get("display_order", []) + + if order: + # Use custom order, but only include active non-hidden tickers + active_set = set(active) - hidden + ordered = [t for t in order if t in active_set] + # Append any active tickers not in the custom order + for t in active: + if t not in hidden and t not in ordered: + ordered.append(t) + return ordered + else: + return [t for t in active if t not in hidden] + + def get_asset_meta(self, ticker: str) -> dict: + """ + Get metadata for a single ticker. + Merges static defaults with any user custom_ticker overrides. + """ + base = self._asset_metadata.get(ticker, { + "class": "OTHER", + "decimals": 2, + "aliases": [ticker], + }) + custom = ( + self._user_config + .get("universe", {}) + .get("custom_tickers", {}) + .get(ticker, {}) + ) + return {**base, **custom} + + def _get_full_asset_metadata(self, merged_config: dict) -> Dict[str, dict]: + """ + Build complete asset metadata for all active + custom tickers. + Merges static server_config data with user custom_ticker overrides. + """ + result = {} + active = merged_config.get("universe", {}).get("active_tickers", []) + custom = merged_config.get("universe", {}).get("custom_tickers", {}) + + for ticker in active: + # v4: if the ticker has no static metadata, infer its class from + # the naming pattern (AAPL โ†’ STOCK, BTCUSD โ†’ CRYPTO, XAUUSD โ†’ + # COMMODITY, โ€ฆ) so the Settings UI can group it properly. + base = self._asset_metadata.get(ticker, { + "class": _infer_asset_class(ticker), + "decimals": 2, + "aliases": [ticker], + }) + override = custom.get(ticker, {}) + result[ticker] = {**base, **override} + + # Add custom tickers not in default universe + for ticker, meta in custom.items(): + if ticker not in result: + result[ticker] = { + "class": meta.get("class", "OTHER"), + "decimals": meta.get("decimals", 2), + "aliases": meta.get("aliases", [ticker]), + **meta, + } + + return result + + # โ”€โ”€ Config Mutation โ”€โ”€ + + def get_broker_symbol(self, ticker: str) -> Optional[str]: + """v2: Return user-configured broker symbol override for a ticker. + Looks up universe.custom_tickers[TICKER].broker_symbol; returns a + non-empty trimmed string, or None if unset. + """ + if not ticker: + return None + cfg = self.get_config() + custom = cfg.get("universe", {}).get("custom_tickers", {}) or {} + entry = custom.get(ticker.upper()) or {} + val = entry.get("broker_symbol") + if isinstance(val, str): + s = val.strip() + if s: + return s + return None + + def update(self, patch: Dict[str, Any]) -> Dict[str, Any]: + """ + Apply a partial update to user config. + Deep-merges the patch into existing user config, saves to disk. + + Args: + patch: Partial config dict (same shape as user_config.json) + Only specified fields are updated; others preserved. + + Returns: + Updated full config (same as get_config()) + """ + self._user_config = self._deep_merge(self._user_config, patch) + self._save_user_config() + log.info(f"Config updated: {list(patch.keys())}") + return self.get_config() + + # โ”€โ”€ Universe Mutation Helpers โ”€โ”€ + + def add_ticker(self, ticker: str, metadata: Optional[dict] = None) -> bool: + """ + Add a ticker to the active universe. + If it's not in the default universe, it goes into custom_tickers. + """ + uni = self._user_config.setdefault("universe", {}) + active = uni.setdefault("active_tickers", list(ASSET_UNIVERSE)) + + if ticker in active: + log.info(f"Ticker {ticker} already in universe") + return False + + active.append(ticker) + + # If not a default ticker, store metadata + if ticker not in ASSET_UNIVERSE: + custom = uni.setdefault("custom_tickers", {}) + custom[ticker] = metadata or { + "class": "OTHER", + "decimals": 2, + "aliases": [ticker], + } + + self._save_user_config() + log.info(f"Added ticker: {ticker}") + return True + + def remove_ticker(self, ticker: str) -> bool: + """Remove a ticker from the active universe.""" + uni = self._user_config.get("universe", {}) + active = uni.get("active_tickers", []) + + if ticker not in active: + return False + + active.remove(ticker) + + # Also remove from custom_tickers and display_order + custom = uni.get("custom_tickers", {}) + custom.pop(ticker, None) + order = uni.get("display_order", []) + if ticker in order: + order.remove(ticker) + hidden = uni.get("hidden_tickers", []) + if ticker in hidden: + hidden.remove(ticker) + + self._save_user_config() + log.info(f"Removed ticker: {ticker}") + return True + + def toggle_ticker_visibility(self, ticker: str) -> bool: + """Toggle a ticker between visible and hidden.""" + uni = self._user_config.setdefault("universe", {}) + hidden = uni.setdefault("hidden_tickers", []) + + if ticker in hidden: + hidden.remove(ticker) + visible = True + else: + hidden.append(ticker) + visible = False + + self._save_user_config() + log.info(f"Ticker {ticker} visibility: {'visible' if visible else 'hidden'}") + return visible + + def reorder_tickers(self, order: List[str]) -> None: + """Set custom display order for tickers.""" + uni = self._user_config.setdefault("universe", {}) + uni["display_order"] = order + self._save_user_config() + log.info(f"Display order updated: {len(order)} tickers") + + # โ”€โ”€ Provider Management โ”€โ”€ + + def init_providers(self) -> Dict[str, Any]: + """ + Instantiate and connect all enabled providers from config. + Called once at server startup. + + Returns: + Dict of provider_id โ†’ provider instance + """ + from providers import create_provider + + merged = self._deep_merge( + copy.deepcopy(DEFAULT_USER_CONFIG.get("providers", {})), + copy.deepcopy(self._user_config.get("providers", {})), + ) + + accounts = merged.get("accounts", {}) + self._providers.clear() + + for pid, acfg in accounts.items(): + if not acfg.get("enabled", True): + log.info(f"Provider {pid} disabled โ€” skipping") + continue + + try: + # Merge default aliases with account-specific aliases + acfg_with_aliases = {**acfg} + merged_aliases = {**SYMBOL_ALIASES} + merged_aliases.update(acfg.get("aliases", {})) + acfg_with_aliases["aliases"] = merged_aliases + + # Read terminal_path from local_config.ini if not set + if acfg.get("type") == "mt5" and not acfg.get("terminal_path"): + from server_config import MT5_TERMINAL_PATH + acfg_with_aliases["terminal_path"] = MT5_TERMINAL_PATH + + # Read Rithmic credentials from local_config.ini if not set + if acfg.get("type") == "rithmic" and not acfg.get("user"): + rithmic_cfg = _read_rithmic_config() + if rithmic_cfg: + acfg_with_aliases.update(rithmic_cfg) + log.info(f"Rithmic credentials loaded from local_config.ini") + + provider = create_provider(acfg_with_aliases) + self._providers[pid] = provider + # v2: give provider a live lookup for user-configured broker + # symbol overrides (Settings panel โ†’ custom_tickers[X].broker_symbol). + if hasattr(provider, "set_broker_symbol_lookup"): + try: + provider.set_broker_symbol_lookup(self.get_broker_symbol) + except Exception as e: + log.warning(f"Provider {pid}: broker_symbol lookup wiring failed: {e}") + log.info(f"Provider created: {pid} ({provider.provider_type})") + + except KeyError as e: + log.error(f"Failed to create provider {pid}: {e}") + + return self._providers + + def get_provider(self, provider_id: Optional[str] = None) -> Optional[Any]: + """ + Get a provider by ID. + If no ID given, returns the active data provider. + """ + if provider_id: + return self._providers.get(provider_id) + + # Default: active data provider + merged = self._deep_merge( + copy.deepcopy(DEFAULT_USER_CONFIG.get("providers", {})), + copy.deepcopy(self._user_config.get("providers", {})), + ) + active_id = merged.get("active_data", "mt5_default") + return self._providers.get(active_id) + + def get_execution_provider(self) -> Optional[Any]: + """Get the active execution provider.""" + merged = self._deep_merge( + copy.deepcopy(DEFAULT_USER_CONFIG.get("providers", {})), + copy.deepcopy(self._user_config.get("providers", {})), + ) + active_id = merged.get("active_execution", "mt5_default") + return self._providers.get(active_id) + + def get_all_providers(self) -> Dict[str, Any]: + """Return all instantiated providers.""" + return dict(self._providers) + + # โ”€โ”€ Profile Management โ”€โ”€ + + def save_profile(self, name: str) -> None: + """v11: save profile as one file under Documents\\Quantum Terminal\\profiles\\.""" + # Snapshot everything except profiles themselves AND the user-owned + # saves (which now live in their own per-file stores). + snapshot = { + k: copy.deepcopy(v) for k, v in self._user_config.items() + if k not in ("profiles", "chart_presets", "chart_templates") + } + _user_data_save("profiles", name, snapshot) + log.info(f"Saved profile: {name}") + + def load_profile(self, name: str) -> bool: + """v11: load profile from per-file store.""" + payload = _user_data_load("profiles", name) + if payload is None: + log.warning(f"Profile '{name}' not found") + return False + snapshot = dict(payload) + snapshot.pop("saved_at", None) + # Replace runtime config (workspace / universe / providers / display) + # but DO NOT touch the saves stores โ€” they live in files now. + self._user_config = copy.deepcopy(snapshot) + self._save_user_config() + log.info(f"Loaded profile: {name}") + return True + + def delete_profile(self, name: str) -> bool: + """v11: delete profile from per-file store.""" + ok = _user_data_delete("profiles", name) + if ok: + log.info(f"Deleted profile: {name}") + return ok + + def list_profiles(self) -> List[dict]: + """v11: list profiles from per-file store.""" + items = _user_data_list("profiles") + result = [] + for name, data in items.items(): + result.append({ + "name": name, + "saved_at": data.get("saved_at", "unknown"), + "ticker_count": len( + (data.get("universe") or {}).get("active_tickers", []) + ), + }) + return result + + # โ”€โ”€ Utilities โ”€โ”€ + + @staticmethod + def _deep_merge(base: dict, override: dict) -> dict: + """ + Recursively merge override into base. + Lists in override REPLACE base lists (not append). + Dicts are merged recursively. + Scalar values in override win. + """ + result = copy.deepcopy(base) + for key, value in override.items(): + if ( + key in result + and isinstance(result[key], dict) + and isinstance(value, dict) + ): + result[key] = ConfigManager._deep_merge(result[key], value) + else: + result[key] = copy.deepcopy(value) + return result + + +# โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ• +# Module-level chart_presets API +# โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ• +# +# v7: chart presets are a small, self-contained slice of user_config.json +# that the chart-settings UI reads/writes via /api/chart-presets. A thin +# module-level layer (path resolver + load/save helpers) gives the routes +# module a clean import-time API. +# +# v8: the v7 implementation kept its own `_cached_config` dict next to the +# ConfigManager instance's `self._user_config` โ€” both layers ultimately +# wrote the same file. That was a lost-write race: a chart-preset write +# refreshed the module cache, but the ConfigManager instance still held a +# stale snapshot, and the next instance-side save would clobber the disk +# copy and silently delete the just-saved preset. v8 removes the cache +# entirely. Every read re-loads from disk; every write atomically replaces +# the file via tempfile + os.replace. Slower (one disk read per preset +# query) but correct under multi-writer use. +# +# Tests patch `_user_config_path()` to a temp file โ€” see +# tests/test_config_manager_chart_presets.py. + + +def _user_config_path() -> Path: + """Return the absolute path to user_config.json. Wraps + `_appdata_user_config_path()` so tests can monkey-patch this single + name without touching the AppData resolver used by ConfigManager.""" + return _appdata_user_config_path() + + +def _load_config() -> Dict[str, Any]: + """Always read fresh from disk. No cache โ€” see v8 header note for + rationale (dual-layer cache hazard).""" + path = _user_config_path() + if not path.exists(): + return {"profiles": {}, "chart_presets": {}, "chart_templates": {}, "_last_profile": None} + try: + with open(path, "r", encoding="utf-8") as f: + data = json.load(f) + except (OSError, json.JSONDecodeError): + return {"profiles": {}, "chart_presets": {}, "chart_templates": {}, "_last_profile": None} + if "chart_presets" not in data: + data["chart_presets"] = {} + if "chart_templates" not in data: + data["chart_templates"] = {} + return data + + +def _save_config(cfg: Dict[str, Any]) -> None: + """Atomically write user config to disk via tempfile + os.replace, so + a crash mid-write can never leave a half-written user_config.json on + disk. No cache to refresh โ€” see v8 header note.""" + path = _user_config_path() + path.parent.mkdir(parents=True, exist_ok=True) + fd, tmp_path = tempfile.mkstemp( + prefix=path.stem + ".", suffix=".tmp", dir=str(path.parent) + ) + try: + with os.fdopen(fd, "w", encoding="utf-8") as f: + json.dump(cfg, f, indent=2, sort_keys=True) + os.replace(tmp_path, path) + except Exception: + if os.path.exists(tmp_path): + try: + os.unlink(tmp_path) + except OSError: + pass + raise + + +def get_chart_presets() -> Dict[str, Any]: + """Return the dict of named chart-style presets. {} if none.""" + return _user_data_list("chart_presets") + + +def save_chart_preset(name: str, style: Dict[str, Any]) -> None: + """Create or replace a preset under `name`.""" + _user_data_save("chart_presets", name, dict(style)) + + +def delete_chart_preset(name: str) -> None: + """Remove a preset. No-op if it doesn't exist.""" + _user_data_delete("chart_presets", name) + + +def get_chart_templates() -> Dict[str, Any]: + """Return the dict of named chart templates. {} if none.""" + return _user_data_list("chart_templates") + + +def save_chart_template(name: str, template: Dict[str, Any]) -> None: + """Create or replace a chart template under `name`. `template` is the + full template object: {sourceTicker, config, savedAt}.""" + _user_data_save("chart_templates", name, dict(template)) + + +def delete_chart_template(name: str) -> None: + """Remove a chart template. No-op if it doesn't exist.""" + _user_data_delete("chart_templates", name) \ No newline at end of file diff --git a/backend/config_routes.py b/backend/config_routes.py new file mode 100644 index 0000000..c2270b8 --- /dev/null +++ b/backend/config_routes.py @@ -0,0 +1,203 @@ +# version: v2 +""" +config_routes.py โ€” Terminal Configuration API Routes +===================================================== + +v2 โ€” PATCH handler now detects broker_symbol changes in + universe.custom_tickers[*].broker_symbol and, for each affected ticker, + calls provider.reset_symbol() on every live provider so MT5 picks up + the new mapping at the next tick/bar fetch (no restart required). + Also triggers a data_sync_complete broadcast so the frontend refetches + bars for the affected tickers. +Serves GET /api/config and all mutation endpoints used by useConfig.js. + +Endpoints: + GET /api/config โ†’ full merged config + PATCH /api/config โ†’ partial update + POST /api/config/tickers โ†’ add ticker + DELETE /api/config/tickers/{ticker} โ†’ remove ticker + POST /api/config/tickers/{ticker}/toggle โ†’ hide/show + PUT /api/config/tickers/order โ†’ reorder + POST /api/config/profiles โ†’ save profile + POST /api/config/profiles/{name}/load โ†’ load profile + DELETE /api/config/profiles/{name} โ†’ delete profile + +This module does NOT introduce any calculation triggers. +""" + +import logging +from typing import Callable, Optional, Set +from fastapi import APIRouter, Request, HTTPException + +log = logging.getLogger("config_routes") + + +# v2 โ€” helpers for broker_symbol hot-rewiring. +def _collect_broker_symbol_changes(patch: dict) -> Set[str]: + """Walk a PATCH body and return the set of tickers whose broker_symbol + is being set (value may be null to clear). Safe for any patch shape.""" + out = set() + try: + custom = (patch.get("universe", {}) or {}).get("custom_tickers", {}) or {} + for ticker, fields in custom.items(): + if isinstance(fields, dict) and "broker_symbol" in fields: + out.add(str(ticker).upper()) + except Exception as e: + log.warning(f"Failed to scan patch for broker_symbol changes: {e}") + return out + + +def _apply_broker_symbol_resets(cfg_manager, tickers: Set[str]) -> None: + """For each live provider that supports it, evict and re-resolve each + ticker so the new broker_symbol override takes effect at the next fetch.""" + try: + providers = cfg_manager.get_all_providers() + except Exception as e: + log.warning(f"get_all_providers failed: {e}") + return + for pid, prov in (providers or {}).items(): + if not hasattr(prov, "reset_symbol"): + continue + for t in tickers: + try: + resolved = prov.reset_symbol(t) + log.info(f"[{pid}] reset_symbol({t}) โ†’ {resolved}") + except Exception as e: + log.warning(f"[{pid}] reset_symbol({t}) failed: {e}") + + +def create_config_router(cfg_manager, broadcast_event: Optional[Callable] = None) -> APIRouter: + """ + Create config API router wired to the given ConfigManager instance. + broadcast_event: async callable to push WS events on config change. + """ + router = APIRouter(prefix="/api/config", tags=["config"]) + + async def _notify(): + """Push config_update event to all WS clients.""" + if broadcast_event: + try: + await broadcast_event({"type": "config_update"}) + except Exception: + pass # Non-critical + + # โ”€โ”€ GET /api/config โ€” full merged config โ”€โ”€ + @router.get("") + async def get_config(): + return cfg_manager.get_config() + + # โ”€โ”€ PATCH /api/config โ€” partial update โ”€โ”€ + @router.patch("") + async def patch_config(request: Request): + try: + patch = await request.json() + except Exception: + raise HTTPException(400, "Invalid JSON body") + + # v2: detect broker_symbol changes in custom_tickers so we can re-resolve + # the affected tickers on the live MT5 provider without a restart. + affected_tickers = _collect_broker_symbol_changes(patch) + + cfg_manager.update(patch) + + if affected_tickers: + _apply_broker_symbol_resets(cfg_manager, affected_tickers) + try: + if broadcast_event: + await broadcast_event({ + "type": "data_sync_complete", + "source": "broker_symbol_change", + "tickers": sorted(affected_tickers), + }) + except Exception: + pass # non-critical + + await _notify() + return cfg_manager.get_config() + + # โ”€โ”€ POST /api/config/tickers โ€” add ticker โ”€โ”€ + @router.post("/tickers") + async def add_ticker(request: Request): + try: + body = await request.json() + except Exception: + raise HTTPException(400, "Invalid JSON body") + + ticker = body.get("ticker", "").strip().upper() + if not ticker: + raise HTTPException(400, "ticker is required") + + metadata = body.get("metadata") + success = cfg_manager.add_ticker(ticker, metadata) + if success: + await _notify() + return {"success": success, "ticker": ticker} + + # โ”€โ”€ DELETE /api/config/tickers/{ticker} โ€” remove ticker โ”€โ”€ + @router.delete("/tickers/{ticker}") + async def remove_ticker(ticker: str): + canonical = ticker.upper() + success = cfg_manager.remove_ticker(canonical) + if success: + await _notify() + return {"success": success, "ticker": canonical} + + # โ”€โ”€ POST /api/config/tickers/{ticker}/toggle โ€” hide/show โ”€โ”€ + @router.post("/tickers/{ticker}/toggle") + async def toggle_ticker(ticker: str): + canonical = ticker.upper() + visible = cfg_manager.toggle_ticker_visibility(canonical) + await _notify() + return {"ticker": canonical, "visible": visible} + + # โ”€โ”€ PUT /api/config/tickers/order โ€” reorder display โ”€โ”€ + @router.put("/tickers/order") + async def reorder_tickers(request: Request): + try: + body = await request.json() + except Exception: + raise HTTPException(400, "Invalid JSON body") + + order = body.get("order", []) + if not isinstance(order, list): + raise HTTPException(400, "order must be a list of ticker strings") + + cfg_manager.reorder_tickers(order) + await _notify() + return {"success": True, "order": order} + + # โ”€โ”€ POST /api/config/profiles โ€” save profile โ”€โ”€ + @router.post("/profiles") + async def save_profile(request: Request): + try: + body = await request.json() + except Exception: + raise HTTPException(400, "Invalid JSON body") + + name = body.get("name", "").strip() + if not name: + raise HTTPException(400, "name is required") + + cfg_manager.save_profile(name) + await _notify() + return {"success": True, "name": name} + + # โ”€โ”€ POST /api/config/profiles/{name}/load โ€” load profile โ”€โ”€ + @router.post("/profiles/{name}/load") + async def load_profile(name: str): + success = cfg_manager.load_profile(name) + if not success: + raise HTTPException(404, f"Profile '{name}' not found") + await _notify() + return {"success": True, "name": name} + + # โ”€โ”€ DELETE /api/config/profiles/{name} โ€” delete profile โ”€โ”€ + @router.delete("/profiles/{name}") + async def delete_profile(name: str): + success = cfg_manager.delete_profile(name) + if not success: + raise HTTPException(404, f"Profile '{name}' not found") + await _notify() + return {"success": True, "name": name} + + return router \ No newline at end of file diff --git a/backend/consumer_startup.py b/backend/consumer_startup.py new file mode 100644 index 0000000..b8e7ae2 --- /dev/null +++ b/backend/consumer_startup.py @@ -0,0 +1,251 @@ +# version: v3 +""" +================================================================================ +Quantum Terminal Consumer โ€” Startup Gate & Route Wiring +================================================================================ +v3 โ€” consumer_gate() now accepts `skip_sync=False`. data_server's lifespan + calls it with `skip_sync=True` so the auth phase runs synchronously + (needed for login routing) while the slow sync_all() is kicked off in + a background thread after lifespan returns. HTTP routes become + available immediately; frontend polls /api/sync/status for the SYNCING + pill. Eliminates the "15-minute loading screen" symptom caused by + persistent 5xx responses on specific data endpoints. + +v2 โ€” Wire /api/version (version_info.py) so the app can report its running + version and check for updates via /api/version endpoint. +Bridges the consumer-specific modules (data_sync_client) +into the existing data_server.py with MINIMAL changes to the PRO codebase. + +Usage in data_server.py lifespan: + from consumer_startup import consumer_gate, wire_consumer_routes + + # In lifespan, BEFORE provider connect: + gate_result = consumer_gate() + if not gate_result["ok"]: + # Terminal won't render โ€” gate_result has details + pass + + # After app creation: + wire_consumer_routes(app) + +This module does NOT introduce any calculation triggers. +================================================================================ +""" + +import logging +from typing import Optional + +log = logging.getLogger("mk.consumer_startup") + + +# ============================================================ +# 1. CONSUMER STARTUP GATE (Rule C2) +# ============================================================ + +def consumer_gate(skip_sync: bool = False) -> dict: + """ + Run the consumer startup gate (Rule C2). + + Three conditions must be met before terminal renders: + 1. Auth always passes (Quantum Terminal is free) + 2. data_sync_client completes download OR loads from cache + 3. MT5 connection established (handled by existing data_server code) + + This function handles conditions 1 and 2. + + v3: when `skip_sync=True`, only the auth phase runs (fast). Caller is + responsible for running sync in a background thread so the FastAPI + lifespan isn't blocked by a slow/unhealthy server. `sync_summary` + will be returned as {"in_progress": True} in that case โ€” populate it + from the thread when sync completes. + Returns dict with gate status: + { + "ok": bool, + "auth_result": AuthResult dict, + "sync_summary": dict, + "needs_login": bool, + "needs_update": bool, + "offline_mode": bool, + } + """ + result = { + "ok": False, + "auth_result": None, + "sync_summary": None, + "needs_login": False, + "needs_update": False, + "offline_mode": False, + } + + # โ”€โ”€ Gate 1: Auth validation โ”€โ”€ + # Removed in Quantum Terminal: Always authenticated + result["auth_result"] = {"status": "ok", "message": "Quantum Terminal - Free Edition"} + result["needs_login"] = False + + # โ”€โ”€ Gate 2: Data sync โ”€โ”€ + # v3: skip_sync=True โ†’ lifespan will run sync_all() in a background + # thread instead, so the HTTP server is responsive immediately. + if skip_sync: + result["sync_summary"] = {"in_progress": True} + else: + try: + from data_sync_client import get_sync_client + sync_client = get_sync_client() + sync_summary = sync_client.sync_all() + result["sync_summary"] = sync_summary + + # Rule C6: format version mismatch blocks terminal + if not sync_summary.get("format_version_ok", True): + log.error("Startup gate: FORMAT VERSION MISMATCH โ€” update required") + result["needs_update"] = True + result["ok"] = False + return result + + if sync_summary.get("offline_mode", False): + result["offline_mode"] = True + + except ImportError: + log.warning("data_sync_client not available โ€” skipping data sync (dev mode)") + result["sync_summary"] = {"note": "dev mode โ€” no sync"} + except Exception as e: + log.error(f"Data sync error: {e}") + result["sync_summary"] = {"error": str(e)} + + result["ok"] = True + log.info("Consumer startup gate PASSED") + return result + + +# ============================================================ +# 2. ROUTE WIRING +# ============================================================ + +def wire_consumer_routes(app): + """ + Wire consumer-specific FastAPI routes into the existing app. + Call this AFTER app = FastAPI() in data_server.py. + + Adds: + /api/auth/* โ€” login, register, logout, status, refresh + /api/sync/* โ€” sync status, cone/options data, staleness, refresh + /api/consumer/* โ€” consumer-specific status endpoints + """ + routes_added = [] + + # Auth routes + # Removed in Quantum Terminal + + # Data sync routes + try: + from data_sync_client import create_data_sync_routes + app.include_router(create_data_sync_routes()) + routes_added.append("data_sync") + log.info("Consumer data sync routes wired: /api/sync/*") + except ImportError: + log.warning("data_sync_client not available โ€” sync routes not wired") + except Exception as e: + log.error(f"Failed to wire data sync routes: {e}") + + # v2: version/update-check route + try: + from version_info import create_version_router, CURRENT_VERSION + app.include_router(create_version_router()) + routes_added.append("version") + log.info(f"Consumer version route wired: /api/version (running v{CURRENT_VERSION})") + except ImportError: + log.warning("version_info not available โ€” /api/version not wired") + except Exception as e: + log.error(f"Failed to wire version route: {e}") + + # Consumer status endpoint + from fastapi import APIRouter + consumer_router = APIRouter(tags=["consumer"]) + + @consumer_router.get("/api/consumer/status") + async def consumer_status(): + """Consumer terminal status โ€” used by frontend to decide what to show.""" + status = { + "is_consumer": True, + "routes_available": routes_added, + } + + # Auth status + status["authenticated"] = True + status["user_email"] = "free@quantumterminal" + status["subscription_status"] = "active" + + # Sync status + try: + from data_sync_client import get_sync_client + sc = get_sync_client() + status["synced_tickers"] = sc.synced_tickers + status["offline_mode"] = sc.is_offline + status["format_version_ok"] = sc.format_version_ok + status["sync_in_progress"] = sc.is_syncing # drives the top-bar pill + status["last_synced_at"] = sc.last_synced_at + + staleness = sc.get_overall_staleness() + status["analysis_date"] = staleness.display_date + status["staleness_level"] = staleness.level.value + except Exception: + status["synced_tickers"] = [] + status["offline_mode"] = True + status["sync_in_progress"] = False + + return status + + @consumer_router.post("/api/consumer/restart") + async def consumer_restart(): + """ + Restart the terminal backend process. Spawns a fresh instance and + exits the current one after a brief delay so the HTTP response can + flush before the process exits. + + Rule C1 compliance: this restarts the display server only โ€” no + calculation modules are involved. + """ + import sys, os, subprocess, threading + import time as _time + + def _do_restart(): + _time.sleep(1.0) + try: + creationflags = 0 + if os.name == "nt": + creationflags = getattr(subprocess, "DETACHED_PROCESS", 0) | \ + getattr(subprocess, "CREATE_NEW_PROCESS_GROUP", 0) + subprocess.Popen( + [sys.executable] + sys.argv, + close_fds=True, + creationflags=creationflags, + ) + except Exception as e: + log.error(f"Restart spawn failed: {e}") + os._exit(0) + + threading.Thread(target=_do_restart, daemon=True).start() + log.warning("Consumer restart requested via /api/consumer/restart") + return {"status": "restarting", "delay_ms": 1000} + + app.include_router(consumer_router) + log.info("Consumer status route wired: /api/consumer/status") + + return routes_added + + +# ============================================================ +# 3. BLOCKED ENDPOINTS (Rule C1) +# ============================================================ + +# These endpoint prefixes must NEVER exist in the consumer terminal. +# consumer_verify.py test script checks for them. +BLOCKED_ENDPOINT_PREFIXES = [ + "/api/calculate", + "/api/calibrate", + "/api/calibration/run", + "/api/run-backtest", + "/api/wf/run", + "/api/flow/calibrate", + "/api/anchor/calibrate", + "/api/dev/clear-cache", # calc target clears calculation data +] diff --git a/backend/data_manager/__init__.py b/backend/data_manager/__init__.py new file mode 100644 index 0000000..1cd6b7a --- /dev/null +++ b/backend/data_manager/__init__.py @@ -0,0 +1,50 @@ +""" +Data Management Module (DMM) +============================ + +Unified data access layer for the Quantum Terminal quant platform. +Parquet-cached, gap-filling, source-agnostic data delivery. + +Quick start (singleton โ€” shares one cache across all scripts): + from data_manager import dm + df = dm.get_bars("EURUSD", "D1", n_bars=1500) + +Or instantiate with custom settings: + from data_manager import DataManager + dm = DataManager(library_root="D:/MyData", auto_init_mt5=True) + df = dm.get_bars("XAUUSD", "M15", date_from=..., date_to=...) +""" + +from .data_manager import DataManager +from .data_store import DataStore +from .data_catalog import DataCatalog + +import logging + +# Configure DMM logging โ€” INFO by default, callers can override +logging.getLogger("dmm").setLevel(logging.INFO) + +# โ”€โ”€ Lazy singleton โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +# Created on first access. Uses default library_root (../data_library/). +# The forecaster's MT5 init is expected to happen BEFORE first dm.get_bars(). + +_dm_instance = None + + +def get_dm(**kwargs) -> DataManager: + """Get or create the module-level DataManager singleton.""" + global _dm_instance + if _dm_instance is None: + _dm_instance = DataManager(**kwargs) + return _dm_instance + + +# Convenience alias โ€” `from data_manager import dm` gives you the singleton +class _LazyDM: + """Proxy that creates the real DataManager on first attribute access.""" + def __getattr__(self, name): + return getattr(get_dm(), name) + +dm = _LazyDM() + +__all__ = ["DataManager", "DataStore", "DataCatalog", "dm", "get_dm"] diff --git a/backend/data_manager/cli.py b/backend/data_manager/cli.py new file mode 100644 index 0000000..440790f --- /dev/null +++ b/backend/data_manager/cli.py @@ -0,0 +1,134 @@ +""" +Data Manager CLI โ€” command-line interface for data library management. + +Usage: + python -m data_manager.cli --status + python -m data_manager.cli --prefetch --symbols EURUSD XAUUSD --tf D1 M15 + python -m data_manager.cli --history --symbol EURUSD --limit 20 + python -m data_manager.cli --purge --symbol EURUSD --tf D1 + python -m data_manager.cli --validate +""" + +import argparse +import sys +import logging +from datetime import datetime, timedelta, timezone +from pathlib import Path + +# Ensure parent dir is on path for relative imports +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) + +from data_manager import DataManager + + +def cmd_status(dm: DataManager): + """Print catalog summary.""" + summary = dm.get_catalog_summary() + if not summary: + print("\n [DATA LIBRARY] Empty โ€” no datasets cached yet.\n") + return + print(f"\n [DATA LIBRARY] {dm.library_root}") + print(f" {'Symbol':<10} {'TF':<6} {'Type':<12} {'From':<12} {'To':<12} {'Rows':>8} {'Source':<6}") + print(" " + "โ”€" * 72) + for r in summary: + d_from = r.get("date_from", "")[:10] if r.get("date_from") else "โ€”" + d_to = r.get("date_to", "")[:10] if r.get("date_to") else "โ€”" + print(f" {r['symbol']:<10} {r['timeframe']:<6} {r['data_type']:<12} " + f"{d_from:<12} {d_to:<12} {r.get('row_count',0):>8} {r.get('source','?'):<6}") + print() + + +def cmd_prefetch(dm: DataManager, symbols: list, timeframes: list, days: int): + """Batch warm-up cache for given symbols/timeframes.""" + date_from = datetime.now(timezone.utc) - timedelta(days=days) + date_to = datetime.now(timezone.utc) + print(f"\n Prefetching {len(symbols)} symbols ร— {len(timeframes)} timeframes " + f"({days} days back)...\n") + dm.prefetch(symbols, timeframes, date_from, date_to) + print("\n Done.\n") + + +def cmd_history(dm: DataManager, symbol: str, limit: int): + """Show fetch history.""" + history = dm.get_fetch_history(symbol, limit) + if not history: + print(f"\n No fetch history{' for '+symbol if symbol else ''}.\n") + return + print(f"\n {'Time':<22} {'Symbol':<10} {'TF':<6} {'Rows':>8} {'ms':>8} {'Status':<10}") + print(" " + "โ”€" * 68) + for r in history: + ts = r.get("timestamp", "")[:19] + print(f" {ts:<22} {r['symbol']:<10} {r['timeframe']:<6} " + f"{r.get('rows_returned',0):>8} {r.get('duration_ms',0):>8} {r.get('status','?'):<10}") + print() + + +def cmd_validate(dm: DataManager): + """Validate all cached Parquet files.""" + summary = dm.get_catalog_summary() + if not summary: + print("\n Nothing to validate โ€” library is empty.\n") + return + print(f"\n Validating {len(summary)} datasets...") + all_ok = True + for r in summary: + result = dm.store.validate(r["symbol"], r["timeframe"], r.get("data_type", "ohlcv")) + tag = "โœ“" if result.valid else "โœ—" + print(f" {tag} {r['symbol']:<10} {r['timeframe']:<6} rows={result.row_count}", end="") + if not result.valid: + all_ok = False + print(f" ISSUES: {'; '.join(result.issues)}") + else: + print() + print(f"\n {'All datasets valid.' if all_ok else 'Some datasets have issues โ€” see above.'}\n") + + +def cmd_purge(dm: DataManager, symbol: str, timeframe: str, data_type: str): + """Remove a specific dataset from cache.""" + dm.purge(symbol, timeframe, data_type) + print(f"\n Purged {symbol} {timeframe} {data_type}\n") + + +def main(): + parser = argparse.ArgumentParser(description="Data Manager CLI") + parser.add_argument("--status", action="store_true", help="Show catalog summary") + parser.add_argument("--prefetch", action="store_true", help="Batch prefetch data") + parser.add_argument("--history", action="store_true", help="Show fetch history") + parser.add_argument("--validate", action="store_true", help="Validate all cached files") + parser.add_argument("--purge", action="store_true", help="Purge a dataset") + + parser.add_argument("--symbols", nargs="+", default=[]) + parser.add_argument("--symbol", type=str, default=None) + parser.add_argument("--tf", nargs="+", default=["D1"]) + parser.add_argument("--days", type=int, default=1500, help="Days back for prefetch") + parser.add_argument("--limit", type=int, default=50) + parser.add_argument("--data-type", type=str, default="ohlcv") + parser.add_argument("--mt5-path", type=str, default=None) + + args = parser.parse_args() + + logging.basicConfig(level=logging.INFO, format="%(message)s") + dm = DataManager(mt5_terminal_path=args.mt5_path, auto_init_mt5=True) + + if args.status: + cmd_status(dm) + elif args.prefetch: + if not args.symbols: + print(" --prefetch requires --symbols") + sys.exit(1) + cmd_prefetch(dm, args.symbols, args.tf, args.days) + elif args.history: + cmd_history(dm, args.symbol, args.limit) + elif args.validate: + cmd_validate(dm) + elif args.purge: + if not args.symbol or not args.tf: + print(" --purge requires --symbol and --tf") + sys.exit(1) + cmd_purge(dm, args.symbol, args.tf[0], args.data_type) + else: + parser.print_help() + + +if __name__ == "__main__": + main() diff --git a/backend/data_manager/data_catalog.py b/backend/data_manager/data_catalog.py new file mode 100644 index 0000000..a0b5ea0 --- /dev/null +++ b/backend/data_manager/data_catalog.py @@ -0,0 +1,253 @@ +""" +DataCatalog โ€” SQLite-backed metadata catalog for the data library. + +Tracks what data exists, date coverage, row counts, gaps, and fetch history. +All paths stored as relative strings for portability. +""" + +import sqlite3 +import json +from pathlib import Path +from datetime import datetime, timezone +from dataclasses import dataclass, field +from typing import Optional, List + +import logging +log = logging.getLogger("dmm.catalog") + + +@dataclass +class CoverageResult: + """What the catalog knows about a (symbol, timeframe, data_type) tuple.""" + symbol: str + timeframe: str + data_type: str + covered: bool = False + date_from: Optional[datetime] = None + date_to: Optional[datetime] = None + row_count: int = 0 + file_path: Optional[str] = None + # Gaps that need filling relative to a request + gaps: List[dict] = field(default_factory=list) + + @property + def needs_fetch(self) -> bool: + return not self.covered or len(self.gaps) > 0 + + +class DataCatalog: + """ + SQLite metadata catalog. + + Parameters + ---------- + db_path : Path + Absolute path to catalog.db (created if missing). + """ + + def __init__(self, db_path: Path): + self.db_path = Path(db_path) + self.db_path.parent.mkdir(parents=True, exist_ok=True) + self._conn = sqlite3.connect(str(self.db_path), check_same_thread=False) + self._conn.row_factory = sqlite3.Row + self._create_tables() + + # โ”€โ”€ schema โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + + def _create_tables(self): + c = self._conn.cursor() + c.execute(""" + CREATE TABLE IF NOT EXISTS data_registry ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + symbol TEXT NOT NULL, + timeframe TEXT NOT NULL, + data_type TEXT NOT NULL DEFAULT 'ohlcv', + source TEXT NOT NULL DEFAULT 'mt5', + date_from TEXT, + date_to TEXT, + row_count INTEGER DEFAULT 0, + file_path TEXT, + last_updated TEXT, + checksum TEXT, + gaps TEXT DEFAULT '[]', + UNIQUE(symbol, timeframe, data_type) + ) + """) + c.execute(""" + CREATE TABLE IF NOT EXISTS fetch_log ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + symbol TEXT NOT NULL, + timeframe TEXT NOT NULL, + source TEXT NOT NULL, + fetch_from TEXT, + fetch_to TEXT, + rows_returned INTEGER DEFAULT 0, + duration_ms INTEGER DEFAULT 0, + status TEXT DEFAULT 'success', + error_msg TEXT, + timestamp TEXT + ) + """) + self._conn.commit() + + # โ”€โ”€ coverage API โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + + def check_coverage( + self, + symbol: str, + timeframe: str, + date_from: datetime, + date_to: datetime, + data_type: str = "ohlcv", + ) -> CoverageResult: + """ + Check what coverage exists for a request and identify gaps to fill. + """ + result = CoverageResult( + symbol=symbol.upper(), + timeframe=timeframe.upper(), + data_type=data_type, + ) + + row = self._conn.execute( + "SELECT * FROM data_registry WHERE symbol=? AND timeframe=? AND data_type=?", + (symbol.upper(), timeframe.upper(), data_type), + ).fetchone() + + if row is None: + # Nothing cached โ€” entire range is a gap + result.gaps.append({ + "from": date_from.isoformat(), + "to": date_to.isoformat(), + }) + return result + + result.covered = True + result.date_from = datetime.fromisoformat(row["date_from"]) if row["date_from"] else None + result.date_to = datetime.fromisoformat(row["date_to"]) if row["date_to"] else None + result.row_count = row["row_count"] or 0 + result.file_path = row["file_path"] + + # Identify leading gap (request starts before cached range) + if result.date_from and date_from < result.date_from: + result.gaps.append({ + "from": date_from.isoformat(), + "to": result.date_from.isoformat(), + }) + + # Identify trailing gap (request ends after cached range) + if result.date_to and date_to > result.date_to: + result.gaps.append({ + "from": result.date_to.isoformat(), + "to": date_to.isoformat(), + }) + + return result + + def check_latest(self, symbol: str, timeframe: str, data_type: str = "ohlcv") -> CoverageResult: + """Quick check: what's the latest bar we have?""" + result = CoverageResult(symbol=symbol.upper(), timeframe=timeframe.upper(), data_type=data_type) + row = self._conn.execute( + "SELECT * FROM data_registry WHERE symbol=? AND timeframe=? AND data_type=?", + (symbol.upper(), timeframe.upper(), data_type), + ).fetchone() + + if row: + result.covered = True + result.date_from = datetime.fromisoformat(row["date_from"]) if row["date_from"] else None + result.date_to = datetime.fromisoformat(row["date_to"]) if row["date_to"] else None + result.row_count = row["row_count"] or 0 + result.file_path = row["file_path"] + + return result + + # โ”€โ”€ registration & updates โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + + def register_dataset( + self, + symbol: str, + timeframe: str, + data_type: str, + source: str, + date_from: datetime, + date_to: datetime, + row_count: int, + file_path: str, + checksum: Optional[str] = None, + ): + """Insert or update a dataset record in the registry.""" + now = datetime.now(timezone.utc).isoformat() + self._conn.execute(""" + INSERT INTO data_registry (symbol, timeframe, data_type, source, + date_from, date_to, row_count, file_path, + last_updated, checksum) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(symbol, timeframe, data_type) DO UPDATE SET + source=excluded.source, + date_from=excluded.date_from, + date_to=excluded.date_to, + row_count=excluded.row_count, + file_path=excluded.file_path, + last_updated=excluded.last_updated, + checksum=excluded.checksum + """, ( + symbol.upper(), timeframe.upper(), data_type, source, + date_from.isoformat(), date_to.isoformat(), row_count, + file_path, now, checksum, + )) + self._conn.commit() + + def log_fetch( + self, + symbol: str, + timeframe: str, + source: str, + fetch_from: datetime, + fetch_to: datetime, + rows_returned: int, + duration_ms: int, + status: str = "success", + error_msg: Optional[str] = None, + ): + """Record a fetch attempt in the audit log.""" + self._conn.execute(""" + INSERT INTO fetch_log (symbol, timeframe, source, fetch_from, fetch_to, + rows_returned, duration_ms, status, error_msg, timestamp) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + """, ( + symbol.upper(), timeframe.upper(), source, + fetch_from.isoformat(), fetch_to.isoformat(), + rows_returned, duration_ms, status, error_msg, + datetime.now(timezone.utc).isoformat(), + )) + self._conn.commit() + + # โ”€โ”€ summary / status โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + + def get_summary(self) -> list: + """Return all registry entries as a list of dicts.""" + rows = self._conn.execute("SELECT * FROM data_registry ORDER BY symbol, timeframe").fetchall() + return [dict(r) for r in rows] + + def get_fetch_history(self, symbol: Optional[str] = None, limit: int = 50) -> list: + if symbol: + rows = self._conn.execute( + "SELECT * FROM fetch_log WHERE symbol=? ORDER BY timestamp DESC LIMIT ?", + (symbol.upper(), limit), + ).fetchall() + else: + rows = self._conn.execute( + "SELECT * FROM fetch_log ORDER BY timestamp DESC LIMIT ?", (limit,) + ).fetchall() + return [dict(r) for r in rows] + + def purge(self, symbol: str, timeframe: str, data_type: str = "ohlcv"): + """Remove a dataset from the registry (caller must also delete the Parquet file).""" + self._conn.execute( + "DELETE FROM data_registry WHERE symbol=? AND timeframe=? AND data_type=?", + (symbol.upper(), timeframe.upper(), data_type), + ) + self._conn.commit() + + def close(self): + self._conn.close() diff --git a/backend/data_manager/data_manager.py b/backend/data_manager/data_manager.py new file mode 100644 index 0000000..bf95c19 --- /dev/null +++ b/backend/data_manager/data_manager.py @@ -0,0 +1,332 @@ +""" +DataManager โ€” Main entry point for the Data Management Module. + +Callers use this class exclusively. It coordinates: + - DataCatalog (knows what's cached) + - DataStore (reads/writes Parquet) + - Fetchers (fill gaps from live sources) + +All paths are derived relative to a configurable library root. +Default root is /data_library/ for full portability. + +Usage +----- + from data_manager import dm # pre-wired singleton + df = dm.get_bars("EURUSD", "D1", 1500) # last 1500 daily bars + df = dm.get_bars("XAUUSD", "M15", # date range fetch + date_from=datetime(2024,1,1), + date_to=datetime(2025,1,1)) +""" + +import time +import pandas as pd +from pathlib import Path +from datetime import datetime, timedelta, timezone +from typing import Optional, List, Union + +from .data_catalog import DataCatalog, CoverageResult +from .data_store import DataStore +from .fetchers.base_fetcher import BaseFetcher +from .fetchers.mt5_fetcher import MT5Fetcher, DEFAULT_MT5_PATH + +import logging +log = logging.getLogger("dmm.manager") + + +class DataManager: + """ + Unified data access layer. + + Parameters + ---------- + library_root : Path or str or None + Where Parquet files and catalog.db live. + Default: /../data_library/ + mt5_terminal_path : str or None + Path to MT5 terminal64.exe. If None, uses DEFAULT_MT5_PATH + from mt5_fetcher.py (single source of truth for the terminal path). + auto_init_mt5 : bool + If True, DataManager will call mt5.initialize() automatically. + If False (default), the caller is responsible for MT5 init โ€” + this is the typical pattern since the forecaster already inits MT5. + cache_enabled : bool + If False, always fetch fresh from source (passthrough mode for testing). + """ + + def __init__( + self, + library_root: Union[str, Path, None] = None, + mt5_terminal_path: Optional[str] = None, + auto_init_mt5: bool = False, + cache_enabled: bool = True, + ): + # Resolve library root relative to this file (portable) + if library_root is None: + library_root = Path(__file__).resolve().parent.parent / "data_library" + self.library_root = Path(library_root).resolve() + + self.cache_enabled = cache_enabled + self.store = DataStore(self.library_root) + self.catalog = DataCatalog(self.library_root / "catalog.db") + + # Primary fetcher: MT5 + self._fetcher = MT5Fetcher( + terminal_path=mt5_terminal_path, + auto_init=auto_init_mt5, + ) + + # Clean up any interrupted writes from previous runs + self.store.cleanup_temp() + + log.info(f"DataManager ready | library: {self.library_root} | cache: {self.cache_enabled}") + + # โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ• + # PUBLIC API โ€” what callers use + # โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ• + + def get_bars( + self, + symbol: str, + timeframe: str, + n_bars: Optional[int] = None, + date_from: Optional[datetime] = None, + date_to: Optional[datetime] = None, + data_type: str = "ohlcv", + ) -> Optional[pd.DataFrame]: + """ + Get OHLCV bars โ€” the main workhorse method. + + Two calling modes: + 1. n_bars mode: dm.get_bars("EURUSD", "D1", n_bars=1500) + 2. date range mode: dm.get_bars("EURUSD", "M15", date_from=..., date_to=...) + + Returns a DataFrame with DatetimeIndex('time') and lowercase columns, + identical to what MT5 returns (drop-in compatible). + """ + symbol = symbol.upper() + timeframe = timeframe.upper() + + # โ”€โ”€ Mode 1: last N bars โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + if n_bars is not None: + return self._get_bars_n(symbol, timeframe, n_bars, data_type) + + # โ”€โ”€ Mode 2: date range โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + if date_from is None: + raise ValueError("Must provide either n_bars or date_from") + + if date_to is None: + date_to = datetime.now(timezone.utc) + + # Ensure UTC + if date_from.tzinfo is None: + date_from = date_from.replace(tzinfo=timezone.utc) + if date_to.tzinfo is None: + date_to = date_to.replace(tzinfo=timezone.utc) + + # Clamp future dates to now + now = datetime.now(timezone.utc) + if date_to > now: + date_to = now + + return self._get_bars_range(symbol, timeframe, date_from, date_to, data_type) + + def get_bars_as_forecaster( + self, + symbol: str, + timeframe: str, + n_bars: Optional[int] = None, + date_from: Optional[datetime] = None, + date_to: Optional[datetime] = None, + ) -> Optional[pd.DataFrame]: + """ + Same as get_bars() but returns with capitalized column names + (Open, High, Low, Close, Volume) and 'time' as the index โ€” + matching what the forecaster currently expects from fetch_daily_data() + and fetch_intraday_data(). + """ + df = self.get_bars(symbol, timeframe, n_bars=n_bars, + date_from=date_from, date_to=date_to) + if df is None: + return None + + # Rename to match forecaster convention + rename_map = { + "open": "Open", "high": "High", "low": "Low", + "close": "Close", "tick_volume": "Volume", + } + df.rename(columns=rename_map, inplace=True) + + # Strip timezone for compat with existing code that uses tz-naive datetimes + if df.index.tz is not None: + df.index = df.index.tz_localize(None) + + return df + + def prefetch( + self, + symbols: List[str], + timeframes: List[str], + date_from: datetime, + date_to: datetime, + data_type: str = "ohlcv", + ): + """Batch warm-up: ensure all (symbol, tf) combos are cached.""" + total = len(symbols) * len(timeframes) + done = 0 + for sym in symbols: + for tf in timeframes: + done += 1 + log.info(f"[{done}/{total}] Prefetching {sym} {tf}...") + self.get_bars(sym, tf, date_from=date_from, date_to=date_to, data_type=data_type) + + def get_catalog_summary(self) -> list: + """Return a summary of all cached datasets.""" + return self.catalog.get_summary() + + def get_fetch_history(self, symbol: Optional[str] = None, limit: int = 50) -> list: + return self.catalog.get_fetch_history(symbol, limit) + + def purge(self, symbol: str, timeframe: str, data_type: str = "ohlcv"): + """Remove cached data for a specific (symbol, tf, data_type) tuple.""" + fpath = self.store._parquet_path(symbol, timeframe, data_type) + if fpath.exists(): + fpath.unlink() + meta = self.store._meta_path(symbol, timeframe, data_type) + if meta.exists(): + meta.unlink() + self.catalog.purge(symbol, timeframe, data_type) + log.info(f"Purged {symbol} {timeframe} {data_type}") + + # โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ• + # INTERNAL โ€” fetch orchestration + # โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ• + + def _get_bars_n(self, symbol: str, timeframe: str, n_bars: int, + data_type: str) -> Optional[pd.DataFrame]: + """ + Get last N bars. Strategy: + 1. If cache has enough rows and is recent enough, return from cache. + 2. Otherwise, fetch from source, cache, return. + """ + if self.cache_enabled: + coverage = self.catalog.check_latest(symbol, timeframe, data_type) + if coverage.covered and coverage.row_count >= n_bars: + # Check if cache is reasonably fresh (within the timeframe interval) + # For D1, "fresh" means updated today; for M15, within the last hour + df = self.store.load(symbol, timeframe, data_type) + if df is not None and len(df) >= n_bars: + # Still fetch a small update to catch new bars + self._update_trailing(symbol, timeframe, data_type, coverage) + df = self.store.load(symbol, timeframe, data_type) + if df is not None: + return df.tail(n_bars) + + # Cache miss or disabled โ€” fetch from source + t0 = time.time() + df = self._fetcher.fetch_bars_n(symbol, timeframe, n_bars) + elapsed_ms = int((time.time() - t0) * 1000) + + if df is None or df.empty: + self.catalog.log_fetch(symbol, timeframe, self._fetcher.name, + datetime.now(timezone.utc), datetime.now(timezone.utc), + 0, elapsed_ms, "failed", "No data returned") + return None + + # Cache the result + if self.cache_enabled: + self.store.append(symbol, timeframe, df, data_type) + checksum = self.store.compute_checksum(symbol, timeframe, data_type) + file_path = str(self.store._parquet_path(symbol, timeframe, data_type).relative_to(self.library_root)) + self.catalog.register_dataset( + symbol, timeframe, data_type, self._fetcher.name, + df.index.min(), df.index.max(), len(df), + file_path, checksum, + ) + + self.catalog.log_fetch(symbol, timeframe, self._fetcher.name, + df.index.min(), df.index.max(), + len(df), elapsed_ms, "success") + + return df.tail(n_bars) + + def _get_bars_range(self, symbol: str, timeframe: str, + date_from: datetime, date_to: datetime, + data_type: str) -> Optional[pd.DataFrame]: + """ + Get bars for a date range. Strategy: + 1. Check catalog for coverage. + 2. Fetch only gaps (smart gap filling). + 3. Return full range from cache. + """ + if not self.cache_enabled: + return self._fetch_and_log(symbol, timeframe, date_from, date_to, data_type) + + coverage = self.catalog.check_coverage(symbol, timeframe, date_from, date_to, data_type) + + if coverage.needs_fetch: + for gap in coverage.gaps: + gap_from = datetime.fromisoformat(gap["from"]) + gap_to = datetime.fromisoformat(gap["to"]) + log.info(f"Filling gap: {symbol} {timeframe} {gap_from.date()}โ†’{gap_to.date()}") + fetched = self._fetch_and_log(symbol, timeframe, gap_from, gap_to, data_type) + if fetched is not None and not fetched.empty: + self.store.append(symbol, timeframe, fetched, data_type) + + # Update catalog with new combined coverage + full_df = self.store.load(symbol, timeframe, data_type) + if full_df is not None and not full_df.empty: + checksum = self.store.compute_checksum(symbol, timeframe, data_type) + file_path = str(self.store._parquet_path(symbol, timeframe, data_type).relative_to(self.library_root)) + self.catalog.register_dataset( + symbol, timeframe, data_type, self._fetcher.name, + full_df.index.min(), full_df.index.max(), len(full_df), + file_path, checksum, + ) + + # Return requested slice from cache + df = self.store.load(symbol, timeframe, data_type, date_from, date_to) + return df + + def _update_trailing(self, symbol: str, timeframe: str, data_type: str, + coverage: CoverageResult): + """Fetch only bars newer than what's already cached.""" + if coverage.date_to is None: + return + # Start from last cached bar (will be deduped) + fetch_from = coverage.date_to - timedelta(hours=1) # small overlap for safety + fetch_to = datetime.now(timezone.utc) + if fetch_to <= fetch_from: + return + + fetched = self._fetch_and_log(symbol, timeframe, fetch_from, fetch_to, data_type) + if fetched is not None and not fetched.empty: + self.store.append(symbol, timeframe, fetched, data_type) + # Re-read and update catalog + full_df = self.store.load(symbol, timeframe, data_type) + if full_df is not None and not full_df.empty: + checksum = self.store.compute_checksum(symbol, timeframe, data_type) + file_path = str(self.store._parquet_path(symbol, timeframe, data_type).relative_to(self.library_root)) + self.catalog.register_dataset( + symbol, timeframe, data_type, self._fetcher.name, + full_df.index.min(), full_df.index.max(), len(full_df), + file_path, checksum, + ) + + def _fetch_and_log(self, symbol: str, timeframe: str, + date_from: datetime, date_to: datetime, + data_type: str) -> Optional[pd.DataFrame]: + """Fetch from source with timing and audit logging.""" + t0 = time.time() + df = self._fetcher.fetch_bars(symbol, timeframe, date_from, date_to) + elapsed_ms = int((time.time() - t0) * 1000) + + if df is None or df.empty: + self.catalog.log_fetch(symbol, timeframe, self._fetcher.name, + date_from, date_to, 0, elapsed_ms, + "failed", "No data returned") + return None + + self.catalog.log_fetch(symbol, timeframe, self._fetcher.name, + date_from, date_to, len(df), elapsed_ms, "success") + log.info(f"Fetched {len(df)} rows {symbol} {timeframe} ({elapsed_ms}ms)") + return df diff --git a/backend/data_manager/data_store.py b/backend/data_manager/data_store.py new file mode 100644 index 0000000..0eb4819 --- /dev/null +++ b/backend/data_manager/data_store.py @@ -0,0 +1,278 @@ +""" +DataStore โ€” Format-agnostic data storage engine with append, dedup, and validation. + +Storage format priority: + 1. Parquet (pyarrow) โ€” production default, industry-standard columnar format + 2. Pickle โ€” fallback when pyarrow is unavailable (testing / lightweight setups) + +All file operations use relative paths anchored to the data_library root. +Designed for portability: no hardcoded absolute paths anywhere. +""" + +import pandas as pd +import numpy as np +import json +import hashlib +from pathlib import Path +from datetime import datetime, timezone +from typing import Optional, List + +import logging +log = logging.getLogger("dmm.store") + +# โ”€โ”€โ”€ Detect best available engine โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +_ENGINE = "pickle" # safe default +try: + import pyarrow + _ENGINE = "parquet" +except ImportError: + try: + import fastparquet + _ENGINE = "parquet" + except ImportError: + pass # stay on pickle + +_EXT = ".parquet" if _ENGINE == "parquet" else ".pkl" + +# โ”€โ”€โ”€ Expected OHLCV schema from MT5 โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +OHLCV_COLUMNS = ["time", "open", "high", "low", "close", "tick_volume", "spread", "real_volume"] +OHLCV_DTYPES = { + "open": "float64", "high": "float64", "low": "float64", "close": "float64", + "tick_volume": "int64", "spread": "int64", "real_volume": "int64", +} + + +class ValidationResult: + """Result of a data file validation check.""" + def __init__(self, valid: bool, issues: List[str] = None, row_count: int = 0): + self.valid = valid + self.issues = issues or [] + self.row_count = row_count + + def __repr__(self): + tag = "OK" if self.valid else f"FAIL({len(self.issues)})" + return f"" + + +class DataStore: + """ + Data store with auto-detected format (Parquet or Pickle). + + Parameters + ---------- + library_root : Path + Absolute path to the data_library/ directory (created if missing). + """ + + def __init__(self, library_root: Path): + self.root = Path(library_root) + self.root.mkdir(parents=True, exist_ok=True) + self._temp = self.root / "_temp" + self._temp.mkdir(exist_ok=True) + self.engine = _ENGINE + self.ext = _EXT + log.info(f"DataStore engine: {self.engine} (ext: {self.ext})") + + # โ”€โ”€ path helpers โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + + def _data_path(self, symbol: str, timeframe: str, data_type: str = "ohlcv") -> Path: + return self.root / symbol.upper() / timeframe.upper() / f"{data_type}{self.ext}" + + def _meta_path(self, symbol: str, timeframe: str, data_type: str = "ohlcv") -> Path: + return self.root / symbol.upper() / timeframe.upper() / f"{data_type}.meta.json" + + def _temp_path(self, symbol: str, timeframe: str, data_type: str = "ohlcv") -> Path: + return self._temp / f"{symbol}_{timeframe}_{data_type}{self.ext}" + + # Backward-compat alias used by DataManager + def _parquet_path(self, symbol: str, timeframe: str, data_type: str = "ohlcv") -> Path: + return self._data_path(symbol, timeframe, data_type) + + # โ”€โ”€ low-level IO โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + + def _write_df(self, df: pd.DataFrame, path: Path): + if self.engine == "parquet": + df.to_parquet(path, engine="pyarrow", compression="snappy", index=True) + else: + df.to_pickle(path) + + def _read_df(self, path: Path, columns: Optional[List[str]] = None) -> pd.DataFrame: + if self.engine == "parquet": + return pd.read_parquet(path, columns=columns) + else: + df = pd.read_pickle(path) + if columns is not None: + avail = [c for c in columns if c in df.columns] + df = df[avail] + return df + + # โ”€โ”€ core API โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + + def exists(self, symbol: str, timeframe: str, data_type: str = "ohlcv") -> bool: + return self._data_path(symbol, timeframe, data_type).exists() + + def load( + self, + symbol: str, + timeframe: str, + data_type: str = "ohlcv", + date_from: Optional[datetime] = None, + date_to: Optional[datetime] = None, + columns: Optional[List[str]] = None, + ) -> Optional[pd.DataFrame]: + """Load data with optional date and column filtering. Returns None if missing.""" + fpath = self._data_path(symbol, timeframe, data_type) + if not fpath.exists(): + return None + + try: + df = self._read_df(fpath, columns) + except Exception as e: + log.error(f"Failed to read {fpath}: {e}") + return None + + # Ensure DatetimeIndex + if "time" in df.columns and not isinstance(df.index, pd.DatetimeIndex): + df["time"] = pd.to_datetime(df["time"], utc=True) + df.set_index("time", inplace=True) + elif isinstance(df.index, pd.DatetimeIndex): + if df.index.tz is None: + df.index = df.index.tz_localize("UTC") + + if date_from is not None: + if hasattr(date_from, 'tzinfo') and date_from.tzinfo is None: + date_from = date_from.replace(tzinfo=timezone.utc) + df = df[df.index >= date_from] + if date_to is not None: + if hasattr(date_to, 'tzinfo') and date_to.tzinfo is None: + date_to = date_to.replace(tzinfo=timezone.utc) + df = df[df.index <= date_to] + + return df + + def save(self, symbol: str, timeframe: str, df: pd.DataFrame, + data_type: str = "ohlcv") -> Path: + """Full write. Writes to temp first, then atomic move to final path.""" + df = self._normalize(df) + fpath = self._data_path(symbol, timeframe, data_type) + fpath.parent.mkdir(parents=True, exist_ok=True) + + tmp = self._temp_path(symbol, timeframe, data_type) + self._write_df(df, tmp) + tmp.replace(fpath) + + self._write_meta(symbol, timeframe, data_type, df) + log.info(f"Saved {len(df)} rows -> {fpath.relative_to(self.root)} [{self.engine}]") + return fpath + + def append(self, symbol: str, timeframe: str, new_data: pd.DataFrame, + data_type: str = "ohlcv") -> Path: + """Append new rows with dedup at the seam. Creates file if missing.""" + new_data = self._normalize(new_data) + fpath = self._data_path(symbol, timeframe, data_type) + + if not fpath.exists(): + return self.save(symbol, timeframe, new_data, data_type) + + existing = self._read_df(fpath) + if "time" in existing.columns: + existing["time"] = pd.to_datetime(existing["time"], utc=True) + existing.set_index("time", inplace=True) + if isinstance(existing.index, pd.DatetimeIndex) and existing.index.tz is None: + existing.index = existing.index.tz_localize("UTC") + + combined = pd.concat([existing, new_data]) + combined = combined[~combined.index.duplicated(keep="last")] + combined.sort_index(inplace=True) + + return self.save(symbol, timeframe, combined, data_type) + + def validate(self, symbol: str, timeframe: str, data_type: str = "ohlcv") -> ValidationResult: + """Zero-trust validation: readability, schema, NaN, monotonic timestamps.""" + issues = [] + fpath = self._data_path(symbol, timeframe, data_type) + + if not fpath.exists(): + return ValidationResult(False, ["File does not exist"], 0) + + try: + df = self._read_df(fpath) + except Exception as e: + return ValidationResult(False, [f"Cannot read file: {e}"], 0) + + row_count = len(df) + if row_count == 0: + issues.append("File is empty") + + if "time" in df.columns: + df["time"] = pd.to_datetime(df["time"]) + if not df["time"].is_monotonic_increasing: + issues.append("Timestamps are not monotonic") + elif isinstance(df.index, pd.DatetimeIndex): + if not df.index.is_monotonic_increasing: + issues.append("Timestamps are not monotonic") + else: + issues.append("No 'time' column or DatetimeIndex found") + + if data_type == "ohlcv": + for col in ["open", "high", "low", "close"]: + if col in df.columns and df[col].isna().any(): + n = df[col].isna().sum() + issues.append(f"NaN in '{col}': {n} rows") + + return ValidationResult(valid=len(issues) == 0, issues=issues, row_count=row_count) + + def compute_checksum(self, symbol: str, timeframe: str, data_type: str = "ohlcv") -> Optional[str]: + fpath = self._data_path(symbol, timeframe, data_type) + if not fpath.exists(): + return None + h = hashlib.md5() + with open(fpath, "rb") as f: + for chunk in iter(lambda: f.read(8192), b""): + h.update(chunk) + return h.hexdigest() + + # โ”€โ”€ internals โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + + def _normalize(self, df: pd.DataFrame) -> pd.DataFrame: + """Ensure consistent format: DatetimeIndex named 'time', UTC, sorted.""" + df = df.copy() + + if "time" in df.columns and not isinstance(df.index, pd.DatetimeIndex): + df["time"] = pd.to_datetime(df["time"], utc=True) + df.set_index("time", inplace=True) + + if isinstance(df.index, pd.DatetimeIndex): + if df.index.tz is None: + df.index = df.index.tz_localize("UTC") + elif str(df.index.tz) != "UTC": + df.index = df.index.tz_convert("UTC") + + df.index.name = "time" + df.sort_index(inplace=True) + df.columns = [c.lower() for c in df.columns] + + return df + + def _write_meta(self, symbol: str, timeframe: str, data_type: str, df: pd.DataFrame): + meta_path = self._meta_path(symbol, timeframe, data_type) + meta = { + "symbol": symbol.upper(), + "timeframe": timeframe.upper(), + "data_type": data_type, + "storage_engine": self.engine, + "row_count": len(df), + "date_from": str(df.index.min()), + "date_to": str(df.index.max()), + "last_updated": datetime.now(timezone.utc).isoformat(), + "schema_version": 1, + } + with open(meta_path, "w") as f: + json.dump(meta, f, indent=2) + + def cleanup_temp(self): + """Remove any leftover temp files from interrupted writes.""" + for f in self._temp.iterdir(): + if f.is_file(): + f.unlink() + log.info(f"Cleaned temp file: {f.name}") diff --git a/backend/data_manager/fetchers/base_fetcher.py b/backend/data_manager/fetchers/base_fetcher.py new file mode 100644 index 0000000..7153e9e --- /dev/null +++ b/backend/data_manager/fetchers/base_fetcher.py @@ -0,0 +1,54 @@ +""" +BaseFetcher โ€” abstract interface for data source adapters. + +Every fetcher (MT5, DXFeed, futures bridge, CSV import) implements this. +""" + +from abc import ABC, abstractmethod +from datetime import datetime +from typing import Optional +import pandas as pd + + +class BaseFetcher(ABC): + """ + Abstract data fetcher interface. + + Implementations must return DataFrames with: + - DatetimeIndex named 'time' (UTC) + - Lowercase column names: open, high, low, close, tick_volume, spread, real_volume + """ + + name: str = "base" # Override in subclass + + @abstractmethod + def fetch_bars( + self, + symbol: str, + timeframe: str, + date_from: datetime, + date_to: datetime, + ) -> Optional[pd.DataFrame]: + """ + Fetch OHLCV bars for the given symbol/timeframe/date range. + Returns None on failure. + """ + ... + + @abstractmethod + def fetch_bars_n( + self, + symbol: str, + timeframe: str, + n_bars: int, + ) -> Optional[pd.DataFrame]: + """ + Fetch the most recent N bars. + Returns None on failure. + """ + ... + + @abstractmethod + def is_available(self) -> bool: + """Check if this data source is currently reachable.""" + ... diff --git a/backend/data_manager/fetchers/mt5_fetcher.py b/backend/data_manager/fetchers/mt5_fetcher.py new file mode 100644 index 0000000..2319582 --- /dev/null +++ b/backend/data_manager/fetchers/mt5_fetcher.py @@ -0,0 +1,272 @@ +""" +MT5Fetcher โ€” MetaTrader 5 data source adapter. + +Wraps mt5.copy_rates_range() and mt5.copy_rates_from_pos() with: + - Automatic timezone conversion to UTC + - Consistent lowercase column names + - Graceful failure handling + - Symbol alias resolution (GER40 โ†’ DE40 etc.) via server_config.SYMBOL_ALIASES +""" + +import pandas as pd +import numpy as np +from datetime import datetime, timezone +from typing import Optional, Dict, List +from pathlib import Path + +from .base_fetcher import BaseFetcher + +import logging +log = logging.getLogger("dmm.mt5_fetcher") + +# โ”€โ”€ Load MT5 terminal path from local_config.ini (Rule 3 โ€” no hardcoded paths) โ”€โ”€ +def _load_mt5_path_from_ini() -> Optional[str]: + """Read MT5 terminal path from local_config.ini if it exists.""" + import configparser + ini = Path(__file__).resolve().parent.parent.parent / "local_config.ini" + if not ini.exists(): + return None + cp = configparser.ConfigParser() + cp.read(str(ini)) + return cp.get("mt5", "terminal_path", fallback=None) + +DEFAULT_MT5_PATH = _load_mt5_path_from_ini() + +# โ”€โ”€ Symbol alias map โ€” imported from server_config if available โ”€โ”€ +_SYMBOL_ALIASES: Dict[str, List[str]] = {} + +def _load_aliases(): + """Load SYMBOL_ALIASES from server_config.py (same project root).""" + global _SYMBOL_ALIASES + if _SYMBOL_ALIASES: + return + try: + import sys + project_root = str(Path(__file__).resolve().parent.parent.parent) + if project_root not in sys.path: + sys.path.insert(0, project_root) + from server_config import SYMBOL_ALIASES + _SYMBOL_ALIASES = SYMBOL_ALIASES + except ImportError: + log.info("server_config not found โ€” symbol aliases unavailable in MT5Fetcher") + +# โ”€โ”€ MT5 timeframe mapping โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +_TF_MAP = {} # Populated lazily after mt5 import + + +def _ensure_tf_map(): + """Build timeframe string โ†’ MT5 constant map. Safe to call repeatedly.""" + global _TF_MAP + if _TF_MAP: + return + try: + import MetaTrader5 as mt5 + _TF_MAP.update({ + "M1": mt5.TIMEFRAME_M1, + "M2": mt5.TIMEFRAME_M2, + "M3": mt5.TIMEFRAME_M3, + "M4": mt5.TIMEFRAME_M4, + "M5": mt5.TIMEFRAME_M5, + "M6": mt5.TIMEFRAME_M6, + "M10": mt5.TIMEFRAME_M10, + "M12": mt5.TIMEFRAME_M12, + "M15": mt5.TIMEFRAME_M15, + "M20": mt5.TIMEFRAME_M20, + "M30": mt5.TIMEFRAME_M30, + "H1": mt5.TIMEFRAME_H1, + "H2": mt5.TIMEFRAME_H2, + "H3": mt5.TIMEFRAME_H3, + "H4": mt5.TIMEFRAME_H4, + "H6": mt5.TIMEFRAME_H6, + "H8": mt5.TIMEFRAME_H8, + "H12": mt5.TIMEFRAME_H12, + "D1": mt5.TIMEFRAME_D1, + "W1": mt5.TIMEFRAME_W1, + "MN": mt5.TIMEFRAME_MN1, + }) + except (ModuleNotFoundError, AttributeError): + log.warning("MetaTrader5 module not available โ€” MT5Fetcher will be non-functional") + + +class MT5Fetcher(BaseFetcher): + """ + MetaTrader 5 data fetcher. + + Parameters + ---------- + terminal_path : str or None + Path to terminal64.exe. If None, reads from local_config.ini. + auto_init : bool + If True, call mt5.initialize() on first fetch. If False, caller + is responsible for ensuring MT5 is already initialized. + """ + + name = "mt5" + + def __init__(self, terminal_path: Optional[str] = None, auto_init: bool = False): + self.terminal_path = terminal_path or DEFAULT_MT5_PATH + self.auto_init = auto_init + self._initialized = False + self._symbol_cache: Dict[str, str] = {} # canonical โ†’ broker symbol + + def _resolve_symbol(self, canonical: str) -> str: + """ + Resolve canonical ticker to broker symbol via alias lookup. + Caches successful resolutions. Returns canonical if no alias found. + """ + if canonical in self._symbol_cache: + return self._symbol_cache[canonical] + + _load_aliases() + + try: + import MetaTrader5 as mt5 + + # Try canonical name first + info = mt5.symbol_info(canonical) + if info is not None: + if not info.visible: + mt5.symbol_select(canonical, True) + self._symbol_cache[canonical] = canonical + return canonical + + # Try aliases + aliases = _SYMBOL_ALIASES.get(canonical, []) + for alias in aliases: + if alias == canonical: + continue + info = mt5.symbol_info(alias) + if info is not None: + if not info.visible: + mt5.symbol_select(alias, True) + log.info(f"Symbol alias resolved: {canonical} โ†’ {alias}") + self._symbol_cache[canonical] = alias + return alias + + # No alias found โ€” return canonical (will fail downstream) + log.warning(f"No broker symbol found for {canonical}") + self._symbol_cache[canonical] = canonical + return canonical + + except Exception: + return canonical + + def _ensure_init(self) -> bool: + """Initialize MT5 if auto_init is on and we haven't yet.""" + if self._initialized: + return True + if not self.auto_init: + # Assume caller already initialized MT5 + self._initialized = True + return True + try: + import MetaTrader5 as mt5 + kwargs = {} + if self.terminal_path: + kwargs["path"] = self.terminal_path + if mt5.initialize(**kwargs): + self._initialized = True + return True + else: + log.error(f"MT5 init failed: {mt5.last_error()}") + return False + except ModuleNotFoundError: + log.error("MetaTrader5 package not installed") + return False + + def is_available(self) -> bool: + return self._ensure_init() + + def fetch_bars( + self, + symbol: str, + timeframe: str, + date_from: datetime, + date_to: datetime, + ) -> Optional[pd.DataFrame]: + """ + Fetch OHLCV bars via mt5.copy_rates_range(). + """ + _ensure_tf_map() + if not self._ensure_init(): + return None + + tf_const = _TF_MAP.get(timeframe.upper()) + if tf_const is None: + log.error(f"Unknown timeframe: {timeframe}") + return None + + try: + import MetaTrader5 as mt5 + + broker_sym = self._resolve_symbol(symbol) + + if date_from.tzinfo is None: + date_from = date_from.replace(tzinfo=timezone.utc) + if date_to.tzinfo is None: + date_to = date_to.replace(tzinfo=timezone.utc) + + rates = mt5.copy_rates_range(broker_sym, tf_const, date_from, date_to) + if rates is None or len(rates) == 0: + log.warning(f"No data returned for {symbol} ({broker_sym}) {timeframe} " + f"{date_from.date()}โ†’{date_to.date()}") + return None + + return self._to_dataframe(rates) + + except Exception as e: + log.error(f"MT5 fetch_bars error for {symbol}: {e}") + return None + + def fetch_bars_n( + self, + symbol: str, + timeframe: str, + n_bars: int, + ) -> Optional[pd.DataFrame]: + """ + Fetch the most recent N bars via mt5.copy_rates_from_pos(). + """ + _ensure_tf_map() + if not self._ensure_init(): + return None + + tf_const = _TF_MAP.get(timeframe.upper()) + if tf_const is None: + log.error(f"Unknown timeframe: {timeframe}") + return None + + try: + import MetaTrader5 as mt5 + + broker_sym = self._resolve_symbol(symbol) + + rates = mt5.copy_rates_from_pos(broker_sym, tf_const, 0, n_bars) + if rates is None or len(rates) == 0: + log.warning(f"No data returned for {symbol} ({broker_sym}) {timeframe} (last {n_bars} bars)") + return None + + return self._to_dataframe(rates) + + except Exception as e: + log.error(f"MT5 fetch_bars_n error for {symbol}: {e}") + return None + + # โ”€โ”€ internal โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + + @staticmethod + def _to_dataframe(rates) -> pd.DataFrame: + """Convert MT5 rates array to standardized DataFrame.""" + df = pd.DataFrame(rates) + df["time"] = pd.to_datetime(df["time"], unit="s", utc=True) + df.set_index("time", inplace=True) + df.index.name = "time" + + # Ensure lowercase column names + df.columns = [c.lower() for c in df.columns] + + # Keep only standard columns if present + keep = ["open", "high", "low", "close", "tick_volume", "spread", "real_volume"] + df = df[[c for c in keep if c in df.columns]] + + return df \ No newline at end of file diff --git a/backend/data_manager/forcaster_dmm_patch.py b/backend/data_manager/forcaster_dmm_patch.py new file mode 100644 index 0000000..4582bdc --- /dev/null +++ b/backend/data_manager/forcaster_dmm_patch.py @@ -0,0 +1,207 @@ +""" +forcaster_dmm_patch.py โ€” Data Management Module integration for forcasterv21. + +This module provides drop-in replacement functions for the forecaster's +fetch_daily_data() and fetch_intraday_data() that route through the DMM +instead of calling MT5 directly. + +INTEGRATION (3 steps in forcasterv21_institutional_anchors.py): +โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + +STEP 1 โ€” Add import near line 28 (after other module imports): + + from data_manager import DataManager + +STEP 2 โ€” Replace the fetch functions (lines 66-93) with: + + # ========================================== + # ๐Ÿ“Š DATA INGESTION (via Data Management Module) + # ========================================== + _dm = None # Initialized in __main__ after MT5 init + + def fetch_daily_data(ticker, days_back=1500): + if _dm is None: + return _fetch_daily_data_legacy(ticker, days_back) + df = _dm.get_bars_as_forecaster(ticker, "D1", n_bars=days_back) + return df + + def fetch_intraday_data(ticker, start_time, end_time=None): + if _dm is None: + return _fetch_intraday_data_legacy(ticker, start_time, end_time) + df = _dm.get_bars_as_forecaster(ticker, "M15", n_bars=20000) + if df is None: return None + df = df[df.index >= start_time] + if end_time is not None: + df = df[df.index <= end_time] + return df + + # Legacy fallback (direct MT5) โ€” used if DMM init fails + def _fetch_daily_data_legacy(ticker, days_back=1500): + if not mt5.initialize(path=MT5_TERMINAL_PATH): + print(f"[!] MT5 Init failed for {ticker}: {mt5.last_error()}") + return None + rates = mt5.copy_rates_from_pos(ticker, mt5.TIMEFRAME_D1, 0, days_back) + if rates is None or len(rates) == 0: return None + df = pd.DataFrame(rates) + df['time'] = pd.to_datetime(df['time'], unit='s') + df.rename(columns={'open': 'Open', 'high': 'High', 'low': 'Low', + 'close': 'Close', 'tick_volume': 'Volume'}, inplace=True) + df.set_index('time', inplace=True) + return df + + def _fetch_intraday_data_legacy(ticker, start_time, end_time=None): + rates = mt5.copy_rates_from_pos(ticker, mt5.TIMEFRAME_M15, 0, 20000) + if rates is None or len(rates) == 0: return None + df = pd.DataFrame(rates) + df['time'] = pd.to_datetime(df['time'], unit='s') + df.rename(columns={'open': 'Open', 'high': 'High', 'low': 'Low', + 'close': 'Close'}, inplace=True) + df.set_index('time', inplace=True) + df = df[df.index >= start_time] + if end_time is not None: + df = df[df.index <= end_time] + return df + + +STEP 3 โ€” In __main__ block (after MT5 init, ~line 1855), add DMM initialization: + + if __name__ == "__main__": + if not mt5.initialize(path=MT5_TERMINAL_PATH): + print("[!] MT5 Init failed.") + quit() + + # >>> NEW: Initialize Data Management Module <<< + try: + _dm = DataManager(mt5_terminal_path=MT5_TERMINAL_PATH) + print(f"[DMM] Data library: {_dm.library_root} | Engine: {_dm.store.engine}") + except Exception as e: + print(f"[DMM] Init failed ({e}) โ€” falling back to direct MT5") + _dm = None + + # ... rest of __main__ unchanged ... + +That's it. Three changes. Everything else stays the same. +""" + +# โ”€โ”€โ”€ Standalone helper for quick integration testing โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +# You can run this file directly to test DMM + forecaster compatibility: +# python forcaster_dmm_patch.py + +import sys +from pathlib import Path + +def test_integration(): + """Test that DMM produces forecaster-compatible DataFrames.""" + sys.path.insert(0, str(Path(__file__).resolve().parent)) + + from data_manager import DataManager + from datetime import datetime, timezone + import pandas as pd + import numpy as np + + print("=" * 60) + print(" FORECASTER โ†” DMM INTEGRATION TEST") + print("=" * 60) + + dm = DataManager( + library_root=Path(__file__).resolve().parent / "_test_fcst_lib", + cache_enabled=True, + ) + + # Inject mock to simulate MT5 + class MockMT5Fetcher: + name = "mock_mt5" + def fetch_bars(self, symbol, timeframe, date_from, date_to): + n = max(1, (date_to - date_from).days + 1) + return _make_synthetic(date_from, n) + def fetch_bars_n(self, symbol, timeframe, n_bars): + return _make_synthetic(datetime(2020, 1, 1, tzinfo=timezone.utc), n_bars) + def is_available(self): + return True + + def _make_synthetic(start, n): + if hasattr(start, 'tzinfo') and start.tzinfo is None: + start = start.replace(tzinfo=timezone.utc) + from datetime import timedelta + dates = pd.DatetimeIndex( + [start + timedelta(days=i) for i in range(n)], + name="time" + ) + rng = np.random.default_rng(42) + close = 1800.0 + np.cumsum(rng.normal(0, 10, n)) + return pd.DataFrame({ + "open": close + rng.normal(0, 2, n), + "high": close + np.abs(rng.normal(0, 5, n)), + "low": close - np.abs(rng.normal(0, 5, n)), + "close": close, + "tick_volume": rng.integers(100, 5000, n), + "spread": rng.integers(1, 30, n), + "real_volume": np.zeros(n, dtype=int), + }, index=dates) + + dm._fetcher = MockMT5Fetcher() + passed = 0 + failed = 0 + + # Test 1: get_bars_as_forecaster matches expected schema + df = dm.get_bars_as_forecaster("XAUUSD", "D1", n_bars=1500) + if df is not None and all(c in df.columns for c in ["Open", "High", "Low", "Close", "Volume"]): + print(" โœ“ Column names: Open, High, Low, Close, Volume") + passed += 1 + else: + print(f" โœ— Column names wrong: {df.columns.tolist() if df is not None else 'None'}") + failed += 1 + + if df is not None and df.index.tz is None: + print(" โœ“ Timezone-naive index (forecaster compat)") + passed += 1 + else: + print(" โœ— Index has timezone (forecaster expects tz-naive)") + failed += 1 + + if df is not None and len(df) == 1500: + print(" โœ“ Correct row count (1500)") + passed += 1 + else: + print(f" โœ— Row count: {len(df) if df is not None else 'None'}") + failed += 1 + + if df is not None and isinstance(df.index, pd.DatetimeIndex): + print(" โœ“ DatetimeIndex preserved") + passed += 1 + else: + print(" โœ— Index is not DatetimeIndex") + failed += 1 + + # Test 2: Intraday M15 with date filtering + from datetime import timedelta + df15 = dm.get_bars_as_forecaster("XAUUSD", "M15", n_bars=500) + if df15 is not None and len(df15) == 500: + print(" โœ“ M15 intraday fetch (500 bars)") + passed += 1 + else: + print(f" โœ— M15 fetch: {len(df15) if df15 is not None else 'None'}") + failed += 1 + + # Test 3: Second fetch is faster (from cache) + import time + t0 = time.time() + dm.get_bars_as_forecaster("XAUUSD", "D1", n_bars=1500) + elapsed = time.time() - t0 + print(f" โœ“ Cached D1 fetch: {elapsed*1000:.0f}ms" if elapsed < 1 else f" โš  Cached fetch slow: {elapsed:.1f}s") + passed += 1 + + # Cleanup + import shutil + test_lib = Path(__file__).resolve().parent / "_test_fcst_lib" + if test_lib.exists(): + shutil.rmtree(test_lib) + + print(f"\n RESULTS: {passed} passed, {failed} failed") + print("=" * 60) + return failed == 0 + + +if __name__ == "__main__": + success = test_integration() + sys.exit(0 if success else 1) diff --git a/backend/data_manager/test_data_manager.py b/backend/data_manager/test_data_manager.py new file mode 100644 index 0000000..37a2076 --- /dev/null +++ b/backend/data_manager/test_data_manager.py @@ -0,0 +1,318 @@ +""" +Data Management Module โ€” Integration Test Suite + +Validates the full stack (DataStore โ†’ DataCatalog โ†’ DataManager) using +synthetic data. No MT5 connection required. + +Run from project root: + python test_data_manager.py +""" + +import sys +import shutil +import traceback +from pathlib import Path +from datetime import datetime, timedelta, timezone + +import pandas as pd +import numpy as np + +# Ensure data_manager is importable +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +from data_manager.data_store import DataStore, ValidationResult +from data_manager.data_catalog import DataCatalog, CoverageResult +from data_manager.data_manager import DataManager + +# โ”€โ”€ Test infrastructure โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +TEST_DIR = Path(__file__).resolve().parent / "_test_library" +PASS = 0 +FAIL = 0 + + +def setup(): + """Clean test directory.""" + if TEST_DIR.exists(): + shutil.rmtree(TEST_DIR) + TEST_DIR.mkdir(parents=True) + + +def teardown(): + """Remove test directory.""" + if TEST_DIR.exists(): + shutil.rmtree(TEST_DIR) + + +def check(name: str, condition: bool, detail: str = ""): + global PASS, FAIL + if condition: + PASS += 1 + print(f" โœ“ {name}") + else: + FAIL += 1 + print(f" โœ— {name} โ€” {detail}") + + +def make_ohlcv(start: datetime, n_bars: int, freq_hours: float = 24.0) -> pd.DataFrame: + """Generate synthetic OHLCV data.""" + dates = [start + timedelta(hours=freq_hours * i) for i in range(n_bars)] + dates = [d.replace(tzinfo=timezone.utc) for d in dates] + rng = np.random.default_rng(42) + close = 100.0 + np.cumsum(rng.normal(0, 1, n_bars)) + df = pd.DataFrame({ + "open": close + rng.normal(0, 0.5, n_bars), + "high": close + np.abs(rng.normal(0, 1, n_bars)), + "low": close - np.abs(rng.normal(0, 1, n_bars)), + "close": close, + "tick_volume": rng.integers(100, 10000, n_bars), + "spread": rng.integers(1, 20, n_bars), + "real_volume": np.zeros(n_bars, dtype=int), + }, index=pd.DatetimeIndex(dates, name="time")) + return df + + +# โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ• +# TEST 1 โ€” DataStore basics +# โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ• +def test_datastore(): + print("\nโ”€โ”€โ”€ TEST 1: DataStore (Parquet read/write) โ”€โ”€โ”€") + store = DataStore(TEST_DIR / "store_test") + + # Save + df = make_ohlcv(datetime(2024, 1, 1), 500) + store.save("EURUSD", "D1", df) + check("Save creates file", store.exists("EURUSD", "D1")) + + # Load full + loaded = store.load("EURUSD", "D1") + check("Load returns correct rows", loaded is not None and len(loaded) == 500, + f"got {len(loaded) if loaded is not None else 'None'}") + + # Load with date filter + mid = datetime(2024, 6, 1, tzinfo=timezone.utc) + filtered = store.load("EURUSD", "D1", date_from=mid) + check("Date filter works", filtered is not None and filtered.index.min() >= mid, + f"min={filtered.index.min() if filtered is not None else 'None'}") + + # Append + dedup + overlap_start = df.index[-50] + new_data = make_ohlcv(overlap_start.replace(tzinfo=None), 100, freq_hours=24.0) + store.append("EURUSD", "D1", new_data) + appended = store.load("EURUSD", "D1") + check("Append deduplicates overlap", appended is not None and len(appended) < 600, + f"expected <600, got {len(appended) if appended is not None else 'None'}") + check("Append is sorted", appended.index.is_monotonic_increasing if appended is not None else False) + + # Validate + result = store.validate("EURUSD", "D1") + check("Validation passes", result.valid, f"issues: {result.issues}") + + # Checksum + cs = store.compute_checksum("EURUSD", "D1") + check("Checksum computed", cs is not None and len(cs) == 32) + + # Non-existent + check("Non-existent returns None", store.load("FAKE", "D1") is None) + + # Column filter + cols_only = store.load("EURUSD", "D1", columns=["close"]) + check("Column filter works", cols_only is not None and list(cols_only.columns) == ["close"]) + + +# โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ• +# TEST 2 โ€” DataCatalog +# โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ• +def test_datacatalog(): + print("\nโ”€โ”€โ”€ TEST 2: DataCatalog (SQLite metadata) โ”€โ”€โ”€") + db_path = TEST_DIR / "catalog_test" / "catalog.db" + cat = DataCatalog(db_path) + + # Register + d_from = datetime(2024, 1, 1, tzinfo=timezone.utc) + d_to = datetime(2024, 6, 1, tzinfo=timezone.utc) + cat.register_dataset("EURUSD", "D1", "ohlcv", "mt5", d_from, d_to, 120, + "EURUSD/D1/ohlcv.parquet") + summary = cat.get_summary() + check("Register creates record", len(summary) == 1) + check("Symbol stored correctly", summary[0]["symbol"] == "EURUSD") + + # Coverage โ€” full hit + cov = cat.check_coverage("EURUSD", "D1", d_from, d_to) + check("Full coverage detected", cov.covered and len(cov.gaps) == 0, + f"gaps={cov.gaps}") + + # Coverage โ€” trailing gap + future = datetime(2024, 9, 1, tzinfo=timezone.utc) + cov2 = cat.check_coverage("EURUSD", "D1", d_from, future) + check("Trailing gap detected", cov2.needs_fetch and len(cov2.gaps) == 1, + f"gaps={cov2.gaps}") + + # Coverage โ€” leading gap + earlier = datetime(2023, 6, 1, tzinfo=timezone.utc) + cov3 = cat.check_coverage("EURUSD", "D1", earlier, d_to) + check("Leading gap detected", cov3.needs_fetch and len(cov3.gaps) == 1) + + # Coverage โ€” both gaps + cov4 = cat.check_coverage("EURUSD", "D1", earlier, future) + check("Both gaps detected", len(cov4.gaps) == 2, f"gaps={len(cov4.gaps)}") + + # Coverage โ€” unknown symbol + cov5 = cat.check_coverage("FAKE", "D1", d_from, d_to) + check("Unknown symbol = needs fetch", cov5.needs_fetch) + + # Fetch log + cat.log_fetch("EURUSD", "D1", "mt5", d_from, d_to, 120, 350, "success") + history = cat.get_fetch_history("EURUSD") + check("Fetch log recorded", len(history) == 1) + + # Upsert (update existing record) + d_to_new = datetime(2024, 9, 1, tzinfo=timezone.utc) + cat.register_dataset("EURUSD", "D1", "ohlcv", "mt5", d_from, d_to_new, 200, + "EURUSD/D1/ohlcv.parquet") + summary2 = cat.get_summary() + check("Upsert updates (not duplicates)", len(summary2) == 1 and summary2[0]["row_count"] == 200) + + cat.close() + + +# โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ• +# TEST 3 โ€” DataManager (full orchestration, mock fetcher) +# โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ• +def test_data_manager(): + print("\nโ”€โ”€โ”€ TEST 3: DataManager (orchestration with mock fetcher) โ”€โ”€โ”€") + + dm = DataManager(library_root=TEST_DIR / "dm_test", cache_enabled=True) + + # Inject a mock fetcher that returns synthetic data + class MockFetcher: + name = "mock" + call_count = 0 + def fetch_bars(self, symbol, timeframe, date_from, date_to): + self.call_count += 1 + # +1 to ensure we cover date_to inclusively + n_bars = max(1, (date_to - date_from).days + 1) + return make_ohlcv(date_from.replace(tzinfo=None), n_bars) + def fetch_bars_n(self, symbol, timeframe, n_bars): + self.call_count += 1 + return make_ohlcv(datetime(2024, 1, 1), n_bars) + def is_available(self): + return True + + mock = MockFetcher() + dm._fetcher = mock + + # First call โ€” should hit fetcher + d_from = datetime(2024, 1, 1, tzinfo=timezone.utc) + d_to = datetime(2024, 6, 1, tzinfo=timezone.utc) + df1 = dm.get_bars("EURUSD", "D1", date_from=d_from, date_to=d_to) + check("First call returns data", df1 is not None and len(df1) > 0) + first_calls = mock.call_count + + # Second identical call โ€” cache covers bulk of range; may do small trailing fill + mock.call_count = 0 + df2 = dm.get_bars("EURUSD", "D1", date_from=d_from, date_to=d_to) + check("Second call: at most 1 trailing gap fill", mock.call_count <= 1, + f"fetcher called {mock.call_count} times") + check("Cached data matches or grows", df2 is not None and len(df2) >= len(df1), + f"df2={len(df2) if df2 is not None else 0}, df1={len(df1)}") + + # Third call โ€” now cache fully covers range, zero fetches expected + mock.call_count = 0 + df2b = dm.get_bars("EURUSD", "D1", date_from=d_from, date_to=d_to) + check("Third call uses cache (zero fetches)", mock.call_count == 0, + f"fetcher called {mock.call_count} times") + + # Extend range โ€” should only fetch the gap + mock.call_count = 0 + d_to_ext = datetime(2024, 9, 1, tzinfo=timezone.utc) + df3 = dm.get_bars("EURUSD", "D1", date_from=d_from, date_to=d_to_ext) + check("Extended range triggers gap fetch", mock.call_count == 1, + f"expected 1 fetch, got {mock.call_count}") + check("Extended result is longer", df3 is not None and len(df3) > len(df1), + f"df3={len(df3) if df3 is not None else 0}, df1={len(df1)}") + + # n_bars mode + mock.call_count = 0 + df4 = dm.get_bars("XAUUSD", "M15", n_bars=500) + check("n_bars mode works", df4 is not None and len(df4) == 500) + + # Catalog summary + summary = dm.get_catalog_summary() + check("Catalog has 2 datasets", len(summary) == 2, + f"got {len(summary)}") + + # Forecaster-compatible output + df_fc = dm.get_bars_as_forecaster("EURUSD", "D1", date_from=d_from, date_to=d_to) + check("Forecaster compat: has 'Close'", df_fc is not None and "Close" in df_fc.columns) + check("Forecaster compat: tz-naive index", + df_fc is not None and df_fc.index.tz is None) + + # Purge + dm.purge("XAUUSD", "M15") + check("Purge removes dataset", len(dm.get_catalog_summary()) == 1) + + # Fetch history + history = dm.get_fetch_history() + check("Fetch history recorded", len(history) > 0) + + +# โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ• +# TEST 4 โ€” Path portability +# โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ• +def test_portability(): + print("\nโ”€โ”€โ”€ TEST 4: Path portability โ”€โ”€โ”€") + lib_root = TEST_DIR / "portable_test" + dm = DataManager(library_root=lib_root) + + # All paths should be under lib_root + check("Library root exists", lib_root.exists()) + check("Catalog in library root", (lib_root / "catalog.db").exists()) + check("Temp dir in library root", (lib_root / "_temp").exists()) + + # No absolute paths in catalog file_path entries + class MockFetcher: + name = "mock" + def fetch_bars(self, s, tf, df, dt): + return make_ohlcv(df.replace(tzinfo=None), 10) + def fetch_bars_n(self, s, tf, n): + return make_ohlcv(datetime(2024, 1, 1), n) + def is_available(self): + return True + + dm._fetcher = MockFetcher() + dm.get_bars("TEST", "D1", n_bars=10) + + summary = dm.get_catalog_summary() + if summary: + fp = summary[0].get("file_path", "") + check("Catalog stores relative path", not fp.startswith("/") and not fp.startswith("C:"), + f"got: {fp}") + else: + check("Catalog stores relative path", False, "no summary entries") + + +# โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ• +# RUN ALL +# โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ• +if __name__ == "__main__": + print("=" * 60) + print(" DATA MANAGEMENT MODULE โ€” INTEGRATION TESTS") + print("=" * 60) + + setup() + try: + test_datastore() + test_datacatalog() + test_data_manager() + test_portability() + except Exception as e: + print(f"\n โœ— UNEXPECTED ERROR: {e}") + traceback.print_exc() + FAIL += 1 + finally: + teardown() + + print("\n" + "=" * 60) + print(f" RESULTS: {PASS} passed, {FAIL} failed") + print("=" * 60) + sys.exit(1 if FAIL > 0 else 0) diff --git a/backend/data_server.py b/backend/data_server.py new file mode 100644 index 0000000..efce0fd --- /dev/null +++ b/backend/data_server.py @@ -0,0 +1,2097 @@ +# version: v27 +# v27 โ€” Registered the /api/chart-templates router (chart_templates_routes.py). +# Companion to /api/chart-presets โ€” named full-chart-config snapshots +# the operator saves/loads via the TPL โ–พ button. (2026-05-11) +# v26 โ€” /api/bands attaches `_last_modified` (unix epoch float) sourced from +# the resolved JSON file's mtime (PRO local OR sync cache). Frontend +# uses it to detect when bands are older than the most recent market +# close and flag the BANDS toolbar button red. Best-effort: if the +# cache file path can't be resolved (legacy fallback or unknown sync +# layout), the field is omitted and the frontend treats the bands as +# fresh (no red flag, no false alarms). +""" +================================================================================ +Quantum Terminal โ€” Data Server (Consumer Build) +================================================================================ +FastAPI backend for the consumer terminal. Display-only โ€” no computation. + +v25 โ€” Added env-gated install of debug_telemetry harness during lifespan + startup. No effect when MK_DEBUG_TELEMETRY != "1". Disposable โ€” + see docs/specs/2026-05-05-debug-telemetry-harness-design.md. + +v19 โ€” Non-blocking initial sync. Previous lifespan ran consumer_gate() end- + to-end before the FastAPI HTTP server became responsive, so every + retry/slow endpoint in sync_all() added to the user-visible loading + screen (up to 15 min in the field). Now: + 1. consumer_gate(skip_sync=True) runs synchronously โ€” auth only, + fast. Lifespan returns, HTTP routes go live immediately. + 2. A daemon thread runs sync_all() in the background; when it + finishes, `app.state.consumer_gate["sync_summary"]` is + populated. Frontend polls /api/sync/status for the SYNCING + pill (already wired) and shows the terminal UI the whole time. + Net: users always see the terminal within seconds, regardless of + server-side data-endpoint health. Per CLAUDE.md Rule C2 relaxed. + +v18 โ€” Force OPTIONS_AVAILABLE = False unconditionally. The PRO + `options_routes.create_options_router` serves `/api/options/{ticker}/full` + directly from `compute_all_multi(snap)` and never applies the ETFโ†’CFD + conversion โ€” so if its `options_data_manager` + `options_engine` + dependencies happen to be bundled into the consumer's PyInstaller + payload (older builds shipped them), the router takes over the route + and every GEX level, wall, max pain etc. renders in raw ETF price + space. Bug symptom reported in the field: XAUUSD candles stream fine + (MT5 connected) but options levels are in ETF numbers (GLD ~200 vs + XAUUSD ~4700). Consumer is display-only (Rule C1) โ€” the PRO options + engine should never run here, period. We keep the import attempt (so + upstream module resolution doesn't change) but set OPTIONS_AVAILABLE + to False after, guaranteeing the sync-cache fallback route (which + applies the CFD conversion) always handles /api/options/*. + +v2 โ€” Data Center transition: /api/bands/{ticker} now reads standalone + {TICKER}_bands.json from the sync cache (Data Center schema), with a + legacy fallback to the old nested bands key inside {TICKER}_cones.json. + /api/cones was already shape-flexible (wrapped or flat) โ€” no change. + +v3 โ€” Cache directory watcher: new background task started in consumer lifespan + that polls %APPDATA%\\QuantumTerminal\\cache\\ and broadcasts data_sync_complete + when any {TICKER}_{category}.json changes. Enables live updates without + browser refresh for manual drops and future auto-uploaders. Feature-flagged + via consumer_config.ini [sync] cache_watcher_enabled (default: on). + +v4 โ€” New endpoint /api/historical_cones/{ticker} for the REPLAY sub-tab of + quant analysis. Serves per-anchor historical cone payloads from the + sync cache (weekly + monthly anchors, each with gbm/mjd/bates models). + +v9 โ€” Focused-symbol fast tick loop. Per-symbol MT5 polling is the bottleneck + (โ‰ˆ5ms ร— universe_size per iteration) so dropping the global tick interval + hits a wall โ‰ˆ 5-7 fps at universe=20. New focus_tick_loop polls ONLY the + symbol the user is watching every FOCUS_TICK_INTERVAL (0.05s = 20 fps), + while the existing tick_loop keeps the rest of the universe fresh at the + normal 1s rate. Frontend sets focus via POST /api/focus/{ticker}. + +v8 โ€” Tick loop reads providers.accounts.mt5_default.use_tick_data on every + iteration. When True, polls MT5 every 0.2s instead of 1.0s, giving the + chart MT5-style live tick movement. Toggle surfaces in Settings โ†’ + PROVIDERS. Default OFF; switching it via PATCH /api/mt5/config takes + effect on the next loop tick โ€” no restart required. + +v7 โ€” /api/quarterly_cones/{ticker} now also resolves the Data Center's + GLOBAL__quarterly_cones.json naming via the sync client + a + direct second-path fallback. Schema upgraded to quarterly_cones_v2 + (4 anchors ร— 2 models). Frontend maps the 8 keys into the QUARTERLY + row of the CONES dropdown with per-anchor sub-toggles. + +v6 โ€” New endpoint /api/quarterly_cones/{ticker} โ€” reads + {TICKER}_quarterly_cones.json directly from the consumer cache. Temporary + wiring: the frontend maps gbm_quarter_curr / mjd_quarter_curr / + bates_quarter_curr into the existing "MANUAL" cone slots so the user can + see the quarterly forecast by toggling MANUAL in the CONES dropdown. + +v5 โ€” New endpoint /api/scalp_bands/{ticker} for the MICRO toggle in the + bands dropdown. Serves the scalp bands JSON (per-horizon point values + anchored at a recent timestamp) from the sync cache. + +Responsibilities: + 1. MT5 Price Feed โ€” poll ticks and bars, detect new closed bars + 2. WebSocket Broadcasting โ€” push prices and state-change events to clients + 3. REST API โ€” serve pre-computed data (cones, bands, anchors, probability field) + 4. File Watching โ€” monitor JSON state files for changes, notify clients + 5. Consumer Auth โ€” JWT login/register via license_client proxy + 6. Data Sync โ€” download pre-computed data from server via data_sync_client + +REMOVED from PRO (Rule C1 โ€” no calculation triggers): + - Signal engine evaluation (signal_engine_v2, signal_live_manager) + - OU analysis (ou_mean_reversion) + - Flow confirmation (flow_confirmation) + - Auto executor (auto_executor) + - Orchestrator endpoints (/api/orchestrator/*) + - Calculate endpoints (/api/calculate/*) + - Signal evaluate endpoints (/api/signals/evaluate/*) + - Anchor calibration endpoints (/api/anchor/calibrate) + - Manual cone computation (/api/cones/manual/*) + - Dev endpoints (/api/dev/*) + - Trade reconciliation loop + - Options archive scheduler + - Broker DPP cache population + +Usage: + python data_server.py --port 8502 + +This file does NOT introduce any calculation triggers. +================================================================================ +""" + +import json +import asyncio +import logging +import time +import argparse +import os +from pathlib import Path +from datetime import datetime, timezone +from typing import Dict, Set, Optional, Any +from contextlib import asynccontextmanager + +# โ”€โ”€ Project root (portable โ€” Rule 3) โ”€โ”€ +PROJECT_ROOT = Path(__file__).resolve().parent + +import sys +if str(PROJECT_ROOT) not in sys.path: + sys.path.insert(0, str(PROJECT_ROOT)) + +from server_config import ServerConfig, PROJECT_ROOT +from config_manager import ConfigManager +from config_routes import create_config_router +from execution_routes import create_execution_router +from regime_routes import create_regime_router +from system_routes import create_system_router +from account_routes import create_account_router +# v18: keep the import attempt (some downstream module trees expect the +# symbol to resolve) but force OPTIONS_AVAILABLE = False so the PRO +# router can never register over the sync-cache fallback. The fallback +# at `/api/options/{ticker}/full` is the ONLY path that applies the +# ETFโ†’CFD conversion for strikes/walls/levels/GEX. +try: + from options_routes import create_options_router # noqa: F401 +except ImportError: + create_options_router = None # noqa: F811 +OPTIONS_AVAILABLE = False +from freshness_routes import create_freshness_router +from trade_mapper_routes import create_trade_mapper_router +from pulse_room.routes import create_pulse_router +from chart_presets_routes import create_chart_presets_router +from chart_templates_routes import create_chart_templates_router + +# โ”€โ”€ Macro rotation data (yfinance sector + country ETFs) โ”€โ”€ +try: + from macro_data import get_sector_data, get_country_data + MACRO_DATA_AVAILABLE = True +except ImportError: + MACRO_DATA_AVAILABLE = False + +# โ”€โ”€ Consumer terminal wiring (Project B) โ”€โ”€ +try: + from consumer_startup import consumer_gate, wire_consumer_routes + CONSUMER_MODE = True +except ImportError: + CONSUMER_MODE = False + + +# โ”€โ”€ Third-party imports โ”€โ”€ +from fastapi import FastAPI, WebSocket, WebSocketDisconnect, HTTPException, Request +from fastapi.middleware.cors import CORSMiddleware +import uvicorn + +# โ”€โ”€ MT5 availability flag (for provider type detection, no direct calls) โ”€โ”€ +try: + import MetaTrader5 as _mt5_check + MT5_AVAILABLE = True + del _mt5_check +except ImportError: + MT5_AVAILABLE = False + +# โ”€โ”€ File watcher โ”€โ”€ +try: + from watchfiles import awatch, Change + WATCHFILES_AVAILABLE = True +except ImportError: + WATCHFILES_AVAILABLE = False + +# โ”€โ”€ Signal live manager โ€” REMOVED in consumer (Rule C1) โ”€โ”€ +SIGNAL_MANAGER_AVAILABLE = False +lifecycle_mgr = None +signal_evaluator = None + +# โ”€โ”€ Auto executor โ€” REMOVED in consumer (Rule C1) โ”€โ”€ +AUTO_EXECUTOR_AVAILABLE = False + + +log = logging.getLogger("data_server") + +# โ”€โ”€ Session file logging โ”€โ”€ +try: + from session_logger import setup_session_logging, create_logs_router + _session_log_path = setup_session_logging() + log.info(f"Session log: {_session_log_path}") +except ImportError: + log.info("session_logger not available โ€” console only") + create_logs_router = None +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s [%(name)s] %(levelname)s: %(message)s", + datefmt="%Y-%m-%d %H:%M:%S", +) + + + + +# ============================================================ +# 3. WEBSOCKET CONNECTION MANAGER +# ============================================================ + +class ConnectionManager: + """Manages WebSocket connections across channels.""" + + def __init__(self): + self._price_clients: Set[WebSocket] = set() + self._event_clients: Set[WebSocket] = set() + + async def connect_prices(self, ws: WebSocket): + await ws.accept() + self._price_clients.add(ws) + log.info(f"Price client connected ({len(self._price_clients)} total)") + + async def connect_events(self, ws: WebSocket): + await ws.accept() + self._event_clients.add(ws) + log.info(f"Event client connected ({len(self._event_clients)} total)") + + def disconnect_prices(self, ws: WebSocket): + self._price_clients.discard(ws) + log.info(f"Price client disconnected ({len(self._price_clients)} remaining)") + + def disconnect_events(self, ws: WebSocket): + self._event_clients.discard(ws) + log.info(f"Event client disconnected ({len(self._event_clients)} remaining)") + + async def broadcast_prices(self, message: dict): + """Send to all price channel subscribers.""" + if not self._price_clients: + return + data = json.dumps(message) + dead = [] + for ws in self._price_clients: + try: + await ws.send_text(data) + except Exception: + dead.append(ws) + for ws in dead: + self._price_clients.discard(ws) + + async def broadcast_event(self, message: dict): + """Send to all event channel subscribers.""" + if not self._event_clients: + return + data = json.dumps(message) + dead = [] + for ws in self._event_clients: + try: + await ws.send_text(data) + except Exception: + dead.append(ws) + for ws in dead: + self._event_clients.discard(ws) + + @property + def price_client_count(self) -> int: + return len(self._price_clients) + + @property + def event_client_count(self) -> int: + return len(self._event_clients) + + +# ============================================================ +# 4. FILE STATE READER +# ============================================================ + +def read_json_safe(path: Path) -> Optional[dict]: + """Read a JSON file, returning None on any error.""" + try: + if not path.exists(): + return None + with open(path, "r") as f: + return json.load(f) + except (json.JSONDecodeError, IOError, OSError) as e: + log.warning(f"Failed to read {path.name}: {e}") + return None + + +# ============================================================ +# 5. BACKGROUND TASKS +# ============================================================ + +# โ”€โ”€ Tick price cache (populated by tick_loop, read by signal lifecycle) โ”€โ”€ +_latest_ticks: Dict[str, float] = {} + + +FAST_TICK_INTERVAL = 1.0 # v9: full-universe loop stays at config rate when + # USE TICK DATA is on (fast loop now handles the + # focused chart at FOCUS_TICK_INTERVAL). +FOCUS_TICK_INTERVAL = 0.1 # v9/v10: focused-symbol fast loop โ€” 10 fps. + # Was 0.05 (20 fps) but that saturated React's + # main thread with re-renders, causing chart drag + # to freeze. 10 fps still feels live and leaves + # event-loop budget for mouse input. True MT5-style + # fluidity needs Option C (bypass React state on + # the price hot path) โ€” defer until requested. + +# v9: focused symbol the user is actively watching. None = no focus โ†’ fast +# loop idles. Set via POST /api/focus/{ticker}. +_focused_symbol: Optional[str] = None + + +def set_focused_symbol(ticker: Optional[str]): + global _focused_symbol + _focused_symbol = ticker.upper() if ticker else None + log.info(f"Focused symbol = {_focused_symbol or '(none)'}") + + +def get_focused_symbol() -> Optional[str]: + return _focused_symbol + + +def _resolve_tick_interval(cfg_manager: ConfigManager, base: float) -> float: + """v9: Main full-universe tick loop always uses base config.tick_interval. + The fast path is now focus_tick_loop, which polls only the focused symbol + at FOCUS_TICK_INTERVAL. (Kept for backward-compat / future hooks.)""" + return base + + +async def tick_loop(app_state, manager: ConnectionManager, config: ServerConfig, cfg_manager: ConfigManager): + """ + Polls MT5 for latest ticks. Default interval = config.tick_interval (1.0s). + v8: When user enables 'USE TICK DATA' in Settings โ†’ PROVIDERS, switches + to FAST_TICK_INTERVAL (0.2s) on the next iteration โ€” no restart needed. + Broadcasts to all /ws/prices subscribers. + """ + log.info(f"Tick loop started (base interval={config.tick_interval}s, " + f"fast interval={FAST_TICK_INTERVAL}s when use_tick_data=True)") + last_mode = None + while True: + try: + provider = app_state.provider + if provider and provider.connected and manager.price_client_count > 0: + universe = cfg_manager.get_active_universe() + ticks = await asyncio.to_thread(provider.get_latest_ticks, universe) + if ticks: + # Update tick cache for signal lifecycle price checks + for t, tick in ticks.items(): + _latest_ticks[t] = tick.last + # Send all ticks as one batch โ€” avoids React render-frame drops + await manager.broadcast_prices({ + "type": "ticks_batch", + "ticks": {t: tick.to_dict() for t, tick in ticks.items()}, + }) + elif provider and provider.connected: + # Still update cache even if no WS clients (signals need prices) + universe = cfg_manager.get_active_universe() + ticks = await asyncio.to_thread(provider.get_latest_ticks, universe) + if ticks: + for t, tick in ticks.items(): + _latest_ticks[t] = tick.last + except Exception as e: + log.error(f"Tick loop error: {e}") + # v8: dynamic sleep โ€” re-read each iteration so the user toggle is live. + interval = _resolve_tick_interval(cfg_manager, config.tick_interval) + if interval != last_mode: + log.info(f"Tick interval = {interval}s " + f"({'FAST / use_tick_data ON' if interval == FAST_TICK_INTERVAL else 'NORMAL'})") + last_mode = interval + await asyncio.sleep(interval) + + +async def focus_tick_loop(app_state, manager: ConnectionManager, cfg_manager: ConfigManager): + """v9: Fast loop dedicated to the symbol the user is actively watching. + Polls ONLY the focused symbol every FOCUS_TICK_INTERVAL when USE TICK DATA + is on. This avoids the universe-size ceiling that caps the main tick loop. + + Sleeps 1s when: + - USE TICK DATA is off + - No focused symbol set + - No WS price clients connected + - MT5 disconnected + """ + log.info(f"Focus tick loop started (interval={FOCUS_TICK_INTERVAL}s when active)") + last_active = None + while True: + try: + mt5_cfg = (cfg_manager.get_config().get("providers", {}) + .get("accounts", {}).get("mt5_default", {}) or {}) + use_tick = bool(mt5_cfg.get("use_tick_data", False)) + sym = get_focused_symbol() + provider = app_state.provider + + active = ( + use_tick + and sym is not None + and provider is not None + and provider.connected + and manager.price_client_count > 0 + ) + if active != last_active: + log.info( + f"Focus tick loop: {'ACTIVE' if active else 'IDLE'} " + f"(use_tick={use_tick}, sym={sym}, " + f"connected={provider.connected if provider else False}, " + f"clients={manager.price_client_count})" + ) + last_active = active + + if active: + ticks = await asyncio.to_thread(provider.get_latest_ticks, [sym]) + if ticks: + for t, tick in ticks.items(): + _latest_ticks[t] = tick.last + await manager.broadcast_prices({ + "type": "ticks_batch", + "ticks": {t: tick.to_dict() for t, tick in ticks.items()}, + }) + await asyncio.sleep(FOCUS_TICK_INTERVAL) + else: + await asyncio.sleep(1.0) + except Exception as e: + log.error(f"Focus tick loop error: {e}") + await asyncio.sleep(1.0) + + +async def bar_check_loop( + app_state, manager: ConnectionManager, config: ServerConfig, cfg_manager: ConfigManager +): + """ + Checks for newly closed M15 bars at config.bar_check_interval. + Broadcasts new bars to /ws/prices subscribers. + """ + log.info(f"Bar check loop started (interval={config.bar_check_interval}s)") + while True: + try: + provider = app_state.provider + if provider and provider.connected: + universe = cfg_manager.get_active_universe() + new_bars = await asyncio.to_thread( + provider.check_new_bars, universe, config.default_timeframe + ) + for bar_msg in new_bars: + await manager.broadcast_prices(bar_msg) + log.debug( + f"New {bar_msg['timeframe']} bar: " + f"{bar_msg['ticker']} @ {bar_msg['bar']['time']}" + ) + except Exception as e: + log.error(f"Bar check error: {e}") + await asyncio.sleep(config.bar_check_interval) + + +async def mt5_reconnect_loop(app_state, config: ServerConfig): + """ + Monitors MT5 connection. Reconnects with backoff if dropped. + """ + while True: + await asyncio.sleep(10) + provider = app_state.provider + if provider and not provider.connected: + log.info("MT5 disconnected โ€” attempting reconnect...") + for attempt in range(config.mt5_max_reconnect_attempts): + success = await asyncio.to_thread(provider.reconnect) + if success: + log.info("MT5 reconnected successfully") + break + delay = min(config.mt5_reconnect_delay * (2 ** attempt), 60) + log.warning( + f"MT5 reconnect attempt {attempt + 1} failed, " + f"retrying in {delay:.0f}s" + ) + await asyncio.sleep(delay) + elif provider: + # Heartbeat check โ€” ask the provider to verify its connection + try: + alive = await asyncio.to_thread(provider.heartbeat) + if not alive: + log.warning("Provider heartbeat failed โ€” will reconnect next cycle") + except Exception: + log.warning("Provider heartbeat exception โ€” will reconnect next cycle") + + +async def file_watch_loop(manager: ConnectionManager, config: ServerConfig): + """ + Watches state JSON files for changes and pushes events to /ws/events. + Falls back to polling if watchfiles is unavailable. + """ + watched_paths = config.watched_files + path_to_source = { + config.resolve_path(config.terminal_payload): "terminal_payload", + config.resolve_path(config.daily_signals): "daily_signals", + config.resolve_path(config.signal_lifecycle): "signal_lifecycle", + config.resolve_path(config.weekly_state): "weekly_state", + } + + if WATCHFILES_AVAILABLE: + log.info(f"File watcher started (watchfiles, {len(watched_paths)} paths)") + # Watch the project root directory, filter for our files + watch_dir = PROJECT_ROOT + + async for changes in awatch(watch_dir, debounce=1000): + for change_type, changed_path in changes: + changed = Path(changed_path) + source = path_to_source.get(changed) + if source is None: + continue + + log.info(f"File change detected: {changed.name} ({change_type.name})") + now = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%S") + + if source == "terminal_payload": + # Read which tickers were updated + data = read_json_safe(changed) + tickers = list(data.get("assets", {}).keys()) if data else [] + await manager.broadcast_event({ + "type": "payload_update", + "source": source, + "timestamp": now, + "tickers_updated": tickers, + }) + elif source in ("daily_signals", "signal_lifecycle"): + data = read_json_safe(changed) + summary = {} + if data and isinstance(data, dict): + # Count signals per state if available + all_signals = [] + for ticker_signals in data.values(): + if isinstance(ticker_signals, list): + all_signals.extend(ticker_signals) + summary = { + "total_signals": len(all_signals), + "tickers_affected": list(data.keys()), + } + await manager.broadcast_event({ + "type": "state_update", + "source": source, + "timestamp": now, + "summary": summary, + }) + else: + await manager.broadcast_event({ + "type": "state_update", + "source": source, + "timestamp": now, + "summary": {}, + }) + else: + # Fallback: polling-based file watcher + log.info("File watcher started (polling mode โ€” install watchfiles for efficiency)") + mtimes: Dict[str, float] = {} + + while True: + for path in watched_paths: + if not path.exists(): + continue + mtime = path.stat().st_mtime + key = str(path) + if key in mtimes and mtime > mtimes[key]: + source = path_to_source.get(path, path.stem) + log.info(f"File change detected (poll): {path.name}") + now = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%S") + await manager.broadcast_event({ + "type": "state_update", + "source": source, + "timestamp": now, + "summary": {}, + }) + mtimes[key] = mtime + + await asyncio.sleep(config.file_watch_debounce) + + +# ============================================================ +# 6. FASTAPI APPLICATION +# ============================================================ + +server_config = ServerConfig() +cfg_manager = ConfigManager() +manager = ConnectionManager() +start_time = time.time() + +def _get_current_prices() -> dict: + """Return latest known prices for all tickers from tick cache.""" + return dict(_latest_ticks) + + + +@asynccontextmanager +async def lifespan(app: FastAPI): + """Startup and shutdown logic.""" + # โ”€โ”€ Startup โ”€โ”€ + log.info("=" * 60) + log.info(" Quantum Terminal Data Server โ€” Starting") + + + # โ”€โ”€ Consumer startup gate (Rule C2, relaxed โ€” v19) โ”€โ”€ + # Run auth synchronously (need the result for login routing); spawn the + # slow sync_all() in a daemon thread so this lifespan returns in <1s and + # the HTTP server starts serving API routes immediately. + if CONSUMER_MODE: + gate_result = consumer_gate(skip_sync=True) + app.state.consumer_gate = gate_result + log.info( + f"Consumer gate (auth phase): ok={gate_result['ok']}, " + f"login={gate_result.get('needs_login')}, " + f"offline={gate_result.get('offline_mode')}" + ) + + # Spawn background sync only when auth succeeded (no login needed). + # If login is required, sync waits โ€” user logs in, periodic_sync.py + # or a post-login hook picks it up. + if gate_result.get("ok") and not gate_result.get("needs_login"): + import threading + def _run_initial_sync(): + try: + from data_sync_client import get_sync_client + sc = get_sync_client() + log.info("[initial-sync] thread started") + summary = sc.sync_all() + # Persist summary onto app.state so /api/sync/status can surface it. + try: + app.state.consumer_gate["sync_summary"] = summary + if summary.get("offline_mode"): + app.state.consumer_gate["offline_mode"] = True + # Rule C6 โ€” format version mismatch surfaces after sync too. + if not summary.get("format_version_ok", True): + app.state.consumer_gate["needs_update"] = True + except Exception: + pass + log.info( + f"[initial-sync] complete: " + f"synced={summary.get('synced', 0)} " + f"skipped={summary.get('skipped', 0)} " + f"offline={summary.get('offline_mode', False)}" + ) + except Exception as e: + log.error(f"[initial-sync] failed: {e}") + try: + app.state.consumer_gate["sync_summary"] = {"error": str(e)} + except Exception: + pass + threading.Thread( + target=_run_initial_sync, + daemon=True, + name="consumer-initial-sync", + ).start() + log.info("Consumer gate: initial sync spawned in background") + log.info("=" * 60) + + # Initialize providers from config (graceful โ€” server starts even if MT5 fails) + provider = None + connected = False + try: + providers = await asyncio.to_thread(cfg_manager.init_providers) + provider = cfg_manager.get_provider() # Active data provider + + if provider: + try: + connected = await asyncio.wait_for( + asyncio.to_thread(provider.connect), + timeout=15.0, # Don't hang forever if MT5 is stuck + ) + except asyncio.TimeoutError: + log.warning("MT5 connection timed out (15s) โ€” continuing without live prices") + connected = False + except Exception as e: + log.warning(f"MT5 connection failed: {e} โ€” continuing without live prices") + connected = False + + if connected: + # Resolve universe symbols through the provider + universe = cfg_manager.get_active_universe() + resolved = await asyncio.to_thread(provider.resolve_universe, universe) + log.info(f"Provider ready: {len(resolved)} symbols resolved") + else: + log.warning("Running in DEMO MODE โ€” provider failed to connect") + else: + log.warning("Running in DEMO MODE โ€” no provider configured") + except Exception as e: + log.warning(f"Provider initialization failed: {e} โ€” server will start without MT5") + provider = None + connected = False + + # Store provider reference for background tasks + app.state.provider = provider + app.state.cfg_manager = cfg_manager + + # Launch background tasks (consumer: tick loop + bar check + file watch only) + tasks = [ + asyncio.create_task(tick_loop(app.state, manager, server_config, cfg_manager)), + asyncio.create_task(focus_tick_loop(app.state, manager, cfg_manager)), # v9 + asyncio.create_task(bar_check_loop(app.state, manager, server_config, cfg_manager)), + asyncio.create_task(file_watch_loop(manager, server_config)), + ] + if provider and provider.connected: + tasks.append(asyncio.create_task(mt5_reconnect_loop(app.state, server_config))) + + # โ”€โ”€ Consumer periodic sync (Rule C1: display-only refresh, 5 min default) โ”€โ”€ + # Polls the VPS at a configurable interval and broadcasts data_sync_complete + # WS event when fresh files arrive. Frontend listens and refreshes panels. + if CONSUMER_MODE: + try: + from periodic_sync import create_periodic_sync_task + sync_task_factory = create_periodic_sync_task(manager.broadcast_event) + tasks.append(asyncio.create_task(sync_task_factory())) + log.info("Consumer periodic sync task started") + except Exception as e: + log.warning(f"Failed to start periodic sync task: {e}") + + # โ”€โ”€ Cache directory watcher (v3) โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + # Live updates whenever a file in the cache dir changes on disk. + # Covers manual drops and future automated DC uploaders. Disable via + # consumer_config.ini โ†’ [sync] cache_watcher_enabled = false. + try: + from cache_watcher import create_cache_watcher_task + watcher_factory = create_cache_watcher_task(manager.broadcast_event) + if watcher_factory is not None: + tasks.append(asyncio.create_task(watcher_factory())) + log.info("Cache watcher task started") + except Exception as e: + log.warning(f"Failed to start cache watcher task: {e}") + + # NOTE: Signal lifecycle, auto executor, options archive, trade reconciliation + # are all PRO-only features โ€” removed in consumer (Rule C1). + + log.info( + f"Data server ready โ€” " + f"{len(cfg_manager.get_active_universe())} assets, " + f"Provider {'connected' if (provider and provider.connected) else 'disconnected (demo mode)'}" + ) + log.info(f"Listening on http://{server_config.host}:{server_config.port}") + + # --- Disposable debug telemetry harness (env-gated) --- + if os.environ.get("MK_DEBUG_TELEMETRY") == "1": + import debug_telemetry + debug_telemetry.install(app) + + yield + + # โ”€โ”€ Shutdown โ”€โ”€ + log.info("Shutting down...") + for t in tasks: + t.cancel() + # Disconnect all providers + for pid, prov in cfg_manager.get_all_providers().items(): + prov.disconnect() + log.info("Data server stopped") + + +app = FastAPI( + title="Quantum Terminal Data Server", + version="1.0.0", + lifespan=lifespan, +) + +app.add_middleware( + CORSMiddleware, + allow_origins=["*"], + allow_credentials=False, + allow_methods=["*"], + allow_headers=["*"], +) + +# โ”€โ”€ Wire consumer routes (auth + sync + status) โ”€โ”€ +if CONSUMER_MODE: + wire_consumer_routes(app) + log.info("Consumer routes wired") + +# โ”€โ”€ Wire API routers (must be at module level, not inside lifespan) โ”€โ”€ +app.include_router(create_config_router(cfg_manager, manager.broadcast_event)) +app.include_router(create_execution_router(cfg_manager, app, manager.broadcast_event)) +app.include_router(create_regime_router(cfg_manager, app)) +app.include_router(create_system_router()) +app.include_router(create_account_router(manager.broadcast_event)) +app.include_router(create_freshness_router(cfg_manager)) +app.include_router(create_trade_mapper_router(app)) +app.include_router(create_pulse_router()) +app.include_router(create_chart_presets_router()) +app.include_router(create_chart_templates_router()) + +# โ”€โ”€ MT5 connection management routes (consumer settings panel) โ”€โ”€ +try: + from mt5_routes import create_mt5_router + app.include_router(create_mt5_router(cfg_manager, app)) + log.info("MT5 management routes registered (/api/mt5/*)") +except ImportError: + log.warning("mt5_routes not available โ€” MT5 settings panel won't work") + +# โ”€โ”€ Tradovate POC routes (consumer settings panel) โ”€โ”€ +# v13: standalone proof-of-work integration. Removing this block + the file +# backend/tradovate_routes.py + providers/tradovate_provider.py fully +# disables Tradovate without touching anything else. +try: + from tradovate_routes import create_tradovate_router + app.include_router(create_tradovate_router(cfg_manager)) + log.info("Tradovate POC routes registered (/api/tradovate/*)") +except ImportError as _e: + log.warning(f"tradovate_routes not available: {_e}") + +if create_logs_router is not None: + app.include_router(create_logs_router()) + +if OPTIONS_AVAILABLE: + try: + app.include_router(create_options_router()) + log.info("Options quant routes registered (/api/options/*)") + except Exception as e: + log.warning(f"Options routes failed to load: {e}") + OPTIONS_AVAILABLE = False + +# โ”€โ”€ Fallback options from sync cache (consumer mode) โ”€โ”€ +# v12: convert ETF strikes โ†’ live CFD prices server-side so frontend never +# does ratio math. Ratio = mt5_live_price / options.spot (ETF). Frozen +# per request so both MainChart's sauce lines and OptionsPanel eat the +# exact same numbers. When MT5 is offline we pass the raw payload +# through with _conversion_applied=false and the frontend can flag it. + +def _mul_options_prices_inplace(data: dict, ratio: float) -> None: + """Multiply known strike/level/price fields by ratio in-place.""" + def mul(v): + return v * ratio if isinstance(v, (int, float)) and v is not None else v + + for k in ("hvl", "max_pain", "gex_flip"): + if k in data and data[k] is not None: + data[k] = mul(data[k]) + + w = data.get("walls") or {} + for k in ("primary_call_wall", "primary_put_wall"): + if k in w and w[k] is not None: + w[k] = mul(w[k]) + for arr_key in ("call_walls", "put_walls"): + for item in (w.get(arr_key) or []): + if isinstance(item, dict) and item.get("strike") is not None: + item["strike"] = mul(item["strike"]) + + # v15: convert both swing `expected_move` and the new `expected_move_0dte` + # block (shipped by DC 2026-04-21). Same shape โ€” only `pct` is + # dimensionless and stays untouched. + for em_key in ("expected_move", "expected_move_0dte"): + em = data.get(em_key) or {} + for k in ("atm_strike", "move_up", "move_down", "straddle"): + if k in em and em[k] is not None: + em[k] = mul(em[k]) + + for iv_key in ("iv_divergence", "iv_skew"): + iv = data.get(iv_key) or {} + for k in ("atm_strike", "call_25d_strike", "put_25d_strike"): + if k in iv and iv[k] is not None: + iv[k] = mul(iv[k]) + + dh = data.get("dealer_heatmap") or {} + for k in ("max_buy_strike", "max_sell_strike"): + if k in dh and dh[k] is not None: + dh[k] = mul(dh[k]) + if isinstance(dh.get("strikes"), list): + dh["strikes"] = [mul(s) for s in dh["strikes"]] + + for gex_key in ("gex", "gex_0dte", "gex_stack"): + g = data.get(gex_key) or {} + if isinstance(g.get("strikes"), list): + g["strikes"] = [mul(s) for s in g["strikes"]] + + # v16: convert both `sauce` (swing) and the new `sauce_0dte` block + # (shipped by DC 2026-04-21). Same per-level strike conversion โ€” no + # mixing. sauce_0dte is null when today โˆ‰ chain (expected). + for sauce_key in ("sauce", "sauce_0dte"): + s_block = data.get(sauce_key) or {} + for s_key in ("volatility_gravity", "structural_integrity"): + s = s_block.get(s_key) or {} + for item in (s.get("levels") or []): + if isinstance(item, dict) and item.get("strike") is not None: + item["strike"] = mul(item["strike"]) + + +def _mt5_mid_for(ticker: str) -> Optional[float]: + """Fetch current MT5 mid price ((bid+ask)/2) for a canonical ticker. + Returns None if provider offline or tick unavailable.""" + prov = app.state.provider + if not prov or not getattr(prov, "connected", False): + return None + try: + ticks = prov.get_latest_ticks([ticker]) + t = ticks.get(ticker) if ticks else None + if t is None: + return None + bid = getattr(t, "bid", None) + ask = getattr(t, "ask", None) + if bid and ask: + return (bid + ask) / 2.0 + return getattr(t, "last", None) + except Exception: + return None + + +if not OPTIONS_AVAILABLE: + @app.get("/api/options/{ticker}/full") + async def get_options_from_sync(ticker: str, n: int = 8): + """Serve options data from consumer sync cache with ETFโ†’CFD conversion + baked in (v12). All strikes / levels / walls in the returned JSON are + in MT5 CFD price space โ€” frontend consumes as-is.""" + canonical = ticker.upper() + try: + import copy + from data_sync_client import get_sync_client + raw = get_sync_client().get_options(canonical) + if not raw: + raise HTTPException(404, f"No options data for {canonical}") + # v12 fix: sync_client returns the live cached reference โ€” mutating + # it double-multiplies strikes on every subsequent request. Deep + # copy so conversion only applies to the response, not the cache. + data = copy.deepcopy(raw) + + # Normalize keys: sync cache may use spot_etf; ensure we have an ETF spot. + etf_spot = data.get("spot_etf") or data.get("spot") + if etf_spot is None: + raise HTTPException(502, f"Options data for {canonical} missing spot price") + + if "canonical" not in data: + data["canonical"] = canonical + if "cboe_ticker" not in data: + cboe_map = {"US500": "SPY", "USTEC": "QQQ", "XAUUSD": "GLD"} + data["cboe_ticker"] = cboe_map.get(canonical) + + mt5_price = await asyncio.to_thread(_mt5_mid_for, canonical) + levels_space = data.get("levels_space") + + if levels_space == "broker": + # Producer (v2.0+) pre-multiplied levels to broker space; consumer no-op. + if mt5_price: + data["spot"] = mt5_price + data["_conversion_applied"] = True + data["_conversion_source"] = "broker_native" + elif mt5_price and etf_spot > 0: + ratio = mt5_price / etf_spot + _mul_options_prices_inplace(data, ratio) + # Always preserve the raw ETF spot; replace top-level spot with the + # live MT5 price so frontend sees CFD space consistently. + data["spot_etf"] = etf_spot + data["spot"] = mt5_price + data["_conversion_applied"] = True + data["_conversion_ratio"] = ratio + data["_conversion_source"] = "mt5_live" + else: + # No MT5 price available โ€” fall through with raw ETF values so + # the panel still renders, but flagged so the UI can warn. + if "spot" not in data: + data["spot"] = etf_spot + data["_conversion_applied"] = False + data["_conversion_source"] = "mt5_unavailable" + return data + except HTTPException: + raise + except Exception as e: + log.warning(f"options fallback failed for {canonical}: {e}") + raise HTTPException(404, f"No options data for {canonical}") + + log.info("Options fallback route registered (sync cache, v12 CFD-converted)") + + +# โ”€โ”€ Macro Rotation Endpoints (Money Flow Tabs 2 & 3) โ”€โ”€ +if MACRO_DATA_AVAILABLE: + @app.get("/api/macro/sectors") + async def api_macro_sectors(refresh: bool = False): + try: + from data_sync_client import get_sync_client + data = get_sync_client().get_macro_sectors() + if data: + return data + except Exception as e: + log.warning(f"[MACRO] macro_sectors failed: {e}") + raise HTTPException(404, "No sector data available") + + @app.get("/api/macro/countries") + async def api_macro_countries(refresh: bool = False): + try: + from data_sync_client import get_sync_client + data = get_sync_client().get_macro_countries() + if data: + return data + except Exception as e: + log.warning(f"[MACRO] macro_countries failed: {e}") + raise HTTPException(404, "No country data available") + + @app.get("/api/macro/flows") + async def api_macro_flows(period: str = "1W"): + try: + from data_sync_client import get_sync_client + data = get_sync_client().get_macro_flows() + if data: + key = "4W" if period.upper() == "4W" else "1W" + flows_obj = data.get(key) + if flows_obj and "flows" in flows_obj: + return flows_obj + except Exception as e: + log.warning(f"[MACRO] macro_flows failed: {e}") + raise HTTPException(404, "No country flow data available") + + log.info("Macro rotation routes registered (/api/macro/sectors, /api/macro/countries, /api/macro/flows)") +else: + log.info("macro_data not available -- yfinance may not be installed") + +# -- Money Flow from sync cache (consumer mode) -- +@app.get("/api/fundamental/money-flow") +async def get_money_flow(period: str = "macro"): + """Serve capital flow data from consumer sync cache.""" + try: + from data_sync_client import get_sync_client + data = get_sync_client().get_global("macro_sector_flows") + if data: + # Data is keyed by period (1W, 4W). Extract requested period. + period_map = {"macro": "macro", "weekly": "weekly", "1w": "macro", "4w": "weekly"} + key = period_map.get(period.lower(), period) + if key in data: + return data[key] + # If only one period exists, return it + for k, v in data.items(): + if isinstance(v, dict) and "flows" in v: + return v + return data + except Exception: + pass + raise HTTPException(404, "No money flow data available") + + +# -- Fundamental state (macro regime, liquidity, yields, COT, scores) -- +@app.get("/api/fundamental_state") +async def get_fundamental_state(): + """Serve fundamental_state.json from consumer sync cache (Rule C1: read-only).""" + try: + from data_sync_client import get_sync_client + data = get_sync_client().get_fundamentals() + if data: + return data + except Exception as e: + log.warning(f"[FUNDAMENTAL] fundamental_state failed: {e}") + raise HTTPException(404, "No fundamental state data available") + + + + +# โ”€โ”€ REST ENDPOINTS โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + +@app.get("/api/health") +async def health(): + """Server health check โ€” backwards-compatible top-level fields plus + a v11+ `components` block where each subsystem reports its own + status. Frontends that want detailed diagnostics read components; + legacy clients that just check `status == "ok"` still work.""" + uptime = time.time() - start_time + provider = app.state.provider + + # โ”€โ”€ Component health โ”€โ”€ + components: Dict[str, Any] = {} + + # MT5 / data provider + mt5_connected = bool(provider and provider.connected) + components["mt5"] = { + "status": "ok" if mt5_connected else "disconnected", + "type": provider.provider_type if provider else None, + "label": provider.label if provider else None, + "connected": mt5_connected, + "active_symbols": len(cfg_manager.get_active_universe()), + } + + # License / auth โ€” Quantum Terminal: always free + components[\"license\"] = { + \"status\": \"ok\", + \"user_email\": \"free@quantumterminal\", + \"subscription_status\": \"active\", + \"subscription_type\": \"free\", + \"offline\": False, + } + + # Sync status + try: + from data_sync_client import get_sync_client as _gsc + _sc = _gsc() + sync_in_progress = bool(getattr(_sc, "_sync_in_progress", False)) + sync_total = int(getattr(_sc, "_sync_total", 0)) + sync_done = int(getattr(_sc, "_sync_done", 0)) + sync_errors = list(getattr(_sc, "_sync_errors", []) or []) + last_synced = getattr(_sc, "last_synced_at", None) + # Status: error if errors and not in progress, syncing if in progress, ok otherwise + if sync_in_progress: + sync_status = "syncing" + elif sync_errors: + sync_status = "degraded" + else: + sync_status = "ok" + components["sync"] = { + "status": sync_status, + "in_progress": sync_in_progress, + "progress": (sync_done, sync_total), + "last_synced_at": last_synced, + "format_version_ok": bool(getattr(_sc, "format_version_ok", True)), + "errors_count": len(sync_errors), + "offline": bool(getattr(_sc, "is_offline", False)), + } + except Exception as e: + components["sync"] = {"status": "error", "error": str(e)} + + # WebSocket clients + components["websockets"] = { + "status": "ok", + "price_clients": manager.price_client_count, + "event_clients": manager.event_client_count, + } + + # Build version + version = None + try: + from pathlib import Path as _Path + for vp in [_Path(__file__).parent / "VERSION", + _Path(__file__).parent.parent.parent / "VERSION", + _Path(__file__).parent.parent / "VERSION"]: + if vp.exists(): + version = vp.read_text(encoding="utf-8").strip() + break + except Exception: + pass + + # Roll up overall status โ€” error if any component errored, degraded + # if any is in a non-ok-but-recoverable state, otherwise ok. + overall = "ok" + for c in components.values(): + s = c.get("status") + if s == "error": + overall = "error" + break + if s in ("disconnected", "auth_required", "degraded", "syncing"): + overall = "degraded" + + return { + # Back-compat top-level fields (v1 and earlier readers) + "status": "ok" if overall != "error" else "error", + "mt5_connected": mt5_connected, + "uptime_seconds": round(uptime, 1), + "active_symbols": len(cfg_manager.get_active_universe()), + "ws_price_clients": manager.price_client_count, + "ws_event_clients": manager.event_client_count, + "universe": cfg_manager.get_display_order(), + "signal_engine_available": False, + "active_signals": 0, + "provider": { + "type": provider.provider_type if provider else None, + "label": provider.label if provider else None, + "connected": mt5_connected, + }, + # v11+ structured component view + "overall": overall, + "version": version, + "components": components, + } + + +@app.get("/api/ticks") +async def get_all_ticks(): + """ + Latest tick for every symbol in the universe. + Used by the frontend on initial load to seed prices + (before the WebSocket starts streaming). + """ + provider = app.state.provider + if not provider or not provider.connected: + return {"ticks": {}, "note": "MT5 not connected"} + universe = cfg_manager.get_active_universe() + ticks = await asyncio.to_thread(provider.get_latest_ticks, universe) + return { + "ticks": {t: tick.to_dict() for t, tick in ticks.items()}, + } + + +@app.post("/api/focus/{ticker}") +async def post_focus(ticker: str): + """v9: Tell the backend which symbol the user is actively watching. + The focus_tick_loop polls only this symbol at FOCUS_TICK_INTERVAL when + USE TICK DATA is on, giving the chart MT5-style live tick movement + without paying universe-size MT5 polling cost.""" + set_focused_symbol(ticker) + return {"focused": get_focused_symbol(), "interval": FOCUS_TICK_INTERVAL} + + +@app.delete("/api/focus") +async def delete_focus(): + """v9: Clear the focused symbol (focus_tick_loop goes idle).""" + set_focused_symbol(None) + return {"focused": None} + + +@app.get("/api/symbol-info/{ticker}") +async def get_symbol_info(ticker: str): + """ + Real broker symbol metadata โ€” tick_value, contract_size, etc. + Used by OrderTicket for accurate P&L preview. + """ + provider = app.state.provider + if not provider or not provider.connected: + raise HTTPException(503, "Provider not connected") + canonical = ticker.upper() + info = await asyncio.to_thread(provider.get_symbol_info, canonical) + if info is None: + raise HTTPException(404, f"Symbol info not found for {canonical}") + return info.to_dict() + + +@app.get("/api/symbol-info") +async def get_all_symbol_info(): + """Batch symbol info for the entire universe.""" + provider = app.state.provider + if not provider or not provider.connected: + return {"symbols": {}, "note": "Provider not connected"} + universe = cfg_manager.get_active_universe() + infos = await asyncio.to_thread(provider.get_all_symbol_info, universe) + return { + "symbols": {t: info.to_dict() for t, info in infos.items()}, + } + + +@app.get("/api/payload") +async def get_payload(): + """Full terminal_payload.json contents. Returns empty in consumer mode.""" + path = server_config.resolve_path(server_config.terminal_payload) + data = read_json_safe(path) + if data is None: + # Consumer mode: no local payload file โ€” data comes via sync cache + return {"assets": {}, "note": "Consumer mode โ€” data served via /api/cones and /api/sync"} + return data + + +@app.get("/api/payload/{ticker}") +async def get_payload_ticker(ticker: str): + """Single ticker's payload entry.""" + path = server_config.resolve_path(server_config.terminal_payload) + data = read_json_safe(path) + if data is None: + return {"note": "Consumer mode โ€” use /api/sync/cones/{ticker}"} + assets = data.get("assets", {}) + canonical = ticker.upper() + if canonical not in assets: + return {"note": f"No payload entry for {canonical}"} + return assets[canonical] + + +@app.get("/api/signals") +async def get_signals(): + """All signals from daily_signals.json.""" + path = server_config.resolve_path(server_config.daily_signals) + data = read_json_safe(path) + if data is None: + return {"signals": {}, "note": "No signals file found"} + return data + + +@app.get("/api/signals/{ticker}") +async def get_signals_ticker(ticker: str): + """Signals for one ticker.""" + path = server_config.resolve_path(server_config.daily_signals) + data = read_json_safe(path) + if data is None: + raise HTTPException(404, "daily_signals.json not found") + canonical = ticker.upper() + if canonical not in data: + return {"ticker": canonical, "signals": []} + return {"ticker": canonical, "signals": data[canonical]} + + +# โ”€โ”€ Stress Lab (v10) โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +# Phase 1: consume DC's stress_lab JSON files from the standard AppData +# cache. Files land there either via data_sync_client (future wiring) or +# manual copy for testing. No computation, no synthesis โ€” display-only (C1). +# +# Expected files in %APPDATA%\QuantumTerminal\cache\ : +# _stress_lab_index.json โ€” watchlist +# {TICKER}_stress_lab.json โ€” per-asset thesis + + +def _stress_lab_cache_dir() -> Path: + """Same cache dir the sync_client writes to โ€” flat layout.""" + from data_sync_client import _get_cache_dir + return _get_cache_dir() + + +@app.get("/api/stress_lab/index") +async def get_stress_lab_index(): + """Watchlist โ€” list of assets with conviction + spot + one-liner.""" + path = _stress_lab_cache_dir() / "_stress_lab_index.json" + data = read_json_safe(path) + if data is None: + raise HTTPException(404, "Stress Lab index not available yet โ€” waiting on first sync") + return data + + +@app.get("/api/stress_lab/{ticker}") +async def get_stress_lab_thesis(ticker: str): + """Full per-asset thesis JSON produced by DC stress_lab_engine.""" + canonical = ticker.upper() + path = _stress_lab_cache_dir() / f"{canonical}_stress_lab.json" + data = read_json_safe(path) + if data is None: + raise HTTPException(404, f"No Stress Lab thesis for {canonical}") + return data + + +# v11: Historical backtest positions for the "last filled signal" replay panel. +@app.get("/api/stress_lab/backtest/positions") +async def get_stress_lab_backtest_positions(): + path = _stress_lab_cache_dir() / "_stress_lab_backtest_positions.json" + data = read_json_safe(path) + if data is None: + raise HTTPException(404, "Backtest positions not available yet") + return data + + +# v11: Historical bar range โ€” replay chart on Stress Lab last-signal panel. +# `from` is a Python keyword, so we accept it via Query(alias="from"). +from fastapi import Query as _Query +from datetime import datetime as _dt, timezone as _tz + +@app.get("/api/bars_range/{ticker}") +async def get_bars_range( + ticker: str, + timeframe: str = "M15", + from_: str = _Query(..., alias="from"), + to: str = _Query(...), +): + """OHLCV bars between two ISO-8601 UTC datetimes. + Used by the Stress Lab last-signal replay chart.""" + canonical = ticker.upper() + provider = app.state.provider + if not provider or not provider.connected: + raise HTTPException(503, "No data provider connected") + try: + f_dt = _dt.fromisoformat(from_.replace("Z", "+00:00")) + t_dt = _dt.fromisoformat(to.replace("Z", "+00:00")) + except ValueError as e: + raise HTTPException(400, f"bad ISO datetime: {e}") + if f_dt.tzinfo is None: f_dt = f_dt.replace(tzinfo=_tz.utc) + if t_dt.tzinfo is None: t_dt = t_dt.replace(tzinfo=_tz.utc) + if not hasattr(provider, "get_bars_range"): + raise HTTPException(501, "provider does not support range fetch") + bars = await asyncio.to_thread( + provider.get_bars_range, canonical, timeframe, f_dt, t_dt + ) + if not bars: + return {"bars": [], "ticker": canonical, "timeframe": timeframe, + "from": from_, "to": to} + # Normalize to the shape frontend expects (LWC candles). + return { + "ticker": canonical, + "timeframe": timeframe, + "from": from_, + "to": to, + "bars": [{ + "time": b.time, "open": b.open, "high": b.high, + "low": b.low, "close": b.close, "volume": b.volume, + } for b in bars], + } + + +@app.get("/api/bands/{ticker}") +async def get_bands(ticker: str): + """Probability bands data for a ticker. Falls back to sync cache.""" + canonical = ticker.upper() + # Try PRO local file first + bands_dir = server_config.resolve_path(server_config.bands_data_dir) + bands_file = bands_dir / f"{canonical}_bands.json" + data = read_json_safe(bands_file) + if data: + # v26: stamp file mtime for staleness detection on the frontend. + try: + data["_last_modified"] = bands_file.stat().st_mtime + except Exception: + pass + return data + + # Fallback 1 (v2 โ€” Data Center): standalone {TICKER}_bands.json in sync cache. + try: + from data_sync_client import get_sync_client + client = get_sync_client() + bands_data = client.get_bands(canonical) + if bands_data: + # v26: best-effort mtime from the sync cache file. Naming + # patterns observed: _bands.json or GLOBAL__bands.json. + try: + cache_dir = getattr(client, "cache_dir", None) + if cache_dir is not None: + from pathlib import Path as _P + cache_dir_p = _P(cache_dir) if not isinstance(cache_dir, _P) else cache_dir + for name in (f"{canonical}_bands.json", + f"GLOBAL_{canonical.lower()}_bands.json"): + candidate = cache_dir_p / name + if candidate.exists(): + bands_data["_last_modified"] = candidate.stat().st_mtime + break + except Exception: + pass + return bands_data + except Exception: + pass + + # Fallback 2 (legacy): bands nested inside the cones sync blob. + # Retained so any cache still holding pre-transition files keeps working. + try: + sync_data = client.get_cones(canonical) + if sync_data and "bands" in sync_data: + return sync_data["bands"] + except Exception: + pass + + raise HTTPException(404, f"No bands data for {canonical}") + + +def _sanitize_json(data): + """Replace NaN/Infinity with None โ€” these are not valid JSON.""" + raw = json.dumps(data, allow_nan=True, default=str) + raw = raw.replace(": NaN", ": null").replace(":NaN", ":null") + raw = raw.replace(": Infinity", ": null").replace(":Infinity", ":null") + raw = raw.replace(": -Infinity", ": null").replace(":-Infinity", ":null") + return json.loads(raw) + + +@app.get("/api/state") +async def get_state(): + """Weekly state (regime, anchors, WF params).""" + path = server_config.resolve_path(server_config.weekly_state) + data = read_json_safe(path) + if data is None: + return {"assets": {}, "note": "No weekly state found โ€” run orchestrator weekly tier"} + return _sanitize_json(data) + + +@app.get("/api/bars/{ticker}") +async def get_bars(ticker: str, timeframe: str = "M15", count: int = 200): + """ + Fetch historical bars directly from MT5. + Used for initial chart load and timeframe switches. + """ + canonical = ticker.upper() + if canonical not in cfg_manager.get_active_universe(): + raise HTTPException(404, f"Ticker {canonical} not in universe") + + provider = app.state.provider + if not provider or not provider.connected: + raise HTTPException(503, "No data provider connected") + + bars = await asyncio.to_thread(provider.get_bars, canonical, timeframe, count) + if not bars: + raise HTTPException(404, f"No bar data for {canonical} {timeframe}") + + return { + "ticker": canonical, + "timeframe": timeframe, + "count": len(bars), + "bars": [b.to_dict() for b in bars], + } + + +@app.get("/api/lifecycle") +async def get_lifecycle(): + """Signal lifecycle state.""" + path = server_config.resolve_path(server_config.signal_lifecycle) + data = read_json_safe(path) + if data is None: + return {"signals": {}, "note": "No lifecycle file found"} + return data + + + +# REMOVED: /api/signals/evaluate, /api/orchestrator, /api/calculate (Rule C1) + + +@app.get("/api/anchors/{ticker}") +async def get_anchors(ticker: str): + """Institutional anchors + extremes. Falls back to sync cache.""" + canonical = ticker.upper() + # Try PRO local file first + anchors_path = PROJECT_ROOT / "terminal_anchors.json" + data = read_json_safe(anchors_path) + if data and canonical in data: + return data[canonical] + + # Fallback: consumer sync cache โ€” anchors are inside the cones response + try: + from data_sync_client import get_sync_client + sync_data = get_sync_client().get_cones(canonical) + if sync_data: + result = {} + if "anchors" in sync_data: + result["anchors"] = sync_data["anchors"] + if "extremes" in sync_data: + result["extremes"] = sync_data["extremes"] + if result: + return result + except Exception: + pass + + raise HTTPException(404, f"No anchor data for {canonical}") + + +@app.get("/api/anchors") +async def get_all_anchors(): + """All tickers' anchor data. Falls back to sync cache.""" + anchors_path = PROJECT_ROOT / "terminal_anchors.json" + data = read_json_safe(anchors_path) + if data: + return data + # Fallback: consumer sync cache + try: + from data_sync_client import get_sync_client + client = get_sync_client() + result = {} + for ticker in client.synced_tickers: + sync_data = client.get_cones(ticker) + if sync_data: + entry = {} + if "anchors" in sync_data: + entry["anchors"] = sync_data["anchors"] + if "extremes" in sync_data: + entry["extremes"] = sync_data["extremes"] + if entry: + result[ticker] = entry + if result: + return result + except Exception: + pass + return {"anchors": {}, "note": "No anchor data found"} + + +@app.get("/api/regime_live/{ticker}") +async def get_regime_live_stub(ticker: str, timeframe: str = "M15", bars: int = 500): + """Stub โ€” regime_live is a PRO-only endpoint (Rule C1). + Returns empty payload so the frontend stops 404-retrying.""" + return { + "ticker": ticker.upper(), + "timeframe": timeframe, + "data": [], + "note": "regime_live unavailable in consumer build" + } + + +@app.get("/api/cones/{ticker}") +async def get_cones(ticker: str): + """Probability cone data for a ticker. Falls back to sync cache in consumer mode. + Also merges GEX cones from the options sync cache (gex_cones lives in + options JSON, not cones JSON, on the producer side).""" + canonical = ticker.upper() + # Try PRO local file first + cones_file = server_config.resolve_path("terminal_cones.json") + data = read_json_safe(cones_file) + if data and canonical in data: + return data[canonical] + + # Fallback: consumer sync cache (Rule C5) + result = None + try: + from data_sync_client import get_sync_client + client = get_sync_client() + sync_data = client.get_cones(canonical) + if sync_data and "cones" in sync_data: + result = dict(sync_data.get("cones", {})) + elif sync_data: + result = dict(sync_data) + + # Merge gex_cones from the options sync cache. The producer writes + # gex_now / gex_weekly_1 / gex_weekly_2 into the options JSON, not + # the cones JSON. Their shape (median, sd1_high, sd1_low, sd2_high, + # sd2_low, sd3_high, sd3_low, dates) matches the standard cone shape + # the frontend renderer expects, so a flat merge is enough. + opt_data = client.get_options(canonical) + if opt_data and isinstance(opt_data.get("gex_cones"), dict): + if result is None: + result = {} + for k, v in opt_data["gex_cones"].items(): + # Don't clobber any pre-existing key (cones JSON wins if both exist) + if k not in result: + result[k] = v + except Exception: + pass + + if result: + return result + raise HTTPException(404, f"No cone data for {canonical}") + + +@app.get("/api/quarterly_cones/{ticker}") +async def get_quarterly_cones(ticker: str): + """Quarterly cone forecast (183-day+ horizon, 4 anchors ร— 2 models). + v6 โ€” Reads {TICKER}_quarterly_cones.json from the consumer cache. + v7 โ€” Also handles the Data Center's GLOBAL__quarterly_cones.json + naming convention (same legacy pattern as historical_cones / scalp_bands). + Routes through the sync client first (in-memory store populated by + resolve_cache_filename), then falls back to direct file reads. + Shape: median + sd1/2/3 high/low + dates per model, keys prefixed + gbm_quarter_*, mjd_quarter_*.""" + import json as _json + canonical = ticker.upper() + try: + from data_sync_client import get_sync_client, _get_cache_dir + client = get_sync_client() + # (1) in-memory store (populated by sync_all + _load_all_from_cache) + data = client._get_data(canonical, "quarterly_cones") + if data: + return data + # (2) direct disk fallback โ€” try both naming conventions explicitly, + # in case the cache scan hasn't run yet for this session. + cache_dir = _get_cache_dir() + for path in ( + cache_dir / f"{canonical}_quarterly_cones.json", + cache_dir / f"GLOBAL_{canonical.lower()}_quarterly_cones.json", + ): + if path.is_file(): + return _json.loads(path.read_text(encoding="utf-8")) + except Exception as e: + log.warning(f"quarterly_cones load failed for {canonical}: {e}") + raise HTTPException(404, f"No quarterly cone data for {canonical}") + + +# v14: TEMPORARY dev-only route. Reads the producer's bands-replay JSON +# directly from D:\MK_DATA_CENTER\temp_out so the replay view can +# render the new per-day anchor schema before the real sync pipeline +# exists. Delete this function (and its import side-effects) to revert. +@app.get("/api/bands_replay/{ticker}") +async def get_bands_replay(ticker: str): + """Bands-replay prototype โ€” reads {TICKER}_bands_replay.json from + D:\\MK_DATA_CENTER\\temp_out (dev path only). Schema matches + /api/historical_cones but with a 'daily' bucket, a 'head' warm-start + block, and fusion_instructions metadata. Returns 404 if the file + isn't present (non-XAUUSD tickers, production boxes, etc).""" + import json as _json + from pathlib import Path as _Path + canonical = ticker.upper() + test_dir = _Path(r"D:\MK_DATA_CENTER\temp_out") + path = test_dir / f"{canonical}_bands_replay.json" + if not path.is_file(): + raise HTTPException(404, f"No bands_replay data for {canonical}") + try: + return _json.loads(path.read_text(encoding="utf-8")) + except Exception as e: + log.warning(f"bands_replay load failed for {canonical}: {e}") + raise HTTPException(500, f"bands_replay parse failed for {canonical}") + + +@app.get("/api/scalp_bands/{ticker}") +async def get_scalp_bands(ticker: str): + """Scalp bands (5-min anchor + per-horizon projections) for the MICRO + toggle in the BANDS dropdown. Served from the sync cache. + Source file: GLOBAL__scalp_bands.json (resolver-routed).""" + canonical = ticker.upper() + try: + from data_sync_client import get_sync_client + client = get_sync_client() + data = client._get_data(canonical, "scalp_bands") + if data: + return data + except Exception: + pass + raise HTTPException(404, f"No scalp bands data for {canonical}") + + +@app.get("/api/historical_cones/{ticker}") +async def get_historical_cones(ticker: str): + """Historical cone anchors (weekly + monthly) for the REPLAY sub-tab. + Each anchor date maps to a dict with gbm / mjd / bates model payloads + (median + sd1/2/3 high/low + dates). Served from the sync cache. + v17: mirrors the quarterly_cones endpoint โ€” falls back to direct file + reads (both `{TICKER}_*.json` and `GLOBAL__*.json` shapes) + when the in-memory store hasn't been populated yet. Without this, a + cold-boot or stalled-sync session would 404 on tickers that have a + file-on-disk but no store entry yet (e.g. EURUSD). + """ + import json as _json + canonical = ticker.upper() + try: + from data_sync_client import get_sync_client, _get_cache_dir + client = get_sync_client() + # (1) in-memory store (populated by sync_all + _load_all_from_cache) + data = client._get_data(canonical, "historical_cones") + if data: + return data + # (2) direct disk fallback โ€” handles legacy GLOBAL_ naming. + cache_dir = _get_cache_dir() + for path in ( + cache_dir / f"{canonical}_historical_cones.json", + cache_dir / f"GLOBAL_{canonical.lower()}_historical_cones.json", + ): + if path.is_file(): + return _json.loads(path.read_text(encoding="utf-8")) + except Exception as e: + log.warning(f"historical_cones load failed for {canonical}: {e}") + raise HTTPException(404, f"No historical cone data for {canonical}") + + +@app.get("/api/cones") +async def get_all_cones(): + """All computed cone data. Falls back to sync cache.""" + cones_file = server_config.resolve_path("terminal_cones.json") + data = read_json_safe(cones_file) + if data: + return data + # Fallback: consumer sync cache + try: + from data_sync_client import get_sync_client + client = get_sync_client() + result = {} + for ticker in client.synced_tickers: + sync_data = client.get_cones(ticker) + if sync_data and "cones" in sync_data: + result[ticker] = sync_data["cones"] + elif sync_data: + result[ticker] = sync_data + if result: + return result + except Exception: + pass + return {"note": "No cones data found"} + + +# โ”€โ”€ PROBABILITY FIELD ENDPOINT โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + +@app.get("/api/probability-field/{ticker}") +async def get_probability_field(ticker: str): + """ + Probability field (v2.0) โ€” consumer version. + + Serves the full pre-computed payload from sync cache. + Both modes (month_curr, month_prev) are bundled in one response โ€” + the frontend picks which mode to render from the cached payload. + + Rule C1: no live computation. Rule C5: cache-only serving. + """ + canonical = ticker.upper() + + try: + from data_sync_client import get_sync_client + probfield = get_sync_client().get_probfield(canonical) + except Exception as e: + log.warning(f"[probfield] get_probfield failed for {canonical}: {e}") + probfield = None + + if not probfield: + raise HTTPException( + 404, + f"No probability field data for {canonical}. " + f"Data may not have synced yet." + ) + + # Return the full v2.0 payload unchanged. + # Frontend reads probfield.modes.month_curr / month_prev directly. + return probfield + +# โ”€โ”€ PATH OUTCOME FOREST (PRO) โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +# v20: serve the per-ticker path_forest forecast from sync cache. Producer +# writes GLOBAL__path_forest.json once live; meanwhile DC has shipped +# 5 scenario fixtures (trend / range / stress / mixed / miscal) so the +# frontend can develop against varied conditions. The `scenario` query param +# picks among the fixtures; live filename takes priority when present. + +@app.get("/api/forest") +async def list_forest_assets(): + """ + v21: List every ticker that has a path_forest forecast in cache. + Used by the Stress Lab โ†’ Forest Path sub-tab so its asset selector + auto-populates with whatever DC has shipped instead of relying on a + hardcoded probe list. Returns ISO sorted upper-case tickers. + """ + import re + from data_sync_client import _get_cache_dir + cache_dir = _get_cache_dir() + if not cache_dir or not cache_dir.exists(): + return {"tickers": []} + pattern = re.compile(r"^GLOBAL_([a-z0-9_]+)_path_forest(?:_[a-z]+)?\.json$") + found = set() + try: + for p in cache_dir.iterdir(): + if not p.is_file(): + continue + m = pattern.match(p.name) + if m: + found.add(m.group(1).upper()) + except Exception as e: + log.warning(f"[forest] index scan failed: {e}") + return {"tickers": sorted(found)} + + +@app.get("/api/forest/{ticker}") +async def get_forest(ticker: str, scenario: str = "mixed"): + """ + Path Outcome Forest forecast (PRO feature). + Reads the cached path_forest JSON for the given ticker. + Rule C1: no live computation. Rule C5: cache-only. + """ + from data_sync_client import _get_cache_dir + canonical = (ticker or "").lower() + if not canonical: + raise HTTPException(400, "ticker required") + + valid = {"trend", "range", "stress", "mixed", "miscal"} + if scenario not in valid: + scenario = "mixed" + + cache_dir = _get_cache_dir() + candidates = [ + cache_dir / f"GLOBAL_{canonical}_path_forest.json", + cache_dir / f"GLOBAL_{canonical}_path_forest_{scenario}.json", + ] + for p in candidates: + try: + if p.exists(): + with open(p, "r", encoding="utf-8") as f: + return json.load(f) + except Exception as e: + log.warning(f"[forest] read failed for {p}: {e}") + continue + + raise HTTPException( + 404, + f"No path_forest data for {ticker} (tried scenario '{scenario}'). " + f"Producer may not have fired yet." + ) + +# โ”€โ”€ MANUAL CONE ENDPOINTS โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + + +# REMOVED: /api/cones/manual (Rule C1 โ€” computation endpoint) + + +@app.get("/api/performance/summary") +async def get_performance_summary(days: int = 90): + """ + Comprehensive performance summary from auto_executor_log.json. + Includes win/loss metrics in $ and pips, gate funnel, daily timeline. + Used by the PerformanceDashboard component. + Note: win/loss are EXPECTED (based on TP/SL), not realised (no outcome tracking yet). + """ + log_path = PROJECT_ROOT / "auto_executor_log.json" + if not log_path.exists(): + return {"total": 0, "days": days} + + try: + with open(log_path) as f: + all_records = json.load(f) + except Exception: + return {"total": 0, "error": "Failed to read log"} + + from datetime import timedelta + cutoff = (datetime.now(timezone.utc) - timedelta(days=days)).isoformat() + records = [r for r in all_records if r.get("timestamp", "") >= cutoff] + + if not records: + return {"total": 0, "days": days} + + # โ”€โ”€ Gate funnel (all records) โ”€โ”€ + funnel: dict = {} + for r in records: + action = r.get("action", "unknown") + gate = r.get("gate", action) + key = gate if action in ("skipped", "blocked") else action + funnel[key] = funnel.get(key, 0) + 1 + + # โ”€โ”€ Executed trades ONLY for P&L metrics โ”€โ”€ + executed = [r for r in records if r.get("action") == "executed"] + # Dry-run trades for activity display only + dry_run = [r for r in records if r.get("action") == "dry_run"] + trades = executed + dry_run # combined for non-P&L distributions + + ne = len(executed) # count of real executions + + # โ”€โ”€ Win/loss helpers (expected, based on TP/SL distances) โ”€โ”€ + def _win_usd(r): + return round(r.get("risk_usd", 0) * r.get("risk_reward", 0), 2) + + def _loss_usd(r): + return round(r.get("risk_usd", 0), 2) + + def _win_pips(r): + entry = r.get("fill_price") or r.get("entry_price", 0) + target = r.get("take_profit", 0) + if not entry or not target: + return 0.0 + return _to_pips(r.get("ticker", ""), abs(target - entry)) + + def _loss_pips(r): + entry = r.get("fill_price") or r.get("entry_price", 0) + sl = r.get("stop_loss", 0) + if not entry or not sl: + return 0.0 + return _to_pips(r.get("ticker", ""), abs(entry - sl)) + + avg_win_usd = round(sum(_win_usd(r) for r in executed) / ne, 2) if ne else 0 + avg_loss_usd = round(sum(_loss_usd(r) for r in executed) / ne, 2) if ne else 0 + avg_win_pips = round(sum(_win_pips(r) for r in executed) / ne, 1) if ne else 0 + avg_loss_pips = round(sum(_loss_pips(r) for r in executed) / ne, 1) if ne else 0 + total_risk_usd = round(sum(_loss_usd(r) for r in executed), 2) + avg_risk_pct = round(sum(r.get("risk_pct", 0) for r in executed) / ne, 4) if ne else 0 + avg_conf = round(sum(r.get("confidence", 0) for r in executed) / ne, 4) if ne else 0 + avg_rr = round(sum(r.get("risk_reward", 0) for r in executed) / ne, 2) if ne else 0 + + # โ”€โ”€ By ticker (executed only) โ”€โ”€ + by_ticker: dict = {} + for r in executed: + t = r.get("ticker", "?") + by_ticker.setdefault(t, { + "count": 0, "risk_usd": 0.0, "win_usd": 0.0, + "win_pips": 0.0, "loss_pips": 0.0, "confidence": [], + }) + by_ticker[t]["count"] += 1 + by_ticker[t]["risk_usd"] = round(by_ticker[t]["risk_usd"] + _loss_usd(r), 2) + by_ticker[t]["win_usd"] = round(by_ticker[t]["win_usd"] + _win_usd(r), 2) + by_ticker[t]["win_pips"] = round(by_ticker[t]["win_pips"] + _win_pips(r), 1) + by_ticker[t]["loss_pips"] = round(by_ticker[t]["loss_pips"] + _loss_pips(r), 1) + by_ticker[t]["confidence"].append(r.get("confidence", 0)) + for t, d in by_ticker.items(): + c = d.pop("confidence") + cnt = d["count"] + d["avg_confidence"] = round(sum(c) / len(c), 4) if c else 0 + d["avg_win_usd"] = round(d["win_usd"] / cnt, 2) if cnt else 0 + d["avg_loss_usd"] = round(d["risk_usd"] / cnt, 2) if cnt else 0 + d["avg_win_pips"] = round(d["win_pips"] / cnt, 1) if cnt else 0 + d["avg_loss_pips"] = round(d["loss_pips"] / cnt, 1) if cnt else 0 + + # โ”€โ”€ By signal type (executed only) โ”€โ”€ + by_type: dict = {} + for r in executed: + st = r.get("signal_type", "?") + by_type[st] = by_type.get(st, 0) + 1 + + # โ”€โ”€ By calibration grade (executed only) โ”€โ”€ + by_grade: dict = {} + for r in executed: + g = r.get("calibration_grade", "?") or "?" + by_grade[g] = by_grade.get(g, 0) + 1 + + # โ”€โ”€ By regime (executed only) โ”€โ”€ + by_regime: dict = {} + for r in executed: + reg = r.get("regime", "?") or "?" + by_regime[reg] = by_regime.get(reg, 0) + 1 + + # โ”€โ”€ By direction (executed only) โ”€โ”€ + by_direction: dict = {"long": 0, "short": 0} + for r in executed: + d = r.get("direction", "") + if d in by_direction: + by_direction[d] += 1 + + # โ”€โ”€ Daily timeline (all records for activity view) โ”€โ”€ + daily: dict = {} + for r in records: + ts = r.get("timestamp", "")[:10] + if not ts: + continue + if ts not in daily: + daily[ts] = {"executed": 0, "dry_run": 0, "skipped": 0, "blocked": 0, "awaiting": 0} + action = r.get("action", "") + if action == "executed": daily[ts]["executed"] += 1 + elif action == "dry_run": daily[ts]["dry_run"] += 1 + elif action == "skipped": daily[ts]["skipped"] += 1 + elif action == "blocked": daily[ts]["blocked"] += 1 + elif action == "awaiting_confirmation": daily[ts]["awaiting"] += 1 + timeline = [{"date": d, **counts} for d, counts in sorted(daily.items())] + + # โ”€โ”€ Skipped by gate โ”€โ”€ + by_gate: dict = {} + for r in records: + if r.get("action") == "skipped": + g = r.get("gate", "unknown") + by_gate[g] = by_gate.get(g, 0) + 1 + + return { + "days": days, + "total_records": len(records), + "total_executed": ne, + "total_dry_run": len(dry_run), + "avg_confidence": avg_conf, + "avg_risk_reward": avg_rr, + "avg_win_usd": avg_win_usd, + "avg_loss_usd": avg_loss_usd, + "avg_win_pips": avg_win_pips, + "avg_loss_pips": avg_loss_pips, + "total_risk_usd": total_risk_usd, + "avg_risk_pct": avg_risk_pct, + "funnel": funnel, + "by_gate": by_gate, + "timeline": timeline, + "by_ticker": by_ticker, + "by_type": by_type, + "by_grade": by_grade, + "by_regime": by_regime, + "by_direction": by_direction, + } + + +# โ”€โ”€ WEBSOCKET ENDPOINTS โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + +@app.websocket("/ws/prices") +async def ws_prices(ws: WebSocket): + """ + Live price stream. + Client can optionally send subscribe messages to filter tickers. + By default, all universe tickers are streamed. + """ + await manager.connect_prices(ws) + try: + while True: + # Keep connection alive; process client messages + try: + raw = await asyncio.wait_for(ws.receive_text(), timeout=30.0) + # Parse client commands (subscribe/unsubscribe) + try: + msg = json.loads(raw) + if msg.get("action") == "subscribe": + # Future: per-client filtering + log.debug(f"Client subscribe: {msg.get('tickers', [])}") + except json.JSONDecodeError: + pass + except asyncio.TimeoutError: + # Send keepalive ping + try: + await ws.send_text(json.dumps({"type": "ping"})) + except Exception: + break + except WebSocketDisconnect: + pass + finally: + manager.disconnect_prices(ws) + + +@app.websocket("/ws/events") +async def ws_events(ws: WebSocket): + """ + State change event stream. + Pushes notifications when state files change. + """ + await manager.connect_events(ws) + try: + while True: + try: + # Just keep alive โ€” events are pushed server-side + await asyncio.wait_for(ws.receive_text(), timeout=30.0) + except asyncio.TimeoutError: + try: + await ws.send_text(json.dumps({"type": "ping"})) + except Exception: + break + except WebSocketDisconnect: + pass + finally: + manager.disconnect_events(ws) + + +# ============================================================ +# 7. ENTRY POINT +# ============================================================ + +def main(): + parser = argparse.ArgumentParser(description="Quantum Terminal Data Server") + parser.add_argument("--host", default=server_config.host, help="Bind address") + parser.add_argument("--port", type=int, default=server_config.port, help="Port number") + parser.add_argument("--reload", action="store_true", help="Auto-reload on code changes") + args = parser.parse_args() + + server_config.host = args.host + server_config.port = args.port + + uvicorn.run( + "data_server:app", + host=args.host, + port=args.port, + reload=args.reload, + log_level="info", + ) + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/backend/data_sync_client.py b/backend/data_sync_client.py new file mode 100644 index 0000000..d6e96da --- /dev/null +++ b/backend/data_sync_client.py @@ -0,0 +1,1378 @@ +# version: v12 +""" +================================================================================ +Quantum Terminal Consumer โ€” Data Sync Client +================================================================================ +Downloads pre-computed data from the Quantum Terminal VPS and caches it locally +for offline resilience. + +v11 โ€” Parallel sync via httpx + asyncio.gather. The manifest path now + issues all per-file GETs concurrently (semaphore-bounded at 8) using + a single httpx.AsyncClient. On a typical fresh install (~150 files), + drops total sync time from ~3 min serial to ~20-30 s. Public + sync_all() stays a synchronous method; it spins up its own event + loop via asyncio.run() so existing callers (lifespan thread, + asyncio.to_thread wrappers in route handlers, periodic_sync) keep + working unchanged. The legacy path (manifest 404) stays serial + using `requests` โ€” it's rarely hit in production and not worth the + additional surface area. + +v10 โ€” Download-speed instrumentation. Adds `sync_bytes_total` (cumulative + bytes synced this run) to /api/sync/status. The frontend remembers + the previous poll's value + timestamp via useRef and computes a + live KB/s or MB/s readout next to the progress bar. Each successful + download adds the JSON-encoded payload size โ€” an estimate (no + transport headers / compression accounted for) but close enough + for a user-visible speed indicator. + +v9 โ€” Sync-progress instrumentation. Tracks `_sync_total` (files the + manifest says we need) and `_sync_done` (files completed so far), + plus exposes both alongside `sync_in_progress` in /api/sync/status. + Drives the new top-toolbar download progress bar in the consumer UI. + +v8 โ€” Async-handler fix. The /api/sync/refresh, /api/sync/refresh/{ticker} + and /api/sync/clear-cache routes were declared `async def` but called + the synchronous `client.sync_all()` / `client.sync_ticker()` / + `client.clear_cache()` directly. That FROZE the FastAPI event loop + for the entire duration of sync (5+ min on first install), starving + every other endpoint โ€” including /api/sync/status, which is what the + LoginScreen v9 fast-path polls to detect format-version mismatch. + Symptom: user passes login, sees "Verifying data versionโ€ฆ" forever + because the status poll was queued behind the blocked refresh. + Fix: wrap every blocking sync call in `asyncio.to_thread()` so it + runs on the worker thread pool while the event loop stays free to + serve other requests. + +v7 โ€” Startup-latency fix. Previous behavior made the initial sync take up + to 15 minutes for users whose servers returned persistent 5xx on + specific endpoints (notably /data/{ticker}/probfield returning 500 + instead of 404 โ€” known server bug). Each failing endpoint cost up to + 70 s (30 s timeout + 10 s sleep + 30 s retry). Changes: + โ€ข Per-file `timeout` dropped 30 s โ†’ 8 s. Consumer is display-only; + a healthy server responds in well under a second. + โ€ข Retry sleep on 5xx dropped 10 s โ†’ 1 s (retains transient-blip + protection without stalling). + โ€ข Manifest fetch timeout dropped 15 s โ†’ 8 s (same reasoning). + Worst case per failing endpoint now ~17 s (8 + 1 + 8) vs prior 70 s. + Across a universe with several persistent 500s this turns a 15-min + startup into roughly ~2 min. + +v2.1 โ€” Incremental sync with full data category support: + - Manifest-first sync (only download what changed) + - If-Modified-Since headers (server returns 304) + - Categories: cones, options, probfield, bands, fundamentals, signals, + lifecycle, weeklystate + - Global data (fundamentals, signals, etc.) stored under ticker "GLOBAL" + - Falls back to legacy full sync if manifest unavailable + +v2 โ€” Data Center transition: bands externalized to {TICKER}_bands.json + (previously nested inside {TICKER}_cones.json). Bands is now a first-class + per-ticker category synced alongside cones/options/probfield. + +v6 โ€” Fix: `total` counter in _load_all_from_cache was referenced but never + initialized after the v5 rewrite, causing an UnboundLocalError the first + time the sweep matched a file. This crashed the end-of-sync_all orphan + sweep (added in v5), so misnamed cache files like GLOBAL_eurusd_bands.json + never got loaded โ†’ bands empty for those tickers. Just adds `total = 0` + above the loop. + +v5 โ€” _load_all_from_cache() is now also invoked at the END of every + successful sync_all() (online path), not just on offline/failure. The + manifest-driven sync only knows about files the server advertises โ€” + orphan cache files (e.g. legacy GLOBAL__bands.json drops that + aren't in the manifest) were never getting into the in-memory stores. + Now they are, via the shared resolver. HTTP-fresh data is still written + first, so the reload confirms the same content for advertised files + and adds any orphans as a second pass. Slot guard prevents the reload + from clobbering fresher HTTP data. + +v4 โ€” Filename-resolution helper `resolve_cache_filename()` added and used by + `_load_all_from_cache()` (and reused from cache_watcher.py). Handles: + - bare globals e.g. "money_flow.json" + - prefixed globals e.g. "GLOBAL_probability_state.json" + (fixes mis-parse of multi-word categories) + - legacy per-ticker misnaming e.g. "GLOBAL_eurusd_bands.json" โ†’ + reroutes in-memory to (EURUSD, bands) + - normal per-ticker e.g. "EURUSD_bands.json" + Keeps stale/mislabeled cache files usable without touching disk. + +v3 โ€” Data Center transition: five new GLOBAL categories registered for the + forward-looking probability layer (shipped under temp_out/probability/ + on the DC side, served as /data/GLOBAL/ and cached as + GLOBAL_.json): + - probability_state (aggregator containing the 4 below) + - regime_transitions (Markov model between regimes) + - score_cones (per-asset probability cones for 15 tickers) + - leading_composite (leading vs coincident indicator composite) + - liquidity_forecast (6-mo net-liquidity projection with cones) + Each file is independently synced, watched, and served โ€” lets the DC + recompute components at different cadences without redownloading the + whole bundle. Exposed to the frontend via /api/data/global/{key}. + +Storage: + %APPDATA%\\QuantumTerminal\\cache\\{TICKER}_cones.json + %APPDATA%\\QuantumTerminal\\cache\\{TICKER}_bands.json + %APPDATA%\\QuantumTerminal\\cache\\{TICKER}_options.json + %APPDATA%\\QuantumTerminal\\cache\\{TICKER}_probfield.json + %APPDATA%\\QuantumTerminal\\cache\\GLOBAL_fundamentals.json + %APPDATA%\\QuantumTerminal\\cache\\GLOBAL_signals.json + %APPDATA%\\QuantumTerminal\\cache\\GLOBAL_lifecycle.json + %APPDATA%\\QuantumTerminal\\cache\\GLOBAL_weeklystate.json + %APPDATA%\\QuantumTerminal\\cache\\_data_status.json + %APPDATA%\\QuantumTerminal\\cache\\_sync_timestamps.json + +This module does NOT introduce any calculation triggers. +================================================================================ +""" + +import asyncio # v8: needed for asyncio.to_thread to keep async handlers from blocking +import json +import logging +import os +import time +import configparser +from pathlib import Path + +# AppData directory name. +APP_DIR = os.environ.get("MK_APP_DIR_NAME", "QuantumTerminal") +from datetime import datetime, timezone, timedelta +from email.utils import formatdate +from typing import Optional, Dict, List, Any, Tuple +from dataclasses import dataclass, field, asdict +from enum import Enum + +log = logging.getLogger("mk.data_sync") + +PROJECT_ROOT = Path(__file__).resolve().parent + +# -- Format version this client expects (Rule C6) -- +EXPECTED_MAJOR_VERSION = 1 + +# Per-ticker data categories (well-known types) +TICKER_CATEGORIES = ("cones", "options", "probfield", "bands") + +# Known global categories โ€” but the system accepts ANY category from the manifest. +# New global files are auto-discovered; no code change needed. +KNOWN_GLOBAL = ("fundamentals", "fundamental_state", "daily_signals", + "signal_lifecycle", "weekly_state", "money_flow", + "central_bank_rates", "macro_sectors", "macro_countries", "macro_flows", + "macro_sector_flows", + # v3: probability layer (Data Center forward-looking globals) + "probability_state", "regime_transitions", "score_cones", + "leading_composite", "liquidity_forecast") + +# โ”€โ”€ Global data endpoints (universe-level, not per-ticker) โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +GLOBAL_ENDPOINTS = { + "fundamentals": "/data/GLOBAL/fundamentals", + "fundamental_state": "/data/GLOBAL/fundamental_state", + "money_flow": "/data/GLOBAL/money_flow", + "macro_sectors": "/data/GLOBAL/macro_sectors", + "macro_countries": "/data/GLOBAL/macro_countries", + "macro_flows": "/data/GLOBAL/macro_flows", + "macro_sector_flows": "/data/GLOBAL/global_macro_sector_flows", + "central_bank_rates": "/data/GLOBAL/central_bank_rates", + # v3: probability layer + "probability_state": "/data/GLOBAL/probability_state", + "regime_transitions": "/data/GLOBAL/regime_transitions", + "score_cones": "/data/GLOBAL/score_cones", + "leading_composite": "/data/GLOBAL/leading_composite", + "liquidity_forecast": "/data/GLOBAL/liquidity_forecast", +} + +# Cache filenames for global data +GLOBAL_CACHE_NAMES = { + "fundamentals": "GLOBAL_fundamentals.json", + "fundamental_state": "GLOBAL_fundamental_state.json", + "money_flow": "GLOBAL_money_flow.json", + "macro_sectors": "GLOBAL_macro_sectors.json", + "macro_countries": "GLOBAL_macro_countries.json", + "macro_flows": "GLOBAL_macro_flows.json", + "macro_sector_flows": "GLOBAL_macro_sector_flows.json", + "central_bank_rates": "GLOBAL_central_bank_rates.json", + # v3: probability layer + "probability_state": "GLOBAL_probability_state.json", + "regime_transitions": "GLOBAL_regime_transitions.json", + "score_cones": "GLOBAL_score_cones.json", + "leading_composite": "GLOBAL_leading_composite.json", + "liquidity_forecast": "GLOBAL_liquidity_forecast.json", +} + + +# ============================================================ +# 1. STALENESS LEVELS (Rule C4) +# ============================================================ + +class FreshnessLevel(str, Enum): + FRESH = "fresh" + WARNING = "warning" + ERROR = "error" + UNKNOWN = "unknown" + + +@dataclass +class StalenessInfo: + ticker: str + computed_at: Optional[str] = None + age_hours: float = 0.0 + level: FreshnessLevel = FreshnessLevel.UNKNOWN + display_date: str = "N/A" + + +# ============================================================ +# 2. CONFIG +# ============================================================ + +def _read_config() -> configparser.ConfigParser: + import platform as _plat + config = configparser.ConfigParser() + + search_paths = [ + PROJECT_ROOT / "consumer_config.ini", + PROJECT_ROOT.parent / "consumer_config.ini", + ] + + import sys as _sys + if getattr(_sys, 'frozen', False): + search_paths.append(Path(_sys.executable).parent / "consumer_config.ini") + + if _plat.system() == "Windows": + import os as _os + appdata = _os.environ.get("APPDATA", "") + if appdata: + search_paths.append(Path(appdata) / APP_DIR / "consumer_config.ini") + + for p in _sys.path: + candidate = Path(p) / "consumer_config.ini" + if candidate not in search_paths: + search_paths.append(candidate) + + for path in search_paths: + if path.exists(): + config.read(str(path), encoding="utf-8-sig") + return config + + config.read_dict({ + "data": { + "cache_dir": f"%APPDATA%\\{APP_DIR}\\cache", + "stale_warning_hours": "48", + "stale_error_hours": "72", + }, + }) + return config + + +def _get_stale_thresholds() -> Tuple[float, float]: + cfg = _read_config() + warn = float(cfg.get("data", "stale_warning_hours", fallback="48")) + err = float(cfg.get("data", "stale_error_hours", fallback="72")) + return warn, err + + +# ============================================================ +# 3. CACHE DIRECTORY +# ============================================================ + +def _get_cache_dir() -> Path: + import platform, os + if platform.system() == "Windows": + appdata = os.environ.get("APPDATA", "") + if appdata: + base = Path(appdata) / APP_DIR / "cache" + else: + base = Path.home() / "AppData" / "Roaming" / APP_DIR / "cache" + else: + base = Path.home() / ".QuantumTerminal" / "cache" + + base.mkdir(parents=True, exist_ok=True) + return base + + +def _cache_path(ticker: str, data_type: str) -> Path: + return _get_cache_dir() / f"{ticker.upper()}_{data_type}.json" + + +def _status_cache_path() -> Path: + return _get_cache_dir() / "_data_status.json" + + +def _sync_timestamps_path() -> Path: + return _get_cache_dir() / "_sync_timestamps.json" + + +# ============================================================ +# 4. CACHE I/O +# ============================================================ + +def _write_cache(path: Path, data: dict): + try: + path.write_text(json.dumps(data, separators=(",", ":")), encoding="utf-8") + except Exception as e: + log.warning(f"Failed to write cache {path.name}: {e}") + + +def _read_cache(path: Path) -> Optional[dict]: + if not path.exists(): + return None + try: + return json.loads(path.read_text(encoding="utf-8")) + except Exception as e: + log.warning(f"Failed to read cache {path.name}: {e}") + return None + + +# ============================================================ +# 5. SYNC TIMESTAMP TRACKER +# ============================================================ + +def _load_sync_timestamps() -> dict: + data = _read_cache(_sync_timestamps_path()) + return data if data else {} + + +def _save_sync_timestamps(timestamps: dict): + _write_cache(_sync_timestamps_path(), timestamps) + + +def _ts_key(category: str, ticker: str) -> str: + return f"{category}:{ticker.upper()}" + + +# v4: single source of truth for parsing cache filenames. Returns (ticker, category) +# or None. Handles bare globals, prefixed globals (incl. multi-word categories), +# legacy GLOBAL__ misnaming, and normal per-ticker. +def resolve_cache_filename(stem: str): + """Map a cache filename stem (without .json) to (ticker, category). + + Examples: + "EURUSD_bands" โ†’ ("EURUSD", "bands") + "GLOBAL_fundamentals" โ†’ ("GLOBAL", "fundamentals") + "GLOBAL_fundamental_state" โ†’ ("GLOBAL", "fundamental_state") + "GLOBAL_probability_state" โ†’ ("GLOBAL", "probability_state") + "GLOBAL_money_flow" โ†’ ("GLOBAL", "money_flow") + "GLOBAL_eurusd_bands" โ†’ ("EURUSD", "bands") [legacy reroute] + "GLOBAL_xauusd_historical_cones" โ†’ ("XAUUSD", "historical_cones") [legacy reroute] + "money_flow" โ†’ ("GLOBAL", "money_flow") [bare global] + "probability_state" โ†’ ("GLOBAL", "probability_state") [bare global] + """ + import re as _re + if not stem: + return None + low = stem.lower() + known_global_cats = set(KNOWN_GLOBAL) | set(GLOBAL_ENDPOINTS.keys()) + + # (a) Bare global โ€” DC writes "money_flow.json" instead of "GLOBAL_money_flow.json". + if low in known_global_cats: + return ("GLOBAL", low) + + # (b) Prefixed global โ€” "GLOBAL_". Must come BEFORE the greedy + # regex (which would misparse multi-word categories). + if low.startswith("global_"): + suffix = low[len("global_"):] + if suffix in known_global_cats: + return ("GLOBAL", suffix) + # (c) Legacy misnaming โ€” "GLOBAL__". + m = _re.match(r'^([a-z0-9]+)_([a-z_]+)$', suffix) + if m: + return (m.group(1).upper(), m.group(2)) + return None + + # (d) Normal per-ticker โ€” greedy split on the last "_" suffix. + m = _re.match(r'^(.+)_([a-z_]+)$', stem, _re.IGNORECASE) + if m: + return (m.group(1).upper(), m.group(2).lower()) + return None + + +# ============================================================ +# 6. FORMAT VERSION CHECK (Rule C6) +# ============================================================ + +def _check_format_version(data: dict) -> bool: + """Telemetry only โ€” logs format_version but never gates (Rule C6 reversal 2026-05-03).""" + version_str = data.get("format_version", "1.0") + try: + major = int(version_str.split(".")[0]) + if major != EXPECTED_MAJOR_VERSION: + log.info( + f"format_version telemetry: server={version_str}, baseline major={EXPECTED_MAJOR_VERSION}" + ) + except (ValueError, IndexError): + log.warning(f"Could not parse format_version: {version_str}") + return True + + +# ============================================================ +# 7. STALENESS CALCULATOR (Rule C4) +# ============================================================ + +def _compute_staleness(computed_at: Optional[str], ticker: str) -> StalenessInfo: + if not computed_at: + return StalenessInfo(ticker=ticker) + + try: + ts = datetime.fromisoformat(computed_at.replace("Z", "+00:00")) + if ts.tzinfo is None: + ts = ts.replace(tzinfo=timezone.utc) + now = datetime.now(timezone.utc) + age = (now - ts).total_seconds() / 3600.0 + + warn_h, err_h = _get_stale_thresholds() + + if age > err_h: + level = FreshnessLevel.ERROR + elif age > warn_h: + level = FreshnessLevel.WARNING + else: + level = FreshnessLevel.FRESH + + display_date = ts.strftime("%Y-%m-%d") + + return StalenessInfo( + ticker=ticker, + computed_at=computed_at, + age_hours=round(age, 1), + level=level, + display_date=display_date, + ) + except Exception: + return StalenessInfo(ticker=ticker, computed_at=computed_at) + + +# ============================================================ +# 8. DATA SYNC CLIENT +# ============================================================ + +class DataSyncClient: + """ + Singleton data sync client with incremental sync. + + v2.1 flow (sync_all): + 1. GET /data/manifest -> per-file metadata with updated_at + 2. Compare each file's updated_at against local _sync_timestamps.json + 3. Only download files where server version is newer + 4. Each request includes If-Modified-Since header (server returns 304) + 5. Global data (fundamentals, signals, etc.) fetched as GLOBAL ticker + 6. On any failure -> fall back to cache (Rule C5) + """ + + def __init__(self): + # In-memory data stores by category (dynamic โ€” no hardcoded list) + self._stores: Dict[str, Dict[str, dict]] = {} + self._data_status: Optional[dict] = None + self._sync_errors: List[str] = [] + self._format_version_ok = True + self._offline_mode = False + self._last_synced_at: Optional[str] = None + # Tracks background sync activity so the terminal top-bar pill can + # show "SYNCING" / "READY" without the user being stuck on a loader. + self._sync_in_progress: bool = False + # v9: progress counters โ€” drive the toolbar progress bar. + self._sync_total: int = 0 + self._sync_done: int = 0 + # v10: cumulative bytes downloaded this sync โ€” frontend computes + # speed from the delta between two consecutive /api/sync/status polls. + self._sync_bytes_total: int = 0 + self._sync_timestamps: Dict[str, str] = _load_sync_timestamps() + # Track which categories we've seen (from manifest or cache) + self._known_categories: set = set(TICKER_CATEGORIES) + + def _get_store(self, category: str) -> Dict[str, dict]: + if category not in self._stores: + self._stores[category] = {} + return self._stores[category] + + # ---- Public: sync all (manifest-first) ---- + + def sync_all(self) -> dict: + """Public entry โ€” runs the async core in a fresh event loop via + asyncio.run() so existing synchronous callers stay unchanged. + Caller MUST NOT be inside a running event loop; wrap with + asyncio.to_thread(client.sync_all) if calling from async context.""" + self._sync_in_progress = True + # v9: reset counters on each entry so a re-sync starts at 0. + self._sync_total = 0 + self._sync_done = 0 + # v10: reset cumulative bytes too. + self._sync_bytes_total = 0 + try: + return asyncio.run(self._sync_all_async()) + finally: + self._sync_in_progress = False + # v9: clamp `done` to `total` so the bar reads 100% on completion + # even if the manifest path bailed out before populating total. + if self._sync_total > 0: + self._sync_done = self._sync_total + + async def _sync_all_async(self) -> dict: + """v11: parallel manifest-based sync. Issues all per-file GETs + concurrently via httpx.AsyncClient + asyncio.gather, semaphore- + bounded at 8 to avoid hammering the server. Falls back to the + synchronous legacy path on manifest 404.""" + import httpx + + self._sync_errors = [] + summary = {"offline_mode": False, "format_version_ok": True, + "synced": 0, "skipped": 0} + + # Quantum Terminal: No auth required โ€” load from cache directly. + jwt = "" + + if not jwt: + log.warning("No JWT โ€” loading from cache (Rule C5)") + self._offline_mode = True + summary["offline_mode"] = True + self._load_all_from_cache() + return summary + + base_url = _read_config().get("server", "base_url", fallback="").rstrip("/") + headers = {"Authorization": f"Bearer {jwt}"} + timeout = httpx.Timeout(connect=8.0, read=8.0, write=8.0, pool=8.0) + # Pool limits โ€” keep headroom over the semaphore concurrency so we + # never starve. + limits = httpx.Limits(max_connections=20, max_keepalive_connections=12) + + async with httpx.AsyncClient(base_url=base_url, headers=headers, + timeout=timeout, limits=limits) as http: + # Step 1: manifest + try: + r = await http.get("/data/manifest") + except Exception as e: + log.warning(f"Manifest unreachable: {e}") + self._offline_mode = True + summary["offline_mode"] = True + self._load_all_from_cache() + return summary + + m_status = r.status_code + + if m_status in (401, 403): + log.warning(f"Auth error during data sync (HTTP {m_status})") + self._load_all_from_cache() + return summary + + if m_status == 404: + log.info("Manifest not found โ€” falling back to legacy serial sync") + # Legacy path is sync (uses requests) โ€” run it on a worker thread. + return await asyncio.to_thread(self._sync_all_legacy_sync, summary) + + if m_status != 200: + log.warning(f"Manifest failed (HTTP {m_status}) โ€” using cache") + self._offline_mode = True + summary["offline_mode"] = True + self._load_all_from_cache() + return summary + + try: + manifest = r.json() + except Exception as e: + log.warning(f"Manifest JSON parse failed: {e}") + self._offline_mode = True + summary["offline_mode"] = True + self._load_all_from_cache() + return summary + + # Step 2: format-version check at the manifest level. + if not _check_format_version(manifest): + self._format_version_ok = False + summary["format_version_ok"] = False + return summary + + # Step 3: parallel downloads. + server_files = manifest.get("files", {}) + self._sync_total = sum(len(c) for c in server_files.values()) + self._sync_done = 0 + + sem = asyncio.Semaphore(8) + tasks = [] + + async def fetch_one(ticker: str, category: str, server_updated: str): + async with sem: + local_key = _ts_key(category, ticker) + local_updated = self._sync_timestamps.get(local_key, "") + # Skip if local is current + if local_updated and server_updated and local_updated >= server_updated: + cached = _read_cache(_cache_path(ticker, category)) + if cached: + self._get_store(category)[ticker.upper()] = cached + summary["skipped"] += 1 + self._sync_done += 1 + return + result = await self._sync_one_async(http, ticker, category) + if result in ("synced", "not_modified"): + self._sync_timestamps[local_key] = server_updated + if result == "not_modified": + cached = _read_cache(_cache_path(ticker, category)) + if cached: + self._get_store(category)[ticker.upper()] = cached + summary["skipped"] += 1 + else: + summary["synced"] += 1 + self._sync_done += 1 + + for category, cat_files in server_files.items(): + self._known_categories.add(category) + for ticker, meta in cat_files.items(): + tasks.append(fetch_one(ticker, category, meta.get("updated_at", ""))) + + await asyncio.gather(*tasks, return_exceptions=True) + + _save_sync_timestamps(self._sync_timestamps) + # Sweep cache for orphan files (legacy misnaming, manual drops). + self._load_all_from_cache(skip_existing=True) + + log.info(f"Parallel sync: {summary['synced']} downloaded, {summary['skipped']} skipped (concurrency=8)") + self._last_synced_at = datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M UTC") + return summary + + async def _sync_one_async(self, http, ticker: str, data_type: str) -> str: + """v11: async equivalent of _sync_one using httpx. Same retry / 304 / + 404 / 5xx behavior as the sync version.""" + canonical = ticker.upper() + endpoint = f"/data/{canonical}/{data_type}" + + extra_headers = {} + local_key = _ts_key(data_type, canonical) + local_ts = self._sync_timestamps.get(local_key) + if local_ts: + try: + ts_dt = datetime.fromisoformat(local_ts) + extra_headers["If-Modified-Since"] = formatdate(ts_dt.timestamp(), usegmt=True) + except (ValueError, TypeError): + pass + + async def _do_request(): + return await http.get(endpoint, headers=extra_headers) + + try: + r = await _do_request() + except Exception as e: + log.warning(f"Sync request failed for {canonical}/{data_type}: {e}") + await asyncio.sleep(1) + try: + r = await _do_request() + except Exception as e2: + log.warning(f"Sync retry failed: {e2}") + cached = _read_cache(_cache_path(canonical, data_type)) + if cached: + self._get_store(data_type)[canonical] = cached + return "cached" + return "failed" + + status = r.status_code + + # v8: retry once on 5xx with a short sleep (matches sync version). + if status >= 500: + log.warning(f"Server error {status} for {endpoint} โ€” retrying in 1s") + await asyncio.sleep(1) + try: + r = await _do_request() + status = r.status_code + except Exception: + pass + + if status == 304: + return "not_modified" + + if status == 200: + try: + data = r.json() + except Exception as e: + log.warning(f"JSON parse failed for {canonical}/{data_type}: {e}") + return "failed" + if not _check_format_version(data): + self._format_version_ok = False + return "failed" + self._get_store(data_type)[canonical] = data + _write_cache(_cache_path(canonical, data_type), data) + try: + self._sync_bytes_total += len(json.dumps(data).encode("utf-8")) + except Exception: + pass + return "synced" + + if status == 404: + return "cached" + + # Other / unexpected โ€” fall back to disk cache if present. + log.warning(f"Failed to sync {data_type} for {canonical} (HTTP {status})") + self._sync_errors.append(f"{canonical}/{data_type}: HTTP {status}") + cached = _read_cache(_cache_path(canonical, data_type)) + if cached: + self._get_store(data_type)[canonical] = cached + return "cached" + return "failed" + + def _sync_all_legacy_sync(self, summary: dict) -> dict: + """v11: thin wrapper invoking the existing sync legacy path on the + worker thread. The legacy path uses `requests` and is rarely hit + (only when the server returns 404 on /data/manifest).""" + log.warning("Legacy sync unavailable in Quantum Terminal โ€” using cache") + self._load_all_from_cache() + return summary + + def _sync_all_unchecked(self) -> dict: + # Quantum Terminal: no license client needed + client = None + + self._sync_errors = [] + summary = {"offline_mode": False, "format_version_ok": True, + "synced": 0, "skipped": 0} + + # Step 1: Try manifest + manifest, m_status = client.auth_request("GET", "/data/manifest", timeout=8) # v7 + + if m_status == 0: + log.warning("Server unreachable -- loading from cache (Rule C5)") + self._offline_mode = True + summary["offline_mode"] = True + self._load_all_from_cache() + return summary + + if m_status in (401, 403): + log.warning(f"Auth error during data sync (HTTP {m_status})") + self._load_all_from_cache() + return summary + + if m_status == 404: + log.info("Manifest not found -- falling back to legacy sync") + return self._sync_all_legacy(client, summary) + + if m_status != 200 or manifest is None: + log.warning(f"Manifest failed (HTTP {m_status}) -- using cache") + self._offline_mode = True + summary["offline_mode"] = True + self._load_all_from_cache() + return summary + + # Step 2: Format version check + if not _check_format_version(manifest): + self._format_version_ok = False + summary["format_version_ok"] = False + return summary + + # Step 3: Download only changed files โ€” iterate ALL categories from manifest + server_files = manifest.get("files", {}) + + # v9: count total files in this sync up front so the toolbar + # progress bar can render `done / total`. + self._sync_total = sum(len(c) for c in server_files.values()) + self._sync_done = 0 + + for category, cat_files in server_files.items(): + self._known_categories.add(category) + + for ticker, meta in cat_files.items(): + server_updated = meta.get("updated_at", "") + local_key = _ts_key(category, ticker) + local_updated = self._sync_timestamps.get(local_key, "") + + # Skip if local is up to date + if local_updated and server_updated and local_updated >= server_updated: + cached = _read_cache(_cache_path(ticker, category)) + if cached: + self._get_store(category)[ticker.upper()] = cached + summary["skipped"] += 1 + self._sync_done += 1 # v9 + continue + + # Download this file + result = self._sync_one(client, ticker, category) + if result in ("synced", "not_modified"): + self._sync_timestamps[local_key] = server_updated + if result == "not_modified": + cached = _read_cache(_cache_path(ticker, category)) + if cached: + self._get_store(category)[ticker.upper()] = cached + summary["skipped"] += 1 + else: + summary["synced"] += 1 + self._sync_done += 1 # v9 โ€” count even on failed/cached fall-through + + _save_sync_timestamps(self._sync_timestamps) + + # v5: sweep cache for orphan files not in the manifest (legacy + # misnaming, manually dropped files, older categories). HTTP-fresh + # entries already in the store are preserved via skip_existing=True. + self._load_all_from_cache(skip_existing=True) + + log.info(f"Incremental sync: {summary['synced']} downloaded, {summary['skipped']} skipped") + self._last_synced_at = datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M UTC") + return summary + + # ---- Legacy full sync (fallback) ---- + + def _sync_all_legacy(self, client, summary: dict) -> dict: + data, status = client.auth_request("GET", "/data/status") + + if status != 200 or data is None: + self._offline_mode = True + summary["offline_mode"] = True + self._load_all_from_cache() + return summary + + if not _check_format_version(data): + self._format_version_ok = False + summary["format_version_ok"] = False + return summary + + self._data_status = data + _write_cache(_status_cache_path(), data) + + for ticker in data.get("assets_available", []): + for cat in TICKER_CATEGORIES: + result = self._sync_one(client, ticker, cat) + if result == "synced": + summary["synced"] = summary.get("synced", 0) + 1 + + # Try known global data too + for cat in KNOWN_GLOBAL: + self._sync_one(client, "GLOBAL", cat) + + # Sync global datasets (fundamentals, macro, money flow, rates) + self.sync_global() + + self._last_synced_at = datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M UTC") + return summary + + # ---- Public: sync single ticker ---- + + def sync_ticker(self, ticker: str) -> dict: + # Quantum Terminal: no license client needed + client = None + + result = {} + if ticker.upper() == "GLOBAL": + # Sync all known global categories + for cat in self._known_categories: + if cat not in TICKER_CATEGORIES: + result[cat] = self._sync_one(client, ticker, cat) + else: + for cat in TICKER_CATEGORIES: + result[cat] = self._sync_one(client, ticker, cat) + return result + + # ---- Public: read data ---- + + def get_cones(self, ticker: str) -> Optional[dict]: + return self._get_data(ticker, "cones") + + def get_bands(self, ticker: str) -> Optional[dict]: + return self._get_data(ticker, "bands") + + def get_options(self, ticker: str) -> Optional[dict]: + return self._get_data(ticker, "options") + + def get_probfield(self, ticker: str) -> Optional[dict]: + return self._get_data(ticker, "probfield") + + def get_fundamentals(self) -> Optional[dict]: + """Get global fundamental data (regime, liquidity, yields, COT, etc.).""" + return self._get_data("GLOBAL", "fundamental_state") + + def get_signals(self) -> Optional[dict]: + """Get daily signals data.""" + return self._get_data("GLOBAL", "daily_signals") + + def get_lifecycle(self) -> Optional[dict]: + """Get signal lifecycle data.""" + return self._get_data("GLOBAL", "signal_lifecycle") + + def get_weekly_state(self) -> Optional[dict]: + """Get weekly state data.""" + return self._get_data("GLOBAL", "weekly_state") + + def get_global(self, category: str) -> Optional[dict]: + """ + Get any global data by category name. + Works for any file synced from the server โ€” no code changes needed. + Examples: get_global("central_bank_rates"), get_global("macro_flows") + """ + return self._get_data("GLOBAL", category) + + # โ”€โ”€ Global data getters โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + + def _fetch_global(self, key: str) -> dict | None: + """ + Fetch one global (GLOBAL ticker) dataset from the server. + Falls back to cache on any error. Returns None if unavailable. + """ + endpoint = GLOBAL_ENDPOINTS.get(key) + cache_file = _get_cache_dir() / GLOBAL_CACHE_NAMES.get(key, f"GLOBAL_{key}.json") + + if not endpoint: + log.warning(f"[DataSync] Unknown global key: {key}") + return None + + try: + jwt = self._get_jwt() + url = f"{self.base_url}{endpoint}" + resp = requests.get( + url, + headers={"Authorization": f"Bearer {jwt}"}, + timeout=8, # v7: was 15 โ€” consumer is display-only, fast fail beats long wait + ) + + if resp.status_code == 200: + data = resp.json() + # Write to cache + cache_file.parent.mkdir(parents=True, exist_ok=True) + cache_file.write_text(json.dumps(data), encoding="utf-8") + log.info(f"[DataSync] Global {key}: fetched and cached ({len(resp.content)} bytes)") + return data + + elif resp.status_code == 304: + log.info(f"[DataSync] Global {key}: 304 Not Modified โ€” using cache") + return self._load_global_cache(key) + + elif resp.status_code == 404: + log.warning(f"[DataSync] Global {key}: 404 โ€” not on server yet") + return self._load_global_cache(key) + + else: + log.warning(f"[DataSync] Global {key}: HTTP {resp.status_code} โ€” using cache") + return self._load_global_cache(key) + + except Exception as e: + log.warning(f"[DataSync] Global {key}: fetch failed ({e}) โ€” using cache") + return self._load_global_cache(key) + + def _load_global_cache(self, key: str) -> dict | None: + """Load global data from AppData cache. Returns None if not cached.""" + cache_file = _get_cache_dir() / GLOBAL_CACHE_NAMES.get(key, f"GLOBAL_{key}.json") + if cache_file.exists(): + try: + return json.loads(cache_file.read_text(encoding="utf-8")) + except Exception as e: + log.warning(f"[DataSync] Cache read failed for {key}: {e}") + return None + + def sync_global(self) -> dict: + """ + Fetch all global datasets from server. + Called inside sync_all() โ€” runs once at startup, not per-ticker. + Returns dict of {key: success_bool}. + """ + results = {} + for key in GLOBAL_ENDPOINTS: + data = self._fetch_global(key) + results[key] = data is not None + return results + + def get_macro_sectors(self) -> dict | None: + """Get macro sector rotation data.""" + return self._load_global_cache("macro_sectors") or self._fetch_global("macro_sectors") + + def get_macro_countries(self) -> dict | None: + """Get macro country data.""" + return self._load_global_cache("macro_countries") or self._fetch_global("macro_countries") + + def get_macro_flows(self) -> dict | None: + """Get country capital flow data (1W and 4W periods).""" + return self._load_global_cache("macro_flows") or self._fetch_global("macro_flows") + + def get_macro_sector_flows(self) -> dict | None: + """Get inter-sector money flow data (macro + weekly periods).""" + return self._load_global_cache("macro_sector_flows") or self._fetch_global("macro_sector_flows") + + def get_central_bank_rates(self) -> dict | None: + """Get central bank rate differential data.""" + return self._load_global_cache("central_bank_rates") or self._fetch_global("central_bank_rates") + + def _get_data(self, ticker: str, category: str) -> Optional[dict]: + canonical = ticker.upper() + store = self._get_store(category) + if canonical in store: + return store[canonical] + cached = _read_cache(_cache_path(canonical, category)) + if cached: + store[canonical] = cached + return cached + + def get_data_status(self) -> Optional[dict]: + if self._data_status: + return self._data_status + return _read_cache(_status_cache_path()) + + # ---- Public: staleness (Rule C4) ---- + + def get_staleness_info(self) -> List[StalenessInfo]: + results = [] + for ticker, data in self._get_store("cones").items(): + computed_at = data.get("computed_at") + results.append(_compute_staleness(computed_at, ticker)) + return results + + def get_overall_staleness(self) -> StalenessInfo: + infos = self.get_staleness_info() + # Use last_synced_at as the authoritative freshness signal + sync_time = getattr(self, "_last_synced_at", None) or getattr(self, "last_synced_at", None) + if sync_time: + return _compute_staleness(sync_time, "ALL") + if not infos: + return StalenessInfo(ticker="ALL", level=FreshnessLevel.FRESH) + worst = max(infos, key=lambda x: x.age_hours) + worst.ticker = "ALL" + return worst + + # ---- Public: status flags ---- + + @property + def is_offline(self) -> bool: + return self._offline_mode + + @property + def format_version_ok(self) -> bool: + return self._format_version_ok + + @property + def sync_errors(self) -> List[str]: + return list(self._sync_errors) + + @property + def is_syncing(self) -> bool: + """True while sync_all is running โ€” top-bar pill polls via + /api/consumer/status.sync_in_progress.""" + return bool(getattr(self, "_sync_in_progress", False)) + + @property + def synced_tickers(self) -> List[str]: + return sorted(self._get_store("cones").keys()) + + @property + def last_synced_at(self) -> Optional[str]: + return self._last_synced_at + + # ---- Internal: sync one file ---- + + def _sync_one(self, client, ticker: str, data_type: str) -> str: + canonical = ticker.upper() + endpoint = f"/data/{canonical}/{data_type}" + + extra_headers = {} + local_key = _ts_key(data_type, canonical) + local_ts = self._sync_timestamps.get(local_key) + if local_ts: + try: + ts_dt = datetime.fromisoformat(local_ts) + extra_headers["If-Modified-Since"] = formatdate( + ts_dt.timestamp(), usegmt=True + ) + except (ValueError, TypeError): + pass + + data, status = client.auth_request( + "GET", endpoint, timeout=8, extra_headers=extra_headers # v7: was 30 + ) + + if status == 304: + log.info(f"304 Not Modified: {canonical}/{data_type}") + return "not_modified" + + # v7: retry once on 5xx with a SHORT sleep (was 10s). A genuine + # transient blip recovers in <1 s; a persistent 500 (e.g. a + # mis-served /probfield) would previously cost the user ~70 s per + # failure, stacking into multi-minute startup delays. Now worst + # case per failing endpoint is ~17 s (8 + 1 + 8). + if status >= 500: + log.warning(f"Server error {status} for {endpoint} -- retrying in 1s") + time.sleep(1) + data, status = client.auth_request( + "GET", endpoint, timeout=8, extra_headers=extra_headers + ) + + if status == 304: + return "not_modified" + + if status == 200 and data is not None: + if not _check_format_version(data): + self._format_version_ok = False + return "failed" + + self._get_store(data_type)[canonical] = data + _write_cache(_cache_path(canonical, data_type), data) + # v10: rough byte count for the toolbar speed readout. Uses the + # serialized JSON length โ€” close enough to wire bytes for a UX + # indicator (no headers / compression accounted for). + try: + self._sync_bytes_total += len(json.dumps(data).encode("utf-8")) + except Exception: + pass + return "synced" + + elif status == 404: + log.info(f"No {data_type} data for {canonical} (404)") + return "cached" + + else: + log.warning(f"Failed to sync {data_type} for {canonical} (HTTP {status})") + self._sync_errors.append(f"{canonical}/{data_type}: HTTP {status}") + cached = _read_cache(_cache_path(canonical, data_type)) + if cached: + self._get_store(data_type)[canonical] = cached + log.info(f"Loaded {data_type} for {canonical} from cache") + return "cached" + return "failed" + + # ---- Internal: load from cache (offline) ---- + + def _load_all_from_cache(self, skip_existing: bool = False): + """Load all cache files into in-memory stores via resolve_cache_filename(). + + If skip_existing=True, slots already populated (typically by a fresh + HTTP sync in the same tick) are preserved โ€” only orphan/missing slots + get filled. Used by v5's end-of-sync_all sweep. + """ + cache_dir = _get_cache_dir() + if not cache_dir.exists(): + return + + self._data_status = _read_cache(_status_cache_path()) + + total = 0 + for f in cache_dir.glob("*.json"): + if f.name.startswith("_"): + continue + if not f.name.lower().endswith(".json"): + continue + stem = f.name[:-5] + resolved = resolve_cache_filename(stem) + if not resolved: + continue + ticker, category = resolved + if skip_existing and ticker in self._get_store(category): + continue + data = _read_cache(f) + if data: + self._get_store(category)[ticker] = data + self._known_categories.add(category) + total += 1 + + if total: + log.info(f"Loaded {total} data files from cache (offline mode)") + + # ---- Public: clear cache (Rule C9 โ€” reversible via _trash) ---- + + def clear_cache(self) -> dict: + """ + Move all cached data files to a timestamped trash folder. + Resets in-memory stores and sync timestamps so the next sync_all() + pulls everything fresh from the server. + + Files are NEVER deleted โ€” they are preserved in + %APPDATA%\\QuantumTerminal\\_trash\\cache_\\ for recovery + per Rule C9. Operator can manually purge _trash/ when confident. + + Returns: + {"moved": int, "trash_dir": str | None} + """ + cache_dir = _get_cache_dir() + if not cache_dir.exists(): + return {"moved": 0, "trash_dir": None} + + ts = datetime.now(timezone.utc).strftime("%Y%m%d_%H%M%S") + trash_dir = cache_dir.parent / "_trash" / f"cache_{ts}" + try: + trash_dir.mkdir(parents=True, exist_ok=True) + except Exception as e: + log.error(f"Failed to create trash dir {trash_dir}: {e}") + return {"moved": 0, "trash_dir": None} + + moved = 0 + for f in cache_dir.glob("*.json"): + try: + f.replace(trash_dir / f.name) + moved += 1 + except Exception as e: + log.warning(f"Failed to move {f.name} to trash: {e}") + + # Reset in-memory state so next sync_all is a clean pull + self._stores.clear() + self._sync_timestamps.clear() + try: + _save_sync_timestamps(self._sync_timestamps) + except Exception: + pass + self._data_status = None + self._sync_errors = [] + self._offline_mode = False + self._last_synced_at = None + + log.info(f"Cleared cache: {moved} files moved to {trash_dir}") + return {"moved": moved, "trash_dir": str(trash_dir)} + + +# ============================================================ +# 9. SINGLETON + REST API ROUTES +# ============================================================ + +_sync_client: Optional[DataSyncClient] = None + + +def get_sync_client() -> DataSyncClient: + global _sync_client + if _sync_client is None: + _sync_client = DataSyncClient() + return _sync_client + + +def create_data_sync_routes(): + from fastapi import APIRouter + + router = APIRouter(tags=["data_sync"]) + + @router.get("/api/sync/status") + async def api_sync_status(): + client = get_sync_client() + staleness = client.get_overall_staleness() + # v9: surface in-progress flag + progress counters so the top-toolbar + # download bar can render `done / total` and the post-sync "please + # refresh" prompt can fire on transition true โ†’ false. + return { + "offline_mode": client.is_offline, + "format_version_ok": client.format_version_ok, + "synced_tickers": client.synced_tickers, + "sync_errors": client.sync_errors, + "analysis_date": staleness.display_date, + "staleness_level": staleness.level.value, + "staleness_hours": staleness.age_hours, + "last_synced_at": client.last_synced_at, + "sync_in_progress": bool(getattr(client, "_sync_in_progress", False)), + "sync_total": int(getattr(client, "_sync_total", 0)), + "sync_done": int(getattr(client, "_sync_done", 0)), + "sync_bytes_total": int(getattr(client, "_sync_bytes_total", 0)), # v10 + } + + @router.get("/api/sync/cones/{ticker}") + async def api_get_cones(ticker: str): + client = get_sync_client() + data = client.get_cones(ticker.upper()) + if data is None: + from fastapi import HTTPException + raise HTTPException(404, f"No cone data for {ticker}") + return data + + @router.get("/api/sync/options/{ticker}") + async def api_get_options(ticker: str): + client = get_sync_client() + data = client.get_options(ticker.upper()) + if data is None: + from fastapi import HTTPException + raise HTTPException(404, f"No options data for {ticker}") + return data + + @router.get("/api/sync/probfield/{ticker}") + async def api_get_probfield(ticker: str): + client = get_sync_client() + data = client.get_probfield(ticker.upper()) + if data is None: + from fastapi import HTTPException + raise HTTPException(404, f"No probfield data for {ticker}") + return data + + @router.get("/api/sync/fundamentals") + async def api_get_fundamentals(): + client = get_sync_client() + data = client.get_fundamentals() + if data is None: + from fastapi import HTTPException + raise HTTPException(404, "No fundamentals data") + return data + + @router.get("/api/sync/signals") + async def api_get_signals(): + client = get_sync_client() + data = client.get_signals() + if data is None: + from fastapi import HTTPException + raise HTTPException(404, "No signals data") + return data + + @router.get("/api/sync/lifecycle") + async def api_get_lifecycle(): + client = get_sync_client() + data = client.get_lifecycle() + if data is None: + from fastapi import HTTPException + raise HTTPException(404, "No lifecycle data") + return data + + @router.get("/api/sync/weeklystate") + async def api_get_weekly_state(): + client = get_sync_client() + data = client.get_weekly_state() + if data is None: + from fastapi import HTTPException + raise HTTPException(404, "No weekly state data") + return data + + @router.get("/api/sync/staleness") + async def api_staleness(): + client = get_sync_client() + infos = client.get_staleness_info() + overall = client.get_overall_staleness() + return { + "overall": asdict(overall), + "per_ticker": [asdict(i) for i in infos], + } + + @router.post("/api/sync/refresh") + async def api_refresh_sync(): + # v8: run on the worker thread pool so the event loop is free to + # serve concurrent requests (notably /api/sync/status, which the + # LoginScreen polls to detect format-version mismatch). Without + # this, sync_all blocks ALL other endpoints for ~5 min on a + # first-install full pull. + client = get_sync_client() + summary = await asyncio.to_thread(client.sync_all) + return summary + + @router.post("/api/sync/refresh/{ticker}") + async def api_refresh_ticker(ticker: str): + # v8: same off-loop reasoning as /api/sync/refresh. + client = get_sync_client() + result = await asyncio.to_thread(client.sync_ticker, ticker.upper()) + return result + + @router.post("/api/sync/clear-cache") + async def api_clear_cache(): + """ + Move cache to _trash and re-sync from server. + Rule C9: files are preserved in _trash/, never deleted. + """ + # v8: both clear_cache and sync_all are blocking I/O โ€” keep them + # off the event loop so /api/sync/status keeps responding. + client = get_sync_client() + clear_result = await asyncio.to_thread(client.clear_cache) + sync_summary = await asyncio.to_thread(client.sync_all) + return { + "cleared": clear_result, + "resync": sync_summary, + } + + @router.get("/api/sync/poll-config") + async def api_poll_config(): + """Returns the current periodic sync interval (for UI display).""" + try: + from periodic_sync import _read_interval + return {"interval_seconds": _read_interval()} + except Exception: + return {"interval_seconds": 300} + + @router.get("/api/data/global/{key}") + async def get_global_data(key: str): + """Proxy global data from local cache to React frontend.""" + valid_keys = set(GLOBAL_ENDPOINTS.keys()) + if key not in valid_keys: + from fastapi import HTTPException + raise HTTPException(status_code=404, detail=f"Unknown global key: {key}") + client = get_sync_client() + data = client._load_global_cache(key) + if data is None: + # Try live fetch as fallback + data = client._fetch_global(key) + if data is None: + from fastapi import HTTPException + raise HTTPException(status_code=503, detail=f"{key} not available yet") + return data + + return router \ No newline at end of file diff --git a/backend/execution_routes.py b/backend/execution_routes.py new file mode 100644 index 0000000..e503bc8 --- /dev/null +++ b/backend/execution_routes.py @@ -0,0 +1,312 @@ +# version: v3 +""" +execution_routes.py โ€” MT5 trade execution HTTP layer (v3). + +v1 was a stub (PRO-only). v2 exposes the provider's execution methods as +HTTP endpoints so the consumer OrderTicket / ExposureStrip / quick-trade +buttons can act on the user's live MT5 account. +v3 adds the pending-order lifecycle: list / modify-price-or-SLTP / cancel. + +Endpoints: + GET /api/positions โ€” list open positions + POST /api/orders โ€” place a market/limit/stop order + POST /api/positions/{ticket}/close โ€” close (full or partial) a position + PATCH /api/positions/{ticket} โ€” modify SL / TP on an open position + GET /api/orders/pending โ€” list resting pending orders (v3) + PATCH /api/orders/{ticket} โ€” modify pending-order price / SL / TP (v3) + DELETE /api/orders/{ticket} โ€” cancel a pending order (v3) + +All endpoints are thin wrappers over `provider.place_order` / `get_positions` +/ `close_position` / `modify_position`. Results flow back as JSON. + +Consumer-safety notes: + - Every request goes through the cached MT5 provider โ€” no fresh connect. + - provider None or disconnected โ†’ returns a structured error, not 500. + - Magic number defaults to 777 (Quantum Terminal tag) unless overridden. +""" + +import logging +from typing import Optional + +from fastapi import APIRouter, HTTPException +from pydantic import BaseModel + +from models import OrderRequest + +log = logging.getLogger("execution_routes") + +MK_MAGIC = 777 # Quantum Terminal tag on broker side for attribution / filtering + + +class OrderBody(BaseModel): + ticker: str + direction: str # "BUY" | "SELL" + order_type: str = "MARKET" # "MARKET" | "LIMIT" | "STOP" + lots: float + price: Optional[float] = None # required for LIMIT/STOP + stop_loss: Optional[float] = None + take_profit: Optional[float] = None + comment: Optional[str] = "Quantum Terminal" + # Signal attribution (optional โ€” logged only) + signal_id: Optional[str] = None + signal_type: Optional[str] = None + signal_confidence: Optional[float] = None + + +class ClosePositionBody(BaseModel): + lots: Optional[float] = None # None = full close + + +class ModifyPositionBody(BaseModel): + stop_loss: Optional[float] = None + take_profit: Optional[float] = None + + +class ModifyOrderBody(BaseModel): + """v3: Modify a pending order โ€” any field left None is preserved server-side.""" + price: Optional[float] = None + stop_loss: Optional[float] = None + take_profit: Optional[float] = None + + +def _pending_to_dict(o) -> dict: + """Normalize a PendingOrder object to the JSON shape the frontend expects.""" + def g(obj, *names, default=None): + for n in names: + v = getattr(obj, n, None) + if v is not None and v != 0: + return v + return default + return { + "ticket": str(g(o, "ticket") or ""), + "ticker": g(o, "ticker", "symbol", default=""), + "direction": g(o, "direction", default=""), + "order_type": g(o, "order_type", default=""), + "lots": float(g(o, "lots", "volume", default=0) or 0), + "price": float(g(o, "price", "price_open", default=0) or 0), + "stop_loss": float(g(o, "stop_loss", "sl", default=0) or 0), + "take_profit": float(g(o, "take_profit", "tp", default=0) or 0), + "comment": str(getattr(o, "comment", "") or ""), + "time_setup": str(g(o, "time_setup", "time_open") or ""), + } + + +def _position_to_dict(p) -> dict: + """Normalize a Position object to the JSON shape the frontend expects.""" + def g(obj, *names, default=None): + for n in names: + v = getattr(obj, n, None) + if v is not None and v != 0: + return v + return default + return { + "ticket": str(g(p, "ticket") or ""), + "ticker": g(p, "ticker", "symbol", default=""), + "direction": g(p, "direction", default=""), + "lots": float(g(p, "lots", "volume", default=0) or 0), + "open_price": float(g(p, "open_price", "price_open", default=0) or 0), + "current_price":float(g(p, "current_price", "price_current", default=0) or 0), + "stop_loss": float(g(p, "stop_loss", "sl", default=0) or 0), + "take_profit": float(g(p, "take_profit", "tp", default=0) or 0), + "profit": float(getattr(p, "profit", 0) or 0), + "swap": float(getattr(p, "swap", 0) or 0), + "commission": float(getattr(p, "commission", 0) or 0), + "open_time": str(g(p, "open_time") or getattr(p, "time_open", "") or ""), + "comment": str(getattr(p, "comment", "") or ""), + } + + +def create_execution_router(cfg_manager=None, app=None, broadcast_event=None) -> APIRouter: + router = APIRouter(tags=["execution"]) + + def _provider(): + if cfg_manager is None: + return None + try: + return cfg_manager.get_provider("mt5_default") + except Exception: + return None + + # โ”€โ”€ GET /api/positions โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + @router.get("/api/positions") + async def get_positions(): + import asyncio + prov = _provider() + if prov is None or not getattr(prov, "connected", False): + return {"positions": [], "mt5_connected": False} + try: + positions = await asyncio.to_thread(prov.get_positions) + except Exception as e: + log.warning(f"get_positions failed: {e}") + return {"positions": [], "mt5_connected": True, "error": str(e)} + return { + "positions": [_position_to_dict(p) for p in (positions or [])], + "mt5_connected": True, + } + + # โ”€โ”€ POST /api/orders โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + @router.post("/api/orders") + async def place_order(body: OrderBody): + import asyncio + prov = _provider() + if prov is None or not getattr(prov, "connected", False): + raise HTTPException(503, {"error": "mt5_disconnected", + "message": "MT5 is not connected. Connect in Settings โ†’ Providers."}) + + # Build the provider's OrderRequest. FlexModel accepts extra fields. + req = OrderRequest( + ticker=body.ticker.upper(), + direction=body.direction.upper(), + order_type=body.order_type.upper(), + lots=float(body.lots), + volume=float(body.lots), # dual-name for safety + price=body.price or 0, + stop_loss=body.stop_loss, + take_profit=body.take_profit, + sl=float(body.stop_loss) if body.stop_loss else 0, + tp=float(body.take_profit) if body.take_profit else 0, + comment=body.comment or "Quantum Terminal", + magic=MK_MAGIC, + ) + + log.info( + f"ORDER {req.direction} {req.lots} {req.ticker} @ " + f"{body.price or 'market'} SL={body.stop_loss} TP={body.take_profit}" + + (f" signal={body.signal_id}" if body.signal_id else "") + ) + + try: + result = await asyncio.to_thread(prov.place_order, req) + except Exception as e: + log.error(f"place_order exception: {e}") + raise HTTPException(500, {"error": "place_order_failed", "message": str(e)}) + + # Normalize response โ€” mt5_provider returns OrderResult (FlexModel). + ok = bool(getattr(result, "success", False)) + payload = { + "success": ok, + "order_id": str(getattr(result, "order_id", "") or ""), + "price": float(getattr(result, "price", 0) or 0), + "ticker": getattr(result, "ticker", req.ticker) or req.ticker, + "direction": getattr(result, "direction", req.direction) or req.direction, + "lots": float(getattr(result, "lots", req.lots) or req.lots), + } + if not ok: + payload["error"] = str( + getattr(result, "error", None) + or getattr(result, "message", None) + or "Order rejected" + ) + return payload + + # โ”€โ”€ POST /api/positions/{ticket}/close โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + @router.post("/api/positions/{ticket}/close") + async def close_position(ticket: str, body: Optional[ClosePositionBody] = None): + import asyncio + prov = _provider() + if prov is None or not getattr(prov, "connected", False): + raise HTTPException(503, {"error": "mt5_disconnected"}) + + lots = body.lots if body else None + log.info(f"CLOSE position {ticket} lots={lots or 'full'}") + try: + result = await asyncio.to_thread(prov.close_position, ticket, lots) + except Exception as e: + log.error(f"close_position exception: {e}") + raise HTTPException(500, {"error": "close_failed", "message": str(e)}) + + ok = bool(getattr(result, "success", False)) + return { + "success": ok, + "ticket": ticket, + "price": float(getattr(result, "price", 0) or 0), + **({"error": str(getattr(result, "error", None) or "")} if not ok else {}), + } + + # โ”€โ”€ PATCH /api/positions/{ticket} โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + @router.patch("/api/positions/{ticket}") + async def modify_position(ticket: str, body: ModifyPositionBody): + import asyncio + prov = _provider() + if prov is None or not getattr(prov, "connected", False): + raise HTTPException(503, {"error": "mt5_disconnected"}) + + if not hasattr(prov, "modify_position"): + raise HTTPException(501, {"error": "modify_unsupported"}) + + log.info(f"MODIFY position {ticket} SL={body.stop_loss} TP={body.take_profit}") + try: + result = await asyncio.to_thread( + prov.modify_position, ticket, body.stop_loss, body.take_profit + ) + except Exception as e: + log.error(f"modify_position exception: {e}") + raise HTTPException(500, {"error": "modify_failed", "message": str(e)}) + + # modify_position returns a dict per mt5_provider + if isinstance(result, dict): + return {"success": bool(result.get("success", False)), **result} + return {"success": True} + + # โ”€โ”€ GET /api/orders/pending โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ v3 โ”€โ”€ + @router.get("/api/orders/pending") + async def list_pending_orders(): + import asyncio + prov = _provider() + if prov is None or not getattr(prov, "connected", False): + return {"orders": [], "mt5_connected": False} + try: + orders = await asyncio.to_thread(prov.get_pending_orders) + except Exception as e: + log.warning(f"get_pending_orders failed: {e}") + return {"orders": [], "mt5_connected": True, "error": str(e)} + return { + "orders": [_pending_to_dict(o) for o in (orders or [])], + "mt5_connected": True, + } + + # โ”€โ”€ PATCH /api/orders/{ticket} โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ v3 โ”€โ”€ + @router.patch("/api/orders/{ticket}") + async def modify_pending_order(ticket: str, body: ModifyOrderBody): + import asyncio + prov = _provider() + if prov is None or not getattr(prov, "connected", False): + raise HTTPException(503, {"error": "mt5_disconnected"}) + if not hasattr(prov, "modify_order"): + raise HTTPException(501, {"error": "modify_order_unsupported"}) + log.info(f"MODIFY order {ticket} price={body.price} SL={body.stop_loss} TP={body.take_profit}") + try: + result = await asyncio.to_thread( + prov.modify_order, ticket, body.price, body.stop_loss, body.take_profit + ) + except Exception as e: + log.error(f"modify_order exception: {e}") + raise HTTPException(500, {"error": "modify_order_failed", "message": str(e)}) + ok = bool(getattr(result, "success", False)) + payload = {"success": ok, "ticket": ticket} + if not ok: + payload["error"] = str(getattr(result, "error", None) or "Modify rejected") + return payload + + # โ”€โ”€ DELETE /api/orders/{ticket} โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ v3 โ”€โ”€ + @router.delete("/api/orders/{ticket}") + async def cancel_pending_order(ticket: str): + import asyncio + prov = _provider() + if prov is None or not getattr(prov, "connected", False): + raise HTTPException(503, {"error": "mt5_disconnected"}) + if not hasattr(prov, "cancel_order"): + raise HTTPException(501, {"error": "cancel_order_unsupported"}) + log.info(f"CANCEL order {ticket}") + try: + result = await asyncio.to_thread(prov.cancel_order, ticket) + except Exception as e: + log.error(f"cancel_order exception: {e}") + raise HTTPException(500, {"error": "cancel_order_failed", "message": str(e)}) + ok = bool(getattr(result, "success", False)) + payload = {"success": ok, "ticket": ticket} + if not ok: + payload["error"] = str(getattr(result, "error", None) or "Cancel rejected") + return payload + + return router diff --git a/backend/freshness_routes.py b/backend/freshness_routes.py new file mode 100644 index 0000000..ef4fe79 --- /dev/null +++ b/backend/freshness_routes.py @@ -0,0 +1,10 @@ +""" +freshness_routes.py โ€” Stub for consumer terminal. +PRO-only routes. Consumer does not use these endpoints. +""" +from fastapi import APIRouter + + +def create_freshness_router(*args, **kwargs) -> APIRouter: + """Return empty router โ€” PRO endpoints not available in consumer.""" + return APIRouter(tags=["freshness"]) diff --git a/backend/launcher.py b/backend/launcher.py new file mode 100644 index 0000000..c60a6a5 --- /dev/null +++ b/backend/launcher.py @@ -0,0 +1,323 @@ +""" +================================================================================ +Quantum Terminal Consumer โ€” Launcher +================================================================================ +Single entry point for the packaged .exe. +Starts data server + serves React frontend + opens browser. +Auto-creates consumer_config.ini on first run. + +Usage: + python launcher.py # Dev mode + QuantumTerminal.exe # Packaged mode + +This module does NOT introduce any calculation triggers. +================================================================================ +""" + +import sys +import os +import threading +import time +import webbrowser +import shutil +import logging +from pathlib import Path + +log = logging.getLogger("mk.launcher") + +# โ€” Determine base paths (works for both dev and PyInstaller) โ€” +if getattr(sys, 'frozen', False): + # Running as packaged .exe โ€” everything is flat in _MEIPASS + BASE_DIR = Path(sys._MEIPASS) + EXE_DIR = Path(sys.executable).parent + BACKEND_DIR = BASE_DIR # PyInstaller bundles flat, no backend/ subfolder + FRONTEND_DIR = BASE_DIR / "frontend" +else: + # Running as script (dev mode) + BASE_DIR = Path(__file__).resolve().parent + EXE_DIR = BASE_DIR + BACKEND_DIR = BASE_DIR + FRONTEND_DIR = BASE_DIR.parent / "terminal_app" / "build" + +PORT = int(os.environ.get("MK_PORT", "8502")) # v2: Electron passes MK_PORT + +# v2: AppData dir name (env-var driven; default "QuantumTerminal" preserves v1 path). +APP_DIR = os.environ.get("MK_APP_DIR_NAME", "QuantumTerminal") + +# โ€” Default config content (f-string so APP_DIR is substituted at module load) โ€” +DEFAULT_CONFIG = f"""\ +[server] +# No external server needed for Quantum Terminal (open source) +base_url = + +[data] +cache_dir = %APPDATA%\\{APP_DIR}\\cache +stale_warning_hours = 48 +stale_error_hours = 72 +""" + + +def ensure_config(): + """Create consumer_config.ini on first run. User never touches this.""" + # Check next to .exe first (packaged mode) + config_path = EXE_DIR / "consumer_config.ini" + + if config_path.exists(): + return config_path + + # Also check AppData (fallback location) + appdata_config = _get_appdata_base() / "consumer_config.ini" + if appdata_config.exists(): + return appdata_config + + # Try to write next to .exe + try: + template = BASE_DIR / "consumer_config.ini.template" + if template.exists(): + shutil.copy2(template, config_path) + else: + config_path.write_text(DEFAULT_CONFIG, encoding="utf-8") + print(f" Config created: {config_path}") + return config_path + except PermissionError: + # Program Files is write-protected โ€” use AppData instead + appdata_dir = _get_appdata_base() + appdata_dir.mkdir(parents=True, exist_ok=True) + appdata_config = appdata_dir / "consumer_config.ini" + try: + template = BASE_DIR / "consumer_config.ini.template" + if template.exists(): + shutil.copy2(template, appdata_config) + else: + appdata_config.write_text(DEFAULT_CONFIG, encoding="utf-8") + print(f" Config created (AppData): {appdata_config}") + return appdata_config + except Exception as e: + print(f" [WARN] Could not create config: {e}") + return appdata_config + + +def _get_appdata_base() -> Path: + """Get AppData\\Roaming\\ path. APP_DIR comes from MK_APP_DIR_NAME env var.""" + import platform + if platform.system() == "Windows": + appdata = os.environ.get("APPDATA", "") + if appdata: + return Path(appdata) / APP_DIR + return Path.home() / f".{APP_DIR.lower()}" + + +def ensure_appdata(): + """Create AppData directories for auth cache and data cache.""" + appdata = os.environ.get("APPDATA", "") + if appdata: + base = Path(appdata) / APP_DIR + else: + base = Path.home() / "AppData" / "Roaming" / APP_DIR + + for subdir in ["", "cache"]: + d = base / subdir if subdir else base + d.mkdir(parents=True, exist_ok=True) + + return base + + +def open_browser(): + """Wait for server to start, then open browser.""" + for _ in range(30): # Wait up to 30 seconds + time.sleep(1) + try: + import urllib.request + urllib.request.urlopen(f"http://127.0.0.1:{PORT}/api/health", timeout=2) + break + except Exception: + continue + + webbrowser.open(f"http://127.0.0.1:{PORT}") + + +def main(): + # v2 Phase D: persistent rotating log file under %APPDATA%\\logs\. + # Console behavior unchanged; file logging is additive. + try: + from logging_setup import setup_logging + _log_file = setup_logging() + except Exception as _e: + _log_file = None + print(f"[launcher] logging setup skipped: {_e}") + + print() + print("=" * 50) + print(" Quantum Terminal Consumer Terminal") + print("=" * 50) + if _log_file: + print(f" Logs : {_log_file}") + + # Step 1: Ensure config exists + config_path = ensure_config() + print(f" Config : {config_path}") + + # Step 2: Ensure AppData dirs exist + appdata = ensure_appdata() + print(f" AppData: {appdata}") + + # Step 3: Setup Python path + sys.path.insert(0, str(BACKEND_DIR)) + os.chdir(str(BACKEND_DIR)) + + # Also add EXE_DIR so consumer_config.ini is found + if str(EXE_DIR) not in sys.path: + sys.path.insert(0, str(EXE_DIR)) + + # Add config directory to path so modules find it + config_dir = str(config_path.parent) + if config_dir not in sys.path: + sys.path.insert(0, config_dir) + + # Copy config next to backend modules if packaged + if getattr(sys, 'frozen', False): + backend_config = BACKEND_DIR / "consumer_config.ini" + if not backend_config.exists() and config_path.exists(): + try: + shutil.copy2(config_path, backend_config) + except PermissionError: + pass # Read-only _internal โ€” config found via sys.path instead + + print(f" Server : http://127.0.0.1:{PORT}") + print(f" Press Ctrl+C to stop") + print("=" * 50) + print() + + # Step 4: Open browser in background โ€” skipped when running under Electron. + if not os.environ.get("MK_NO_BROWSER"): + threading.Thread(target=open_browser, daemon=True).start() + + # Step 5: Import and start server + from data_server import app + + # Serve React build as static files (production mode) + # IMPORTANT: Use StaticFiles mount (not catch-all routes) so API routes + # always take priority. The catch-all @app.get("/{path}") pattern causes + # 405 errors on PATCH/POST to /api/* endpoints. + if FRONTEND_DIR.exists() and (FRONTEND_DIR / "index.html").exists(): + from fastapi.staticfiles import StaticFiles + from fastapi.responses import FileResponse + from starlette.middleware.base import BaseHTTPMiddleware + from starlette.responses import Response as StarletteResponse + + # Helper: serve index.html with no-cache headers so browser + # always fetches the latest after an update. JS/CSS files have + # content hashes in their filenames so they cache safely. + def _serve_index(): + return FileResponse( + str(FRONTEND_DIR / "index.html"), + headers={ + "Cache-Control": "no-cache, no-store, must-revalidate", + "Pragma": "no-cache", + "Expires": "0", + }, + ) + + # Serve /static/* assets (JS, CSS, images) + app.mount("/static", StaticFiles(directory=str(FRONTEND_DIR / "static")), name="static") + + # SPA middleware: for non-API, non-static GET requests that don't match + # a file, serve index.html. This replaces the old @app.get("/{full_path:path}") + # catch-all which caused 405 errors on API PATCH/POST requests. + class SPAMiddleware(BaseHTTPMiddleware): + async def dispatch(self, request, call_next): + response = await call_next(request) + + # If the API returned 404 and this is a GET request + # that doesn't target /api/ or /ws/, serve index.html + # (React client-side routing) + if ( + response.status_code == 404 + and request.method == "GET" + and not request.url.path.startswith("/api/") + and not request.url.path.startswith("/ws/") + and not request.url.path.startswith("/static/") + ): + # Check if it's a real file in the build directory + file_path = FRONTEND_DIR / request.url.path.lstrip("/") + if file_path.exists() and file_path.is_file(): + return FileResponse(str(file_path)) + # Otherwise serve index.html for React Router + return _serve_index() + + return response + + app.add_middleware(SPAMiddleware) + + # Root route โ€” serve index.html + @app.get("/", include_in_schema=False) + async def serve_root(): + return _serve_index() + + # Favicon + favicon_path = FRONTEND_DIR / "favicon.ico" + if favicon_path.exists(): + @app.get("/favicon.ico", include_in_schema=False) + async def serve_favicon(): + return FileResponse(str(favicon_path)) + + print(" Frontend: serving React build") + else: + print(" Frontend: not found โ€” use npm start for dev mode") + + # v2 Phase E: graceful shutdown. + # Electron sends SIGTERM (Windows: terminates the process group) when + # the user closes the window. uvicorn handles SIGTERM by setting its + # `should_exit` flag and draining in-flight requests. We additionally + # register a Python-level signal handler that flushes logs and lets + # any pending sync_timestamps write finish before exit. SIGINT (Ctrl+C + # in dev) goes through the same path. + import uvicorn + import signal + + config = uvicorn.Config( + app=app, + host="127.0.0.1", + port=PORT, + log_level="info", + # Give in-flight HTTP requests up to 5s to finish before forcing exit. + timeout_graceful_shutdown=5, + ) + server = uvicorn.Server(config) + + def _on_signal(signum, frame): + log.info(f"Received signal {signum} โ€” initiating graceful shutdown") + server.should_exit = True + + for sig in (signal.SIGINT, signal.SIGTERM): + try: + signal.signal(sig, _on_signal) + except Exception: + # Some platforms don't allow custom handlers in non-main threads. + pass + + try: + server.run() + finally: + # Final flush โ€” make sure log records on disk are complete and + # any in-memory state (sync timestamps) has been persisted. + try: + from data_sync_client import get_sync_client as _gsc + _sc = _gsc() + try: + # Best-effort: persist sync timestamps if anything was changed + # mid-request when the signal arrived. + from data_sync_client import _save_sync_timestamps + _save_sync_timestamps(_sc._sync_timestamps) + except Exception: + pass + except Exception: + pass + try: + logging.shutdown() + except Exception: + pass + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/backend/logging_setup.py b/backend/logging_setup.py new file mode 100644 index 0000000..ea785a1 --- /dev/null +++ b/backend/logging_setup.py @@ -0,0 +1,91 @@ +""" +================================================================================ +Quantum Terminal Consumer v2 โ€” Logging setup +================================================================================ +Centralized logging configuration. Adds a daily-rotating file handler so logs +survive across runs and can be attached to support tickets, while keeping the +console output identical to v1. + +Log location: %APPDATA%\\\\logs\\backend-YYYY-MM-DD.log +Rotation: daily at midnight, 14 days retained. +Format: same as console โ€” `YYYY-MM-DD HH:MM:SS [logger.name] LEVEL: msg` + +Call setup_logging() ONCE at process startup โ€” launcher.py does this. +================================================================================ +""" +from __future__ import annotations + +import logging +import logging.handlers +import os +import sys +from pathlib import Path + +# Match the APP_DIR convention used elsewhere (license_client / data_sync_client) +APP_DIR = os.environ.get("MK_APP_DIR_NAME", "QuantumTerminal") + +_LOG_FORMAT = "%(asctime)s [%(name)s] %(levelname)s: %(message)s" +_DATE_FORMAT = "%Y-%m-%d %H:%M:%S" +_RETENTION_DAYS = 14 +_FILE_MAX_BYTES = 50 * 1024 * 1024 # 50 MB hard cap per file as a backstop + + +def _appdata_logs_dir() -> Path: + """Resolve %APPDATA%\\\\logs\\ (or ~/./logs/ off-Windows).""" + appdata = os.environ.get("APPDATA", "") + if appdata: + return Path(appdata) / APP_DIR / "logs" + return Path.home() / f".{APP_DIR.lower()}" / "logs" + + +def setup_logging(level: int = logging.INFO) -> Path: + """Configure root logging once. Returns the log file path so callers can + print it on startup (`launcher.py` does).""" + log_dir = _appdata_logs_dir() + log_dir.mkdir(parents=True, exist_ok=True) + log_file = log_dir / "backend.log" + + root = logging.getLogger() + root.setLevel(level) + + # If already configured (e.g., re-import in tests), don't double-attach. + if any(getattr(h, "_mk_managed", False) for h in root.handlers): + return log_file + + fmt = logging.Formatter(_LOG_FORMAT, datefmt=_DATE_FORMAT) + + # Console handler โ€” stderr keeps the existing capture path used by Electron. + ch = logging.StreamHandler(stream=sys.stderr) + ch.setLevel(level) + ch.setFormatter(fmt) + ch._mk_managed = True + root.addHandler(ch) + + # Daily rotating file handler. Use TimedRotatingFileHandler (rolls at + # midnight) with 14 backups. Hard size cap as a backstop in case of + # log floods. + try: + fh = logging.handlers.TimedRotatingFileHandler( + filename=str(log_file), + when="midnight", + interval=1, + backupCount=_RETENTION_DAYS, + encoding="utf-8", + delay=True, # don't open until first emit (avoids a stale empty file) + ) + fh.setLevel(level) + fh.setFormatter(fmt) + fh._mk_managed = True + root.addHandler(fh) + except Exception as e: + # File logging is a nice-to-have โ€” don't kill startup if disk is read-only. + print(f"[logging] file handler unavailable: {e}", file=sys.stderr) + + # uvicorn / fastapi loggers tend to set up their own handlers; let them + # bubble up to root by stripping their handlers and unsetting `propagate=False`. + for noisy in ("uvicorn", "uvicorn.error", "uvicorn.access", "fastapi"): + lg = logging.getLogger(noisy) + lg.handlers.clear() + lg.propagate = True + + return log_file diff --git a/backend/macro_data.py b/backend/macro_data.py new file mode 100644 index 0000000..c26959b --- /dev/null +++ b/backend/macro_data.py @@ -0,0 +1,304 @@ +""" +macro_data.py โ€” Macro Rotation Data Provider +Quantum Terminal Analytical Terminal + +Fetches sector ETF and country ETF performance data from yfinance. +Computes 1-week and 4-week percentage changes. +Results cached with configurable TTL to avoid API hammering. + +Endpoints served: + GET /api/macro/sectors โ†’ sector ETF performance + GET /api/macro/countries โ†’ country ETF performance +""" + +import logging +import time +from typing import Dict, List, Optional + +log = logging.getLogger("mk.macro_data") + +# โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ• +# ETF UNIVERSE โ€” no hardcoded assets, but ETFs are +# benchmark instruments (not part of trading universe) +# โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ• + +SECTOR_ETFS = { + "XLE": {"name": "Energy", "color": "#e67e22"}, + "XLF": {"name": "Financials", "color": "#3498db"}, + "XLK": {"name": "Technology", "color": "#9b59b6"}, + "XLRE": {"name": "Real Estate", "color": "#1abc9c"}, + "XLU": {"name": "Utilities", "color": "#f39c12"}, + "XLV": {"name": "Healthcare", "color": "#2ecc71"}, + "XLB": {"name": "Materials", "color": "#e74c3c"}, + "XLI": {"name": "Industrials", "color": "#34495e"}, + "XLC": {"name": "Communication", "color": "#e84393"}, + "XLY": {"name": "Consumer Discretionary", "color": "#00cec9"}, + "XLP": {"name": "Consumer Staples", "color": "#fdcb6e"}, +} + +COUNTRY_ETFS = { + "SPY": {"name": "United States", "code": "US"}, + "EWJ": {"name": "Japan", "code": "JP"}, + "FXI": {"name": "China", "code": "CN"}, + "EWG": {"name": "Germany", "code": "DE"}, + "EWU": {"name": "United Kingdom", "code": "GB"}, + "EWZ": {"name": "Brazil", "code": "BR"}, + "EWA": {"name": "Australia", "code": "AU"}, + "EWC": {"name": "Canada", "code": "CA"}, + "EWY": {"name": "South Korea", "code": "KR"}, + "EWT": {"name": "Taiwan", "code": "TW"}, + "INDA": {"name": "India", "code": "IN"}, + "EWQ": {"name": "France", "code": "FR"}, + "EWI": {"name": "Italy", "code": "IT"}, + "EWP": {"name": "Spain", "code": "ES"}, + "EWW": {"name": "Mexico", "code": "MX"}, + "EZA": {"name": "South Africa", "code": "ZA"}, + "TUR": {"name": "Turkey", "code": "TR"}, + "KSA": {"name": "Saudi Arabia", "code": "SA"}, + "EWS": {"name": "Singapore", "code": "SG"}, + "EWH": {"name": "Hong Kong", "code": "HK"}, + "EWM": {"name": "Malaysia", "code": "MY"}, + "EWN": {"name": "Netherlands", "code": "NL"}, + "EWD": {"name": "Sweden", "code": "SE"}, + "EWL": {"name": "Switzerland", "code": "CH"}, + "THD": {"name": "Thailand", "code": "TH"}, +} + + +# โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ• +# CACHE +# โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ• + +_cache: Dict[str, dict] = {} +CACHE_TTL_SECONDS = 3600 # 1 hour + + +def _is_cache_valid(key: str) -> bool: + if key not in _cache: + return False + return (time.time() - _cache[key]["ts"]) < CACHE_TTL_SECONDS + + +# โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ• +# DATA FETCH +# โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ• + +def _fetch_etf_changes(tickers: List[str], period: str = "2mo") -> Dict[str, dict]: + """ + Fetch OHLCV for a list of ETF tickers and compute + 1-week and 4-week percentage changes from latest close. + + Returns dict keyed by ticker: + {ticker: {"close": float, "chg_1w": float, "chg_4w": float}} + """ + try: + import yfinance as yf + except ImportError: + log.warning("yfinance not installed โ€” macro data unavailable") + return {} + + results = {} + try: + # Batch download โ€” single API call for all tickers + data = yf.download(tickers, period=period, interval="1d", + auto_adjust=True, progress=False, threads=True) + + if data.empty: + log.warning("yfinance returned empty data for macro ETFs") + return {} + + close = data["Close"] if "Close" in data.columns else data.get("close") + if close is None or close.empty: + log.warning("No Close column in yfinance macro data") + return {} + + # Handle single ticker case (returns Series, not DataFrame) + if isinstance(close, type(data)): + pass # already DataFrame + else: + # Single ticker returns Series โ€” unlikely but handle + close = close.to_frame(name=tickers[0]) + + for ticker in tickers: + if ticker not in close.columns: + continue + + series = close[ticker].dropna() + if len(series) < 2: + continue + + latest = float(series.iloc[-1]) + + # 1-week change (5 trading days back) + idx_1w = min(5, len(series) - 1) + close_1w = float(series.iloc[-1 - idx_1w]) + chg_1w = ((latest - close_1w) / close_1w) * 100 if close_1w != 0 else 0.0 + + # 4-week change (20 trading days back) + idx_4w = min(20, len(series) - 1) + close_4w = float(series.iloc[-1 - idx_4w]) + chg_4w = ((latest - close_4w) / close_4w) * 100 if close_4w != 0 else 0.0 + + results[ticker] = { + "close": round(latest, 2), + "chg_1w": round(chg_1w, 2), + "chg_4w": round(chg_4w, 2), + } + + except Exception as e: + log.warning(f"yfinance macro fetch failed: {e}") + + return results + + +# โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ• +# PUBLIC API +# โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ• + +def get_sector_data(force_refresh: bool = False) -> dict: + """ + Returns sector performance data. + + Response shape: + { + "sectors": [ + {"ticker": "XLE", "name": "Energy", "color": "#e67e22", + "close": 88.5, "chg_1w": 2.3, "chg_4w": -1.1}, + ... + ], + "top5_1w": [...], + "bottom5_1w": [...], + "top5_4w": [...], + "bottom5_4w": [...], + "updated_at": 1700000000.0 + } + """ + cache_key = "sectors" + + if not force_refresh and _is_cache_valid(cache_key): + return _cache[cache_key]["data"] + + tickers = list(SECTOR_ETFS.keys()) + raw = _fetch_etf_changes(tickers) + + sectors = [] + for ticker, meta in SECTOR_ETFS.items(): + perf = raw.get(ticker, {}) + sectors.append({ + "ticker": ticker, + "name": meta["name"], + "color": meta["color"], + "close": perf.get("close", 0), + "chg_1w": perf.get("chg_1w", 0), + "chg_4w": perf.get("chg_4w", 0), + }) + + # Sort helpers + sorted_1w = sorted(sectors, key=lambda s: s["chg_1w"], reverse=True) + sorted_4w = sorted(sectors, key=lambda s: s["chg_4w"], reverse=True) + + result = { + "sectors": sectors, + "top5_1w": sorted_1w[:5], + "bottom5_1w": sorted_1w[-5:], + "top5_4w": sorted_4w[:5], + "bottom5_4w": sorted_4w[-5:], + "updated_at": time.time(), + } + + _cache[cache_key] = {"data": result, "ts": time.time()} + log.info(f"Sector data refreshed โ€” {len(sectors)} sectors loaded") + return result + + +def get_country_data(force_refresh: bool = False) -> dict: + """ + Returns country ETF performance data. + + Response shape: + { + "countries": [ + {"ticker": "SPY", "name": "United States", "code": "US", + "close": 440.5, "chg_1w": 1.2, "chg_4w": 3.5}, + ... + ], + "top5_1w": [...], + "bottom5_1w": [...], + "top5_4w": [...], + "bottom5_4w": [...], + "updated_at": 1700000000.0 + } + """ + cache_key = "countries" + + if not force_refresh and _is_cache_valid(cache_key): + return _cache[cache_key]["data"] + + tickers = list(COUNTRY_ETFS.keys()) + raw = _fetch_etf_changes(tickers) + + countries = [] + for ticker, meta in COUNTRY_ETFS.items(): + perf = raw.get(ticker, {}) + countries.append({ + "ticker": ticker, + "name": meta["name"], + "code": meta["code"], + "close": perf.get("close", 0), + "chg_1w": perf.get("chg_1w", 0), + "chg_4w": perf.get("chg_4w", 0), + }) + + sorted_1w = sorted(countries, key=lambda c: c["chg_1w"], reverse=True) + sorted_4w = sorted(countries, key=lambda c: c["chg_4w"], reverse=True) + + result = { + "countries": countries, + "top5_1w": sorted_1w[:5], + "bottom5_1w": sorted_1w[-5:], + "top5_4w": sorted_4w[:5], + "bottom5_4w": sorted_4w[-5:], + "updated_at": time.time(), + } + + _cache[cache_key] = {"data": result, "ts": time.time()} + log.info(f"Country data refreshed โ€” {len(countries)} countries loaded") + return result + + +# โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ• +# CLI DIAGNOSTIC +# โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ• + +if __name__ == "__main__": + logging.basicConfig(level=logging.INFO, + format="%(asctime)s [%(name)s] %(message)s") + + print("=" * 60) + print("MACRO DATA โ€” SECTOR ETFS") + print("=" * 60) + sd = get_sector_data(force_refresh=True) + for s in sd["sectors"]: + arrow_1w = "โ–ฒ" if s["chg_1w"] >= 0 else "โ–ผ" + arrow_4w = "โ–ฒ" if s["chg_4w"] >= 0 else "โ–ผ" + print(f" {s['ticker']:5s} {s['name']:28s} " + f"1W: {arrow_1w} {s['chg_1w']:+6.2f}% " + f"4W: {arrow_4w} {s['chg_4w']:+6.2f}%") + + print(f"\n TOP 5 (1W): {', '.join(s['name'] for s in sd['top5_1w'])}") + print(f" BOTTOM 5 (1W): {', '.join(s['name'] for s in sd['bottom5_1w'])}") + + print("\n" + "=" * 60) + print("MACRO DATA โ€” COUNTRY ETFS") + print("=" * 60) + cd = get_country_data(force_refresh=True) + for c in cd["countries"]: + arrow_1w = "โ–ฒ" if c["chg_1w"] >= 0 else "โ–ผ" + arrow_4w = "โ–ฒ" if c["chg_4w"] >= 0 else "โ–ผ" + print(f" {c['ticker']:5s} {c['code']:3s} {c['name']:20s} " + f"1W: {arrow_1w} {c['chg_1w']:+6.2f}% " + f"4W: {arrow_4w} {c['chg_4w']:+6.2f}%") + + print(f"\n TOP 5 (1W): {', '.join(c['name'] for c in cd['top5_1w'])}") + print(f" BOTTOM 5 (1W): {', '.join(c['name'] for c in cd['bottom5_1w'])}") + print("\nDONE") diff --git a/backend/models.py b/backend/models.py new file mode 100644 index 0000000..1deae2c --- /dev/null +++ b/backend/models.py @@ -0,0 +1,178 @@ +""" +models.py โ€” Data models for consumer terminal. +Provides TickData, BarData, etc. that providers and data_server import. + +All classes accept and silently ignore unknown keyword arguments +so PRO provider code that passes extra fields won't crash. +""" +from typing import Optional, Dict, Any +from datetime import datetime + + +def _make_flex(cls): + """Decorator: wraps __init__ to silently drop unknown kwargs.""" + orig = cls.__init__ + import inspect + params = set(inspect.signature(orig).parameters.keys()) - {'self'} + + def flex_init(self, *args, **kwargs): + valid = {k: v for k, v in kwargs.items() if k in params} + orig(self, *args, **valid) + # Store extras as attributes + for k, v in kwargs.items(): + if k not in params: + setattr(self, k, v) + + cls.__init__ = flex_init + + # Add to_dict if not present + if not hasattr(cls, 'to_dict'): + def to_dict(self): + d = {} + for k in params: + d[k] = getattr(self, k, None) + for k, v in self.__dict__.items(): + if k not in d: + d[k] = v + return d + cls.to_dict = to_dict + + return cls + + +from dataclasses import dataclass + + +@_make_flex +@dataclass +class TickData: + ticker: str = "" + symbol: str = "" + bid: float = 0.0 + ask: float = 0.0 + last: float = 0.0 + volume: float = 0.0 + time: Optional[datetime] = None + spread: float = 0.0 + time_msc: int = 0 + flags: int = 0 + volume_real: float = 0.0 + + +@_make_flex +@dataclass +class BarData: + ticker: str = "" + symbol: str = "" + timeframe: str = "" + time: Optional[datetime] = None + open: float = 0.0 + high: float = 0.0 + low: float = 0.0 + close: float = 0.0 + volume: float = 0.0 + tick_volume: float = 0.0 + spread: int = 0 + real_volume: float = 0.0 + + +@_make_flex +@dataclass +class AccountInfo: + balance: float = 0.0 + equity: float = 0.0 + margin: float = 0.0 + free_margin: float = 0.0 + currency: str = "USD" + leverage: int = 100 + server: str = "" + login: int = 0 + name: str = "" + company: str = "" + profit: float = 0.0 + margin_level: float = 0.0 + + +@_make_flex +@dataclass +class SymbolInfo: + ticker: str = "" + symbol: str = "" + description: str = "" + point: float = 0.0 + digits: int = 5 + tick_size: float = 0.0 + tick_value: float = 0.0 + volume_min: float = 0.01 + volume_max: float = 100.0 + volume_step: float = 0.01 + trade_contract_size: float = 100000.0 + currency_base: str = "" + currency_profit: str = "" + currency_margin: str = "" + spread: int = 0 + trade_mode: int = 0 + trade_stops_level: int = 0 + swap_long: float = 0.0 + swap_short: float = 0.0 + + +@_make_flex +@dataclass +class OrderRequest: + symbol: str = "" + ticker: str = "" + direction: str = "" + volume: float = 0.01 + price: float = 0.0 + sl: float = 0.0 + tp: float = 0.0 + comment: str = "" + order_type: str = "market" + + +@_make_flex +@dataclass +class OrderResult: + success: bool = False + order_id: int = 0 + message: str = "" + price: float = 0.0 + volume: float = 0.0 + retcode: int = 0 + + +@_make_flex +@dataclass +class Position: + ticket: int = 0 + symbol: str = "" + ticker: str = "" + direction: str = "" + volume: float = 0.0 + price_open: float = 0.0 + price_current: float = 0.0 + sl: float = 0.0 + tp: float = 0.0 + profit: float = 0.0 + comment: str = "" + time_open: Optional[datetime] = None + swap: float = 0.0 + commission: float = 0.0 + + +@_make_flex +@dataclass +class PendingOrder: + """A resting (unfilled) order โ€” LIMIT or STOP โ€” on the broker.""" + ticket: int = 0 + symbol: str = "" + ticker: str = "" + direction: str = "" # "BUY" | "SELL" + order_type: str = "" # "LIMIT" | "STOP" + volume: float = 0.0 + price_open: float = 0.0 + sl: float = 0.0 + tp: float = 0.0 + comment: str = "" + time_setup: Optional[datetime] = None \ No newline at end of file diff --git a/backend/mt5_routes.py b/backend/mt5_routes.py new file mode 100644 index 0000000..1bf1dcb --- /dev/null +++ b/backend/mt5_routes.py @@ -0,0 +1,346 @@ +# version: v6 +""" +mt5_routes.py โ€” MT5 Connection Management API +================================================ +Endpoints for the consumer Settings โ†’ Providers panel. + + GET /api/mt5/config โ†’ read MT5 connection settings + PATCH /api/mt5/config โ†’ save MT5 connection settings + POST /api/mt5/connect โ†’ connect to MT5 with current config + POST /api/mt5/disconnect โ†’ disconnect from MT5 + +Settings are stored in user_config.json under providers.accounts.mt5_default. +This module does NOT introduce any calculation triggers. + +v2 โ€” PATCH /config now disconnects and reinitializes the MT5 provider after + saving, so the next /connect picks up the new terminal_path / server / + login immediately. Previously the provider cached the old path in memory + and stale connects kept firing until the backend was restarted โ€” e.g. + user couldn't revert an invalid GoDo path back to auto-detect. + +v3 โ€” Added `use_tick_data` to the GET/PATCH config payload. When True, the + backend tick loop polls MT5 every 0.2s instead of 1.0s, giving the chart + a live MT5-style "tick feel" on the last candle. Default OFF. Toggle + surfaces in Settings โ†’ PROVIDERS. + +v4 โ€” PATCH /config no longer disconnects + reinits the MT5 provider on every + change. Reinit now triggers ONLY when a connection-relevant field + (connection_method / terminal_path / server / login / password) is + touched. Flipping `use_tick_data` (and any other future cosmetic field) + leaves the live connection alone. Without this, toggling USE TICK DATA + in the UI caused a ~5s outage with a flood of 503 responses while the + reconnect loop healed the connection. + +v5 โ€” POST /browse-terminal opens a native Windows file-picker so users with + multiple MT5 installations can select the correct terminal64.exe without + manually typing (or pasting) a long path. Runs tkinter.filedialog on a + worker thread, returns the chosen absolute path. +""" + +import logging +from fastapi import APIRouter, Request, HTTPException + +log = logging.getLogger("mt5_routes") + + +def create_mt5_router(cfg_manager, app) -> APIRouter: + router = APIRouter(prefix="/api/mt5", tags=["mt5"]) + + def _get_mt5_config() -> dict: + """Read MT5 config from user_config providers section.""" + config = cfg_manager.get_config() + providers = config.get("providers", {}) + accounts = providers.get("accounts", {}) + mt5_acct = accounts.get("mt5_default", {}) + return { + "connection_method": mt5_acct.get("connection_method", "path"), + "terminal_path": mt5_acct.get("terminal_path") or "", + "server": mt5_acct.get("server", ""), + "login": mt5_acct.get("login", ""), + "password": "", # Never send password back + "enabled": mt5_acct.get("enabled", True), + # v3: USE TICK DATA โ€” when ON, tick loop polls MT5 5ร—/sec instead + # of 1ร—/sec. Default OFF; user opts in via Settings โ†’ PROVIDERS. + "use_tick_data": bool(mt5_acct.get("use_tick_data", False)), + } + + # โ”€โ”€ GET /api/mt5/config โ”€โ”€ + @router.get("/config") + async def get_mt5_config(): + return _get_mt5_config() + + # โ”€โ”€ PATCH /api/mt5/config โ”€โ”€ + @router.patch("/config") + async def patch_mt5_config(request: Request): + import asyncio + + try: + patch = await request.json() + except Exception: + raise HTTPException(400, "Invalid JSON") + + # Build the nested update for providers.accounts.mt5_default + mt5_update = {} + for key in ["connection_method", "terminal_path", "server", "login", "password", + "use_tick_data"]: # v3 + if key in patch: + mt5_update[key] = patch[key] + + if mt5_update: + cfg_manager.update({ + "providers": { + "accounts": { + "mt5_default": mt5_update + } + } + }) + log.info(f"MT5 config updated: {list(mt5_update.keys())}") + + # v4: reinit only when a connection-relevant field changed. + # use_tick_data is read live by focus_tick_loop on every iteration + # โ€” flipping it must NOT bounce the MT5 connection (5s 503 outage). + CONNECTION_FIELDS = {"connection_method", "terminal_path", + "server", "login", "password"} + if CONNECTION_FIELDS.intersection(mt5_update.keys()): + try: + prov = cfg_manager.get_provider("mt5_default") + if prov is not None: + try: + if getattr(prov, "connected", False): + await asyncio.to_thread(prov.disconnect) + except Exception as e: + log.warning(f"MT5 disconnect during reinit failed: {e}") + await asyncio.to_thread(cfg_manager.init_providers) + log.info("MT5 provider reinitialized with new config") + except Exception as e: + log.warning(f"MT5 provider reinit failed: {e}") + + return {"config": _get_mt5_config()} + + # โ”€โ”€ POST /api/mt5/browse-terminal โ”€โ”€ + @router.post("/browse-terminal") + async def browse_terminal(): + """v5: Open a native OS file picker for the user to select terminal64.exe. + Runs tkinter.filedialog on a worker thread (FastAPI's async loop can't + block on a dialog). Returns {"path": "..."} on pick, {"path": null, + "cancelled": true} on cancel. Windows-only for now โ€” defaults to the + most likely MT5 install dir if nothing is configured yet.""" + import asyncio + import os + import platform + + if platform.system() != "Windows": + raise HTTPException(501, "Browse dialog is Windows-only for now.") + + # Pick a sensible initial directory: current configured path โ†’ its dir, + # or %ProgramFiles% as a fallback. + current = _get_mt5_config().get("terminal_path") or "" + if current and os.path.isfile(current): + initial_dir = os.path.dirname(current) + elif current and os.path.isdir(current): + initial_dir = current + else: + pf = os.environ.get("ProgramFiles", r"C:\Program Files") + initial_dir = pf + + def _pick() -> dict: + # v6: switched tkinter โ†’ native comdlg32 via ctypes. + # tkinter is not bundled in the PyInstaller build (strips by + # default unless explicitly hidden-imported), so the installed + # .exe would fail with "No module named 'tkinter'". comdlg32.dll + # is core Windows โ€” always available, no dependencies, no build + # change needed. Still runs on a worker thread because the native + # call blocks until the user picks or cancels. + try: + import ctypes + from ctypes import wintypes, Structure, byref, sizeof, c_void_p + + class OPENFILENAMEW(Structure): + _fields_ = [ + ("lStructSize", wintypes.DWORD), + ("hwndOwner", wintypes.HWND), + ("hInstance", wintypes.HINSTANCE), + ("lpstrFilter", wintypes.LPCWSTR), + ("lpstrCustomFilter",wintypes.LPWSTR), + ("nMaxCustFilter", wintypes.DWORD), + ("nFilterIndex", wintypes.DWORD), + ("lpstrFile", wintypes.LPWSTR), + ("nMaxFile", wintypes.DWORD), + ("lpstrFileTitle", wintypes.LPWSTR), + ("nMaxFileTitle", wintypes.DWORD), + ("lpstrInitialDir", wintypes.LPCWSTR), + ("lpstrTitle", wintypes.LPCWSTR), + ("Flags", wintypes.DWORD), + ("nFileOffset", wintypes.WORD), + ("nFileExtension", wintypes.WORD), + ("lpstrDefExt", wintypes.LPCWSTR), + ("lCustData", wintypes.LPARAM), + ("lpfnHook", c_void_p), + ("lpTemplateName", wintypes.LPCWSTR), + ("pvReserved", c_void_p), + ("dwReserved", wintypes.DWORD), + ("FlagsEx", wintypes.DWORD), + ] + + OFN_FILEMUSTEXIST = 0x00001000 + OFN_PATHMUSTEXIST = 0x00000800 + OFN_EXPLORER = 0x00080000 + OFN_NOCHANGEDIR = 0x00000008 + + buf = ctypes.create_unicode_buffer(4096) + ofn = OPENFILENAMEW() + ctypes.memset(byref(ofn), 0, sizeof(ofn)) + ofn.lStructSize = sizeof(OPENFILENAMEW) + ofn.lpstrFile = ctypes.cast(buf, wintypes.LPWSTR) + ofn.nMaxFile = 4096 + # Filter string: pairs of "label\0pattern\0" terminated with an extra \0. + ofn.lpstrFilter = ( + "MT5 terminal\0terminal64.exe;terminal.exe\0" + "Executables\0*.exe\0" + "All files\0*.*\0\0" + ) + ofn.lpstrInitialDir = initial_dir + ofn.lpstrTitle = "Select MT5 terminal (terminal64.exe)" + ofn.Flags = (OFN_FILEMUSTEXIST | OFN_PATHMUSTEXIST + | OFN_EXPLORER | OFN_NOCHANGEDIR) + + ok = ctypes.windll.comdlg32.GetOpenFileNameW(byref(ofn)) + if not ok: + # User cancelled OR dialog failed. CommDlgExtendedError + # returns 0 for cancel, non-zero for actual errors. + err = ctypes.windll.comdlg32.CommDlgExtendedError() + if err != 0: + return {"path": None, "error": f"comdlg32 error {err:#x}"} + return {"path": None, "cancelled": True} + + picked = buf.value + if not picked: + return {"path": None, "cancelled": True} + return {"path": os.path.normpath(picked), "cancelled": False} + except Exception as e: + log.error(f"browse-terminal dialog failed: {e}") + return {"path": None, "error": str(e)} + + try: + result = await asyncio.to_thread(_pick) + except Exception as e: + raise HTTPException(500, f"Dialog launch failed: {e}") + if result.get("error"): + raise HTTPException(500, result["error"]) + return result + + + # โ”€โ”€ POST /api/mt5/connect โ”€โ”€ + @router.post("/connect") + async def connect_mt5(request: Request): + import asyncio + + provider = cfg_manager.get_provider("mt5_default") + if provider is None: + # Try to init providers first + try: + await asyncio.to_thread(cfg_manager.init_providers) + provider = cfg_manager.get_provider("mt5_default") + except Exception as e: + raise HTTPException(500, f"Failed to initialize MT5 provider: {e}") + + if provider is None: + raise HTTPException(500, "MT5 provider not available") + + # If already connected, return current state + if provider.connected: + account = None + try: + import MetaTrader5 as mt5 + info = mt5.account_info() + if info: + account = { + "login": info.login, + "server": info.server, + "balance": info.balance, + "equity": info.equity, + "currency": info.currency, + } + except Exception: + pass + return {"connected": True, "account": account} + + # Try to connect + try: + connected = await asyncio.wait_for( + asyncio.to_thread(provider.connect), + timeout=15.0, + ) + except asyncio.TimeoutError: + return {"connected": False, "error": "Connection timed out (15s)"} + except Exception as e: + return {"connected": False, "error": str(e)} + + if not connected: + return {"connected": False, "error": "MT5 connection failed โ€” check terminal path or credentials"} + + # Update app state + app.state.provider = provider + + # Resolve symbols + try: + universe = cfg_manager.get_active_universe() + await asyncio.to_thread(provider.resolve_universe, universe) + except Exception: + pass + + # Get account info + account = None + try: + import MetaTrader5 as mt5 + info = mt5.account_info() + if info: + account = { + "login": info.login, + "server": info.server, + "balance": info.balance, + "equity": info.equity, + "currency": info.currency, + } + except Exception: + pass + + log.info("MT5 connected via settings panel") + return {"connected": True, "account": account} + + # โ”€โ”€ POST /api/mt5/disconnect โ”€โ”€ + @router.post("/disconnect") + async def disconnect_mt5(): + import asyncio + provider = cfg_manager.get_provider("mt5_default") + if provider and provider.connected: + try: + await asyncio.to_thread(provider.disconnect) + log.info("MT5 disconnected via settings panel") + except Exception as e: + log.warning(f"MT5 disconnect error: {e}") + return {"connected": False} + + # โ”€โ”€ GET /api/mt5/status โ”€โ”€ + @router.get("/status") + async def get_mt5_status(): + """Quick status check โ€” used by settings panel.""" + provider = cfg_manager.get_provider("mt5_default") + connected = provider.connected if provider else False + account = None + if connected: + try: + import MetaTrader5 as mt5 + info = mt5.account_info() + if info: + account = { + "login": info.login, + "server": info.server, + "balance": info.balance, + "equity": info.equity, + } + except Exception: + pass + return {"connected": connected, "account": account} + + return router diff --git a/backend/options_routes.py b/backend/options_routes.py new file mode 100644 index 0000000..b585f36 --- /dev/null +++ b/backend/options_routes.py @@ -0,0 +1,142 @@ +""" +options_routes.py โ€” Options Quant REST Endpoints +================================================= +Quantum Terminal | Phase 4F โ€” Options Quant Engine + +Wire into data_server.py: + from options_routes import create_options_router + app.include_router(create_options_router()) + +Endpoints: + GET /api/options/symbols โ€” supported canonical assets + Yahoo mapping + GET /api/options/{ticker}/snapshot โ€” full metrics (GEX + levels + IV divergence) + GET /api/options/{ticker}/chain โ€” raw calls/puts chain data + GET /api/options/{ticker}/expiries โ€” available expiry dates from Yahoo +""" + +import logging +from pathlib import Path +from fastapi import APIRouter, HTTPException, Query +from typing import Optional + +log = logging.getLogger("mk.options_routes") + +PROJECT_ROOT = Path(__file__).resolve().parent + + +def create_options_router() -> APIRouter: + router = APIRouter(tags=["options"]) + + from options_data_manager import get_options_dm, OPTIONS_SYMBOL_MAP + from options_engine import compute_all, compute_all_multi + + @router.get("/api/options/symbols") + async def get_options_symbols(): + """ + Return list of assets with options support and their Yahoo mappings. + """ + return { + "supported": [ + {"canonical": k, "cboe_ticker": v} + for k, v in OPTIONS_SYMBOL_MAP.items() + ] + } + + @router.get("/api/options/{ticker}/snapshot") + async def get_options_snapshot( + ticker: str, + expiry: Optional[str] = Query(None, description="ISO date e.g. 2025-03-21"), + ): + """ + Full options analytics snapshot for one asset. + Includes GEX, flip level, max pain, walls, P/C ratio, IV divergence. + Response is cached 15 minutes โ€” stale data served outside market hours. + """ + canonical = ticker.upper() + if canonical not in OPTIONS_SYMBOL_MAP: + raise HTTPException( + 404, + f"{canonical} not supported. Supported: {list(OPTIONS_SYMBOL_MAP.keys())}" + ) + + odm = get_options_dm() + snap = odm.get_snapshot(canonical, expiry=expiry) + + if snap is None: + raise HTTPException(503, f"Could not fetch options data for {canonical}. " + f"Check yfinance install and network.") + + try: + metrics = compute_all(snap) + except Exception as e: + log.error(f"Options engine error for {canonical}: {e}") + raise HTTPException(500, f"Options calculation failed: {e}") + + return metrics + + @router.get("/api/options/{ticker}/chain") + async def get_options_chain( + ticker: str, + expiry: Optional[str] = Query(None), + ): + """ + Raw options chain (calls + puts) for one asset and expiry. + Useful for frontend to render custom strike tables. + """ + canonical = ticker.upper() + if canonical not in OPTIONS_SYMBOL_MAP: + raise HTTPException(404, f"{canonical} not supported") + + odm = get_options_dm() + snap = odm.get_snapshot(canonical, expiry=expiry) + + if snap is None: + raise HTTPException(503, f"Could not fetch chain for {canonical}") + + return { + "canonical": snap["canonical"], + "cboe_ticker": snap.get("cboe_ticker"), + "expiry": snap.get("expiry"), + "spot": snap.get("spot"), + "calls": snap.get("calls", []), + "puts": snap.get("puts", []), + "cached_at": snap.get("cached_at"), + "is_stale": snap.get("is_stale", False), + } + + @router.get("/api/options/{ticker}/expiries") + async def get_options_expiries(ticker: str): + """Available expiry dates. Returns list of ISO date strings.""" + canonical = ticker.upper() + if canonical not in OPTIONS_SYMBOL_MAP: + raise HTTPException(404, f"{canonical} not supported") + odm = get_options_dm() + return {"canonical": canonical, "expiries": odm.get_available_expiries(canonical)} + + @router.get("/api/options/{ticker}/full") + async def get_options_full( + ticker: str, + n: int = Query(8, description="Number of expiries for GEX stack (1-12)"), + ): + """ + Full multi-expiry options analytics. + Includes GEX stack (GEX1โ€“GEXn), 0DTE GEX, HVL, expected move. + This is the primary endpoint for the enhanced OPTIONS QUANT UI. + """ + canonical = ticker.upper() + if canonical not in OPTIONS_SYMBOL_MAP: + raise HTTPException(404, f"{canonical} not supported") + + odm = get_options_dm() + n = max(1, min(n, 12)) + snap = odm.get_full_snapshot(canonical, n_expiries=n) + if snap is None: + raise HTTPException(503, f"Could not fetch options data for {canonical}") + + try: + return compute_all_multi(snap) + except Exception as e: + log.error(f"Options multi-expiry calc error {canonical}: {e}") + raise HTTPException(500, f"Calculation failed: {e}") + + return router \ No newline at end of file diff --git a/backend/periodic_sync.py b/backend/periodic_sync.py new file mode 100644 index 0000000..145915a --- /dev/null +++ b/backend/periodic_sync.py @@ -0,0 +1,172 @@ +# version: v2 +""" +================================================================================ +Quantum Terminal Consumer โ€” Periodic Data Sync Poller +================================================================================ +Background asyncio task that polls the VPS at a configurable interval and +broadcasts a 'data_sync_complete' WebSocket event whenever fresh data arrives. +This solves the problem where a long-running terminal would never re-check +the server after the initial startup gate. + +Configurable via consumer_config.ini: + [sync] + poll_interval_seconds = 300 + +Default interval: 300s (5 minutes). Floor: 60s (no hammering the server). + +Rule C1 compliance: this module performs ZERO calculations. It only re-runs +the same display-data sync as the startup gate (consumer_gate -> sync_all). + +This module does NOT introduce any calculation triggers. +================================================================================ +""" + +import asyncio +import logging +import os + +# v2: AppData dir name (env-var driven; default "QuantumTerminal" preserves v1 path). +APP_DIR = os.environ.get("MK_APP_DIR_NAME", "QuantumTerminal") +import configparser +from datetime import datetime, timezone +from pathlib import Path +from typing import Awaitable, Callable + +from pulse_room.sync_hook import on_sync_complete as _pulse_on_sync_complete + +log = logging.getLogger("mk.periodic_sync") + +DEFAULT_INTERVAL_SECONDS = 300 # 5 minutes +MIN_INTERVAL_SECONDS = 60 # safety floor + + +def _read_interval() -> int: + """ + Read sync.poll_interval_seconds from consumer_config.ini. + Searches the same locations as data_sync_client._read_config(). + Returns DEFAULT_INTERVAL_SECONDS if not configured. + Floor is enforced at MIN_INTERVAL_SECONDS. + """ + project_root = Path(__file__).resolve().parent + + candidates = [ + project_root / "consumer_config.ini", + project_root.parent / "consumer_config.ini", + ] + + import sys as _sys + if getattr(_sys, "frozen", False): + candidates.append(Path(_sys.executable).parent / "consumer_config.ini") + + appdata = os.environ.get("APPDATA", "") + if appdata: + candidates.append(Path(appdata) / APP_DIR / "consumer_config.ini") + + for p in candidates: + if not p.exists(): + continue + try: + cp = configparser.ConfigParser() + cp.read(str(p), encoding="utf-8-sig") + if cp.has_option("sync", "poll_interval_seconds"): + val = cp.getint("sync", "poll_interval_seconds") + return max(MIN_INTERVAL_SECONDS, val) + break + except Exception as e: + log.warning(f"Could not parse sync interval from {p.name}: {e}") + break + + return DEFAULT_INTERVAL_SECONDS + + +def create_periodic_sync_task( + broadcast_event: Callable[[dict], Awaitable[None]], +) -> Callable: + """ + Returns an async coroutine factory that polls sync_all() at a configurable + interval. Pass the result to asyncio.create_task() inside the FastAPI + lifespan. + + Usage in data_server.py lifespan: + from periodic_sync import create_periodic_sync_task + sync_task = create_periodic_sync_task(manager.broadcast_event) + tasks.append(asyncio.create_task(sync_task())) + + Each tick the task: + 1. Calls data_sync_client.sync_all() in a worker thread + 2. Logs the result (synced / skipped / offline) + 3. If new files arrived (synced > 0), broadcasts WS event: + {"type": "data_sync_complete", "synced": N, ...} + Frontend (useTerminalData.js) listens and refreshes panels. + + Errors do not kill the task โ€” they back off and retry on the next interval. + """ + interval = _read_interval() + log.info(f"Periodic sync task configured: interval={interval}s") + + async def _task(): + # Wait one full interval before first poll โ€” + # the startup gate has already done the initial sync. + try: + await asyncio.sleep(interval) + except asyncio.CancelledError: + return + + while True: + try: + from data_sync_client import get_sync_client + client = get_sync_client() + + # sync_all() does blocking HTTP โ€” run in a thread + summary = await asyncio.to_thread(client.sync_all) + + synced = summary.get("synced", 0) + skipped = summary.get("skipped", 0) + offline = summary.get("offline_mode", False) + + log.info( + f"Periodic sync tick: synced={synced}, " + f"skipped={skipped}, offline={offline}" + ) + + # Only broadcast when fresh data actually arrived. + # Skipping silent ticks keeps frontend quiet during quiet hours. + if synced > 0: + try: + await broadcast_event({ + "type": "data_sync_complete", + "synced": synced, + "skipped": skipped, + "offline_mode": offline, + "timestamp": datetime.now(timezone.utc).isoformat(), + }) + except Exception as e: + log.warning(f"Failed to broadcast data_sync_complete: {e}") + + # Pulse Room auto-rebuild โ€” only when fresh source data + # arrived. Fires only if scanner ON + watchlist non-empty + # (eligibility check is inside on_sync_complete itself); + # daemon thread, fire-and-forget; never breaks sync pipeline. + try: + _pulse_on_sync_complete() + except Exception: + log.exception("pulse_room.sync_hook threw") + + except asyncio.CancelledError: + log.info("Periodic sync task cancelled") + break + except Exception as e: + log.error(f"Periodic sync task error: {e}", exc_info=True) + # Back off on error but never longer than the configured interval + try: + await asyncio.sleep(min(interval, 120)) + except asyncio.CancelledError: + break + continue + + try: + await asyncio.sleep(interval) + except asyncio.CancelledError: + break + + return _task diff --git a/backend/providers/__init__.py b/backend/providers/__init__.py new file mode 100644 index 0000000..7e45164 --- /dev/null +++ b/backend/providers/__init__.py @@ -0,0 +1,68 @@ +""" +================================================================================ +Quantum Terminal โ€” Provider Registry +================================================================================ +Maps provider type names to their implementation classes. + +To add a new provider: + 1. Create providers/my_provider.py implementing BaseProvider + 2. Add "my_provider": MyProvider to PROVIDER_REGISTRY below + 3. Add account config in user_config.json under providers.accounts + +That's it โ€” the config_manager will instantiate and manage it. +================================================================================ +""" + +from providers.base_provider import BaseProvider +from providers.mt5_provider import MT5Provider + +# โ”€โ”€ Rithmic provider (graceful if async_rithmic not installed) โ”€โ”€ +try: + from providers.rithmic_provider import RithmicProvider + RITHMIC_AVAILABLE = True +except ImportError: + RithmicProvider = None + RITHMIC_AVAILABLE = False + +# โ”€โ”€ Provider type โ†’ class mapping โ”€โ”€ +# Key = the "type" field in account config +# Value = class that implements BaseProvider +PROVIDER_REGISTRY = { + "mt5": MT5Provider, +} + +# Register Rithmic only if async_rithmic is installed +if RITHMIC_AVAILABLE: + PROVIDER_REGISTRY["rithmic"] = RithmicProvider + + +def create_provider(account_config: dict) -> BaseProvider: + """ + Factory: create a provider instance from account config dict. + + Expected config shape: + { + "id": "mt5_primary", + "type": "mt5", + "label": "MT5 โ€” CFI (Live)", + "terminal_path": null, + "aliases": {} + } + + Raises KeyError if provider type is not registered. + """ + provider_type = account_config.get("type", "") + if provider_type not in PROVIDER_REGISTRY: + registered = ", ".join(PROVIDER_REGISTRY.keys()) + raise KeyError( + f"Unknown provider type: '{provider_type}'. " + f"Registered types: {registered}" + ) + + cls = PROVIDER_REGISTRY[provider_type] + return cls(account_config) + + +def list_provider_types() -> list: + """Return list of registered provider type names.""" + return list(PROVIDER_REGISTRY.keys()) diff --git a/backend/providers/base_provider.py b/backend/providers/base_provider.py new file mode 100644 index 0000000..5d8f287 --- /dev/null +++ b/backend/providers/base_provider.py @@ -0,0 +1,276 @@ +""" +================================================================================ +Quantum Terminal โ€” Base Provider Interface +================================================================================ +Abstract contract that every data/execution provider must implement. + +The data_server and config_manager talk to providers ONLY through this +interface. MT5, Binance, Polygon, or any future source plugs in by +subclassing BaseProvider and implementing the required methods. + +Design principles: + - All methods are synchronous (callers use asyncio.to_thread) + - Providers manage their own connection lifecycle + - Canonical ticker names everywhere โ€” providers map internally + - Providers declare their capabilities (data-only vs data+execution) + +Usage: + from providers.base_provider import BaseProvider + from providers.mt5_provider import MT5Provider + + provider = MT5Provider(account_config) + provider.connect() + ticks = provider.get_latest_ticks(["XAUUSD", "EURUSD"]) +================================================================================ +""" + +from abc import ABC, abstractmethod +from typing import Dict, List, Optional +from models import ( + TickData, BarData, AccountInfo, SymbolInfo, + OrderRequest, OrderResult, Position, PendingOrder, +) + + +class BaseProvider(ABC): + """ + Abstract provider interface. + + Every provider has: + - A type name (e.g., "mt5", "binance") + - A unique instance ID (e.g., "mt5_primary", "binance_spot") + - Connection lifecycle (connect/disconnect/reconnect) + - Market data methods (ticks, bars, symbol info) + - Optional execution methods (orders, positions) + + Providers are synchronous. The data_server wraps calls in + asyncio.to_thread() to avoid blocking the event loop. + """ + + # โ”€โ”€ Identity โ”€โ”€ + + @property + @abstractmethod + def provider_type(self) -> str: + """Provider type identifier. E.g., 'mt5', 'binance', 'polygon'.""" + ... + + @property + @abstractmethod + def provider_id(self) -> str: + """Unique instance ID. E.g., 'mt5_primary'. Set from account config.""" + ... + + @property + @abstractmethod + def label(self) -> str: + """Human-readable label. E.g., 'MT5 โ€” CFI (Live)'.""" + ... + + # โ”€โ”€ Capabilities โ”€โ”€ + + @property + def can_stream_ticks(self) -> bool: + """Whether this provider supports live tick polling.""" + return True + + @property + def can_execute(self) -> bool: + """Whether this provider supports order execution.""" + return False + + @property + def supported_timeframes(self) -> List[str]: + """List of timeframe strings this provider supports.""" + return ["M1", "M5", "M15", "M30", "H1", "H4", "D1", "W1"] + + # โ”€โ”€ Connection Lifecycle โ”€โ”€ + + @property + @abstractmethod + def connected(self) -> bool: + """Whether the provider is currently connected.""" + ... + + @abstractmethod + def connect(self) -> bool: + """ + Establish connection to the data/execution source. + Returns True on success, False on failure. + Must be idempotent โ€” calling connect() when already connected is safe. + """ + ... + + @abstractmethod + def disconnect(self) -> None: + """Cleanly shut down the connection.""" + ... + + def reconnect(self) -> bool: + """Disconnect and reconnect. Override for custom reconnect logic.""" + self.disconnect() + return self.connect() + + def heartbeat(self) -> bool: + """ + Lightweight connection health check. + Returns True if the connection is alive, False otherwise. + If False, sets internal connected state to False so reconnect_loop picks it up. + Override in subclasses for provider-specific health checks. + """ + return self.connected + + # โ”€โ”€ Market Data โ”€โ”€ + + @abstractmethod + def get_latest_ticks(self, symbols: List[str]) -> Dict[str, TickData]: + """ + Fetch latest tick for each symbol. + + Args: + symbols: List of canonical ticker names (e.g., ["XAUUSD", "EURUSD"]) + + Returns: + Dict mapping canonical ticker โ†’ TickData. + Missing/failed symbols are simply omitted. + """ + ... + + @abstractmethod + def get_bars( + self, ticker: str, timeframe: str = "M15", count: int = 200 + ) -> List[BarData]: + """ + Fetch recent OHLCV bars for a canonical ticker. + + Args: + ticker: Canonical symbol name + timeframe: Timeframe string (M1, M5, M15, H1, H4, D1, etc.) + count: Number of bars to fetch + + Returns: + List of BarData, oldest first. Empty list on failure. + """ + ... + + @abstractmethod + def check_new_bars( + self, symbols: List[str], timeframe: str = "M15" + ) -> List[dict]: + """ + Detect newly closed bars since last check. + + Returns list of dicts: + {"type": "bar", "ticker": str, "timeframe": str, "bar": BarData.to_dict()} + + Implementation must track last-seen bar timestamps internally. + """ + ... + + @abstractmethod + def get_symbol_info(self, ticker: str) -> Optional[SymbolInfo]: + """ + Get metadata for a symbol (decimals, lot sizing, contract size, etc.) + Returns None if symbol not found. + """ + ... + + def get_all_symbol_info(self, symbols: List[str]) -> Dict[str, SymbolInfo]: + """ + Batch symbol info for multiple tickers. + Default implementation calls get_symbol_info() in a loop. + Override for providers that support batch queries. + """ + result = {} + for s in symbols: + info = self.get_symbol_info(s) + if info is not None: + result[s] = info + return result + + # โ”€โ”€ Account Info โ”€โ”€ + + @abstractmethod + def get_account_info(self) -> Optional[AccountInfo]: + """ + Get current account snapshot (balance, equity, margin, etc.) + Returns None if not connected or not applicable. + """ + ... + + # โ”€โ”€ Execution (optional โ€” override if can_execute is True) โ”€โ”€ + + def place_order(self, order: OrderRequest) -> OrderResult: + """Place an order. Override in execution-capable providers.""" + return OrderResult( + success=False, + error=f"Provider '{self.provider_type}' does not support execution", + ) + + def get_positions(self) -> List[Position]: + """Get all open positions. Override in execution-capable providers.""" + return [] + + def close_position(self, ticket: str, lots: Optional[float] = None) -> OrderResult: + """ + Close a position (fully or partially). + Override in execution-capable providers. + """ + return OrderResult( + success=False, + error=f"Provider '{self.provider_type}' does not support execution", + ) + + def get_bars_range(self, ticker, timeframe, from_dt, to_dt): + """Fetch OHLCV bars between two datetimes. + Override in providers that support historical range queries.""" + return [] + + def get_pending_orders(self) -> List[PendingOrder]: + """ + List all resting (non-filled) pending orders (LIMIT / STOP). + Override in execution-capable providers. + """ + return [] + + def cancel_order(self, ticket: str) -> OrderResult: + """Cancel a resting pending order. Override in execution-capable providers.""" + return OrderResult( + success=False, + error=f"Provider '{self.provider_type}' does not support execution", + ) + + def modify_order( + self, ticket: str, + price: Optional[float] = None, + stop_loss: Optional[float] = None, + take_profit: Optional[float] = None, + ) -> OrderResult: + """Modify price / SL / TP on a pending order. Override in execution-capable providers.""" + return OrderResult( + success=False, + error=f"Provider '{self.provider_type}' does not support execution", + ) + + # โ”€โ”€ Symbol Resolution โ”€โ”€ + + @abstractmethod + def resolve_symbol(self, canonical: str) -> Optional[str]: + """ + Map a canonical ticker name to the provider's native symbol. + Returns None if the symbol is not available in this provider. + """ + ... + + def get_available_symbols(self) -> List[str]: + """ + Return list of all canonical symbols this provider can serve. + Default: empty (override to support symbol discovery). + """ + return [] + + # โ”€โ”€ String Representation โ”€โ”€ + + def __repr__(self) -> str: + status = "connected" if self.connected else "disconnected" + return f"<{self.__class__.__name__} id='{self.provider_id}' [{status}]>" \ No newline at end of file diff --git a/backend/providers/mt5_provider.py b/backend/providers/mt5_provider.py new file mode 100644 index 0000000..0e5f749 --- /dev/null +++ b/backend/providers/mt5_provider.py @@ -0,0 +1,1039 @@ +# version: v6 +""" +================================================================================ +Quantum Terminal โ€” MetaTrader 5 Provider + +v6 โ€” Auto-link broker symbol via fuzzy match. Brokers wrap canonical names + in arbitrary suffixes / case variants โ€” XAUUSD might be stored as + `xauusd`, `XAUUSD.x`, `XAUUSD.cash`, `XAUUSDi`, `XAUUSD!`, `XAUUSD-cfd` + etc.; common-name swaps like USTEC โ†” NAS100 / US100, US30 โ†” DOW are + also widespread. Previously these required manual entry in Settings โ†’ + Custom Tickers โ†’ broker_symbol. Now `resolve_symbol()` enumerates the + broker's symbol catalog (`mt5.symbols_get()`), normalizes each name + (lowercased, alphanumeric-only), and matches against the canonical + plus a known-alias list. First hit wins, gets cached, logs an INFO + line so the operator can see the auto-link. User overrides from + Settings still take precedence. + +v2 โ€” Added broker-symbol override hook. set_broker_symbol_lookup(fn) lets + the ConfigManager wire a runtime lookup for user-configured overrides + (Settings panel โ†’ custom_tickers[X].broker_symbol). resolve_symbol() + checks the override FIRST before the canonical / alias chain. New + reset_symbol(canonical) method evicts a ticker's cached mapping so the + next resolve picks up a changed override without requiring a restart. +================================================================================ +================================================================================ +Implements BaseProvider for MetaTrader 5. + +This is the refactored MT5Adapter from data_server.py. Same proven logic: + - Symbol alias resolution (XAUUSD โ†’ GOLD, GER40 โ†’ DE40, etc.) + - Tick polling, bar fetching, new bar detection + - Synchronous API wrapped by callers in asyncio.to_thread() + +New in provider version: + - Implements BaseProvider interface (swappable) + - Account info reporting + - Execution methods (place_order, get_positions, close_position) + - Symbol info extraction from MT5 symbol_info() + - Reads config from account dict, not ServerConfig + +Usage: + from providers.mt5_provider import MT5Provider + + provider = MT5Provider({ + "id": "mt5_primary", + "label": "MT5 โ€” CFI (Live)", + "terminal_path": None, # auto-detect + "aliases": { "GER40": ["GER40", "DE40", "DAX40"] }, + }) + provider.connect() + ticks = provider.get_latest_ticks(["XAUUSD", "EURUSD"]) +================================================================================ +""" + +import logging +from datetime import datetime, timezone +from typing import Dict, List, Optional, Any + +from models import ( + TickData, BarData, AccountInfo, SymbolInfo, + OrderRequest, OrderResult, Position, PendingOrder, +) +from providers.base_provider import BaseProvider + +log = logging.getLogger("provider.mt5") + +# โ”€โ”€ MT5 imported lazily โ€” provider works in degraded mode if not installed โ”€โ”€ +try: + import MetaTrader5 as mt5 + MT5_AVAILABLE = True +except ImportError: + mt5 = None + MT5_AVAILABLE = False + +# โ”€โ”€ MT5 timeframe constants โ”€โ”€ +MT5_TIMEFRAMES = { + "M1": 1, + "M5": 5, + "M15": 15, + "M30": 30, + "H1": 16385, + "H4": 16388, + "D1": 16408, + "W1": 32769, +} + +# โ”€โ”€ MT5 order type mapping โ”€โ”€ +MT5_ORDER_TYPES = { + "MARKET_BUY": None, # Populated after mt5 import check + "MARKET_SELL": None, + "LIMIT_BUY": None, + "LIMIT_SELL": None, + "STOP_BUY": None, + "STOP_SELL": None, +} + +if MT5_AVAILABLE: + MT5_ORDER_TYPES.update({ + "MARKET_BUY": mt5.ORDER_TYPE_BUY, + "MARKET_SELL": mt5.ORDER_TYPE_SELL, + "LIMIT_BUY": mt5.ORDER_TYPE_BUY_LIMIT, + "LIMIT_SELL": mt5.ORDER_TYPE_SELL_LIMIT, + "STOP_BUY": mt5.ORDER_TYPE_BUY_STOP, + "STOP_SELL": mt5.ORDER_TYPE_SELL_STOP, + }) + + +# v6: known-alias table for the auto-link fuzzy match. When a canonical +# ticker's exact name isn't found at the broker, resolve_symbol() will +# look for any of these names too, in addition to fuzzy-normalizing both +# sides (lowercased, alphanumeric-only) so suffix/case quirks +# (XAUUSD.x, XAUUSDi, xauusd, XAUUSD-cfd, etc.) auto-resolve. +# Keep entries CASE-INSENSITIVE โ€” they're normalized on lookup. The +# canonical itself doesn't need to be repeated here. +KNOWN_ALIASES: Dict[str, List[str]] = { + "US500": ["SPX500", "SP500", "USA500", "ES500", "WS500", "S&P500", "USA500IDX"], + "USTEC": ["NAS100", "US100", "USTECH100", "USTEC100", "NDX", "NQ100", "NAS100IDX"], + "US30": ["DOW", "DJ30", "WS30", "INDU", "DJ30IDX", "US30Cash"], + "GER40": ["DE40", "GER30", "DAX", "DAX40", "DE30", "GER30Cash"], + "UK100": ["FTSE100", "FTSE", "UK100Cash"], + "JP225": ["NIK225", "NIKKEI", "NIKKEI225", "JP225Cash"], + "FRA40": ["CAC40", "CAC"], + "ESP35": ["IBEX35", "IBEX"], + "AUS200": ["ASX200", "AU200"], + "HK50": ["HKG33", "HKG50", "HSI"], + "XAUUSD": ["GOLD", "XAU", "XAUUSDX"], + "XAGUSD": ["SILVER", "XAG", "XAGUSDX"], + "XPTUSD": ["PLATINUM"], + "XPDUSD": ["PALLADIUM"], + "XTIUSD": ["WTI", "USOIL", "OIL", "CRUDE", "WTIUSD", "OILUSD"], + "BRENT": ["UKOIL", "BCOUSD", "BRENTUSD", "UKOIL.cash"], + "BTCUSD": ["BITCOIN", "BTC"], + "ETHUSD": ["ETHEREUM", "ETH"], + "NATGAS": ["NGAS", "NATURALGAS", "GAS"], + "COPPER": ["XCUUSD", "HG"], +} + + +class MT5Provider(BaseProvider): + """ + MetaTrader 5 data and execution provider. + + Config dict keys: + id: str โ€” unique provider instance ID (e.g., "mt5_primary") + label: str โ€” display name (e.g., "MT5 โ€” CFI (Live)") + terminal_path: str or None โ€” path to terminal64.exe (None = auto-detect) + aliases: dict โ€” canonical โ†’ [broker_names] override map + """ + + def __init__(self, account_config: Dict[str, Any]): + self._id = account_config.get("id", "mt5_default") + self._label = account_config.get("label", "MetaTrader 5") + self._terminal_path = account_config.get("terminal_path", None) + self._aliases = account_config.get("aliases", {}) + self._connected = False + + # Symbol resolution caches + self._symbol_map: Dict[str, str] = {} # canonical โ†’ broker name + self._reverse_map: Dict[str, str] = {} # broker name โ†’ canonical + self._resolved_symbols: List[str] = [] # canonical names that resolved + # v6: lazy normalized index of the broker's full symbol catalog so + # resolve_symbol() can do an alphanumeric-only case-insensitive + # match. Built on first use after connect; cleared on disconnect. + self._broker_symbols_norm: Dict[str, str] = {} # normalized โ†’ real broker name + + # Bar tracking for check_new_bars + self._last_bar_times: Dict[str, int] = {} # "TICKER_TF" โ†’ epoch + + # v2: runtime broker-symbol override lookup (wired by ConfigManager). + # Callable(canonical: str) -> Optional[str]. + self._broker_symbol_lookup = None + + # โ”€โ”€ v6: fuzzy auto-link helpers โ”€โ”€ + + @staticmethod + def _normalize_symbol_name(name: str) -> str: + """Lower-case, alphanumeric-only. So `XAUUSD.x`, `xauusd!`, + `XAUUSD-cfd`, `xau usd` all collapse to `xauusd`.""" + if not name: + return "" + return "".join(ch.lower() for ch in str(name) if ch.isalnum()) + + def _ensure_broker_index(self) -> None: + """Lazy-build the normalized โ†’ real-name lookup over every symbol + the broker exposes. Cheap (< 50ms for 1000+ symbols) and runs only + once per connection. Cleared on disconnect / reset.""" + if self._broker_symbols_norm or not self._connected: + return + try: + all_syms = mt5.symbols_get() + except Exception as e: + log.warning(f"[symbol auto-link] symbols_get() failed: {e}") + return + if not all_syms: + return + idx = {} + for s in all_syms: + try: + key = self._normalize_symbol_name(s.name) + if key and key not in idx: + idx[key] = s.name + except Exception: + continue + self._broker_symbols_norm = idx + log.info(f"[symbol auto-link] indexed {len(idx)} broker symbols") + + def _fuzzy_resolve(self, canonical: str) -> Optional[str]: + """Match `canonical` (and KNOWN_ALIASES[canonical]) against the + broker's full catalog with normalize-and-compare. First hit wins.""" + self._ensure_broker_index() + if not self._broker_symbols_norm: + return None + candidates: List[str] = [canonical] + KNOWN_ALIASES.get(canonical.upper(), []) + seen_keys = set() + for c in candidates: + key = self._normalize_symbol_name(c) + if not key or key in seen_keys: + continue + seen_keys.add(key) + real = self._broker_symbols_norm.get(key) + if real: + return real + return None + + # โ”€โ”€ Identity โ”€โ”€ + + @property + def provider_type(self) -> str: + return "mt5" + + @property + def provider_id(self) -> str: + return self._id + + @property + def label(self) -> str: + return self._label + + # โ”€โ”€ Capabilities โ”€โ”€ + + @property + def can_execute(self) -> bool: + return True # MT5 supports order execution + + @property + def supported_timeframes(self) -> List[str]: + return list(MT5_TIMEFRAMES.keys()) + + # โ”€โ”€ Connection โ”€โ”€ + + @property + def connected(self) -> bool: + return self._connected + + def connect(self) -> bool: + """Initialize MT5 connection. Returns True on success.""" + if not MT5_AVAILABLE: + log.warning("MetaTrader5 package not installed โ€” provider unavailable") + return False + + if self._connected: + return True # Idempotent + + init_kwargs = {} + if self._terminal_path: + init_kwargs["path"] = self._terminal_path + log.info(f"MT5 terminal path: {self._terminal_path}") + else: + log.info("MT5 terminal path: auto-detect") + + if not mt5.initialize(**init_kwargs): + log.error(f"MT5 initialize() failed: {mt5.last_error()}") + return False + + info = mt5.terminal_info() + if info is None: + log.error("MT5 terminal_info() returned None") + mt5.shutdown() + return False + + self._connected = True + log.info( + f"MT5 connected: {info.name} | " + f"Company: {info.company} | " + f"Build: {info.build}" + ) + return True + + def disconnect(self) -> None: + """Shutdown MT5 connection.""" + if self._connected and MT5_AVAILABLE: + mt5.shutdown() + self._connected = False + self._symbol_map.clear() + self._reverse_map.clear() + self._resolved_symbols.clear() + self._broker_symbols_norm.clear() # v6: drop broker-specific index on disconnect + log.info("MT5 disconnected") + + def heartbeat(self) -> bool: + """ + Lightweight MT5 health check. + Calls mt5.terminal_info() โ€” fast and read-only. + Returns False and marks disconnected if MT5 is unresponsive. + """ + if not self._connected or not MT5_AVAILABLE: + return False + try: + info = mt5.terminal_info() + if info is None: + self._connected = False + log.warning("MT5 heartbeat failed โ€” terminal_info() returned None") + return False + return True + except Exception as e: + self._connected = False + log.warning(f"MT5 heartbeat exception: {e}") + return False + + # โ”€โ”€ Symbol Resolution โ”€โ”€ + + def set_broker_symbol_lookup(self, fn) -> None: + """v2: Register a callable(canonical) -> Optional[str] that returns + the user-configured broker-symbol override for a ticker, if any. + Called on every cache miss in resolve_symbol so runtime config + changes take effect without provider restart.""" + self._broker_symbol_lookup = fn + + def reset_symbol(self, canonical: str) -> Optional[str]: + """v2: Evict a ticker's cached mapping and re-resolve. Call this + after the user changes broker_symbol in Settings so the next + tick/bar request uses the new mapping.""" + canonical = canonical.upper() + old = self._symbol_map.pop(canonical, None) + if old is not None: + self._reverse_map.pop(old, None) + # Also drop cached bar time so the next poll re-seeds. + for k in list(self._last_bar_times.keys()): + if k.startswith(canonical + "_"): + self._last_bar_times.pop(k, None) + return self.resolve_symbol(canonical) + + def resolve_symbol(self, canonical: str) -> Optional[str]: + """ + Map canonical name to broker symbol. + Order: + 0. user override (Settings panel) via broker_symbol_lookup callable โ€” v2 + 1. cache + 2. canonical name itself + 3. provider-level aliases (from account_config["aliases"]) + Caches result in _symbol_map for fast lookups. + """ + if not self._connected: + return None + + # v2: (0) user override takes precedence over cache โ€” so changing + # broker_symbol in Settings and calling reset_symbol() picks it up. + if self._broker_symbol_lookup is not None: + try: + override = self._broker_symbol_lookup(canonical) + except Exception as e: + log.warning(f"broker_symbol_lookup failed for {canonical}: {e}") + override = None + if override: + sym = mt5.symbol_info(override) + if sym is not None: + if not sym.visible: + mt5.symbol_select(override, True) + self._symbol_map[canonical] = override + self._reverse_map[override] = canonical + log.info(f"Symbol resolved via user override: {canonical} โ†’ {override}") + return override + else: + log.warning(f"User override {canonical} โ†’ {override} not found in MT5; falling back") + + # (1) cache + if canonical in self._symbol_map: + return self._symbol_map[canonical] + + # (2) canonical name directly + sym = mt5.symbol_info(canonical) + if sym is not None: + if not sym.visible: + mt5.symbol_select(canonical, True) + self._symbol_map[canonical] = canonical + self._reverse_map[canonical] = canonical + return canonical + + # v6: (2.5) fuzzy auto-link โ€” handle suffix/case variants and known + # common aliases (USTEC โ†” NAS100/US100, US30 โ†” DOW, etc.) without + # the user needing to set Settings โ†’ broker_symbol manually. + auto = self._fuzzy_resolve(canonical) + if auto is not None: + sym = mt5.symbol_info(auto) + if sym is not None: + if not sym.visible: + mt5.symbol_select(auto, True) + self._symbol_map[canonical] = auto + self._reverse_map[auto] = canonical + log.info(f"[symbol auto-link] {canonical} โ†’ {auto}") + return auto + + # (3) provider aliases + aliases = self._aliases.get(canonical, []) + for alias in aliases: + if alias == canonical: + continue + sym = mt5.symbol_info(alias) + if sym is not None: + if not sym.visible: + mt5.symbol_select(alias, True) + self._symbol_map[canonical] = alias + self._reverse_map[alias] = canonical + log.info(f"Symbol resolved: {canonical} โ†’ {alias}") + return alias + + log.warning(f"Symbol {canonical} not found in MT5 (tried {len(aliases)} aliases)") + return None + + def resolve_universe(self, universe: List[str]) -> List[str]: + """ + Resolve a full universe of canonical tickers. + Returns list of canonical names that successfully resolved. + Populates internal symbol maps. + """ + self._symbol_map.clear() + self._reverse_map.clear() + self._resolved_symbols.clear() + + for canonical in universe: + broker_name = self.resolve_symbol(canonical) + if broker_name is not None: + self._resolved_symbols.append(canonical) + if broker_name != canonical: + log.info(f" โœ“ {canonical} โ†’ {broker_name}") + else: + log.info(f" โœ“ {canonical}") + else: + log.warning(f" โœ— {canonical} โ€” not found") + + log.info(f"Resolved {len(self._resolved_symbols)}/{len(universe)} symbols") + return list(self._resolved_symbols) + + def get_available_symbols(self) -> List[str]: + """Return canonical names that have been successfully resolved.""" + return list(self._resolved_symbols) + + def _broker_symbol(self, canonical: str) -> str: + """Quick lookup: canonical โ†’ broker symbol (assumes already resolved).""" + return self._symbol_map.get(canonical, canonical) + + # โ”€โ”€ Market Data โ”€โ”€ + + def get_latest_ticks(self, symbols: List[str]) -> Dict[str, TickData]: + """Fetch latest tick for each symbol.""" + if not self._connected: + return {} + + ticks = {} + for canonical in symbols: + broker_sym = self._broker_symbol(canonical) + tick = mt5.symbol_info_tick(broker_sym) + if tick is None: + continue + + sym_info = mt5.symbol_info(broker_sym) + dec = sym_info.digits if sym_info else 5 + + ticks[canonical] = TickData( + ticker=canonical, + bid=round(tick.bid, dec), + ask=round(tick.ask, dec), + last=round(tick.last or tick.bid, dec), + time=datetime.fromtimestamp( + tick.time, tz=timezone.utc + ).strftime("%Y-%m-%dT%H:%M:%S"), + spread=round(tick.ask - tick.bid, dec), + ) + return ticks + + def get_bars( + self, ticker: str, timeframe: str = "M15", count: int = 200 + ) -> List[BarData]: + """Fetch recent OHLCV bars for a canonical ticker.""" + if not self._connected: + return [] + + tf_value = MT5_TIMEFRAMES.get(timeframe) + if tf_value is None: + log.error(f"Unknown timeframe: {timeframe}") + return [] + + broker_sym = self._broker_symbol(ticker) + rates = mt5.copy_rates_from_pos(broker_sym, tf_value, 0, count) + if rates is None or len(rates) == 0: + return [] + + sym_info = mt5.symbol_info(broker_sym) + dec = sym_info.digits if sym_info else 5 + + bars = [] + for r in rates: + bars.append(BarData( + time=datetime.fromtimestamp( + r[0], tz=timezone.utc + ).strftime("%Y-%m-%dT%H:%M:%S"), + open=round(float(r[1]), dec), + high=round(float(r[2]), dec), + low=round(float(r[3]), dec), + close=round(float(r[4]), dec), + volume=int(r[5]), + )) + return bars + + def get_bars_range( + self, ticker: str, timeframe: str, from_dt: datetime, to_dt: datetime, + ) -> List[BarData]: + """v5: Fetch OHLCV bars between two UTC datetimes via copy_rates_range. + Used by the Stress Lab last-signal panel to replay historical trades.""" + if not self._connected: + return [] + tf_value = MT5_TIMEFRAMES.get(timeframe) + if tf_value is None: + log.error(f"Unknown timeframe: {timeframe}") + return [] + broker_sym = self._broker_symbol(ticker) + # Ensure tz-aware UTC for the MT5 call + if from_dt.tzinfo is None: from_dt = from_dt.replace(tzinfo=timezone.utc) + if to_dt.tzinfo is None: to_dt = to_dt.replace(tzinfo=timezone.utc) + rates = mt5.copy_rates_range(broker_sym, tf_value, from_dt, to_dt) + if rates is None or len(rates) == 0: + return [] + sym_info = mt5.symbol_info(broker_sym) + dec = sym_info.digits if sym_info else 5 + bars = [] + for r in rates: + bars.append(BarData( + time=datetime.fromtimestamp(r[0], tz=timezone.utc) + .strftime("%Y-%m-%dT%H:%M:%S"), + open=round(float(r[1]), dec), + high=round(float(r[2]), dec), + low=round(float(r[3]), dec), + close=round(float(r[4]), dec), + volume=int(r[5]), + )) + return bars + + def check_new_bars( + self, symbols: List[str], timeframe: str = "M15" + ) -> List[dict]: + """ + Detect newly closed bars since last check. + Returns list of bar event dicts for broadcasting. + """ + if not self._connected: + return [] + + tf_value = MT5_TIMEFRAMES.get(timeframe) + if tf_value is None: + return [] + + new_bars = [] + for canonical in symbols: + broker_sym = self._broker_symbol(canonical) + rates = mt5.copy_rates_from_pos(broker_sym, tf_value, 0, 2) + if rates is None or len(rates) < 2: + continue + + # Second-to-last bar = most recently CLOSED bar + closed_bar = rates[-2] + bar_epoch = int(closed_bar[0]) + + cache_key = f"{canonical}_{timeframe}" + if cache_key in self._last_bar_times: + if bar_epoch > self._last_bar_times[cache_key]: + sym_info = mt5.symbol_info(broker_sym) + dec = sym_info.digits if sym_info else 5 + bar = BarData( + time=datetime.fromtimestamp( + bar_epoch, tz=timezone.utc + ).strftime("%Y-%m-%dT%H:%M:%S"), + open=round(float(closed_bar[1]), dec), + high=round(float(closed_bar[2]), dec), + low=round(float(closed_bar[3]), dec), + close=round(float(closed_bar[4]), dec), + volume=int(closed_bar[5]), + ) + new_bars.append({ + "type": "bar", + "ticker": canonical, + "timeframe": timeframe, + "bar": bar.to_dict(), + }) + + self._last_bar_times[cache_key] = bar_epoch + + return new_bars + + # โ”€โ”€ Symbol Info โ”€โ”€ + + def get_symbol_info(self, ticker: str) -> Optional[SymbolInfo]: + """Extract full symbol metadata from MT5.""" + if not self._connected: + return None + + broker_sym = self._broker_symbol(ticker) + sym = mt5.symbol_info(broker_sym) + if sym is None: + return None + + # Determine asset class from symbol path or properties + asset_class = self._classify_symbol(sym) + + return SymbolInfo( + ticker=ticker, + broker_symbol=broker_sym, + asset_class=asset_class, + decimals=sym.digits, + description=sym.description or ticker, + trade_allowed=sym.trade_mode != 0, + min_lot=sym.volume_min, + max_lot=sym.volume_max, + lot_step=sym.volume_step, + contract_size=sym.trade_contract_size, + currency_profit=sym.currency_profit, + currency_margin=sym.currency_margin, + tick_size=sym.trade_tick_size, + tick_value=sym.trade_tick_value, + ) + + def _classify_symbol(self, sym) -> str: + """Guess asset class from MT5 symbol properties.""" + path = (sym.path or "").lower() + desc = (sym.description or "").lower() + + if "forex" in path or "currencies" in path: + return "FX" + if "index" in path or "indices" in path: + return "INDEX" + if "commodit" in path or "metals" in path or "energy" in path: + return "COMMODITY" + if "crypto" in path: + return "CRYPTO" + if "stock" in path or "equit" in path: + return "STOCK" + # Heuristic fallbacks + if "gold" in desc or "silver" in desc or "oil" in desc: + return "COMMODITY" + if sym.currency_profit == sym.currency_margin and sym.digits >= 4: + return "FX" + return "OTHER" + + # โ”€โ”€ Account Info โ”€โ”€ + + def get_account_info(self) -> Optional[AccountInfo]: + """Get MT5 account snapshot.""" + if not self._connected: + return None + + info = mt5.account_info() + if info is None: + return None + + term = mt5.terminal_info() + return AccountInfo( + provider_type="mt5", + account_id=str(info.login), + label=self._label, + broker=info.company, + server=info.server, + currency=info.currency, + balance=info.balance, + equity=info.equity, + margin_free=info.margin_free, + leverage=info.leverage, + connected=True, + ) + + # โ”€โ”€ Execution โ”€โ”€ + + def place_order(self, order: OrderRequest) -> OrderResult: + """Place a trade order through MT5.""" + if not self._connected: + return OrderResult(success=False, error="MT5 not connected") + + broker_sym = self._broker_symbol(order.ticker) + sym_info = mt5.symbol_info(broker_sym) + if sym_info is None: + return OrderResult( + success=False, + error=f"Symbol {order.ticker} not found", + ) + + # Determine MT5 order type + type_key = f"{order.order_type}_{order.direction}" + mt5_type = MT5_ORDER_TYPES.get(type_key) + if mt5_type is None: + return OrderResult( + success=False, + error=f"Unsupported order type: {type_key}", + ) + + # Build request + request = { + "action": mt5.TRADE_ACTION_DEAL if order.order_type == "MARKET" else mt5.TRADE_ACTION_PENDING, + "symbol": broker_sym, + "volume": order.lots, + "type": mt5_type, + "magic": order.magic, + "comment": order.comment or "Quantum Terminal", + "type_time": mt5.ORDER_TIME_GTC, + "type_filling": mt5.ORDER_FILLING_IOC, + } + + if order.price is not None: + request["price"] = order.price + elif order.order_type == "MARKET": + # Use current ask/bid for market orders + tick = mt5.symbol_info_tick(broker_sym) + if tick: + request["price"] = tick.ask if order.direction == "BUY" else tick.bid + + if order.stop_loss is not None: + request["sl"] = order.stop_loss + if order.take_profit is not None: + request["tp"] = order.take_profit + + # Send order + result = mt5.order_send(request) + if result is None: + return OrderResult( + success=False, + error=f"MT5 order_send returned None: {mt5.last_error()}", + ) + + if result.retcode != mt5.TRADE_RETCODE_DONE: + return OrderResult( + success=False, + error=f"MT5 order rejected: {result.retcode} โ€” {result.comment}", + raw=result._asdict() if hasattr(result, '_asdict') else None, + ) + + return OrderResult( + success=True, + order_id=str(result.deal), + ticker=order.ticker, + direction=order.direction, + lots=order.lots, + price=result.price, + ) + + def get_positions(self) -> List[Position]: + """Get all open positions.""" + if not self._connected: + return [] + + positions = mt5.positions_get() + if positions is None: + return [] + + result = [] + for pos in positions: + # Map broker symbol back to canonical + canonical = self._reverse_map.get(pos.symbol, pos.symbol) + result.append(Position( + ticket=str(pos.ticket), + ticker=canonical, + direction="BUY" if pos.type == 0 else "SELL", + lots=pos.volume, + open_price=pos.price_open, + current_price=pos.price_current, + stop_loss=pos.sl if pos.sl != 0 else None, + take_profit=pos.tp if pos.tp != 0 else None, + profit=pos.profit, + swap=pos.swap, + open_time=datetime.fromtimestamp( + pos.time, tz=timezone.utc + ).strftime("%Y-%m-%dT%H:%M:%S"), + )) + return result + + def close_position( + self, ticket: str, lots: Optional[float] = None + ) -> OrderResult: + """Close a position by ticket (full or partial).""" + if not self._connected: + return OrderResult(success=False, error="MT5 not connected") + + # Find the position + positions = mt5.positions_get(ticket=int(ticket)) + if not positions or len(positions) == 0: + return OrderResult( + success=False, + error=f"Position {ticket} not found", + ) + + pos = positions[0] + close_lots = lots if lots is not None else pos.volume + + # Determine close direction (opposite of position) + close_type = mt5.ORDER_TYPE_SELL if pos.type == 0 else mt5.ORDER_TYPE_BUY + tick = mt5.symbol_info_tick(pos.symbol) + price = tick.bid if pos.type == 0 else tick.ask + + request = { + "action": mt5.TRADE_ACTION_DEAL, + "symbol": pos.symbol, + "volume": close_lots, + "type": close_type, + "position": pos.ticket, + "price": price, + "comment": "Quantum Terminal close", + "type_time": mt5.ORDER_TIME_GTC, + "type_filling": mt5.ORDER_FILLING_IOC, + } + + result = mt5.order_send(request) + if result is None: + return OrderResult( + success=False, + error=f"Close failed: {mt5.last_error()}", + ) + + if result.retcode != mt5.TRADE_RETCODE_DONE: + return OrderResult( + success=False, + error=f"Close rejected: {result.retcode} โ€” {result.comment}", + ) + + canonical = self._reverse_map.get(pos.symbol, pos.symbol) + return OrderResult( + success=True, + order_id=str(result.deal), + ticker=canonical, + direction="SELL" if pos.type == 0 else "BUY", + lots=close_lots, + price=result.price, + ) + + def modify_position( + self, ticket: str, stop_loss=None, take_profit=None + ) -> dict: + """Modify SL/TP on an open position.""" + if not self._connected: + return {"success": False, "error": "MT5 not connected"} + + positions = mt5.positions_get(ticket=int(ticket)) + if not positions or len(positions) == 0: + return {"success": False, "error": f"Position {ticket} not found"} + + pos = positions[0] + tick = mt5.symbol_info_tick(pos.symbol) + if not tick: + return {"success": False, "error": f"No tick data for {pos.symbol}"} + + # v3: Preserve the other side when only one is being modified. + # MT5 treats sl=0.0 / tp=0.0 as "clear the level", so passing None + # through as 0 would silently remove whichever side the caller + # omitted. Read the live pos.sl / pos.tp and pass them when the + # corresponding parameter is None. + request = { + "action": mt5.TRADE_ACTION_SLTP, + "symbol": pos.symbol, + "position": pos.ticket, + "sl": float(stop_loss) if stop_loss is not None else float(getattr(pos, "sl", 0.0) or 0.0), + "tp": float(take_profit) if take_profit is not None else float(getattr(pos, "tp", 0.0) or 0.0), + } + + result = mt5.order_send(request) + if result is None: + return {"success": False, "error": f"Modify failed: {mt5.last_error()}"} + + if result.retcode != mt5.TRADE_RETCODE_DONE: + return {"success": False, "error": f"Modify rejected: {result.retcode} โ€” {result.comment}"} + + return { + "success": True, + "ticket": ticket, + "stop_loss": stop_loss, + "take_profit": take_profit, + } + + # โ”€โ”€ Pending orders (LIMIT / STOP) โ€” v4 โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + + def get_pending_orders(self) -> List[PendingOrder]: + """List all resting pending orders (LIMIT / STOP) on this account.""" + if not self._connected: + return [] + orders = mt5.orders_get() + if orders is None: + return [] + out = [] + for o in orders: + t = o.type + if t == mt5.ORDER_TYPE_BUY_LIMIT: direction, kind = "BUY", "LIMIT" + elif t == mt5.ORDER_TYPE_SELL_LIMIT: direction, kind = "SELL", "LIMIT" + elif t == mt5.ORDER_TYPE_BUY_STOP: direction, kind = "BUY", "STOP" + elif t == mt5.ORDER_TYPE_SELL_STOP: direction, kind = "SELL", "STOP" + else: + continue # skip stop-limit / market / closed types + canonical = self._reverse_map.get(o.symbol, o.symbol) + out.append(PendingOrder( + ticket=str(o.ticket), + symbol=o.symbol, + ticker=canonical, + direction=direction, + order_type=kind, + lots=o.volume_current, + price=o.price_open, + stop_loss=o.sl if o.sl != 0 else None, + take_profit=o.tp if o.tp != 0 else None, + comment=o.comment or "", + time_setup=datetime.fromtimestamp( + o.time_setup, tz=timezone.utc + ).strftime("%Y-%m-%dT%H:%M:%S"), + )) + return out + + def cancel_order(self, ticket: str) -> OrderResult: + """Remove (cancel) a resting pending order.""" + if not self._connected: + return OrderResult(success=False, error="MT5 not connected") + orders = mt5.orders_get(ticket=int(ticket)) + if not orders: + return OrderResult(success=False, error=f"Order {ticket} not found") + req = {"action": mt5.TRADE_ACTION_REMOVE, "order": int(ticket)} + result = mt5.order_send(req) + if result is None: + return OrderResult(success=False, error=f"Cancel failed: {mt5.last_error()}") + if result.retcode != mt5.TRADE_RETCODE_DONE: + return OrderResult( + success=False, + error=f"Cancel rejected: {result.retcode} โ€” {result.comment}", + ) + return OrderResult(success=True, order_id=int(ticket)) + + def modify_order( + self, ticket: str, + price: Optional[float] = None, + stop_loss: Optional[float] = None, + take_profit: Optional[float] = None, + ) -> OrderResult: + """ + Modify a resting pending order's trigger price and/or SL/TP. + Parameters left as None are preserved from the existing order so + the caller can patch a single field without clobbering the rest. + """ + if not self._connected: + return OrderResult(success=False, error="MT5 not connected") + orders = mt5.orders_get(ticket=int(ticket)) + if not orders: + return OrderResult(success=False, error=f"Order {ticket} not found") + o = orders[0] + req = { + "action": mt5.TRADE_ACTION_MODIFY, + "order": int(ticket), + "symbol": o.symbol, + "price": float(price) if price is not None else float(o.price_open), + "sl": float(stop_loss) if stop_loss is not None else float(o.sl or 0.0), + "tp": float(take_profit) if take_profit is not None else float(o.tp or 0.0), + "type_time": o.type_time, + "type_filling": o.type_filling, + } + result = mt5.order_send(req) + if result is None: + return OrderResult(success=False, error=f"Modify failed: {mt5.last_error()}") + if result.retcode != mt5.TRADE_RETCODE_DONE: + return OrderResult( + success=False, + error=f"Modify rejected: {result.retcode} โ€” {result.comment}", + ) + return OrderResult(success=True, order_id=int(ticket)) + + def get_trade_history(self, days: int = 7) -> list: + """ + Get closed trade history from MT5 using history_deals_get(). + Groups DEAL_ENTRY_IN + DEAL_ENTRY_OUT by position_id into ClosedTrade records. + """ + from models import ClosedTrade + from datetime import timedelta + + if not self._connected: + return [] + + utc_to = datetime.now(timezone.utc) + utc_from = utc_to - timedelta(days=days) + + deals = mt5.history_deals_get(utc_from, utc_to) + if deals is None or len(deals) == 0: + return [] + + # Group deals by position_id + positions = {} # position_id -> {"in": deal, "out": deal} + for deal in deals: + pid = deal.position_id + if pid == 0: + continue # Balance/correction operations + if pid not in positions: + positions[pid] = {"in": None, "out": None} + # entry=0 is DEAL_ENTRY_IN, entry=1 is DEAL_ENTRY_OUT + if deal.entry == 0: + positions[pid]["in"] = deal + elif deal.entry == 1: + positions[pid]["out"] = deal + + # Build ClosedTrade records (only complete round-trips) + trades = [] + for pid, pair in positions.items(): + entry = pair["in"] + exit_deal = pair["out"] + if entry is None or exit_deal is None: + continue # Partial โ€” still open or missing leg + + # Resolve canonical ticker from broker symbol + canonical = entry.symbol + for canon, aliases in self._aliases.items(): + if entry.symbol in aliases: + canonical = canon + break + + direction = "BUY" if entry.type == 0 else "SELL" # DEAL_TYPE_BUY=0 + trades.append(ClosedTrade( + ticket=str(pid), + ticker=canonical, + direction=direction, + lots=entry.volume, + open_price=entry.price, + close_price=exit_deal.price, + profit=exit_deal.profit, + commission=round((entry.commission or 0) + (exit_deal.commission or 0), 2), + swap=round((entry.swap or 0) + (exit_deal.swap or 0), 2), + open_time=datetime.fromtimestamp(entry.time, tz=timezone.utc).isoformat(), + close_time=datetime.fromtimestamp(exit_deal.time, tz=timezone.utc).isoformat(), + comment=entry.comment or "", + magic=entry.magic, + )) + + # Sort newest first + trades.sort(key=lambda t: t.close_time, reverse=True) + return trades \ No newline at end of file diff --git a/backend/providers/rithmic_provider.py b/backend/providers/rithmic_provider.py new file mode 100644 index 0000000..1eb8ab8 --- /dev/null +++ b/backend/providers/rithmic_provider.py @@ -0,0 +1,758 @@ +""" +================================================================================ +Quantum Terminal โ€” Rithmic Provider +================================================================================ +Implements BaseProvider for Rithmic futures data feed. + +Uses the async_rithmic library (Protocol Buffer API over WebSocket) to: + - Stream live BBO ticks for futures instruments + - Stream live time bars (M1/M5/M15/M30/H1) + - Fetch historical bars on demand + - Auto-resolve front month contracts (ES โ†’ ESM6, etc.) + +Architecture: + async_rithmic requires running inside a proper asyncio event loop. + This provider exposes async_connect()/async_disconnect() methods that + run directly in the caller's event loop (FastAPI's uvicorn loop). + + data_server.py lifespan calls: + await provider.async_connect() # in the main event loop + + Sync methods (get_latest_ticks, get_bars, etc.) read from caches + populated by streaming callbacks running in the same event loop. + + For non-async contexts (test scripts), connect() falls back to + asyncio.run() which works but blocks the calling thread. + +Config dict keys: + id: str โ€” unique provider ID (e.g., "rithmic_default") + type: str โ€” "rithmic" + label: str โ€” display name (e.g., "Rithmic โ€” Paper Trading") + enabled: bool โ€” True + user: str โ€” Rithmic username (from local_config.ini) + password: str โ€” Rithmic password (from local_config.ini) + system_name: str โ€” "Rithmic Test" / "Rithmic 01" etc. + url: str โ€” server URL (e.g., "rituz00100.rithmic.com:443") + app_name: str โ€” "Quantum Terminal" (default) + app_version: str โ€” "1.0" (default) + symbol_map: dict โ€” canonical โ†’ {base, exchange} overrides (optional) + +Dependencies: + pip install async_rithmic +================================================================================ +""" + +import logging +import threading +import asyncio +from collections import deque +from datetime import datetime, timezone, timedelta +from typing import Dict, List, Optional, Any + +from models import ( + TickData, BarData, AccountInfo, SymbolInfo, + OrderRequest, OrderResult, Position, +) +from providers.base_provider import BaseProvider + +log = logging.getLogger("provider.rithmic") + +# โ”€โ”€ async_rithmic imported lazily โ”€โ”€ +try: + from async_rithmic import ( + RithmicClient, + TimeBarType, + DataType, + LastTradePresenceBits, + BestBidOfferPresenceBits, + ) + RITHMIC_AVAILABLE = True +except ImportError: + RITHMIC_AVAILABLE = False + RithmicClient = None + TimeBarType = None + DataType = None + + +# ============================================================ +# 1. SYMBOL MAPPING โ€” Canonical โ†’ Rithmic +# ============================================================ + +# Maps Quantum Terminal canonical tickers to Rithmic base symbols + exchange. +# get_front_month_contract() auto-resolves the actual contract code +# (e.g., "ES" โ†’ "ESM6" for June 2026). +# +# IMPORTANT: Futures canonical tickers are SEPARATE from CFD tickers. +# ES = CME E-mini S&P 500 futures (Rithmic) +# US500 = S&P 500 CFD (MT5/CFI) +# They coexist in the universe as independent instruments. +RITHMIC_SYMBOL_MAP = { + # Equity index futures + "ES": {"base": "ES", "exchange": "CME"}, + "NQ": {"base": "NQ", "exchange": "CME"}, + "YM": {"base": "YM", "exchange": "CBOT"}, + # Metal futures + "GC": {"base": "GC", "exchange": "COMEX"}, + "SI": {"base": "SI", "exchange": "COMEX"}, + # Energy futures + "CL": {"base": "CL", "exchange": "NYMEX"}, + "BZ": {"base": "BZ", "exchange": "NYMEX"}, + # European index futures (Eurex โ€” may not be available on all accounts) + "FDAX": {"base": "FDAX", "exchange": "EUREX"}, + "Z": {"base": "Z", "exchange": "LIFFE"}, +} + +# Maps our timeframe strings to (TimeBarType enum name, period) pairs. +RITHMIC_TF_MAP = { + "M1": ("MINUTE_BAR", 1), + "M5": ("MINUTE_BAR", 5), + "M15": ("MINUTE_BAR", 15), + "M30": ("MINUTE_BAR", 30), + "H1": ("MINUTE_BAR", 60), + "H4": ("MINUTE_BAR", 240), + "D1": ("DAILY_BAR", 1), +} + +# Futures contract specs (static โ€” used for SymbolInfo). +FUTURES_SPECS = { + "ES": {"decimals": 2, "tick_size": 0.25, "tick_value": 12.50, + "contract_size": 50.0, "description": "E-mini S&P 500", + "currency": "USD", "min_lot": 1, "lot_step": 1, "max_lot": 100}, + "NQ": {"decimals": 2, "tick_size": 0.25, "tick_value": 5.00, + "contract_size": 20.0, "description": "E-mini NASDAQ-100", + "currency": "USD", "min_lot": 1, "lot_step": 1, "max_lot": 100}, + "YM": {"decimals": 0, "tick_size": 1.0, "tick_value": 5.00, + "contract_size": 5.0, "description": "E-mini Dow", + "currency": "USD", "min_lot": 1, "lot_step": 1, "max_lot": 100}, + "GC": {"decimals": 2, "tick_size": 0.10, "tick_value": 10.00, + "contract_size": 100.0, "description": "Gold Futures", + "currency": "USD", "min_lot": 1, "lot_step": 1, "max_lot": 100}, + "SI": {"decimals": 3, "tick_size": 0.005, "tick_value": 25.00, + "contract_size": 5000.0, "description": "Silver Futures", + "currency": "USD", "min_lot": 1, "lot_step": 1, "max_lot": 100}, + "CL": {"decimals": 2, "tick_size": 0.01, "tick_value": 10.00, + "contract_size": 1000.0, "description": "Crude Oil WTI", + "currency": "USD", "min_lot": 1, "lot_step": 1, "max_lot": 100}, + "BZ": {"decimals": 2, "tick_size": 0.01, "tick_value": 10.00, + "contract_size": 1000.0, "description": "Brent Crude Oil", + "currency": "USD", "min_lot": 1, "lot_step": 1, "max_lot": 100}, + "FDAX": {"decimals": 1, "tick_size": 0.5, "tick_value": 12.50, + "contract_size": 25.0, "description": "DAX Futures", + "currency": "EUR", "min_lot": 1, "lot_step": 1, "max_lot": 50}, + "Z": {"decimals": 1, "tick_size": 0.5, "tick_value": 5.00, + "contract_size": 10.0, "description": "FTSE 100 Futures", + "currency": "GBP", "min_lot": 1, "lot_step": 1, "max_lot": 50}, +} + +# Minutes per timeframe โ€” used to compute bar fetch windows +TF_MINUTES = { + "M1": 1, "M5": 5, "M15": 15, "M30": 30, + "H1": 60, "H4": 240, "D1": 1440, "W1": 10080, +} + + +# ============================================================ +# 2. RITHMIC PROVIDER +# ============================================================ + +class RithmicProvider(BaseProvider): + """ + Rithmic futures data provider. + + Data-only provider (can_execute=False for now). Streams live + ticks and bars via async_rithmic, serves data through the + synchronous BaseProvider interface. + + IMPORTANT: async_rithmic must run inside a proper asyncio event + loop. Use async_connect() from FastAPI's lifespan, or connect() + which falls back to asyncio.run() for standalone scripts. + """ + + def __init__(self, account_config: Dict[str, Any]): + self._id = account_config.get("id", "rithmic_default") + self._label = account_config.get("label", "Rithmic") + self._user = account_config.get("user", "") + self._password = account_config.get("password", "") + self._system_name = account_config.get("system_name", "Rithmic Test") + self._url = account_config.get("url", "") + self._app_name = account_config.get("app_name", "Quantum Terminal") + self._app_version = account_config.get("app_version", "1.0") + + # Symbol map: merge defaults with user overrides + self._symbol_map = dict(RITHMIC_SYMBOL_MAP) + user_map = account_config.get("symbol_map", {}) + self._symbol_map.update(user_map) + + # Connection state + self._connected = False + self._client: Optional[Any] = None # RithmicClient instance + + # Reference to the event loop we're running in (set during connect) + self._loop: Optional[asyncio.AbstractEventLoop] = None + + # Resolved front month contracts: canonical โ†’ "ESM6" etc. + self._front_months: Dict[str, str] = {} + # Reverse map: "ESM6" โ†’ "ES" + self._reverse_map: Dict[str, str] = {} + + # Tick cache: canonical โ†’ TickData (written from async callbacks, + # read from sync methods โ€” both in the same thread in production) + self._tick_cache: Dict[str, TickData] = {} + + # Bar buffer: "TICKER_TF" โ†’ deque of BarData (ring buffer) + self._bar_buffers: Dict[str, deque] = {} + self._max_bar_buffer = 500 + + # Bar tracking for check_new_bars + self._last_bar_times: Dict[str, str] = {} + + # Subscribed symbols (canonical names that resolved successfully) + self._subscribed: List[str] = [] + + # โ”€โ”€ Identity โ”€โ”€ + + @property + def provider_type(self) -> str: + return "rithmic" + + @property + def provider_id(self) -> str: + return self._id + + @property + def label(self) -> str: + return self._label + + # โ”€โ”€ Capabilities โ”€โ”€ + + @property + def can_execute(self) -> bool: + return False # Data-only for now + + @property + def can_stream_ticks(self) -> bool: + return True + + @property + def supported_timeframes(self) -> List[str]: + return list(RITHMIC_TF_MAP.keys()) + + # โ”€โ”€ Connection Lifecycle โ”€โ”€ + + @property + def connected(self) -> bool: + return self._connected + + def connect(self) -> bool: + """ + Synchronous connect โ€” for use in non-async contexts (test scripts). + + In production (data_server.py), use async_connect() instead. + This method uses asyncio.run() which blocks the calling thread. + """ + if not RITHMIC_AVAILABLE: + log.warning("async_rithmic package not installed โ€” " + "run: pip install async_rithmic") + return False + + if self._connected: + return True + + if not self._user or not self._password or not self._url: + log.error("Rithmic credentials missing โ€” " + "check [rithmic] in local_config.ini") + return False + + try: + return asyncio.run(self.async_connect()) + except Exception as e: + log.error(f"Rithmic connect error: {e}") + return False + + async def async_connect(self) -> bool: + """ + Async connect โ€” runs in the caller's event loop. + + Called from data_server.py lifespan: + connected = await provider.async_connect() + + 1. Create RithmicClient + 2. Connect to server + 3. Resolve front month contracts + 4. Subscribe to BBO ticks + M1 time bars + """ + if not RITHMIC_AVAILABLE: + log.warning("async_rithmic not installed") + return False + + if self._connected: + return True + + if not self._user or not self._password or not self._url: + log.error("Rithmic credentials missing") + return False + + try: + self._loop = asyncio.get_running_loop() + + self._client = RithmicClient( + user=self._user, + password=self._password, + system_name=self._system_name, + app_name=self._app_name, + app_version=self._app_version, + url=self._url, + ) + + await self._client.connect() + log.info("Rithmic client connected to server") + + # Register event callbacks + self._client.on_tick += self._on_tick + self._client.on_time_bar += self._on_time_bar + + # Resolve front month contracts + for canonical, mapping in self._symbol_map.items(): + base = mapping["base"] + exchange = mapping["exchange"] + try: + front = await self._client.get_front_month_contract( + base, exchange + ) + self._front_months[canonical] = front + self._reverse_map[front] = canonical + log.info(f" {canonical} -> {front} ({exchange})") + except Exception as e: + log.warning( + f" {canonical} -> FAILED to resolve " + f"{base}@{exchange}: {e}" + ) + + # Subscribe to live data for resolved symbols + for canonical, contract in self._front_months.items(): + exchange = self._symbol_map[canonical]["exchange"] + try: + # Subscribe to BBO ticks + data_type = DataType.LAST_TRADE | DataType.BBO + await self._client.subscribe_to_market_data( + contract, exchange, data_type + ) + + # Subscribe to M1 time bars + await self._client.subscribe_to_time_bar_data( + contract, exchange, TimeBarType.MINUTE_BAR, 1 + ) + + self._subscribed.append(canonical) + + # Initialize bar buffer + self._bar_buffers[f"{canonical}_M1"] = deque( + maxlen=self._max_bar_buffer + ) + + log.info( + f" Subscribed: {canonical} ({contract}@{exchange})" + ) + except Exception as e: + log.warning( + f" Subscribe failed for {canonical} " + f"({contract}@{exchange}): {e}" + ) + + if self._subscribed: + self._connected = True + log.info( + f"Rithmic connected: {self._system_name} | " + f"Subscribed: {len(self._subscribed)} symbols | " + f"URL: {self._url}" + ) + return True + else: + log.error("No symbols subscribed โ€” connection not useful") + await self._async_disconnect() + return False + + except Exception as e: + log.error(f"Rithmic async connect failed: {e}") + return False + + def disconnect(self) -> None: + """ + Synchronous disconnect โ€” for use in non-async contexts. + + In production, use async_disconnect() instead, or this method + will schedule the disconnect in the running event loop. + """ + if not self._connected: + return + + # If we have a reference to the event loop and it's running, + # schedule the async disconnect + if self._loop and self._loop.is_running(): + future = asyncio.run_coroutine_threadsafe( + self._async_disconnect(), self._loop + ) + try: + future.result(timeout=10) + except Exception as e: + log.warning(f"Rithmic disconnect error: {e}") + else: + # No running loop โ€” try asyncio.run as last resort + try: + asyncio.run(self._async_disconnect()) + except Exception: + pass + + self._connected = False + self._subscribed.clear() + self._front_months.clear() + self._reverse_map.clear() + self._tick_cache.clear() + self._bar_buffers.clear() + self._client = None + log.info("Rithmic disconnected") + + async def async_disconnect(self) -> None: + """Async disconnect โ€” called from lifespan shutdown.""" + await self._async_disconnect() + self._connected = False + self._subscribed.clear() + self._front_months.clear() + self._reverse_map.clear() + self._tick_cache.clear() + self._bar_buffers.clear() + self._client = None + log.info("Rithmic disconnected") + + def heartbeat(self) -> bool: + """Check if the Rithmic connection is alive.""" + # async_rithmic handles heartbeats internally + return self._connected + + # โ”€โ”€ Market Data โ”€โ”€ + + def get_latest_ticks(self, symbols: List[str]) -> Dict[str, TickData]: + """Return latest cached ticks for requested symbols.""" + result = {} + for s in symbols: + if s in self._tick_cache: + result[s] = self._tick_cache[s] + return result + + def get_bars( + self, ticker: str, timeframe: str = "M15", count: int = 200 + ) -> List[BarData]: + """ + Fetch bars for a canonical ticker. + + Checks local buffer first. If not enough bars, fetches from + Rithmic history plant via the event loop. + """ + if ticker not in self._front_months: + return [] + + # Check buffer first + buf_key = f"{ticker}_{timeframe}" + if buf_key in self._bar_buffers: + bars = list(self._bar_buffers[buf_key]) + if len(bars) >= count: + return bars[-count:] + + # Fetch from history โ€” need the event loop + if not self._loop or not self._loop.is_running(): + return [] + + try: + future = asyncio.run_coroutine_threadsafe( + self._async_get_bars(ticker, timeframe, count), + self._loop, + ) + return future.result(timeout=30) + except Exception as e: + log.warning(f"[{ticker}] Historical bar fetch failed: {e}") + return [] + + def check_new_bars( + self, symbols: List[str], timeframe: str = "M15" + ) -> List[dict]: + """Return bars received since last check.""" + new_bars = [] + for s in symbols: + buf_key = f"{s}_{timeframe}" + if buf_key not in self._bar_buffers: + continue + + track_key = f"{s}_{timeframe}" + last_time = self._last_bar_times.get(track_key) + buf = self._bar_buffers[buf_key] + + for bar in buf: + if last_time is None or bar.time > last_time: + new_bars.append({ + "type": "bar", + "ticker": s, + "timeframe": timeframe, + "bar": bar.to_dict(), + }) + self._last_bar_times[track_key] = bar.time + + return new_bars + + def get_symbol_info(self, ticker: str) -> Optional[SymbolInfo]: + """Return static futures contract metadata.""" + if ticker not in self._symbol_map: + return None + + mapping = self._symbol_map[ticker] + base = mapping["base"] + spec = FUTURES_SPECS.get(base) + if not spec: + return None + + broker_symbol = self._front_months.get(ticker, base) + + return SymbolInfo( + ticker=ticker, + broker_symbol=broker_symbol, + asset_class="FUTURES", + decimals=spec["decimals"], + description=spec["description"], + trade_allowed=False, + min_lot=spec["min_lot"], + max_lot=spec["max_lot"], + lot_step=spec["lot_step"], + contract_size=spec["contract_size"], + currency_profit=spec["currency"], + currency_margin=spec["currency"], + tick_size=spec["tick_size"], + tick_value=spec["tick_value"], + ) + + def get_account_info(self) -> Optional[AccountInfo]: + """Not applicable for data-only provider.""" + return None + + # โ”€โ”€ Symbol Resolution โ”€โ”€ + + def resolve_symbol(self, canonical: str) -> Optional[str]: + """Map canonical ticker to resolved Rithmic contract code.""" + return self._front_months.get(canonical) + + def resolve_universe(self, universe: List[str]) -> List[str]: + """Return which universe tickers this provider can serve.""" + return [t for t in universe if t in self._front_months] + + def get_available_symbols(self) -> List[str]: + """Return canonical symbols that were successfully subscribed.""" + return list(self._subscribed) + + # ============================================================ + # INTERNAL โ€” Async Operations + # ============================================================ + + async def _async_disconnect(self) -> None: + """Async cleanup: unsubscribe and disconnect.""" + if not self._client: + return + + try: + for canonical in self._subscribed: + if canonical not in self._front_months: + continue + contract = self._front_months[canonical] + exchange = self._symbol_map[canonical]["exchange"] + try: + await self._client.unsubscribe_from_market_data( + contract, exchange, + DataType.LAST_TRADE | DataType.BBO, + ) + await self._client.unsubscribe_from_time_bar_data( + contract, exchange, TimeBarType.MINUTE_BAR, 1, + ) + except Exception: + pass + + await self._client.disconnect() + except Exception as e: + log.warning(f"Rithmic async disconnect error: {e}") + + async def _async_get_bars( + self, ticker: str, timeframe: str, count: int + ) -> List[BarData]: + """Fetch historical time bars from Rithmic history plant.""" + if ticker not in self._front_months or not self._client: + return [] + + contract = self._front_months[ticker] + exchange = self._symbol_map[ticker]["exchange"] + + tf_info = RITHMIC_TF_MAP.get(timeframe) + if not tf_info: + log.warning(f"[{ticker}] Unsupported timeframe: {timeframe}") + return [] + + bar_type_name, period = tf_info + + if bar_type_name == "MINUTE_BAR": + bar_type = TimeBarType.MINUTE_BAR + elif bar_type_name == "DAILY_BAR": + bar_type = TimeBarType.DAILY_BAR + else: + bar_type = TimeBarType.MINUTE_BAR + + # Calculate time window + minutes_per_bar = TF_MINUTES.get(timeframe, 15) + total_minutes = int(count * minutes_per_bar * 1.2) # 20% buffer + end_time = datetime.now(timezone.utc) + start_time = end_time - timedelta(minutes=total_minutes) + + try: + raw_bars = await self._client.get_time_bars( + contract, exchange, bar_type, period, + start_time, end_time, + ) + + bars = [] + for rb in raw_bars: + bar = self._parse_bar(rb) + if bar: + bars.append(bar) + + bars.sort(key=lambda b: b.time) + bars = bars[-count:] + + # Cache in buffer + buf_key = f"{ticker}_{timeframe}" + self._bar_buffers[buf_key] = deque(bars, maxlen=self._max_bar_buffer) + + log.info( + f"[{ticker}] Fetched {len(bars)} {timeframe} bars " + f"from Rithmic history" + ) + return bars + + except AttributeError: + log.warning( + f"[{ticker}] get_time_bars() not available. " + f"Bars will accumulate from live stream." + ) + return [] + except Exception as e: + log.warning(f"[{ticker}] Historical bar fetch error: {e}") + return [] + + # ============================================================ + # INTERNAL โ€” Streaming Callbacks + # ============================================================ + + async def _on_tick(self, data: dict) -> None: + """Callback for live tick data. Updates tick cache.""" + try: + security_code = data.get("symbol", "") + canonical = self._reverse_map.get(security_code) + if not canonical: + return + + data_type = data.get("data_type") + presence = data.get("presence_bits", 0) + + # Preserve existing values + existing = self._tick_cache.get(canonical) + bid = existing.bid if existing else 0.0 + ask = existing.ask if existing else 0.0 + last = existing.last if existing else 0.0 + + if data_type == DataType.BBO: + if presence & BestBidOfferPresenceBits.BID: + bid = float(data.get("bid_price", bid)) + if presence & BestBidOfferPresenceBits.ASK: + ask = float(data.get("ask_price", ask)) + elif data_type == DataType.LAST_TRADE: + if presence & LastTradePresenceBits.LAST_TRADE: + last = float(data.get("trade_price", last)) + + if bid > 0 or ask > 0 or last > 0: + now = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%S") + spread = round(ask - bid, 6) if (bid > 0 and ask > 0) else 0.0 + if last == 0 and bid > 0: + last = (bid + ask) / 2 + + self._tick_cache[canonical] = TickData( + ticker=canonical, + bid=bid, + ask=ask, + last=last, + time=now, + spread=spread, + ) + + except Exception as e: + log.debug(f"Tick callback error: {e}") + + async def _on_time_bar(self, data: dict) -> None: + """Callback for live time bar data. Adds to ring buffer.""" + try: + security_code = data.get("symbol", "") + canonical = self._reverse_map.get(security_code) + if not canonical: + return + + bar = self._parse_bar(data) + if not bar: + return + + period = data.get("period", 1) + tf_key = self._period_to_tf(period) + buf_key = f"{canonical}_{tf_key}" + + if buf_key not in self._bar_buffers: + self._bar_buffers[buf_key] = deque( + maxlen=self._max_bar_buffer + ) + self._bar_buffers[buf_key].append(bar) + + except Exception as e: + log.debug(f"Time bar callback error: {e}") + + # ============================================================ + # INTERNAL โ€” Helpers + # ============================================================ + + def _parse_bar(self, data: dict) -> Optional[BarData]: + """Parse a raw Rithmic bar dict into a BarData object.""" + try: + open_p = float(data.get("open_price", data.get("open", 0))) + high_p = float(data.get("high_price", data.get("high", 0))) + low_p = float(data.get("low_price", data.get("low", 0))) + close_p = float(data.get("close_price", data.get("close", 0))) + volume = int(data.get("volume", 0)) + + bar_time = data.get("bar_end_time", data.get("time", "")) + if isinstance(bar_time, datetime): + time_str = bar_time.strftime("%Y-%m-%dT%H:%M:%S") + elif isinstance(bar_time, (int, float)): + time_str = datetime.fromtimestamp( + bar_time, tz=timezone.utc + ).strftime("%Y-%m-%dT%H:%M:%S") + else: + time_str = str(bar_time) + + if open_p == 0 and close_p == 0: + return None + + return BarData( + time=time_str, + open=open_p, + high=high_p, + low=low_p, + close=close_p, + volume=volume, + ) + except Exception: + return None + + @staticmethod + def _period_to_tf(period: int) -> str: + """Convert minute period to our timeframe string.""" + tf_map = {1: "M1", 5: "M5", 15: "M15", 30: "M30", 60: "H1", 240: "H4"} + return tf_map.get(period, "M1") diff --git a/backend/providers/tradovate_provider.py b/backend/providers/tradovate_provider.py new file mode 100644 index 0000000..c53576e --- /dev/null +++ b/backend/providers/tradovate_provider.py @@ -0,0 +1,454 @@ +# version: v1 +""" +================================================================================ +Quantum Terminal โ€” Tradovate Provider (POC) + +Proof-of-work integration with Tradovate for users who don't run MT5. + +What this does: + ยท Authenticates against /auth/accesstokenrequest (REST) + ยท Resolves CFD-style tickers (XAUUSD, US500, ...) โ†’ Tradovate futures roots + (GC, ES, ...) โ†’ front-month contracts (GCM6, ESM6, ...) + ยท Fetches historical bars over the market-data WebSocket (md/getChart) + +What this does NOT do (deferred to later phases): + ยท Live tick streaming (comes next) + ยท Order placement (futures orders are quite different from CFDs) + ยท Price conversion between CFD and futures price space + ยท Rollover-adjusted continuous contracts + ยท Settlement/margin accounting + +Reversibility: this file is standalone. Nothing in MT5 or the base terminal +touches it. Delete the file + one route-include line and Tradovate goes +away entirely. +================================================================================ +""" + +import asyncio +import json +import logging +import time +from dataclasses import dataclass, field +from datetime import datetime, timedelta, timezone +from typing import Any, Dict, List, Optional + +log = logging.getLogger("tradovate_provider") + +# โ”€โ”€โ”€ Dependencies (both already in consumer_venv) โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +try: + import httpx +except ImportError: + httpx = None + log.warning("httpx not installed โ€” Tradovate provider will be inert") + +try: + import websockets +except ImportError: + websockets = None + log.warning("websockets not installed โ€” Tradovate provider will be inert") + + +# โ”€โ”€โ”€ Endpoint config โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +ENDPOINTS = { + "demo": { + "rest": "https://demo.tradovateapi.com/v1", + "md_ws": "wss://md-demo.tradovateapi.com/v1/websocket", + }, + "live": { + "rest": "https://live.tradovateapi.com/v1", + "md_ws": "wss://md.tradovateapi.com/v1/websocket", + }, +} + + +# โ”€โ”€โ”€ CFD โ†’ Futures root map (POC v1) โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +# Add more entries as we validate them. Unmapped tickers raise a clean +# "symbol not supported by Tradovate" error rather than crashing. +CFD_TO_FUTURES_ROOT = { + "XAUUSD": "GC", # Gold futures (100 oz, COMEX) + "XAGUSD": "SI", # Silver futures (5,000 oz, COMEX) + "XTIUSD": "CL", # Crude oil futures (1,000 bbl, NYMEX) + "US500": "ES", # E-mini S&P 500 (CME) + "USTEC": "NQ", # E-mini Nasdaq 100 (CME) + "GER40": "FDAX", # DAX futures (Eurex) โ€” requires exchange subscription + "BTCUSD": "BTC", # Micro Bitcoin or BTC futures (CME) + # FX / UK100 / SOLUSD intentionally unmapped for POC. +} + +# โ”€โ”€โ”€ Futures month codes โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +MONTH_CODES = {1:"F", 2:"G", 3:"H", 4:"J", 5:"K", 6:"M", + 7:"N", 8:"Q", 9:"U", 10:"V", 11:"X", 12:"Z"} + +# Which months does each product expire in? +# Quarterlies = Mar/Jun/Sep/Dec, monthly = every month, bi-monthly = even months. +PRODUCT_EXPIRY_MONTHS = { + "ES": [3, 6, 9, 12], # E-mini S&P โ€” quarterly + "NQ": [3, 6, 9, 12], + "GC": [2, 4, 6, 8, 10, 12], # Gold โ€” bi-monthly + "SI": [1, 3, 5, 7, 9, 12], # Silver โ€” not every month + "CL": list(range(1, 13)), # Crude โ€” monthly + "BTC": [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], # Bitcoin โ€” monthly + "FDAX":[3, 6, 9, 12], +} + + +# โ”€โ”€โ”€ Timeframe โ†’ Tradovate chart description โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +def _tf_to_chart_desc(timeframe: str) -> Dict[str, Any]: + tf_map = { + "M1": ("MinuteBar", 1), + "M5": ("MinuteBar", 5), + "M15": ("MinuteBar", 15), + "M30": ("MinuteBar", 30), + "H1": ("MinuteBar", 60), + "H4": ("MinuteBar", 240), + "D1": ("DailyBar", 1), + } + underlying, size = tf_map.get(timeframe.upper(), ("MinuteBar", 15)) + return { + "underlyingType": underlying, + "elementSize": size, + "elementSizeUnit": "UnderlyingUnits", + "withHistogram": False, + } + + +@dataclass +class TradovateAuth: + access_token: str = "" + md_access_token: str = "" + user_id: int = 0 + name: str = "" + has_live: bool = False + user_status: str = "" + expires_at: float = 0.0 # epoch seconds + md_expires_at: float = 0.0 + + def is_valid(self) -> bool: + return bool(self.access_token) and time.time() < self.expires_at - 30 + + +class TradovateProvider: + """POC Tradovate provider. Not a full BaseProvider subclass yet โ€” + we'll upgrade to that once the smoke test passes. For now it exposes + the methods tradovate_routes.py needs to service the POC endpoint.""" + + def __init__(self, config: Dict[str, Any]): + self._config = config + self._auth = TradovateAuth() + self._last_error: str = "" + + # โ”€โ”€ Config updates (called from PATCH /api/tradovate/config) โ”€โ”€ + def update_config(self, new_config: Dict[str, Any]) -> None: + self._config.update(new_config or {}) + # Invalidate cached auth so the next connect uses new creds. + self._auth = TradovateAuth() + + @property + def connected(self) -> bool: + return self._auth.is_valid() + + @property + def is_delayed(self) -> bool: + # Free Tradovate demo accounts are delayed 10 minutes for most CME + # products. hasLive=True means the user has paid live market data; + # False means delayed. Real instrument-by-instrument delay info comes + # from md/getContract-like queries โ€” POC just surfaces the boolean. + return not self._auth.has_live + + @property + def status_dict(self) -> Dict[str, Any]: + return { + "connected": self.connected, + "user_id": self._auth.user_id, + "name": self._auth.name, + "user_status": self._auth.user_status, + "has_live": self._auth.has_live, + "delayed": self.is_delayed, + "env": self._config.get("env", "demo"), + "expires_at": self._auth.expires_at, + "error": self._last_error, + } + + # โ”€โ”€ Authentication โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + async def authenticate(self) -> Dict[str, Any]: + self._last_error = "" + if httpx is None: + self._last_error = "httpx not installed in venv" + return {"success": False, "error": self._last_error} + cfg = self._config + env = (cfg.get("env") or "demo").lower() + if env not in ENDPOINTS: + self._last_error = f"unknown env '{env}' (expected demo or live)" + return {"success": False, "error": self._last_error} + required = ["app_id", "cid", "sec", "username", "password"] + missing = [k for k in required if not cfg.get(k)] + if missing: + self._last_error = f"missing creds: {', '.join(missing)}" + return {"success": False, "error": self._last_error} + + url = ENDPOINTS[env]["rest"] + "/auth/accesstokenrequest" + body = { + "name": cfg["username"], + "password": cfg["password"], + "appId": cfg["app_id"], + "appVersion": cfg.get("app_version", "1.0"), + "cid": int(cfg["cid"]), + "sec": cfg["sec"], + "deviceId": cfg.get("device_id", "QuantumTerminal-consumer-poc"), + } + try: + async with httpx.AsyncClient(timeout=15.0) as client: + r = await client.post(url, json=body) + if r.status_code != 200: + self._last_error = f"HTTP {r.status_code}: {r.text[:200]}" + return {"success": False, "error": self._last_error} + data = r.json() + except Exception as e: + self._last_error = f"auth request failed: {e}" + log.error(self._last_error) + return {"success": False, "error": self._last_error} + + # Tradovate returns error info inside a 200 response for bad creds + err_code = data.get("errorText") or data.get("p-ticket") + if err_code and not data.get("accessToken"): + self._last_error = f"tradovate rejected auth: {err_code}" + return {"success": False, "error": self._last_error} + + # Parse token expiration โ€” Tradovate returns ISO8601 in expirationTime + def _parse_iso(s): + if not s: + return time.time() + 4000 # ~66 min fallback + try: + s = s.replace("Z", "+00:00") + return datetime.fromisoformat(s).timestamp() + except Exception: + return time.time() + 4000 + + self._auth = TradovateAuth( + access_token = data.get("accessToken", ""), + md_access_token= data.get("mdAccessToken", ""), + user_id = data.get("userId", 0) or 0, + name = data.get("name", "") or cfg["username"], + has_live = bool(data.get("hasLive", False)), + user_status = data.get("userStatus", "") or "", + expires_at = _parse_iso(data.get("expirationTime")), + md_expires_at = _parse_iso(data.get("expirationTime")), + ) + if not self._auth.access_token: + self._last_error = "no accessToken in response" + return {"success": False, "error": self._last_error} + return {"success": True, **self.status_dict} + + def disconnect(self) -> None: + self._auth = TradovateAuth() + self._last_error = "" + + # โ”€โ”€ Symbol resolution โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + def _resolve_root(self, ticker: str) -> Optional[str]: + t = (ticker or "").upper() + if t in CFD_TO_FUTURES_ROOT: + return CFD_TO_FUTURES_ROOT[t] + # Pass-through: if the user passes a root directly (e.g. "GC"), accept + if t in PRODUCT_EXPIRY_MONTHS: + return t + return None + + def _front_month_contract(self, root: str, today: Optional[datetime] = None) -> str: + """Cheap calendar-based front-month resolver. Good enough for POC. + For production we should hit /contract/suggest for accurate roll + timing (contracts roll a few days before the last notice date). + """ + months = PRODUCT_EXPIRY_MONTHS.get(root) + if not months: + # Default to quarterly if unknown + months = [3, 6, 9, 12] + now = today or datetime.now(timezone.utc) + y, m = now.year, now.month + # Pick the next expiry month that's >= current month. Add 5-day lead + # to avoid the last-trading-day rush โ€” we want the LIQUID contract. + lead_day = now.day >= 10 + candidate_month = None + for mm in months: + if mm > m or (mm == m and not lead_day): + candidate_month = mm + break + if candidate_month is None: + # Wrap to next year's first expiry month + candidate_month = months[0] + y += 1 + return f"{root}{MONTH_CODES[candidate_month]}{y % 10}" + + async def resolve_contract_via_api(self, root: str) -> Optional[str]: + """Ask Tradovate to suggest the current tradeable contract. Used as + a fallback/verification โ€” if this disagrees with the calendar + heuristic we prefer this answer.""" + if httpx is None or not self._auth.access_token: + return None + url = ENDPOINTS[self._config.get("env", "demo")]["rest"] + "/contract/suggest" + try: + async with httpx.AsyncClient(timeout=10.0) as client: + r = await client.get( + url, params={"t": root, "l": 5}, + headers={"Authorization": f"Bearer {self._auth.access_token}"}, + ) + if r.status_code != 200: + return None + contracts = r.json() or [] + # Return the shortest-name contract (usually the front month). + if contracts: + contracts.sort(key=lambda c: len(c.get("name", ""))) + return contracts[0].get("name") + except Exception as e: + log.warning(f"contract/suggest failed for {root}: {e}") + return None + + # โ”€โ”€ Market-data WebSocket helpers โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + async def _ws_fetch_bars(self, contract: str, timeframe: str, count: int) -> List[Dict[str, Any]]: + """Open a one-shot WebSocket, auth, request bars, return them.""" + if websockets is None or not self._auth.md_access_token: + raise RuntimeError("not authenticated or websockets missing") + env = self._config.get("env", "demo") + url = ENDPOINTS[env]["md_ws"] + + # Tradovate WebSocket protocol: each message is a plaintext frame + # \n\n\n + # Responses come as JSON arrays prefixed by "a" (for "array frames") + # and single-letter frames "o" (open), "h" (heartbeat), "c" (close). + def _encode(endpoint: str, msg_id: int, body: Any = "") -> str: + body_str = json.dumps(body) if not isinstance(body, str) else body + return f"{endpoint}\n{msg_id}\n\n{body_str}" + + bars: List[Dict[str, Any]] = [] + base_price = 0.0 + tick_size = 0.01 + tick_mult = 1.0 + + try: + async with websockets.connect(url, ping_interval=20, ping_timeout=20) as ws: + # 1st server frame should be "o" + open_frame = await asyncio.wait_for(ws.recv(), timeout=10.0) + if not (isinstance(open_frame, str) and open_frame.startswith("o")): + raise RuntimeError(f"unexpected open frame: {open_frame!r}") + + # Authorize with MD token + await ws.send(_encode("authorize", 1, self._auth.md_access_token)) + + # Request chart + chart_req = { + "symbol": contract, + "chartDescription": _tf_to_chart_desc(timeframe), + "timeRange": {"asMuchAsElements": max(1, min(int(count), 2000))}, + } + await ws.send(_encode("md/getChart", 2, chart_req)) + + # Read frames until we receive the charts packet for our id + deadline = time.time() + 20.0 + subscription_id: Optional[int] = None + while time.time() < deadline and len(bars) < count: + try: + frame = await asyncio.wait_for(ws.recv(), timeout=5.0) + except asyncio.TimeoutError: + break + if not isinstance(frame, str): + continue + # "h" = heartbeat โ€” send one back to stay alive + if frame == "h": + await ws.send("[]") + continue + if frame.startswith("a"): + try: + arr = json.loads(frame[1:]) + except Exception: + continue + for item in arr or []: + # Response to md/getChart (id=2) returns subscription id + if item.get("i") == 2 and item.get("s") == 200: + body = item.get("d") or {} + subscription_id = body.get("subscriptionId") or body.get("id") + # Streaming chart data comes with "e": "chart" + if item.get("e") == "chart": + cd = item.get("d") or {} + charts = cd.get("charts") or [] + for ch in charts: + base_price = ch.get("bp", base_price) + tick_size = ch.get("ts", tick_size) or tick_size + tick_mult = ch.get("tm", tick_mult) or tick_mult + raw_bars = ch.get("bars") or [] + for b in raw_bars: + bars.append(_decode_bar(b, base_price, tick_size, tick_mult)) + + # Clean up โ€” unsubscribe if we got a subscription id + if subscription_id is not None: + try: + await ws.send(_encode("md/cancelChart", 3, {"subscriptionId": subscription_id})) + except Exception: + pass + except Exception as e: + self._last_error = f"ws fetch failed: {e}" + log.error(self._last_error) + raise + + return bars[-count:] if len(bars) > count else bars + + async def get_bars(self, ticker: str, timeframe: str = "M15", count: int = 200) -> List[Dict[str, Any]]: + """Main entry point. Maps ticker โ†’ contract, fetches bars via WS.""" + if not self._auth.is_valid(): + raise RuntimeError("not authenticated") + root = self._resolve_root(ticker) + if not root: + raise ValueError(f"{ticker} not mapped to a Tradovate futures root") + # Try API-based resolution first, fall back to calendar heuristic. + contract = await self.resolve_contract_via_api(root) + if not contract: + contract = self._front_month_contract(root) + bars = await self._ws_fetch_bars(contract, timeframe, count) + return bars + + +# โ”€โ”€โ”€ Bar decoding helper โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +def _decode_bar(b: Dict[str, Any], base_price: float, tick_size: float, tick_mult: float) -> Dict[str, Any]: + """Tradovate returns bars with fields encoded as offsets from a base + price in tick units. Decode to absolute OHLC floats.""" + # Some Tradovate responses send bars already in absolute prices; others + # send deltas. Handle both by checking if base_price + tick_size make + # the output make sense. + def _px(val): + if val is None: + return None + # Heuristic: if val is very small (|val| < 1e6) and base_price > 0 + # and tick_size > 0, treat as offset. Otherwise treat as absolute. + if base_price > 0 and tick_size > 0 and abs(val) < 1_000_000: + return base_price + (val * tick_size * (tick_mult or 1)) + return float(val) + + ts = b.get("timestamp") or b.get("t") + # timestamp can be ISO string or epoch ms + if isinstance(ts, (int, float)): + iso = datetime.fromtimestamp(ts / (1000 if ts > 1e11 else 1), tz=timezone.utc) \ + .strftime("%Y-%m-%dT%H:%M:%S") + elif isinstance(ts, str): + iso = ts.replace("Z", "").split(".")[0] + else: + iso = "" + + return { + "time": iso, + "open": _px(b.get("open")) or 0.0, + "high": _px(b.get("high")) or 0.0, + "low": _px(b.get("low")) or 0.0, + "close": _px(b.get("close")) or 0.0, + "volume": int(b.get("upVolume", 0) or 0) + int(b.get("downVolume", 0) or 0) + or int(b.get("volume", 0) or 0), + } + + +# โ”€โ”€โ”€ Singleton accessor (one provider instance per process) โ”€โ”€โ”€โ”€โ”€ +_instance: Optional[TradovateProvider] = None + +def get_tradovate_provider(config: Optional[Dict[str, Any]] = None) -> TradovateProvider: + global _instance + if _instance is None: + _instance = TradovateProvider(config or {}) + elif config: + _instance.update_config(config) + return _instance diff --git a/backend/pulse_room/__init__.py b/backend/pulse_room/__init__.py new file mode 100644 index 0000000..ad08c17 --- /dev/null +++ b/backend/pulse_room/__init__.py @@ -0,0 +1,10 @@ +# version: v1 +"""Pulse Room โ€” Phase 1 foundation. + +Reads cached cones/bands JSONs, builds a pulse snapshot, +serves REST endpoints for the React Pulse Room UI. + +See docs/specs/2026-05-04-pulse-room-design.md for full design. +""" + +# Re-exports added as submodules land. Until then this package is empty. diff --git a/backend/pulse_room/builder.py b/backend/pulse_room/builder.py new file mode 100644 index 0000000..e868a28 --- /dev/null +++ b/backend/pulse_room/builder.py @@ -0,0 +1,463 @@ +# version: v4 +"""Pulse snapshot builder. + +Reads cached cones/bands JSONs, extracts SD edges as forward curves, +and writes pulse_snapshot.json to AppData. Atomic write โ€” partial +runs never corrupt the live snapshot. +""" + +import json +import logging +import os +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Optional + +from . import state + + +log = logging.getLogger("mk.pulse_room.builder") + + +# ---------- path resolvers ---------- + +def _default_cache_dir() -> Path: + appdata = os.environ.get("APPDATA") + if not appdata: + appdata = str(Path.home() / ".config") + dir_name = os.environ.get("MK_APP_DIR_NAME", "QuantumTerminal-v2") + return Path(appdata) / dir_name / "cache" + + +def cones_path(ticker: str, cache_dir: Optional[Path] = None) -> Path: + cd = cache_dir or _default_cache_dir() + return cd / f"{ticker.upper()}_cones.json" + + +def bands_path(ticker: str, cache_dir: Optional[Path] = None) -> Path: + cd = cache_dir or _default_cache_dir() + return cd / f"GLOBAL_{ticker.lower()}_bands.json" + + +# ---------- safe JSON loader ---------- + +def load_json(path: Path) -> Optional[dict]: + """Load JSON; return None on missing or malformed.""" + if not path.exists(): + return None + try: + with open(path, "r", encoding="utf-8") as f: + return json.load(f) + except (OSError, json.JSONDecodeError) as e: + log.warning("load_json failed for %s: %s", path, e) + return None + + +# ---------- cone extraction ---------- + +def _parse_iso(s: str) -> datetime: + """Parse a cone-format date like '2026-04-27 00:00' as UTC.""" + # Cones use 'YYYY-MM-DD HH:MM' (no seconds, no TZ) + if "T" in s: + s_normalized = s.replace("Z", "+00:00") + try: + dt = datetime.fromisoformat(s_normalized) + if dt.tzinfo is None: + dt = dt.replace(tzinfo=timezone.utc) + return dt + except ValueError: + pass + return datetime.strptime(s, "%Y-%m-%d %H:%M").replace(tzinfo=timezone.utc) + + +def _date_offsets_minutes(dates: list, base: Optional[datetime] = None) -> list: + """Convert a list of date strings to minutes-from-base. + + If `base` is None, uses dates[0] (legacy behavior, kept so existing tests + that don't pass a `now` continue to work). When the production code path + in `build_snapshot` calls this, it always passes `base=now` so that t_min + is minutes-from-computed_at (per spec ยง3.1) โ€” past timestamps get t_min<0. + """ + parsed = [_parse_iso(d) for d in dates] + base_dt = base if base is not None else parsed[0] + return [int((d - base_dt).total_seconds() // 60) for d in parsed] + + +def _index_at_offset_minutes(dates: list, target_minutes: int, base: Optional[datetime] = None) -> int: + """Return the smallest index whose offset (relative to `base`, default dates[0]) + is at or after `target_minutes`.""" + offsets = _date_offsets_minutes(dates, base=base) + for i, m in enumerate(offsets): + if m >= target_minutes: + return i + return len(offsets) - 1 + + +# ---------- cone origins ---------- +# 8 origin variants โ€” each is a separate cone scope inside the same +# _cones.json file. All use the gbm model. +# Format: (json_key, origin_id, display_label) +# Aligned with the producer's canonical schema: +# gbm_now โ€” live, anchored at last D1 close / current px (uncached) +# gbm_curr โ€” Monday open of current trading week (cached) +# gbm_prev โ€” Monday open of last week (cached) +# gbm_week_prev_2 โ€” 2 weeks back (conditionally cached) +# gbm_month_curr โ€” first trading day of current month (cached) +# gbm_month_prev โ€” first trading day of last month (cached) +# gbm_month_prev_2 โ€” first trading day of month -2 (cached) +# gbm_extreme_high โ€” bar of highest high in last 45 days (cached) +# gbm_extreme_low โ€” bar of lowest low in last 45 days (cached) +CONE_ORIGINS = [ + ("gbm_now", "now", "now"), + ("gbm_curr", "current_week", "current week"), + ("gbm_prev", "prev_week", "previous week"), + ("gbm_week_prev_2", "week_prev_2", "week -2"), + ("gbm_month_curr", "current_month", "current month"), + ("gbm_month_prev", "prev_month", "previous month"), + ("gbm_month_prev_2", "prev_month_2", "month -2"), + ("gbm_extreme_high", "extreme_high", "extreme high"), + ("gbm_extreme_low", "extreme_low", "extreme low"), +] + +# Priority order for sourcing the canonical 1-day volatility yardstick when +# multiple origins are available. gbm_curr is the historical canonical key; +# fall back to gbm_now (this-week scope), then gbm_month_curr. +_CONE_1D_SD_PRIORITY = ("gbm_curr", "gbm_now", "gbm_month_curr") + + +def _compute_cone_1d_sd( + cones_json: dict, + now: Optional[datetime] = None, +) -> Optional[float]: + """Compute cone_1d_sd from the highest-priority origin variant present. + + Returns None if no priority variant is available or extraction fails. + """ + for key in _CONE_1D_SD_PRIORITY: + v = cones_json.get(key) + if not v: + continue + try: + dates = v["dates"] + idx_1d = _index_at_offset_minutes(dates, 1440, base=now) + up_width = v["sd1_high"][idx_1d] - v["median"][idx_1d] + dn_width = v["median"][idx_1d] - v["sd1_low"][idx_1d] + return float((up_width + dn_width) / 2.0) + except (KeyError, IndexError, TypeError, ValueError): + continue + return None + + +def extract_cone_elements( + cones_json: dict, + now: Optional[datetime] = None, +) -> tuple[list, float]: + """Return (elements_list, cone_1d_sd). + + Iterates CONE_ORIGINS and emits 7 series per present origin variant: + median, sd1_upper/lower, sd2_upper/lower, sd3_upper/lower. Each + element carries an `origin` field (e.g. "now", "current_week"). + Missing variant keys are silently skipped โ€” some assets may have + fewer variants in their cones JSON. + + cone_1d_sd is sourced from the highest-priority variant present + (gbm_curr โ†’ gbm_now โ†’ gbm_month_curr). + + If `now` is provided, t_min values are re-based so t_min=0 corresponds + to `now`; past timestamps get t_min<0, future get t_min>0. If `now` is + None, falls back to legacy behavior (t_min=0 at dates[0]) so existing + test fixtures keep working. + """ + elements = [] + for json_key, origin_id, display_label in CONE_ORIGINS: + v = cones_json.get(json_key) + if not v: + continue + try: + dates = v["dates"] + except (KeyError, TypeError): + continue + offsets = _date_offsets_minutes(dates, base=now) + + # Median series โ†’ "