update data main fx

Quant_Hiber_Risk_Stargy/src/data/__init__.py

@@ -0,0 +1,3 @@
+"""
+Data package for quant FX bot.
+"""

Quant_Hiber_Risk_Stargy/src/data/mt5_fetcher.py

@@ -0,0 +1,184 @@
+"""
+src/data/mt5_fetcher.py
+
+MT5 historical data fetcher for Forex quant bot.
+- Connects to MetaTrader 5 terminal
+- Downloads OHLCV bars
+- Exports to parquet/csv
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from datetime import datetime
+from pathlib import Path
+from typing import Optional
+
+import pandas as pd
+
+try:
+    import MetaTrader5 as mt5
+except ImportError as exc:
+    raise ImportError(
+        "MetaTrader5 package is not installed. "
+        "Run: pip install MetaTrader5"
+    ) from exc
+
+
+TIMEFRAME_MAP: dict[str, int] = {
+    "M1": mt5.TIMEFRAME_M1,
+    "M5": mt5.TIMEFRAME_M5,
+    "M15": mt5.TIMEFRAME_M15,
+    "M30": mt5.TIMEFRAME_M30,
+    "H1": mt5.TIMEFRAME_H1,
+    "H4": mt5.TIMEFRAME_H4,
+    "D1": mt5.TIMEFRAME_D1,
+}
+
+
+@dataclass
+class MT5Config:
+    terminal_path: Optional[str] = None
+    login: Optional[int] = None
+    password: Optional[str] = None
+    server: Optional[str] = None
+    timezone: str = "UTC"
+
+
+class MT5Fetcher:
+    def __init__(self, config: MT5Config):
+        self.config = config
+        self.initialized = False
+
+    def initialize(self) -> None:
+        kwargs = {}
+
+        if self.config.terminal_path:
+            kwargs["path"] = self.config.terminal_path
+        if self.config.login:
+            kwargs["login"] = self.config.login
+        if self.config.password:
+            kwargs["password"] = self.config.password
+        if self.config.server:
+            kwargs["server"] = self.config.server
+
+        ok = mt5.initialize(**kwargs)
+        if not ok:
+            code, msg = mt5.last_error()
+            raise RuntimeError(f"MT5 initialize failed: {code} - {msg}")
+
+        self.initialized = True
+
+    def shutdown(self) -> None:
+        if self.initialized:
+            mt5.shutdown()
+            self.initialized = False
+
+    def ensure_symbol(self, symbol: str) -> None:
+        info = mt5.symbol_info(symbol)
+        if info is None:
+            raise ValueError(f"Symbol not found in MT5 terminal: {symbol}")
+
+        if not info.visible:
+            selected = mt5.symbol_select(symbol, True)
+            if not selected:
+                raise RuntimeError(f"Failed to enable symbol in Market Watch: {symbol}")
+
+    def fetch_range(
+        self,
+        symbol: str,
+        timeframe: str,
+        start: datetime,
+        end: datetime,
+    ) -> pd.DataFrame:
+        if timeframe not in TIMEFRAME_MAP:
+            raise ValueError(f"Unsupported timeframe: {timeframe}")
+
+        self.ensure_symbol(symbol)
+
+        rates = mt5.copy_rates_range(symbol, TIMEFRAME_MAP[timeframe], start, end)
+        if rates is None:
+            code, msg = mt5.last_error()
+            raise RuntimeError(
+                f"Failed to fetch rates for {symbol} {timeframe}: {code} - {msg}"
+            )
+
+        df = pd.DataFrame(rates)
+        if df.empty:
+            raise ValueError(f"No data returned for {symbol} {timeframe}")
+
+        df["time"] = pd.to_datetime(df["time"], unit="s", utc=True)
+        df = df.rename(
+            columns={
+                "time": "timestamp",
+                "tick_volume": "tick_volume",
+                "real_volume": "real_volume",
+                "spread": "spread",
+            }
+        )
+
+        ordered_cols = [
+            "timestamp",
+            "open",
+            "high",
+            "low",
+            "close",
+            "tick_volume",
+            "spread",
+            "real_volume",
+        ]
+        existing_cols = [c for c in ordered_cols if c in df.columns]
+        df = df[existing_cols].copy()
+
+        df["symbol"] = symbol
+        df["timeframe"] = timeframe
+
+        return df
+
+    @staticmethod
+    def validate_bars(df: pd.DataFrame) -> dict:
+        required_cols = {"timestamp", "open", "high", "low", "close"}
+        missing = required_cols - set(df.columns)
+        if missing:
+            raise ValueError(f"Missing required columns: {sorted(missing)}")
+
+        stats = {
+            "rows": int(len(df)),
+            "start": str(df["timestamp"].min()),
+            "end": str(df["timestamp"].max()),
+            "duplicate_timestamps": int(df["timestamp"].duplicated().sum()),
+            "null_count": int(df.isnull().sum().sum()),
+        }
+
+        bad_ohlc = (
+            (df["high"] < df["low"])
+            | (df["open"] > df["high"])
+            | (df["open"] < df["low"])
+            | (df["close"] > df["high"])
+            | (df["close"] < df["low"])
+        )
+        stats["bad_ohlc_rows"] = int(bad_ohlc.sum())
+
+        return stats
+
+    @staticmethod
+    def save_dataframe(
+        df: pd.DataFrame,
+        output_dir: Path,
+        symbol: str,
+        timeframe: str,
+        fmt: str = "parquet",
+    ) -> Path:
+        output_dir.mkdir(parents=True, exist_ok=True)
+
+        filename = f"{symbol}_{timeframe}.{fmt}"
+        path = output_dir / filename
+
+        if fmt == "parquet":
+            df.to_parquet(path, index=False)
+        elif fmt == "csv":
+            df.to_csv(path, index=False)
+        else:
+            raise ValueError("fmt must be 'parquet' or 'csv'")
+
+        return path