update data main fx

Quant_Hiber_Risk_Stargy/data/preprocessor.py

@@ -0,0 +1,52 @@
+import pandas as pd
+import os
+from pathlib import Path
+
+def preprocess_all_symbols():
+    symbols = ['EURUSDm', 'GBPUSDm', 'USDJPYm', 'USDCHFm', 'USDCADm', 'NZDUSDm']
+    timeframes = ['M15', 'H1', 'H4']
+    raw_base = Path("data/raw")
+    processed_base = Path("data/processed")
+    
+    print("--- BẮT ĐẦU TIỀN XỬ LÝ DỮ LIỆU ---")
+
+    for symbol in symbols:
+        for tf in timeframes:
+            file_path = raw_base / symbol / tf / f"{symbol}_{tf}.parquet"
+            
+            if not file_path.exists():
+                continue
+                
+            # 1. Đọc dữ liệu
+            df = pd.read_parquet(file_path)
+            initial_count = len(df)
+            
+            # 2. Xử lý logic cơ bản
+            # Chuyển index sang datetime nếu chưa có
+            df.index = pd.to_datetime(df['timestamp'])
+            df = df.sort_index()
+            
+            # Loại bỏ trùng lặp
+            df = df[~df.index.duplicated(keep='first')]
+            
+            # Loại bỏ nến lỗi (High < Low)
+            df = df[df['high'] >= df['low']]
+            
+            # Lọc từ năm 2022 trở lại đây như bạn yêu cầu
+            df = df.loc['2022-01-01':]
+            
+            # 3. Loại bỏ ngày cuối tuần (Forex nghỉ)
+            df = df[df.index.dayofweek < 5]
+
+            # 4. Lưu dữ liệu sạch
+            output_dir = processed_base / symbol / tf
+            output_dir.mkdir(parents=True, exist_ok=True)
+            output_path = output_dir / f"{symbol}_{tf}_clean.parquet"
+            df.to_parquet(output_path)
+            
+            print(f"[OK] {symbol} {tf}: {initial_count} -> {len(df)} nến (Đã lưu tại {output_path})")
+
+    print("--- HOÀN THÀNH ---")
+
+if __name__ == "__main__":
+    preprocess_all_symbols()

Quant_Hiber_Risk_Stargy/data/raw/script.py

@@ -0,0 +1,14 @@
+import pandas as pd
+
+# Đường dẫn đến file vừa tải
+file_path = r"data\raw\EURUSDm\M15\EURUSDm_M15.parquet"
+
+# Đọc dữ liệu
+df = pd.read_parquet(file_path)
+
+# Xem 5 dòng đầu và 5 dòng cuối
+print(df.head())
+print(df.tail())
+
+# Kiểm tra xem có ngày nào bị thiếu không
+print(f"Tổng số nến: {len(df)}")

Quant_Hiber_Risk_Stargy/scripts/download_data.py

@@ -18,13 +18,26 @@ import sys
 from datetime import datetime
 from pathlib import Path
 
-ROOT = Path(__file__).resolve().parents[1]
+
+def _find_repo_root(start: Path) -> Path:
+    """Walk up from start until a directory containing `src` is found."""
+    current = start.resolve()
+    while True:
+        if (current / "src").is_dir():
+            return current
+        if current.parent == current:
+            break
+        current = current.parent
+
+    raise RuntimeError("Cannot locate repository root (directory containing 'src').")
+
+
+ROOT = _find_repo_root(Path(__file__).parent)
 if str(ROOT) not in sys.path:
     sys.path.insert(0, str(ROOT))
 
 from src.data.mt5_fetcher import MT5Config, MT5Fetcher  # noqa: E402
 
-
 DEFAULT_SYMBOLS = ["EURUSD", "USDJPY", "GBPUSD", "USDCHF", "AUDUSD", "NZDUSD"]
 DEFAULT_TIMEFRAMES = ["M15", "H1", "H4"]
 
@@ -147,4 +160,4 @@ def main() -> None:
 
 
 if __name__ == "__main__":
-    main()
+    main()

Quant_Hiber_Risk_Stargy/src/data/__init__.py

@@ -0,0 +1,3 @@
+"""
+Data package for quant FX bot.
+"""

Quant_Hiber_Risk_Stargy/src/data/mt5_fetcher.py

@@ -0,0 +1,184 @@
+"""
+src/data/mt5_fetcher.py
+
+MT5 historical data fetcher for Forex quant bot.
+- Connects to MetaTrader 5 terminal
+- Downloads OHLCV bars
+- Exports to parquet/csv
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from datetime import datetime
+from pathlib import Path
+from typing import Optional
+
+import pandas as pd
+
+try:
+    import MetaTrader5 as mt5
+except ImportError as exc:
+    raise ImportError(
+        "MetaTrader5 package is not installed. "
+        "Run: pip install MetaTrader5"
+    ) from exc
+
+
+TIMEFRAME_MAP: dict[str, int] = {
+    "M1": mt5.TIMEFRAME_M1,
+    "M5": mt5.TIMEFRAME_M5,
+    "M15": mt5.TIMEFRAME_M15,
+    "M30": mt5.TIMEFRAME_M30,
+    "H1": mt5.TIMEFRAME_H1,
+    "H4": mt5.TIMEFRAME_H4,
+    "D1": mt5.TIMEFRAME_D1,
+}
+
+
+@dataclass
+class MT5Config:
+    terminal_path: Optional[str] = None
+    login: Optional[int] = None
+    password: Optional[str] = None
+    server: Optional[str] = None
+    timezone: str = "UTC"
+
+
+class MT5Fetcher:
+    def __init__(self, config: MT5Config):
+        self.config = config
+        self.initialized = False
+
+    def initialize(self) -> None:
+        kwargs = {}
+
+        if self.config.terminal_path:
+            kwargs["path"] = self.config.terminal_path
+        if self.config.login:
+            kwargs["login"] = self.config.login
+        if self.config.password:
+            kwargs["password"] = self.config.password
+        if self.config.server:
+            kwargs["server"] = self.config.server
+
+        ok = mt5.initialize(**kwargs)
+        if not ok:
+            code, msg = mt5.last_error()
+            raise RuntimeError(f"MT5 initialize failed: {code} - {msg}")
+
+        self.initialized = True
+
+    def shutdown(self) -> None:
+        if self.initialized:
+            mt5.shutdown()
+            self.initialized = False
+
+    def ensure_symbol(self, symbol: str) -> None:
+        info = mt5.symbol_info(symbol)
+        if info is None:
+            raise ValueError(f"Symbol not found in MT5 terminal: {symbol}")
+
+        if not info.visible:
+            selected = mt5.symbol_select(symbol, True)
+            if not selected:
+                raise RuntimeError(f"Failed to enable symbol in Market Watch: {symbol}")
+
+    def fetch_range(
+        self,
+        symbol: str,
+        timeframe: str,
+        start: datetime,
+        end: datetime,
+    ) -> pd.DataFrame:
+        if timeframe not in TIMEFRAME_MAP:
+            raise ValueError(f"Unsupported timeframe: {timeframe}")
+
+        self.ensure_symbol(symbol)
+
+        rates = mt5.copy_rates_range(symbol, TIMEFRAME_MAP[timeframe], start, end)
+        if rates is None:
+            code, msg = mt5.last_error()
+            raise RuntimeError(
+                f"Failed to fetch rates for {symbol} {timeframe}: {code} - {msg}"
+            )
+
+        df = pd.DataFrame(rates)
+        if df.empty:
+            raise ValueError(f"No data returned for {symbol} {timeframe}")
+
+        df["time"] = pd.to_datetime(df["time"], unit="s", utc=True)
+        df = df.rename(
+            columns={
+                "time": "timestamp",
+                "tick_volume": "tick_volume",
+                "real_volume": "real_volume",
+                "spread": "spread",
+            }
+        )
+
+        ordered_cols = [
+            "timestamp",
+            "open",
+            "high",
+            "low",
+            "close",
+            "tick_volume",
+            "spread",
+            "real_volume",
+        ]
+        existing_cols = [c for c in ordered_cols if c in df.columns]
+        df = df[existing_cols].copy()
+
+        df["symbol"] = symbol
+        df["timeframe"] = timeframe
+
+        return df
+
+    @staticmethod
+    def validate_bars(df: pd.DataFrame) -> dict:
+        required_cols = {"timestamp", "open", "high", "low", "close"}
+        missing = required_cols - set(df.columns)
+        if missing:
+            raise ValueError(f"Missing required columns: {sorted(missing)}")
+
+        stats = {
+            "rows": int(len(df)),
+            "start": str(df["timestamp"].min()),
+            "end": str(df["timestamp"].max()),
+            "duplicate_timestamps": int(df["timestamp"].duplicated().sum()),
+            "null_count": int(df.isnull().sum().sum()),
+        }
+
+        bad_ohlc = (
+            (df["high"] < df["low"])
+            | (df["open"] > df["high"])
+            | (df["open"] < df["low"])
+            | (df["close"] > df["high"])
+            | (df["close"] < df["low"])
+        )
+        stats["bad_ohlc_rows"] = int(bad_ohlc.sum())
+
+        return stats
+
+    @staticmethod
+    def save_dataframe(
+        df: pd.DataFrame,
+        output_dir: Path,
+        symbol: str,
+        timeframe: str,
+        fmt: str = "parquet",
+    ) -> Path:
+        output_dir.mkdir(parents=True, exist_ok=True)
+
+        filename = f"{symbol}_{timeframe}.{fmt}"
+        path = output_dir / filename
+
+        if fmt == "parquet":
+            df.to_parquet(path, index=False)
+        elif fmt == "csv":
+            df.to_csv(path, index=False)
+        else:
+            raise ValueError("fmt must be 'parquet' or 'csv'")
+
+        return path