diff --git a/Quant_Hiber_Risk_Stargy/data/preprocessor.py b/Quant_Hiber_Risk_Stargy/data/preprocessor.py new file mode 100644 index 0000000..702a9eb --- /dev/null +++ b/Quant_Hiber_Risk_Stargy/data/preprocessor.py @@ -0,0 +1,52 @@ +import pandas as pd +import os +from pathlib import Path + +def preprocess_all_symbols(): + symbols = ['EURUSDm', 'GBPUSDm', 'USDJPYm', 'USDCHFm', 'USDCADm', 'NZDUSDm'] + timeframes = ['M15', 'H1', 'H4'] + raw_base = Path("data/raw") + processed_base = Path("data/processed") + + print("--- BẮT ĐẦU TIỀN XỬ LÝ DỮ LIỆU ---") + + for symbol in symbols: + for tf in timeframes: + file_path = raw_base / symbol / tf / f"{symbol}_{tf}.parquet" + + if not file_path.exists(): + continue + + # 1. Đọc dữ liệu + df = pd.read_parquet(file_path) + initial_count = len(df) + + # 2. Xử lý logic cơ bản + # Chuyển index sang datetime nếu chưa có + df.index = pd.to_datetime(df['timestamp']) + df = df.sort_index() + + # Loại bỏ trùng lặp + df = df[~df.index.duplicated(keep='first')] + + # Loại bỏ nến lỗi (High < Low) + df = df[df['high'] >= df['low']] + + # Lọc từ năm 2022 trở lại đây như bạn yêu cầu + df = df.loc['2022-01-01':] + + # 3. Loại bỏ ngày cuối tuần (Forex nghỉ) + df = df[df.index.dayofweek < 5] + + # 4. Lưu dữ liệu sạch + output_dir = processed_base / symbol / tf + output_dir.mkdir(parents=True, exist_ok=True) + output_path = output_dir / f"{symbol}_{tf}_clean.parquet" + df.to_parquet(output_path) + + print(f"[OK] {symbol} {tf}: {initial_count} -> {len(df)} nến (Đã lưu tại {output_path})") + + print("--- HOÀN THÀNH ---") + +if __name__ == "__main__": + preprocess_all_symbols() \ No newline at end of file diff --git a/Quant_Hiber_Risk_Stargy/data/processed/EURUSDm/H1/EURUSDm_H1_clean.parquet b/Quant_Hiber_Risk_Stargy/data/processed/EURUSDm/H1/EURUSDm_H1_clean.parquet new file mode 100644 index 0000000..6f409e2 Binary files /dev/null and b/Quant_Hiber_Risk_Stargy/data/processed/EURUSDm/H1/EURUSDm_H1_clean.parquet differ diff --git a/Quant_Hiber_Risk_Stargy/data/processed/EURUSDm/H4/EURUSDm_H4_clean.parquet b/Quant_Hiber_Risk_Stargy/data/processed/EURUSDm/H4/EURUSDm_H4_clean.parquet new file mode 100644 index 0000000..c7644bf Binary files /dev/null and b/Quant_Hiber_Risk_Stargy/data/processed/EURUSDm/H4/EURUSDm_H4_clean.parquet differ diff --git a/Quant_Hiber_Risk_Stargy/data/processed/EURUSDm/M15/EURUSDm_M15_clean.parquet b/Quant_Hiber_Risk_Stargy/data/processed/EURUSDm/M15/EURUSDm_M15_clean.parquet new file mode 100644 index 0000000..63f27a5 Binary files /dev/null and b/Quant_Hiber_Risk_Stargy/data/processed/EURUSDm/M15/EURUSDm_M15_clean.parquet differ diff --git a/Quant_Hiber_Risk_Stargy/data/processed/GBPUSDm/H1/GBPUSDm_H1_clean.parquet b/Quant_Hiber_Risk_Stargy/data/processed/GBPUSDm/H1/GBPUSDm_H1_clean.parquet new file mode 100644 index 0000000..21a648d Binary files /dev/null and b/Quant_Hiber_Risk_Stargy/data/processed/GBPUSDm/H1/GBPUSDm_H1_clean.parquet differ diff --git a/Quant_Hiber_Risk_Stargy/data/processed/GBPUSDm/H4/GBPUSDm_H4_clean.parquet b/Quant_Hiber_Risk_Stargy/data/processed/GBPUSDm/H4/GBPUSDm_H4_clean.parquet new file mode 100644 index 0000000..06f9cf3 Binary files /dev/null and b/Quant_Hiber_Risk_Stargy/data/processed/GBPUSDm/H4/GBPUSDm_H4_clean.parquet differ diff --git a/Quant_Hiber_Risk_Stargy/data/processed/GBPUSDm/M15/GBPUSDm_M15_clean.parquet b/Quant_Hiber_Risk_Stargy/data/processed/GBPUSDm/M15/GBPUSDm_M15_clean.parquet new file mode 100644 index 0000000..9e6ebb3 Binary files /dev/null and b/Quant_Hiber_Risk_Stargy/data/processed/GBPUSDm/M15/GBPUSDm_M15_clean.parquet differ diff --git a/Quant_Hiber_Risk_Stargy/data/processed/NZDUSDm/H1/NZDUSDm_H1_clean.parquet b/Quant_Hiber_Risk_Stargy/data/processed/NZDUSDm/H1/NZDUSDm_H1_clean.parquet new file mode 100644 index 0000000..d54b64f Binary files /dev/null and b/Quant_Hiber_Risk_Stargy/data/processed/NZDUSDm/H1/NZDUSDm_H1_clean.parquet differ diff --git a/Quant_Hiber_Risk_Stargy/data/processed/NZDUSDm/H4/NZDUSDm_H4_clean.parquet b/Quant_Hiber_Risk_Stargy/data/processed/NZDUSDm/H4/NZDUSDm_H4_clean.parquet new file mode 100644 index 0000000..538e117 Binary files /dev/null and b/Quant_Hiber_Risk_Stargy/data/processed/NZDUSDm/H4/NZDUSDm_H4_clean.parquet differ diff --git a/Quant_Hiber_Risk_Stargy/data/processed/NZDUSDm/M15/NZDUSDm_M15_clean.parquet b/Quant_Hiber_Risk_Stargy/data/processed/NZDUSDm/M15/NZDUSDm_M15_clean.parquet new file mode 100644 index 0000000..a12b39d Binary files /dev/null and b/Quant_Hiber_Risk_Stargy/data/processed/NZDUSDm/M15/NZDUSDm_M15_clean.parquet differ diff --git a/Quant_Hiber_Risk_Stargy/data/processed/USDCADm/H1/USDCADm_H1_clean.parquet b/Quant_Hiber_Risk_Stargy/data/processed/USDCADm/H1/USDCADm_H1_clean.parquet new file mode 100644 index 0000000..97b84b7 Binary files /dev/null and b/Quant_Hiber_Risk_Stargy/data/processed/USDCADm/H1/USDCADm_H1_clean.parquet differ diff --git a/Quant_Hiber_Risk_Stargy/data/processed/USDCADm/H4/USDCADm_H4_clean.parquet b/Quant_Hiber_Risk_Stargy/data/processed/USDCADm/H4/USDCADm_H4_clean.parquet new file mode 100644 index 0000000..86fa6fb Binary files /dev/null and b/Quant_Hiber_Risk_Stargy/data/processed/USDCADm/H4/USDCADm_H4_clean.parquet differ diff --git a/Quant_Hiber_Risk_Stargy/data/processed/USDCADm/M15/USDCADm_M15_clean.parquet b/Quant_Hiber_Risk_Stargy/data/processed/USDCADm/M15/USDCADm_M15_clean.parquet new file mode 100644 index 0000000..42a4ea6 Binary files /dev/null and b/Quant_Hiber_Risk_Stargy/data/processed/USDCADm/M15/USDCADm_M15_clean.parquet differ diff --git a/Quant_Hiber_Risk_Stargy/data/processed/USDCHFm/H1/USDCHFm_H1_clean.parquet b/Quant_Hiber_Risk_Stargy/data/processed/USDCHFm/H1/USDCHFm_H1_clean.parquet new file mode 100644 index 0000000..92965c7 Binary files /dev/null and b/Quant_Hiber_Risk_Stargy/data/processed/USDCHFm/H1/USDCHFm_H1_clean.parquet differ diff --git a/Quant_Hiber_Risk_Stargy/data/processed/USDCHFm/H4/USDCHFm_H4_clean.parquet b/Quant_Hiber_Risk_Stargy/data/processed/USDCHFm/H4/USDCHFm_H4_clean.parquet new file mode 100644 index 0000000..481c66a Binary files /dev/null and b/Quant_Hiber_Risk_Stargy/data/processed/USDCHFm/H4/USDCHFm_H4_clean.parquet differ diff --git a/Quant_Hiber_Risk_Stargy/data/processed/USDCHFm/M15/USDCHFm_M15_clean.parquet b/Quant_Hiber_Risk_Stargy/data/processed/USDCHFm/M15/USDCHFm_M15_clean.parquet new file mode 100644 index 0000000..5f27245 Binary files /dev/null and b/Quant_Hiber_Risk_Stargy/data/processed/USDCHFm/M15/USDCHFm_M15_clean.parquet differ diff --git a/Quant_Hiber_Risk_Stargy/data/processed/USDJPYm/H1/USDJPYm_H1_clean.parquet b/Quant_Hiber_Risk_Stargy/data/processed/USDJPYm/H1/USDJPYm_H1_clean.parquet new file mode 100644 index 0000000..7780f5b Binary files /dev/null and b/Quant_Hiber_Risk_Stargy/data/processed/USDJPYm/H1/USDJPYm_H1_clean.parquet differ diff --git a/Quant_Hiber_Risk_Stargy/data/processed/USDJPYm/H4/USDJPYm_H4_clean.parquet b/Quant_Hiber_Risk_Stargy/data/processed/USDJPYm/H4/USDJPYm_H4_clean.parquet new file mode 100644 index 0000000..85b40ca Binary files /dev/null and b/Quant_Hiber_Risk_Stargy/data/processed/USDJPYm/H4/USDJPYm_H4_clean.parquet differ diff --git a/Quant_Hiber_Risk_Stargy/data/processed/USDJPYm/M15/USDJPYm_M15_clean.parquet b/Quant_Hiber_Risk_Stargy/data/processed/USDJPYm/M15/USDJPYm_M15_clean.parquet new file mode 100644 index 0000000..799ccf7 Binary files /dev/null and b/Quant_Hiber_Risk_Stargy/data/processed/USDJPYm/M15/USDJPYm_M15_clean.parquet differ diff --git a/Quant_Hiber_Risk_Stargy/data/raw/EURUSDm/H1/EURUSDm_H1.parquet b/Quant_Hiber_Risk_Stargy/data/raw/EURUSDm/H1/EURUSDm_H1.parquet new file mode 100644 index 0000000..e7c7b50 Binary files /dev/null and b/Quant_Hiber_Risk_Stargy/data/raw/EURUSDm/H1/EURUSDm_H1.parquet differ diff --git a/Quant_Hiber_Risk_Stargy/data/raw/EURUSDm/H4/EURUSDm_H4.parquet b/Quant_Hiber_Risk_Stargy/data/raw/EURUSDm/H4/EURUSDm_H4.parquet new file mode 100644 index 0000000..724625e Binary files /dev/null and b/Quant_Hiber_Risk_Stargy/data/raw/EURUSDm/H4/EURUSDm_H4.parquet differ diff --git a/Quant_Hiber_Risk_Stargy/data/raw/EURUSDm/M15/EURUSDm_M15.parquet b/Quant_Hiber_Risk_Stargy/data/raw/EURUSDm/M15/EURUSDm_M15.parquet new file mode 100644 index 0000000..87204bd Binary files /dev/null and b/Quant_Hiber_Risk_Stargy/data/raw/EURUSDm/M15/EURUSDm_M15.parquet differ diff --git a/Quant_Hiber_Risk_Stargy/data/raw/GBPUSDm/H1/GBPUSDm_H1.parquet b/Quant_Hiber_Risk_Stargy/data/raw/GBPUSDm/H1/GBPUSDm_H1.parquet new file mode 100644 index 0000000..01dd8df Binary files /dev/null and b/Quant_Hiber_Risk_Stargy/data/raw/GBPUSDm/H1/GBPUSDm_H1.parquet differ diff --git a/Quant_Hiber_Risk_Stargy/data/raw/GBPUSDm/H4/GBPUSDm_H4.parquet b/Quant_Hiber_Risk_Stargy/data/raw/GBPUSDm/H4/GBPUSDm_H4.parquet new file mode 100644 index 0000000..a799eb9 Binary files /dev/null and b/Quant_Hiber_Risk_Stargy/data/raw/GBPUSDm/H4/GBPUSDm_H4.parquet differ diff --git a/Quant_Hiber_Risk_Stargy/data/raw/GBPUSDm/M15/GBPUSDm_M15.parquet b/Quant_Hiber_Risk_Stargy/data/raw/GBPUSDm/M15/GBPUSDm_M15.parquet new file mode 100644 index 0000000..331becd Binary files /dev/null and b/Quant_Hiber_Risk_Stargy/data/raw/GBPUSDm/M15/GBPUSDm_M15.parquet differ diff --git a/Quant_Hiber_Risk_Stargy/data/raw/NZDUSDm/H1/NZDUSDm_H1.parquet b/Quant_Hiber_Risk_Stargy/data/raw/NZDUSDm/H1/NZDUSDm_H1.parquet new file mode 100644 index 0000000..bb30823 Binary files /dev/null and b/Quant_Hiber_Risk_Stargy/data/raw/NZDUSDm/H1/NZDUSDm_H1.parquet differ diff --git a/Quant_Hiber_Risk_Stargy/data/raw/NZDUSDm/H4/NZDUSDm_H4.parquet b/Quant_Hiber_Risk_Stargy/data/raw/NZDUSDm/H4/NZDUSDm_H4.parquet new file mode 100644 index 0000000..aa43792 Binary files /dev/null and b/Quant_Hiber_Risk_Stargy/data/raw/NZDUSDm/H4/NZDUSDm_H4.parquet differ diff --git a/Quant_Hiber_Risk_Stargy/data/raw/NZDUSDm/M15/NZDUSDm_M15.parquet b/Quant_Hiber_Risk_Stargy/data/raw/NZDUSDm/M15/NZDUSDm_M15.parquet new file mode 100644 index 0000000..9147389 Binary files /dev/null and b/Quant_Hiber_Risk_Stargy/data/raw/NZDUSDm/M15/NZDUSDm_M15.parquet differ diff --git a/Quant_Hiber_Risk_Stargy/data/raw/USDCADm/H1/USDCADm_H1.parquet b/Quant_Hiber_Risk_Stargy/data/raw/USDCADm/H1/USDCADm_H1.parquet new file mode 100644 index 0000000..7b79852 Binary files /dev/null and b/Quant_Hiber_Risk_Stargy/data/raw/USDCADm/H1/USDCADm_H1.parquet differ diff --git a/Quant_Hiber_Risk_Stargy/data/raw/USDCADm/H4/USDCADm_H4.parquet b/Quant_Hiber_Risk_Stargy/data/raw/USDCADm/H4/USDCADm_H4.parquet new file mode 100644 index 0000000..da23a09 Binary files /dev/null and b/Quant_Hiber_Risk_Stargy/data/raw/USDCADm/H4/USDCADm_H4.parquet differ diff --git a/Quant_Hiber_Risk_Stargy/data/raw/USDCADm/M15/USDCADm_M15.parquet b/Quant_Hiber_Risk_Stargy/data/raw/USDCADm/M15/USDCADm_M15.parquet new file mode 100644 index 0000000..64977df Binary files /dev/null and b/Quant_Hiber_Risk_Stargy/data/raw/USDCADm/M15/USDCADm_M15.parquet differ diff --git a/Quant_Hiber_Risk_Stargy/data/raw/USDCHFm/H1/USDCHFm_H1.parquet b/Quant_Hiber_Risk_Stargy/data/raw/USDCHFm/H1/USDCHFm_H1.parquet new file mode 100644 index 0000000..a836cc9 Binary files /dev/null and b/Quant_Hiber_Risk_Stargy/data/raw/USDCHFm/H1/USDCHFm_H1.parquet differ diff --git a/Quant_Hiber_Risk_Stargy/data/raw/USDCHFm/H4/USDCHFm_H4.parquet b/Quant_Hiber_Risk_Stargy/data/raw/USDCHFm/H4/USDCHFm_H4.parquet new file mode 100644 index 0000000..1f39d64 Binary files /dev/null and b/Quant_Hiber_Risk_Stargy/data/raw/USDCHFm/H4/USDCHFm_H4.parquet differ diff --git a/Quant_Hiber_Risk_Stargy/data/raw/USDCHFm/M15/USDCHFm_M15.parquet b/Quant_Hiber_Risk_Stargy/data/raw/USDCHFm/M15/USDCHFm_M15.parquet new file mode 100644 index 0000000..ba4118e Binary files /dev/null and b/Quant_Hiber_Risk_Stargy/data/raw/USDCHFm/M15/USDCHFm_M15.parquet differ diff --git a/Quant_Hiber_Risk_Stargy/data/raw/USDJPYm/H1/USDJPYm_H1.parquet b/Quant_Hiber_Risk_Stargy/data/raw/USDJPYm/H1/USDJPYm_H1.parquet new file mode 100644 index 0000000..f0b52f5 Binary files /dev/null and b/Quant_Hiber_Risk_Stargy/data/raw/USDJPYm/H1/USDJPYm_H1.parquet differ diff --git a/Quant_Hiber_Risk_Stargy/data/raw/USDJPYm/H4/USDJPYm_H4.parquet b/Quant_Hiber_Risk_Stargy/data/raw/USDJPYm/H4/USDJPYm_H4.parquet new file mode 100644 index 0000000..59819fa Binary files /dev/null and b/Quant_Hiber_Risk_Stargy/data/raw/USDJPYm/H4/USDJPYm_H4.parquet differ diff --git a/Quant_Hiber_Risk_Stargy/data/raw/USDJPYm/M15/USDJPYm_M15.parquet b/Quant_Hiber_Risk_Stargy/data/raw/USDJPYm/M15/USDJPYm_M15.parquet new file mode 100644 index 0000000..8383f7e Binary files /dev/null and b/Quant_Hiber_Risk_Stargy/data/raw/USDJPYm/M15/USDJPYm_M15.parquet differ diff --git a/Quant_Hiber_Risk_Stargy/data/raw/script.py b/Quant_Hiber_Risk_Stargy/data/raw/script.py new file mode 100644 index 0000000..8573143 --- /dev/null +++ b/Quant_Hiber_Risk_Stargy/data/raw/script.py @@ -0,0 +1,14 @@ +import pandas as pd + +# Đường dẫn đến file vừa tải +file_path = r"data\raw\EURUSDm\M15\EURUSDm_M15.parquet" + +# Đọc dữ liệu +df = pd.read_parquet(file_path) + +# Xem 5 dòng đầu và 5 dòng cuối +print(df.head()) +print(df.tail()) + +# Kiểm tra xem có ngày nào bị thiếu không +print(f"Tổng số nến: {len(df)}") \ No newline at end of file diff --git a/Quant_Hiber_Risk_Stargy/scripts/download_data.py/download_data.py b/Quant_Hiber_Risk_Stargy/scripts/download_data.py similarity index 89% rename from Quant_Hiber_Risk_Stargy/scripts/download_data.py/download_data.py rename to Quant_Hiber_Risk_Stargy/scripts/download_data.py index ef64447..efc9263 100644 --- a/Quant_Hiber_Risk_Stargy/scripts/download_data.py/download_data.py +++ b/Quant_Hiber_Risk_Stargy/scripts/download_data.py @@ -18,13 +18,26 @@ import sys from datetime import datetime from pathlib import Path -ROOT = Path(__file__).resolve().parents[1] + +def _find_repo_root(start: Path) -> Path: + """Walk up from start until a directory containing `src` is found.""" + current = start.resolve() + while True: + if (current / "src").is_dir(): + return current + if current.parent == current: + break + current = current.parent + + raise RuntimeError("Cannot locate repository root (directory containing 'src').") + + +ROOT = _find_repo_root(Path(__file__).parent) if str(ROOT) not in sys.path: sys.path.insert(0, str(ROOT)) from src.data.mt5_fetcher import MT5Config, MT5Fetcher # noqa: E402 - DEFAULT_SYMBOLS = ["EURUSD", "USDJPY", "GBPUSD", "USDCHF", "AUDUSD", "NZDUSD"] DEFAULT_TIMEFRAMES = ["M15", "H1", "H4"] @@ -147,4 +160,4 @@ def main() -> None: if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/Quant_Hiber_Risk_Stargy/src/__pycache__/__init__.cpython-313.pyc b/Quant_Hiber_Risk_Stargy/src/__pycache__/__init__.cpython-313.pyc new file mode 100644 index 0000000..55232c4 Binary files /dev/null and b/Quant_Hiber_Risk_Stargy/src/__pycache__/__init__.cpython-313.pyc differ diff --git a/Quant_Hiber_Risk_Stargy/src/__pycache__/__init__.cpython-314.pyc b/Quant_Hiber_Risk_Stargy/src/__pycache__/__init__.cpython-314.pyc new file mode 100644 index 0000000..b91ee49 Binary files /dev/null and b/Quant_Hiber_Risk_Stargy/src/__pycache__/__init__.cpython-314.pyc differ diff --git a/Quant_Hiber_Risk_Stargy/src/data/__init__.py b/Quant_Hiber_Risk_Stargy/src/data/__init__.py index e69de29..d505b31 100644 --- a/Quant_Hiber_Risk_Stargy/src/data/__init__.py +++ b/Quant_Hiber_Risk_Stargy/src/data/__init__.py @@ -0,0 +1,3 @@ +""" +Data package for quant FX bot. +""" \ No newline at end of file diff --git a/Quant_Hiber_Risk_Stargy/src/data/__pycache__/__init__.cpython-313.pyc b/Quant_Hiber_Risk_Stargy/src/data/__pycache__/__init__.cpython-313.pyc new file mode 100644 index 0000000..d60f7c9 Binary files /dev/null and b/Quant_Hiber_Risk_Stargy/src/data/__pycache__/__init__.cpython-313.pyc differ diff --git a/Quant_Hiber_Risk_Stargy/src/data/__pycache__/__init__.cpython-314.pyc b/Quant_Hiber_Risk_Stargy/src/data/__pycache__/__init__.cpython-314.pyc new file mode 100644 index 0000000..92f0aec Binary files /dev/null and b/Quant_Hiber_Risk_Stargy/src/data/__pycache__/__init__.cpython-314.pyc differ diff --git a/Quant_Hiber_Risk_Stargy/src/data/__pycache__/mt5_fetcher.cpython-313.pyc b/Quant_Hiber_Risk_Stargy/src/data/__pycache__/mt5_fetcher.cpython-313.pyc new file mode 100644 index 0000000..d7d5955 Binary files /dev/null and b/Quant_Hiber_Risk_Stargy/src/data/__pycache__/mt5_fetcher.cpython-313.pyc differ diff --git a/Quant_Hiber_Risk_Stargy/src/data/__pycache__/mt5_fetcher.cpython-314.pyc b/Quant_Hiber_Risk_Stargy/src/data/__pycache__/mt5_fetcher.cpython-314.pyc new file mode 100644 index 0000000..d1b867f Binary files /dev/null and b/Quant_Hiber_Risk_Stargy/src/data/__pycache__/mt5_fetcher.cpython-314.pyc differ diff --git a/Quant_Hiber_Risk_Stargy/src/data/mt5_fetcher.py b/Quant_Hiber_Risk_Stargy/src/data/mt5_fetcher.py index e69de29..12864ef 100644 --- a/Quant_Hiber_Risk_Stargy/src/data/mt5_fetcher.py +++ b/Quant_Hiber_Risk_Stargy/src/data/mt5_fetcher.py @@ -0,0 +1,184 @@ +""" +src/data/mt5_fetcher.py + +MT5 historical data fetcher for Forex quant bot. +- Connects to MetaTrader 5 terminal +- Downloads OHLCV bars +- Exports to parquet/csv +""" + +from __future__ import annotations + +from dataclasses import dataclass +from datetime import datetime +from pathlib import Path +from typing import Optional + +import pandas as pd + +try: + import MetaTrader5 as mt5 +except ImportError as exc: + raise ImportError( + "MetaTrader5 package is not installed. " + "Run: pip install MetaTrader5" + ) from exc + + +TIMEFRAME_MAP: dict[str, int] = { + "M1": mt5.TIMEFRAME_M1, + "M5": mt5.TIMEFRAME_M5, + "M15": mt5.TIMEFRAME_M15, + "M30": mt5.TIMEFRAME_M30, + "H1": mt5.TIMEFRAME_H1, + "H4": mt5.TIMEFRAME_H4, + "D1": mt5.TIMEFRAME_D1, +} + + +@dataclass +class MT5Config: + terminal_path: Optional[str] = None + login: Optional[int] = None + password: Optional[str] = None + server: Optional[str] = None + timezone: str = "UTC" + + +class MT5Fetcher: + def __init__(self, config: MT5Config): + self.config = config + self.initialized = False + + def initialize(self) -> None: + kwargs = {} + + if self.config.terminal_path: + kwargs["path"] = self.config.terminal_path + if self.config.login: + kwargs["login"] = self.config.login + if self.config.password: + kwargs["password"] = self.config.password + if self.config.server: + kwargs["server"] = self.config.server + + ok = mt5.initialize(**kwargs) + if not ok: + code, msg = mt5.last_error() + raise RuntimeError(f"MT5 initialize failed: {code} - {msg}") + + self.initialized = True + + def shutdown(self) -> None: + if self.initialized: + mt5.shutdown() + self.initialized = False + + def ensure_symbol(self, symbol: str) -> None: + info = mt5.symbol_info(symbol) + if info is None: + raise ValueError(f"Symbol not found in MT5 terminal: {symbol}") + + if not info.visible: + selected = mt5.symbol_select(symbol, True) + if not selected: + raise RuntimeError(f"Failed to enable symbol in Market Watch: {symbol}") + + def fetch_range( + self, + symbol: str, + timeframe: str, + start: datetime, + end: datetime, + ) -> pd.DataFrame: + if timeframe not in TIMEFRAME_MAP: + raise ValueError(f"Unsupported timeframe: {timeframe}") + + self.ensure_symbol(symbol) + + rates = mt5.copy_rates_range(symbol, TIMEFRAME_MAP[timeframe], start, end) + if rates is None: + code, msg = mt5.last_error() + raise RuntimeError( + f"Failed to fetch rates for {symbol} {timeframe}: {code} - {msg}" + ) + + df = pd.DataFrame(rates) + if df.empty: + raise ValueError(f"No data returned for {symbol} {timeframe}") + + df["time"] = pd.to_datetime(df["time"], unit="s", utc=True) + df = df.rename( + columns={ + "time": "timestamp", + "tick_volume": "tick_volume", + "real_volume": "real_volume", + "spread": "spread", + } + ) + + ordered_cols = [ + "timestamp", + "open", + "high", + "low", + "close", + "tick_volume", + "spread", + "real_volume", + ] + existing_cols = [c for c in ordered_cols if c in df.columns] + df = df[existing_cols].copy() + + df["symbol"] = symbol + df["timeframe"] = timeframe + + return df + + @staticmethod + def validate_bars(df: pd.DataFrame) -> dict: + required_cols = {"timestamp", "open", "high", "low", "close"} + missing = required_cols - set(df.columns) + if missing: + raise ValueError(f"Missing required columns: {sorted(missing)}") + + stats = { + "rows": int(len(df)), + "start": str(df["timestamp"].min()), + "end": str(df["timestamp"].max()), + "duplicate_timestamps": int(df["timestamp"].duplicated().sum()), + "null_count": int(df.isnull().sum().sum()), + } + + bad_ohlc = ( + (df["high"] < df["low"]) + | (df["open"] > df["high"]) + | (df["open"] < df["low"]) + | (df["close"] > df["high"]) + | (df["close"] < df["low"]) + ) + stats["bad_ohlc_rows"] = int(bad_ohlc.sum()) + + return stats + + @staticmethod + def save_dataframe( + df: pd.DataFrame, + output_dir: Path, + symbol: str, + timeframe: str, + fmt: str = "parquet", + ) -> Path: + output_dir.mkdir(parents=True, exist_ok=True) + + filename = f"{symbol}_{timeframe}.{fmt}" + path = output_dir / filename + + if fmt == "parquet": + df.to_parquet(path, index=False) + elif fmt == "csv": + df.to_csv(path, index=False) + else: + raise ValueError("fmt must be 'parquet' or 'csv'") + + return path \ No newline at end of file