Initial commit - AHAD QUANT v1

2026-06-25 14:00:20 +03:00
commit 71a885dfe3
48 changed files with 22322 additions and 0 deletions
@@ -0,0 +1,523 @@
+"""
+AHAD QUANT — Data Downloader (Forex Edition)
+Downloads 1h candle data from multiple sources for Forex pairs.
+
+Sources (by priority):
+  1. OANDA v20 REST API  — if OANDA_API_KEY is set (recommended)
+  2. Yahoo Finance        — free, no key required (yfinance)
+
+Usage:
+    python download_data.py
+
+"""
+
+import json
+import os
+import sys
+import time
+import requests
+
+# ── Fix encodage Windows ──────────────────────────────────────────────────────
+if hasattr(sys.stdout, 'reconfigure'):
+    try:
+        sys.stdout.reconfigure(encoding='utf-8', errors='replace')
+    except Exception:
+        pass
+
+import config
+
+# ── Constants ─────────────────────────────────────────────────────────────────
+DAYS_BACK = 730          # yfinance 1h limit (2 years)
+MAX_RETRIES = 5
+RETRY_DELAY = 3
+
+# OANDA candle granularities map
+OANDA_GRANULARITY = {
+    "1m": "M1", "5m": "M5", "15m": "M15", "30m": "M30",
+    "1h": "H1", "4h": "H4", "1d": "D",
+}
+
+# Yahoo Finance Forex suffix and ticker format
+# EUR/USD → "EURUSD=X"
+YAHOO_SUFFIX = "=X"
+
+
+# ─── OANDA source ─────────────────────────────────────────────────────────────
+
+def _oanda_to_instrument(pair: str) -> str:
+    """EURUSD → EUR_USD (OANDA format)."""
+    return f"{pair[:3]}_{pair[3:]}"
+
+
+def get_candles_oanda(pair: str, interval: str = "1h",
+                      days: int = DAYS_BACK) -> list[dict]:
+    """
+    Fetch historical candles from OANDA v20 API.
+    Requires OANDA_API_KEY and OANDA_ACCOUNT_ID in environment.
+    """
+    api_key = config.OANDA_API_KEY
+    practice = config.OANDA_PRACTICE
+
+    base_url = (
+        "https://api-fxpractice.oanda.com"
+        if practice else
+        "https://api-fxtrade.oanda.com"
+    )
+
+    instrument = _oanda_to_instrument(pair)
+    granularity = OANDA_GRANULARITY.get(interval, "H1")
+
+    # OANDA max per request = 5000 candles
+    candles_per_req = 5000
+    end_ts = int(time.time())
+    start_ts = end_ts - (days * 24 * 3600)
+
+    headers = {
+        "Authorization": f"Bearer {api_key}",
+        "Content-Type": "application/json",
+    }
+
+    all_candles: list[dict] = []
+    seen: set = set()
+    current_start = start_ts
+
+    interval_seconds = {"M1": 60, "M5": 300, "M15": 900, "M30": 1800,
+                         "H1": 3600, "H4": 14400, "D": 86400}.get(granularity, 3600)
+
+    while current_start < end_ts:
+        url = (
+            f"{base_url}/v3/instruments/{instrument}/candles"
+            f"?granularity={granularity}"
+            f"&from={current_start}"
+            f"&to={min(current_start + candles_per_req * interval_seconds, end_ts)}"
+            f"&price=M"   # midpoint (bid+ask)/2
+        )
+        for attempt in range(1, MAX_RETRIES + 1):
+            try:
+                resp = requests.get(url, headers=headers, timeout=30)
+                resp.raise_for_status()
+                data = resp.json()
+                break
+            except Exception as e:
+                if attempt < MAX_RETRIES:
+                    time.sleep(RETRY_DELAY * attempt)
+                else:
+                    raise e
+
+        raw = data.get("candles", [])
+        if not raw:
+            break
+
+        for c in raw:
+            if not c.get("complete", True):
+                continue
+            t = int(c["time"].split(".")[0]) if "." in c["time"] else int(c["time"][:10])
+            if t not in seen:
+                seen.add(t)
+                mid = c["mid"]
+                all_candles.append({
+                    "t": t * 1000,  # milliseconds (matches Binance format)
+                    "o": float(mid["o"]),
+                    "h": float(mid["h"]),
+                    "l": float(mid["l"]),
+                    "c": float(mid["c"]),
+                    "v": float(c.get("volume", 0)),
+                })
+
+        last_t = int(raw[-1]["time"][:10])
+        if last_t <= current_start:
+            break
+        current_start = last_t + interval_seconds
+
+        if len(raw) < candles_per_req:
+            break
+
+        time.sleep(0.1)
+
+    all_candles.sort(key=lambda x: x["t"])
+    return all_candles
+
+
+# ─── Yahoo Finance source ─────────────────────────────────────────────────────
+
+def get_candles_yfinance(pair: str, interval: str = "1h",
+                         days: int = DAYS_BACK) -> list[dict]:
+    """
+    Fetch historical candles from Yahoo Finance (free, no key required).
+    1h data available up to 730 days back.
+    """
+    try:
+        import yfinance as yf
+    except ImportError:
+        raise ImportError(
+            "yfinance is required for free data download.\n"
+            "Install with: pip install yfinance"
+        )
+
+    ticker_sym = f"{pair}{YAHOO_SUFFIX}"
+
+    # yfinance 1h: max period "730d"
+    period_map = {"1m": "7d", "5m": "60d", "15m": "60d", "30m": "60d",
+                  "1h": "730d", "4h": "730d", "1d": "max"}
+    period = period_map.get(interval, "730d")
+
+    # yfinance interval notation
+    yf_interval_map = {"1h": "1h", "4h": "1h", "1d": "1d",
+                       "1m": "1m", "5m": "5m", "15m": "15m"}
+    yf_interval = yf_interval_map.get(interval, "1h")
+
+    for attempt in range(1, MAX_RETRIES + 1):
+        try:
+            ticker = yf.Ticker(ticker_sym)
+            df = ticker.history(period=period, interval=yf_interval,
+                                auto_adjust=True, prepost=False)
+            break
+        except Exception as e:
+            if attempt < MAX_RETRIES:
+                time.sleep(RETRY_DELAY * attempt)
+            else:
+                raise e
+
+    if df is None or df.empty:
+        raise ValueError(f"No data returned from Yahoo Finance for {pair}")
+
+    # If 4h was requested but yfinance only gives 1h, resample
+    if interval == "4h" and yf_interval == "1h":
+        df = df.resample("4h").agg({
+            "Open": "first", "High": "max", "Low": "min",
+            "Close": "last", "Volume": "sum",
+        }).dropna()
+
+    candles: list[dict] = []
+    for ts, row in df.iterrows():
+        # Normalize timestamp to milliseconds
+        if hasattr(ts, "timestamp"):
+            t_ms = int(ts.timestamp() * 1000)
+        else:
+            t_ms = int(ts) * 1000
+
+        candles.append({
+            "t": t_ms,
+            "o": float(row["Open"]),
+            "h": float(row["High"]),
+            "l": float(row["Low"]),
+            "c": float(row["Close"]),
+            "v": float(row.get("Volume", 0)),
+        })
+
+    candles.sort(key=lambda x: x["t"])
+    return candles
+
+
+
+# ─── Twelve Data source ───────────────────────────────────────────────────────
+
+def get_candles_twelvedata(pair: str, interval: str = "1h",
+                           days: int = DAYS_BACK) -> list[dict]:
+    """
+    Fetch historical Forex candles from Twelve Data API.
+    Plan Free : 800 req/jour, données Forex 1h sans clé (rate-limited).
+    Inscription gratuite : https://twelvedata.com/
+    """
+    api_key = config.TWELVE_DATA_API_KEY
+    # Twelve Data interval notation
+    _interval_map = {
+        "1m": "1min", "5m": "5min", "15m": "15min", "30m": "30min",
+        "1h": "1h", "4h": "4h", "1d": "1day",
+    }
+    td_interval = _interval_map.get(interval, "1h")
+
+    # Format paire : EURUSD → EUR/USD
+    symbol = f"{pair[:3]}/{pair[3:]}"
+
+    # Nombre de points à récupérer (max 5000 par requête)
+    outputsize = min(days * 24, 5000)
+
+    base_url = "https://api.twelvedata.com/time_series"
+    params: dict = {
+        "symbol": symbol,
+        "interval": td_interval,
+        "outputsize": outputsize,
+        "format": "JSON",
+        "timezone": "UTC",
+    }
+    if api_key:
+        params["apikey"] = api_key
+
+    from datetime import datetime as _dt   # import unique, hors boucle
+
+    for attempt in range(1, MAX_RETRIES + 1):
+        try:
+            resp = requests.get(base_url, params=params, timeout=30)
+            resp.raise_for_status()
+            data = resp.json()
+            if data.get("status") == "error" or "values" not in data:
+                raise ValueError(f"Twelve Data error: {data.get('message', data)}")
+            values = data["values"]
+            candles = []
+            for bar in reversed(values):   # API retourne du plus récent au plus ancien
+                dt = _dt.strptime(bar["datetime"], "%Y-%m-%d %H:%M:%S")
+                candles.append({
+                    "t": int(dt.timestamp() * 1000),
+                    "o": float(bar["open"]),
+                    "h": float(bar["high"]),
+                    "l": float(bar["low"]),
+                    "c": float(bar["close"]),
+                    "v": float(bar.get("volume", 0)),
+                })
+            candles.sort(key=lambda x: x["t"])
+            return candles
+        except Exception as e:
+            if attempt < MAX_RETRIES:
+                time.sleep(RETRY_DELAY * attempt)
+            else:
+                raise ValueError(f"Twelve Data — {pair}: {e}") from e
+    return []
+
+
+# ─── Alpha Vantage source ─────────────────────────────────────────────────────
+
+def get_candles_alphavantage(pair: str, interval: str = "1h",
+                              days: int = DAYS_BACK) -> list[dict]:
+    """
+    Fetch Forex candles from Alpha Vantage.
+    Plan Free : 25 req/jour. ALPHA_VANTAGE_API_KEY requis.
+    Inscription : https://www.alphavantage.co/support/#api-key
+    """
+    api_key = config.ALPHA_VANTAGE_API_KEY
+    if not api_key:
+        raise ValueError(
+            "ALPHA_VANTAGE_API_KEY est requis pour cette source.\n"
+            "Inscription gratuite : https://www.alphavantage.co/support/#api-key"
+        )
+
+    from_currency = pair[:3]
+    to_currency = pair[3:]
+
+    # Alpha Vantage FX intervals
+    _interval_map = {
+        "1m": "1min", "5m": "5min", "15m": "15min", "30m": "30min", "1h": "60min",
+    }
+
+    candles = []
+
+    if interval == "1d":
+        # FX_DAILY endpoint
+        url = "https://www.alphavantage.co/query"
+        params = {
+            "function": "FX_DAILY",
+            "from_symbol": from_currency,
+            "to_symbol": to_currency,
+            "outputsize": "full",
+            "apikey": api_key,
+        }
+        resp = requests.get(url, params=params, timeout=30)
+        data = resp.json()
+        ts_key = "Time Series FX (Daily)"
+        if ts_key not in data:
+            raise ValueError(f"Alpha Vantage error: {data}")
+        for date_str, bar in sorted(data[ts_key].items()):
+            from datetime import datetime
+            dt = datetime.strptime(date_str, "%Y-%m-%d")
+            candles.append({
+                "t": int(dt.timestamp() * 1000),
+                "o": float(bar["1. open"]),
+                "h": float(bar["2. high"]),
+                "l": float(bar["3. low"]),
+                "c": float(bar["4. close"]),
+                "v": 0.0,
+            })
+    else:
+        av_interval = _interval_map.get(interval, "60min")
+        url = "https://www.alphavantage.co/query"
+        params = {
+            "function": "FX_INTRADAY",
+            "from_symbol": from_currency,
+            "to_symbol": to_currency,
+            "interval": av_interval,
+            "outputsize": "full",
+            "apikey": api_key,
+        }
+        resp = requests.get(url, params=params, timeout=30)
+        data = resp.json()
+        ts_key = f"Time Series FX ({av_interval})"
+        if ts_key not in data:
+            raise ValueError(f"Alpha Vantage error: {data}")
+        from datetime import datetime
+        for dt_str, bar in sorted(data[ts_key].items()):
+            dt = datetime.strptime(dt_str, "%Y-%m-%d %H:%M:%S")
+            candles.append({
+                "t": int(dt.timestamp() * 1000),
+                "o": float(bar["1. open"]),
+                "h": float(bar["2. high"]),
+                "l": float(bar["3. low"]),
+                "c": float(bar["4. close"]),
+                "v": 0.0,
+            })
+
+    candles.sort(key=lambda x: x["t"])
+    return candles
+
+
+# ─── Unified getter (DATA_SOURCE agnostic) ────────────────────────────────────
+
+def get_candles(pair: str, interval: str = "1h",
+                days: int = DAYS_BACK) -> list[dict]:
+    """
+    Download candle data for a Forex pair.
+
+    La source est contrôlée par DATA_SOURCE dans .env (indépendant du broker) :
+      auto         → OANDA si OANDA_API_KEY défini, sinon yfinance
+      oanda        → OANDA v20 REST API (OANDA_API_KEY requis)
+      yfinance     → Yahoo Finance (gratuit, 2 ans max)
+      twelvedata   → Twelve Data (800 req/j gratuit, clé recommandée)
+      alphavantage → Alpha Vantage (25 req/j, ALPHA_VANTAGE_API_KEY requis)
+      mt5          → MetaTrader 5 Python SDK (Windows uniquement)
+      ccxt         → via broker CCXT_BROKER
+    """
+    source = getattr(config, "DATA_SOURCE", "auto").lower()
+
+    if source == "oanda":
+        return get_candles_oanda(pair, interval, days)
+    elif source == "yfinance":
+        return get_candles_yfinance(pair, interval, days)
+    elif source == "twelvedata":
+        return get_candles_twelvedata(pair, interval, days)
+    elif source == "alphavantage":
+        return get_candles_alphavantage(pair, interval, days)
+    elif source in ("mt5", "metatrader5"):
+        # Données directement depuis MT5 Python SDK
+        try:
+            import MetaTrader5 as mt5
+            tf_map = {"1m": 1, "5m": 5, "15m": 15, "30m": 30,
+                      "1h": 16385, "4h": 16388, "1d": 16408}
+            tf = tf_map.get(interval, 16385)
+            rates = mt5.copy_rates_from_pos(pair, tf, 0, days * 24)
+            if rates is None:
+                raise ValueError(f"MT5: aucune donnée pour {pair}")
+            return [
+                {"t": int(r["time"]) * 1000, "o": float(r["open"]),
+                 "h": float(r["high"]), "l": float(r["low"]),
+                 "c": float(r["close"]), "v": float(r.get("tick_volume", 0))}
+                for r in rates
+            ]
+        except ImportError:
+            raise ImportError("MetaTrader5 SDK requis. pip install MetaTrader5 (Windows uniquement)")
+    elif source == "ccxt":
+        # Données via ccxt broker
+        try:
+            import ccxt
+            broker = getattr(config, "CCXT_BROKER", "")
+            if not broker:
+                raise ValueError("CCXT_BROKER doit être défini dans .env (ex: 'ig')")
+            cls = getattr(ccxt, broker)
+            client = cls({"apiKey": config.CCXT_API_KEY, "secret": config.CCXT_API_SECRET,
+                          "enableRateLimit": True})
+            ohlcv = client.fetch_ohlcv(f"{pair[:3]}/{pair[3:]}", interval, limit=min(days * 24, 1000))
+            return [{"t": c[0], "o": c[1], "h": c[2], "l": c[3], "c": c[4], "v": c[5]}
+                    for c in ohlcv]
+        except Exception as e:
+            raise ValueError(f"CCXT data error: {e}") from e
+    else:
+        # auto : OANDA si clé disponible, sinon yfinance
+        if config.OANDA_API_KEY:
+            return get_candles_oanda(pair, interval, days)
+        elif getattr(config, "TWELVE_DATA_API_KEY", ""):
+            return get_candles_twelvedata(pair, interval, days)
+        else:
+            return get_candles_yfinance(pair, interval, days)
+
+
+
+# ─── Main download loop ───────────────────────────────────────────────────────
+
+def download_all() -> None:
+    """Download candle data for all configured Forex pairs and save to data/."""
+    os.makedirs(config.DATA_DIR, exist_ok=True)
+
+    # Détermination de la source affichée
+    _src = getattr(config, "DATA_SOURCE", "auto").lower()
+    if _src == "auto":
+        if config.OANDA_API_KEY:
+            source_label = "OANDA v20 (auto-sélectionné)"
+        elif getattr(config, "TWELVE_DATA_API_KEY", ""):
+            source_label = "Twelve Data (auto-sélectionné)"
+        else:
+            source_label = "Yahoo Finance (auto-sélectionné, gratuit)"
+    elif _src == "oanda":
+        source_label = "OANDA v20"
+    elif _src == "yfinance":
+        source_label = "Yahoo Finance"
+    elif _src == "twelvedata":
+        source_label = "Twelve Data"
+    elif _src == "alphavantage":
+        source_label = "Alpha Vantage"
+    elif _src in ("mt5", "metatrader5"):
+        source_label = "MetaTrader 5 SDK"
+    elif _src == "ccxt":
+        source_label = f"CCXT ({getattr(config, 'CCXT_BROKER', '?')})"
+    else:
+        source_label = _src
+
+    print(f"Downloading {DAYS_BACK} days of {config.CANDLE_INTERVAL} candles "
+          f"for {len(config.PAIRS)} Forex pairs...")
+    print(f"Source  : {source_label}  (DATA_SOURCE={_src})")
+    print(f"Broker  : {config.EXCHANGE}  (pour l'exécution)")
+    print(f"Saving  : {config.DATA_DIR}/\n")
+
+    failed: list[str] = []
+
+    # Délai adaptatif selon la source (éviter le rate-limiting)
+    _src = getattr(config, "DATA_SOURCE", "auto").lower()
+    if _src in ("yfinance", "auto") and not config.OANDA_API_KEY:
+        _sleep = 1.5   # yfinance : 1.5s entre paires (rate limit discret)
+    elif _src == "alphavantage":
+        _sleep = 15.0  # Alpha Vantage free : 25 req/j → 4 req/min max
+    elif _src == "twelvedata":
+        _sleep = 8.0 if not getattr(config, "TWELVE_DATA_API_KEY", "") else 0.8
+        # Sans clé : ~8 req/min | Avec clé plan free : 800 req/j → ~1 req/s
+    elif _src == "oanda":
+        _sleep = 0.3   # OANDA : API rapide avec clé
+    else:
+        _sleep = 1.0   # défaut raisonnable
+
+    print(f"  (délai entre paires : {_sleep}s pour éviter le rate-limiting)\n")
+
+    for i, pair in enumerate(config.PAIRS, 1):
+        try:
+            candles = get_candles(pair, config.CANDLE_INTERVAL, DAYS_BACK)
+            path = os.path.join(config.DATA_DIR, f"{pair}_1h.json")
+            with open(path, "w", encoding="utf-8") as f:
+                json.dump(candles, f)
+            days_actual = round(len(candles) / 24)
+            status = "OK" if days_actual >= 600 else "[données limitées]"
+            print(f"  [{i:2d}/{len(config.PAIRS)}] {pair:8s} -- "
+                  f"{len(candles):,} candles (~{days_actual} jours) {status}")
+        except Exception as e:
+            print(f"  [{i:2d}/{len(config.PAIRS)}] {pair:8s} -- ERREUR: {e}")
+            failed.append(pair)
+
+        time.sleep(_sleep)   # rate limit adaptatif selon DATA_SOURCE
+
+    # Retry failed pairs
+    if failed:
+        print(f"\n[!] {len(failed)} paire(s) échouée(s) : {', '.join(failed)}")
+        print("Retry dans 10 secondes...\n")
+        time.sleep(10)
+
+        for pair in failed:
+            try:
+                candles = get_candles(pair, config.CANDLE_INTERVAL, DAYS_BACK)
+                path = os.path.join(config.DATA_DIR, f"{pair}_1h.json")
+                with open(path, "w", encoding="utf-8") as f:
+                    json.dump(candles, f)
+                days_actual = round(len(candles) / 24)
+                print(f"  [OK] {pair:8s} -- {len(candles):,} candles récupérés!")
+            except Exception as e:
+                print(f"  [FAIL] {pair:8s} -- Échec définitif: {e}")
+
+    print("\n[DONE] Données sauvegardées dans data/")
+
+
+if __name__ == "__main__":
+    download_all()