""" AHAD QUANT — Data Downloader (Forex Edition) Downloads 1h candle data from multiple sources for Forex pairs. Sources (by priority): 1. OANDA v20 REST API — if OANDA_API_KEY is set (recommended) 2. Yahoo Finance — free, no key required (yfinance) Usage: python download_data.py """ import json import os import sys import time import requests # ── Fix encodage Windows ────────────────────────────────────────────────────── if hasattr(sys.stdout, 'reconfigure'): try: sys.stdout.reconfigure(encoding='utf-8', errors='replace') except Exception: pass import config # ── Constants ───────────────────────────────────────────────────────────────── DAYS_BACK = 730 # yfinance 1h limit (2 years) MAX_RETRIES = 5 RETRY_DELAY = 3 # OANDA candle granularities map OANDA_GRANULARITY = { "1m": "M1", "5m": "M5", "15m": "M15", "30m": "M30", "1h": "H1", "4h": "H4", "1d": "D", } # Yahoo Finance Forex suffix and ticker format # EUR/USD → "EURUSD=X" YAHOO_SUFFIX = "=X" # ─── OANDA source ───────────────────────────────────────────────────────────── def _oanda_to_instrument(pair: str) -> str: """EURUSD → EUR_USD (OANDA format).""" return f"{pair[:3]}_{pair[3:]}" def get_candles_oanda(pair: str, interval: str = "1h", days: int = DAYS_BACK) -> list[dict]: """ Fetch historical candles from OANDA v20 API. Requires OANDA_API_KEY and OANDA_ACCOUNT_ID in environment. """ api_key = config.OANDA_API_KEY practice = config.OANDA_PRACTICE base_url = ( "https://api-fxpractice.oanda.com" if practice else "https://api-fxtrade.oanda.com" ) instrument = _oanda_to_instrument(pair) granularity = OANDA_GRANULARITY.get(interval, "H1") # OANDA max per request = 5000 candles candles_per_req = 5000 end_ts = int(time.time()) start_ts = end_ts - (days * 24 * 3600) headers = { "Authorization": f"Bearer {api_key}", "Content-Type": "application/json", } all_candles: list[dict] = [] seen: set = set() current_start = start_ts interval_seconds = {"M1": 60, "M5": 300, "M15": 900, "M30": 1800, "H1": 3600, "H4": 14400, "D": 86400}.get(granularity, 3600) while current_start < end_ts: url = ( f"{base_url}/v3/instruments/{instrument}/candles" f"?granularity={granularity}" f"&from={current_start}" f"&to={min(current_start + candles_per_req * interval_seconds, end_ts)}" f"&price=M" # midpoint (bid+ask)/2 ) for attempt in range(1, MAX_RETRIES + 1): try: resp = requests.get(url, headers=headers, timeout=30) resp.raise_for_status() data = resp.json() break except Exception as e: if attempt < MAX_RETRIES: time.sleep(RETRY_DELAY * attempt) else: raise e raw = data.get("candles", []) if not raw: break for c in raw: if not c.get("complete", True): continue t = int(c["time"].split(".")[0]) if "." in c["time"] else int(c["time"][:10]) if t not in seen: seen.add(t) mid = c["mid"] all_candles.append({ "t": t * 1000, # milliseconds (matches Binance format) "o": float(mid["o"]), "h": float(mid["h"]), "l": float(mid["l"]), "c": float(mid["c"]), "v": float(c.get("volume", 0)), }) last_t = int(raw[-1]["time"][:10]) if last_t <= current_start: break current_start = last_t + interval_seconds if len(raw) < candles_per_req: break time.sleep(0.1) all_candles.sort(key=lambda x: x["t"]) return all_candles # ─── Yahoo Finance source ───────────────────────────────────────────────────── def get_candles_yfinance(pair: str, interval: str = "1h", days: int = DAYS_BACK) -> list[dict]: """ Fetch historical candles from Yahoo Finance (free, no key required). 1h data available up to 730 days back. """ try: import yfinance as yf except ImportError: raise ImportError( "yfinance is required for free data download.\n" "Install with: pip install yfinance" ) ticker_sym = f"{pair}{YAHOO_SUFFIX}" # yfinance 1h: max period "730d" period_map = {"1m": "7d", "5m": "60d", "15m": "60d", "30m": "60d", "1h": "730d", "4h": "730d", "1d": "max"} period = period_map.get(interval, "730d") # yfinance interval notation yf_interval_map = {"1h": "1h", "4h": "1h", "1d": "1d", "1m": "1m", "5m": "5m", "15m": "15m"} yf_interval = yf_interval_map.get(interval, "1h") for attempt in range(1, MAX_RETRIES + 1): try: ticker = yf.Ticker(ticker_sym) df = ticker.history(period=period, interval=yf_interval, auto_adjust=True, prepost=False) break except Exception as e: if attempt < MAX_RETRIES: time.sleep(RETRY_DELAY * attempt) else: raise e if df is None or df.empty: raise ValueError(f"No data returned from Yahoo Finance for {pair}") # If 4h was requested but yfinance only gives 1h, resample if interval == "4h" and yf_interval == "1h": df = df.resample("4h").agg({ "Open": "first", "High": "max", "Low": "min", "Close": "last", "Volume": "sum", }).dropna() candles: list[dict] = [] for ts, row in df.iterrows(): # Normalize timestamp to milliseconds if hasattr(ts, "timestamp"): t_ms = int(ts.timestamp() * 1000) else: t_ms = int(ts) * 1000 candles.append({ "t": t_ms, "o": float(row["Open"]), "h": float(row["High"]), "l": float(row["Low"]), "c": float(row["Close"]), "v": float(row.get("Volume", 0)), }) candles.sort(key=lambda x: x["t"]) return candles # ─── Twelve Data source ─────────────────────────────────────────────────────── def get_candles_twelvedata(pair: str, interval: str = "1h", days: int = DAYS_BACK) -> list[dict]: """ Fetch historical Forex candles from Twelve Data API. Plan Free : 800 req/jour, données Forex 1h sans clé (rate-limited). Inscription gratuite : https://twelvedata.com/ """ api_key = config.TWELVE_DATA_API_KEY # Twelve Data interval notation _interval_map = { "1m": "1min", "5m": "5min", "15m": "15min", "30m": "30min", "1h": "1h", "4h": "4h", "1d": "1day", } td_interval = _interval_map.get(interval, "1h") # Format paire : EURUSD → EUR/USD symbol = f"{pair[:3]}/{pair[3:]}" # Nombre de points à récupérer (max 5000 par requête) outputsize = min(days * 24, 5000) base_url = "https://api.twelvedata.com/time_series" params: dict = { "symbol": symbol, "interval": td_interval, "outputsize": outputsize, "format": "JSON", "timezone": "UTC", } if api_key: params["apikey"] = api_key from datetime import datetime as _dt # import unique, hors boucle for attempt in range(1, MAX_RETRIES + 1): try: resp = requests.get(base_url, params=params, timeout=30) resp.raise_for_status() data = resp.json() if data.get("status") == "error" or "values" not in data: raise ValueError(f"Twelve Data error: {data.get('message', data)}") values = data["values"] candles = [] for bar in reversed(values): # API retourne du plus récent au plus ancien dt = _dt.strptime(bar["datetime"], "%Y-%m-%d %H:%M:%S") candles.append({ "t": int(dt.timestamp() * 1000), "o": float(bar["open"]), "h": float(bar["high"]), "l": float(bar["low"]), "c": float(bar["close"]), "v": float(bar.get("volume", 0)), }) candles.sort(key=lambda x: x["t"]) return candles except Exception as e: if attempt < MAX_RETRIES: time.sleep(RETRY_DELAY * attempt) else: raise ValueError(f"Twelve Data — {pair}: {e}") from e return [] # ─── Alpha Vantage source ───────────────────────────────────────────────────── def get_candles_alphavantage(pair: str, interval: str = "1h", days: int = DAYS_BACK) -> list[dict]: """ Fetch Forex candles from Alpha Vantage. Plan Free : 25 req/jour. ALPHA_VANTAGE_API_KEY requis. Inscription : https://www.alphavantage.co/support/#api-key """ api_key = config.ALPHA_VANTAGE_API_KEY if not api_key: raise ValueError( "ALPHA_VANTAGE_API_KEY est requis pour cette source.\n" "Inscription gratuite : https://www.alphavantage.co/support/#api-key" ) from_currency = pair[:3] to_currency = pair[3:] # Alpha Vantage FX intervals _interval_map = { "1m": "1min", "5m": "5min", "15m": "15min", "30m": "30min", "1h": "60min", } candles = [] if interval == "1d": # FX_DAILY endpoint url = "https://www.alphavantage.co/query" params = { "function": "FX_DAILY", "from_symbol": from_currency, "to_symbol": to_currency, "outputsize": "full", "apikey": api_key, } resp = requests.get(url, params=params, timeout=30) data = resp.json() ts_key = "Time Series FX (Daily)" if ts_key not in data: raise ValueError(f"Alpha Vantage error: {data}") for date_str, bar in sorted(data[ts_key].items()): from datetime import datetime dt = datetime.strptime(date_str, "%Y-%m-%d") candles.append({ "t": int(dt.timestamp() * 1000), "o": float(bar["1. open"]), "h": float(bar["2. high"]), "l": float(bar["3. low"]), "c": float(bar["4. close"]), "v": 0.0, }) else: av_interval = _interval_map.get(interval, "60min") url = "https://www.alphavantage.co/query" params = { "function": "FX_INTRADAY", "from_symbol": from_currency, "to_symbol": to_currency, "interval": av_interval, "outputsize": "full", "apikey": api_key, } resp = requests.get(url, params=params, timeout=30) data = resp.json() ts_key = f"Time Series FX ({av_interval})" if ts_key not in data: raise ValueError(f"Alpha Vantage error: {data}") from datetime import datetime for dt_str, bar in sorted(data[ts_key].items()): dt = datetime.strptime(dt_str, "%Y-%m-%d %H:%M:%S") candles.append({ "t": int(dt.timestamp() * 1000), "o": float(bar["1. open"]), "h": float(bar["2. high"]), "l": float(bar["3. low"]), "c": float(bar["4. close"]), "v": 0.0, }) candles.sort(key=lambda x: x["t"]) return candles # ─── Unified getter (DATA_SOURCE agnostic) ──────────────────────────────────── def get_candles(pair: str, interval: str = "1h", days: int = DAYS_BACK) -> list[dict]: """ Download candle data for a Forex pair. La source est contrôlée par DATA_SOURCE dans .env (indépendant du broker) : auto → OANDA si OANDA_API_KEY défini, sinon yfinance oanda → OANDA v20 REST API (OANDA_API_KEY requis) yfinance → Yahoo Finance (gratuit, 2 ans max) twelvedata → Twelve Data (800 req/j gratuit, clé recommandée) alphavantage → Alpha Vantage (25 req/j, ALPHA_VANTAGE_API_KEY requis) mt5 → MetaTrader 5 Python SDK (Windows uniquement) ccxt → via broker CCXT_BROKER """ source = getattr(config, "DATA_SOURCE", "auto").lower() if source == "oanda": return get_candles_oanda(pair, interval, days) elif source == "yfinance": return get_candles_yfinance(pair, interval, days) elif source == "twelvedata": return get_candles_twelvedata(pair, interval, days) elif source == "alphavantage": return get_candles_alphavantage(pair, interval, days) elif source in ("mt5", "metatrader5"): # Données directement depuis MT5 Python SDK try: import MetaTrader5 as mt5 tf_map = {"1m": 1, "5m": 5, "15m": 15, "30m": 30, "1h": 16385, "4h": 16388, "1d": 16408} tf = tf_map.get(interval, 16385) rates = mt5.copy_rates_from_pos(pair, tf, 0, days * 24) if rates is None: raise ValueError(f"MT5: aucune donnée pour {pair}") return [ {"t": int(r["time"]) * 1000, "o": float(r["open"]), "h": float(r["high"]), "l": float(r["low"]), "c": float(r["close"]), "v": float(r.get("tick_volume", 0))} for r in rates ] except ImportError: raise ImportError("MetaTrader5 SDK requis. pip install MetaTrader5 (Windows uniquement)") elif source == "ccxt": # Données via ccxt broker try: import ccxt broker = getattr(config, "CCXT_BROKER", "") if not broker: raise ValueError("CCXT_BROKER doit être défini dans .env (ex: 'ig')") cls = getattr(ccxt, broker) client = cls({"apiKey": config.CCXT_API_KEY, "secret": config.CCXT_API_SECRET, "enableRateLimit": True}) ohlcv = client.fetch_ohlcv(f"{pair[:3]}/{pair[3:]}", interval, limit=min(days * 24, 1000)) return [{"t": c[0], "o": c[1], "h": c[2], "l": c[3], "c": c[4], "v": c[5]} for c in ohlcv] except Exception as e: raise ValueError(f"CCXT data error: {e}") from e else: # auto : OANDA si clé disponible, sinon yfinance if config.OANDA_API_KEY: return get_candles_oanda(pair, interval, days) elif getattr(config, "TWELVE_DATA_API_KEY", ""): return get_candles_twelvedata(pair, interval, days) else: return get_candles_yfinance(pair, interval, days) # ─── Main download loop ─────────────────────────────────────────────────────── def download_all() -> None: """Download candle data for all configured Forex pairs and save to data/.""" os.makedirs(config.DATA_DIR, exist_ok=True) # Détermination de la source affichée _src = getattr(config, "DATA_SOURCE", "auto").lower() if _src == "auto": if config.OANDA_API_KEY: source_label = "OANDA v20 (auto-sélectionné)" elif getattr(config, "TWELVE_DATA_API_KEY", ""): source_label = "Twelve Data (auto-sélectionné)" else: source_label = "Yahoo Finance (auto-sélectionné, gratuit)" elif _src == "oanda": source_label = "OANDA v20" elif _src == "yfinance": source_label = "Yahoo Finance" elif _src == "twelvedata": source_label = "Twelve Data" elif _src == "alphavantage": source_label = "Alpha Vantage" elif _src in ("mt5", "metatrader5"): source_label = "MetaTrader 5 SDK" elif _src == "ccxt": source_label = f"CCXT ({getattr(config, 'CCXT_BROKER', '?')})" else: source_label = _src print(f"Downloading {DAYS_BACK} days of {config.CANDLE_INTERVAL} candles " f"for {len(config.PAIRS)} Forex pairs...") print(f"Source : {source_label} (DATA_SOURCE={_src})") print(f"Broker : {config.EXCHANGE} (pour l'exécution)") print(f"Saving : {config.DATA_DIR}/\n") failed: list[str] = [] # Délai adaptatif selon la source (éviter le rate-limiting) _src = getattr(config, "DATA_SOURCE", "auto").lower() if _src in ("yfinance", "auto") and not config.OANDA_API_KEY: _sleep = 1.5 # yfinance : 1.5s entre paires (rate limit discret) elif _src == "alphavantage": _sleep = 15.0 # Alpha Vantage free : 25 req/j → 4 req/min max elif _src == "twelvedata": _sleep = 8.0 if not getattr(config, "TWELVE_DATA_API_KEY", "") else 0.8 # Sans clé : ~8 req/min | Avec clé plan free : 800 req/j → ~1 req/s elif _src == "oanda": _sleep = 0.3 # OANDA : API rapide avec clé else: _sleep = 1.0 # défaut raisonnable print(f" (délai entre paires : {_sleep}s pour éviter le rate-limiting)\n") for i, pair in enumerate(config.PAIRS, 1): try: candles = get_candles(pair, config.CANDLE_INTERVAL, DAYS_BACK) path = os.path.join(config.DATA_DIR, f"{pair}_1h.json") with open(path, "w", encoding="utf-8") as f: json.dump(candles, f) days_actual = round(len(candles) / 24) status = "OK" if days_actual >= 600 else "[données limitées]" print(f" [{i:2d}/{len(config.PAIRS)}] {pair:8s} -- " f"{len(candles):,} candles (~{days_actual} jours) {status}") except Exception as e: print(f" [{i:2d}/{len(config.PAIRS)}] {pair:8s} -- ERREUR: {e}") failed.append(pair) time.sleep(_sleep) # rate limit adaptatif selon DATA_SOURCE # Retry failed pairs if failed: print(f"\n[!] {len(failed)} paire(s) échouée(s) : {', '.join(failed)}") print("Retry dans 10 secondes...\n") time.sleep(10) for pair in failed: try: candles = get_candles(pair, config.CANDLE_INTERVAL, DAYS_BACK) path = os.path.join(config.DATA_DIR, f"{pair}_1h.json") with open(path, "w", encoding="utf-8") as f: json.dump(candles, f) days_actual = round(len(candles) / 24) print(f" [OK] {pair:8s} -- {len(candles):,} candles récupérés!") except Exception as e: print(f" [FAIL] {pair:8s} -- Échec définitif: {e}") print("\n[DONE] Données sauvegardées dans data/") if __name__ == "__main__": download_all()