Abdoul Ahad Binizi
524 lines
19 KiB
Python
524 lines
19 KiB
Python
"""
|
|
AHAD QUANT — Data Downloader (Forex Edition)
|
|
Downloads 1h candle data from multiple sources for Forex pairs.
|
|
|
|
Sources (by priority):
|
|
1. OANDA v20 REST API — if OANDA_API_KEY is set (recommended)
|
|
2. Yahoo Finance — free, no key required (yfinance)
|
|
|
|
Usage:
|
|
python download_data.py
|
|
|
|
"""
|
|
|
|
import json
|
|
import os
|
|
import sys
|
|
import time
|
|
import requests
|
|
|
|
# ── Fix encodage Windows ──────────────────────────────────────────────────────
|
|
if hasattr(sys.stdout, 'reconfigure'):
|
|
try:
|
|
sys.stdout.reconfigure(encoding='utf-8', errors='replace')
|
|
except Exception:
|
|
pass
|
|
|
|
import config
|
|
|
|
# ── Constants ─────────────────────────────────────────────────────────────────
|
|
DAYS_BACK = 730 # yfinance 1h limit (2 years)
|
|
MAX_RETRIES = 5
|
|
RETRY_DELAY = 3
|
|
|
|
# OANDA candle granularities map
|
|
OANDA_GRANULARITY = {
|
|
"1m": "M1", "5m": "M5", "15m": "M15", "30m": "M30",
|
|
"1h": "H1", "4h": "H4", "1d": "D",
|
|
}
|
|
|
|
# Yahoo Finance Forex suffix and ticker format
|
|
# EUR/USD → "EURUSD=X"
|
|
YAHOO_SUFFIX = "=X"
|
|
|
|
|
|
# ─── OANDA source ─────────────────────────────────────────────────────────────
|
|
|
|
def _oanda_to_instrument(pair: str) -> str:
|
|
"""EURUSD → EUR_USD (OANDA format)."""
|
|
return f"{pair[:3]}_{pair[3:]}"
|
|
|
|
|
|
def get_candles_oanda(pair: str, interval: str = "1h",
|
|
days: int = DAYS_BACK) -> list[dict]:
|
|
"""
|
|
Fetch historical candles from OANDA v20 API.
|
|
Requires OANDA_API_KEY and OANDA_ACCOUNT_ID in environment.
|
|
"""
|
|
api_key = config.OANDA_API_KEY
|
|
practice = config.OANDA_PRACTICE
|
|
|
|
base_url = (
|
|
"https://api-fxpractice.oanda.com"
|
|
if practice else
|
|
"https://api-fxtrade.oanda.com"
|
|
)
|
|
|
|
instrument = _oanda_to_instrument(pair)
|
|
granularity = OANDA_GRANULARITY.get(interval, "H1")
|
|
|
|
# OANDA max per request = 5000 candles
|
|
candles_per_req = 5000
|
|
end_ts = int(time.time())
|
|
start_ts = end_ts - (days * 24 * 3600)
|
|
|
|
headers = {
|
|
"Authorization": f"Bearer {api_key}",
|
|
"Content-Type": "application/json",
|
|
}
|
|
|
|
all_candles: list[dict] = []
|
|
seen: set = set()
|
|
current_start = start_ts
|
|
|
|
interval_seconds = {"M1": 60, "M5": 300, "M15": 900, "M30": 1800,
|
|
"H1": 3600, "H4": 14400, "D": 86400}.get(granularity, 3600)
|
|
|
|
while current_start < end_ts:
|
|
url = (
|
|
f"{base_url}/v3/instruments/{instrument}/candles"
|
|
f"?granularity={granularity}"
|
|
f"&from={current_start}"
|
|
f"&to={min(current_start + candles_per_req * interval_seconds, end_ts)}"
|
|
f"&price=M" # midpoint (bid+ask)/2
|
|
)
|
|
for attempt in range(1, MAX_RETRIES + 1):
|
|
try:
|
|
resp = requests.get(url, headers=headers, timeout=30)
|
|
resp.raise_for_status()
|
|
data = resp.json()
|
|
break
|
|
except Exception as e:
|
|
if attempt < MAX_RETRIES:
|
|
time.sleep(RETRY_DELAY * attempt)
|
|
else:
|
|
raise e
|
|
|
|
raw = data.get("candles", [])
|
|
if not raw:
|
|
break
|
|
|
|
for c in raw:
|
|
if not c.get("complete", True):
|
|
continue
|
|
t = int(c["time"].split(".")[0]) if "." in c["time"] else int(c["time"][:10])
|
|
if t not in seen:
|
|
seen.add(t)
|
|
mid = c["mid"]
|
|
all_candles.append({
|
|
"t": t * 1000, # milliseconds (matches Binance format)
|
|
"o": float(mid["o"]),
|
|
"h": float(mid["h"]),
|
|
"l": float(mid["l"]),
|
|
"c": float(mid["c"]),
|
|
"v": float(c.get("volume", 0)),
|
|
})
|
|
|
|
last_t = int(raw[-1]["time"][:10])
|
|
if last_t <= current_start:
|
|
break
|
|
current_start = last_t + interval_seconds
|
|
|
|
if len(raw) < candles_per_req:
|
|
break
|
|
|
|
time.sleep(0.1)
|
|
|
|
all_candles.sort(key=lambda x: x["t"])
|
|
return all_candles
|
|
|
|
|
|
# ─── Yahoo Finance source ─────────────────────────────────────────────────────
|
|
|
|
def get_candles_yfinance(pair: str, interval: str = "1h",
|
|
days: int = DAYS_BACK) -> list[dict]:
|
|
"""
|
|
Fetch historical candles from Yahoo Finance (free, no key required).
|
|
1h data available up to 730 days back.
|
|
"""
|
|
try:
|
|
import yfinance as yf
|
|
except ImportError:
|
|
raise ImportError(
|
|
"yfinance is required for free data download.\n"
|
|
"Install with: pip install yfinance"
|
|
)
|
|
|
|
ticker_sym = f"{pair}{YAHOO_SUFFIX}"
|
|
|
|
# yfinance 1h: max period "730d"
|
|
period_map = {"1m": "7d", "5m": "60d", "15m": "60d", "30m": "60d",
|
|
"1h": "730d", "4h": "730d", "1d": "max"}
|
|
period = period_map.get(interval, "730d")
|
|
|
|
# yfinance interval notation
|
|
yf_interval_map = {"1h": "1h", "4h": "1h", "1d": "1d",
|
|
"1m": "1m", "5m": "5m", "15m": "15m"}
|
|
yf_interval = yf_interval_map.get(interval, "1h")
|
|
|
|
for attempt in range(1, MAX_RETRIES + 1):
|
|
try:
|
|
ticker = yf.Ticker(ticker_sym)
|
|
df = ticker.history(period=period, interval=yf_interval,
|
|
auto_adjust=True, prepost=False)
|
|
break
|
|
except Exception as e:
|
|
if attempt < MAX_RETRIES:
|
|
time.sleep(RETRY_DELAY * attempt)
|
|
else:
|
|
raise e
|
|
|
|
if df is None or df.empty:
|
|
raise ValueError(f"No data returned from Yahoo Finance for {pair}")
|
|
|
|
# If 4h was requested but yfinance only gives 1h, resample
|
|
if interval == "4h" and yf_interval == "1h":
|
|
df = df.resample("4h").agg({
|
|
"Open": "first", "High": "max", "Low": "min",
|
|
"Close": "last", "Volume": "sum",
|
|
}).dropna()
|
|
|
|
candles: list[dict] = []
|
|
for ts, row in df.iterrows():
|
|
# Normalize timestamp to milliseconds
|
|
if hasattr(ts, "timestamp"):
|
|
t_ms = int(ts.timestamp() * 1000)
|
|
else:
|
|
t_ms = int(ts) * 1000
|
|
|
|
candles.append({
|
|
"t": t_ms,
|
|
"o": float(row["Open"]),
|
|
"h": float(row["High"]),
|
|
"l": float(row["Low"]),
|
|
"c": float(row["Close"]),
|
|
"v": float(row.get("Volume", 0)),
|
|
})
|
|
|
|
candles.sort(key=lambda x: x["t"])
|
|
return candles
|
|
|
|
|
|
|
|
# ─── Twelve Data source ───────────────────────────────────────────────────────
|
|
|
|
def get_candles_twelvedata(pair: str, interval: str = "1h",
|
|
days: int = DAYS_BACK) -> list[dict]:
|
|
"""
|
|
Fetch historical Forex candles from Twelve Data API.
|
|
Plan Free : 800 req/jour, données Forex 1h sans clé (rate-limited).
|
|
Inscription gratuite : https://twelvedata.com/
|
|
"""
|
|
api_key = config.TWELVE_DATA_API_KEY
|
|
# Twelve Data interval notation
|
|
_interval_map = {
|
|
"1m": "1min", "5m": "5min", "15m": "15min", "30m": "30min",
|
|
"1h": "1h", "4h": "4h", "1d": "1day",
|
|
}
|
|
td_interval = _interval_map.get(interval, "1h")
|
|
|
|
# Format paire : EURUSD → EUR/USD
|
|
symbol = f"{pair[:3]}/{pair[3:]}"
|
|
|
|
# Nombre de points à récupérer (max 5000 par requête)
|
|
outputsize = min(days * 24, 5000)
|
|
|
|
base_url = "https://api.twelvedata.com/time_series"
|
|
params: dict = {
|
|
"symbol": symbol,
|
|
"interval": td_interval,
|
|
"outputsize": outputsize,
|
|
"format": "JSON",
|
|
"timezone": "UTC",
|
|
}
|
|
if api_key:
|
|
params["apikey"] = api_key
|
|
|
|
from datetime import datetime as _dt # import unique, hors boucle
|
|
|
|
for attempt in range(1, MAX_RETRIES + 1):
|
|
try:
|
|
resp = requests.get(base_url, params=params, timeout=30)
|
|
resp.raise_for_status()
|
|
data = resp.json()
|
|
if data.get("status") == "error" or "values" not in data:
|
|
raise ValueError(f"Twelve Data error: {data.get('message', data)}")
|
|
values = data["values"]
|
|
candles = []
|
|
for bar in reversed(values): # API retourne du plus récent au plus ancien
|
|
dt = _dt.strptime(bar["datetime"], "%Y-%m-%d %H:%M:%S")
|
|
candles.append({
|
|
"t": int(dt.timestamp() * 1000),
|
|
"o": float(bar["open"]),
|
|
"h": float(bar["high"]),
|
|
"l": float(bar["low"]),
|
|
"c": float(bar["close"]),
|
|
"v": float(bar.get("volume", 0)),
|
|
})
|
|
candles.sort(key=lambda x: x["t"])
|
|
return candles
|
|
except Exception as e:
|
|
if attempt < MAX_RETRIES:
|
|
time.sleep(RETRY_DELAY * attempt)
|
|
else:
|
|
raise ValueError(f"Twelve Data — {pair}: {e}") from e
|
|
return []
|
|
|
|
|
|
# ─── Alpha Vantage source ─────────────────────────────────────────────────────
|
|
|
|
def get_candles_alphavantage(pair: str, interval: str = "1h",
|
|
days: int = DAYS_BACK) -> list[dict]:
|
|
"""
|
|
Fetch Forex candles from Alpha Vantage.
|
|
Plan Free : 25 req/jour. ALPHA_VANTAGE_API_KEY requis.
|
|
Inscription : https://www.alphavantage.co/support/#api-key
|
|
"""
|
|
api_key = config.ALPHA_VANTAGE_API_KEY
|
|
if not api_key:
|
|
raise ValueError(
|
|
"ALPHA_VANTAGE_API_KEY est requis pour cette source.\n"
|
|
"Inscription gratuite : https://www.alphavantage.co/support/#api-key"
|
|
)
|
|
|
|
from_currency = pair[:3]
|
|
to_currency = pair[3:]
|
|
|
|
# Alpha Vantage FX intervals
|
|
_interval_map = {
|
|
"1m": "1min", "5m": "5min", "15m": "15min", "30m": "30min", "1h": "60min",
|
|
}
|
|
|
|
candles = []
|
|
|
|
if interval == "1d":
|
|
# FX_DAILY endpoint
|
|
url = "https://www.alphavantage.co/query"
|
|
params = {
|
|
"function": "FX_DAILY",
|
|
"from_symbol": from_currency,
|
|
"to_symbol": to_currency,
|
|
"outputsize": "full",
|
|
"apikey": api_key,
|
|
}
|
|
resp = requests.get(url, params=params, timeout=30)
|
|
data = resp.json()
|
|
ts_key = "Time Series FX (Daily)"
|
|
if ts_key not in data:
|
|
raise ValueError(f"Alpha Vantage error: {data}")
|
|
for date_str, bar in sorted(data[ts_key].items()):
|
|
from datetime import datetime
|
|
dt = datetime.strptime(date_str, "%Y-%m-%d")
|
|
candles.append({
|
|
"t": int(dt.timestamp() * 1000),
|
|
"o": float(bar["1. open"]),
|
|
"h": float(bar["2. high"]),
|
|
"l": float(bar["3. low"]),
|
|
"c": float(bar["4. close"]),
|
|
"v": 0.0,
|
|
})
|
|
else:
|
|
av_interval = _interval_map.get(interval, "60min")
|
|
url = "https://www.alphavantage.co/query"
|
|
params = {
|
|
"function": "FX_INTRADAY",
|
|
"from_symbol": from_currency,
|
|
"to_symbol": to_currency,
|
|
"interval": av_interval,
|
|
"outputsize": "full",
|
|
"apikey": api_key,
|
|
}
|
|
resp = requests.get(url, params=params, timeout=30)
|
|
data = resp.json()
|
|
ts_key = f"Time Series FX ({av_interval})"
|
|
if ts_key not in data:
|
|
raise ValueError(f"Alpha Vantage error: {data}")
|
|
from datetime import datetime
|
|
for dt_str, bar in sorted(data[ts_key].items()):
|
|
dt = datetime.strptime(dt_str, "%Y-%m-%d %H:%M:%S")
|
|
candles.append({
|
|
"t": int(dt.timestamp() * 1000),
|
|
"o": float(bar["1. open"]),
|
|
"h": float(bar["2. high"]),
|
|
"l": float(bar["3. low"]),
|
|
"c": float(bar["4. close"]),
|
|
"v": 0.0,
|
|
})
|
|
|
|
candles.sort(key=lambda x: x["t"])
|
|
return candles
|
|
|
|
|
|
# ─── Unified getter (DATA_SOURCE agnostic) ────────────────────────────────────
|
|
|
|
def get_candles(pair: str, interval: str = "1h",
|
|
days: int = DAYS_BACK) -> list[dict]:
|
|
"""
|
|
Download candle data for a Forex pair.
|
|
|
|
La source est contrôlée par DATA_SOURCE dans .env (indépendant du broker) :
|
|
auto → OANDA si OANDA_API_KEY défini, sinon yfinance
|
|
oanda → OANDA v20 REST API (OANDA_API_KEY requis)
|
|
yfinance → Yahoo Finance (gratuit, 2 ans max)
|
|
twelvedata → Twelve Data (800 req/j gratuit, clé recommandée)
|
|
alphavantage → Alpha Vantage (25 req/j, ALPHA_VANTAGE_API_KEY requis)
|
|
mt5 → MetaTrader 5 Python SDK (Windows uniquement)
|
|
ccxt → via broker CCXT_BROKER
|
|
"""
|
|
source = getattr(config, "DATA_SOURCE", "auto").lower()
|
|
|
|
if source == "oanda":
|
|
return get_candles_oanda(pair, interval, days)
|
|
elif source == "yfinance":
|
|
return get_candles_yfinance(pair, interval, days)
|
|
elif source == "twelvedata":
|
|
return get_candles_twelvedata(pair, interval, days)
|
|
elif source == "alphavantage":
|
|
return get_candles_alphavantage(pair, interval, days)
|
|
elif source in ("mt5", "metatrader5"):
|
|
# Données directement depuis MT5 Python SDK
|
|
try:
|
|
import MetaTrader5 as mt5
|
|
tf_map = {"1m": 1, "5m": 5, "15m": 15, "30m": 30,
|
|
"1h": 16385, "4h": 16388, "1d": 16408}
|
|
tf = tf_map.get(interval, 16385)
|
|
rates = mt5.copy_rates_from_pos(pair, tf, 0, days * 24)
|
|
if rates is None:
|
|
raise ValueError(f"MT5: aucune donnée pour {pair}")
|
|
return [
|
|
{"t": int(r["time"]) * 1000, "o": float(r["open"]),
|
|
"h": float(r["high"]), "l": float(r["low"]),
|
|
"c": float(r["close"]), "v": float(r.get("tick_volume", 0))}
|
|
for r in rates
|
|
]
|
|
except ImportError:
|
|
raise ImportError("MetaTrader5 SDK requis. pip install MetaTrader5 (Windows uniquement)")
|
|
elif source == "ccxt":
|
|
# Données via ccxt broker
|
|
try:
|
|
import ccxt
|
|
broker = getattr(config, "CCXT_BROKER", "")
|
|
if not broker:
|
|
raise ValueError("CCXT_BROKER doit être défini dans .env (ex: 'ig')")
|
|
cls = getattr(ccxt, broker)
|
|
client = cls({"apiKey": config.CCXT_API_KEY, "secret": config.CCXT_API_SECRET,
|
|
"enableRateLimit": True})
|
|
ohlcv = client.fetch_ohlcv(f"{pair[:3]}/{pair[3:]}", interval, limit=min(days * 24, 1000))
|
|
return [{"t": c[0], "o": c[1], "h": c[2], "l": c[3], "c": c[4], "v": c[5]}
|
|
for c in ohlcv]
|
|
except Exception as e:
|
|
raise ValueError(f"CCXT data error: {e}") from e
|
|
else:
|
|
# auto : OANDA si clé disponible, sinon yfinance
|
|
if config.OANDA_API_KEY:
|
|
return get_candles_oanda(pair, interval, days)
|
|
elif getattr(config, "TWELVE_DATA_API_KEY", ""):
|
|
return get_candles_twelvedata(pair, interval, days)
|
|
else:
|
|
return get_candles_yfinance(pair, interval, days)
|
|
|
|
|
|
|
|
# ─── Main download loop ───────────────────────────────────────────────────────
|
|
|
|
def download_all() -> None:
|
|
"""Download candle data for all configured Forex pairs and save to data/."""
|
|
os.makedirs(config.DATA_DIR, exist_ok=True)
|
|
|
|
# Détermination de la source affichée
|
|
_src = getattr(config, "DATA_SOURCE", "auto").lower()
|
|
if _src == "auto":
|
|
if config.OANDA_API_KEY:
|
|
source_label = "OANDA v20 (auto-sélectionné)"
|
|
elif getattr(config, "TWELVE_DATA_API_KEY", ""):
|
|
source_label = "Twelve Data (auto-sélectionné)"
|
|
else:
|
|
source_label = "Yahoo Finance (auto-sélectionné, gratuit)"
|
|
elif _src == "oanda":
|
|
source_label = "OANDA v20"
|
|
elif _src == "yfinance":
|
|
source_label = "Yahoo Finance"
|
|
elif _src == "twelvedata":
|
|
source_label = "Twelve Data"
|
|
elif _src == "alphavantage":
|
|
source_label = "Alpha Vantage"
|
|
elif _src in ("mt5", "metatrader5"):
|
|
source_label = "MetaTrader 5 SDK"
|
|
elif _src == "ccxt":
|
|
source_label = f"CCXT ({getattr(config, 'CCXT_BROKER', '?')})"
|
|
else:
|
|
source_label = _src
|
|
|
|
print(f"Downloading {DAYS_BACK} days of {config.CANDLE_INTERVAL} candles "
|
|
f"for {len(config.PAIRS)} Forex pairs...")
|
|
print(f"Source : {source_label} (DATA_SOURCE={_src})")
|
|
print(f"Broker : {config.EXCHANGE} (pour l'exécution)")
|
|
print(f"Saving : {config.DATA_DIR}/\n")
|
|
|
|
failed: list[str] = []
|
|
|
|
# Délai adaptatif selon la source (éviter le rate-limiting)
|
|
_src = getattr(config, "DATA_SOURCE", "auto").lower()
|
|
if _src in ("yfinance", "auto") and not config.OANDA_API_KEY:
|
|
_sleep = 1.5 # yfinance : 1.5s entre paires (rate limit discret)
|
|
elif _src == "alphavantage":
|
|
_sleep = 15.0 # Alpha Vantage free : 25 req/j → 4 req/min max
|
|
elif _src == "twelvedata":
|
|
_sleep = 8.0 if not getattr(config, "TWELVE_DATA_API_KEY", "") else 0.8
|
|
# Sans clé : ~8 req/min | Avec clé plan free : 800 req/j → ~1 req/s
|
|
elif _src == "oanda":
|
|
_sleep = 0.3 # OANDA : API rapide avec clé
|
|
else:
|
|
_sleep = 1.0 # défaut raisonnable
|
|
|
|
print(f" (délai entre paires : {_sleep}s pour éviter le rate-limiting)\n")
|
|
|
|
for i, pair in enumerate(config.PAIRS, 1):
|
|
try:
|
|
candles = get_candles(pair, config.CANDLE_INTERVAL, DAYS_BACK)
|
|
path = os.path.join(config.DATA_DIR, f"{pair}_1h.json")
|
|
with open(path, "w", encoding="utf-8") as f:
|
|
json.dump(candles, f)
|
|
days_actual = round(len(candles) / 24)
|
|
status = "OK" if days_actual >= 600 else "[données limitées]"
|
|
print(f" [{i:2d}/{len(config.PAIRS)}] {pair:8s} -- "
|
|
f"{len(candles):,} candles (~{days_actual} jours) {status}")
|
|
except Exception as e:
|
|
print(f" [{i:2d}/{len(config.PAIRS)}] {pair:8s} -- ERREUR: {e}")
|
|
failed.append(pair)
|
|
|
|
time.sleep(_sleep) # rate limit adaptatif selon DATA_SOURCE
|
|
|
|
# Retry failed pairs
|
|
if failed:
|
|
print(f"\n[!] {len(failed)} paire(s) échouée(s) : {', '.join(failed)}")
|
|
print("Retry dans 10 secondes...\n")
|
|
time.sleep(10)
|
|
|
|
for pair in failed:
|
|
try:
|
|
candles = get_candles(pair, config.CANDLE_INTERVAL, DAYS_BACK)
|
|
path = os.path.join(config.DATA_DIR, f"{pair}_1h.json")
|
|
with open(path, "w", encoding="utf-8") as f:
|
|
json.dump(candles, f)
|
|
days_actual = round(len(candles) / 24)
|
|
print(f" [OK] {pair:8s} -- {len(candles):,} candles récupérés!")
|
|
except Exception as e:
|
|
print(f" [FAIL] {pair:8s} -- Échec définitif: {e}")
|
|
|
|
print("\n[DONE] Données sauvegardées dans data/")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
download_all()
|