dienakdz
479 lines
20 KiB
Python
479 lines
20 KiB
Python
"""
|
||
Forex data source
|
||
Get Forex Data with Tiingo
|
||
"""
|
||
from typing import Dict, List, Any, Optional
|
||
from datetime import datetime, timedelta
|
||
import time
|
||
import requests
|
||
import threading
|
||
|
||
from app.data_sources.base import BaseDataSource, TIMEFRAME_SECONDS
|
||
from app.utils.logger import get_logger
|
||
from app.config import TiingoConfig, APIKeys
|
||
|
||
logger = get_logger(__name__)
|
||
|
||
# Global Cache - Reduce Tiingo API calls
|
||
_forex_cache: Dict[str, Dict[str, Any]] = {}
|
||
_forex_cache_lock = threading.Lock()
|
||
_FOREX_CACHE_TTL = 60 # Forex price caching for 60 seconds (Tiingo free API has strict limits)
|
||
|
||
|
||
class ForexDataSource(BaseDataSource):
|
||
"""Forex data source (Tiingo)"""
|
||
|
||
name = "Forex/Tiingo"
|
||
|
||
# Tiingo resampleFreq mapping
|
||
# Tiingo free account support: 5min, 15min, 30min, 1hour, 4hour, 1day
|
||
# Note: 1min requires paid subscription, 1week/1month is not supported by Tiingo FX API
|
||
TIMEFRAME_MAP = {
|
||
'1m': '1min', # Paid subscription required
|
||
'5m': '5min',
|
||
'15m': '15min',
|
||
'30m': '30min',
|
||
'1H': '1hour',
|
||
'4H': '4hour',
|
||
'1D': '1day',
|
||
'1W': None, # Tiingo does not support it and needs to be aggregated.
|
||
'1M': None # Tiingo does not support it and needs to be aggregated.
|
||
}
|
||
|
||
# Forex pair mapping (Tiingo uses standard tickers such as eurusd, audusd)
|
||
# Uppercase letters are also acceptable. Tiingo is usually not case-sensitive, but uniformity is recommended.
|
||
SYMBOL_MAP = {
|
||
# Precious metals (Tiingo does not necessarily support all precious metals in OANDA format, usually XAUUSD)
|
||
'XAUUSD': 'xauusd',
|
||
'XAGUSD': 'xagusd',
|
||
# major currency pairs
|
||
'EURUSD': 'eurusd',
|
||
'GBPUSD': 'gbpusd',
|
||
'USDJPY': 'usdjpy',
|
||
'AUDUSD': 'audusd',
|
||
'USDCAD': 'usdcad',
|
||
'USDCHF': 'usdchf',
|
||
'NZDUSD': 'nzdusd',
|
||
}
|
||
|
||
def __init__(self):
|
||
self.base_url = TiingoConfig.BASE_URL
|
||
if not APIKeys.TIINGO_API_KEY:
|
||
logger.warning("Tiingo API key is not configured; FX data will be unavailable")
|
||
|
||
def get_ticker(self, symbol: str) -> Dict[str, Any]:
|
||
"""
|
||
Get realtime quotes for foreign exchange
|
||
|
||
Get realtime quotes using the Tiingo FX Top-of-Book API
|
||
Comes with 60 second cache to avoid triggering Tiingo rate limit frequently
|
||
|
||
Returns:
|
||
dict: {
|
||
'last': current price (mid price),
|
||
'bid': buying price,
|
||
'ask': selling price,
|
||
'change': change amount,
|
||
'changePercent': increase or decrease
|
||
}
|
||
"""
|
||
api_key = APIKeys.TIINGO_API_KEY
|
||
if not api_key:
|
||
logger.warning("Tiingo API key not configured")
|
||
return {'last': 0, 'symbol': symbol}
|
||
|
||
# Check cache
|
||
cache_key = f"ticker_{symbol}"
|
||
with _forex_cache_lock:
|
||
cached = _forex_cache.get(cache_key)
|
||
if cached:
|
||
cache_time = cached.get('_cache_time', 0)
|
||
if time.time() - cache_time < _FOREX_CACHE_TTL:
|
||
logger.debug(f"Using cached forex ticker for {symbol}")
|
||
return cached
|
||
|
||
try:
|
||
# parse symbol
|
||
tiingo_symbol = self.SYMBOL_MAP.get(symbol)
|
||
if not tiingo_symbol:
|
||
tiingo_symbol = symbol.lower()
|
||
|
||
# Tiingo FX Top-of-Book API
|
||
# https://api.tiingo.com/tiingo/fx/top?tickers=eurusd&token=...
|
||
url = f"{self.base_url}/fx/top"
|
||
params = {
|
||
'tickers': tiingo_symbol,
|
||
'token': api_key
|
||
}
|
||
|
||
# Retry logic: Handling 429 rate limiting
|
||
for attempt in range(3):
|
||
response = requests.get(url, params=params, timeout=TiingoConfig.TIMEOUT)
|
||
if response.status_code == 429:
|
||
wait_time = 2 * (attempt + 1)
|
||
logger.warning(f"Tiingo rate limit (429), waiting {wait_time}s before retry ({attempt+1}/3)")
|
||
time.sleep(wait_time)
|
||
continue
|
||
break
|
||
|
||
if response.status_code == 429:
|
||
logger.warning("Tiingo rate limit exceeded for ticker request")
|
||
logger.info("Note: Tiingo 1-minute forex data requires a paid subscription")
|
||
# Return cached data (if available, even if expired)
|
||
with _forex_cache_lock:
|
||
if cache_key in _forex_cache:
|
||
logger.info(f"Returning stale cache for {symbol} due to rate limit")
|
||
return _forex_cache[cache_key]
|
||
return {'last': 0, 'symbol': symbol}
|
||
|
||
response.raise_for_status()
|
||
data = response.json()
|
||
|
||
if data and isinstance(data, list) and len(data) > 0:
|
||
item = data[0]
|
||
# Tiingo FX top returns: ticker, quoteTimestamp, bidPrice, bidSize, askPrice, askSize, midPrice
|
||
bid = float(item.get('bidPrice', 0) or 0)
|
||
ask = float(item.get('askPrice', 0) or 0)
|
||
mid = float(item.get('midPrice', 0) or 0)
|
||
|
||
# If there is no midPrice, calculate the mid price
|
||
if not mid and bid and ask:
|
||
mid = (bid + ask) / 2
|
||
|
||
last_price = mid or bid or ask
|
||
|
||
# Get the closing price of the previous day to calculate the rise and fall (additional request for daily data is required)
|
||
prev_close = 0
|
||
change = 0
|
||
change_pct = 0
|
||
|
||
try:
|
||
# Get yesterday's closing price
|
||
yesterday = (datetime.now() - timedelta(days=2)).strftime('%Y-%m-%d')
|
||
today = datetime.now().strftime('%Y-%m-%d')
|
||
price_url = f"{self.base_url}/fx/{tiingo_symbol}/prices"
|
||
price_params = {
|
||
'startDate': yesterday,
|
||
'endDate': today,
|
||
'resampleFreq': '1day',
|
||
'token': api_key
|
||
}
|
||
price_resp = requests.get(price_url, params=price_params, timeout=TiingoConfig.TIMEOUT)
|
||
if price_resp.status_code == 200:
|
||
price_data = price_resp.json()
|
||
if price_data and len(price_data) > 0:
|
||
prev_close = float(price_data[-1].get('close', 0) or 0)
|
||
if prev_close and last_price:
|
||
change = last_price - prev_close
|
||
change_pct = (change / prev_close) * 100
|
||
except Exception:
|
||
pass # Failure to calculate the rise or fall does not affect the main functions
|
||
|
||
result = {
|
||
'last': round(last_price, 5),
|
||
'bid': round(bid, 5),
|
||
'ask': round(ask, 5),
|
||
'change': round(change, 5),
|
||
'changePercent': round(change_pct, 2),
|
||
'previousClose': round(prev_close, 5) if prev_close else 0,
|
||
'_cache_time': time.time()
|
||
}
|
||
|
||
# cache results
|
||
with _forex_cache_lock:
|
||
_forex_cache[cache_key] = result
|
||
|
||
return result
|
||
|
||
except Exception as e:
|
||
logger.error(f"Failed to get forex ticker for {symbol}: {e}")
|
||
|
||
return {'last': 0, 'symbol': symbol}
|
||
|
||
def _get_timeframe_seconds(self, timeframe: str) -> int:
|
||
"""Get the number of seconds corresponding to the time period"""
|
||
return TIMEFRAME_SECONDS.get(timeframe, 86400)
|
||
|
||
def get_kline(
|
||
self,
|
||
symbol: str,
|
||
timeframe: str,
|
||
limit: int,
|
||
before_time: Optional[int] = None
|
||
) -> List[Dict[str, Any]]:
|
||
"""
|
||
Get foreign exchange K-line data
|
||
|
||
Args:
|
||
symbol: Forex pair symbol (such as XAUUSD, EURUSD)
|
||
timeframe: time period
|
||
limit: number of data items
|
||
before_time: end timestamp
|
||
"""
|
||
# Dynamically obtain API Key
|
||
api_key = APIKeys.TIINGO_API_KEY
|
||
if not api_key:
|
||
logger.error("Tiingo API key is not configured")
|
||
return []
|
||
|
||
try:
|
||
# 1. Parse Symbol
|
||
tiingo_symbol = self.SYMBOL_MAP.get(symbol)
|
||
if not tiingo_symbol:
|
||
# Try smart conversion: EURUSD -> eurusd
|
||
tiingo_symbol = symbol.lower()
|
||
|
||
# 2. Analysis Resolution (resampleFreq)
|
||
resample_freq = self.TIMEFRAME_MAP.get(timeframe)
|
||
|
||
# Special treatment: 1W/1M requires daily aggregation
|
||
aggregate_to_weekly = (timeframe == '1W')
|
||
aggregate_to_monthly = (timeframe == '1M')
|
||
original_limit = limit # Save original request quantity
|
||
|
||
if aggregate_to_weekly or aggregate_to_monthly:
|
||
# Aggregate using daily data
|
||
resample_freq = '1day'
|
||
# Limit the maximum number of weekly/monthly requests (Tiingo free API has data volume limit)
|
||
# The maximum weekly request is 100 weeks = 700 days ≈ 2 years
|
||
# The maximum monthly request is 36 months = 1080 days ≈ 3 years
|
||
max_limit = 100 if aggregate_to_weekly else 36
|
||
original_limit = min(original_limit, max_limit)
|
||
# More daily data is needed to aggregate (weekly lines require 7 days, monthly lines require 30 days)
|
||
limit = original_limit * (7 if aggregate_to_weekly else 30)
|
||
|
||
if not resample_freq:
|
||
logger.warning(f"Tiingo does not support timeframe: {timeframe}")
|
||
return []
|
||
|
||
# 1 minute data requires paid subscription reminder
|
||
if timeframe == '1m':
|
||
logger.info(f"Note: Tiingo 1-minute forex data requires a paid subscription")
|
||
|
||
# 3. Calculation time range
|
||
if before_time:
|
||
end_dt = datetime.fromtimestamp(before_time)
|
||
else:
|
||
end_dt = datetime.now()
|
||
|
||
# Calculate start time based on period and quantity
|
||
# Note: Use daily seconds calculation in aggregation mode
|
||
if aggregate_to_weekly or aggregate_to_monthly:
|
||
tf_seconds = 86400 # daily seconds
|
||
else:
|
||
tf_seconds = self._get_timeframe_seconds(timeframe)
|
||
# Get more buffer time (1.5 times, foreign exchange does not trade on weekends)
|
||
start_dt = end_dt - timedelta(seconds=limit * tf_seconds * 1.5)
|
||
|
||
# Tiingo free API supports up to about 5 years of data, limiting the maximum time range
|
||
max_days = 365 * 3 # up to 3 years
|
||
if (end_dt - start_dt).days > max_days:
|
||
start_dt = end_dt - timedelta(days=max_days)
|
||
logger.info(f"Tiingo: Limited date range to {max_days} days")
|
||
|
||
# Format the date as YYYY-MM-DD (Tiingo supports this format)
|
||
start_date_str = start_dt.strftime('%Y-%m-%d')
|
||
end_date_str = end_dt.strftime('%Y-%m-%d')
|
||
|
||
# 4. API request (with retry logic)
|
||
# URL: https://api.tiingo.com/tiingo/fx/{ticker}/prices
|
||
url = f"{self.base_url}/fx/{tiingo_symbol}/prices"
|
||
|
||
params = {
|
||
'startDate': start_date_str,
|
||
'endDate': end_date_str,
|
||
'resampleFreq': resample_freq,
|
||
'token': api_key,
|
||
'format': 'json'
|
||
}
|
||
|
||
# logger.info(f"Tiingo Request: {url} params={params}")
|
||
|
||
# Retry logic: Handling 429 rate limiting
|
||
max_retries = 3
|
||
retry_delay = 2 # Second
|
||
response = None
|
||
|
||
for attempt in range(max_retries):
|
||
try:
|
||
response = requests.get(url, params=params, timeout=TiingoConfig.TIMEOUT)
|
||
|
||
if response.status_code == 429:
|
||
# Rate limit, wait and try again
|
||
wait_time = retry_delay * (attempt + 1)
|
||
logger.warning(f"Tiingo rate limit (429), waiting {wait_time}s before retry ({attempt + 1}/{max_retries})")
|
||
time.sleep(wait_time)
|
||
continue
|
||
|
||
break # Success or other errors, exit the retry loop
|
||
|
||
except requests.exceptions.Timeout:
|
||
if attempt < max_retries - 1:
|
||
logger.warning(f"Tiingo request timeout, retrying ({attempt + 1}/{max_retries})")
|
||
time.sleep(retry_delay)
|
||
continue
|
||
raise
|
||
|
||
if response is None:
|
||
logger.error("Tiingo API request failed after all retries")
|
||
return []
|
||
|
||
if response.status_code == 429:
|
||
logger.error("Tiingo API rate limit exceeded. Please wait a moment before retrying.")
|
||
return []
|
||
|
||
if response.status_code == 403:
|
||
logger.error("Tiingo API permission error (403): check whether your API key is valid and has access to this dataset.")
|
||
return []
|
||
|
||
response.raise_for_status()
|
||
data = response.json()
|
||
|
||
# 5. Process the response
|
||
# Tiingo returns a list of dicts:
|
||
# [
|
||
# {
|
||
# "date": "2023-01-01T00:00:00.000Z",
|
||
# "ticker": "eurusd",
|
||
# "open": 1.07,
|
||
# "high": 1.08,
|
||
# "low": 1.06,
|
||
# "close": 1.07
|
||
# "mid": ... (optional, depends on settings, usually OHLC are bid or mid)
|
||
# }, ...
|
||
# ]
|
||
# Note: Tiingo FX prices objects keys: date, open, high, low, close.
|
||
|
||
if not isinstance(data, list):
|
||
logger.warning(f"Tiingo response is not a list: {data}")
|
||
return []
|
||
|
||
klines = []
|
||
for item in data:
|
||
# Parsing time: "2023-01-01T00:00:00.000Z"
|
||
dt_str = item.get('date')
|
||
# Tiingo returns UTC time in ISO format and needs to handle the time zone correctly.
|
||
# Convert UTC time to local timestamp
|
||
if dt_str.endswith('Z'):
|
||
dt_str = dt_str[:-1] + '+00:00' # Replace Z with +00:00 for UTC
|
||
|
||
dt = datetime.fromisoformat(dt_str)
|
||
ts = int(dt.timestamp()) # UTC time zone is now handled correctly
|
||
|
||
klines.append({
|
||
'time': ts,
|
||
'open': float(item.get('open')),
|
||
'high': float(item.get('high')),
|
||
'low': float(item.get('low')),
|
||
'close': float(item.get('close')),
|
||
'volume': 0.0 # Tiingo FX usually does not have volume
|
||
})
|
||
|
||
# Sort by time
|
||
klines.sort(key=lambda x: x['time'])
|
||
|
||
# If you need to aggregate to weekly or monthly lines
|
||
if aggregate_to_weekly:
|
||
klines = self._aggregate_to_weekly(klines)
|
||
logger.debug(f"Aggregated {len(klines)} weekly candles from daily data")
|
||
elif aggregate_to_monthly:
|
||
klines = self._aggregate_to_monthly(klines)
|
||
logger.debug(f"Aggregated {len(klines)} monthly candles from daily data")
|
||
|
||
# Filter to original request count
|
||
if len(klines) > original_limit:
|
||
klines = klines[-original_limit:]
|
||
|
||
# logger.info(f"obtained {len(klines)} pieces of Tiingo foreign exchange data")
|
||
return klines
|
||
|
||
except requests.exceptions.RequestException as e:
|
||
logger.error(f"Tiingo API request failed: {e}")
|
||
return []
|
||
except Exception as e:
|
||
logger.error(f"Failed to process Tiingo data: {e}")
|
||
return []
|
||
|
||
def _aggregate_to_weekly(self, daily_klines: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
|
||
"""Aggregate daily data into weekly data"""
|
||
if not daily_klines:
|
||
return []
|
||
|
||
weekly_klines = []
|
||
current_week = None
|
||
week_data = None
|
||
|
||
for kline in daily_klines:
|
||
dt = datetime.fromtimestamp(kline['time'])
|
||
# Get the Monday of the week in which the date is located
|
||
week_start = dt - timedelta(days=dt.weekday())
|
||
week_key = week_start.strftime('%Y-%W')
|
||
|
||
if week_key != current_week:
|
||
# Save data from last week
|
||
if week_data:
|
||
weekly_klines.append(week_data)
|
||
# start a new week
|
||
current_week = week_key
|
||
week_data = {
|
||
'time': int(week_start.timestamp()),
|
||
'open': kline['open'],
|
||
'high': kline['high'],
|
||
'low': kline['low'],
|
||
'close': kline['close'],
|
||
'volume': kline['volume']
|
||
}
|
||
else:
|
||
# Update this week's data
|
||
week_data['high'] = max(week_data['high'], kline['high'])
|
||
week_data['low'] = min(week_data['low'], kline['low'])
|
||
week_data['close'] = kline['close']
|
||
week_data['volume'] += kline['volume']
|
||
|
||
# Add last week
|
||
if week_data:
|
||
weekly_klines.append(week_data)
|
||
|
||
return weekly_klines
|
||
|
||
def _aggregate_to_monthly(self, daily_klines: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
|
||
"""Aggregate daily data into monthly data"""
|
||
if not daily_klines:
|
||
return []
|
||
|
||
monthly_klines = []
|
||
current_month = None
|
||
month_data = None
|
||
|
||
for kline in daily_klines:
|
||
dt = datetime.fromtimestamp(kline['time'])
|
||
month_key = dt.strftime('%Y-%m')
|
||
|
||
if month_key != current_month:
|
||
# Save last month’s data
|
||
if month_data:
|
||
monthly_klines.append(month_data)
|
||
# start a new month
|
||
current_month = month_key
|
||
month_start = dt.replace(day=1, hour=0, minute=0, second=0)
|
||
month_data = {
|
||
'time': int(month_start.timestamp()),
|
||
'open': kline['open'],
|
||
'high': kline['high'],
|
||
'low': kline['low'],
|
||
'close': kline['close'],
|
||
'volume': kline['volume']
|
||
}
|
||
else:
|
||
# Update this month's data
|
||
month_data['high'] = max(month_data['high'], kline['high'])
|
||
month_data['low'] = min(month_data['low'], kline['low'])
|
||
month_data['close'] = kline['close']
|
||
month_data['volume'] += kline['volume']
|
||
|
||
# Add last month
|
||
if month_data:
|
||
monthly_klines.append(month_data)
|
||
|
||
return monthly_klines
|