""" Forex data source Get Forex Data with Tiingo """ import threading import time from datetime import datetime, timedelta from typing import Any, Dict, List, Optional import requests from app.config import APIKeys, TiingoConfig from app.data_sources.base import TIMEFRAME_SECONDS, BaseDataSource from app.utils.logger import get_logger logger = get_logger(__name__) # Global Cache - Reduce Tiingo API calls _forex_cache: Dict[str, Dict[str, Any]] = {} _forex_cache_lock = threading.Lock() _FOREX_CACHE_TTL = 60 # Forex price caching for 60 seconds (Tiingo free API has strict limits) class ForexDataSource(BaseDataSource): """Forex data source (Tiingo)""" name = "Forex/Tiingo" # Tiingo resampleFreq mapping # Tiingo free account support: 5min, 15min, 30min, 1hour, 4hour, 1day # Note: 1min requires paid subscription, 1week/1month is not supported by Tiingo FX API TIMEFRAME_MAP = { "1m": "1min", # Paid subscription required "5m": "5min", "15m": "15min", "30m": "30min", "1H": "1hour", "4H": "4hour", "1D": "1day", "1W": None, # Tiingo does not support it and needs to be aggregated. "1M": None, # Tiingo does not support it and needs to be aggregated. } # Forex pair mapping (Tiingo uses standard tickers such as eurusd, audusd) # Uppercase letters are also acceptable. Tiingo is usually not case-sensitive, but uniformity is recommended. SYMBOL_MAP = { # Precious metals (Tiingo does not necessarily support all precious metals in OANDA format, usually XAUUSD) "XAUUSD": "xauusd", "XAGUSD": "xagusd", # major currency pairs "EURUSD": "eurusd", "GBPUSD": "gbpusd", "USDJPY": "usdjpy", "AUDUSD": "audusd", "USDCAD": "usdcad", "USDCHF": "usdchf", "NZDUSD": "nzdusd", } def __init__(self): self.base_url = TiingoConfig.BASE_URL if not APIKeys.TIINGO_API_KEY: logger.warning("Tiingo API key is not configured; FX data will be unavailable") def get_ticker(self, symbol: str) -> Dict[str, Any]: """ Get realtime quotes for foreign exchange Get realtime quotes using the Tiingo FX Top-of-Book API Comes with 60 second cache to avoid triggering Tiingo rate limit frequently Returns: dict: { 'last': current price (mid price), 'bid': buying price, 'ask': selling price, 'change': change amount, 'changePercent': increase or decrease } """ api_key = APIKeys.TIINGO_API_KEY if not api_key: logger.warning("Tiingo API key not configured") return {"last": 0, "symbol": symbol} # Check cache cache_key = f"ticker_{symbol}" with _forex_cache_lock: cached = _forex_cache.get(cache_key) if cached: cache_time = cached.get("_cache_time", 0) if time.time() - cache_time < _FOREX_CACHE_TTL: logger.debug(f"Using cached forex ticker for {symbol}") return cached try: # parse symbol tiingo_symbol = self.SYMBOL_MAP.get(symbol) if not tiingo_symbol: tiingo_symbol = symbol.lower() # Tiingo FX Top-of-Book API # https://api.tiingo.com/tiingo/fx/top?tickers=eurusd&token=... url = f"{self.base_url}/fx/top" params = {"tickers": tiingo_symbol, "token": api_key} # Retry logic: Handling 429 rate limiting for attempt in range(3): response = requests.get(url, params=params, timeout=TiingoConfig.TIMEOUT) if response.status_code == 429: wait_time = 2 * (attempt + 1) logger.warning(f"Tiingo rate limit (429), waiting {wait_time}s before retry ({attempt + 1}/3)") time.sleep(wait_time) continue break if response.status_code == 429: logger.warning("Tiingo rate limit exceeded for ticker request") logger.info("Note: Tiingo 1-minute forex data requires a paid subscription") # Return cached data (if available, even if expired) with _forex_cache_lock: if cache_key in _forex_cache: logger.info(f"Returning stale cache for {symbol} due to rate limit") return _forex_cache[cache_key] return {"last": 0, "symbol": symbol} response.raise_for_status() data = response.json() if data and isinstance(data, list) and len(data) > 0: item = data[0] # Tiingo FX top returns: ticker, quoteTimestamp, bidPrice, bidSize, askPrice, askSize, midPrice bid = float(item.get("bidPrice", 0) or 0) ask = float(item.get("askPrice", 0) or 0) mid = float(item.get("midPrice", 0) or 0) # If there is no midPrice, calculate the mid price if not mid and bid and ask: mid = (bid + ask) / 2 last_price = mid or bid or ask # Get the closing price of the previous day to calculate the rise and fall (additional request for daily data is required) prev_close = 0 change = 0 change_pct = 0 try: # Get yesterday's closing price yesterday = (datetime.now() - timedelta(days=2)).strftime("%Y-%m-%d") today = datetime.now().strftime("%Y-%m-%d") price_url = f"{self.base_url}/fx/{tiingo_symbol}/prices" price_params = {"startDate": yesterday, "endDate": today, "resampleFreq": "1day", "token": api_key} price_resp = requests.get(price_url, params=price_params, timeout=TiingoConfig.TIMEOUT) if price_resp.status_code == 200: price_data = price_resp.json() if price_data and len(price_data) > 0: prev_close = float(price_data[-1].get("close", 0) or 0) if prev_close and last_price: change = last_price - prev_close change_pct = (change / prev_close) * 100 except Exception: pass # Failure to calculate the rise or fall does not affect the main functions result = { "last": round(last_price, 5), "bid": round(bid, 5), "ask": round(ask, 5), "change": round(change, 5), "changePercent": round(change_pct, 2), "previousClose": round(prev_close, 5) if prev_close else 0, "_cache_time": time.time(), } # cache results with _forex_cache_lock: _forex_cache[cache_key] = result return result except Exception as e: logger.error(f"Failed to get forex ticker for {symbol}: {e}") return {"last": 0, "symbol": symbol} def _get_timeframe_seconds(self, timeframe: str) -> int: """Get the number of seconds corresponding to the time period""" return TIMEFRAME_SECONDS.get(timeframe, 86400) def get_kline( self, symbol: str, timeframe: str, limit: int, before_time: Optional[int] = None ) -> List[Dict[str, Any]]: """ Get foreign exchange K-line data Args: symbol: Forex pair symbol (such as XAUUSD, EURUSD) timeframe: time period limit: number of data items before_time: end timestamp """ # Dynamically obtain API Key api_key = APIKeys.TIINGO_API_KEY if not api_key: logger.error("Tiingo API key is not configured") return [] try: # 1. Parse Symbol tiingo_symbol = self.SYMBOL_MAP.get(symbol) if not tiingo_symbol: # Try smart conversion: EURUSD -> eurusd tiingo_symbol = symbol.lower() # 2. Analysis Resolution (resampleFreq) resample_freq = self.TIMEFRAME_MAP.get(timeframe) # Special treatment: 1W/1M requires daily aggregation aggregate_to_weekly = timeframe == "1W" aggregate_to_monthly = timeframe == "1M" original_limit = limit # Save original request quantity if aggregate_to_weekly or aggregate_to_monthly: # Aggregate using daily data resample_freq = "1day" # Limit the maximum number of weekly/monthly requests (Tiingo free API has data volume limit) # The maximum weekly request is 100 weeks = 700 days ≈ 2 years # The maximum monthly request is 36 months = 1080 days ≈ 3 years max_limit = 100 if aggregate_to_weekly else 36 original_limit = min(original_limit, max_limit) # More daily data is needed to aggregate (weekly lines require 7 days, monthly lines require 30 days) limit = original_limit * (7 if aggregate_to_weekly else 30) if not resample_freq: logger.warning(f"Tiingo does not support timeframe: {timeframe}") return [] # 1 minute data requires paid subscription reminder if timeframe == "1m": logger.info("Note: Tiingo 1-minute forex data requires a paid subscription") # 3. Calculation time range if before_time: end_dt = datetime.fromtimestamp(before_time) else: end_dt = datetime.now() # Calculate start time based on period and quantity # Note: Use daily seconds calculation in aggregation mode if aggregate_to_weekly or aggregate_to_monthly: tf_seconds = 86400 # daily seconds else: tf_seconds = self._get_timeframe_seconds(timeframe) # Get more buffer time (1.5 times, foreign exchange does not trade on weekends) start_dt = end_dt - timedelta(seconds=limit * tf_seconds * 1.5) # Tiingo free API supports up to about 5 years of data, limiting the maximum time range max_days = 365 * 3 # up to 3 years if (end_dt - start_dt).days > max_days: start_dt = end_dt - timedelta(days=max_days) logger.info(f"Tiingo: Limited date range to {max_days} days") # Format the date as YYYY-MM-DD (Tiingo supports this format) start_date_str = start_dt.strftime("%Y-%m-%d") end_date_str = end_dt.strftime("%Y-%m-%d") # 4. API request (with retry logic) # URL: https://api.tiingo.com/tiingo/fx/{ticker}/prices url = f"{self.base_url}/fx/{tiingo_symbol}/prices" params = { "startDate": start_date_str, "endDate": end_date_str, "resampleFreq": resample_freq, "token": api_key, "format": "json", } # logger.info(f"Tiingo Request: {url} params={params}") # Retry logic: Handling 429 rate limiting max_retries = 3 retry_delay = 2 # Second response = None for attempt in range(max_retries): try: response = requests.get(url, params=params, timeout=TiingoConfig.TIMEOUT) if response.status_code == 429: # Rate limit, wait and try again wait_time = retry_delay * (attempt + 1) logger.warning( f"Tiingo rate limit (429), waiting {wait_time}s before retry ({attempt + 1}/{max_retries})" ) time.sleep(wait_time) continue break # Success or other errors, exit the retry loop except requests.exceptions.Timeout: if attempt < max_retries - 1: logger.warning(f"Tiingo request timeout, retrying ({attempt + 1}/{max_retries})") time.sleep(retry_delay) continue raise if response is None: logger.error("Tiingo API request failed after all retries") return [] if response.status_code == 429: logger.error("Tiingo API rate limit exceeded. Please wait a moment before retrying.") return [] if response.status_code == 403: logger.error( "Tiingo API permission error (403): check whether your API key is valid and has access to this dataset." ) return [] response.raise_for_status() data = response.json() # 5. Process the response # Tiingo returns a list of dicts: # [ # { # "date": "2023-01-01T00:00:00.000Z", # "ticker": "eurusd", # "open": 1.07, # "high": 1.08, # "low": 1.06, # "close": 1.07 # "mid": ... (optional, depends on settings, usually OHLC are bid or mid) # }, ... # ] # Note: Tiingo FX prices objects keys: date, open, high, low, close. if not isinstance(data, list): logger.warning(f"Tiingo response is not a list: {data}") return [] klines = [] for item in data: # Parsing time: "2023-01-01T00:00:00.000Z" dt_str = item.get("date") # Tiingo returns UTC time in ISO format and needs to handle the time zone correctly. # Convert UTC time to local timestamp if dt_str.endswith("Z"): dt_str = dt_str[:-1] + "+00:00" # Replace Z with +00:00 for UTC dt = datetime.fromisoformat(dt_str) ts = int(dt.timestamp()) # UTC time zone is now handled correctly klines.append( { "time": ts, "open": float(item.get("open")), "high": float(item.get("high")), "low": float(item.get("low")), "close": float(item.get("close")), "volume": 0.0, # Tiingo FX usually does not have volume } ) # Sort by time klines.sort(key=lambda x: x["time"]) # If you need to aggregate to weekly or monthly lines if aggregate_to_weekly: klines = self._aggregate_to_weekly(klines) logger.debug(f"Aggregated {len(klines)} weekly candles from daily data") elif aggregate_to_monthly: klines = self._aggregate_to_monthly(klines) logger.debug(f"Aggregated {len(klines)} monthly candles from daily data") # Filter to original request count if len(klines) > original_limit: klines = klines[-original_limit:] # logger.info(f"obtained {len(klines)} pieces of Tiingo foreign exchange data") return klines except requests.exceptions.RequestException as e: logger.error(f"Tiingo API request failed: {e}") return [] except Exception as e: logger.error(f"Failed to process Tiingo data: {e}") return [] def _aggregate_to_weekly(self, daily_klines: List[Dict[str, Any]]) -> List[Dict[str, Any]]: """Aggregate daily data into weekly data""" if not daily_klines: return [] weekly_klines = [] current_week = None week_data = None for kline in daily_klines: dt = datetime.fromtimestamp(kline["time"]) # Get the Monday of the week in which the date is located week_start = dt - timedelta(days=dt.weekday()) week_key = week_start.strftime("%Y-%W") if week_key != current_week: # Save data from last week if week_data: weekly_klines.append(week_data) # start a new week current_week = week_key week_data = { "time": int(week_start.timestamp()), "open": kline["open"], "high": kline["high"], "low": kline["low"], "close": kline["close"], "volume": kline["volume"], } else: # Update this week's data week_data["high"] = max(week_data["high"], kline["high"]) week_data["low"] = min(week_data["low"], kline["low"]) week_data["close"] = kline["close"] week_data["volume"] += kline["volume"] # Add last week if week_data: weekly_klines.append(week_data) return weekly_klines def _aggregate_to_monthly(self, daily_klines: List[Dict[str, Any]]) -> List[Dict[str, Any]]: """Aggregate daily data into monthly data""" if not daily_klines: return [] monthly_klines = [] current_month = None month_data = None for kline in daily_klines: dt = datetime.fromtimestamp(kline["time"]) month_key = dt.strftime("%Y-%m") if month_key != current_month: # Save last month’s data if month_data: monthly_klines.append(month_data) # start a new month current_month = month_key month_start = dt.replace(day=1, hour=0, minute=0, second=0) month_data = { "time": int(month_start.timestamp()), "open": kline["open"], "high": kline["high"], "low": kline["low"], "close": kline["close"], "volume": kline["volume"], } else: # Update this month's data month_data["high"] = max(month_data["high"], kline["high"]) month_data["low"] = min(month_data["low"], kline["low"]) month_data["close"] = kline["close"] month_data["volume"] += kline["volume"] # Add last month if month_data: monthly_klines.append(month_data) return monthly_klines