DinQuant/backend_api_python/app/data_sources/forex.py

"""
Forex data source
Get Forex Data with Tiingo
"""
from typing import Dict, List, Any, Optional
from datetime import datetime, timedelta
import time
import requests
import threading

from app.data_sources.base import BaseDataSource, TIMEFRAME_SECONDS
from app.utils.logger import get_logger
from app.config import TiingoConfig, APIKeys

logger = get_logger(__name__)

# Global Cache - Reduce Tiingo API calls
_forex_cache: Dict[str, Dict[str, Any]] = {}
_forex_cache_lock = threading.Lock()
_FOREX_CACHE_TTL = 60  # Forex price caching for 60 seconds (Tiingo free API has strict limits)


class ForexDataSource(BaseDataSource):
    """Forex data source (Tiingo)"""
    
    name = "Forex/Tiingo"
    
    # Tiingo resampleFreq mapping
    # Tiingo free account support: 5min, 15min, 30min, 1hour, 4hour, 1day
    # Note: 1min requires paid subscription, 1week/1month is not supported by Tiingo FX API
    TIMEFRAME_MAP = {
        '1m': '1min',      # Paid subscription required
        '5m': '5min',
        '15m': '15min',
        '30m': '30min',
        '1H': '1hour',
        '4H': '4hour',
        '1D': '1day',
        '1W': None,        # Tiingo does not support it and needs to be aggregated.
        '1M': None         # Tiingo does not support it and needs to be aggregated.
    }
    
    # Forex pair mapping (Tiingo uses standard tickers such as eurusd, audusd)
    # Uppercase letters are also acceptable. Tiingo is usually not case-sensitive, but uniformity is recommended.
    SYMBOL_MAP = {
        # Precious metals (Tiingo does not necessarily support all precious metals in OANDA format, usually XAUUSD)
        'XAUUSD': 'xauusd',
        'XAGUSD': 'xagusd',
        # major currency pairs
        'EURUSD': 'eurusd',
        'GBPUSD': 'gbpusd',
        'USDJPY': 'usdjpy',
        'AUDUSD': 'audusd',
        'USDCAD': 'usdcad',
        'USDCHF': 'usdchf',
        'NZDUSD': 'nzdusd',
    }
    
    def __init__(self):
        self.base_url = TiingoConfig.BASE_URL
        if not APIKeys.TIINGO_API_KEY:
             logger.warning("Tiingo API key is not configured; FX data will be unavailable")
    
    def get_ticker(self, symbol: str) -> Dict[str, Any]:
        """
        Get realtime quotes for foreign exchange
        
        Get realtime quotes using the Tiingo FX Top-of-Book API
        Comes with 60 second cache to avoid triggering Tiingo rate limit frequently
        
        Returns:
            dict: {
                'last': current price (mid price),
                'bid': buying price,
                'ask': selling price,
                'change': change amount,
                'changePercent': increase or decrease
            }
        """
        api_key = APIKeys.TIINGO_API_KEY
        if not api_key:
            logger.warning("Tiingo API key not configured")
            return {'last': 0, 'symbol': symbol}
        
        # Check cache
        cache_key = f"ticker_{symbol}"
        with _forex_cache_lock:
            cached = _forex_cache.get(cache_key)
            if cached:
                cache_time = cached.get('_cache_time', 0)
                if time.time() - cache_time < _FOREX_CACHE_TTL:
                    logger.debug(f"Using cached forex ticker for {symbol}")
                    return cached
        
        try:
            # parse symbol
            tiingo_symbol = self.SYMBOL_MAP.get(symbol)
            if not tiingo_symbol:
                tiingo_symbol = symbol.lower()
            
            # Tiingo FX Top-of-Book API
            # https://api.tiingo.com/tiingo/fx/top?tickers=eurusd&token=...
            url = f"{self.base_url}/fx/top"
            params = {
                'tickers': tiingo_symbol,
                'token': api_key
            }
            
            # Retry logic: Handling 429 rate limiting
            for attempt in range(3):
                response = requests.get(url, params=params, timeout=TiingoConfig.TIMEOUT)
                if response.status_code == 429:
                    wait_time = 2 * (attempt + 1)
                    logger.warning(f"Tiingo rate limit (429), waiting {wait_time}s before retry ({attempt+1}/3)")
                    time.sleep(wait_time)
                    continue
                break
            
            if response.status_code == 429:
                logger.warning("Tiingo rate limit exceeded for ticker request")
                logger.info("Note: Tiingo 1-minute forex data requires a paid subscription")
                # Return cached data (if available, even if expired)
                with _forex_cache_lock:
                    if cache_key in _forex_cache:
                        logger.info(f"Returning stale cache for {symbol} due to rate limit")
                        return _forex_cache[cache_key]
                return {'last': 0, 'symbol': symbol}
            
            response.raise_for_status()
            data = response.json()
            
            if data and isinstance(data, list) and len(data) > 0:
                item = data[0]
                # Tiingo FX top returns: ticker, quoteTimestamp, bidPrice, bidSize, askPrice, askSize, midPrice
                bid = float(item.get('bidPrice', 0) or 0)
                ask = float(item.get('askPrice', 0) or 0)
                mid = float(item.get('midPrice', 0) or 0)
                
                # If there is no midPrice, calculate the mid price
                if not mid and bid and ask:
                    mid = (bid + ask) / 2
                
                last_price = mid or bid or ask
                
                # Get the closing price of the previous day to calculate the rise and fall (additional request for daily data is required)
                prev_close = 0
                change = 0
                change_pct = 0
                
                try:
                    # Get yesterday's closing price
                    yesterday = (datetime.now() - timedelta(days=2)).strftime('%Y-%m-%d')
                    today = datetime.now().strftime('%Y-%m-%d')
                    price_url = f"{self.base_url}/fx/{tiingo_symbol}/prices"
                    price_params = {
                        'startDate': yesterday,
                        'endDate': today,
                        'resampleFreq': '1day',
                        'token': api_key
                    }
                    price_resp = requests.get(price_url, params=price_params, timeout=TiingoConfig.TIMEOUT)
                    if price_resp.status_code == 200:
                        price_data = price_resp.json()
                        if price_data and len(price_data) > 0:
                            prev_close = float(price_data[-1].get('close', 0) or 0)
                            if prev_close and last_price:
                                change = last_price - prev_close
                                change_pct = (change / prev_close) * 100
                except Exception:
                    pass  # Failure to calculate the rise or fall does not affect the main functions
                
                result = {
                    'last': round(last_price, 5),
                    'bid': round(bid, 5),
                    'ask': round(ask, 5),
                    'change': round(change, 5),
                    'changePercent': round(change_pct, 2),
                    'previousClose': round(prev_close, 5) if prev_close else 0,
                    '_cache_time': time.time()
                }
                
                # cache results
                with _forex_cache_lock:
                    _forex_cache[cache_key] = result
                
                return result
                
        except Exception as e:
            logger.error(f"Failed to get forex ticker for {symbol}: {e}")
        
        return {'last': 0, 'symbol': symbol}
    
    def _get_timeframe_seconds(self, timeframe: str) -> int:
        """Get the number of seconds corresponding to the time period"""
        return TIMEFRAME_SECONDS.get(timeframe, 86400)
    
    def get_kline(
        self,
        symbol: str,
        timeframe: str,
        limit: int,
        before_time: Optional[int] = None
    ) -> List[Dict[str, Any]]:
        """
        Get foreign exchange K-line data
        
        Args:
            symbol: Forex pair symbol (such as XAUUSD, EURUSD)
            timeframe: time period
            limit: number of data items
            before_time: end timestamp
        """
        # Dynamically obtain API Key
        api_key = APIKeys.TIINGO_API_KEY
        if not api_key:
            logger.error("Tiingo API key is not configured")
            return []
            
        try:
            # 1. Parse Symbol
            tiingo_symbol = self.SYMBOL_MAP.get(symbol)
            if not tiingo_symbol:
                # Try smart conversion: EURUSD -> eurusd
                tiingo_symbol = symbol.lower()

            # 2. Analysis Resolution (resampleFreq)
            resample_freq = self.TIMEFRAME_MAP.get(timeframe)
            
            # Special treatment: 1W/1M requires daily aggregation
            aggregate_to_weekly = (timeframe == '1W')
            aggregate_to_monthly = (timeframe == '1M')
            original_limit = limit  # Save original request quantity
            
            if aggregate_to_weekly or aggregate_to_monthly:
                # Aggregate using daily data
                resample_freq = '1day'
                # Limit the maximum number of weekly/monthly requests (Tiingo free API has data volume limit)
                # The maximum weekly request is 100 weeks = 700 days ≈ 2 years
                # The maximum monthly request is 36 months = 1080 days ≈ 3 years
                max_limit = 100 if aggregate_to_weekly else 36
                original_limit = min(original_limit, max_limit)
                # More daily data is needed to aggregate (weekly lines require 7 days, monthly lines require 30 days)
                limit = original_limit * (7 if aggregate_to_weekly else 30)
            
            if not resample_freq:
                logger.warning(f"Tiingo does not support timeframe: {timeframe}")
                return []
            
            # 1 minute data requires paid subscription reminder
            if timeframe == '1m':
                logger.info(f"Note: Tiingo 1-minute forex data requires a paid subscription")
            
            # 3. Calculation time range
            if before_time:
                end_dt = datetime.fromtimestamp(before_time)
            else:
                end_dt = datetime.now()
            
            # Calculate start time based on period and quantity
            # Note: Use daily seconds calculation in aggregation mode
            if aggregate_to_weekly or aggregate_to_monthly:
                tf_seconds = 86400  # daily seconds
            else:
                tf_seconds = self._get_timeframe_seconds(timeframe)
            # Get more buffer time (1.5 times, foreign exchange does not trade on weekends)
            start_dt = end_dt - timedelta(seconds=limit * tf_seconds * 1.5)
            
            # Tiingo free API supports up to about 5 years of data, limiting the maximum time range
            max_days = 365 * 3  # up to 3 years
            if (end_dt - start_dt).days > max_days:
                start_dt = end_dt - timedelta(days=max_days)
                logger.info(f"Tiingo: Limited date range to {max_days} days")
            
            # Format the date as YYYY-MM-DD (Tiingo supports this format)
            start_date_str = start_dt.strftime('%Y-%m-%d')
            end_date_str = end_dt.strftime('%Y-%m-%d')
            
            # 4. API request (with retry logic)
            # URL: https://api.tiingo.com/tiingo/fx/{ticker}/prices
            url = f"{self.base_url}/fx/{tiingo_symbol}/prices"
            
            params = {
                'startDate': start_date_str,
                'endDate': end_date_str,
                'resampleFreq': resample_freq,
                'token': api_key,
                'format': 'json'
            }
            
            # logger.info(f"Tiingo Request: {url} params={params}")
            
            # Retry logic: Handling 429 rate limiting
            max_retries = 3
            retry_delay = 2  # Second
            response = None
            
            for attempt in range(max_retries):
                try:
                    response = requests.get(url, params=params, timeout=TiingoConfig.TIMEOUT)
                    
                    if response.status_code == 429:
                        # Rate limit, wait and try again
                        wait_time = retry_delay * (attempt + 1)
                        logger.warning(f"Tiingo rate limit (429), waiting {wait_time}s before retry ({attempt + 1}/{max_retries})")
                        time.sleep(wait_time)
                        continue
                    
                    break  # Success or other errors, exit the retry loop
                    
                except requests.exceptions.Timeout:
                    if attempt < max_retries - 1:
                        logger.warning(f"Tiingo request timeout, retrying ({attempt + 1}/{max_retries})")
                        time.sleep(retry_delay)
                        continue
                    raise
            
            if response is None:
                logger.error("Tiingo API request failed after all retries")
                return []
            
            if response.status_code == 429:
                logger.error("Tiingo API rate limit exceeded. Please wait a moment before retrying.")
                return []
            
            if response.status_code == 403:
                logger.error("Tiingo API permission error (403): check whether your API key is valid and has access to this dataset.")
                return []
                 
            response.raise_for_status()
            data = response.json()
            
            # 5. Process the response
            # Tiingo returns a list of dicts:
            # [
            #   {
            #     "date": "2023-01-01T00:00:00.000Z",
            #     "ticker": "eurusd",
            #     "open": 1.07,
            #     "high": 1.08,
            #     "low": 1.06,
            #     "close": 1.07
            #     "mid": ... (optional, depends on settings, usually OHLC are bid or mid)
            #   }, ...
            # ]
            # Note: Tiingo FX prices objects keys: date, open, high, low, close.
            
            if not isinstance(data, list):
                logger.warning(f"Tiingo response is not a list: {data}")
                return []
                
            klines = []
            for item in data:
                # Parsing time: "2023-01-01T00:00:00.000Z"
                dt_str = item.get('date')
                # Tiingo returns UTC time in ISO format and needs to handle the time zone correctly.
                # Convert UTC time to local timestamp
                if dt_str.endswith('Z'):
                    dt_str = dt_str[:-1] + '+00:00'  # Replace Z with +00:00 for UTC
                
                dt = datetime.fromisoformat(dt_str)
                ts = int(dt.timestamp())  # UTC time zone is now handled correctly
                
                klines.append({
                    'time': ts,
                    'open': float(item.get('open')),
                    'high': float(item.get('high')),
                    'low': float(item.get('low')),
                    'close': float(item.get('close')),
                    'volume': 0.0 # Tiingo FX usually does not have volume
                })
            
            # Sort by time
            klines.sort(key=lambda x: x['time'])
            
            # If you need to aggregate to weekly or monthly lines
            if aggregate_to_weekly:
                klines = self._aggregate_to_weekly(klines)
                logger.debug(f"Aggregated {len(klines)} weekly candles from daily data")
            elif aggregate_to_monthly:
                klines = self._aggregate_to_monthly(klines)
                logger.debug(f"Aggregated {len(klines)} monthly candles from daily data")
            
            # Filter to original request count
            if len(klines) > original_limit:
                klines = klines[-original_limit:]
            
            # logger.info(f"obtained {len(klines)} pieces of Tiingo foreign exchange data")
            return klines
            
        except requests.exceptions.RequestException as e:
            logger.error(f"Tiingo API request failed: {e}")
            return []
        except Exception as e:
            logger.error(f"Failed to process Tiingo data: {e}")
            return []
    
    def _aggregate_to_weekly(self, daily_klines: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
        """Aggregate daily data into weekly data"""
        if not daily_klines:
            return []
        
        weekly_klines = []
        current_week = None
        week_data = None
        
        for kline in daily_klines:
            dt = datetime.fromtimestamp(kline['time'])
            # Get the Monday of the week in which the date is located
            week_start = dt - timedelta(days=dt.weekday())
            week_key = week_start.strftime('%Y-%W')
            
            if week_key != current_week:
                # Save data from last week
                if week_data:
                    weekly_klines.append(week_data)
                # start a new week
                current_week = week_key
                week_data = {
                    'time': int(week_start.timestamp()),
                    'open': kline['open'],
                    'high': kline['high'],
                    'low': kline['low'],
                    'close': kline['close'],
                    'volume': kline['volume']
                }
            else:
                # Update this week's data
                week_data['high'] = max(week_data['high'], kline['high'])
                week_data['low'] = min(week_data['low'], kline['low'])
                week_data['close'] = kline['close']
                week_data['volume'] += kline['volume']
        
        # Add last week
        if week_data:
            weekly_klines.append(week_data)
        
        return weekly_klines
    
    def _aggregate_to_monthly(self, daily_klines: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
        """Aggregate daily data into monthly data"""
        if not daily_klines:
            return []
        
        monthly_klines = []
        current_month = None
        month_data = None
        
        for kline in daily_klines:
            dt = datetime.fromtimestamp(kline['time'])
            month_key = dt.strftime('%Y-%m')
            
            if month_key != current_month:
                # Save last month’s data
                if month_data:
                    monthly_klines.append(month_data)
                # start a new month
                current_month = month_key
                month_start = dt.replace(day=1, hour=0, minute=0, second=0)
                month_data = {
                    'time': int(month_start.timestamp()),
                    'open': kline['open'],
                    'high': kline['high'],
                    'low': kline['low'],
                    'close': kline['close'],
                    'volume': kline['volume']
                }
            else:
                # Update this month's data
                month_data['high'] = max(month_data['high'], kline['high'])
                month_data['low'] = min(month_data['low'], kline['low'])
                month_data['close'] = kline['close']
                month_data['volume'] += kline['volume']
        
        # Add last month
        if month_data:
            monthly_klines.append(month_data)
        
        return monthly_klines