dienakdz
376 lines
14 KiB
Python
376 lines
14 KiB
Python
"""
|
|
Cryptocurrency data source
|
|
Get data using CCXT (Coinbase)
|
|
"""
|
|
from typing import Dict, List, Any, Optional, Tuple
|
|
from datetime import datetime, timedelta
|
|
import ccxt
|
|
|
|
from app.data_sources.base import BaseDataSource, TIMEFRAME_SECONDS
|
|
from app.utils.logger import get_logger
|
|
from app.config import CCXTConfig, APIKeys
|
|
|
|
logger = get_logger(__name__)
|
|
|
|
|
|
class CryptoDataSource(BaseDataSource):
|
|
"""Cryptocurrency data source"""
|
|
|
|
name = "Crypto/CCXT"
|
|
|
|
# time period mapping
|
|
TIMEFRAME_MAP = CCXTConfig.TIMEFRAME_MAP
|
|
|
|
# List of common quote currencies (sorted by priority)
|
|
COMMON_QUOTES = ['USDT', 'USD', 'BTC', 'ETH', 'BUSD', 'USDC', 'BNB', 'EUR', 'GBP']
|
|
|
|
def __init__(self):
|
|
config = {
|
|
'timeout': CCXTConfig.TIMEOUT,
|
|
'enableRateLimit': CCXTConfig.ENABLE_RATE_LIMIT
|
|
}
|
|
|
|
# If a proxy is configured
|
|
if CCXTConfig.PROXY:
|
|
config['proxies'] = {
|
|
'http': CCXTConfig.PROXY,
|
|
'https': CCXTConfig.PROXY
|
|
}
|
|
|
|
exchange_id = CCXTConfig.DEFAULT_EXCHANGE
|
|
|
|
# Dynamically loading exchange classes
|
|
if not hasattr(ccxt, exchange_id):
|
|
logger.warning(f"CCXT exchange '{exchange_id}' not found, falling back to 'coinbase'")
|
|
exchange_id = 'coinbase'
|
|
|
|
exchange_class = getattr(ccxt, exchange_id)
|
|
self.exchange = exchange_class(config)
|
|
|
|
# Lazy loading of markets (loaded on first use)
|
|
self._markets_loaded = False
|
|
self._markets_cache = None
|
|
|
|
def _ensure_markets_loaded(self) -> bool:
|
|
"""Make sure markets are loaded (for symbol verification)"""
|
|
if self._markets_loaded and self._markets_cache is not None:
|
|
return True
|
|
|
|
try:
|
|
# Some exchanges require explicit loading of markets
|
|
if hasattr(self.exchange, 'load_markets'):
|
|
self.exchange.load_markets(reload=False)
|
|
self._markets_cache = getattr(self.exchange, 'markets', {})
|
|
self._markets_loaded = True
|
|
return True
|
|
except Exception as e:
|
|
logger.debug(f"Failed to load markets for {self.exchange.id}: {e}")
|
|
return False
|
|
|
|
def _normalize_symbol(self, symbol: str) -> Tuple[str, str]:
|
|
"""
|
|
Normalized symbol format, returns (normalized_symbol, base_currency)
|
|
|
|
Handles various input formats:
|
|
- BTC/USDT -> BTC/USDT
|
|
- BTCUSDT -> BTC/USDT
|
|
- BTC/USDT:USDT -> BTC/USDT
|
|
- BTC -> BTC/USDT (default)
|
|
- PI, TRX -> PI/USDT, TRX/USDT
|
|
"""
|
|
if not symbol:
|
|
return '', ''
|
|
|
|
sym = symbol.strip()
|
|
|
|
# Remove swap/futures suffix
|
|
if ':' in sym:
|
|
sym = sym.split(':', 1)[0]
|
|
|
|
sym = sym.upper()
|
|
|
|
# If there is already a separator, parse it directly
|
|
if '/' in sym:
|
|
parts = sym.split('/', 1)
|
|
base = parts[0].strip()
|
|
quote = parts[1].strip() if len(parts) > 1 else ''
|
|
if base and quote:
|
|
return f"{base}/{quote}", base
|
|
|
|
# Try to identify from common quote currencies
|
|
for quote in self.COMMON_QUOTES:
|
|
if sym.endswith(quote) and len(sym) > len(quote):
|
|
base = sym[:-len(quote)]
|
|
if base:
|
|
return f"{base}/{quote}", base
|
|
|
|
# If not recognized, USDT will be used by default.
|
|
return f"{sym}/USDT", sym
|
|
|
|
def _find_valid_symbol(self, base: str, preferred_quote: str = 'USDT') -> Optional[str]:
|
|
"""
|
|
Find valid symbols in the exchange's markets
|
|
|
|
Args:
|
|
base: base currency (e.g. 'PI', 'TRX')
|
|
preferred_quote: preferred quote currency
|
|
|
|
Returns:
|
|
A valid symbol found, or None if not found
|
|
"""
|
|
if not self._ensure_markets_loaded():
|
|
return None
|
|
|
|
markets = self._markets_cache or {}
|
|
if not markets:
|
|
return None
|
|
|
|
# Try different quote currencies by priority
|
|
quotes_to_try = [preferred_quote] + [q for q in self.COMMON_QUOTES if q != preferred_quote]
|
|
|
|
for quote in quotes_to_try:
|
|
candidate = f"{base}/{quote}"
|
|
if candidate in markets:
|
|
market = markets[candidate]
|
|
# Check if the market is active
|
|
if market.get('active', True):
|
|
return candidate
|
|
|
|
return None
|
|
|
|
def _normalize_symbol_for_exchange(self, symbol: str) -> str:
|
|
"""
|
|
Standardize symbols based on exchange characteristics
|
|
|
|
Symbol format requirements for different exchanges:
|
|
- Binance: BTC/USDT (standard format)
|
|
- OKX: BTC/USDT (standard format, but some currencies may not be supported)
|
|
- Coinbase: BTC/USD (usually use USD instead of USDT)
|
|
- Kraken: XBT/USD (BTC is mapped to XBT)
|
|
- Bitfinex: tBTCUST (special format)
|
|
"""
|
|
normalized, base = self._normalize_symbol(symbol)
|
|
|
|
if not normalized or not base:
|
|
return symbol
|
|
|
|
exchange_id = getattr(self.exchange, 'id', '').lower()
|
|
|
|
# Special handling: symbol mapping for certain exchanges
|
|
if exchange_id == 'coinbase':
|
|
# Coinbase usually uses USD instead of USDT
|
|
if normalized.endswith('/USDT'):
|
|
usd_version = normalized.replace('/USDT', '/USD')
|
|
if self._ensure_markets_loaded():
|
|
markets = self._markets_cache or {}
|
|
if usd_version in markets:
|
|
return usd_version
|
|
|
|
# Try to find a valid symbol on the exchange
|
|
if self._ensure_markets_loaded():
|
|
valid_symbol = self._find_valid_symbol(base, normalized.split('/')[1] if '/' in normalized else 'USDT')
|
|
if valid_symbol:
|
|
return valid_symbol
|
|
|
|
return normalized
|
|
|
|
def get_ticker(self, symbol: str) -> Dict[str, Any]:
|
|
"""
|
|
Get latest ticker for a crypto symbol via CCXT.
|
|
|
|
Accepts common formats:
|
|
- BTC/USDT, BTCUSDT, BTC/USDT:USDT
|
|
- PI, TRX (will be normalized and searched across exchanges)
|
|
- Automatically adapt to the symbol format requirements of different exchanges
|
|
"""
|
|
if not symbol or not symbol.strip():
|
|
return {'last': 0, 'symbol': symbol}
|
|
|
|
# normalized notation
|
|
normalized = self._normalize_symbol_for_exchange(symbol)
|
|
|
|
if not normalized:
|
|
logger.warning(f"Failed to normalize symbol: {symbol}")
|
|
return {'last': 0, 'symbol': symbol}
|
|
|
|
# Try to get ticker
|
|
try:
|
|
ticker = self.exchange.fetch_ticker(normalized)
|
|
if ticker and isinstance(ticker, dict):
|
|
return ticker
|
|
except Exception as e:
|
|
error_msg = str(e).lower()
|
|
is_symbol_error = any(keyword in error_msg for keyword in [
|
|
'does not have market symbol',
|
|
'symbol not found',
|
|
'invalid symbol',
|
|
'market does not exist',
|
|
'trading pair not found'
|
|
])
|
|
|
|
if is_symbol_error:
|
|
# Try to find alternative symbols
|
|
base = normalized.split('/')[0] if '/' in normalized else normalized
|
|
if self._ensure_markets_loaded():
|
|
valid_symbol = self._find_valid_symbol(base)
|
|
if valid_symbol and valid_symbol != normalized:
|
|
try:
|
|
logger.debug(f"Trying alternative symbol: {valid_symbol} (original: {symbol}, first attempt: {normalized})")
|
|
ticker = self.exchange.fetch_ticker(valid_symbol)
|
|
if ticker and isinstance(ticker, dict):
|
|
return ticker
|
|
except Exception as e2:
|
|
logger.debug(f"Alternative symbol {valid_symbol} also failed: {e2}")
|
|
|
|
# If all attempts fail, log a warning and return a default value
|
|
logger.warning(
|
|
f"Symbol '{symbol}' (normalized: {normalized}) not found on {self.exchange.id}. "
|
|
f"Error: {str(e)[:100]}"
|
|
)
|
|
|
|
return {'last': 0, 'symbol': symbol}
|
|
|
|
def get_kline(
|
|
self,
|
|
symbol: str,
|
|
timeframe: str,
|
|
limit: int,
|
|
before_time: Optional[int] = None
|
|
) -> List[Dict[str, Any]]:
|
|
"""Get cryptocurrency K-line data"""
|
|
klines = []
|
|
|
|
try:
|
|
ccxt_timeframe = self.TIMEFRAME_MAP.get(timeframe, '1d')
|
|
|
|
# Use a unified symbol normalization method
|
|
symbol_pair = self._normalize_symbol_for_exchange(symbol)
|
|
|
|
if not symbol_pair:
|
|
logger.warning(f"Failed to normalize symbol for K-line: {symbol}")
|
|
return []
|
|
|
|
# logger.info(f"Get cryptocurrency K-line: {symbol_pair}, period: {ccxt_timeframe}, number of bars: {limit}")
|
|
|
|
ohlcv = self._fetch_ohlcv(symbol_pair, ccxt_timeframe, limit, before_time, timeframe)
|
|
|
|
if not ohlcv:
|
|
logger.warning(f"CCXT returned no K-lines: {symbol_pair}")
|
|
return []
|
|
|
|
# Convert data format
|
|
for candle in ohlcv:
|
|
if len(candle) < 6:
|
|
continue
|
|
klines.append(self.format_kline(
|
|
timestamp=int(candle[0] / 1000), # Milliseconds to seconds
|
|
open_price=candle[1],
|
|
high=candle[2],
|
|
low=candle[3],
|
|
close=candle[4],
|
|
volume=candle[5]
|
|
))
|
|
|
|
# Filter and restrict
|
|
klines = self.filter_and_limit(klines, limit, before_time)
|
|
|
|
# Record results
|
|
self.log_result(symbol, klines, timeframe)
|
|
|
|
except Exception as e:
|
|
logger.error(f"Failed to fetch crypto K-lines {symbol}: {str(e)}")
|
|
import traceback
|
|
logger.error(traceback.format_exc())
|
|
|
|
return klines
|
|
|
|
def _fetch_ohlcv(
|
|
self,
|
|
symbol_pair: str,
|
|
ccxt_timeframe: str,
|
|
limit: int,
|
|
before_time: Optional[int],
|
|
timeframe: str
|
|
) -> List:
|
|
"""Obtain OHLCV data (supports paging to obtain complete data)"""
|
|
try:
|
|
if before_time:
|
|
# Calculation time range
|
|
total_seconds = self.calculate_time_range(timeframe, limit)
|
|
end_time = datetime.fromtimestamp(before_time)
|
|
start_time = end_time - timedelta(seconds=total_seconds)
|
|
since = int(start_time.timestamp() * 1000)
|
|
end_ms = before_time * 1000
|
|
|
|
# logger.info(f"Historical data request: since={since//1000}, end={before_time}, time span={total_seconds/86400:.1f} days")
|
|
|
|
# Fetch data in pages until the complete time range is covered
|
|
all_ohlcv = []
|
|
batch_limit = 300 # Coinbase limit is often 300, safer than 1000
|
|
current_since = since
|
|
|
|
while current_since < end_ms:
|
|
batch = self.exchange.fetch_ohlcv(
|
|
symbol_pair,
|
|
ccxt_timeframe,
|
|
since=current_since,
|
|
limit=batch_limit
|
|
)
|
|
|
|
if not batch:
|
|
break
|
|
|
|
all_ohlcv.extend(batch)
|
|
|
|
# The time when the last piece of data is obtained is used as the starting time of the next request
|
|
last_timestamp = batch[-1][0]
|
|
|
|
# If the time of the last data exceeds the end time, or the returned data is less than the requested amount, it means that the acquisition has been completed.
|
|
# if last_timestamp >= end_ms or len(batch) < batch_limit:
|
|
if last_timestamp >= end_ms:
|
|
break
|
|
|
|
# Next time, start from the next time point of the last item
|
|
timeframe_ms = TIMEFRAME_SECONDS.get(timeframe, 86400) * 1000
|
|
current_since = last_timestamp + timeframe_ms
|
|
|
|
# logger.info(f"Getting in paging: {len(all_ohlcv)} items have been obtained, continue from {datetime.fromtimestamp(current_since/1000)}")
|
|
|
|
ohlcv = all_ohlcv
|
|
else:
|
|
ohlcv = self.exchange.fetch_ohlcv(symbol_pair, ccxt_timeframe, limit=limit)
|
|
|
|
# logger.info(f"CCXT returns {len(ohlcv) if ohlcv else 0} pieces of data")
|
|
return ohlcv
|
|
|
|
except Exception as e:
|
|
logger.warning(f"CCXT fetch_ohlcv failed: {str(e)}; trying fallback")
|
|
return self._fetch_ohlcv_fallback(symbol_pair, ccxt_timeframe, limit, before_time, timeframe)
|
|
|
|
def _fetch_ohlcv_fallback(
|
|
self,
|
|
symbol_pair: str,
|
|
ccxt_timeframe: str,
|
|
limit: int,
|
|
before_time: Optional[int],
|
|
timeframe: str
|
|
) -> List:
|
|
"""Alternate acquisition method"""
|
|
try:
|
|
total_seconds = self.calculate_time_range(timeframe, limit)
|
|
|
|
if before_time:
|
|
end_time = datetime.fromtimestamp(before_time)
|
|
start_time = end_time - timedelta(seconds=total_seconds)
|
|
since = int(start_time.timestamp() * 1000)
|
|
else:
|
|
since = int((datetime.now() - timedelta(seconds=total_seconds)).timestamp() * 1000)
|
|
|
|
ohlcv = self.exchange.fetch_ohlcv(symbol_pair, ccxt_timeframe, since=since, limit=limit)
|
|
# logger.info(f"CCXT alternative method returns {len(ohlcv) if ohlcv else 0} pieces of data")
|
|
return ohlcv
|
|
except Exception as e:
|
|
logger.error(f"CCXT fallback method also failed: {str(e)}")
|
|
return []
|
|
|