DinQuant/backend_api_python/app/data_sources/data_manager.py

# -*- coding: utf-8 -*-
"""
===================================
A股数据源管理器 (Data Manager)
===================================

参考 daily_stock_analysis 项目的 DataFetcherManager 实现
统一管理多个A股数据源，实现自动故障切换

数据源优先级：
1. 东方财富 (Eastmoney) - 数据最全，首选
2. 腾讯财经 (Tencent) - 稳定可靠
3. 新浪财经 (Sina) - 轻量级
4. Akshare - 功能丰富，但容易被封
5. yfinance - 国际数据源，兜底
"""

import logging
from typing import Dict, List, Any, Optional, Tuple
from datetime import datetime
import requests

from app.data_sources.circuit_breaker import (
    CircuitBreaker,
    get_ashare_circuit_breaker,
    get_realtime_circuit_breaker
)
from app.data_sources.cache_manager import (
    DataCache,
    get_realtime_cache,
    get_kline_cache,
    generate_kline_cache_key
)
from app.data_sources.rate_limiter import (
    RateLimiter,
    get_eastmoney_limiter,
    get_tencent_limiter,
    get_akshare_limiter,
    get_request_headers,
    retry_with_backoff
)
from app.utils.logger import get_logger
from app.utils.http import get_retry_session

logger = get_logger(__name__)


# ============================================
# 数据源常量
# ============================================

class DataSource:
    """数据源标识"""
    EASTMONEY = "eastmoney"
    TENCENT = "tencent"
    SINA = "sina"
    AKSHARE = "akshare"
    YFINANCE = "yfinance"


# 数据源优先级（数字越小优先级越高）
DATA_SOURCE_PRIORITY = {
    DataSource.EASTMONEY: 0,
    DataSource.TENCENT: 1,
    DataSource.SINA: 2,
    DataSource.AKSHARE: 3,
    DataSource.YFINANCE: 4,
}


# ============================================
# A股数据管理器
# ============================================

class AShareDataManager:
    """
    A股数据源管理器
    
    功能：
    1. 多数据源自动切换（按优先级）
    2. 熔断器保护
    3. 数据缓存
    4. 防封禁策略
    """
    
    # 东方财富 K 线周期映射
    EM_PERIOD_MAP = {
        '1m': '1',
        '5m': '5',
        '15m': '15',
        '30m': '30',
        '1H': '60',
        '4H': '240',
        '1D': '101',
        '1W': '102',
    }
    
    def __init__(self):
        # 熔断器
        self._circuit_breaker = get_ashare_circuit_breaker()
        self._realtime_cb = get_realtime_circuit_breaker()
        
        # 缓存
        self._realtime_cache = get_realtime_cache()
        self._kline_cache = get_kline_cache()
        
        # 限流器
        self._em_limiter = get_eastmoney_limiter()
        self._tencent_limiter = get_tencent_limiter()
        self._akshare_limiter = get_akshare_limiter()
        
        # Akshare 可用性检查
        self._has_akshare = self._check_akshare()
    
    def _check_akshare(self) -> bool:
        """检查 akshare 是否可用"""
        try:
            import akshare
            return True
        except ImportError:
            logger.debug("akshare 未安装，相关功能已禁用")
            return False
    
    def get_kline(
        self,
        symbol: str,
        timeframe: str,
        limit: int,
        before_time: Optional[int] = None,
        use_cache: bool = True
    ) -> Tuple[List[Dict[str, Any]], str]:
        """
        获取K线数据（自动切换数据源）
        
        Args:
            symbol: 股票代码
            timeframe: 时间周期
            limit: 数据条数
            before_time: 获取此时间之前的数据
            use_cache: 是否使用缓存
            
        Returns:
            (K线数据列表, 成功的数据源名称)
        """
        # 检查缓存
        if use_cache:
            cache_key = generate_kline_cache_key(symbol, timeframe, limit, before_time)
            cached = self._kline_cache.get(cache_key)
            if cached:
                logger.debug(f"[缓存命中] K线数据 {symbol}:{timeframe}")
                return cached, "cache"
        
        errors = []
        
        # 按优先级尝试各个数据源
        sources = [
            (DataSource.EASTMONEY, self._fetch_eastmoney_kline),
            (DataSource.TENCENT, self._fetch_tencent_kline),
            (DataSource.AKSHARE, self._fetch_akshare_kline),
            (DataSource.YFINANCE, self._fetch_yfinance_kline),
        ]
        
        for source_name, fetch_func in sources:
            # 检查熔断器
            if not self._circuit_breaker.is_available(source_name):
                logger.debug(f"[熔断] {source_name} 处于熔断状态，跳过")
                continue
            
            # 跳过不可用的 akshare
            if source_name == DataSource.AKSHARE and not self._has_akshare:
                continue
            
            try:
                logger.debug(f"[数据源] 尝试 {source_name} 获取 {symbol}")
                klines = fetch_func(symbol, timeframe, limit, before_time)
                
                if klines:
                    self._circuit_breaker.record_success(source_name)
                    
                    # 更新缓存
                    if use_cache:
                        cache_key = generate_kline_cache_key(symbol, timeframe, limit, before_time)
                        self._kline_cache.set(cache_key, klines)
                    
                    logger.info(f"[数据源] {source_name} 成功获取 {symbol} {len(klines)} 条数据")
                    return klines, source_name
                    
            except Exception as e:
                error_msg = f"{source_name}: {str(e)}"
                errors.append(error_msg)
                self._circuit_breaker.record_failure(source_name, str(e))
                logger.warning(f"[数据源] {error_msg}")
        
        # 所有数据源都失败
        logger.error(f"[数据源] 所有数据源获取 {symbol} 失败: {errors}")
        return [], ""
    
    def _fetch_eastmoney_kline(
        self,
        symbol: str,
        timeframe: str,
        limit: int,
        before_time: Optional[int] = None
    ) -> List[Dict[str, Any]]:
        """使用东方财富获取K线数据"""
        klines = []
        
        period = self.EM_PERIOD_MAP.get(timeframe)
        if not period:
            raise ValueError(f"Eastmoney 不支持时间周期: {timeframe}")
        
        # 限流
        self._em_limiter.wait()
        
        # 确定市场代码
        if symbol.startswith('6'):
            secid = f"1.{symbol}"  # 上海
        else:
            secid = f"0.{symbol}"  # 深圳
        
        url = "https://push2his.eastmoney.com/api/qt/stock/kline/get"
        params = {
            'secid': secid,
            'fields1': 'f1,f2,f3,f4,f5,f6',
            'fields2': 'f51,f52,f53,f54,f55,f56,f57',
            'klt': period,
            'fqt': '1',  # 前复权
            'end': '20500101',
            'lmt': limit,
        }
        
        headers = get_request_headers(referer='https://quote.eastmoney.com/')
        
        session = get_retry_session()
        response = session.get(url, params=params, headers=headers, timeout=15)
        
        if response.status_code != 200:
            raise ConnectionError(f"HTTP {response.status_code}")
        
        data = response.json()
        
        if data.get('data') and data['data'].get('klines'):
            for line in data['data']['klines']:
                try:
                    parts = line.split(',')
                    if len(parts) >= 6:
                        time_str = parts[0]
                        if ' ' in time_str:
                            dt = datetime.strptime(time_str, '%Y-%m-%d %H:%M')
                        else:
                            dt = datetime.strptime(time_str, '%Y-%m-%d')
                        
                        klines.append({
                            'time': int(dt.timestamp()),
                            'open': round(float(parts[1]), 4),
                            'high': round(float(parts[3]), 4),
                            'low': round(float(parts[4]), 4),
                            'close': round(float(parts[2]), 4),
                            'volume': round(float(parts[5]), 2)
                        })
                except (ValueError, IndexError):
                    continue
        
        # 过滤和排序
        klines.sort(key=lambda x: x['time'])
        if before_time:
            klines = [k for k in klines if k['time'] < before_time]
        if len(klines) > limit:
            klines = klines[-limit:]
        
        return klines
    
    def _fetch_tencent_kline(
        self,
        symbol: str,
        timeframe: str,
        limit: int,
        before_time: Optional[int] = None
    ) -> List[Dict[str, Any]]:
        """使用腾讯财经获取K线数据"""
        # 腾讯周期映射
        period_map = {
            '1m': 1, '5m': 5, '15m': 15, '30m': 30,
            '1H': 60, '1D': 'day', '1W': 'week'
        }
        
        period = period_map.get(timeframe)
        if period is None:
            raise ValueError(f"腾讯财经不支持时间周期: {timeframe}")
        
        # 转换代码格式
        if symbol.startswith('6'):
            tencent_symbol = f"sh{symbol}"
        elif symbol.startswith('0') or symbol.startswith('3'):
            tencent_symbol = f"sz{symbol}"
        else:
            tencent_symbol = f"bj{symbol}"
        
        # 限流
        self._tencent_limiter.wait()
        
        # 构建URL
        if isinstance(period, int):
            url = f"http://ifzq.gtimg.cn/appstock/app/kline/mkline?param={tencent_symbol},m{period},,{limit}"
        else:
            url = f"http://web.ifzq.gtimg.cn/appstock/app/fqkline/get?param={tencent_symbol},{period},,,{limit},qfq"
        
        response = requests.get(url, timeout=10)
        
        if response.status_code != 200:
            raise ConnectionError(f"HTTP {response.status_code}")
        
        data = response.json()
        klines = []
        
        if data.get('code') == 0 and 'data' in data:
            stock_data = data['data'].get(tencent_symbol)
            if stock_data:
                if isinstance(period, int):
                    candles = stock_data.get(f'm{period}', [])
                else:
                    candles = stock_data.get('qfqday', stock_data.get('day', []))
                
                for candle in candles:
                    if len(candle) >= 5:
                        try:
                            time_str = str(candle[0])
                            if len(time_str) == 12:
                                dt = datetime.strptime(time_str, '%Y%m%d%H%M')
                            elif len(time_str) == 10:
                                dt = datetime.strptime(time_str, '%Y-%m-%d')
                            else:
                                continue
                            
                            klines.append({
                                'time': int(dt.timestamp()),
                                'open': round(float(candle[1]), 4),
                                'high': round(float(candle[3]), 4),
                                'low': round(float(candle[4]), 4),
                                'close': round(float(candle[2]), 4),
                                'volume': round(float(candle[5]), 2) if len(candle) > 5 else 0
                            })
                        except (ValueError, IndexError):
                            continue
        
        # 过滤和排序
        klines.sort(key=lambda x: x['time'])
        if before_time:
            klines = [k for k in klines if k['time'] < before_time]
        if len(klines) > limit:
            klines = klines[-limit:]
        
        return klines
    
    def _fetch_akshare_kline(
        self,
        symbol: str,
        timeframe: str,
        limit: int,
        before_time: Optional[int] = None
    ) -> List[Dict[str, Any]]:
        """使用 Akshare 获取K线数据"""
        if not self._has_akshare:
            raise RuntimeError("akshare 未安装")
        
        import akshare as ak
        from datetime import timedelta
        
        # Akshare 只支持日线/周线
        period_map = {'1D': 'daily', '1W': 'weekly'}
        period = period_map.get(timeframe)
        if not period:
            raise ValueError(f"Akshare 不支持时间周期: {timeframe}")
        
        # 限流
        self._akshare_limiter.wait()
        
        # 计算日期范围
        if before_time:
            end_date = datetime.fromtimestamp(before_time).strftime('%Y%m%d')
        else:
            end_date = datetime.now().strftime('%Y%m%d')
        
        days = limit * 2 if timeframe == '1D' else limit * 10
        start_date = (datetime.now() - timedelta(days=days)).strftime('%Y%m%d')
        
        df = ak.stock_zh_a_hist(
            symbol=symbol,
            period=period,
            start_date=start_date,
            end_date=end_date,
            adjust="qfq"
        )
        
        klines = []
        if df is not None and not df.empty:
            df = df.tail(limit)
            for _, row in df.iterrows():
                ts = int(datetime.strptime(str(row['日期']), '%Y-%m-%d').timestamp())
                klines.append({
                    'time': ts,
                    'open': round(float(row['开盘']), 4),
                    'high': round(float(row['最高']), 4),
                    'low': round(float(row['最低']), 4),
                    'close': round(float(row['收盘']), 4),
                    'volume': round(float(row['成交量']), 2)
                })
        
        return klines
    
    def _fetch_yfinance_kline(
        self,
        symbol: str,
        timeframe: str,
        limit: int,
        before_time: Optional[int] = None
    ) -> List[Dict[str, Any]]:
        """使用 yfinance 获取K线数据（兜底）"""
        import yfinance as yf
        
        # 转换为 Yahoo 格式
        if symbol.startswith('6'):
            yahoo_symbol = f"{symbol}.SS"
        elif symbol.startswith('0') or symbol.startswith('3'):
            yahoo_symbol = f"{symbol}.SZ"
        else:
            yahoo_symbol = f"{symbol}.SS"
        
        # 周期映射
        period_map = {
            '1D': ('1d', f'{limit}d'),
            '1W': ('1wk', f'{limit * 7}d'),
            '1H': ('1h', f'{limit}d'),
        }
        
        interval, period = period_map.get(timeframe, ('1d', f'{limit}d'))
        
        ticker = yf.Ticker(yahoo_symbol)
        df = ticker.history(period=period, interval=interval)
        
        klines = []
        if df is not None and not df.empty:
            df = df.tail(limit)
            for idx, row in df.iterrows():
                ts = int(idx.timestamp())
                klines.append({
                    'time': ts,
                    'open': round(float(row['Open']), 4),
                    'high': round(float(row['High']), 4),
                    'low': round(float(row['Low']), 4),
                    'close': round(float(row['Close']), 4),
                    'volume': round(float(row['Volume']), 2)
                })
        
        return klines
    
    def get_realtime_quote(
        self,
        symbol: str,
        use_cache: bool = True
    ) -> Tuple[Dict[str, Any], str]:
        """
        获取实时报价（自动切换数据源）
        
        Args:
            symbol: 股票代码
            use_cache: 是否使用缓存
            
        Returns:
            (报价数据字典, 成功的数据源名称)
        """
        # 检查缓存
        if use_cache:
            cached = self._realtime_cache.get(f"quote:{symbol}")
            if cached:
                return cached, "cache"
        
        errors = []
        
        # 按优先级尝试各个数据源
        sources = [
            (DataSource.EASTMONEY, self._fetch_eastmoney_quote),
            (DataSource.TENCENT, self._fetch_tencent_quote),
            (DataSource.AKSHARE, self._fetch_akshare_quote),
        ]
        
        for source_name, fetch_func in sources:
            if not self._realtime_cb.is_available(source_name):
                continue
            
            if source_name == DataSource.AKSHARE and not self._has_akshare:
                continue
            
            try:
                quote = fetch_func(symbol)
                if quote and quote.get('last', 0) > 0:
                    self._realtime_cb.record_success(source_name)
                    
                    # 更新缓存
                    if use_cache:
                        self._realtime_cache.set(f"quote:{symbol}", quote, ttl=60.0)
                    
                    return quote, source_name
                    
            except Exception as e:
                errors.append(f"{source_name}: {str(e)}")
                self._realtime_cb.record_failure(source_name, str(e))
        
        logger.warning(f"[实时报价] 所有数据源获取 {symbol} 失败")
        return {'last': 0, 'symbol': symbol}, ""
    
    def _fetch_eastmoney_quote(self, symbol: str) -> Dict[str, Any]:
        """使用东方财富获取实时报价"""
        if symbol.startswith('6'):
            secid = f"1.{symbol}"
        else:
            secid = f"0.{symbol}"
        
        url = "https://push2.eastmoney.com/api/qt/stock/get"
        params = {
            'secid': secid,
            'fields': 'f43,f44,f45,f46,f47,f48,f57,f58,f60,f169,f170',
        }
        
        self._em_limiter.wait()
        
        session = get_retry_session()
        response = session.get(url, params=params, headers=get_request_headers(), timeout=10)
        
        if response.status_code == 200:
            data = response.json()
            if data and data.get('data'):
                d = data['data']
                last_price = d.get('f43', 0)
                if last_price and last_price > 0:
                    divisor = 100 if last_price > 1000 else 1
                    return {
                        'last': last_price / divisor,
                        'high': d.get('f44', 0) / divisor,
                        'low': d.get('f45', 0) / divisor,
                        'open': d.get('f46', 0) / divisor,
                        'previousClose': d.get('f60', 0) / divisor,
                        'change': d.get('f169', 0) / divisor,
                        'changePercent': d.get('f170', 0) / 100
                    }
        
        return {}
    
    def _fetch_tencent_quote(self, symbol: str) -> Dict[str, Any]:
        """使用腾讯财经获取实时报价"""
        if symbol.startswith('6'):
            tencent_symbol = f"sh{symbol}"
        else:
            tencent_symbol = f"sz{symbol}"
        
        url = f"http://qt.gtimg.cn/q={tencent_symbol}"
        
        self._tencent_limiter.wait()
        
        response = requests.get(url, timeout=10)
        content = response.content.decode('gbk', errors='ignore')
        
        if '="' in content:
            data_str = content.split('="')[1].strip('";\n')
            if data_str:
                parts = data_str.split('~')
                if len(parts) > 32:
                    return {
                        'last': float(parts[3]) if parts[3] else 0,
                        'change': float(parts[31]) if parts[31] else 0,
                        'changePercent': float(parts[32]) if parts[32] else 0,
                        'high': float(parts[33]) if len(parts) > 33 and parts[33] else 0,
                        'low': float(parts[34]) if len(parts) > 34 and parts[34] else 0,
                        'open': float(parts[5]) if len(parts) > 5 and parts[5] else 0,
                        'previousClose': float(parts[4]) if parts[4] else 0
                    }
        
        return {}
    
    def _fetch_akshare_quote(self, symbol: str) -> Dict[str, Any]:
        """使用 Akshare 获取实时报价"""
        if not self._has_akshare:
            return {}
        
        import akshare as ak
        
        self._akshare_limiter.wait()
        
        df = ak.stock_zh_a_spot_em()
        if df is not None and not df.empty:
            row = df[df['代码'] == symbol]
            if not row.empty:
                row = row.iloc[0]
                return {
                    'last': float(row.get('最新价', 0) or 0),
                    'change': float(row.get('涨跌额', 0) or 0),
                    'changePercent': float(row.get('涨跌幅', 0) or 0),
                    'high': float(row.get('最高', 0) or 0),
                    'low': float(row.get('最低', 0) or 0),
                    'open': float(row.get('今开', 0) or 0),
                    'previousClose': float(row.get('昨收', 0) or 0)
                }
        
        return {}
    
    def get_status(self) -> Dict[str, Any]:
        """获取数据管理器状态"""
        return {
            'circuit_breaker': self._circuit_breaker.get_status(),
            'realtime_circuit_breaker': self._realtime_cb.get_status(),
            'realtime_cache_stats': self._realtime_cache.stats(),
            'kline_cache_stats': self._kline_cache.stats(),
            'has_akshare': self._has_akshare,
        }


# ============================================
# 全局实例
# ============================================

_ashare_data_manager: Optional[AShareDataManager] = None


def get_ashare_data_manager() -> AShareDataManager:
    """获取A股数据管理器单例"""
    global _ashare_data_manager
    if _ashare_data_manager is None:
        _ashare_data_manager = AShareDataManager()
    return _ashare_data_manager