TIANHE
@@ -0,0 +1,8 @@
|
||||
"""
|
||||
数据源模块
|
||||
支持多种市场的K线数据获取
|
||||
"""
|
||||
from app.data_sources.factory import DataSourceFactory
|
||||
|
||||
__all__ = ['DataSourceFactory']
|
||||
|
||||
@@ -0,0 +1,154 @@
|
||||
"""
|
||||
数据源基类
|
||||
定义统一的数据源接口
|
||||
"""
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import Dict, List, Any, Optional
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
from app.utils.logger import get_logger
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
|
||||
# K线周期映射（秒数）
|
||||
TIMEFRAME_SECONDS = {
|
||||
'1m': 60,
|
||||
'5m': 300,
|
||||
'15m': 900,
|
||||
'30m': 1800,
|
||||
'1H': 3600,
|
||||
'4H': 14400,
|
||||
'1D': 86400,
|
||||
'1W': 604800
|
||||
}
|
||||
|
||||
|
||||
class BaseDataSource(ABC):
|
||||
"""数据源基类"""
|
||||
|
||||
name: str = "base"
|
||||
|
||||
@abstractmethod
|
||||
def get_kline(
|
||||
self,
|
||||
symbol: str,
|
||||
timeframe: str,
|
||||
limit: int,
|
||||
before_time: Optional[int] = None
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
获取K线数据
|
||||
|
||||
Args:
|
||||
symbol: 交易对/股票代码
|
||||
timeframe: 时间周期 (1m, 5m, 15m, 30m, 1H, 4H, 1D, 1W)
|
||||
limit: 数据条数
|
||||
before_time: 获取此时间之前的数据（Unix时间戳，秒）
|
||||
|
||||
Returns:
|
||||
K线数据列表，格式:
|
||||
[{"time": int, "open": float, "high": float, "low": float, "close": float, "volume": float}, ...]
|
||||
"""
|
||||
pass
|
||||
|
||||
def get_ticker(self, symbol: str) -> Dict[str, Any]:
|
||||
"""
|
||||
Get latest ticker for a symbol (best-effort).
|
||||
|
||||
This is an optional interface used by the strategy executor for fetching current price.
|
||||
Implementations may return a dict compatible with CCXT `fetch_ticker` shape (e.g. {'last': ...}).
|
||||
"""
|
||||
raise NotImplementedError("get_ticker is not implemented for this data source")
|
||||
|
||||
def format_kline(
|
||||
self,
|
||||
timestamp: int,
|
||||
open_price: float,
|
||||
high: float,
|
||||
low: float,
|
||||
close: float,
|
||||
volume: float
|
||||
) -> Dict[str, Any]:
|
||||
"""格式化单条K线数据"""
|
||||
return {
|
||||
'time': timestamp,
|
||||
'open': round(float(open_price), 4),
|
||||
'high': round(float(high), 4),
|
||||
'low': round(float(low), 4),
|
||||
'close': round(float(close), 4),
|
||||
'volume': round(float(volume), 2)
|
||||
}
|
||||
|
||||
def calculate_time_range(
|
||||
self,
|
||||
timeframe: str,
|
||||
limit: int,
|
||||
buffer_ratio: float = 1.2
|
||||
) -> int:
|
||||
"""
|
||||
计算获取指定数量K线所需的时间范围（秒）
|
||||
|
||||
Args:
|
||||
timeframe: 时间周期
|
||||
limit: K线数量
|
||||
buffer_ratio: 缓冲系数
|
||||
|
||||
Returns:
|
||||
时间范围（秒）
|
||||
"""
|
||||
seconds_per_candle = TIMEFRAME_SECONDS.get(timeframe, 86400)
|
||||
return int(seconds_per_candle * limit * buffer_ratio)
|
||||
|
||||
def filter_and_limit(
|
||||
self,
|
||||
klines: List[Dict[str, Any]],
|
||||
limit: int,
|
||||
before_time: Optional[int] = None
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
过滤和限制K线数据
|
||||
|
||||
Args:
|
||||
klines: K线数据列表
|
||||
limit: 最大数量
|
||||
before_time: 过滤此时间之后的数据
|
||||
|
||||
Returns:
|
||||
处理后的K线数据
|
||||
"""
|
||||
# 按时间排序
|
||||
klines.sort(key=lambda x: x['time'])
|
||||
|
||||
# 过滤时间
|
||||
if before_time:
|
||||
klines = [k for k in klines if k['time'] < before_time]
|
||||
|
||||
# 限制数量（取最新的）
|
||||
if len(klines) > limit:
|
||||
klines = klines[-limit:]
|
||||
|
||||
return klines
|
||||
|
||||
def log_result(
|
||||
self,
|
||||
symbol: str,
|
||||
klines: List[Dict[str, Any]],
|
||||
timeframe: str
|
||||
):
|
||||
"""记录获取结果日志"""
|
||||
if klines:
|
||||
latest_time = datetime.fromtimestamp(klines[-1]['time'])
|
||||
time_diff = (datetime.now() - latest_time).total_seconds()
|
||||
# logger.info(
|
||||
# f"{self.name}: {symbol} 获取 {len(klines)} 条数据, "
|
||||
# f"最新时间: {latest_time}, 延迟: {time_diff:.0f}秒"
|
||||
# )
|
||||
|
||||
# 检查数据是否过旧
|
||||
max_diff = TIMEFRAME_SECONDS.get(timeframe, 3600) * 2
|
||||
if time_diff > max_diff:
|
||||
logger.warning(f"Warning: {symbol} data is delayed ({time_diff:.0f}s)")
|
||||
else:
|
||||
logger.warning(f"{self.name}: no data for {symbol}")
|
||||
|
||||
@@ -0,0 +1,615 @@
|
||||
"""
|
||||
CN/HK stock data source.
|
||||
Supports A-Share and H-Share with multiple public sources.
|
||||
Priority (AShare): Eastmoney (intraday/daily) > yfinance (daily) > akshare (daily, optional).
|
||||
Priority (HShare): Tencent (intraday) > Eastmoney/Tencent (daily) > yfinance (daily) > akshare (daily, optional).
|
||||
"""
|
||||
import json
|
||||
from typing import Dict, List, Any, Optional
|
||||
from datetime import datetime, timedelta
|
||||
import requests
|
||||
|
||||
import yfinance as yf
|
||||
|
||||
from app.data_sources.base import BaseDataSource
|
||||
from app.data_sources.us_stock import USStockDataSource
|
||||
from app.utils.logger import get_logger
|
||||
from app.utils.http import get_retry_session
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
# Optional dependency: akshare
|
||||
try:
|
||||
import akshare as ak # type: ignore
|
||||
HAS_AKSHARE = True
|
||||
logger.debug("akshare is available")
|
||||
except ImportError:
|
||||
HAS_AKSHARE = False
|
||||
# Keep it quiet to avoid noisy startup logs on Windows.
|
||||
logger.debug("akshare is not installed; akshare-based features are disabled")
|
||||
|
||||
|
||||
class TencentDataMixin:
|
||||
"""Tencent quote API mixin (mostly for H-Share and legacy fallback)."""
|
||||
|
||||
# 腾讯 K 线周期映射（注意：腾讯分钟级接口不支持240分钟，4H需要特殊处理）
|
||||
TENCENT_PERIOD_MAP = {
|
||||
'1m': 1,
|
||||
'5m': 5,
|
||||
'15m': 15,
|
||||
'30m': 30,
|
||||
'1H': 60,
|
||||
'1D': 'day',
|
||||
'1W': 'week'
|
||||
}
|
||||
|
||||
def _fetch_tencent_kline(
|
||||
self,
|
||||
symbol_code: str,
|
||||
timeframe: str,
|
||||
limit: int
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
使用腾讯财经接口获取K线数据
|
||||
|
||||
Args:
|
||||
symbol_code: 腾讯格式的代码 (sh600000, sz000001, hk00700)
|
||||
timeframe: 时间周期
|
||||
limit: 数据条数
|
||||
"""
|
||||
klines = []
|
||||
|
||||
# 4H 需要特殊处理：获取1H数据然后聚合
|
||||
if timeframe == '4H':
|
||||
return self._fetch_and_aggregate_4h(symbol_code, limit)
|
||||
|
||||
try:
|
||||
period = self.TENCENT_PERIOD_MAP.get(timeframe)
|
||||
if period is None:
|
||||
logger.warning(f"Unsupported timeframe: {timeframe}")
|
||||
return []
|
||||
|
||||
# 构建请求URL
|
||||
if isinstance(period, int):
|
||||
# 分钟级数据
|
||||
url = f"http://ifzq.gtimg.cn/appstock/app/kline/mkline?param={symbol_code},m{period},,{limit}"
|
||||
else:
|
||||
# 日线/周线数据
|
||||
url = f"http://web.ifzq.gtimg.cn/appstock/app/fqkline/get?param={symbol_code},{period},,,{limit},qfq"
|
||||
|
||||
# logger.info(f"腾讯财经请求: {symbol_code}, 周期: {timeframe}, URL: {url[:80]}...")
|
||||
|
||||
session = get_retry_session()
|
||||
response = session.get(url, timeout=10)
|
||||
|
||||
if response.status_code != 200:
|
||||
logger.warning(f"Tencent quote returned status: {response.status_code}")
|
||||
return []
|
||||
|
||||
data = response.json()
|
||||
|
||||
# 解析响应数据
|
||||
if data.get('code') == 0 and 'data' in data:
|
||||
stock_data = data['data'].get(symbol_code)
|
||||
if stock_data:
|
||||
# 分钟级数据格式
|
||||
if isinstance(period, int):
|
||||
candles = stock_data.get(f'm{period}', [])
|
||||
else:
|
||||
# 日线/周线数据格式
|
||||
candles = stock_data.get('qfqday', stock_data.get('day', []))
|
||||
|
||||
for candle in candles:
|
||||
if len(candle) >= 5:
|
||||
# 解析时间
|
||||
time_str = str(candle[0])
|
||||
try:
|
||||
if len(time_str) == 12: # 分钟级: 202411301430
|
||||
dt = datetime.strptime(time_str, '%Y%m%d%H%M')
|
||||
elif len(time_str) == 10: # 日线: 2024-11-30
|
||||
dt = datetime.strptime(time_str, '%Y-%m-%d')
|
||||
else:
|
||||
continue
|
||||
|
||||
klines.append(self.format_kline(
|
||||
timestamp=int(dt.timestamp()),
|
||||
open_price=float(candle[1]),
|
||||
high=float(candle[3]),
|
||||
low=float(candle[4]),
|
||||
close=float(candle[2]),
|
||||
volume=float(candle[5]) if len(candle) > 5 else 0
|
||||
))
|
||||
except (ValueError, IndexError) as e:
|
||||
logger.debug(f"Failed to parse kline candle: {candle}, error: {e}")
|
||||
continue
|
||||
|
||||
# logger.info(f"腾讯财经返回 {len(klines)} 条数据")
|
||||
else:
|
||||
logger.warning(f"Tencent quote returned unexpected data: code={data.get('code')}")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Tencent quote fetch failed: {e}")
|
||||
import traceback
|
||||
logger.error(traceback.format_exc())
|
||||
|
||||
return klines
|
||||
|
||||
def _fetch_and_aggregate_4h(self, symbol_code: str, limit: int) -> List[Dict[str, Any]]:
|
||||
"""获取1H数据并聚合为4H"""
|
||||
# 获取足够多的1H数据
|
||||
hour_klines = self._fetch_tencent_kline(symbol_code, '1H', limit * 4 + 10)
|
||||
|
||||
if not hour_klines:
|
||||
return []
|
||||
|
||||
# 按4小时聚合
|
||||
aggregated = []
|
||||
i = 0
|
||||
while i < len(hour_klines):
|
||||
# 取4根K线
|
||||
batch = hour_klines[i:i+4]
|
||||
if len(batch) < 4:
|
||||
break
|
||||
|
||||
aggregated.append(self.format_kline(
|
||||
timestamp=batch[0]['time'],
|
||||
open_price=batch[0]['open'],
|
||||
high=max(k['high'] for k in batch),
|
||||
low=min(k['low'] for k in batch),
|
||||
close=batch[-1]['close'],
|
||||
volume=sum(k['volume'] for k in batch)
|
||||
))
|
||||
i += 4
|
||||
|
||||
# logger.info(f"聚合生成 {len(aggregated)} 条 4H 数据")
|
||||
return aggregated[-limit:] if len(aggregated) > limit else aggregated
|
||||
|
||||
|
||||
class AShareDataSource(BaseDataSource, TencentDataMixin):
|
||||
"""A-Share data source."""
|
||||
|
||||
name = "AShare"
|
||||
|
||||
# akshare 时间周期映射
|
||||
AKSHARE_PERIOD_MAP = {
|
||||
'1D': 'daily',
|
||||
'1W': 'weekly'
|
||||
}
|
||||
|
||||
# 东方财富 K 线周期映射
|
||||
EM_PERIOD_MAP = {
|
||||
'1m': '1',
|
||||
'5m': '5',
|
||||
'15m': '15',
|
||||
'30m': '30',
|
||||
'1H': '60',
|
||||
'4H': '240',
|
||||
'1D': '101',
|
||||
'1W': '102',
|
||||
}
|
||||
|
||||
def __init__(self):
|
||||
self.us_stock_source = USStockDataSource()
|
||||
|
||||
def get_kline(
|
||||
self,
|
||||
symbol: str,
|
||||
timeframe: str,
|
||||
limit: int,
|
||||
before_time: Optional[int] = None
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""Fetch A-Share Kline data."""
|
||||
klines = []
|
||||
|
||||
# Prefer Eastmoney (supports most intraday timeframes)
|
||||
klines = self._fetch_eastmoney_ashare(symbol, timeframe, limit)
|
||||
if klines:
|
||||
klines = self.filter_and_limit(klines, limit, before_time)
|
||||
self.log_result(symbol, klines, timeframe)
|
||||
return klines
|
||||
|
||||
# Fallback: yfinance (daily/weekly)
|
||||
if timeframe in ('1D', '1W'):
|
||||
yahoo_symbol = self._to_yahoo_symbol(symbol)
|
||||
if yahoo_symbol:
|
||||
# logger.info(f"尝试使用 yfinance 获取A股: {yahoo_symbol}")
|
||||
klines = self.us_stock_source.get_kline(yahoo_symbol, timeframe, limit, before_time)
|
||||
if klines:
|
||||
# logger.info(f"yfinance 成功获取 {len(klines)} 条A股数据")
|
||||
return klines
|
||||
|
||||
# Fallback: akshare (daily/weekly)
|
||||
if HAS_AKSHARE and timeframe in self.AKSHARE_PERIOD_MAP:
|
||||
klines = self._fetch_akshare(symbol, timeframe, limit, before_time)
|
||||
if klines:
|
||||
return klines
|
||||
|
||||
logger.warning(f"AShare {symbol} data fetch failed")
|
||||
return klines
|
||||
|
||||
def _to_tencent_symbol(self, symbol: str) -> Optional[str]:
|
||||
"""转换为腾讯财经格式"""
|
||||
if symbol.startswith('6'):
|
||||
return f"sh{symbol}"
|
||||
elif symbol.startswith('0') or symbol.startswith('3'):
|
||||
return f"sz{symbol}"
|
||||
elif symbol.startswith('4') or symbol.startswith('8'):
|
||||
return f"bj{symbol}" # 北交所
|
||||
return None
|
||||
|
||||
def _to_yahoo_symbol(self, symbol: str) -> Optional[str]:
|
||||
"""转换为 Yahoo Finance 格式"""
|
||||
if symbol.startswith('6'):
|
||||
return f"{symbol}.SS"
|
||||
elif symbol.startswith('0') or symbol.startswith('3'):
|
||||
return f"{symbol}.SZ"
|
||||
elif symbol.startswith('4') or symbol.startswith('8'):
|
||||
return f"{symbol}.BJ"
|
||||
return None
|
||||
|
||||
def _fetch_eastmoney_ashare(
|
||||
self,
|
||||
symbol: str,
|
||||
timeframe: str,
|
||||
limit: int
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""使用东方财富获取A股数据"""
|
||||
klines = []
|
||||
|
||||
period = self.EM_PERIOD_MAP.get(timeframe)
|
||||
if not period:
|
||||
logger.warning(f"Eastmoney unsupported timeframe: {timeframe}")
|
||||
return []
|
||||
|
||||
try:
|
||||
# 确定市场代码: 上海=1, 深圳=0, 北交所=0
|
||||
if symbol.startswith('6'):
|
||||
secid = f"1.{symbol}"
|
||||
else:
|
||||
secid = f"0.{symbol}"
|
||||
|
||||
# 东方财富K线接口
|
||||
url = f"https://push2his.eastmoney.com/api/qt/stock/kline/get"
|
||||
params = {
|
||||
'secid': secid,
|
||||
'fields1': 'f1,f2,f3,f4,f5,f6',
|
||||
'fields2': 'f51,f52,f53,f54,f55,f56,f57',
|
||||
'klt': period,
|
||||
'fqt': '1', # 前复权
|
||||
'end': '20500101',
|
||||
'lmt': limit,
|
||||
}
|
||||
|
||||
# logger.info(f"东方财富A股请求: {symbol}, 周期: {timeframe}")
|
||||
|
||||
# 添加浏览器请求头
|
||||
headers = {
|
||||
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
|
||||
'Referer': 'https://quote.eastmoney.com/',
|
||||
'Accept': 'application/json, text/plain, */*',
|
||||
'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8',
|
||||
}
|
||||
|
||||
session = get_retry_session()
|
||||
response = session.get(url, params=params, headers=headers, timeout=15)
|
||||
|
||||
if response.status_code != 200:
|
||||
logger.warning(f"Eastmoney HTTP status: {response.status_code}")
|
||||
return []
|
||||
|
||||
data = response.json()
|
||||
|
||||
# 解析响应
|
||||
if data.get('data') and data['data'].get('klines'):
|
||||
for line in data['data']['klines']:
|
||||
try:
|
||||
parts = line.split(',')
|
||||
if len(parts) >= 6:
|
||||
time_str = parts[0]
|
||||
if ' ' in time_str:
|
||||
dt = datetime.strptime(time_str, '%Y-%m-%d %H:%M')
|
||||
else:
|
||||
dt = datetime.strptime(time_str, '%Y-%m-%d')
|
||||
|
||||
klines.append(self.format_kline(
|
||||
timestamp=int(dt.timestamp()),
|
||||
open_price=float(parts[1]),
|
||||
high=float(parts[3]),
|
||||
low=float(parts[4]),
|
||||
close=float(parts[2]),
|
||||
volume=float(parts[5])
|
||||
))
|
||||
except (ValueError, IndexError) as e:
|
||||
logger.debug(f"Failed to parse Eastmoney data line: {line}, error: {e}")
|
||||
continue
|
||||
|
||||
# logger.info(f"东方财富返回 {len(klines)} 条A股数据")
|
||||
else:
|
||||
logger.warning("Eastmoney returned no data")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Eastmoney A-share fetch failed: {e}")
|
||||
import traceback
|
||||
logger.error(traceback.format_exc())
|
||||
|
||||
return klines
|
||||
|
||||
def _fetch_akshare(
|
||||
self,
|
||||
symbol: str,
|
||||
timeframe: str,
|
||||
limit: int,
|
||||
before_time: Optional[int]
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""使用 akshare 获取数据"""
|
||||
klines = []
|
||||
|
||||
try:
|
||||
period = self.AKSHARE_PERIOD_MAP.get(timeframe, 'daily')
|
||||
|
||||
# 计算日期范围
|
||||
if before_time:
|
||||
end_date = datetime.fromtimestamp(before_time).strftime('%Y%m%d')
|
||||
else:
|
||||
end_date = datetime.now().strftime('%Y%m%d')
|
||||
|
||||
days = limit * 2 if timeframe == '1D' else limit * 10
|
||||
start_date = (datetime.now() - timedelta(days=days)).strftime('%Y%m%d')
|
||||
|
||||
# logger.info(f"使用 akshare 获取A股: {symbol}, 周期: {period}")
|
||||
|
||||
df = ak.stock_zh_a_hist(
|
||||
symbol=symbol,
|
||||
period=period,
|
||||
start_date=start_date,
|
||||
end_date=end_date,
|
||||
adjust="qfq" # 前复权
|
||||
)
|
||||
|
||||
if df is not None and not df.empty:
|
||||
df = df.tail(limit)
|
||||
for _, row in df.iterrows():
|
||||
ts = int(datetime.strptime(str(row['日期']), '%Y-%m-%d').timestamp())
|
||||
klines.append(self.format_kline(
|
||||
timestamp=ts,
|
||||
open_price=row['开盘'],
|
||||
high=row['最高'],
|
||||
low=row['最低'],
|
||||
close=row['收盘'],
|
||||
volume=row['成交量']
|
||||
))
|
||||
# logger.info(f"akshare 返回 {len(klines)} 条A股数据")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Akshare A-share fetch failed: {e}")
|
||||
import traceback
|
||||
logger.error(traceback.format_exc())
|
||||
|
||||
return klines
|
||||
|
||||
|
||||
class HShareDataSource(BaseDataSource, TencentDataMixin):
|
||||
"""港股数据源"""
|
||||
|
||||
name = "HShare"
|
||||
|
||||
def __init__(self):
|
||||
self.us_stock_source = USStockDataSource()
|
||||
|
||||
def get_kline(
|
||||
self,
|
||||
symbol: str,
|
||||
timeframe: str,
|
||||
limit: int,
|
||||
before_time: Optional[int] = None
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""获取港股K线数据"""
|
||||
klines = []
|
||||
|
||||
# 方案1: 腾讯财经 (港股日线/周线首选，稳定可靠)
|
||||
if timeframe in ('1D', '1W'):
|
||||
tencent_symbol = self._to_tencent_symbol(symbol)
|
||||
if tencent_symbol:
|
||||
# logger.info(f"尝试使用腾讯财经获取港股: {tencent_symbol}")
|
||||
klines = self._fetch_tencent_kline(tencent_symbol, timeframe, limit)
|
||||
if klines:
|
||||
klines = self.filter_and_limit(klines, limit, before_time)
|
||||
self.log_result(symbol, klines, timeframe)
|
||||
return klines
|
||||
|
||||
# 方案2: 东方财富 (支持所有周期，但可能有地域限制)
|
||||
klines = self._fetch_eastmoney_kline(symbol, timeframe, limit)
|
||||
if klines:
|
||||
klines = self.filter_and_limit(klines, limit, before_time)
|
||||
self.log_result(symbol, klines, timeframe)
|
||||
return klines
|
||||
|
||||
# 方案3: 尝试 yfinance (日线级别备选)
|
||||
if timeframe in ('1D', '1W'):
|
||||
yahoo_symbol = self._to_yahoo_symbol(symbol)
|
||||
if yahoo_symbol:
|
||||
# logger.info(f"尝试使用 yfinance 获取港股: {yahoo_symbol}")
|
||||
klines = self.us_stock_source.get_kline(yahoo_symbol, timeframe, limit, before_time)
|
||||
if klines:
|
||||
# logger.info(f"yfinance 成功获取 {len(klines)} 条港股数据")
|
||||
return klines
|
||||
|
||||
# 方案4: 尝试 akshare (日线级别)
|
||||
if HAS_AKSHARE and timeframe in ('1D', '1W'):
|
||||
klines = self._fetch_akshare(symbol, timeframe, limit, before_time)
|
||||
if klines:
|
||||
return klines
|
||||
|
||||
# 分钟级数据获取失败提示
|
||||
if timeframe not in ('1D', '1W'):
|
||||
logger.warning(f"HK stock {symbol}: minute-level data is not supported (data source limitations)")
|
||||
else:
|
||||
logger.warning(f"HK stock {symbol}: data fetch failed (timeframe: {timeframe})")
|
||||
return klines
|
||||
|
||||
def _to_tencent_symbol(self, symbol: str) -> str:
|
||||
"""转换为腾讯财经格式"""
|
||||
# 港股代码补齐到5位
|
||||
padded = symbol.zfill(5)
|
||||
return f"hk{padded}"
|
||||
|
||||
def _to_yahoo_symbol(self, symbol: str) -> str:
|
||||
"""转换为 Yahoo Finance 格式"""
|
||||
# 港股代码补齐到4位
|
||||
padded = symbol.zfill(4)
|
||||
return f"{padded}.HK"
|
||||
|
||||
def _fetch_eastmoney_kline(
|
||||
self,
|
||||
symbol: str,
|
||||
timeframe: str,
|
||||
limit: int
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""使用东方财富获取港股分钟级数据"""
|
||||
klines = []
|
||||
|
||||
# 东方财富 K 线周期映射
|
||||
em_period_map = {
|
||||
'1m': '1',
|
||||
'5m': '5',
|
||||
'15m': '15',
|
||||
'30m': '30',
|
||||
'1H': '60',
|
||||
'4H': '240',
|
||||
'1D': '101',
|
||||
'1W': '102',
|
||||
}
|
||||
|
||||
period = em_period_map.get(timeframe)
|
||||
if not period:
|
||||
logger.warning(f"Eastmoney unsupported timeframe: {timeframe}")
|
||||
return []
|
||||
|
||||
try:
|
||||
# 港股代码补齐到5位
|
||||
hk_symbol = symbol.zfill(5)
|
||||
# 东方财富港股代码格式: 116.00700 (116是港股市场代码)
|
||||
secid = f"116.{hk_symbol}"
|
||||
|
||||
# 东方财富K线接口
|
||||
url = f"https://push2his.eastmoney.com/api/qt/stock/kline/get"
|
||||
params = {
|
||||
'secid': secid,
|
||||
'fields1': 'f1,f2,f3,f4,f5,f6',
|
||||
'fields2': 'f51,f52,f53,f54,f55,f56,f57',
|
||||
'klt': period, # K线类型
|
||||
'fqt': '1', # 前复权
|
||||
'end': '20500101',
|
||||
'lmt': limit,
|
||||
}
|
||||
|
||||
# logger.info(f"东方财富港股请求: {hk_symbol}, 周期: {timeframe}")
|
||||
|
||||
# 添加浏览器请求头，避免被拒绝
|
||||
headers = {
|
||||
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
|
||||
'Referer': 'https://quote.eastmoney.com/',
|
||||
'Accept': 'application/json, text/plain, */*',
|
||||
'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8',
|
||||
}
|
||||
|
||||
session = get_retry_session()
|
||||
response = session.get(url, params=params, headers=headers, timeout=15)
|
||||
|
||||
if response.status_code != 200:
|
||||
logger.warning(f"Eastmoney HTTP status: {response.status_code}")
|
||||
return []
|
||||
|
||||
data = response.json()
|
||||
|
||||
# 解析响应
|
||||
if data.get('data') and data['data'].get('klines'):
|
||||
for line in data['data']['klines']:
|
||||
try:
|
||||
# 格式: "2025-11-28 15:00,400.0,401.0,399.0,400.5,1000,100000"
|
||||
# 日期,开盘,收盘,最高,最低,成交量,成交额
|
||||
parts = line.split(',')
|
||||
if len(parts) >= 6:
|
||||
time_str = parts[0]
|
||||
# 解析时间
|
||||
if ' ' in time_str:
|
||||
dt = datetime.strptime(time_str, '%Y-%m-%d %H:%M')
|
||||
else:
|
||||
dt = datetime.strptime(time_str, '%Y-%m-%d')
|
||||
|
||||
klines.append(self.format_kline(
|
||||
timestamp=int(dt.timestamp()),
|
||||
open_price=float(parts[1]),
|
||||
high=float(parts[3]),
|
||||
low=float(parts[4]),
|
||||
close=float(parts[2]),
|
||||
volume=float(parts[5])
|
||||
))
|
||||
except (ValueError, IndexError) as e:
|
||||
logger.debug(f"Failed to parse Eastmoney data line: {line}, error: {e}")
|
||||
continue
|
||||
|
||||
# logger.info(f"东方财富返回 {len(klines)} 条港股数据")
|
||||
else:
|
||||
logger.warning("Eastmoney returned no data")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Eastmoney HK stock fetch failed: {e}")
|
||||
import traceback
|
||||
logger.error(traceback.format_exc())
|
||||
|
||||
return klines
|
||||
|
||||
def _fetch_akshare(
|
||||
self,
|
||||
symbol: str,
|
||||
timeframe: str,
|
||||
limit: int,
|
||||
before_time: Optional[int]
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""使用 akshare 获取港股数据"""
|
||||
klines = []
|
||||
|
||||
try:
|
||||
# 计算日期范围
|
||||
if before_time:
|
||||
end_date = datetime.fromtimestamp(before_time).strftime('%Y%m%d')
|
||||
else:
|
||||
end_date = datetime.now().strftime('%Y%m%d')
|
||||
|
||||
days = limit * 2 if timeframe == '1D' else limit * 10
|
||||
start_date = (datetime.now() - timedelta(days=days)).strftime('%Y%m%d')
|
||||
|
||||
# 港股代码补齐到5位
|
||||
hk_symbol = symbol.zfill(5)
|
||||
|
||||
# logger.info(f"使用 akshare 获取港股: {hk_symbol}")
|
||||
|
||||
df = ak.stock_hk_hist(
|
||||
symbol=hk_symbol,
|
||||
period="daily",
|
||||
start_date=start_date,
|
||||
end_date=end_date,
|
||||
adjust="qfq"
|
||||
)
|
||||
|
||||
if df is not None and not df.empty:
|
||||
df = df.tail(limit)
|
||||
for _, row in df.iterrows():
|
||||
ts = int(datetime.strptime(str(row['日期']), '%Y-%m-%d').timestamp())
|
||||
klines.append(self.format_kline(
|
||||
timestamp=ts,
|
||||
open_price=row['开盘'],
|
||||
high=row['最高'],
|
||||
low=row['最低'],
|
||||
close=row['收盘'],
|
||||
volume=row['成交量']
|
||||
))
|
||||
# logger.info(f"akshare 返回 {len(klines)} 条港股数据")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Akshare HK stock fetch failed: {e}")
|
||||
import traceback
|
||||
logger.error(traceback.format_exc())
|
||||
|
||||
return klines
|
||||
@@ -0,0 +1,199 @@
|
||||
"""
|
||||
加密货币数据源
|
||||
使用 CCXT (Binance) 获取数据
|
||||
"""
|
||||
from typing import Dict, List, Any, Optional
|
||||
from datetime import datetime, timedelta
|
||||
import ccxt
|
||||
|
||||
from app.data_sources.base import BaseDataSource, TIMEFRAME_SECONDS
|
||||
from app.utils.logger import get_logger
|
||||
from app.config import CCXTConfig, APIKeys
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
|
||||
class CryptoDataSource(BaseDataSource):
|
||||
"""加密货币数据源"""
|
||||
|
||||
name = "Crypto/CCXT"
|
||||
|
||||
# 时间周期映射
|
||||
TIMEFRAME_MAP = CCXTConfig.TIMEFRAME_MAP
|
||||
|
||||
def __init__(self):
|
||||
config = {
|
||||
'timeout': CCXTConfig.TIMEOUT,
|
||||
'enableRateLimit': CCXTConfig.ENABLE_RATE_LIMIT
|
||||
}
|
||||
|
||||
# 如果配置了代理
|
||||
if CCXTConfig.PROXY:
|
||||
config['proxies'] = {
|
||||
'http': CCXTConfig.PROXY,
|
||||
'https': CCXTConfig.PROXY
|
||||
}
|
||||
|
||||
self.exchange = ccxt.binance(config)
|
||||
|
||||
def get_ticker(self, symbol: str) -> Dict[str, Any]:
|
||||
"""
|
||||
Get latest ticker for a crypto symbol via CCXT (Binance).
|
||||
|
||||
Accepts common formats:
|
||||
- BTC/USDT
|
||||
- BTCUSDT
|
||||
- BTC/USDT:USDT (swap-style suffix, will be normalized)
|
||||
"""
|
||||
sym = (symbol or "").strip()
|
||||
if ":" in sym:
|
||||
sym = sym.split(":", 1)[0]
|
||||
sym = sym.upper()
|
||||
if "/" not in sym:
|
||||
if sym.endswith("USDT") and len(sym) > 4:
|
||||
sym = f"{sym[:-4]}/USDT"
|
||||
elif sym.endswith("USD") and len(sym) > 3:
|
||||
sym = f"{sym[:-3]}/USD"
|
||||
return self.exchange.fetch_ticker(sym)
|
||||
|
||||
def get_kline(
|
||||
self,
|
||||
symbol: str,
|
||||
timeframe: str,
|
||||
limit: int,
|
||||
before_time: Optional[int] = None
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""获取加密货币K线数据"""
|
||||
klines = []
|
||||
|
||||
try:
|
||||
ccxt_timeframe = self.TIMEFRAME_MAP.get(timeframe, '1d')
|
||||
|
||||
# 构建交易对符号
|
||||
if not symbol.endswith('USDT') and not symbol.endswith('USD'):
|
||||
symbol_pair = f'{symbol}/USDT'
|
||||
else:
|
||||
symbol_pair = symbol
|
||||
|
||||
# logger.info(f"获取加密货币K线: {symbol_pair}, 周期: {ccxt_timeframe}, 条数: {limit}")
|
||||
|
||||
ohlcv = self._fetch_ohlcv(symbol_pair, ccxt_timeframe, limit, before_time, timeframe)
|
||||
|
||||
if not ohlcv:
|
||||
logger.warning(f"CCXT returned no K-lines: {symbol_pair}")
|
||||
return []
|
||||
|
||||
# 转换数据格式
|
||||
for candle in ohlcv:
|
||||
if len(candle) < 6:
|
||||
continue
|
||||
klines.append(self.format_kline(
|
||||
timestamp=int(candle[0] / 1000), # 毫秒转秒
|
||||
open_price=candle[1],
|
||||
high=candle[2],
|
||||
low=candle[3],
|
||||
close=candle[4],
|
||||
volume=candle[5]
|
||||
))
|
||||
|
||||
# 过滤和限制
|
||||
klines = self.filter_and_limit(klines, limit, before_time)
|
||||
|
||||
# 记录结果
|
||||
self.log_result(symbol, klines, timeframe)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to fetch crypto K-lines {symbol}: {str(e)}")
|
||||
import traceback
|
||||
logger.error(traceback.format_exc())
|
||||
|
||||
return klines
|
||||
|
||||
def _fetch_ohlcv(
|
||||
self,
|
||||
symbol_pair: str,
|
||||
ccxt_timeframe: str,
|
||||
limit: int,
|
||||
before_time: Optional[int],
|
||||
timeframe: str
|
||||
) -> List:
|
||||
"""获取OHLCV数据（支持分页获取完整数据）"""
|
||||
try:
|
||||
if before_time:
|
||||
# 计算时间范围
|
||||
total_seconds = self.calculate_time_range(timeframe, limit)
|
||||
end_time = datetime.fromtimestamp(before_time)
|
||||
start_time = end_time - timedelta(seconds=total_seconds)
|
||||
since = int(start_time.timestamp() * 1000)
|
||||
end_ms = before_time * 1000
|
||||
|
||||
# logger.info(f"历史数据请求: since={since//1000}, end={before_time}, 时间跨度={total_seconds/86400:.1f}天")
|
||||
|
||||
# 分页获取数据，直到覆盖完整时间范围
|
||||
all_ohlcv = []
|
||||
batch_limit = 1000 # Binance 单次最大返回量
|
||||
current_since = since
|
||||
|
||||
while current_since < end_ms:
|
||||
batch = self.exchange.fetch_ohlcv(
|
||||
symbol_pair,
|
||||
ccxt_timeframe,
|
||||
since=current_since,
|
||||
limit=batch_limit
|
||||
)
|
||||
|
||||
if not batch:
|
||||
break
|
||||
|
||||
all_ohlcv.extend(batch)
|
||||
|
||||
# 获取最后一条数据的时间，作为下次请求的起始时间
|
||||
last_timestamp = batch[-1][0]
|
||||
|
||||
# 如果最后一条数据时间超过了结束时间，或者返回数据少于请求量，说明已经获取完毕
|
||||
if last_timestamp >= end_ms or len(batch) < batch_limit:
|
||||
break
|
||||
|
||||
# 下次从最后一条的下一个时间点开始
|
||||
timeframe_ms = TIMEFRAME_SECONDS.get(timeframe, 86400) * 1000
|
||||
current_since = last_timestamp + timeframe_ms
|
||||
|
||||
# logger.info(f"分页获取中: 已获取 {len(all_ohlcv)} 条, 继续从 {datetime.fromtimestamp(current_since/1000)}")
|
||||
|
||||
ohlcv = all_ohlcv
|
||||
else:
|
||||
ohlcv = self.exchange.fetch_ohlcv(symbol_pair, ccxt_timeframe, limit=limit)
|
||||
|
||||
# logger.info(f"CCXT 返回 {len(ohlcv) if ohlcv else 0} 条数据")
|
||||
return ohlcv
|
||||
|
||||
except Exception as e:
|
||||
logger.warning(f"CCXT fetch_ohlcv failed: {str(e)}; trying fallback")
|
||||
return self._fetch_ohlcv_fallback(symbol_pair, ccxt_timeframe, limit, before_time, timeframe)
|
||||
|
||||
def _fetch_ohlcv_fallback(
|
||||
self,
|
||||
symbol_pair: str,
|
||||
ccxt_timeframe: str,
|
||||
limit: int,
|
||||
before_time: Optional[int],
|
||||
timeframe: str
|
||||
) -> List:
|
||||
"""备用获取方法"""
|
||||
try:
|
||||
total_seconds = self.calculate_time_range(timeframe, limit)
|
||||
|
||||
if before_time:
|
||||
end_time = datetime.fromtimestamp(before_time)
|
||||
start_time = end_time - timedelta(seconds=total_seconds)
|
||||
since = int(start_time.timestamp() * 1000)
|
||||
else:
|
||||
since = int((datetime.now() - timedelta(seconds=total_seconds)).timestamp() * 1000)
|
||||
|
||||
ohlcv = self.exchange.fetch_ohlcv(symbol_pair, ccxt_timeframe, since=since, limit=limit)
|
||||
# logger.info(f"CCXT 备用方法返回 {len(ohlcv) if ohlcv else 0} 条数据")
|
||||
return ohlcv
|
||||
except Exception as e:
|
||||
logger.error(f"CCXT fallback method also failed: {str(e)}")
|
||||
return []
|
||||
|
||||
@@ -0,0 +1,106 @@
|
||||
"""
|
||||
数据源工厂
|
||||
根据市场类型返回对应的数据源
|
||||
"""
|
||||
from typing import Dict, List, Any, Optional
|
||||
|
||||
from app.data_sources.base import BaseDataSource
|
||||
from app.utils.logger import get_logger
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
|
||||
class DataSourceFactory:
|
||||
"""数据源工厂"""
|
||||
|
||||
_sources: Dict[str, BaseDataSource] = {}
|
||||
|
||||
@classmethod
|
||||
def get_source(cls, market: str) -> BaseDataSource:
|
||||
"""
|
||||
获取指定市场的数据源
|
||||
|
||||
Args:
|
||||
market: 市场类型 (Crypto, USStock, AShare, HShare)
|
||||
|
||||
Returns:
|
||||
数据源实例
|
||||
"""
|
||||
if market not in cls._sources:
|
||||
cls._sources[market] = cls._create_source(market)
|
||||
return cls._sources[market]
|
||||
|
||||
@classmethod
|
||||
def get_data_source(cls, name: str) -> BaseDataSource:
|
||||
"""
|
||||
Backward compatible alias used by older code paths.
|
||||
|
||||
Some modules historically called `get_data_source("binance")` to fetch a crypto data source.
|
||||
In the localized Python backend we primarily use `get_source("Crypto")`.
|
||||
"""
|
||||
key = (name or "").strip().lower()
|
||||
if key in ("crypto", "binance", "okx", "bybit", "bitget", "kucoin", "gate", "mexc", "kraken", "coinbase"):
|
||||
return cls.get_source("Crypto")
|
||||
if key in ("futures",):
|
||||
return cls.get_source("Futures")
|
||||
# Default to Crypto for safety (most callers want a ticker for crypto pairs).
|
||||
return cls.get_source("Crypto")
|
||||
|
||||
@classmethod
|
||||
def _create_source(cls, market: str) -> BaseDataSource:
|
||||
"""创建数据源实例"""
|
||||
if market == 'Crypto':
|
||||
from app.data_sources.crypto import CryptoDataSource
|
||||
return CryptoDataSource()
|
||||
elif market == 'USStock':
|
||||
from app.data_sources.us_stock import USStockDataSource
|
||||
return USStockDataSource()
|
||||
elif market == 'AShare':
|
||||
from app.data_sources.cn_stock import AShareDataSource
|
||||
return AShareDataSource()
|
||||
elif market == 'HShare':
|
||||
from app.data_sources.cn_stock import HShareDataSource
|
||||
return HShareDataSource()
|
||||
elif market == 'Forex':
|
||||
from app.data_sources.forex import ForexDataSource
|
||||
return ForexDataSource()
|
||||
elif market == 'Futures':
|
||||
from app.data_sources.futures import FuturesDataSource
|
||||
return FuturesDataSource()
|
||||
else:
|
||||
raise ValueError(f"不支持的市场类型: {market}")
|
||||
|
||||
@classmethod
|
||||
def get_kline(
|
||||
cls,
|
||||
market: str,
|
||||
symbol: str,
|
||||
timeframe: str,
|
||||
limit: int,
|
||||
before_time: Optional[int] = None
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
获取K线数据的便捷方法
|
||||
|
||||
Args:
|
||||
market: 市场类型
|
||||
symbol: 交易对/股票代码
|
||||
timeframe: 时间周期
|
||||
limit: 数据条数
|
||||
before_time: 获取此时间之前的数据
|
||||
|
||||
Returns:
|
||||
K线数据列表
|
||||
"""
|
||||
try:
|
||||
source = cls.get_source(market)
|
||||
klines = source.get_kline(symbol, timeframe, limit, before_time)
|
||||
|
||||
# 确保数据按时间排序
|
||||
klines.sort(key=lambda x: x['time'])
|
||||
|
||||
return klines
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to fetch K-lines {market}:{symbol} - {str(e)}")
|
||||
return []
|
||||
|
||||
@@ -0,0 +1,190 @@
|
||||
"""
|
||||
外汇数据源
|
||||
使用 Tiingo 获取外汇数据
|
||||
"""
|
||||
from typing import Dict, List, Any, Optional
|
||||
from datetime import datetime, timedelta
|
||||
import time
|
||||
import requests
|
||||
|
||||
from app.data_sources.base import BaseDataSource, TIMEFRAME_SECONDS
|
||||
from app.utils.logger import get_logger
|
||||
from app.config import TiingoConfig, APIKeys
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
|
||||
class ForexDataSource(BaseDataSource):
|
||||
"""外汇数据源 (Tiingo)"""
|
||||
|
||||
name = "Forex/Tiingo"
|
||||
|
||||
# Tiingo resampleFreq 映射
|
||||
# Tiingo 支持: 1min, 5min, 15min, 30min, 1hour, 4hour, 1day 等
|
||||
TIMEFRAME_MAP = {
|
||||
'1m': '1min',
|
||||
'5m': '5min',
|
||||
'15m': '15min',
|
||||
'30m': '30min',
|
||||
'1H': '1hour',
|
||||
'4H': '4hour',
|
||||
'1D': '1day',
|
||||
'1W': '1week',
|
||||
'1M': '1month'
|
||||
}
|
||||
|
||||
# 外汇对映射 (Tiingo 使用标准 ticker，如 eurusd, audusd)
|
||||
# 大写也可以，Tiingo 通常不区分大小写，但建议统一
|
||||
SYMBOL_MAP = {
|
||||
# 贵金属 (Tiingo 不一定支持所有 OANDA 格式的贵金属，通常是 XAUUSD)
|
||||
'XAUUSD': 'xauusd',
|
||||
'XAGUSD': 'xagusd',
|
||||
# 主要货币对
|
||||
'EURUSD': 'eurusd',
|
||||
'GBPUSD': 'gbpusd',
|
||||
'USDJPY': 'usdjpy',
|
||||
'AUDUSD': 'audusd',
|
||||
'USDCAD': 'usdcad',
|
||||
'USDCHF': 'usdchf',
|
||||
'NZDUSD': 'nzdusd',
|
||||
}
|
||||
|
||||
def __init__(self):
|
||||
self.base_url = TiingoConfig.BASE_URL
|
||||
if not APIKeys.TIINGO_API_KEY:
|
||||
logger.warning("Tiingo API key is not configured; FX data will be unavailable")
|
||||
|
||||
def _get_timeframe_seconds(self, timeframe: str) -> int:
|
||||
"""获取时间周期对应的秒数"""
|
||||
return TIMEFRAME_SECONDS.get(timeframe, 86400)
|
||||
|
||||
def get_kline(
|
||||
self,
|
||||
symbol: str,
|
||||
timeframe: str,
|
||||
limit: int,
|
||||
before_time: Optional[int] = None
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
获取外汇K线数据
|
||||
|
||||
Args:
|
||||
symbol: 外汇对代码（如 XAUUSD, EURUSD）
|
||||
timeframe: 时间周期
|
||||
limit: 数据条数
|
||||
before_time: 结束时间戳
|
||||
"""
|
||||
# 动态获取 API Key
|
||||
api_key = APIKeys.TIINGO_API_KEY
|
||||
if not api_key:
|
||||
logger.error("Tiingo API key is not configured")
|
||||
return []
|
||||
|
||||
try:
|
||||
# 1. 解析 Symbol
|
||||
tiingo_symbol = self.SYMBOL_MAP.get(symbol)
|
||||
if not tiingo_symbol:
|
||||
# 尝试智能转换: EURUSD -> eurusd
|
||||
tiingo_symbol = symbol.lower()
|
||||
|
||||
# 2. 解析 Resolution (resampleFreq)
|
||||
resample_freq = self.TIMEFRAME_MAP.get(timeframe)
|
||||
if not resample_freq:
|
||||
logger.warning(f"Tiingo does not support timeframe: {timeframe}")
|
||||
return []
|
||||
|
||||
# 3. 计算时间范围
|
||||
if before_time:
|
||||
end_dt = datetime.fromtimestamp(before_time)
|
||||
else:
|
||||
end_dt = datetime.now()
|
||||
|
||||
# 根据周期和数量计算开始时间
|
||||
tf_seconds = self._get_timeframe_seconds(timeframe)
|
||||
# 多取一些缓冲时间
|
||||
start_dt = end_dt - timedelta(seconds=limit * tf_seconds * 2)
|
||||
|
||||
# 格式化日期为 YYYY-MM-DD (Tiingo 支持该格式)
|
||||
start_date_str = start_dt.strftime('%Y-%m-%d')
|
||||
end_date_str = end_dt.strftime('%Y-%m-%d')
|
||||
|
||||
# 4. API 请求
|
||||
# URL: https://api.tiingo.com/tiingo/fx/{ticker}/prices
|
||||
url = f"{self.base_url}/fx/{tiingo_symbol}/prices"
|
||||
|
||||
params = {
|
||||
'startDate': start_date_str,
|
||||
'endDate': end_date_str,
|
||||
'resampleFreq': resample_freq,
|
||||
'token': api_key,
|
||||
'format': 'json'
|
||||
}
|
||||
|
||||
# logger.info(f"Tiingo Request: {url} params={params}")
|
||||
|
||||
response = requests.get(url, params=params, timeout=TiingoConfig.TIMEOUT)
|
||||
|
||||
if response.status_code == 403: # 具体的权限错误
|
||||
logger.error("Tiingo API permission error (403): check whether your API key is valid and has access to this dataset.")
|
||||
return []
|
||||
|
||||
response.raise_for_status()
|
||||
data = response.json()
|
||||
|
||||
# 5. 处理响应
|
||||
# Tiingo returns a list of dicts:
|
||||
# [
|
||||
# {
|
||||
# "date": "2023-01-01T00:00:00.000Z",
|
||||
# "ticker": "eurusd",
|
||||
# "open": 1.07,
|
||||
# "high": 1.08,
|
||||
# "low": 1.06,
|
||||
# "close": 1.07
|
||||
# "mid": ... (optional, depends on settings, usually OHLC are bid or mid)
|
||||
# }, ...
|
||||
# ]
|
||||
# Note: Tiingo FX prices objects keys: date, open, high, low, close.
|
||||
|
||||
if not isinstance(data, list):
|
||||
logger.warning(f"Tiingo response is not a list: {data}")
|
||||
return []
|
||||
|
||||
klines = []
|
||||
for item in data:
|
||||
# 解析时间: "2023-01-01T00:00:00.000Z"
|
||||
dt_str = item.get('date')
|
||||
# 简化处理，Tiingo 返回的是 UTC 时间 ISO 格式
|
||||
# datetime.fromisoformat 在 Py3.7+ 支持，但要注意 Z 的处理
|
||||
# 这里简单处理一下 Z
|
||||
if dt_str.endswith('Z'):
|
||||
dt_str = dt_str[:-1]
|
||||
|
||||
dt = datetime.fromisoformat(dt_str)
|
||||
ts = int(dt.timestamp())
|
||||
|
||||
klines.append({
|
||||
'time': ts,
|
||||
'open': float(item.get('open')),
|
||||
'high': float(item.get('high')),
|
||||
'low': float(item.get('low')),
|
||||
'close': float(item.get('close')),
|
||||
'volume': 0.0 # Tiingo FX 通常没有 volume
|
||||
})
|
||||
|
||||
# 按时间排序
|
||||
klines.sort(key=lambda x: x['time'])
|
||||
|
||||
# 过滤
|
||||
if len(klines) > limit:
|
||||
klines = klines[-limit:]
|
||||
|
||||
# logger.info(f"获取到 {len(klines)} 条 Tiingo 外汇数据")
|
||||
return klines
|
||||
|
||||
except requests.exceptions.RequestException as e:
|
||||
logger.error(f"Tiingo API request failed: {e}")
|
||||
return []
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to process Tiingo data: {e}")
|
||||
return []
|
||||
@@ -0,0 +1,243 @@
|
||||
"""
|
||||
期货数据源
|
||||
支持：
|
||||
1. 加密货币期货（Binance Futures via CCXT）
|
||||
2. 传统期货（Yahoo Finance）
|
||||
"""
|
||||
from typing import Dict, List, Any, Optional
|
||||
from datetime import datetime, timedelta
|
||||
import ccxt
|
||||
import yfinance as yf
|
||||
|
||||
from app.data_sources.base import BaseDataSource, TIMEFRAME_SECONDS
|
||||
from app.utils.logger import get_logger
|
||||
from app.config import CCXTConfig, APIKeys
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
|
||||
class FuturesDataSource(BaseDataSource):
|
||||
"""期货数据源"""
|
||||
|
||||
name = "Futures"
|
||||
|
||||
# Yahoo Finance时间周期映射
|
||||
YF_TIMEFRAME_MAP = {
|
||||
'1m': '1m',
|
||||
'5m': '5m',
|
||||
'15m': '15m',
|
||||
'30m': '30m',
|
||||
'1H': '1h',
|
||||
'4H': '4h',
|
||||
'1D': '1d',
|
||||
'1W': '1wk'
|
||||
}
|
||||
|
||||
# CCXT时间周期映射
|
||||
CCXT_TIMEFRAME_MAP = CCXTConfig.TIMEFRAME_MAP
|
||||
|
||||
# 传统期货合约代码（Yahoo Finance）
|
||||
YF_SYMBOLS = {
|
||||
'GC': 'GC=F', # 黄金期货
|
||||
'SI': 'SI=F', # 白银期货
|
||||
'CL': 'CL=F', # 原油期货
|
||||
'NG': 'NG=F', # 天然气期货
|
||||
'ZC': 'ZC=F', # 玉米期货
|
||||
'ZW': 'ZW=F', # 小麦期货
|
||||
}
|
||||
|
||||
def __init__(self):
|
||||
# 初始化CCXT（用于加密货币期货）
|
||||
config = {
|
||||
'timeout': CCXTConfig.TIMEOUT,
|
||||
'enableRateLimit': CCXTConfig.ENABLE_RATE_LIMIT,
|
||||
'options': {
|
||||
'defaultType': 'future'
|
||||
}
|
||||
}
|
||||
|
||||
if CCXTConfig.PROXY:
|
||||
config['proxies'] = {
|
||||
'http': CCXTConfig.PROXY,
|
||||
'https': CCXTConfig.PROXY
|
||||
}
|
||||
|
||||
self.exchange = ccxt.binance(config)
|
||||
|
||||
def get_ticker(self, symbol: str) -> Dict[str, Any]:
|
||||
"""
|
||||
Get latest ticker for futures symbol.
|
||||
|
||||
- For crypto futures, uses CCXT Binance futures client.
|
||||
- For traditional futures (Yahoo Finance symbols), returns a minimal ticker shape with `last`.
|
||||
"""
|
||||
sym = (symbol or "").strip()
|
||||
if sym in self.YF_SYMBOLS or sym.endswith("=F"):
|
||||
try:
|
||||
yf_symbol = self.YF_SYMBOLS.get(sym, sym)
|
||||
if not yf_symbol.endswith("=F"):
|
||||
yf_symbol = yf_symbol + "=F"
|
||||
t = yf.Ticker(yf_symbol)
|
||||
# Prefer fast_info if available, fall back to last close
|
||||
last = None
|
||||
try:
|
||||
last = getattr(t, "fast_info", {}).get("last_price")
|
||||
except Exception:
|
||||
last = None
|
||||
if last is None:
|
||||
hist = t.history(period="2d", interval="1d")
|
||||
if hist is not None and not hist.empty:
|
||||
last = float(hist["Close"].iloc[-1])
|
||||
return {"symbol": yf_symbol, "last": float(last or 0.0)}
|
||||
except Exception:
|
||||
return {"symbol": sym, "last": 0.0}
|
||||
|
||||
if ":" in sym:
|
||||
sym = sym.split(":", 1)[0]
|
||||
sym = sym.upper()
|
||||
if "/" not in sym:
|
||||
if sym.endswith("USDT") and len(sym) > 4:
|
||||
sym = f"{sym[:-4]}/USDT"
|
||||
elif sym.endswith("USD") and len(sym) > 3:
|
||||
sym = f"{sym[:-3]}/USD"
|
||||
return self.exchange.fetch_ticker(sym)
|
||||
|
||||
def _get_timeframe_seconds(self, timeframe: str) -> int:
|
||||
"""获取时间周期对应的秒数"""
|
||||
return TIMEFRAME_SECONDS.get(timeframe, 86400)
|
||||
|
||||
def get_kline(
|
||||
self,
|
||||
symbol: str,
|
||||
timeframe: str,
|
||||
limit: int,
|
||||
before_time: Optional[int] = None
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
获取期货K线数据
|
||||
|
||||
Args:
|
||||
symbol: 期货合约代码
|
||||
timeframe: 时间周期
|
||||
limit: 数据条数
|
||||
before_time: 结束时间戳
|
||||
"""
|
||||
# 判断是传统期货还是加密货币期货
|
||||
if symbol in self.YF_SYMBOLS or symbol.endswith('=F'):
|
||||
return self._get_traditional_futures(symbol, timeframe, limit, before_time)
|
||||
else:
|
||||
return self._get_crypto_futures(symbol, timeframe, limit, before_time)
|
||||
|
||||
def _get_traditional_futures(
|
||||
self,
|
||||
symbol: str,
|
||||
timeframe: str,
|
||||
limit: int,
|
||||
before_time: Optional[int] = None
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""使用yfinance获取传统期货数据"""
|
||||
try:
|
||||
# 转换symbol格式
|
||||
yf_symbol = self.YF_SYMBOLS.get(symbol, symbol)
|
||||
if not yf_symbol.endswith('=F'):
|
||||
yf_symbol = symbol + '=F'
|
||||
|
||||
# 转换时间周期
|
||||
yf_interval = self.YF_TIMEFRAME_MAP.get(timeframe, '1d')
|
||||
|
||||
# logger.info(f"获取传统期货K线: {yf_symbol}, 周期: {yf_interval}, 条数: {limit}")
|
||||
|
||||
# 计算时间范围
|
||||
if before_time:
|
||||
end_time = datetime.fromtimestamp(before_time)
|
||||
else:
|
||||
end_time = datetime.now()
|
||||
|
||||
tf_seconds = self._get_timeframe_seconds(timeframe)
|
||||
start_time = end_time - timedelta(seconds=tf_seconds * limit * 1.5)
|
||||
|
||||
# 获取数据
|
||||
ticker = yf.Ticker(yf_symbol)
|
||||
df = ticker.history(
|
||||
start=start_time,
|
||||
end=end_time,
|
||||
interval=yf_interval
|
||||
)
|
||||
|
||||
if df.empty:
|
||||
logger.warning(f"No data: {yf_symbol}")
|
||||
return []
|
||||
|
||||
# 转换格式
|
||||
klines = []
|
||||
for index, row in df.iterrows():
|
||||
klines.append({
|
||||
'time': int(index.timestamp()),
|
||||
'open': float(row['Open']),
|
||||
'high': float(row['High']),
|
||||
'low': float(row['Low']),
|
||||
'close': float(row['Close']),
|
||||
'volume': float(row['Volume'])
|
||||
})
|
||||
|
||||
klines.sort(key=lambda x: x['time'])
|
||||
if len(klines) > limit:
|
||||
klines = klines[-limit:]
|
||||
|
||||
# logger.info(f"获取到 {len(klines)} 条传统期货数据")
|
||||
return klines
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to fetch traditional futures data: {e}")
|
||||
return []
|
||||
|
||||
def _get_crypto_futures(
|
||||
self,
|
||||
symbol: str,
|
||||
timeframe: str,
|
||||
limit: int,
|
||||
before_time: Optional[int] = None
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""使用CCXT获取加密货币期货数据"""
|
||||
try:
|
||||
# 确保symbol格式正确
|
||||
ccxt_symbol = symbol if '/' in symbol else f"{symbol}/USDT"
|
||||
ccxt_timeframe = self.CCXT_TIMEFRAME_MAP.get(timeframe, '1d')
|
||||
|
||||
# logger.info(f"获取加密货币期货K线: {ccxt_symbol}, 周期: {ccxt_timeframe}, 条数: {limit}")
|
||||
|
||||
# 获取数据
|
||||
if before_time:
|
||||
since_time = before_time - limit * self._get_timeframe_seconds(timeframe)
|
||||
ohlcv = self.exchange.fetch_ohlcv(
|
||||
ccxt_symbol,
|
||||
ccxt_timeframe,
|
||||
since=since_time * 1000,
|
||||
limit=limit
|
||||
)
|
||||
else:
|
||||
ohlcv = self.exchange.fetch_ohlcv(
|
||||
ccxt_symbol,
|
||||
ccxt_timeframe,
|
||||
limit=limit
|
||||
)
|
||||
|
||||
# 转换格式
|
||||
klines = []
|
||||
for candle in ohlcv:
|
||||
klines.append({
|
||||
'time': int(candle[0] / 1000),
|
||||
'open': float(candle[1]),
|
||||
'high': float(candle[2]),
|
||||
'low': float(candle[3]),
|
||||
'close': float(candle[4]),
|
||||
'volume': float(candle[5])
|
||||
})
|
||||
|
||||
# logger.info(f"获取到 {len(klines)} 条加密货币期货数据")
|
||||
return klines
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to fetch crypto futures data: {e}")
|
||||
return []
|
||||
|
||||
@@ -0,0 +1,194 @@
|
||||
"""
|
||||
美股数据源
|
||||
使用 yfinance 和 finnhub 获取数据
|
||||
"""
|
||||
from typing import Dict, List, Any, Optional
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
import yfinance as yf
|
||||
|
||||
from app.data_sources.base import BaseDataSource
|
||||
from app.utils.logger import get_logger
|
||||
from app.config import APIKeys, YFinanceConfig
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
|
||||
class USStockDataSource(BaseDataSource):
|
||||
"""美股数据源"""
|
||||
|
||||
name = "USStock/yfinance"
|
||||
|
||||
# yfinance 时间周期映射
|
||||
INTERVAL_MAP = {
|
||||
'1m': '1m',
|
||||
'5m': '5m',
|
||||
'15m': '15m',
|
||||
'30m': '30m',
|
||||
'1H': '1h',
|
||||
'4H': '4h',
|
||||
'1D': '1d',
|
||||
'1W': '1wk'
|
||||
}
|
||||
|
||||
# 不同周期获取数据的天数范围
|
||||
DAYS_MAP = {
|
||||
'1m': lambda limit: min(7, max(1, (limit // 390) + 2)),
|
||||
'5m': lambda limit: min(60, max(1, (limit // 78) + 2)),
|
||||
'15m': lambda limit: min(60, max(1, (limit // 26) + 2)),
|
||||
'30m': lambda limit: min(60, max(1, (limit // 13) + 2)),
|
||||
'1H': lambda limit: min(730, max(1, (limit // 24) + 2)),
|
||||
'4H': lambda limit: min(730, max(1, (limit // 6) + 2)),
|
||||
'1D': lambda limit: min(3650, limit + 1),
|
||||
'1W': lambda limit: min(3650, (limit * 7) + 7)
|
||||
}
|
||||
|
||||
def __init__(self):
|
||||
# 初始化 finnhub 作为备选
|
||||
self.finnhub_client = None
|
||||
try:
|
||||
import finnhub
|
||||
if APIKeys.is_configured('FINNHUB_API_KEY'):
|
||||
self.finnhub_client = finnhub.Client(api_key=APIKeys.FINNHUB_API_KEY)
|
||||
logger.info("Finnhub client initialized")
|
||||
except Exception as e:
|
||||
logger.warning(f"Finnhub init failed: {e}")
|
||||
|
||||
def get_kline(
|
||||
self,
|
||||
symbol: str,
|
||||
timeframe: str,
|
||||
limit: int,
|
||||
before_time: Optional[int] = None
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""获取美股K线数据"""
|
||||
klines = []
|
||||
|
||||
try:
|
||||
interval = self.INTERVAL_MAP.get(timeframe, '1d')
|
||||
days_func = self.DAYS_MAP.get(timeframe, lambda x: x + 1)
|
||||
days = days_func(limit)
|
||||
|
||||
# 计算日期范围
|
||||
if before_time:
|
||||
end_date = datetime.fromtimestamp(before_time)
|
||||
start_date = end_date - timedelta(days=days)
|
||||
else:
|
||||
end_date = datetime.now()
|
||||
start_date = end_date - timedelta(days=days)
|
||||
|
||||
# logger.info(f"使用 yfinance 获取 {symbol}, 周期: {interval}, 日期: {start_date.date()} ~ {end_date.date()}")
|
||||
|
||||
# 尝试 yfinance
|
||||
df = self._fetch_yfinance(symbol, interval, start_date, end_date)
|
||||
|
||||
if df is None or df.empty:
|
||||
# 尝试 finnhub
|
||||
if self.finnhub_client and timeframe == '1D':
|
||||
klines = self._fetch_finnhub(symbol, start_date, end_date, limit)
|
||||
if klines:
|
||||
return klines
|
||||
else:
|
||||
klines = self._convert_dataframe(df, limit)
|
||||
|
||||
# 过滤和限制
|
||||
klines = self.filter_and_limit(klines, limit, before_time)
|
||||
|
||||
# 记录结果
|
||||
self.log_result(symbol, klines, timeframe)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to fetch US stock K-lines {symbol}: {str(e)}")
|
||||
import traceback
|
||||
logger.error(traceback.format_exc())
|
||||
|
||||
return klines
|
||||
|
||||
def _fetch_yfinance(self, symbol: str, interval: str, start_date: datetime, end_date: datetime):
|
||||
"""使用 yfinance 获取数据"""
|
||||
try:
|
||||
ticker = yf.Ticker(symbol)
|
||||
df = ticker.history(
|
||||
start=start_date.strftime('%Y-%m-%d'),
|
||||
end=end_date.strftime('%Y-%m-%d'),
|
||||
interval=interval
|
||||
)
|
||||
# logger.info(f"yfinance 返回 {len(df) if df is not None and not df.empty else 0} 条数据")
|
||||
return df
|
||||
except Exception as e:
|
||||
logger.warning(f"yfinance fetch failed: {e}")
|
||||
return None
|
||||
|
||||
def _fetch_finnhub(
|
||||
self,
|
||||
symbol: str,
|
||||
start_date: datetime,
|
||||
end_date: datetime,
|
||||
limit: int
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""使用 finnhub 获取日线数据"""
|
||||
klines = []
|
||||
try:
|
||||
start_ts = int(start_date.timestamp())
|
||||
end_ts = int(end_date.timestamp())
|
||||
|
||||
# logger.info(f"使用 Finnhub 获取 {symbol} 日线数据")
|
||||
candles = self.finnhub_client.stock_candles(symbol, 'D', start_ts, end_ts)
|
||||
|
||||
if candles and candles.get('s') == 'ok':
|
||||
for i in range(len(candles['t'])):
|
||||
klines.append(self.format_kline(
|
||||
timestamp=candles['t'][i],
|
||||
open_price=candles['o'][i],
|
||||
high=candles['h'][i],
|
||||
low=candles['l'][i],
|
||||
close=candles['c'][i],
|
||||
volume=candles['v'][i]
|
||||
))
|
||||
# logger.info(f"Finnhub 返回 {len(klines)} 条数据")
|
||||
except Exception as e:
|
||||
logger.error(f"Finnhub fetch failed: {e}")
|
||||
|
||||
return klines
|
||||
|
||||
def _convert_dataframe(self, df, limit: int) -> List[Dict[str, Any]]:
|
||||
"""转换 DataFrame 为K线列表"""
|
||||
klines = []
|
||||
df = df.tail(limit).reset_index()
|
||||
|
||||
# 确定时间列名（日线是 Date，分钟级是 Datetime）
|
||||
time_col = None
|
||||
if 'Datetime' in df.columns:
|
||||
time_col = 'Datetime'
|
||||
elif 'Date' in df.columns:
|
||||
time_col = 'Date'
|
||||
elif 'index' in df.columns:
|
||||
time_col = 'index'
|
||||
|
||||
if time_col is None:
|
||||
logger.warning(f"Unable to determine time column; available columns: {df.columns.tolist()}")
|
||||
return klines
|
||||
|
||||
for _, row in df.iterrows():
|
||||
try:
|
||||
# 处理时间戳
|
||||
time_value = row[time_col]
|
||||
if hasattr(time_value, 'timestamp'):
|
||||
ts = int(time_value.timestamp())
|
||||
else:
|
||||
continue
|
||||
|
||||
klines.append(self.format_kline(
|
||||
timestamp=ts,
|
||||
open_price=row['Open'],
|
||||
high=row['High'],
|
||||
low=row['Low'],
|
||||
close=row['Close'],
|
||||
volume=row['Volume']
|
||||
))
|
||||
except Exception as e:
|
||||
logger.debug(f"Failed to parse row data: {e}")
|
||||
continue
|
||||
|
||||
return klines
|
||||
|
||||
Reference in New Issue
Block a user