Refactor code for improved readability and consistency
- Cleaned up whitespace and formatting in various files including http.py, language.py, logger.py, safe_exec.py, and SQL migration scripts. - Consolidated import statements and removed unnecessary blank lines. - Updated logging configuration for better clarity. - Enhanced the safe execution code with improved error handling and logging. - Removed commented-out code and unnecessary variables in backfill_zero_trades.py and other scripts. - Added a pyproject.toml for Ruff and Vulture configuration. - Introduced requirements-dev.txt for development dependencies. - Removed commented-out stock entries in init.sql for cleaner migration scripts.
This commit is contained in:
dienakdz
@@ -7,38 +7,26 @@ Improved version (refer to daily_stock_analysis project):
|
||||
- Data cache (cache_manager)
|
||||
- Anti-ban strategy (rate_limiter)
|
||||
"""
|
||||
|
||||
from app.data_sources.cache_manager import DataCache, get_kline_cache, get_realtime_cache, get_stock_info_cache
|
||||
from app.data_sources.circuit_breaker import CircuitBreaker, get_realtime_circuit_breaker
|
||||
from app.data_sources.factory import DataSourceFactory
|
||||
from app.data_sources.circuit_breaker import (
|
||||
CircuitBreaker,
|
||||
get_realtime_circuit_breaker
|
||||
)
|
||||
from app.data_sources.cache_manager import (
|
||||
DataCache,
|
||||
get_realtime_cache,
|
||||
get_kline_cache,
|
||||
get_stock_info_cache
|
||||
)
|
||||
from app.data_sources.rate_limiter import (
|
||||
RateLimiter,
|
||||
get_random_user_agent,
|
||||
random_sleep,
|
||||
retry_with_backoff
|
||||
)
|
||||
from app.data_sources.rate_limiter import RateLimiter, get_random_user_agent, random_sleep, retry_with_backoff
|
||||
|
||||
__all__ = [
|
||||
# factory
|
||||
'DataSourceFactory',
|
||||
"DataSourceFactory",
|
||||
# fuse
|
||||
'CircuitBreaker',
|
||||
'get_realtime_circuit_breaker',
|
||||
"CircuitBreaker",
|
||||
"get_realtime_circuit_breaker",
|
||||
# cache
|
||||
'DataCache',
|
||||
'get_realtime_cache',
|
||||
'get_kline_cache',
|
||||
'get_stock_info_cache',
|
||||
"DataCache",
|
||||
"get_realtime_cache",
|
||||
"get_kline_cache",
|
||||
"get_stock_info_cache",
|
||||
# current limiter
|
||||
'RateLimiter',
|
||||
'get_random_user_agent',
|
||||
'random_sleep',
|
||||
'retry_with_backoff',
|
||||
"RateLimiter",
|
||||
"get_random_user_agent",
|
||||
"random_sleep",
|
||||
"retry_with_backoff",
|
||||
]
|
||||
|
||||
@@ -1,566 +0,0 @@
|
||||
"""
|
||||
A-share / H-share chart K-lines — multi-tier fallback.
|
||||
|
||||
Priority order (when TWELVE_DATA_API_KEY is configured):
|
||||
ALL timeframes → Twelve Data (paid, globally stable) → Tencent daily/weekly → yfinance → AkShare
|
||||
|
||||
Without API key:
|
||||
Daily / Weekly → Tencent fqkline (fast, no key) → yfinance → AkShare
|
||||
Minute / Hour → yfinance → AkShare (Eastmoney, fragile overseas)
|
||||
|
||||
Tencent ``fqkline`` only reliably supports day/week/month.
|
||||
yfinance supports CN (.SS/.SZ) and HK (.HK) at all common intervals.
|
||||
Twelve Data (https://twelvedata.com) supports XSHG/XSHE/XHKG at all intervals.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import time
|
||||
from datetime import datetime, timedelta
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
import pandas as pd
|
||||
import requests
|
||||
|
||||
from app.utils.logger import get_logger
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
_MAX_ATTEMPTS = 3
|
||||
_BACKOFF_BASE_SEC = 1.5
|
||||
_BACKOFF_CAP_SEC = 12.0
|
||||
|
||||
_TRANSIENT_ERR_MARKERS = (
|
||||
"remote end closed connection",
|
||||
"connection aborted",
|
||||
"connection reset",
|
||||
"timed out",
|
||||
"timeout",
|
||||
"max retries exceeded",
|
||||
"temporarily unavailable",
|
||||
"broken pipe",
|
||||
"eof occurred",
|
||||
"remote disconnected",
|
||||
"chunkedencodingerror",
|
||||
"incompleteread",
|
||||
"rate",
|
||||
"too many requests",
|
||||
"429",
|
||||
)
|
||||
|
||||
|
||||
def _is_transient(exc: BaseException) -> bool:
|
||||
return any(m in str(exc).lower() for m in _TRANSIENT_ERR_MARKERS)
|
||||
|
||||
|
||||
_CHART_TF_ALIASES = {
|
||||
"1w": "1W",
|
||||
"1d": "1D",
|
||||
"1h": "1H",
|
||||
"4h": "4H",
|
||||
"d": "1D",
|
||||
"day": "1D",
|
||||
"w": "1W",
|
||||
"week": "1W",
|
||||
"wk": "1W",
|
||||
"60m": "1H",
|
||||
"240m": "4H",
|
||||
"1day": "1D",
|
||||
"1week": "1W",
|
||||
}
|
||||
|
||||
|
||||
def normalize_chart_timeframe(timeframe: str) -> str:
|
||||
t = (timeframe or "1D").strip()
|
||||
if not t:
|
||||
return "1D"
|
||||
key = t.lower()
|
||||
if key in _CHART_TF_ALIASES:
|
||||
return _CHART_TF_ALIASES[key]
|
||||
return t
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# AkShare code converters
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def ak_a_code_from_tencent(tencent_code: str) -> str:
|
||||
c = (tencent_code or "").strip().lower()
|
||||
if len(c) >= 8 and c[:2] in ("sh", "sz"):
|
||||
return c[2:]
|
||||
return c
|
||||
|
||||
|
||||
def ak_hk_code_from_tencent(tencent_code: str) -> str:
|
||||
c = (tencent_code or "").strip().upper().replace(".HK", "")
|
||||
if c.startswith("HK"):
|
||||
num = c[2:]
|
||||
else:
|
||||
num = c
|
||||
if num.isdigit():
|
||||
return num.zfill(5)
|
||||
return num
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Twelve Data (paid, globally reliable — https://twelvedata.com)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _get_twelve_data_api_key() -> str:
|
||||
try:
|
||||
from app.utils.config_loader import load_addon_config
|
||||
key = load_addon_config().get("twelve_data", {}).get("api_key", "")
|
||||
if key:
|
||||
return key
|
||||
except Exception:
|
||||
pass
|
||||
return (os.getenv("TWELVE_DATA_API_KEY") or "").strip()
|
||||
|
||||
|
||||
_TD_INTERVAL_MAP = {
|
||||
"1m": "1min",
|
||||
"5m": "5min",
|
||||
"15m": "15min",
|
||||
"30m": "30min",
|
||||
"1H": "1h",
|
||||
"4H": "4h",
|
||||
"1D": "1day",
|
||||
"1W": "1week",
|
||||
}
|
||||
|
||||
|
||||
def _td_symbol_and_exchange(tencent_code: str, is_hk: bool) -> tuple[str, str]:
|
||||
"""Convert Tencent code to Twelve Data (symbol, exchange)."""
|
||||
c = (tencent_code or "").strip().upper()
|
||||
if is_hk:
|
||||
num = c.replace("HK", "")
|
||||
if num.isdigit():
|
||||
num = str(int(num)).zfill(4)
|
||||
return num, "XHKG"
|
||||
digits = c.lstrip("SHSZ")
|
||||
if c.startswith("SH") or digits.startswith("6"):
|
||||
return digits, "XSHG"
|
||||
return digits, "XSHE"
|
||||
|
||||
|
||||
def fetch_twelvedata_klines(
|
||||
*,
|
||||
is_hk: bool,
|
||||
tencent_code: str,
|
||||
timeframe: str,
|
||||
limit: int,
|
||||
before_time: Optional[int],
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""Fetch K-lines from Twelve Data REST API. Requires TWELVE_DATA_API_KEY."""
|
||||
api_key = _get_twelve_data_api_key()
|
||||
if not api_key:
|
||||
return []
|
||||
|
||||
interval = _TD_INTERVAL_MAP.get(timeframe)
|
||||
if not interval:
|
||||
return []
|
||||
|
||||
symbol, exchange = _td_symbol_and_exchange(tencent_code, is_hk)
|
||||
params: Dict[str, Any] = {
|
||||
"symbol": symbol,
|
||||
"exchange": exchange,
|
||||
"interval": interval,
|
||||
"outputsize": min(int(limit), 5000),
|
||||
"apikey": api_key,
|
||||
"format": "JSON",
|
||||
"dp": "4",
|
||||
}
|
||||
if before_time:
|
||||
end_dt = datetime.fromtimestamp(int(before_time))
|
||||
params["end_date"] = end_dt.strftime("%Y-%m-%d %H:%M:%S")
|
||||
|
||||
url = "https://api.twelvedata.com/time_series"
|
||||
|
||||
for attempt in range(_MAX_ATTEMPTS):
|
||||
try:
|
||||
resp = requests.get(url, params=params, timeout=20)
|
||||
data = resp.json()
|
||||
break
|
||||
except Exception as e:
|
||||
if attempt + 1 < _MAX_ATTEMPTS and _is_transient(e):
|
||||
delay = min(_BACKOFF_CAP_SEC, _BACKOFF_BASE_SEC * (2 ** attempt))
|
||||
logger.debug(
|
||||
"TwelveData transient error %s/%s tf=%s (attempt %s/%s): %s",
|
||||
symbol, exchange, timeframe, attempt + 1, _MAX_ATTEMPTS, e,
|
||||
)
|
||||
time.sleep(delay)
|
||||
continue
|
||||
logger.warning("TwelveData request failed %s/%s tf=%s: %s", symbol, exchange, timeframe, e)
|
||||
return []
|
||||
else:
|
||||
return []
|
||||
|
||||
if data.get("status") != "ok" or "values" not in data:
|
||||
code = data.get("code", "")
|
||||
msg = data.get("message", str(data))
|
||||
if code == 429 or "API credits" in msg or "minute limit" in msg:
|
||||
logger.warning("TwelveData rate limit for %s/%s: %s", symbol, exchange, msg)
|
||||
else:
|
||||
logger.warning("TwelveData error %s/%s tf=%s: %s", symbol, exchange, timeframe, msg)
|
||||
return []
|
||||
|
||||
out: List[Dict[str, Any]] = []
|
||||
for v in data["values"]:
|
||||
try:
|
||||
dt_str = v.get("datetime", "")
|
||||
for fmt in ("%Y-%m-%d %H:%M:%S", "%Y-%m-%d"):
|
||||
try:
|
||||
ts = int(datetime.strptime(dt_str, fmt).timestamp())
|
||||
break
|
||||
except ValueError:
|
||||
continue
|
||||
else:
|
||||
continue
|
||||
o = float(v["open"])
|
||||
h = float(v["high"])
|
||||
low = float(v["low"])
|
||||
c = float(v["close"])
|
||||
vol = float(v.get("volume") or 0)
|
||||
if o == 0 and c == 0:
|
||||
continue
|
||||
out.append({
|
||||
"time": ts,
|
||||
"open": round(o, 4),
|
||||
"high": round(h, 4),
|
||||
"low": round(low, 4),
|
||||
"close": round(c, 4),
|
||||
"volume": round(vol, 2),
|
||||
})
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
out.sort(key=lambda x: x["time"])
|
||||
logger.debug("TwelveData returned %d bars for %s/%s tf=%s", len(out), symbol, exchange, timeframe)
|
||||
return out
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# yfinance helpers (globally accessible — Yahoo CDN)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def yf_symbol_from_tencent(tencent_code: str, is_hk: bool) -> str:
|
||||
"""Convert Tencent-style code (SH600519 / SZ000001 / HK00700) to yfinance ticker."""
|
||||
c = (tencent_code or "").strip().upper()
|
||||
if is_hk:
|
||||
num = c.replace("HK", "")
|
||||
if num.isdigit():
|
||||
return str(int(num)).zfill(4) + ".HK"
|
||||
return num + ".HK"
|
||||
if c.startswith("SH"):
|
||||
return c[2:] + ".SS"
|
||||
if c.startswith("SZ"):
|
||||
return c[2:] + ".SZ"
|
||||
digits = c.lstrip("SHSZ")
|
||||
if digits.startswith("6"):
|
||||
return digits + ".SS"
|
||||
return digits + ".SZ"
|
||||
|
||||
|
||||
_YF_INTERVAL_MAP = {
|
||||
"1m": "1m",
|
||||
"5m": "5m",
|
||||
"15m": "15m",
|
||||
"30m": "30m",
|
||||
"1H": "1h",
|
||||
"4H": "1h",
|
||||
"1D": "1d",
|
||||
"1W": "1wk",
|
||||
}
|
||||
|
||||
_YF_DAYS_MAP = {
|
||||
"1m": lambda lim: min(7, max(2, (lim // 240) + 2)),
|
||||
"5m": lambda lim: min(60, max(3, (lim // 48) + 3)),
|
||||
"15m": lambda lim: min(60, max(3, (lim // 16) + 3)),
|
||||
"30m": lambda lim: min(60, max(5, (lim // 8) + 5)),
|
||||
"1H": lambda lim: min(730, max(8, (lim // 4) + 8)),
|
||||
"4H": lambda lim: min(730, max(20, lim + 10)),
|
||||
"1D": lambda lim: min(3650, lim + 10),
|
||||
"1W": lambda lim: min(3650, lim * 7 + 30),
|
||||
}
|
||||
|
||||
|
||||
def _bars_from_yfinance_df(df: Any) -> List[Dict[str, Any]]:
|
||||
"""Convert a yfinance DataFrame (with DatetimeIndex or Date/Datetime column) to bar dicts."""
|
||||
if df is None or getattr(df, "empty", True):
|
||||
return []
|
||||
df = df.reset_index()
|
||||
time_col = None
|
||||
for candidate in ("Datetime", "Date", "index"):
|
||||
if candidate in df.columns:
|
||||
time_col = candidate
|
||||
break
|
||||
if time_col is None:
|
||||
return []
|
||||
out: List[Dict[str, Any]] = []
|
||||
for _, row in df.iterrows():
|
||||
try:
|
||||
tv = row[time_col]
|
||||
if hasattr(tv, "timestamp"):
|
||||
ts = int(tv.timestamp())
|
||||
else:
|
||||
continue
|
||||
o, h, low, c, v = (
|
||||
float(row["Open"]),
|
||||
float(row["High"]),
|
||||
float(row["Low"]),
|
||||
float(row["Close"]),
|
||||
float(row["Volume"]),
|
||||
)
|
||||
if o == 0 and c == 0:
|
||||
continue
|
||||
out.append({
|
||||
"time": ts,
|
||||
"open": round(o, 4),
|
||||
"high": round(h, 4),
|
||||
"low": round(low, 4),
|
||||
"close": round(c, 4),
|
||||
"volume": round(v, 2),
|
||||
})
|
||||
except Exception:
|
||||
continue
|
||||
out.sort(key=lambda x: x["time"])
|
||||
return out
|
||||
|
||||
|
||||
def fetch_yfinance_klines(
|
||||
*,
|
||||
is_hk: bool,
|
||||
tencent_code: str,
|
||||
timeframe: str,
|
||||
limit: int,
|
||||
before_time: Optional[int],
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""Fetch K-lines via yfinance for CN/HK stocks. Globally accessible, no API key needed."""
|
||||
try:
|
||||
import yfinance as yf
|
||||
except ImportError:
|
||||
logger.debug("yfinance not installed; skipping yfinance K-lines")
|
||||
return []
|
||||
|
||||
interval = _YF_INTERVAL_MAP.get(timeframe)
|
||||
if not interval:
|
||||
return []
|
||||
|
||||
yf_sym = yf_symbol_from_tencent(tencent_code, is_hk)
|
||||
effective_limit = limit * 4 if timeframe == "4H" else limit
|
||||
days_func = _YF_DAYS_MAP.get(timeframe, lambda x: x + 10)
|
||||
days = days_func(effective_limit)
|
||||
|
||||
end = datetime.fromtimestamp(int(before_time)) if before_time else datetime.now()
|
||||
start = end - timedelta(days=days)
|
||||
|
||||
df: Any = None
|
||||
for attempt in range(_MAX_ATTEMPTS):
|
||||
try:
|
||||
ticker = yf.Ticker(yf_sym)
|
||||
df = ticker.history(
|
||||
start=start.strftime("%Y-%m-%d"),
|
||||
end=(end + timedelta(days=1)).strftime("%Y-%m-%d"),
|
||||
interval=interval,
|
||||
)
|
||||
break
|
||||
except Exception as e:
|
||||
if attempt + 1 < _MAX_ATTEMPTS and _is_transient(e):
|
||||
delay = min(_BACKOFF_CAP_SEC, _BACKOFF_BASE_SEC * (2 ** attempt))
|
||||
logger.debug(
|
||||
"yfinance transient error %s tf=%s (attempt %s/%s), retry in %.1fs: %s",
|
||||
yf_sym, timeframe, attempt + 1, _MAX_ATTEMPTS, delay, e,
|
||||
)
|
||||
time.sleep(delay)
|
||||
continue
|
||||
logger.warning("yfinance K-line failed %s tf=%s: %s", yf_sym, timeframe, e)
|
||||
return []
|
||||
|
||||
bars = _bars_from_yfinance_df(df)
|
||||
if timeframe == "4H" and bars:
|
||||
bars = _merge_every_n_sorted_bars(bars, 4)
|
||||
logger.debug("yfinance returned %d bars for %s tf=%s", len(bars), yf_sym, timeframe)
|
||||
return bars
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# AkShare helpers (Eastmoney — unreliable from overseas, used as last resort)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _minute_period_str(timeframe: str) -> Optional[str]:
|
||||
return {"1m": "1", "5m": "5", "15m": "15", "30m": "30", "1H": "60", "4H": "60"}.get(timeframe)
|
||||
|
||||
|
||||
def _min_bar_window(timeframe: str, limit: int, before_time: Optional[int]) -> tuple[str, str]:
|
||||
_ = (timeframe, limit)
|
||||
end = datetime.fromtimestamp(int(before_time)) if before_time else datetime.now()
|
||||
start = end - timedelta(days=16)
|
||||
fmt = "%Y-%m-%d %H:%M:%S"
|
||||
return start.strftime(fmt), end.strftime(fmt)
|
||||
|
||||
|
||||
def _bars_from_ak_min_df(df: Any) -> List[Dict[str, Any]]:
|
||||
if df is None or getattr(df, "empty", True):
|
||||
return []
|
||||
cols = [str(x) for x in df.columns]
|
||||
time_c = "时间" if "时间" in cols else (cols[0] if len(cols) > 5 else None)
|
||||
if not time_c:
|
||||
return []
|
||||
|
||||
def _pick(name_zh: str, idx: int) -> str:
|
||||
return name_zh if name_zh in cols else (cols[idx] if len(cols) > idx else "")
|
||||
|
||||
c_open = _pick("开盘", 1)
|
||||
c_close = _pick("收盘", 2)
|
||||
c_high = _pick("最高", 3)
|
||||
c_low = _pick("最低", 4)
|
||||
c_vol = _pick("成交量", 5)
|
||||
if not all((c_open, c_close, c_high, c_low, c_vol)):
|
||||
return []
|
||||
|
||||
out: List[Dict[str, Any]] = []
|
||||
for _, row in df.iterrows():
|
||||
try:
|
||||
t = pd.Timestamp(row[time_c])
|
||||
ts = int(t.timestamp())
|
||||
o, c, h, low, v = float(row[c_open]), float(row[c_close]), float(row[c_high]), float(row[c_low]), float(row[c_vol])
|
||||
out.append({
|
||||
"time": ts,
|
||||
"open": round(o, 4),
|
||||
"high": round(h, 4),
|
||||
"low": round(low, 4),
|
||||
"close": round(c, 4),
|
||||
"volume": round(v, 2),
|
||||
})
|
||||
except Exception:
|
||||
continue
|
||||
out.sort(key=lambda x: x["time"])
|
||||
return out
|
||||
|
||||
|
||||
def _merge_every_n_sorted_bars(bars: List[Dict[str, Any]], n: int) -> List[Dict[str, Any]]:
|
||||
if n <= 1 or len(bars) < n:
|
||||
return bars
|
||||
out: List[Dict[str, Any]] = []
|
||||
i = 0
|
||||
while i + n <= len(bars):
|
||||
chunk = bars[i : i + n]
|
||||
out.append({
|
||||
"time": chunk[0]["time"],
|
||||
"open": chunk[0]["open"],
|
||||
"high": max(b["high"] for b in chunk),
|
||||
"low": min(b["low"] for b in chunk),
|
||||
"close": chunk[-1]["close"],
|
||||
"volume": round(sum(b["volume"] for b in chunk), 2),
|
||||
})
|
||||
i += n
|
||||
return out
|
||||
|
||||
|
||||
def fetch_akshare_minute_klines(
|
||||
*,
|
||||
is_hk: bool,
|
||||
tencent_code: str,
|
||||
timeframe: str,
|
||||
limit: int,
|
||||
before_time: Optional[int],
|
||||
) -> List[Dict[str, Any]]:
|
||||
p = _minute_period_str(timeframe)
|
||||
if p is None:
|
||||
return []
|
||||
try:
|
||||
import akshare as ak # type: ignore
|
||||
except ImportError:
|
||||
logger.debug("akshare not installed; skipping AkShare minute K-lines")
|
||||
return []
|
||||
|
||||
sym = ak_hk_code_from_tencent(tencent_code) if is_hk else ak_a_code_from_tencent(tencent_code)
|
||||
sd, ed = _min_bar_window(timeframe, limit, before_time)
|
||||
adj = "" if p == "1" else "qfq"
|
||||
|
||||
df: Any = None
|
||||
for attempt in range(_MAX_ATTEMPTS):
|
||||
try:
|
||||
if is_hk:
|
||||
df = ak.stock_hk_hist_min_em(symbol=sym, period=p, adjust=adj, start_date=sd, end_date=ed)
|
||||
else:
|
||||
df = ak.stock_zh_a_hist_min_em(symbol=sym, start_date=sd, end_date=ed, period=p, adjust=adj)
|
||||
break
|
||||
except Exception as e:
|
||||
if attempt + 1 < _MAX_ATTEMPTS and _is_transient(e):
|
||||
delay = min(_BACKOFF_CAP_SEC, _BACKOFF_BASE_SEC * (2 ** attempt))
|
||||
logger.debug(
|
||||
"AkShare minute transient error %s tf=%s sym=%s (attempt %s/%s): %s",
|
||||
tencent_code, timeframe, sym, attempt + 1, _MAX_ATTEMPTS, e,
|
||||
)
|
||||
time.sleep(delay)
|
||||
continue
|
||||
logger.warning("AkShare minute K-line failed %s tf=%s sym=%s: %s", tencent_code, timeframe, sym, e)
|
||||
return []
|
||||
|
||||
bars = _bars_from_ak_min_df(df)
|
||||
if timeframe == "4H" and bars:
|
||||
bars = _merge_every_n_sorted_bars(bars, 4)
|
||||
return bars
|
||||
|
||||
|
||||
def fetch_akshare_weekly_klines(
|
||||
*,
|
||||
is_hk: bool,
|
||||
tencent_code: str,
|
||||
limit: int,
|
||||
before_time: Optional[int],
|
||||
) -> List[Dict[str, Any]]:
|
||||
try:
|
||||
import akshare as ak # type: ignore
|
||||
except ImportError:
|
||||
return []
|
||||
|
||||
sym = ak_hk_code_from_tencent(tencent_code) if is_hk else ak_a_code_from_tencent(tencent_code)
|
||||
end = datetime.fromtimestamp(int(before_time)) if before_time else datetime.now()
|
||||
start = end - timedelta(days=max(int(limit or 300), 1) * 14 + 400)
|
||||
start_s = start.strftime("%Y%m%d")
|
||||
end_s = end.strftime("%Y%m%d")
|
||||
|
||||
df: Any = None
|
||||
for attempt in range(_MAX_ATTEMPTS):
|
||||
try:
|
||||
if is_hk:
|
||||
df = ak.stock_hk_hist(symbol=sym, period="weekly", start_date=start_s, end_date=end_s, adjust="qfq")
|
||||
else:
|
||||
df = ak.stock_zh_a_hist(symbol=sym, period="weekly", start_date=start_s, end_date=end_s, adjust="qfq")
|
||||
break
|
||||
except Exception as e:
|
||||
if attempt + 1 < _MAX_ATTEMPTS and _is_transient(e):
|
||||
delay = min(_BACKOFF_CAP_SEC, _BACKOFF_BASE_SEC * (2 ** attempt))
|
||||
logger.debug(
|
||||
"AkShare weekly transient error sym=%s (attempt %s/%s): %s",
|
||||
sym, attempt + 1, _MAX_ATTEMPTS, e,
|
||||
)
|
||||
time.sleep(delay)
|
||||
continue
|
||||
logger.warning("AkShare weekly K-line failed sym=%s: %s", sym, e)
|
||||
return []
|
||||
|
||||
if df is None or getattr(df, "empty", True) or "日期" not in df.columns:
|
||||
return []
|
||||
out: List[Dict[str, Any]] = []
|
||||
for _, row in df.iterrows():
|
||||
try:
|
||||
t = pd.Timestamp(row["日期"])
|
||||
ts = int(t.timestamp())
|
||||
o, c, h, low = float(row["开盘"]), float(row["收盘"]), float(row["最高"]), float(row["最低"])
|
||||
v = float(row["成交量"])
|
||||
out.append({
|
||||
"time": ts,
|
||||
"open": round(o, 4),
|
||||
"high": round(h, 4),
|
||||
"low": round(low, 4),
|
||||
"close": round(c, 4),
|
||||
"volume": round(v, 2),
|
||||
})
|
||||
except Exception:
|
||||
continue
|
||||
out.sort(key=lambda x: x["time"])
|
||||
return out
|
||||
@@ -2,9 +2,10 @@
|
||||
Data source base class
|
||||
Define a unified data source interface
|
||||
"""
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import Dict, List, Any, Optional
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from app.utils.logger import get_logger
|
||||
|
||||
@@ -12,40 +13,27 @@ logger = get_logger(__name__)
|
||||
|
||||
|
||||
# K-line cycle mapping (seconds)
|
||||
TIMEFRAME_SECONDS = {
|
||||
'1m': 60,
|
||||
'5m': 300,
|
||||
'15m': 900,
|
||||
'30m': 1800,
|
||||
'1H': 3600,
|
||||
'4H': 14400,
|
||||
'1D': 86400,
|
||||
'1W': 604800
|
||||
}
|
||||
TIMEFRAME_SECONDS = {"1m": 60, "5m": 300, "15m": 900, "30m": 1800, "1H": 3600, "4H": 14400, "1D": 86400, "1W": 604800}
|
||||
|
||||
|
||||
class BaseDataSource(ABC):
|
||||
"""Data source base class."""
|
||||
|
||||
name: str = "base"
|
||||
|
||||
|
||||
@abstractmethod
|
||||
def get_kline(
|
||||
self,
|
||||
symbol: str,
|
||||
timeframe: str,
|
||||
limit: int,
|
||||
before_time: Optional[int] = None
|
||||
self, symbol: str, timeframe: str, limit: int, before_time: Optional[int] = None
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
Get K-line data
|
||||
|
||||
|
||||
Args:
|
||||
symbol: trading pair/stock code
|
||||
timeframe: time period (1m, 5m, 15m, 30m, 1H, 4H, 1D, 1W)
|
||||
limit: number of data items
|
||||
before_time: Get data before this time (Unix timestamp, seconds)
|
||||
|
||||
|
||||
Returns:
|
||||
K-line data list, format:
|
||||
[{"time": int, "open": float, "high": float, "low": float, "close": float, "volume": float}, ...]
|
||||
@@ -60,82 +48,63 @@ class BaseDataSource(ABC):
|
||||
Implementations may return a dict compatible with CCXT `fetch_ticker` shape (e.g. {'last': ...}).
|
||||
"""
|
||||
raise NotImplementedError("get_ticker is not implemented for this data source")
|
||||
|
||||
|
||||
def format_kline(
|
||||
self,
|
||||
timestamp: int,
|
||||
open_price: float,
|
||||
high: float,
|
||||
low: float,
|
||||
close: float,
|
||||
volume: float
|
||||
self, timestamp: int, open_price: float, high: float, low: float, close: float, volume: float
|
||||
) -> Dict[str, Any]:
|
||||
"""Format a single K-line record."""
|
||||
return {
|
||||
'time': timestamp,
|
||||
'open': round(float(open_price), 4),
|
||||
'high': round(float(high), 4),
|
||||
'low': round(float(low), 4),
|
||||
'close': round(float(close), 4),
|
||||
'volume': round(float(volume), 2)
|
||||
"time": timestamp,
|
||||
"open": round(float(open_price), 4),
|
||||
"high": round(float(high), 4),
|
||||
"low": round(float(low), 4),
|
||||
"close": round(float(close), 4),
|
||||
"volume": round(float(volume), 2),
|
||||
}
|
||||
|
||||
def calculate_time_range(
|
||||
self,
|
||||
timeframe: str,
|
||||
limit: int,
|
||||
buffer_ratio: float = 1.2
|
||||
) -> int:
|
||||
|
||||
def calculate_time_range(self, timeframe: str, limit: int, buffer_ratio: float = 1.2) -> int:
|
||||
"""
|
||||
Calculate the time range (seconds) required to obtain the specified number of K-lines
|
||||
|
||||
|
||||
Args:
|
||||
timeframe: time period
|
||||
limit: number of K-lines
|
||||
buffer_ratio: buffer coefficient
|
||||
|
||||
|
||||
Returns:
|
||||
Time range (seconds)
|
||||
"""
|
||||
seconds_per_candle = TIMEFRAME_SECONDS.get(timeframe, 86400)
|
||||
return int(seconds_per_candle * limit * buffer_ratio)
|
||||
|
||||
|
||||
def filter_and_limit(
|
||||
self,
|
||||
klines: List[Dict[str, Any]],
|
||||
limit: int,
|
||||
before_time: Optional[int] = None
|
||||
self, klines: List[Dict[str, Any]], limit: int, before_time: Optional[int] = None
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
Filter and limit K-line data
|
||||
|
||||
|
||||
Args:
|
||||
klines: K-line data list
|
||||
limit: maximum quantity
|
||||
before_time: Filter data after this time
|
||||
|
||||
|
||||
Returns:
|
||||
Processed K-line data
|
||||
"""
|
||||
# Sort by time
|
||||
klines.sort(key=lambda x: x['time'])
|
||||
|
||||
klines.sort(key=lambda x: x["time"])
|
||||
|
||||
# filter time
|
||||
if before_time:
|
||||
klines = [k for k in klines if k['time'] < before_time]
|
||||
|
||||
klines = [k for k in klines if k["time"] < before_time]
|
||||
|
||||
# Limit quantity (take the latest)
|
||||
if len(klines) > limit:
|
||||
klines = klines[-limit:]
|
||||
|
||||
|
||||
return klines
|
||||
|
||||
def log_result(
|
||||
self,
|
||||
symbol: str,
|
||||
klines: List[Dict[str, Any]],
|
||||
timeframe: str
|
||||
):
|
||||
|
||||
def log_result(self, symbol: str, klines: List[Dict[str, Any]], timeframe: str):
|
||||
"""Record the result log.
|
||||
|
||||
Delayed judgment:
|
||||
|
||||
@@ -13,13 +13,12 @@ characteristic:
|
||||
3. Partition management by data type
|
||||
"""
|
||||
|
||||
import time
|
||||
import logging
|
||||
from typing import Dict, Any, Optional, List
|
||||
from collections import OrderedDict
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime
|
||||
import threading
|
||||
import time
|
||||
from collections import OrderedDict
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Dict, Optional
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -27,15 +26,16 @@ logger = logging.getLogger(__name__)
|
||||
@dataclass
|
||||
class CacheEntry:
|
||||
"""cache entry"""
|
||||
|
||||
data: Any
|
||||
timestamp: float
|
||||
ttl: float
|
||||
hit_count: int = 0
|
||||
|
||||
|
||||
def is_expired(self) -> bool:
|
||||
"""Check if expired"""
|
||||
return time.time() - self.timestamp > self.ttl
|
||||
|
||||
|
||||
def age(self) -> float:
|
||||
"""Return cache age (seconds)"""
|
||||
return time.time() - self.timestamp
|
||||
@@ -44,34 +44,34 @@ class CacheEntry:
|
||||
class DataCache:
|
||||
"""
|
||||
Data Cache Manager
|
||||
|
||||
|
||||
characteristic:
|
||||
- TTL expiration mechanism
|
||||
- Maximum capacity limit
|
||||
- LRU elimination strategy
|
||||
- Thread safety
|
||||
"""
|
||||
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
name: str = "default",
|
||||
default_ttl: float = 600.0, # Default 10 minutes
|
||||
max_size: int = 1000 # Maximum number of cache entries
|
||||
max_size: int = 1000, # Maximum number of cache entries
|
||||
):
|
||||
self.name = name
|
||||
self.default_ttl = default_ttl
|
||||
self.max_size = max_size
|
||||
self._cache: OrderedDict[str, CacheEntry] = OrderedDict()
|
||||
self._lock = threading.RLock()
|
||||
|
||||
|
||||
# Statistics
|
||||
self._hits = 0
|
||||
self._misses = 0
|
||||
|
||||
|
||||
def get(self, key: str) -> Optional[Any]:
|
||||
"""
|
||||
Get cached data
|
||||
|
||||
|
||||
Returns:
|
||||
Cached data, returns None if it does not exist or has expired.
|
||||
"""
|
||||
@@ -79,33 +79,28 @@ class DataCache:
|
||||
if key not in self._cache:
|
||||
self._misses += 1
|
||||
return None
|
||||
|
||||
|
||||
entry = self._cache[key]
|
||||
|
||||
|
||||
# Check if expired
|
||||
if entry.is_expired():
|
||||
del self._cache[key]
|
||||
self._misses += 1
|
||||
logger.debug(f"[cache] {self.name}:{key} expired and was removed")
|
||||
return None
|
||||
|
||||
|
||||
# Update access order (LRU)
|
||||
self._cache.move_to_end(key)
|
||||
entry.hit_count += 1
|
||||
self._hits += 1
|
||||
|
||||
|
||||
logger.debug(f"[cache hit] {self.name}:{key} (age: {entry.age():.0f}s/{entry.ttl:.0f}s)")
|
||||
return entry.data
|
||||
|
||||
def set(
|
||||
self,
|
||||
key: str,
|
||||
data: Any,
|
||||
ttl: Optional[float] = None
|
||||
) -> None:
|
||||
|
||||
def set(self, key: str, data: Any, ttl: Optional[float] = None) -> None:
|
||||
"""
|
||||
Set cache data
|
||||
|
||||
|
||||
Args:
|
||||
key: cache key
|
||||
data: cache data
|
||||
@@ -116,16 +111,12 @@ class DataCache:
|
||||
while len(self._cache) >= self.max_size:
|
||||
oldest_key, _ = self._cache.popitem(last=False)
|
||||
logger.debug(f"[cache] {self.name} reached capacity, evicted: {oldest_key}")
|
||||
|
||||
|
||||
actual_ttl = ttl if ttl is not None else self.default_ttl
|
||||
self._cache[key] = CacheEntry(
|
||||
data=data,
|
||||
timestamp=time.time(),
|
||||
ttl=actual_ttl
|
||||
)
|
||||
|
||||
self._cache[key] = CacheEntry(data=data, timestamp=time.time(), ttl=actual_ttl)
|
||||
|
||||
logger.debug(f"[cache update] {self.name}:{key} TTL={actual_ttl}s")
|
||||
|
||||
|
||||
def delete(self, key: str) -> bool:
|
||||
"""Delete cache entry"""
|
||||
with self._lock:
|
||||
@@ -134,7 +125,7 @@ class DataCache:
|
||||
logger.debug(f"[cache] {self.name}:{key} deleted")
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def clear(self) -> int:
|
||||
"""Clear cache"""
|
||||
with self._lock:
|
||||
@@ -142,35 +133,32 @@ class DataCache:
|
||||
self._cache.clear()
|
||||
logger.info(f"[cache] {self.name} cleared {count} records")
|
||||
return count
|
||||
|
||||
|
||||
def cleanup_expired(self) -> int:
|
||||
"""Clean up expired entries"""
|
||||
with self._lock:
|
||||
expired_keys = [
|
||||
key for key, entry in self._cache.items()
|
||||
if entry.is_expired()
|
||||
]
|
||||
expired_keys = [key for key, entry in self._cache.items() if entry.is_expired()]
|
||||
for key in expired_keys:
|
||||
del self._cache[key]
|
||||
|
||||
|
||||
if expired_keys:
|
||||
logger.debug(f"[cache] {self.name} cleaned {len(expired_keys)} expired records")
|
||||
return len(expired_keys)
|
||||
|
||||
|
||||
def stats(self) -> Dict[str, Any]:
|
||||
"""Get cache statistics"""
|
||||
with self._lock:
|
||||
total_requests = self._hits + self._misses
|
||||
hit_rate = self._hits / total_requests if total_requests > 0 else 0
|
||||
|
||||
|
||||
return {
|
||||
'name': self.name,
|
||||
'size': len(self._cache),
|
||||
'max_size': self.max_size,
|
||||
'hits': self._hits,
|
||||
'misses': self._misses,
|
||||
'hit_rate': f"{hit_rate:.1%}",
|
||||
'default_ttl': self.default_ttl
|
||||
"name": self.name,
|
||||
"size": len(self._cache),
|
||||
"max_size": self.max_size,
|
||||
"hits": self._hits,
|
||||
"misses": self._misses,
|
||||
"hit_rate": f"{hit_rate:.1%}",
|
||||
"default_ttl": self.default_ttl,
|
||||
}
|
||||
|
||||
|
||||
@@ -182,21 +170,21 @@ class DataCache:
|
||||
_realtime_cache = DataCache(
|
||||
name="realtime",
|
||||
default_ttl=1200.0, # 20 minutes
|
||||
max_size=6000
|
||||
max_size=6000,
|
||||
)
|
||||
|
||||
# K-line data caching (5 minutes TTL, caching on demand)
|
||||
_kline_cache = DataCache(
|
||||
name="kline",
|
||||
default_ttl=300.0, # 5 minutes
|
||||
max_size=500 # Up to 500 trading pairs
|
||||
default_ttl=300.0, # 5 minutes
|
||||
max_size=500, # Up to 500 trading pairs
|
||||
)
|
||||
|
||||
# Stock basic information cache (1 day TTL)
|
||||
_stock_info_cache = DataCache(
|
||||
name="stock_info",
|
||||
default_ttl=86400.0, # 24 hours
|
||||
max_size=6000
|
||||
max_size=6000,
|
||||
)
|
||||
|
||||
|
||||
@@ -215,15 +203,10 @@ def get_stock_info_cache() -> DataCache:
|
||||
return _stock_info_cache
|
||||
|
||||
|
||||
def generate_kline_cache_key(
|
||||
symbol: str,
|
||||
timeframe: str,
|
||||
limit: int,
|
||||
before_time: Optional[int] = None
|
||||
) -> str:
|
||||
def generate_kline_cache_key(symbol: str, timeframe: str, limit: int, before_time: Optional[int] = None) -> str:
|
||||
"""
|
||||
Generate K-line cache key
|
||||
|
||||
|
||||
Format: symbol:timeframe:limit[:before_time]
|
||||
"""
|
||||
key = f"{symbol}:{timeframe}:{limit}"
|
||||
|
||||
@@ -13,139 +13,140 @@ HALF_OPEN --Success--> CLOSED
|
||||
HALF_OPEN --Failure--> OPEN
|
||||
"""
|
||||
|
||||
import time
|
||||
import logging
|
||||
from typing import Dict, Any, Optional
|
||||
import time
|
||||
from enum import Enum
|
||||
from typing import Any, Dict, Optional
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class CircuitState(Enum):
|
||||
"""fuse status"""
|
||||
CLOSED = "closed" # normal state
|
||||
OPEN = "open" # Fuse status (not available)
|
||||
|
||||
CLOSED = "closed" # normal state
|
||||
OPEN = "open" # Fuse status (not available)
|
||||
HALF_OPEN = "half_open" # Half-open state (exploratory request)
|
||||
|
||||
|
||||
class CircuitBreaker:
|
||||
"""
|
||||
Circuit Breakers - Manage the blown/cooling status of data sources
|
||||
|
||||
|
||||
Strategy:
|
||||
- Enter the fuse state after N consecutive failures
|
||||
- Skip this data source during the circuit breaker period
|
||||
- Automatically returns to half-open state after cooling time
|
||||
- In the half-open state, if a single success is successful, it will be fully restored, if it fails, the fuse will continue to be broken.
|
||||
"""
|
||||
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
failure_threshold: int = 3, # Continuous failure threshold
|
||||
failure_threshold: int = 3, # Continuous failure threshold
|
||||
cooldown_seconds: float = 300.0, # Cooling time (seconds), default 5 minutes
|
||||
half_open_max_calls: int = 1 # Maximum number of attempts in half-open state
|
||||
half_open_max_calls: int = 1, # Maximum number of attempts in half-open state
|
||||
):
|
||||
self.failure_threshold = failure_threshold
|
||||
self.cooldown_seconds = cooldown_seconds
|
||||
self.half_open_max_calls = half_open_max_calls
|
||||
|
||||
|
||||
# Status of each data source {source_name: {state, failures, last_failure_time, half_open_calls}}
|
||||
self._states: Dict[str, Dict[str, Any]] = {}
|
||||
|
||||
|
||||
def _get_state(self, source: str) -> Dict[str, Any]:
|
||||
"""Get or initialize data source status"""
|
||||
if source not in self._states:
|
||||
self._states[source] = {
|
||||
'state': CircuitState.CLOSED,
|
||||
'failures': 0,
|
||||
'last_failure_time': 0.0,
|
||||
'half_open_calls': 0,
|
||||
'last_error': None
|
||||
"state": CircuitState.CLOSED,
|
||||
"failures": 0,
|
||||
"last_failure_time": 0.0,
|
||||
"half_open_calls": 0,
|
||||
"last_error": None,
|
||||
}
|
||||
return self._states[source]
|
||||
|
||||
|
||||
def is_available(self, source: str) -> bool:
|
||||
"""
|
||||
Check if the data source is available
|
||||
|
||||
|
||||
Return True to indicate that the request can be attempted
|
||||
Return False to indicate that the data source should be skipped
|
||||
"""
|
||||
state = self._get_state(source)
|
||||
current_time = time.time()
|
||||
|
||||
if state['state'] == CircuitState.CLOSED:
|
||||
|
||||
if state["state"] == CircuitState.CLOSED:
|
||||
return True
|
||||
|
||||
if state['state'] == CircuitState.OPEN:
|
||||
|
||||
if state["state"] == CircuitState.OPEN:
|
||||
# Check cool down time
|
||||
time_since_failure = current_time - state['last_failure_time']
|
||||
time_since_failure = current_time - state["last_failure_time"]
|
||||
if time_since_failure >= self.cooldown_seconds:
|
||||
# Cooling is completed and enters the half-open state
|
||||
state['state'] = CircuitState.HALF_OPEN
|
||||
state['half_open_calls'] = 0
|
||||
state["state"] = CircuitState.HALF_OPEN
|
||||
state["half_open_calls"] = 0
|
||||
logger.info(f"[circuit breaker] {source} cooldown finished, entering half-open state")
|
||||
return True
|
||||
else:
|
||||
remaining = self.cooldown_seconds - time_since_failure
|
||||
logger.debug(f"[circuit breaker] {source} is open, remaining cooldown: {remaining:.0f}s")
|
||||
return False
|
||||
|
||||
if state['state'] == CircuitState.HALF_OPEN:
|
||||
|
||||
if state["state"] == CircuitState.HALF_OPEN:
|
||||
# Limit the number of requests in the half-open state
|
||||
if state['half_open_calls'] < self.half_open_max_calls:
|
||||
if state["half_open_calls"] < self.half_open_max_calls:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
return True
|
||||
|
||||
|
||||
def record_success(self, source: str) -> None:
|
||||
"""Log successful request"""
|
||||
state = self._get_state(source)
|
||||
|
||||
if state['state'] == CircuitState.HALF_OPEN:
|
||||
|
||||
if state["state"] == CircuitState.HALF_OPEN:
|
||||
# Successful in half-open state, full recovery
|
||||
logger.info(f"[circuit breaker] {source} request succeeded in half-open state, fully recovered")
|
||||
|
||||
|
||||
# reset state
|
||||
state['state'] = CircuitState.CLOSED
|
||||
state['failures'] = 0
|
||||
state['half_open_calls'] = 0
|
||||
state['last_error'] = None
|
||||
|
||||
state["state"] = CircuitState.CLOSED
|
||||
state["failures"] = 0
|
||||
state["half_open_calls"] = 0
|
||||
state["last_error"] = None
|
||||
|
||||
def record_failure(self, source: str, error: Optional[str] = None) -> None:
|
||||
"""Logging failed requests"""
|
||||
state = self._get_state(source)
|
||||
current_time = time.time()
|
||||
|
||||
state['failures'] += 1
|
||||
state['last_failure_time'] = current_time
|
||||
state['last_error'] = error
|
||||
|
||||
if state['state'] == CircuitState.HALF_OPEN:
|
||||
|
||||
state["failures"] += 1
|
||||
state["last_failure_time"] = current_time
|
||||
state["last_error"] = error
|
||||
|
||||
if state["state"] == CircuitState.HALF_OPEN:
|
||||
# Fails in half-open state and continues to fuse
|
||||
state['state'] = CircuitState.OPEN
|
||||
state['half_open_calls'] = 0
|
||||
logger.warning(f"[circuit breaker] {source} request failed in half-open state, staying open for {self.cooldown_seconds}s")
|
||||
elif state['failures'] >= self.failure_threshold:
|
||||
state["state"] = CircuitState.OPEN
|
||||
state["half_open_calls"] = 0
|
||||
logger.warning(
|
||||
f"[circuit breaker] {source} request failed in half-open state, staying open for {self.cooldown_seconds}s"
|
||||
)
|
||||
elif state["failures"] >= self.failure_threshold:
|
||||
# reaches the threshold and enters the circuit breaker
|
||||
state['state'] = CircuitState.OPEN
|
||||
logger.warning(f"[circuit breaker] {source} failed {state['failures']} times consecutively and is now open "
|
||||
f"(cooldown {self.cooldown_seconds}s)")
|
||||
state["state"] = CircuitState.OPEN
|
||||
logger.warning(
|
||||
f"[circuit breaker] {source} failed {state['failures']} times consecutively and is now open "
|
||||
f"(cooldown {self.cooldown_seconds}s)"
|
||||
)
|
||||
if error:
|
||||
logger.warning(f"[circuit breaker] last error: {error}")
|
||||
|
||||
|
||||
def get_status(self) -> Dict[str, Dict[str, Any]]:
|
||||
"""Get all data source status"""
|
||||
return {
|
||||
source: {
|
||||
'state': info['state'].value,
|
||||
'failures': info['failures'],
|
||||
'last_error': info['last_error']
|
||||
}
|
||||
source: {"state": info["state"].value, "failures": info["failures"], "last_error": info["last_error"]}
|
||||
for source, info in self._states.items()
|
||||
}
|
||||
|
||||
|
||||
def reset(self, source: Optional[str] = None) -> None:
|
||||
"""Reset fuse status"""
|
||||
if source:
|
||||
@@ -163,9 +164,9 @@ class CircuitBreaker:
|
||||
|
||||
# Real-time market circuit breaker (more stringent strategy)
|
||||
_realtime_circuit_breaker = CircuitBreaker(
|
||||
failure_threshold=2, # Failed 2 times in a row
|
||||
cooldown_seconds=180.0, # Cool for 3 minutes
|
||||
half_open_max_calls=1
|
||||
failure_threshold=2, # Failed 2 times in a row
|
||||
cooldown_seconds=180.0, # Cool for 3 minutes
|
||||
half_open_max_calls=1,
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -1,292 +0,0 @@
|
||||
"""
|
||||
A-share / HK share fundamentals — multi-tier fallback.
|
||||
|
||||
Priority (when TWELVE_DATA_API_KEY configured):
|
||||
Twelve Data /statistics + /profile → AkShare (Eastmoney, fragile overseas)
|
||||
|
||||
Without API key:
|
||||
AkShare only (may fail from overseas servers)
|
||||
|
||||
Keys are aligned with MarketDataCollector expectations (pe_ratio, pb_ratio, etc.).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
import time
|
||||
from typing import Any, Dict, Optional
|
||||
|
||||
import requests
|
||||
|
||||
from app.data_sources.asia_stock_kline import (
|
||||
_get_twelve_data_api_key,
|
||||
_td_symbol_and_exchange,
|
||||
ak_a_code_from_tencent,
|
||||
ak_hk_code_from_tencent,
|
||||
)
|
||||
from app.utils.logger import get_logger
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
_TD_TIMEOUT = 15
|
||||
_TD_MAX_ATTEMPTS = 2
|
||||
_TD_BACKOFF_SEC = 2.0
|
||||
|
||||
|
||||
def _float_clean(x: Any) -> Optional[float]:
|
||||
if x is None or x == "":
|
||||
return None
|
||||
try:
|
||||
v = float(x)
|
||||
if math.isnan(v) or math.isinf(v):
|
||||
return None
|
||||
return v
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Twelve Data fundamentals (globally stable, paid)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _td_request(endpoint: str, symbol: str, exchange: str) -> Optional[Dict[str, Any]]:
|
||||
"""Generic Twelve Data GET with retry."""
|
||||
api_key = _get_twelve_data_api_key()
|
||||
if not api_key:
|
||||
return None
|
||||
url = f"https://api.twelvedata.com{endpoint}"
|
||||
params = {"symbol": symbol, "exchange": exchange, "apikey": api_key}
|
||||
for attempt in range(_TD_MAX_ATTEMPTS):
|
||||
try:
|
||||
resp = requests.get(url, params=params, timeout=_TD_TIMEOUT)
|
||||
data = resp.json()
|
||||
if data.get("status") == "error":
|
||||
code = data.get("code", "")
|
||||
msg = (data.get("message") or "")[:120]
|
||||
if code == 429 or "API credits" in msg or "minute limit" in msg:
|
||||
logger.warning("TwelveData rate limit on %s %s/%s: %s", endpoint, symbol, exchange, msg)
|
||||
else:
|
||||
logger.debug("TwelveData %s error %s/%s: %s", endpoint, symbol, exchange, msg)
|
||||
return None
|
||||
return data
|
||||
except Exception as e:
|
||||
if attempt + 1 < _TD_MAX_ATTEMPTS:
|
||||
time.sleep(_TD_BACKOFF_SEC)
|
||||
continue
|
||||
logger.warning("TwelveData %s request failed %s/%s: %s", endpoint, symbol, exchange, e)
|
||||
return None
|
||||
|
||||
|
||||
def fetch_twelvedata_fundamental(tencent_code: str, is_hk: bool) -> Dict[str, Any]:
|
||||
"""Fetch PE/PB/PS/PEG/ROE/margin/market_cap/52w from Twelve Data /statistics."""
|
||||
symbol, exchange = _td_symbol_and_exchange(tencent_code, is_hk)
|
||||
data = _td_request("/statistics", symbol, exchange)
|
||||
if not data or "statistics" not in data:
|
||||
return {}
|
||||
|
||||
stats = data["statistics"]
|
||||
result: Dict[str, Any] = {"source": "twelvedata"}
|
||||
|
||||
vm = stats.get("valuations_metrics") or {}
|
||||
result["market_cap"] = _float_clean(vm.get("market_capitalization"))
|
||||
result["pe_ratio"] = _float_clean(vm.get("trailing_pe"))
|
||||
result["forward_pe"] = _float_clean(vm.get("forward_pe"))
|
||||
result["pb_ratio"] = _float_clean(vm.get("price_to_book_mrq"))
|
||||
result["ps_ratio"] = _float_clean(vm.get("price_to_sales_ttm"))
|
||||
result["peg"] = _float_clean(vm.get("peg_ratio"))
|
||||
result["enterprise_value"] = _float_clean(vm.get("enterprise_value"))
|
||||
|
||||
fin = stats.get("financials") or {}
|
||||
result["profit_margin"] = _float_clean(fin.get("profit_margin"))
|
||||
result["gross_margin"] = _float_clean(fin.get("gross_margin"))
|
||||
result["operating_margin"] = _float_clean(fin.get("operating_margin"))
|
||||
result["roe"] = _float_clean(fin.get("return_on_equity_ttm"))
|
||||
result["roa"] = _float_clean(fin.get("return_on_assets_ttm"))
|
||||
|
||||
ss = stats.get("stock_statistics") or {}
|
||||
result["total_shares"] = _float_clean(ss.get("shares_outstanding"))
|
||||
result["float_shares"] = _float_clean(ss.get("float_shares"))
|
||||
|
||||
sp = stats.get("stock_price_summary") or {}
|
||||
result["52w_high"] = _float_clean(sp.get("fifty_two_week_high"))
|
||||
result["52w_low"] = _float_clean(sp.get("fifty_two_week_low"))
|
||||
result["beta"] = _float_clean(sp.get("beta"))
|
||||
|
||||
div = stats.get("dividends_and_splits") or {}
|
||||
result["dividend_yield"] = _float_clean(div.get("trailing_annual_dividend_yield"))
|
||||
result["dividend_rate"] = _float_clean(div.get("trailing_annual_dividend_rate"))
|
||||
|
||||
non_null = sum(1 for v in result.values() if v is not None and v != "twelvedata")
|
||||
logger.debug("TwelveData /statistics %s/%s: %d non-null fields", symbol, exchange, non_null)
|
||||
return result
|
||||
|
||||
|
||||
def fetch_twelvedata_profile(tencent_code: str, is_hk: bool) -> Dict[str, Any]:
|
||||
"""Fetch company info from Twelve Data /profile."""
|
||||
symbol, exchange = _td_symbol_and_exchange(tencent_code, is_hk)
|
||||
data = _td_request("/profile", symbol, exchange)
|
||||
if not data or not data.get("name"):
|
||||
return {}
|
||||
|
||||
out: Dict[str, Any] = {"source": "twelvedata"}
|
||||
for src, dst in (
|
||||
("name", "name"),
|
||||
("industry", "industry"),
|
||||
("sector", "sector"),
|
||||
("website", "website"),
|
||||
("description", "description"),
|
||||
("employees", "employees"),
|
||||
("name", "full_name"),
|
||||
):
|
||||
v = data.get(src)
|
||||
if v is not None and str(v).strip():
|
||||
out[dst] = str(v).strip() if isinstance(v, str) else v
|
||||
|
||||
country = data.get("country")
|
||||
if country:
|
||||
out["country"] = country
|
||||
|
||||
logger.debug("TwelveData /profile %s/%s: name=%s industry=%s", symbol, exchange, out.get("name"), out.get("industry"))
|
||||
return out
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# AkShare fundamentals (Eastmoney — fragile overseas, used as fallback)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _eastmoney_a_em_symbol(tencent_code: str) -> str:
|
||||
c = ak_a_code_from_tencent(tencent_code)
|
||||
c = (c or "").zfill(6)
|
||||
if c.startswith("6"):
|
||||
return "SH" + c
|
||||
return "SZ" + c
|
||||
|
||||
|
||||
def _individual_info_map(symbol_6: str) -> Dict[str, Any]:
|
||||
out: Dict[str, Any] = {}
|
||||
try:
|
||||
import akshare as ak # type: ignore
|
||||
df = ak.stock_individual_info_em(symbol=symbol_6)
|
||||
except Exception as e:
|
||||
logger.debug("stock_individual_info_em failed %s: %s", symbol_6, e)
|
||||
return out
|
||||
if df is None or getattr(df, "empty", True) or len(df.columns) < 2:
|
||||
return out
|
||||
kcol, vcol = df.columns[0], df.columns[1]
|
||||
for _, row in df.iterrows():
|
||||
try:
|
||||
k = str(row[kcol]).strip()
|
||||
if k:
|
||||
out[k] = row[vcol]
|
||||
except Exception:
|
||||
continue
|
||||
return out
|
||||
|
||||
|
||||
def fetch_cn_fundamental_akshare(tencent_code: str) -> Dict[str, Any]:
|
||||
"""PE/PB/PS, market cap, ROE proxy, EPS for A-share (best-effort)."""
|
||||
sym6 = ak_a_code_from_tencent(tencent_code)
|
||||
if not sym6:
|
||||
return {}
|
||||
result: Dict[str, Any] = {"source": "akshare_em"}
|
||||
info = _individual_info_map(sym6)
|
||||
if info:
|
||||
result["market_cap"] = _float_clean(info.get("总市值"))
|
||||
result["float_market_cap"] = _float_clean(info.get("流通市值"))
|
||||
ind = info.get("行业")
|
||||
if ind is not None and str(ind).strip():
|
||||
result["industry"] = str(ind).strip()
|
||||
result["total_shares"] = _float_clean(info.get("总股本"))
|
||||
result["float_shares"] = _float_clean(info.get("流通股"))
|
||||
|
||||
em_sym = _eastmoney_a_em_symbol(tencent_code)
|
||||
try:
|
||||
import akshare as ak # type: ignore
|
||||
vdf = ak.stock_zh_valuation_comparison_em(symbol=em_sym)
|
||||
except Exception as e:
|
||||
logger.debug("stock_zh_valuation_comparison_em failed %s: %s", em_sym, e)
|
||||
vdf = None
|
||||
|
||||
if vdf is not None and not vdf.empty and "代码" in vdf.columns:
|
||||
hit = vdf[vdf["代码"].astype(str).str.replace(".0", "", regex=False).str.zfill(6) == sym6.zfill(6)]
|
||||
if not hit.empty:
|
||||
r = hit.iloc[0]
|
||||
pe = _float_clean(r.get("市盈率-TTM"))
|
||||
if pe is not None:
|
||||
result["pe_ratio"] = pe
|
||||
pb = _float_clean(r.get("市净率-MRQ"))
|
||||
if pb is not None:
|
||||
result["pb_ratio"] = pb
|
||||
ps = _float_clean(r.get("市销率-TTM"))
|
||||
if ps is not None:
|
||||
result["ps_ratio"] = ps
|
||||
peg = _float_clean(r.get("PEG"))
|
||||
if peg is not None:
|
||||
result["peg"] = peg
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def fetch_hk_fundamental_akshare(tencent_code: str) -> Dict[str, Any]:
|
||||
hk5 = ak_hk_code_from_tencent(tencent_code)
|
||||
if not hk5:
|
||||
return {}
|
||||
result: Dict[str, Any] = {"source": "akshare_em"}
|
||||
try:
|
||||
import akshare as ak # type: ignore
|
||||
df = ak.stock_hk_financial_indicator_em(symbol=hk5)
|
||||
except Exception as e:
|
||||
logger.debug("stock_hk_financial_indicator_em failed %s: %s", hk5, e)
|
||||
return result
|
||||
if df is None or df.empty:
|
||||
return result
|
||||
r = df.iloc[0]
|
||||
result["pe_ratio"] = _float_clean(r.get("市盈率"))
|
||||
result["pb_ratio"] = _float_clean(r.get("市净率"))
|
||||
result["eps"] = _float_clean(r.get("基本每股收益(元)"))
|
||||
result["roe"] = _float_clean(r.get("股东权益回报率(%)"))
|
||||
result["profit_margin"] = _float_clean(r.get("销售净利率(%)"))
|
||||
mcap = _float_clean(r.get("总市值(港元)")) or _float_clean(r.get("港股市值(港元)"))
|
||||
if mcap is not None:
|
||||
result["market_cap"] = mcap
|
||||
result["dividend_yield"] = _float_clean(r.get("股息率TTM(%)"))
|
||||
return result
|
||||
|
||||
|
||||
def fetch_cn_company_extras(tencent_code: str) -> Dict[str, Any]:
|
||||
sym6 = ak_a_code_from_tencent(tencent_code)
|
||||
if not sym6:
|
||||
return {}
|
||||
info = _individual_info_map(sym6)
|
||||
out: Dict[str, Any] = {}
|
||||
if info.get("行业"):
|
||||
out["industry"] = str(info["行业"]).strip()
|
||||
if info.get("上市时间"):
|
||||
out["ipo_date"] = str(info["上市时间"]).strip()
|
||||
return out
|
||||
|
||||
|
||||
def fetch_hk_company_extras(tencent_code: str) -> Dict[str, Any]:
|
||||
hk5 = ak_hk_code_from_tencent(tencent_code)
|
||||
if not hk5:
|
||||
return {}
|
||||
out: Dict[str, Any] = {}
|
||||
try:
|
||||
import akshare as ak # type: ignore
|
||||
df = ak.stock_hk_company_profile_em(symbol=hk5)
|
||||
except Exception as e:
|
||||
logger.debug("stock_hk_company_profile_em failed %s: %s", hk5, e)
|
||||
return out
|
||||
if df is None or df.empty:
|
||||
return out
|
||||
r = df.iloc[0]
|
||||
for key, col in (
|
||||
("industry", "所属行业"),
|
||||
("ipo_date", "公司成立日期"),
|
||||
("website", "公司网址"),
|
||||
("full_name", "公司名称"),
|
||||
):
|
||||
v = r.get(col)
|
||||
if v is not None and str(v).strip():
|
||||
out[key] = str(v).strip()
|
||||
return out
|
||||
@@ -1,99 +0,0 @@
|
||||
"""
|
||||
中国A股数据源 — 多层 fallback
|
||||
|
||||
有 TWELVE_DATA_API_KEY:
|
||||
所有周期 → Twelve Data（主） → 腾讯日/周线 → yfinance → AkShare
|
||||
|
||||
无 API Key:
|
||||
分钟/小时 → yfinance → AkShare
|
||||
日/周线 → 腾讯 fqkline → yfinance → AkShare
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Dict, List, Any, Optional
|
||||
|
||||
from app.data_sources.base import BaseDataSource
|
||||
from app.data_sources.tencent import normalize_cn_code, fetch_quote, parse_quote_to_ticker, fetch_kline, tencent_kline_rows_to_dicts
|
||||
from app.data_sources.asia_stock_kline import (
|
||||
normalize_chart_timeframe,
|
||||
fetch_twelvedata_klines,
|
||||
fetch_yfinance_klines,
|
||||
fetch_akshare_minute_klines,
|
||||
fetch_akshare_weekly_klines,
|
||||
)
|
||||
from app.utils.logger import get_logger
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
|
||||
class CNStockDataSource(BaseDataSource):
|
||||
"""A股数据源（TwelveData + Tencent + yfinance + AkShare）"""
|
||||
|
||||
name = "CNStock/multi-source"
|
||||
|
||||
def get_ticker(self, symbol: str) -> Dict[str, Any]:
|
||||
code = normalize_cn_code(symbol)
|
||||
parts = fetch_quote(code)
|
||||
if not parts:
|
||||
return {"last": 0, "symbol": code}
|
||||
t = parse_quote_to_ticker(parts)
|
||||
return {
|
||||
"last": t.get("last", 0),
|
||||
"change": t.get("change", 0),
|
||||
"changePercent": t.get("changePercent", 0),
|
||||
"high": t.get("high", 0),
|
||||
"low": t.get("low", 0),
|
||||
"open": t.get("open", 0),
|
||||
"previousClose": t.get("previousClose", 0),
|
||||
"name": t.get("name", ""),
|
||||
"symbol": code,
|
||||
}
|
||||
|
||||
def get_kline(
|
||||
self,
|
||||
symbol: str,
|
||||
timeframe: str,
|
||||
limit: int,
|
||||
before_time: Optional[int] = None,
|
||||
) -> List[Dict[str, Any]]:
|
||||
code = normalize_cn_code(symbol)
|
||||
tf = normalize_chart_timeframe(timeframe)
|
||||
lim = max(int(limit or 300), 1)
|
||||
|
||||
# Tier 1: Twelve Data (paid, most reliable)
|
||||
rows = fetch_twelvedata_klines(
|
||||
is_hk=False, tencent_code=code, timeframe=tf, limit=lim, before_time=before_time
|
||||
)
|
||||
if rows:
|
||||
return self.filter_and_limit(rows, limit=lim, before_time=before_time)
|
||||
|
||||
# Tier 2: Tencent for daily/weekly (fast, free)
|
||||
if tf in ("1D", "1W"):
|
||||
tf_map = {"1D": "day", "1W": "week"}
|
||||
period = tf_map.get(tf, "day")
|
||||
raw_rows = fetch_kline(code, period=period, count=lim, adj="qfq")
|
||||
out = tencent_kline_rows_to_dicts(raw_rows)
|
||||
if out:
|
||||
return self.filter_and_limit(out, limit=lim, before_time=before_time)
|
||||
|
||||
# Tier 3: yfinance (works when Yahoo not rate-limited)
|
||||
rows = fetch_yfinance_klines(
|
||||
is_hk=False, tencent_code=code, timeframe=tf, limit=lim, before_time=before_time
|
||||
)
|
||||
if rows:
|
||||
return self.filter_and_limit(rows, limit=lim, before_time=before_time)
|
||||
|
||||
# Tier 4: AkShare (fragile overseas, last resort)
|
||||
if tf in ("1m", "5m", "15m", "30m", "1H", "4H"):
|
||||
rows = fetch_akshare_minute_klines(
|
||||
is_hk=False, tencent_code=code, timeframe=tf, limit=lim, before_time=before_time
|
||||
)
|
||||
elif tf == "1W":
|
||||
rows = fetch_akshare_weekly_klines(
|
||||
is_hk=False, tencent_code=code, limit=lim, before_time=before_time
|
||||
)
|
||||
else:
|
||||
rows = []
|
||||
|
||||
return self.filter_and_limit(rows, limit=lim, before_time=before_time)
|
||||
@@ -2,75 +2,71 @@
|
||||
Cryptocurrency data source
|
||||
Get data using CCXT (Coinbase)
|
||||
"""
|
||||
from typing import Dict, List, Any, Optional, Tuple
|
||||
|
||||
from datetime import datetime, timedelta
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
|
||||
import ccxt
|
||||
|
||||
from app.data_sources.base import BaseDataSource, TIMEFRAME_SECONDS
|
||||
from app.config import CCXTConfig
|
||||
from app.data_sources.base import TIMEFRAME_SECONDS, BaseDataSource
|
||||
from app.utils.logger import get_logger
|
||||
from app.config import CCXTConfig, APIKeys
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
|
||||
class CryptoDataSource(BaseDataSource):
|
||||
"""Cryptocurrency data source"""
|
||||
|
||||
|
||||
name = "Crypto/CCXT"
|
||||
|
||||
|
||||
# time period mapping
|
||||
TIMEFRAME_MAP = CCXTConfig.TIMEFRAME_MAP
|
||||
|
||||
|
||||
# List of common quote currencies (sorted by priority)
|
||||
COMMON_QUOTES = ['USDT', 'USD', 'BTC', 'ETH', 'BUSD', 'USDC', 'BNB', 'EUR', 'GBP']
|
||||
|
||||
COMMON_QUOTES = ["USDT", "USD", "BTC", "ETH", "BUSD", "USDC", "BNB", "EUR", "GBP"]
|
||||
|
||||
def __init__(self):
|
||||
config = {
|
||||
'timeout': CCXTConfig.TIMEOUT,
|
||||
'enableRateLimit': CCXTConfig.ENABLE_RATE_LIMIT
|
||||
}
|
||||
|
||||
config = {"timeout": CCXTConfig.TIMEOUT, "enableRateLimit": CCXTConfig.ENABLE_RATE_LIMIT}
|
||||
|
||||
# If a proxy is configured
|
||||
if CCXTConfig.PROXY:
|
||||
config['proxies'] = {
|
||||
'http': CCXTConfig.PROXY,
|
||||
'https': CCXTConfig.PROXY
|
||||
}
|
||||
|
||||
config["proxies"] = {"http": CCXTConfig.PROXY, "https": CCXTConfig.PROXY}
|
||||
|
||||
exchange_id = CCXTConfig.DEFAULT_EXCHANGE
|
||||
|
||||
|
||||
# Dynamically loading exchange classes
|
||||
if not hasattr(ccxt, exchange_id):
|
||||
logger.warning(f"CCXT exchange '{exchange_id}' not found, falling back to 'coinbase'")
|
||||
exchange_id = 'coinbase'
|
||||
|
||||
exchange_id = "coinbase"
|
||||
|
||||
exchange_class = getattr(ccxt, exchange_id)
|
||||
self.exchange = exchange_class(config)
|
||||
|
||||
|
||||
# Lazy loading of markets (loaded on first use)
|
||||
self._markets_loaded = False
|
||||
self._markets_cache = None
|
||||
|
||||
|
||||
def _ensure_markets_loaded(self) -> bool:
|
||||
"""Make sure markets are loaded (for symbol verification)"""
|
||||
if self._markets_loaded and self._markets_cache is not None:
|
||||
return True
|
||||
|
||||
|
||||
try:
|
||||
# Some exchanges require explicit loading of markets
|
||||
if hasattr(self.exchange, 'load_markets'):
|
||||
if hasattr(self.exchange, "load_markets"):
|
||||
self.exchange.load_markets(reload=False)
|
||||
self._markets_cache = getattr(self.exchange, 'markets', {})
|
||||
self._markets_cache = getattr(self.exchange, "markets", {})
|
||||
self._markets_loaded = True
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.debug(f"Failed to load markets for {self.exchange.id}: {e}")
|
||||
return False
|
||||
|
||||
|
||||
def _normalize_symbol(self, symbol: str) -> Tuple[str, str]:
|
||||
"""
|
||||
Normalized symbol format, returns (normalized_symbol, base_currency)
|
||||
|
||||
|
||||
Handles various input formats:
|
||||
- BTC/USDT -> BTC/USDT
|
||||
- BTCUSDT -> BTC/USDT
|
||||
@@ -79,69 +75,69 @@ class CryptoDataSource(BaseDataSource):
|
||||
- PI, TRX -> PI/USDT, TRX/USDT
|
||||
"""
|
||||
if not symbol:
|
||||
return '', ''
|
||||
|
||||
return "", ""
|
||||
|
||||
sym = symbol.strip()
|
||||
|
||||
|
||||
# Remove swap/futures suffix
|
||||
if ':' in sym:
|
||||
sym = sym.split(':', 1)[0]
|
||||
|
||||
if ":" in sym:
|
||||
sym = sym.split(":", 1)[0]
|
||||
|
||||
sym = sym.upper()
|
||||
|
||||
|
||||
# If there is already a separator, parse it directly
|
||||
if '/' in sym:
|
||||
parts = sym.split('/', 1)
|
||||
if "/" in sym:
|
||||
parts = sym.split("/", 1)
|
||||
base = parts[0].strip()
|
||||
quote = parts[1].strip() if len(parts) > 1 else ''
|
||||
quote = parts[1].strip() if len(parts) > 1 else ""
|
||||
if base and quote:
|
||||
return f"{base}/{quote}", base
|
||||
|
||||
|
||||
# Try to identify from common quote currencies
|
||||
for quote in self.COMMON_QUOTES:
|
||||
if sym.endswith(quote) and len(sym) > len(quote):
|
||||
base = sym[:-len(quote)]
|
||||
base = sym[: -len(quote)]
|
||||
if base:
|
||||
return f"{base}/{quote}", base
|
||||
|
||||
|
||||
# If not recognized, USDT will be used by default.
|
||||
return f"{sym}/USDT", sym
|
||||
|
||||
def _find_valid_symbol(self, base: str, preferred_quote: str = 'USDT') -> Optional[str]:
|
||||
|
||||
def _find_valid_symbol(self, base: str, preferred_quote: str = "USDT") -> Optional[str]:
|
||||
"""
|
||||
Find valid symbols in the exchange's markets
|
||||
|
||||
|
||||
Args:
|
||||
base: base currency (e.g. 'PI', 'TRX')
|
||||
preferred_quote: preferred quote currency
|
||||
|
||||
|
||||
Returns:
|
||||
A valid symbol found, or None if not found
|
||||
"""
|
||||
if not self._ensure_markets_loaded():
|
||||
return None
|
||||
|
||||
|
||||
markets = self._markets_cache or {}
|
||||
if not markets:
|
||||
return None
|
||||
|
||||
|
||||
# Try different quote currencies by priority
|
||||
quotes_to_try = [preferred_quote] + [q for q in self.COMMON_QUOTES if q != preferred_quote]
|
||||
|
||||
|
||||
for quote in quotes_to_try:
|
||||
candidate = f"{base}/{quote}"
|
||||
if candidate in markets:
|
||||
market = markets[candidate]
|
||||
# Check if the market is active
|
||||
if market.get('active', True):
|
||||
if market.get("active", True):
|
||||
return candidate
|
||||
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def _normalize_symbol_for_exchange(self, symbol: str) -> str:
|
||||
"""
|
||||
Standardize symbols based on exchange characteristics
|
||||
|
||||
|
||||
Symbol format requirements for different exchanges:
|
||||
- Binance: BTC/USDT (standard format)
|
||||
- OKX: BTC/USDT (standard format, but some currencies may not be supported)
|
||||
@@ -150,28 +146,28 @@ class CryptoDataSource(BaseDataSource):
|
||||
- Bitfinex: tBTCUST (special format)
|
||||
"""
|
||||
normalized, base = self._normalize_symbol(symbol)
|
||||
|
||||
|
||||
if not normalized or not base:
|
||||
return symbol
|
||||
|
||||
exchange_id = getattr(self.exchange, 'id', '').lower()
|
||||
|
||||
|
||||
exchange_id = getattr(self.exchange, "id", "").lower()
|
||||
|
||||
# Special handling: symbol mapping for certain exchanges
|
||||
if exchange_id == 'coinbase':
|
||||
if exchange_id == "coinbase":
|
||||
# Coinbase usually uses USD instead of USDT
|
||||
if normalized.endswith('/USDT'):
|
||||
usd_version = normalized.replace('/USDT', '/USD')
|
||||
if normalized.endswith("/USDT"):
|
||||
usd_version = normalized.replace("/USDT", "/USD")
|
||||
if self._ensure_markets_loaded():
|
||||
markets = self._markets_cache or {}
|
||||
if usd_version in markets:
|
||||
return usd_version
|
||||
|
||||
|
||||
# Try to find a valid symbol on the exchange
|
||||
if self._ensure_markets_loaded():
|
||||
valid_symbol = self._find_valid_symbol(base, normalized.split('/')[1] if '/' in normalized else 'USDT')
|
||||
valid_symbol = self._find_valid_symbol(base, normalized.split("/")[1] if "/" in normalized else "USDT")
|
||||
if valid_symbol:
|
||||
return valid_symbol
|
||||
|
||||
|
||||
return normalized
|
||||
|
||||
def get_ticker(self, symbol: str) -> Dict[str, Any]:
|
||||
@@ -184,15 +180,15 @@ class CryptoDataSource(BaseDataSource):
|
||||
- Automatically adapt to the symbol format requirements of different exchanges
|
||||
"""
|
||||
if not symbol or not symbol.strip():
|
||||
return {'last': 0, 'symbol': symbol}
|
||||
|
||||
return {"last": 0, "symbol": symbol}
|
||||
|
||||
# normalized notation
|
||||
normalized = self._normalize_symbol_for_exchange(symbol)
|
||||
|
||||
|
||||
if not normalized:
|
||||
logger.warning(f"Failed to normalize symbol: {symbol}")
|
||||
return {'last': 0, 'symbol': symbol}
|
||||
|
||||
return {"last": 0, "symbol": symbol}
|
||||
|
||||
# Try to get ticker
|
||||
try:
|
||||
ticker = self.exchange.fetch_ticker(normalized)
|
||||
@@ -200,97 +196,95 @@ class CryptoDataSource(BaseDataSource):
|
||||
return ticker
|
||||
except Exception as e:
|
||||
error_msg = str(e).lower()
|
||||
is_symbol_error = any(keyword in error_msg for keyword in [
|
||||
'does not have market symbol',
|
||||
'symbol not found',
|
||||
'invalid symbol',
|
||||
'market does not exist',
|
||||
'trading pair not found'
|
||||
])
|
||||
|
||||
is_symbol_error = any(
|
||||
keyword in error_msg
|
||||
for keyword in [
|
||||
"does not have market symbol",
|
||||
"symbol not found",
|
||||
"invalid symbol",
|
||||
"market does not exist",
|
||||
"trading pair not found",
|
||||
]
|
||||
)
|
||||
|
||||
if is_symbol_error:
|
||||
# Try to find alternative symbols
|
||||
base = normalized.split('/')[0] if '/' in normalized else normalized
|
||||
base = normalized.split("/")[0] if "/" in normalized else normalized
|
||||
if self._ensure_markets_loaded():
|
||||
valid_symbol = self._find_valid_symbol(base)
|
||||
if valid_symbol and valid_symbol != normalized:
|
||||
try:
|
||||
logger.debug(f"Trying alternative symbol: {valid_symbol} (original: {symbol}, first attempt: {normalized})")
|
||||
logger.debug(
|
||||
f"Trying alternative symbol: {valid_symbol} (original: {symbol}, first attempt: {normalized})"
|
||||
)
|
||||
ticker = self.exchange.fetch_ticker(valid_symbol)
|
||||
if ticker and isinstance(ticker, dict):
|
||||
return ticker
|
||||
except Exception as e2:
|
||||
logger.debug(f"Alternative symbol {valid_symbol} also failed: {e2}")
|
||||
|
||||
|
||||
# If all attempts fail, log a warning and return a default value
|
||||
logger.warning(
|
||||
f"Symbol '{symbol}' (normalized: {normalized}) not found on {self.exchange.id}. "
|
||||
f"Error: {str(e)[:100]}"
|
||||
f"Symbol '{symbol}' (normalized: {normalized}) not found on {self.exchange.id}. Error: {str(e)[:100]}"
|
||||
)
|
||||
|
||||
return {'last': 0, 'symbol': symbol}
|
||||
|
||||
|
||||
return {"last": 0, "symbol": symbol}
|
||||
|
||||
def get_kline(
|
||||
self,
|
||||
symbol: str,
|
||||
timeframe: str,
|
||||
limit: int,
|
||||
before_time: Optional[int] = None
|
||||
self, symbol: str, timeframe: str, limit: int, before_time: Optional[int] = None
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""Get cryptocurrency K-line data"""
|
||||
klines = []
|
||||
|
||||
|
||||
try:
|
||||
ccxt_timeframe = self.TIMEFRAME_MAP.get(timeframe, '1d')
|
||||
|
||||
ccxt_timeframe = self.TIMEFRAME_MAP.get(timeframe, "1d")
|
||||
|
||||
# Use a unified symbol normalization method
|
||||
symbol_pair = self._normalize_symbol_for_exchange(symbol)
|
||||
|
||||
|
||||
if not symbol_pair:
|
||||
logger.warning(f"Failed to normalize symbol for K-line: {symbol}")
|
||||
return []
|
||||
|
||||
|
||||
# logger.info(f"Get cryptocurrency K-line: {symbol_pair}, period: {ccxt_timeframe}, number of bars: {limit}")
|
||||
|
||||
|
||||
ohlcv = self._fetch_ohlcv(symbol_pair, ccxt_timeframe, limit, before_time, timeframe)
|
||||
|
||||
|
||||
if not ohlcv:
|
||||
logger.warning(f"CCXT returned no K-lines: {symbol_pair}")
|
||||
return []
|
||||
|
||||
|
||||
# Convert data format
|
||||
for candle in ohlcv:
|
||||
if len(candle) < 6:
|
||||
continue
|
||||
klines.append(self.format_kline(
|
||||
timestamp=int(candle[0] / 1000), # Milliseconds to seconds
|
||||
open_price=candle[1],
|
||||
high=candle[2],
|
||||
low=candle[3],
|
||||
close=candle[4],
|
||||
volume=candle[5]
|
||||
))
|
||||
|
||||
klines.append(
|
||||
self.format_kline(
|
||||
timestamp=int(candle[0] / 1000), # Milliseconds to seconds
|
||||
open_price=candle[1],
|
||||
high=candle[2],
|
||||
low=candle[3],
|
||||
close=candle[4],
|
||||
volume=candle[5],
|
||||
)
|
||||
)
|
||||
|
||||
# Filter and restrict
|
||||
klines = self.filter_and_limit(klines, limit, before_time)
|
||||
|
||||
|
||||
# Record results
|
||||
self.log_result(symbol, klines, timeframe)
|
||||
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to fetch crypto K-lines {symbol}: {str(e)}")
|
||||
import traceback
|
||||
|
||||
logger.error(traceback.format_exc())
|
||||
|
||||
|
||||
return klines
|
||||
|
||||
|
||||
def _fetch_ohlcv(
|
||||
self,
|
||||
symbol_pair: str,
|
||||
ccxt_timeframe: str,
|
||||
limit: int,
|
||||
before_time: Optional[int],
|
||||
timeframe: str
|
||||
self, symbol_pair: str, ccxt_timeframe: str, limit: int, before_time: Optional[int], timeframe: str
|
||||
) -> List:
|
||||
"""Obtain OHLCV data (supports paging to obtain complete data)"""
|
||||
try:
|
||||
@@ -301,75 +295,66 @@ class CryptoDataSource(BaseDataSource):
|
||||
start_time = end_time - timedelta(seconds=total_seconds)
|
||||
since = int(start_time.timestamp() * 1000)
|
||||
end_ms = before_time * 1000
|
||||
|
||||
|
||||
# logger.info(f"Historical data request: since={since//1000}, end={before_time}, time span={total_seconds/86400:.1f} days")
|
||||
|
||||
|
||||
# Fetch data in pages until the complete time range is covered
|
||||
all_ohlcv = []
|
||||
batch_limit = 300 # Coinbase limit is often 300, safer than 1000
|
||||
current_since = since
|
||||
|
||||
|
||||
while current_since < end_ms:
|
||||
batch = self.exchange.fetch_ohlcv(
|
||||
symbol_pair,
|
||||
ccxt_timeframe,
|
||||
since=current_since,
|
||||
limit=batch_limit
|
||||
symbol_pair, ccxt_timeframe, since=current_since, limit=batch_limit
|
||||
)
|
||||
|
||||
|
||||
if not batch:
|
||||
break
|
||||
|
||||
|
||||
all_ohlcv.extend(batch)
|
||||
|
||||
|
||||
# The time when the last piece of data is obtained is used as the starting time of the next request
|
||||
last_timestamp = batch[-1][0]
|
||||
|
||||
|
||||
# If the time of the last data exceeds the end time, or the returned data is less than the requested amount, it means that the acquisition has been completed.
|
||||
# if last_timestamp >= end_ms or len(batch) < batch_limit:
|
||||
if last_timestamp >= end_ms:
|
||||
break
|
||||
|
||||
|
||||
# Next time, start from the next time point of the last item
|
||||
timeframe_ms = TIMEFRAME_SECONDS.get(timeframe, 86400) * 1000
|
||||
current_since = last_timestamp + timeframe_ms
|
||||
|
||||
|
||||
# logger.info(f"Getting in paging: {len(all_ohlcv)} items have been obtained, continue from {datetime.fromtimestamp(current_since/1000)}")
|
||||
|
||||
|
||||
ohlcv = all_ohlcv
|
||||
else:
|
||||
ohlcv = self.exchange.fetch_ohlcv(symbol_pair, ccxt_timeframe, limit=limit)
|
||||
|
||||
|
||||
# logger.info(f"CCXT returns {len(ohlcv) if ohlcv else 0} pieces of data")
|
||||
return ohlcv
|
||||
|
||||
|
||||
except Exception as e:
|
||||
logger.warning(f"CCXT fetch_ohlcv failed: {str(e)}; trying fallback")
|
||||
return self._fetch_ohlcv_fallback(symbol_pair, ccxt_timeframe, limit, before_time, timeframe)
|
||||
|
||||
|
||||
def _fetch_ohlcv_fallback(
|
||||
self,
|
||||
symbol_pair: str,
|
||||
ccxt_timeframe: str,
|
||||
limit: int,
|
||||
before_time: Optional[int],
|
||||
timeframe: str
|
||||
self, symbol_pair: str, ccxt_timeframe: str, limit: int, before_time: Optional[int], timeframe: str
|
||||
) -> List:
|
||||
"""Alternate acquisition method"""
|
||||
try:
|
||||
total_seconds = self.calculate_time_range(timeframe, limit)
|
||||
|
||||
|
||||
if before_time:
|
||||
end_time = datetime.fromtimestamp(before_time)
|
||||
start_time = end_time - timedelta(seconds=total_seconds)
|
||||
since = int(start_time.timestamp() * 1000)
|
||||
else:
|
||||
since = int((datetime.now() - timedelta(seconds=total_seconds)).timestamp() * 1000)
|
||||
|
||||
|
||||
ohlcv = self.exchange.fetch_ohlcv(symbol_pair, ccxt_timeframe, since=since, limit=limit)
|
||||
# logger.info(f"CCXT alternative method returns {len(ohlcv) if ohlcv else 0} pieces of data")
|
||||
return ohlcv
|
||||
except Exception as e:
|
||||
logger.error(f"CCXT fallback method also failed: {str(e)}")
|
||||
return []
|
||||
|
||||
|
||||
@@ -2,7 +2,8 @@
|
||||
data source factory
|
||||
Return the corresponding data source according to the market type
|
||||
"""
|
||||
from typing import Dict, List, Any, Optional
|
||||
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from app.data_sources.base import BaseDataSource
|
||||
from app.utils.logger import get_logger
|
||||
@@ -12,17 +13,17 @@ logger = get_logger(__name__)
|
||||
|
||||
class DataSourceFactory:
|
||||
"""data source factory"""
|
||||
|
||||
|
||||
_sources: Dict[str, BaseDataSource] = {}
|
||||
|
||||
|
||||
@classmethod
|
||||
def get_source(cls, market: str) -> BaseDataSource:
|
||||
"""
|
||||
Get the data source for the specified market
|
||||
|
||||
|
||||
Args:
|
||||
market: market type (Crypto, USStock, Forex, Futures)
|
||||
|
||||
|
||||
Returns:
|
||||
Data source instance
|
||||
"""
|
||||
@@ -45,74 +46,67 @@ class DataSourceFactory:
|
||||
return cls.get_source("Futures")
|
||||
# Default to Crypto for safety (most callers want a ticker for crypto pairs).
|
||||
return cls.get_source("Crypto")
|
||||
|
||||
|
||||
@classmethod
|
||||
def _create_source(cls, market: str) -> BaseDataSource:
|
||||
"""Create data source instance"""
|
||||
if market == 'Crypto':
|
||||
if market == "Crypto":
|
||||
from app.data_sources.crypto import CryptoDataSource
|
||||
|
||||
return CryptoDataSource()
|
||||
elif market == 'CNStock':
|
||||
from app.data_sources.cn_stock import CNStockDataSource
|
||||
return CNStockDataSource()
|
||||
elif market == 'HKStock':
|
||||
from app.data_sources.hk_stock import HKStockDataSource
|
||||
return HKStockDataSource()
|
||||
elif market == 'USStock':
|
||||
elif market == "USStock":
|
||||
from app.data_sources.us_stock import USStockDataSource
|
||||
|
||||
return USStockDataSource()
|
||||
elif market == 'Forex':
|
||||
elif market == "Forex":
|
||||
from app.data_sources.forex import ForexDataSource
|
||||
|
||||
return ForexDataSource()
|
||||
elif market == 'Futures':
|
||||
elif market == "Futures":
|
||||
from app.data_sources.futures import FuturesDataSource
|
||||
|
||||
return FuturesDataSource()
|
||||
else:
|
||||
raise ValueError(f"Unsupported market type: {market}")
|
||||
|
||||
|
||||
@classmethod
|
||||
def get_kline(
|
||||
cls,
|
||||
market: str,
|
||||
symbol: str,
|
||||
timeframe: str,
|
||||
limit: int,
|
||||
before_time: Optional[int] = None
|
||||
cls, market: str, symbol: str, timeframe: str, limit: int, before_time: Optional[int] = None
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
A convenient way to obtain K-line data
|
||||
|
||||
|
||||
Args:
|
||||
market: market type
|
||||
symbol: trading pair/stock code
|
||||
timeframe: time period
|
||||
limit: number of data items
|
||||
before_time: Get data before this time
|
||||
|
||||
|
||||
Returns:
|
||||
K-line data list
|
||||
"""
|
||||
try:
|
||||
source = cls.get_source(market)
|
||||
klines = source.get_kline(symbol, timeframe, limit, before_time)
|
||||
|
||||
|
||||
# Make sure the data is sorted by time
|
||||
klines.sort(key=lambda x: x['time'])
|
||||
|
||||
klines.sort(key=lambda x: x["time"])
|
||||
|
||||
return klines
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to fetch K-lines {market}:{symbol} - {str(e)}")
|
||||
return []
|
||||
|
||||
|
||||
@classmethod
|
||||
def get_ticker(cls, market: str, symbol: str) -> Dict[str, Any]:
|
||||
"""
|
||||
The convenient way to get realtime quotes
|
||||
|
||||
|
||||
Args:
|
||||
market: market type
|
||||
symbol: trading pair/stock code
|
||||
|
||||
|
||||
Returns:
|
||||
Real-time quotation data: {
|
||||
'last': latest price,
|
||||
@@ -126,8 +120,7 @@ class DataSourceFactory:
|
||||
return source.get_ticker(symbol)
|
||||
except NotImplementedError:
|
||||
logger.warning(f"get_ticker not implemented for market: {market}")
|
||||
return {'last': 0, 'symbol': symbol}
|
||||
return {"last": 0, "symbol": symbol}
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to fetch ticker {market}:{symbol} - {str(e)}")
|
||||
return {'last': 0, 'symbol': symbol}
|
||||
|
||||
return {"last": 0, "symbol": symbol}
|
||||
|
||||
@@ -2,15 +2,17 @@
|
||||
Forex data source
|
||||
Get Forex Data with Tiingo
|
||||
"""
|
||||
from typing import Dict, List, Any, Optional
|
||||
from datetime import datetime, timedelta
|
||||
import time
|
||||
import requests
|
||||
import threading
|
||||
|
||||
from app.data_sources.base import BaseDataSource, TIMEFRAME_SECONDS
|
||||
import threading
|
||||
import time
|
||||
from datetime import datetime, timedelta
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
import requests
|
||||
|
||||
from app.config import APIKeys, TiingoConfig
|
||||
from app.data_sources.base import TIMEFRAME_SECONDS, BaseDataSource
|
||||
from app.utils.logger import get_logger
|
||||
from app.config import TiingoConfig, APIKeys
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
@@ -22,52 +24,52 @@ _FOREX_CACHE_TTL = 60 # Forex price caching for 60 seconds (Tiingo free API has
|
||||
|
||||
class ForexDataSource(BaseDataSource):
|
||||
"""Forex data source (Tiingo)"""
|
||||
|
||||
|
||||
name = "Forex/Tiingo"
|
||||
|
||||
|
||||
# Tiingo resampleFreq mapping
|
||||
# Tiingo free account support: 5min, 15min, 30min, 1hour, 4hour, 1day
|
||||
# Note: 1min requires paid subscription, 1week/1month is not supported by Tiingo FX API
|
||||
TIMEFRAME_MAP = {
|
||||
'1m': '1min', # Paid subscription required
|
||||
'5m': '5min',
|
||||
'15m': '15min',
|
||||
'30m': '30min',
|
||||
'1H': '1hour',
|
||||
'4H': '4hour',
|
||||
'1D': '1day',
|
||||
'1W': None, # Tiingo does not support it and needs to be aggregated.
|
||||
'1M': None # Tiingo does not support it and needs to be aggregated.
|
||||
"1m": "1min", # Paid subscription required
|
||||
"5m": "5min",
|
||||
"15m": "15min",
|
||||
"30m": "30min",
|
||||
"1H": "1hour",
|
||||
"4H": "4hour",
|
||||
"1D": "1day",
|
||||
"1W": None, # Tiingo does not support it and needs to be aggregated.
|
||||
"1M": None, # Tiingo does not support it and needs to be aggregated.
|
||||
}
|
||||
|
||||
|
||||
# Forex pair mapping (Tiingo uses standard tickers such as eurusd, audusd)
|
||||
# Uppercase letters are also acceptable. Tiingo is usually not case-sensitive, but uniformity is recommended.
|
||||
SYMBOL_MAP = {
|
||||
# Precious metals (Tiingo does not necessarily support all precious metals in OANDA format, usually XAUUSD)
|
||||
'XAUUSD': 'xauusd',
|
||||
'XAGUSD': 'xagusd',
|
||||
"XAUUSD": "xauusd",
|
||||
"XAGUSD": "xagusd",
|
||||
# major currency pairs
|
||||
'EURUSD': 'eurusd',
|
||||
'GBPUSD': 'gbpusd',
|
||||
'USDJPY': 'usdjpy',
|
||||
'AUDUSD': 'audusd',
|
||||
'USDCAD': 'usdcad',
|
||||
'USDCHF': 'usdchf',
|
||||
'NZDUSD': 'nzdusd',
|
||||
"EURUSD": "eurusd",
|
||||
"GBPUSD": "gbpusd",
|
||||
"USDJPY": "usdjpy",
|
||||
"AUDUSD": "audusd",
|
||||
"USDCAD": "usdcad",
|
||||
"USDCHF": "usdchf",
|
||||
"NZDUSD": "nzdusd",
|
||||
}
|
||||
|
||||
|
||||
def __init__(self):
|
||||
self.base_url = TiingoConfig.BASE_URL
|
||||
if not APIKeys.TIINGO_API_KEY:
|
||||
logger.warning("Tiingo API key is not configured; FX data will be unavailable")
|
||||
|
||||
logger.warning("Tiingo API key is not configured; FX data will be unavailable")
|
||||
|
||||
def get_ticker(self, symbol: str) -> Dict[str, Any]:
|
||||
"""
|
||||
Get realtime quotes for foreign exchange
|
||||
|
||||
|
||||
Get realtime quotes using the Tiingo FX Top-of-Book API
|
||||
Comes with 60 second cache to avoid triggering Tiingo rate limit frequently
|
||||
|
||||
|
||||
Returns:
|
||||
dict: {
|
||||
'last': current price (mid price),
|
||||
@@ -80,42 +82,39 @@ class ForexDataSource(BaseDataSource):
|
||||
api_key = APIKeys.TIINGO_API_KEY
|
||||
if not api_key:
|
||||
logger.warning("Tiingo API key not configured")
|
||||
return {'last': 0, 'symbol': symbol}
|
||||
|
||||
return {"last": 0, "symbol": symbol}
|
||||
|
||||
# Check cache
|
||||
cache_key = f"ticker_{symbol}"
|
||||
with _forex_cache_lock:
|
||||
cached = _forex_cache.get(cache_key)
|
||||
if cached:
|
||||
cache_time = cached.get('_cache_time', 0)
|
||||
cache_time = cached.get("_cache_time", 0)
|
||||
if time.time() - cache_time < _FOREX_CACHE_TTL:
|
||||
logger.debug(f"Using cached forex ticker for {symbol}")
|
||||
return cached
|
||||
|
||||
|
||||
try:
|
||||
# parse symbol
|
||||
tiingo_symbol = self.SYMBOL_MAP.get(symbol)
|
||||
if not tiingo_symbol:
|
||||
tiingo_symbol = symbol.lower()
|
||||
|
||||
|
||||
# Tiingo FX Top-of-Book API
|
||||
# https://api.tiingo.com/tiingo/fx/top?tickers=eurusd&token=...
|
||||
url = f"{self.base_url}/fx/top"
|
||||
params = {
|
||||
'tickers': tiingo_symbol,
|
||||
'token': api_key
|
||||
}
|
||||
|
||||
params = {"tickers": tiingo_symbol, "token": api_key}
|
||||
|
||||
# Retry logic: Handling 429 rate limiting
|
||||
for attempt in range(3):
|
||||
response = requests.get(url, params=params, timeout=TiingoConfig.TIMEOUT)
|
||||
if response.status_code == 429:
|
||||
wait_time = 2 * (attempt + 1)
|
||||
logger.warning(f"Tiingo rate limit (429), waiting {wait_time}s before retry ({attempt+1}/3)")
|
||||
logger.warning(f"Tiingo rate limit (429), waiting {wait_time}s before retry ({attempt + 1}/3)")
|
||||
time.sleep(wait_time)
|
||||
continue
|
||||
break
|
||||
|
||||
|
||||
if response.status_code == 429:
|
||||
logger.warning("Tiingo rate limit exceeded for ticker request")
|
||||
logger.info("Note: Tiingo 1-minute forex data requires a paid subscription")
|
||||
@@ -124,86 +123,77 @@ class ForexDataSource(BaseDataSource):
|
||||
if cache_key in _forex_cache:
|
||||
logger.info(f"Returning stale cache for {symbol} due to rate limit")
|
||||
return _forex_cache[cache_key]
|
||||
return {'last': 0, 'symbol': symbol}
|
||||
|
||||
return {"last": 0, "symbol": symbol}
|
||||
|
||||
response.raise_for_status()
|
||||
data = response.json()
|
||||
|
||||
|
||||
if data and isinstance(data, list) and len(data) > 0:
|
||||
item = data[0]
|
||||
# Tiingo FX top returns: ticker, quoteTimestamp, bidPrice, bidSize, askPrice, askSize, midPrice
|
||||
bid = float(item.get('bidPrice', 0) or 0)
|
||||
ask = float(item.get('askPrice', 0) or 0)
|
||||
mid = float(item.get('midPrice', 0) or 0)
|
||||
|
||||
bid = float(item.get("bidPrice", 0) or 0)
|
||||
ask = float(item.get("askPrice", 0) or 0)
|
||||
mid = float(item.get("midPrice", 0) or 0)
|
||||
|
||||
# If there is no midPrice, calculate the mid price
|
||||
if not mid and bid and ask:
|
||||
mid = (bid + ask) / 2
|
||||
|
||||
|
||||
last_price = mid or bid or ask
|
||||
|
||||
|
||||
# Get the closing price of the previous day to calculate the rise and fall (additional request for daily data is required)
|
||||
prev_close = 0
|
||||
change = 0
|
||||
change_pct = 0
|
||||
|
||||
|
||||
try:
|
||||
# Get yesterday's closing price
|
||||
yesterday = (datetime.now() - timedelta(days=2)).strftime('%Y-%m-%d')
|
||||
today = datetime.now().strftime('%Y-%m-%d')
|
||||
yesterday = (datetime.now() - timedelta(days=2)).strftime("%Y-%m-%d")
|
||||
today = datetime.now().strftime("%Y-%m-%d")
|
||||
price_url = f"{self.base_url}/fx/{tiingo_symbol}/prices"
|
||||
price_params = {
|
||||
'startDate': yesterday,
|
||||
'endDate': today,
|
||||
'resampleFreq': '1day',
|
||||
'token': api_key
|
||||
}
|
||||
price_params = {"startDate": yesterday, "endDate": today, "resampleFreq": "1day", "token": api_key}
|
||||
price_resp = requests.get(price_url, params=price_params, timeout=TiingoConfig.TIMEOUT)
|
||||
if price_resp.status_code == 200:
|
||||
price_data = price_resp.json()
|
||||
if price_data and len(price_data) > 0:
|
||||
prev_close = float(price_data[-1].get('close', 0) or 0)
|
||||
prev_close = float(price_data[-1].get("close", 0) or 0)
|
||||
if prev_close and last_price:
|
||||
change = last_price - prev_close
|
||||
change_pct = (change / prev_close) * 100
|
||||
except Exception:
|
||||
pass # Failure to calculate the rise or fall does not affect the main functions
|
||||
|
||||
|
||||
result = {
|
||||
'last': round(last_price, 5),
|
||||
'bid': round(bid, 5),
|
||||
'ask': round(ask, 5),
|
||||
'change': round(change, 5),
|
||||
'changePercent': round(change_pct, 2),
|
||||
'previousClose': round(prev_close, 5) if prev_close else 0,
|
||||
'_cache_time': time.time()
|
||||
"last": round(last_price, 5),
|
||||
"bid": round(bid, 5),
|
||||
"ask": round(ask, 5),
|
||||
"change": round(change, 5),
|
||||
"changePercent": round(change_pct, 2),
|
||||
"previousClose": round(prev_close, 5) if prev_close else 0,
|
||||
"_cache_time": time.time(),
|
||||
}
|
||||
|
||||
|
||||
# cache results
|
||||
with _forex_cache_lock:
|
||||
_forex_cache[cache_key] = result
|
||||
|
||||
|
||||
return result
|
||||
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to get forex ticker for {symbol}: {e}")
|
||||
|
||||
return {'last': 0, 'symbol': symbol}
|
||||
|
||||
|
||||
return {"last": 0, "symbol": symbol}
|
||||
|
||||
def _get_timeframe_seconds(self, timeframe: str) -> int:
|
||||
"""Get the number of seconds corresponding to the time period"""
|
||||
return TIMEFRAME_SECONDS.get(timeframe, 86400)
|
||||
|
||||
|
||||
def get_kline(
|
||||
self,
|
||||
symbol: str,
|
||||
timeframe: str,
|
||||
limit: int,
|
||||
before_time: Optional[int] = None
|
||||
self, symbol: str, timeframe: str, limit: int, before_time: Optional[int] = None
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
Get foreign exchange K-line data
|
||||
|
||||
|
||||
Args:
|
||||
symbol: Forex pair symbol (such as XAUUSD, EURUSD)
|
||||
timeframe: time period
|
||||
@@ -215,7 +205,7 @@ class ForexDataSource(BaseDataSource):
|
||||
if not api_key:
|
||||
logger.error("Tiingo API key is not configured")
|
||||
return []
|
||||
|
||||
|
||||
try:
|
||||
# 1. Parse Symbol
|
||||
tiingo_symbol = self.SYMBOL_MAP.get(symbol)
|
||||
@@ -225,15 +215,15 @@ class ForexDataSource(BaseDataSource):
|
||||
|
||||
# 2. Analysis Resolution (resampleFreq)
|
||||
resample_freq = self.TIMEFRAME_MAP.get(timeframe)
|
||||
|
||||
|
||||
# Special treatment: 1W/1M requires daily aggregation
|
||||
aggregate_to_weekly = (timeframe == '1W')
|
||||
aggregate_to_monthly = (timeframe == '1M')
|
||||
aggregate_to_weekly = timeframe == "1W"
|
||||
aggregate_to_monthly = timeframe == "1M"
|
||||
original_limit = limit # Save original request quantity
|
||||
|
||||
|
||||
if aggregate_to_weekly or aggregate_to_monthly:
|
||||
# Aggregate using daily data
|
||||
resample_freq = '1day'
|
||||
resample_freq = "1day"
|
||||
# Limit the maximum number of weekly/monthly requests (Tiingo free API has data volume limit)
|
||||
# The maximum weekly request is 100 weeks = 700 days ≈ 2 years
|
||||
# The maximum monthly request is 36 months = 1080 days ≈ 3 years
|
||||
@@ -241,21 +231,21 @@ class ForexDataSource(BaseDataSource):
|
||||
original_limit = min(original_limit, max_limit)
|
||||
# More daily data is needed to aggregate (weekly lines require 7 days, monthly lines require 30 days)
|
||||
limit = original_limit * (7 if aggregate_to_weekly else 30)
|
||||
|
||||
|
||||
if not resample_freq:
|
||||
logger.warning(f"Tiingo does not support timeframe: {timeframe}")
|
||||
return []
|
||||
|
||||
|
||||
# 1 minute data requires paid subscription reminder
|
||||
if timeframe == '1m':
|
||||
logger.info(f"Note: Tiingo 1-minute forex data requires a paid subscription")
|
||||
|
||||
if timeframe == "1m":
|
||||
logger.info("Note: Tiingo 1-minute forex data requires a paid subscription")
|
||||
|
||||
# 3. Calculation time range
|
||||
if before_time:
|
||||
end_dt = datetime.fromtimestamp(before_time)
|
||||
else:
|
||||
end_dt = datetime.now()
|
||||
|
||||
|
||||
# Calculate start time based on period and quantity
|
||||
# Note: Use daily seconds calculation in aggregation mode
|
||||
if aggregate_to_weekly or aggregate_to_monthly:
|
||||
@@ -264,71 +254,75 @@ class ForexDataSource(BaseDataSource):
|
||||
tf_seconds = self._get_timeframe_seconds(timeframe)
|
||||
# Get more buffer time (1.5 times, foreign exchange does not trade on weekends)
|
||||
start_dt = end_dt - timedelta(seconds=limit * tf_seconds * 1.5)
|
||||
|
||||
|
||||
# Tiingo free API supports up to about 5 years of data, limiting the maximum time range
|
||||
max_days = 365 * 3 # up to 3 years
|
||||
if (end_dt - start_dt).days > max_days:
|
||||
start_dt = end_dt - timedelta(days=max_days)
|
||||
logger.info(f"Tiingo: Limited date range to {max_days} days")
|
||||
|
||||
|
||||
# Format the date as YYYY-MM-DD (Tiingo supports this format)
|
||||
start_date_str = start_dt.strftime('%Y-%m-%d')
|
||||
end_date_str = end_dt.strftime('%Y-%m-%d')
|
||||
|
||||
start_date_str = start_dt.strftime("%Y-%m-%d")
|
||||
end_date_str = end_dt.strftime("%Y-%m-%d")
|
||||
|
||||
# 4. API request (with retry logic)
|
||||
# URL: https://api.tiingo.com/tiingo/fx/{ticker}/prices
|
||||
url = f"{self.base_url}/fx/{tiingo_symbol}/prices"
|
||||
|
||||
|
||||
params = {
|
||||
'startDate': start_date_str,
|
||||
'endDate': end_date_str,
|
||||
'resampleFreq': resample_freq,
|
||||
'token': api_key,
|
||||
'format': 'json'
|
||||
"startDate": start_date_str,
|
||||
"endDate": end_date_str,
|
||||
"resampleFreq": resample_freq,
|
||||
"token": api_key,
|
||||
"format": "json",
|
||||
}
|
||||
|
||||
|
||||
# logger.info(f"Tiingo Request: {url} params={params}")
|
||||
|
||||
|
||||
# Retry logic: Handling 429 rate limiting
|
||||
max_retries = 3
|
||||
retry_delay = 2 # Second
|
||||
response = None
|
||||
|
||||
|
||||
for attempt in range(max_retries):
|
||||
try:
|
||||
response = requests.get(url, params=params, timeout=TiingoConfig.TIMEOUT)
|
||||
|
||||
|
||||
if response.status_code == 429:
|
||||
# Rate limit, wait and try again
|
||||
wait_time = retry_delay * (attempt + 1)
|
||||
logger.warning(f"Tiingo rate limit (429), waiting {wait_time}s before retry ({attempt + 1}/{max_retries})")
|
||||
logger.warning(
|
||||
f"Tiingo rate limit (429), waiting {wait_time}s before retry ({attempt + 1}/{max_retries})"
|
||||
)
|
||||
time.sleep(wait_time)
|
||||
continue
|
||||
|
||||
|
||||
break # Success or other errors, exit the retry loop
|
||||
|
||||
|
||||
except requests.exceptions.Timeout:
|
||||
if attempt < max_retries - 1:
|
||||
logger.warning(f"Tiingo request timeout, retrying ({attempt + 1}/{max_retries})")
|
||||
time.sleep(retry_delay)
|
||||
continue
|
||||
raise
|
||||
|
||||
|
||||
if response is None:
|
||||
logger.error("Tiingo API request failed after all retries")
|
||||
return []
|
||||
|
||||
|
||||
if response.status_code == 429:
|
||||
logger.error("Tiingo API rate limit exceeded. Please wait a moment before retrying.")
|
||||
return []
|
||||
|
||||
|
||||
if response.status_code == 403:
|
||||
logger.error("Tiingo API permission error (403): check whether your API key is valid and has access to this dataset.")
|
||||
logger.error(
|
||||
"Tiingo API permission error (403): check whether your API key is valid and has access to this dataset."
|
||||
)
|
||||
return []
|
||||
|
||||
|
||||
response.raise_for_status()
|
||||
data = response.json()
|
||||
|
||||
|
||||
# 5. Process the response
|
||||
# Tiingo returns a list of dicts:
|
||||
# [
|
||||
@@ -343,35 +337,37 @@ class ForexDataSource(BaseDataSource):
|
||||
# }, ...
|
||||
# ]
|
||||
# Note: Tiingo FX prices objects keys: date, open, high, low, close.
|
||||
|
||||
|
||||
if not isinstance(data, list):
|
||||
logger.warning(f"Tiingo response is not a list: {data}")
|
||||
return []
|
||||
|
||||
|
||||
klines = []
|
||||
for item in data:
|
||||
# Parsing time: "2023-01-01T00:00:00.000Z"
|
||||
dt_str = item.get('date')
|
||||
dt_str = item.get("date")
|
||||
# Tiingo returns UTC time in ISO format and needs to handle the time zone correctly.
|
||||
# Convert UTC time to local timestamp
|
||||
if dt_str.endswith('Z'):
|
||||
dt_str = dt_str[:-1] + '+00:00' # Replace Z with +00:00 for UTC
|
||||
|
||||
if dt_str.endswith("Z"):
|
||||
dt_str = dt_str[:-1] + "+00:00" # Replace Z with +00:00 for UTC
|
||||
|
||||
dt = datetime.fromisoformat(dt_str)
|
||||
ts = int(dt.timestamp()) # UTC time zone is now handled correctly
|
||||
|
||||
klines.append({
|
||||
'time': ts,
|
||||
'open': float(item.get('open')),
|
||||
'high': float(item.get('high')),
|
||||
'low': float(item.get('low')),
|
||||
'close': float(item.get('close')),
|
||||
'volume': 0.0 # Tiingo FX usually does not have volume
|
||||
})
|
||||
|
||||
|
||||
klines.append(
|
||||
{
|
||||
"time": ts,
|
||||
"open": float(item.get("open")),
|
||||
"high": float(item.get("high")),
|
||||
"low": float(item.get("low")),
|
||||
"close": float(item.get("close")),
|
||||
"volume": 0.0, # Tiingo FX usually does not have volume
|
||||
}
|
||||
)
|
||||
|
||||
# Sort by time
|
||||
klines.sort(key=lambda x: x['time'])
|
||||
|
||||
klines.sort(key=lambda x: x["time"])
|
||||
|
||||
# If you need to aggregate to weekly or monthly lines
|
||||
if aggregate_to_weekly:
|
||||
klines = self._aggregate_to_weekly(klines)
|
||||
@@ -379,36 +375,36 @@ class ForexDataSource(BaseDataSource):
|
||||
elif aggregate_to_monthly:
|
||||
klines = self._aggregate_to_monthly(klines)
|
||||
logger.debug(f"Aggregated {len(klines)} monthly candles from daily data")
|
||||
|
||||
|
||||
# Filter to original request count
|
||||
if len(klines) > original_limit:
|
||||
klines = klines[-original_limit:]
|
||||
|
||||
|
||||
# logger.info(f"obtained {len(klines)} pieces of Tiingo foreign exchange data")
|
||||
return klines
|
||||
|
||||
|
||||
except requests.exceptions.RequestException as e:
|
||||
logger.error(f"Tiingo API request failed: {e}")
|
||||
return []
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to process Tiingo data: {e}")
|
||||
return []
|
||||
|
||||
|
||||
def _aggregate_to_weekly(self, daily_klines: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
|
||||
"""Aggregate daily data into weekly data"""
|
||||
if not daily_klines:
|
||||
return []
|
||||
|
||||
|
||||
weekly_klines = []
|
||||
current_week = None
|
||||
week_data = None
|
||||
|
||||
|
||||
for kline in daily_klines:
|
||||
dt = datetime.fromtimestamp(kline['time'])
|
||||
dt = datetime.fromtimestamp(kline["time"])
|
||||
# Get the Monday of the week in which the date is located
|
||||
week_start = dt - timedelta(days=dt.weekday())
|
||||
week_key = week_start.strftime('%Y-%W')
|
||||
|
||||
week_key = week_start.strftime("%Y-%W")
|
||||
|
||||
if week_key != current_week:
|
||||
# Save data from last week
|
||||
if week_data:
|
||||
@@ -416,39 +412,39 @@ class ForexDataSource(BaseDataSource):
|
||||
# start a new week
|
||||
current_week = week_key
|
||||
week_data = {
|
||||
'time': int(week_start.timestamp()),
|
||||
'open': kline['open'],
|
||||
'high': kline['high'],
|
||||
'low': kline['low'],
|
||||
'close': kline['close'],
|
||||
'volume': kline['volume']
|
||||
"time": int(week_start.timestamp()),
|
||||
"open": kline["open"],
|
||||
"high": kline["high"],
|
||||
"low": kline["low"],
|
||||
"close": kline["close"],
|
||||
"volume": kline["volume"],
|
||||
}
|
||||
else:
|
||||
# Update this week's data
|
||||
week_data['high'] = max(week_data['high'], kline['high'])
|
||||
week_data['low'] = min(week_data['low'], kline['low'])
|
||||
week_data['close'] = kline['close']
|
||||
week_data['volume'] += kline['volume']
|
||||
|
||||
week_data["high"] = max(week_data["high"], kline["high"])
|
||||
week_data["low"] = min(week_data["low"], kline["low"])
|
||||
week_data["close"] = kline["close"]
|
||||
week_data["volume"] += kline["volume"]
|
||||
|
||||
# Add last week
|
||||
if week_data:
|
||||
weekly_klines.append(week_data)
|
||||
|
||||
|
||||
return weekly_klines
|
||||
|
||||
|
||||
def _aggregate_to_monthly(self, daily_klines: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
|
||||
"""Aggregate daily data into monthly data"""
|
||||
if not daily_klines:
|
||||
return []
|
||||
|
||||
|
||||
monthly_klines = []
|
||||
current_month = None
|
||||
month_data = None
|
||||
|
||||
|
||||
for kline in daily_klines:
|
||||
dt = datetime.fromtimestamp(kline['time'])
|
||||
month_key = dt.strftime('%Y-%m')
|
||||
|
||||
dt = datetime.fromtimestamp(kline["time"])
|
||||
month_key = dt.strftime("%Y-%m")
|
||||
|
||||
if month_key != current_month:
|
||||
# Save last month’s data
|
||||
if month_data:
|
||||
@@ -457,22 +453,22 @@ class ForexDataSource(BaseDataSource):
|
||||
current_month = month_key
|
||||
month_start = dt.replace(day=1, hour=0, minute=0, second=0)
|
||||
month_data = {
|
||||
'time': int(month_start.timestamp()),
|
||||
'open': kline['open'],
|
||||
'high': kline['high'],
|
||||
'low': kline['low'],
|
||||
'close': kline['close'],
|
||||
'volume': kline['volume']
|
||||
"time": int(month_start.timestamp()),
|
||||
"open": kline["open"],
|
||||
"high": kline["high"],
|
||||
"low": kline["low"],
|
||||
"close": kline["close"],
|
||||
"volume": kline["volume"],
|
||||
}
|
||||
else:
|
||||
# Update this month's data
|
||||
month_data['high'] = max(month_data['high'], kline['high'])
|
||||
month_data['low'] = min(month_data['low'], kline['low'])
|
||||
month_data['close'] = kline['close']
|
||||
month_data['volume'] += kline['volume']
|
||||
|
||||
month_data["high"] = max(month_data["high"], kline["high"])
|
||||
month_data["low"] = min(month_data["low"], kline["low"])
|
||||
month_data["close"] = kline["close"]
|
||||
month_data["volume"] += kline["volume"]
|
||||
|
||||
# Add last month
|
||||
if month_data:
|
||||
monthly_klines.append(month_data)
|
||||
|
||||
|
||||
return monthly_klines
|
||||
|
||||
@@ -4,64 +4,61 @@ support:
|
||||
1. Cryptocurrency Futures (Binance Futures via CCXT)
|
||||
2. Traditional futures (Yahoo Finance)
|
||||
"""
|
||||
from typing import Dict, List, Any, Optional
|
||||
|
||||
from datetime import datetime, timedelta
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
import ccxt
|
||||
import yfinance as yf
|
||||
|
||||
from app.data_sources.base import BaseDataSource, TIMEFRAME_SECONDS
|
||||
from app.config import CCXTConfig
|
||||
from app.data_sources.base import TIMEFRAME_SECONDS, BaseDataSource
|
||||
from app.utils.logger import get_logger
|
||||
from app.config import CCXTConfig, APIKeys
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
|
||||
class FuturesDataSource(BaseDataSource):
|
||||
"""Futures data source"""
|
||||
|
||||
|
||||
name = "Futures"
|
||||
|
||||
|
||||
# Yahoo Finance time period mapping
|
||||
YF_TIMEFRAME_MAP = {
|
||||
'1m': '1m',
|
||||
'5m': '5m',
|
||||
'15m': '15m',
|
||||
'30m': '30m',
|
||||
'1H': '1h',
|
||||
'4H': '4h',
|
||||
'1D': '1d',
|
||||
'1W': '1wk'
|
||||
"1m": "1m",
|
||||
"5m": "5m",
|
||||
"15m": "15m",
|
||||
"30m": "30m",
|
||||
"1H": "1h",
|
||||
"4H": "4h",
|
||||
"1D": "1d",
|
||||
"1W": "1wk",
|
||||
}
|
||||
|
||||
|
||||
# CCXT time period mapping
|
||||
CCXT_TIMEFRAME_MAP = CCXTConfig.TIMEFRAME_MAP
|
||||
|
||||
|
||||
# Traditional futures contract code (Yahoo Finance)
|
||||
YF_SYMBOLS = {
|
||||
'GC': 'GC=F', # gold futures
|
||||
'SI': 'SI=F', # Silver futures
|
||||
'CL': 'CL=F', # Crude oil futures
|
||||
'NG': 'NG=F', # Natural gas futures
|
||||
'ZC': 'ZC=F', # Corn futures
|
||||
'ZW': 'ZW=F', # Wheat futures
|
||||
"GC": "GC=F", # gold futures
|
||||
"SI": "SI=F", # Silver futures
|
||||
"CL": "CL=F", # Crude oil futures
|
||||
"NG": "NG=F", # Natural gas futures
|
||||
"ZC": "ZC=F", # Corn futures
|
||||
"ZW": "ZW=F", # Wheat futures
|
||||
}
|
||||
|
||||
|
||||
def __init__(self):
|
||||
# Initialize CCXT (for cryptocurrency futures)
|
||||
config = {
|
||||
'timeout': CCXTConfig.TIMEOUT,
|
||||
'enableRateLimit': CCXTConfig.ENABLE_RATE_LIMIT,
|
||||
'options': {
|
||||
'defaultType': 'future'
|
||||
}
|
||||
"timeout": CCXTConfig.TIMEOUT,
|
||||
"enableRateLimit": CCXTConfig.ENABLE_RATE_LIMIT,
|
||||
"options": {"defaultType": "future"},
|
||||
}
|
||||
|
||||
|
||||
if CCXTConfig.PROXY:
|
||||
config['proxies'] = {
|
||||
'http': CCXTConfig.PROXY,
|
||||
'https': CCXTConfig.PROXY
|
||||
}
|
||||
|
||||
config["proxies"] = {"http": CCXTConfig.PROXY, "https": CCXTConfig.PROXY}
|
||||
|
||||
self.exchange = ccxt.binance(config)
|
||||
|
||||
def get_ticker(self, symbol: str) -> Dict[str, Any]:
|
||||
@@ -101,21 +98,17 @@ class FuturesDataSource(BaseDataSource):
|
||||
elif sym.endswith("USD") and len(sym) > 3:
|
||||
sym = f"{sym[:-3]}/USD"
|
||||
return self.exchange.fetch_ticker(sym)
|
||||
|
||||
|
||||
def _get_timeframe_seconds(self, timeframe: str) -> int:
|
||||
"""Get the number of seconds corresponding to the time period"""
|
||||
return TIMEFRAME_SECONDS.get(timeframe, 86400)
|
||||
|
||||
|
||||
def get_kline(
|
||||
self,
|
||||
symbol: str,
|
||||
timeframe: str,
|
||||
limit: int,
|
||||
before_time: Optional[int] = None
|
||||
self, symbol: str, timeframe: str, limit: int, before_time: Optional[int] = None
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
Get futures K-line data
|
||||
|
||||
|
||||
Args:
|
||||
symbol: futures contract code
|
||||
timeframe: time period
|
||||
@@ -123,124 +116,106 @@ class FuturesDataSource(BaseDataSource):
|
||||
before_time: end timestamp
|
||||
"""
|
||||
# Determine whether it is traditional futures or cryptocurrency futures
|
||||
if symbol in self.YF_SYMBOLS or symbol.endswith('=F'):
|
||||
if symbol in self.YF_SYMBOLS or symbol.endswith("=F"):
|
||||
return self._get_traditional_futures(symbol, timeframe, limit, before_time)
|
||||
else:
|
||||
return self._get_crypto_futures(symbol, timeframe, limit, before_time)
|
||||
|
||||
|
||||
def _get_traditional_futures(
|
||||
self,
|
||||
symbol: str,
|
||||
timeframe: str,
|
||||
limit: int,
|
||||
before_time: Optional[int] = None
|
||||
self, symbol: str, timeframe: str, limit: int, before_time: Optional[int] = None
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""Use yfinance to obtain traditional futures data"""
|
||||
try:
|
||||
# Convert symbol format
|
||||
yf_symbol = self.YF_SYMBOLS.get(symbol, symbol)
|
||||
if not yf_symbol.endswith('=F'):
|
||||
yf_symbol = symbol + '=F'
|
||||
|
||||
if not yf_symbol.endswith("=F"):
|
||||
yf_symbol = symbol + "=F"
|
||||
|
||||
# conversion time period
|
||||
yf_interval = self.YF_TIMEFRAME_MAP.get(timeframe, '1d')
|
||||
|
||||
yf_interval = self.YF_TIMEFRAME_MAP.get(timeframe, "1d")
|
||||
|
||||
# logger.info(f"Get traditional futures K-line: {yf_symbol}, period: {yf_interval}, number of bars: {limit}")
|
||||
|
||||
|
||||
# Calculation time range
|
||||
if before_time:
|
||||
end_time = datetime.fromtimestamp(before_time)
|
||||
else:
|
||||
end_time = datetime.now()
|
||||
|
||||
|
||||
tf_seconds = self._get_timeframe_seconds(timeframe)
|
||||
start_time = end_time - timedelta(seconds=tf_seconds * limit * 1.5)
|
||||
|
||||
|
||||
# The end parameter of yfinance is not included (exclusive), and one day needs to be added.
|
||||
end_time_inclusive = end_time + timedelta(days=1)
|
||||
|
||||
|
||||
# Get data
|
||||
ticker = yf.Ticker(yf_symbol)
|
||||
df = ticker.history(
|
||||
start=start_time,
|
||||
end=end_time_inclusive,
|
||||
interval=yf_interval
|
||||
)
|
||||
|
||||
df = ticker.history(start=start_time, end=end_time_inclusive, interval=yf_interval)
|
||||
|
||||
if df.empty:
|
||||
logger.warning(f"No data: {yf_symbol}")
|
||||
return []
|
||||
|
||||
|
||||
# Convert format
|
||||
klines = []
|
||||
for index, row in df.iterrows():
|
||||
klines.append({
|
||||
'time': int(index.timestamp()),
|
||||
'open': float(row['Open']),
|
||||
'high': float(row['High']),
|
||||
'low': float(row['Low']),
|
||||
'close': float(row['Close']),
|
||||
'volume': float(row['Volume'])
|
||||
})
|
||||
|
||||
klines.sort(key=lambda x: x['time'])
|
||||
klines.append(
|
||||
{
|
||||
"time": int(index.timestamp()),
|
||||
"open": float(row["Open"]),
|
||||
"high": float(row["High"]),
|
||||
"low": float(row["Low"]),
|
||||
"close": float(row["Close"]),
|
||||
"volume": float(row["Volume"]),
|
||||
}
|
||||
)
|
||||
|
||||
klines.sort(key=lambda x: x["time"])
|
||||
if len(klines) > limit:
|
||||
klines = klines[-limit:]
|
||||
|
||||
|
||||
# logger.info(f"obtained {len(klines)} pieces of traditional futures data")
|
||||
return klines
|
||||
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to fetch traditional futures data: {e}")
|
||||
return []
|
||||
|
||||
|
||||
def _get_crypto_futures(
|
||||
self,
|
||||
symbol: str,
|
||||
timeframe: str,
|
||||
limit: int,
|
||||
before_time: Optional[int] = None
|
||||
self, symbol: str, timeframe: str, limit: int, before_time: Optional[int] = None
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""Obtain cryptocurrency futures data using CCXT"""
|
||||
try:
|
||||
# Make sure the symbol format is correct
|
||||
ccxt_symbol = symbol if '/' in symbol else f"{symbol}/USDT"
|
||||
ccxt_timeframe = self.CCXT_TIMEFRAME_MAP.get(timeframe, '1d')
|
||||
|
||||
ccxt_symbol = symbol if "/" in symbol else f"{symbol}/USDT"
|
||||
ccxt_timeframe = self.CCXT_TIMEFRAME_MAP.get(timeframe, "1d")
|
||||
|
||||
# logger.info(f"Get cryptocurrency futures K-line: {ccxt_symbol}, period: {ccxt_timeframe}, number of bars: {limit}")
|
||||
|
||||
|
||||
# Get data
|
||||
if before_time:
|
||||
since_time = before_time - limit * self._get_timeframe_seconds(timeframe)
|
||||
ohlcv = self.exchange.fetch_ohlcv(
|
||||
ccxt_symbol,
|
||||
ccxt_timeframe,
|
||||
since=since_time * 1000,
|
||||
limit=limit
|
||||
)
|
||||
ohlcv = self.exchange.fetch_ohlcv(ccxt_symbol, ccxt_timeframe, since=since_time * 1000, limit=limit)
|
||||
else:
|
||||
ohlcv = self.exchange.fetch_ohlcv(
|
||||
ccxt_symbol,
|
||||
ccxt_timeframe,
|
||||
limit=limit
|
||||
)
|
||||
|
||||
ohlcv = self.exchange.fetch_ohlcv(ccxt_symbol, ccxt_timeframe, limit=limit)
|
||||
|
||||
# Convert format
|
||||
klines = []
|
||||
for candle in ohlcv:
|
||||
klines.append({
|
||||
'time': int(candle[0] / 1000),
|
||||
'open': float(candle[1]),
|
||||
'high': float(candle[2]),
|
||||
'low': float(candle[3]),
|
||||
'close': float(candle[4]),
|
||||
'volume': float(candle[5])
|
||||
})
|
||||
|
||||
klines.append(
|
||||
{
|
||||
"time": int(candle[0] / 1000),
|
||||
"open": float(candle[1]),
|
||||
"high": float(candle[2]),
|
||||
"low": float(candle[3]),
|
||||
"close": float(candle[4]),
|
||||
"volume": float(candle[5]),
|
||||
}
|
||||
)
|
||||
|
||||
# logger.info(f"obtained {len(klines)} pieces of cryptocurrency futures data")
|
||||
return klines
|
||||
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to fetch crypto futures data: {e}")
|
||||
return []
|
||||
|
||||
|
||||
@@ -1,99 +0,0 @@
|
||||
"""
|
||||
港股/H股数据源 — 多层 fallback
|
||||
|
||||
有 TWELVE_DATA_API_KEY:
|
||||
所有周期 → Twelve Data（主） → 腾讯日/周线 → yfinance → AkShare
|
||||
|
||||
无 API Key:
|
||||
分钟/小时 → yfinance → AkShare
|
||||
日/周线 → 腾讯 fqkline → yfinance → AkShare
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Dict, List, Any, Optional
|
||||
|
||||
from app.data_sources.base import BaseDataSource
|
||||
from app.data_sources.tencent import normalize_hk_code, fetch_quote, parse_quote_to_ticker, fetch_kline, tencent_kline_rows_to_dicts
|
||||
from app.data_sources.asia_stock_kline import (
|
||||
normalize_chart_timeframe,
|
||||
fetch_twelvedata_klines,
|
||||
fetch_yfinance_klines,
|
||||
fetch_akshare_minute_klines,
|
||||
fetch_akshare_weekly_klines,
|
||||
)
|
||||
from app.utils.logger import get_logger
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
|
||||
class HKStockDataSource(BaseDataSource):
|
||||
"""港股/H股数据源（TwelveData + Tencent + yfinance + AkShare）"""
|
||||
|
||||
name = "HKStock/multi-source"
|
||||
|
||||
def get_ticker(self, symbol: str) -> Dict[str, Any]:
|
||||
code = normalize_hk_code(symbol)
|
||||
parts = fetch_quote(code)
|
||||
if not parts:
|
||||
return {"last": 0, "symbol": code}
|
||||
t = parse_quote_to_ticker(parts)
|
||||
return {
|
||||
"last": t.get("last", 0),
|
||||
"change": t.get("change", 0),
|
||||
"changePercent": t.get("changePercent", 0),
|
||||
"high": t.get("high", 0),
|
||||
"low": t.get("low", 0),
|
||||
"open": t.get("open", 0),
|
||||
"previousClose": t.get("previousClose", 0),
|
||||
"name": t.get("name", ""),
|
||||
"symbol": code,
|
||||
}
|
||||
|
||||
def get_kline(
|
||||
self,
|
||||
symbol: str,
|
||||
timeframe: str,
|
||||
limit: int,
|
||||
before_time: Optional[int] = None,
|
||||
) -> List[Dict[str, Any]]:
|
||||
code = normalize_hk_code(symbol)
|
||||
tf = normalize_chart_timeframe(timeframe)
|
||||
lim = max(int(limit or 300), 1)
|
||||
|
||||
# Tier 1: Twelve Data (paid, most reliable)
|
||||
rows = fetch_twelvedata_klines(
|
||||
is_hk=True, tencent_code=code, timeframe=tf, limit=lim, before_time=before_time
|
||||
)
|
||||
if rows:
|
||||
return self.filter_and_limit(rows, limit=lim, before_time=before_time)
|
||||
|
||||
# Tier 2: Tencent for daily/weekly (fast, free)
|
||||
if tf in ("1D", "1W"):
|
||||
tf_map = {"1D": "day", "1W": "week"}
|
||||
period = tf_map.get(tf, "day")
|
||||
raw_rows = fetch_kline(code, period=period, count=lim, adj="qfq")
|
||||
out = tencent_kline_rows_to_dicts(raw_rows)
|
||||
if out:
|
||||
return self.filter_and_limit(out, limit=lim, before_time=before_time)
|
||||
|
||||
# Tier 3: yfinance (works when Yahoo not rate-limited)
|
||||
rows = fetch_yfinance_klines(
|
||||
is_hk=True, tencent_code=code, timeframe=tf, limit=lim, before_time=before_time
|
||||
)
|
||||
if rows:
|
||||
return self.filter_and_limit(rows, limit=lim, before_time=before_time)
|
||||
|
||||
# Tier 4: AkShare (fragile overseas, last resort)
|
||||
if tf in ("1m", "5m", "15m", "30m", "1H", "4H"):
|
||||
rows = fetch_akshare_minute_klines(
|
||||
is_hk=True, tencent_code=code, timeframe=tf, limit=lim, before_time=before_time
|
||||
)
|
||||
elif tf == "1W":
|
||||
rows = fetch_akshare_weekly_klines(
|
||||
is_hk=True, tencent_code=code, limit=lim, before_time=before_time
|
||||
)
|
||||
else:
|
||||
rows = []
|
||||
|
||||
return self.filter_and_limit(rows, limit=lim, before_time=before_time)
|
||||
File diff suppressed because it is too large
Load Diff
@@ -12,11 +12,11 @@ Provide anti-crawler strategies:
|
||||
4. Request frequency limit
|
||||
"""
|
||||
|
||||
import time
|
||||
import random
|
||||
import logging
|
||||
from typing import Optional, Callable, Any, Type, Tuple
|
||||
import random
|
||||
import time
|
||||
from functools import wraps
|
||||
from typing import Any, Callable, Optional, Tuple, Type
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -27,23 +27,23 @@ logger = logging.getLogger(__name__)
|
||||
|
||||
USER_AGENTS = [
|
||||
# Chrome Windows
|
||||
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
|
||||
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36',
|
||||
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36',
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36",
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36",
|
||||
# Chrome Mac
|
||||
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
|
||||
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36',
|
||||
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
|
||||
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36",
|
||||
# Firefox
|
||||
'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:121.0) Gecko/20100101 Firefox/121.0',
|
||||
'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:120.0) Gecko/20100101 Firefox/120.0',
|
||||
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:121.0) Gecko/20100101 Firefox/121.0',
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:121.0) Gecko/20100101 Firefox/121.0",
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:120.0) Gecko/20100101 Firefox/120.0",
|
||||
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:121.0) Gecko/20100101 Firefox/121.0",
|
||||
# Safari
|
||||
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.2 Safari/605.1.15',
|
||||
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.1 Safari/605.1.15',
|
||||
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.2 Safari/605.1.15",
|
||||
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.1 Safari/605.1.15",
|
||||
# Edge
|
||||
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36 Edg/120.0.0.0',
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36 Edg/120.0.0.0",
|
||||
# Linux Chrome
|
||||
'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
|
||||
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
|
||||
]
|
||||
|
||||
|
||||
@@ -55,24 +55,24 @@ def get_random_user_agent() -> str:
|
||||
def get_request_headers(referer: Optional[str] = None) -> dict:
|
||||
"""
|
||||
Get request header with random User-Agent
|
||||
|
||||
|
||||
Args:
|
||||
referer: optional Referer header
|
||||
|
||||
|
||||
Returns:
|
||||
Request header dictionary
|
||||
"""
|
||||
headers = {
|
||||
'User-Agent': get_random_user_agent(),
|
||||
'Accept': 'application/json, text/plain, */*',
|
||||
'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8',
|
||||
'Accept-Encoding': 'gzip, deflate',
|
||||
'Connection': 'keep-alive',
|
||||
"User-Agent": get_random_user_agent(),
|
||||
"Accept": "application/json, text/plain, */*",
|
||||
"Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
|
||||
"Accept-Encoding": "gzip, deflate",
|
||||
"Connection": "keep-alive",
|
||||
}
|
||||
|
||||
|
||||
if referer:
|
||||
headers['Referer'] = referer
|
||||
|
||||
headers["Referer"] = referer
|
||||
|
||||
return headers
|
||||
|
||||
|
||||
@@ -80,17 +80,14 @@ def get_request_headers(referer: Optional[str] = None) -> dict:
|
||||
# Random sleep
|
||||
# ============================================
|
||||
|
||||
def random_sleep(
|
||||
min_seconds: float = 1.0,
|
||||
max_seconds: float = 3.0,
|
||||
log: bool = False
|
||||
) -> None:
|
||||
|
||||
def random_sleep(min_seconds: float = 1.0, max_seconds: float = 3.0, log: bool = False) -> None:
|
||||
"""
|
||||
Random sleep (Jitter)
|
||||
|
||||
|
||||
Anti-ban strategy: simulate random delays in human behavior
|
||||
Incorporate irregular wait times between requests
|
||||
|
||||
|
||||
Args:
|
||||
min_seconds: Minimum sleep time (seconds)
|
||||
max_seconds: Maximum sleep time (seconds)
|
||||
@@ -106,22 +103,18 @@ def random_sleep(
|
||||
# Request frequency limiter
|
||||
# ============================================
|
||||
|
||||
|
||||
class RateLimiter:
|
||||
"""
|
||||
Request frequency limiter
|
||||
|
||||
|
||||
Ensure there is a minimum amount of time between requests
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
min_interval: float = 1.0,
|
||||
jitter_min: float = 0.5,
|
||||
jitter_max: float = 1.5
|
||||
):
|
||||
|
||||
def __init__(self, min_interval: float = 1.0, jitter_min: float = 0.5, jitter_max: float = 1.5):
|
||||
"""
|
||||
Initialize frequency limiter
|
||||
|
||||
|
||||
Args:
|
||||
min_interval: Minimum request interval (seconds)
|
||||
jitter_min: minimum random jitter (seconds)
|
||||
@@ -131,33 +124,33 @@ class RateLimiter:
|
||||
self.jitter_min = jitter_min
|
||||
self.jitter_max = jitter_max
|
||||
self._last_request_time: Optional[float] = None
|
||||
|
||||
|
||||
def wait(self) -> float:
|
||||
"""
|
||||
Wait until the next request can be made
|
||||
|
||||
|
||||
Returns:
|
||||
Actual waiting time (seconds)
|
||||
"""
|
||||
wait_time = 0.0
|
||||
|
||||
|
||||
if self._last_request_time is not None:
|
||||
elapsed = time.time() - self._last_request_time
|
||||
if elapsed < self.min_interval:
|
||||
# Supplement sleep to minimum interval
|
||||
wait_time = self.min_interval - elapsed
|
||||
time.sleep(wait_time)
|
||||
|
||||
|
||||
# Add random jitter
|
||||
jitter = random.uniform(self.jitter_min, self.jitter_max)
|
||||
time.sleep(jitter)
|
||||
wait_time += jitter
|
||||
|
||||
|
||||
# Record the time of this request
|
||||
self._last_request_time = time.time()
|
||||
|
||||
|
||||
return wait_time
|
||||
|
||||
|
||||
def reset(self) -> None:
|
||||
"""reset limiter"""
|
||||
self._last_request_time = None
|
||||
@@ -167,17 +160,18 @@ class RateLimiter:
|
||||
# Exponential backoff retry decorator
|
||||
# ============================================
|
||||
|
||||
|
||||
def retry_with_backoff(
|
||||
max_attempts: int = 3,
|
||||
base_delay: float = 2.0,
|
||||
max_delay: float = 30.0,
|
||||
exponential_base: float = 2.0,
|
||||
exceptions: Tuple[Type[Exception], ...] = (Exception,),
|
||||
on_retry: Optional[Callable[[int, Exception], None]] = None
|
||||
on_retry: Optional[Callable[[int, Exception], None]] = None,
|
||||
):
|
||||
"""
|
||||
Exponential backoff retry decorator
|
||||
|
||||
|
||||
Args:
|
||||
max_attempts: Maximum number of retries
|
||||
base_delay: base delay time (seconds)
|
||||
@@ -185,49 +179,50 @@ def retry_with_backoff(
|
||||
exponential_base: exponential base
|
||||
exceptions: Exception types that need to be retried
|
||||
on_retry: callback function when retrying
|
||||
|
||||
|
||||
Usage example:
|
||||
@retry_with_backoff(max_attempts=3, exceptions=(ConnectionError, TimeoutError))
|
||||
def fetch_data():
|
||||
...
|
||||
"""
|
||||
|
||||
def decorator(func: Callable) -> Callable:
|
||||
@wraps(func)
|
||||
def wrapper(*args, **kwargs) -> Any:
|
||||
last_exception = None
|
||||
|
||||
|
||||
for attempt in range(1, max_attempts + 1):
|
||||
try:
|
||||
return func(*args, **kwargs)
|
||||
except exceptions as e:
|
||||
last_exception = e
|
||||
|
||||
|
||||
if attempt == max_attempts:
|
||||
logger.error(f"[Retry] {func.__name__} has reached the maximum number of retries ({max_attempts}), giving up")
|
||||
logger.error(
|
||||
f"[Retry] {func.__name__} has reached the maximum number of retries ({max_attempts}), giving up"
|
||||
)
|
||||
raise
|
||||
|
||||
|
||||
# Calculate the backoff delay: base_delay * (exponential_base ^ (attempt - 1))
|
||||
delay = min(
|
||||
base_delay * (exponential_base ** (attempt - 1)),
|
||||
max_delay
|
||||
)
|
||||
delay = min(base_delay * (exponential_base ** (attempt - 1)), max_delay)
|
||||
# Add random jitter (±20%)
|
||||
delay *= random.uniform(0.8, 1.2)
|
||||
|
||||
|
||||
logger.warning(
|
||||
f"[Retry] {func.__name__} failed for the {attempt}/{max_attempts} time: {e}, "
|
||||
f"waiting {delay:.1f}s before retrying..."
|
||||
)
|
||||
|
||||
|
||||
if on_retry:
|
||||
on_retry(attempt, e)
|
||||
|
||||
|
||||
time.sleep(delay)
|
||||
|
||||
|
||||
# Shouldn't have gotten here
|
||||
raise last_exception
|
||||
|
||||
|
||||
return wrapper
|
||||
|
||||
return decorator
|
||||
|
||||
|
||||
@@ -236,25 +231,13 @@ def retry_with_backoff(
|
||||
# ============================================
|
||||
|
||||
# Oriental Fortune interface current limiter (more stringent)
|
||||
_eastmoney_limiter = RateLimiter(
|
||||
min_interval=2.0,
|
||||
jitter_min=1.0,
|
||||
jitter_max=3.0
|
||||
)
|
||||
_eastmoney_limiter = RateLimiter(min_interval=2.0, jitter_min=1.0, jitter_max=3.0)
|
||||
|
||||
# Tencent Finance interface current limiter (relatively loose)
|
||||
_tencent_limiter = RateLimiter(
|
||||
min_interval=1.0,
|
||||
jitter_min=0.5,
|
||||
jitter_max=1.5
|
||||
)
|
||||
_tencent_limiter = RateLimiter(min_interval=1.0, jitter_min=0.5, jitter_max=1.5)
|
||||
|
||||
# Akshare interface current limiter
|
||||
_akshare_limiter = RateLimiter(
|
||||
min_interval=2.0,
|
||||
jitter_min=1.5,
|
||||
jitter_max=3.5
|
||||
)
|
||||
_akshare_limiter = RateLimiter(min_interval=2.0, jitter_min=1.5, jitter_max=3.5)
|
||||
|
||||
|
||||
def get_eastmoney_limiter() -> RateLimiter:
|
||||
|
||||
@@ -11,61 +11,16 @@ This is used as a stable alternative when Yahoo/yfinance gets rate-limited.
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
import requests
|
||||
|
||||
from app.data_sources.rate_limiter import get_request_headers, retry_with_backoff, get_tencent_limiter
|
||||
from app.data_sources.rate_limiter import get_request_headers, get_tencent_limiter, retry_with_backoff
|
||||
from app.utils.logger import get_logger
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
|
||||
def normalize_cn_code(symbol: str) -> str:
|
||||
"""
|
||||
Normalize A-share symbol to Tencent code: sh600519 / sz000001.
|
||||
Accepts:
|
||||
- 600519 / 600519.SH / 600519.SS
|
||||
- 000001 / 000001.SZ
|
||||
"""
|
||||
s = (symbol or "").strip().upper()
|
||||
if not s:
|
||||
return s
|
||||
if s.endswith(".SH"):
|
||||
s = s[:-3]
|
||||
return f"SH{s}"
|
||||
if s.endswith(".SS"):
|
||||
s = s[:-3]
|
||||
return f"SH{s}"
|
||||
if s.endswith(".SZ"):
|
||||
s = s[:-3]
|
||||
return f"SZ{s}"
|
||||
|
||||
if s.isdigit() and len(s) == 6:
|
||||
return ("SH" + s) if s.startswith("6") else ("SZ" + s)
|
||||
|
||||
return s
|
||||
|
||||
|
||||
def normalize_hk_code(symbol: str) -> str:
|
||||
"""
|
||||
Normalize HK stock symbol to Tencent code: hk00700 (5 digits).
|
||||
Accepts:
|
||||
- 700 / 0700 / 00700.HK / 0700.HK
|
||||
"""
|
||||
s = (symbol or "").strip().upper()
|
||||
if not s:
|
||||
return s
|
||||
if s.endswith(".HK"):
|
||||
s = s[:-3]
|
||||
if s.isdigit():
|
||||
return "HK" + s.zfill(5)
|
||||
# If user already passed HKxxxxx
|
||||
if s.startswith("HK") and s[2:].isdigit():
|
||||
return "HK" + s[2:].zfill(5)
|
||||
return s
|
||||
|
||||
|
||||
def _lower_code(code: str) -> str:
|
||||
return (code or "").strip().lower()
|
||||
|
||||
@@ -109,6 +64,7 @@ def parse_quote_to_ticker(parts: List[str]) -> Dict[str, Any]:
|
||||
"""
|
||||
Best-effort conversion to a unified ticker dict.
|
||||
"""
|
||||
|
||||
def _f(i: int, default: float = 0.0) -> float:
|
||||
try:
|
||||
v = parts[i]
|
||||
@@ -198,7 +154,7 @@ def fetch_kline(code: str, period: str, count: int = 300, adj: str = "qfq", time
|
||||
period examples:
|
||||
- day, week, month (supported by Tencent fqkline)
|
||||
|
||||
Note: Minute periods (m1/m5/…) return **bad params** on this endpoint; use AkShare in ``asia_stock_kline``.
|
||||
Note: Minute periods (m1/m5/…) return **bad params** on this endpoint.
|
||||
"""
|
||||
c = _lower_code(code)
|
||||
if not c:
|
||||
@@ -235,4 +191,3 @@ def fetch_kline(code: str, period: str, count: int = 300, adj: str = "qfq", time
|
||||
if isinstance(v, list) and v and str(k).lower().endswith(str(period).lower()):
|
||||
return v
|
||||
return []
|
||||
|
||||
|
||||
@@ -2,64 +2,57 @@
|
||||
US stock data source
|
||||
Get data using yfinance and finnhub
|
||||
"""
|
||||
from typing import Dict, List, Any, Optional
|
||||
|
||||
from datetime import datetime, timedelta
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
import yfinance as yf
|
||||
|
||||
from app.config import APIKeys
|
||||
from app.data_sources.base import BaseDataSource
|
||||
from app.utils.logger import get_logger
|
||||
from app.config import APIKeys, YFinanceConfig
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
|
||||
class USStockDataSource(BaseDataSource):
|
||||
"""US stock data source"""
|
||||
|
||||
|
||||
name = "USStock/yfinance"
|
||||
|
||||
|
||||
# yfinance time period mapping
|
||||
INTERVAL_MAP = {
|
||||
'1m': '1m',
|
||||
'5m': '5m',
|
||||
'15m': '15m',
|
||||
'30m': '30m',
|
||||
'1H': '1h',
|
||||
'4H': '4h',
|
||||
'1D': '1d',
|
||||
'1W': '1wk'
|
||||
}
|
||||
|
||||
INTERVAL_MAP = {"1m": "1m", "5m": "5m", "15m": "15m", "30m": "30m", "1H": "1h", "4H": "4h", "1D": "1d", "1W": "1wk"}
|
||||
|
||||
# The range of days to obtain data in different periods
|
||||
DAYS_MAP = {
|
||||
'1m': lambda limit: min(7, max(1, (limit // 390) + 2)),
|
||||
'5m': lambda limit: min(60, max(1, (limit // 78) + 2)),
|
||||
'15m': lambda limit: min(60, max(1, (limit // 26) + 2)),
|
||||
'30m': lambda limit: min(60, max(1, (limit // 13) + 2)),
|
||||
'1H': lambda limit: min(730, max(1, (limit // 24) + 2)),
|
||||
'4H': lambda limit: min(730, max(1, (limit // 6) + 2)),
|
||||
'1D': lambda limit: min(3650, limit + 1),
|
||||
'1W': lambda limit: min(3650, (limit * 7) + 7)
|
||||
"1m": lambda limit: min(7, max(1, (limit // 390) + 2)),
|
||||
"5m": lambda limit: min(60, max(1, (limit // 78) + 2)),
|
||||
"15m": lambda limit: min(60, max(1, (limit // 26) + 2)),
|
||||
"30m": lambda limit: min(60, max(1, (limit // 13) + 2)),
|
||||
"1H": lambda limit: min(730, max(1, (limit // 24) + 2)),
|
||||
"4H": lambda limit: min(730, max(1, (limit // 6) + 2)),
|
||||
"1D": lambda limit: min(3650, limit + 1),
|
||||
"1W": lambda limit: min(3650, (limit * 7) + 7),
|
||||
}
|
||||
|
||||
|
||||
def __init__(self):
|
||||
# Initialize finnhub as an alternative
|
||||
self.finnhub_client = None
|
||||
try:
|
||||
import finnhub
|
||||
if APIKeys.is_configured('FINNHUB_API_KEY'):
|
||||
|
||||
if APIKeys.is_configured("FINNHUB_API_KEY"):
|
||||
self.finnhub_client = finnhub.Client(api_key=APIKeys.FINNHUB_API_KEY)
|
||||
logger.info("Finnhub client initialized")
|
||||
except Exception as e:
|
||||
logger.warning(f"Finnhub init failed: {e}")
|
||||
|
||||
|
||||
def get_ticker(self, symbol: str) -> Dict[str, Any]:
|
||||
"""
|
||||
Get realtime quotes for U.S. stocks
|
||||
|
||||
|
||||
Use Finnhub first (more real-time), downgrade to yfinance fast_info
|
||||
|
||||
|
||||
Returns:
|
||||
dict: {
|
||||
'last': current price,
|
||||
@@ -71,21 +64,21 @@ class USStockDataSource(BaseDataSource):
|
||||
'previousClose': yesterday's closing price
|
||||
}
|
||||
"""
|
||||
symbol = (symbol or '').strip().upper()
|
||||
|
||||
symbol = (symbol or "").strip().upper()
|
||||
|
||||
# Prefer using Finnhub (live data)
|
||||
if self.finnhub_client:
|
||||
try:
|
||||
quote = self.finnhub_client.quote(symbol)
|
||||
if quote and quote.get('c'):
|
||||
if quote and quote.get("c"):
|
||||
return {
|
||||
'last': quote.get('c', 0), # current price
|
||||
'change': quote.get('d', 0), # Changes
|
||||
'changePercent': quote.get('dp', 0), # Increase or decrease
|
||||
'high': quote.get('h', 0), # Best in Japan
|
||||
'low': quote.get('l', 0), # Lowest within the day
|
||||
'open': quote.get('o', 0), # opening price
|
||||
'previousClose': quote.get('pc', 0) # Yesterday's closing price
|
||||
"last": quote.get("c", 0), # current price
|
||||
"change": quote.get("d", 0), # Changes
|
||||
"changePercent": quote.get("dp", 0), # Increase or decrease
|
||||
"high": quote.get("h", 0), # Best in Japan
|
||||
"low": quote.get("l", 0), # Lowest within the day
|
||||
"open": quote.get("o", 0), # opening price
|
||||
"previousClose": quote.get("pc", 0), # Yesterday's closing price
|
||||
}
|
||||
except Exception as e:
|
||||
msg = str(e).lower()
|
||||
@@ -93,94 +86,94 @@ class USStockDataSource(BaseDataSource):
|
||||
logger.debug(f"Finnhub quote skipped (no access): {symbol}: {e}")
|
||||
else:
|
||||
logger.warning(f"Finnhub quote failed for {symbol}: {e}")
|
||||
|
||||
|
||||
# Downgrade to use yfinance
|
||||
try:
|
||||
ticker = yf.Ticker(symbol)
|
||||
|
||||
|
||||
# Try fast_info (faster)
|
||||
try:
|
||||
fast_info = ticker.fast_info
|
||||
last_price = fast_info.get('lastPrice') or fast_info.get('last_price')
|
||||
prev_close = fast_info.get('previousClose') or fast_info.get('previous_close') or fast_info.get('regularMarketPreviousClose')
|
||||
|
||||
last_price = fast_info.get("lastPrice") or fast_info.get("last_price")
|
||||
prev_close = (
|
||||
fast_info.get("previousClose")
|
||||
or fast_info.get("previous_close")
|
||||
or fast_info.get("regularMarketPreviousClose")
|
||||
)
|
||||
|
||||
if last_price:
|
||||
change = (last_price - prev_close) if prev_close else 0
|
||||
change_pct = (change / prev_close * 100) if prev_close else 0
|
||||
return {
|
||||
'last': float(last_price),
|
||||
'change': round(change, 4),
|
||||
'changePercent': round(change_pct, 2),
|
||||
'high': float(fast_info.get('dayHigh') or fast_info.get('day_high') or last_price),
|
||||
'low': float(fast_info.get('dayLow') or fast_info.get('day_low') or last_price),
|
||||
'open': float(fast_info.get('open') or fast_info.get('regularMarketOpen') or last_price),
|
||||
'previousClose': float(prev_close) if prev_close else 0
|
||||
"last": float(last_price),
|
||||
"change": round(change, 4),
|
||||
"changePercent": round(change_pct, 2),
|
||||
"high": float(fast_info.get("dayHigh") or fast_info.get("day_high") or last_price),
|
||||
"low": float(fast_info.get("dayLow") or fast_info.get("day_low") or last_price),
|
||||
"open": float(fast_info.get("open") or fast_info.get("regularMarketOpen") or last_price),
|
||||
"previousClose": float(prev_close) if prev_close else 0,
|
||||
}
|
||||
except Exception as e:
|
||||
logger.debug(f"yfinance fast_info failed for {symbol}: {e}")
|
||||
|
||||
|
||||
# Downgrade to use info (slower but more complete data)
|
||||
try:
|
||||
info = ticker.info
|
||||
last_price = info.get('regularMarketPrice') or info.get('currentPrice')
|
||||
prev_close = info.get('regularMarketPreviousClose') or info.get('previousClose')
|
||||
|
||||
last_price = info.get("regularMarketPrice") or info.get("currentPrice")
|
||||
prev_close = info.get("regularMarketPreviousClose") or info.get("previousClose")
|
||||
|
||||
if last_price:
|
||||
change = (last_price - prev_close) if prev_close else 0
|
||||
change_pct = (change / prev_close * 100) if prev_close else 0
|
||||
return {
|
||||
'last': float(last_price),
|
||||
'change': round(change, 4),
|
||||
'changePercent': round(change_pct, 2),
|
||||
'high': float(info.get('regularMarketDayHigh') or info.get('dayHigh') or last_price),
|
||||
'low': float(info.get('regularMarketDayLow') or info.get('dayLow') or last_price),
|
||||
'open': float(info.get('regularMarketOpen') or info.get('open') or last_price),
|
||||
'previousClose': float(prev_close) if prev_close else 0
|
||||
"last": float(last_price),
|
||||
"change": round(change, 4),
|
||||
"changePercent": round(change_pct, 2),
|
||||
"high": float(info.get("regularMarketDayHigh") or info.get("dayHigh") or last_price),
|
||||
"low": float(info.get("regularMarketDayLow") or info.get("dayLow") or last_price),
|
||||
"open": float(info.get("regularMarketOpen") or info.get("open") or last_price),
|
||||
"previousClose": float(prev_close) if prev_close else 0,
|
||||
}
|
||||
except Exception as e:
|
||||
logger.debug(f"yfinance info failed for {symbol}: {e}")
|
||||
|
||||
|
||||
# Last downgrade: use the most recent 1-minute K-line
|
||||
try:
|
||||
hist = ticker.history(period='1d', interval='1m')
|
||||
hist = ticker.history(period="1d", interval="1m")
|
||||
if hist is not None and not hist.empty:
|
||||
last_row = hist.iloc[-1]
|
||||
first_row = hist.iloc[0]
|
||||
last_price = float(last_row['Close'])
|
||||
open_price = float(first_row['Open'])
|
||||
|
||||
last_price = float(last_row["Close"])
|
||||
open_price = float(first_row["Open"])
|
||||
|
||||
return {
|
||||
'last': last_price,
|
||||
'change': round(last_price - open_price, 4),
|
||||
'changePercent': round((last_price - open_price) / open_price * 100, 2) if open_price else 0,
|
||||
'high': float(hist['High'].max()),
|
||||
'low': float(hist['Low'].min()),
|
||||
'open': open_price,
|
||||
'previousClose': open_price # approximate
|
||||
"last": last_price,
|
||||
"change": round(last_price - open_price, 4),
|
||||
"changePercent": round((last_price - open_price) / open_price * 100, 2) if open_price else 0,
|
||||
"high": float(hist["High"].max()),
|
||||
"low": float(hist["Low"].min()),
|
||||
"open": open_price,
|
||||
"previousClose": open_price, # approximate
|
||||
}
|
||||
except Exception as e:
|
||||
logger.debug(f"yfinance history fallback failed for {symbol}: {e}")
|
||||
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to get ticker for {symbol}: {e}")
|
||||
|
||||
return {'last': 0, 'symbol': symbol}
|
||||
|
||||
|
||||
return {"last": 0, "symbol": symbol}
|
||||
|
||||
def get_kline(
|
||||
self,
|
||||
symbol: str,
|
||||
timeframe: str,
|
||||
limit: int,
|
||||
before_time: Optional[int] = None
|
||||
self, symbol: str, timeframe: str, limit: int, before_time: Optional[int] = None
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""Get U.S. stock K-line data"""
|
||||
klines = []
|
||||
|
||||
|
||||
try:
|
||||
interval = self.INTERVAL_MAP.get(timeframe, '1d')
|
||||
interval = self.INTERVAL_MAP.get(timeframe, "1d")
|
||||
days_func = self.DAYS_MAP.get(timeframe, lambda x: x + 1)
|
||||
days = days_func(limit)
|
||||
|
||||
|
||||
# Calculate date range
|
||||
if before_time:
|
||||
end_date = datetime.fromtimestamp(before_time)
|
||||
@@ -188,80 +181,75 @@ class USStockDataSource(BaseDataSource):
|
||||
else:
|
||||
end_date = datetime.now()
|
||||
start_date = end_date - timedelta(days=days)
|
||||
|
||||
|
||||
# logger.info(f"Use yfinance to get {symbol}, period: {interval}, date: {start_date.date()} ~ {end_date.date()}")
|
||||
|
||||
|
||||
# Try yfinance
|
||||
df = self._fetch_yfinance(symbol, interval, start_date, end_date)
|
||||
|
||||
|
||||
if df is None or df.empty:
|
||||
# try finnhub
|
||||
if self.finnhub_client and timeframe == '1D':
|
||||
if self.finnhub_client and timeframe == "1D":
|
||||
klines = self._fetch_finnhub(symbol, start_date, end_date, limit)
|
||||
if klines:
|
||||
return klines
|
||||
else:
|
||||
klines = self._convert_dataframe(df, limit)
|
||||
|
||||
|
||||
# Filter and restrict
|
||||
klines = self.filter_and_limit(klines, limit, before_time)
|
||||
|
||||
|
||||
# Record results
|
||||
self.log_result(symbol, klines, timeframe)
|
||||
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to fetch US stock K-lines {symbol}: {str(e)}")
|
||||
import traceback
|
||||
|
||||
logger.error(traceback.format_exc())
|
||||
|
||||
|
||||
return klines
|
||||
|
||||
|
||||
def _fetch_yfinance(self, symbol: str, interval: str, start_date: datetime, end_date: datetime):
|
||||
"""Use yfinance to get data"""
|
||||
try:
|
||||
ticker = yf.Ticker(symbol)
|
||||
|
||||
|
||||
# The end parameter of yfinance is not included (exclusive), so you need to add one day to include the end_date data of the current day.
|
||||
# For example: end="2026-01-12" actually only returns the data of 2026-01-11
|
||||
end_date_inclusive = end_date + timedelta(days=1)
|
||||
|
||||
|
||||
df = ticker.history(
|
||||
start=start_date.strftime('%Y-%m-%d'),
|
||||
end=end_date_inclusive.strftime('%Y-%m-%d'),
|
||||
interval=interval
|
||||
start=start_date.strftime("%Y-%m-%d"), end=end_date_inclusive.strftime("%Y-%m-%d"), interval=interval
|
||||
)
|
||||
# logger.info(f"yfinance returns {len(df) if df is not None and not df.empty else 0} pieces of data")
|
||||
return df
|
||||
except Exception as e:
|
||||
logger.warning(f"yfinance fetch failed: {e}")
|
||||
return None
|
||||
|
||||
def _fetch_finnhub(
|
||||
self,
|
||||
symbol: str,
|
||||
start_date: datetime,
|
||||
end_date: datetime,
|
||||
limit: int
|
||||
) -> List[Dict[str, Any]]:
|
||||
|
||||
def _fetch_finnhub(self, symbol: str, start_date: datetime, end_date: datetime, limit: int) -> List[Dict[str, Any]]:
|
||||
"""Use finnhub to get daily data"""
|
||||
klines = []
|
||||
try:
|
||||
start_ts = int(start_date.timestamp())
|
||||
end_ts = int(end_date.timestamp())
|
||||
|
||||
|
||||
# logger.info(f"Use Finnhub to obtain {symbol} daily data")
|
||||
candles = self.finnhub_client.stock_candles(symbol, 'D', start_ts, end_ts)
|
||||
|
||||
if candles and candles.get('s') == 'ok':
|
||||
for i in range(len(candles['t'])):
|
||||
klines.append(self.format_kline(
|
||||
timestamp=candles['t'][i],
|
||||
open_price=candles['o'][i],
|
||||
high=candles['h'][i],
|
||||
low=candles['l'][i],
|
||||
close=candles['c'][i],
|
||||
volume=candles['v'][i]
|
||||
))
|
||||
candles = self.finnhub_client.stock_candles(symbol, "D", start_ts, end_ts)
|
||||
|
||||
if candles and candles.get("s") == "ok":
|
||||
for i in range(len(candles["t"])):
|
||||
klines.append(
|
||||
self.format_kline(
|
||||
timestamp=candles["t"][i],
|
||||
open_price=candles["o"][i],
|
||||
high=candles["h"][i],
|
||||
low=candles["l"][i],
|
||||
close=candles["c"][i],
|
||||
volume=candles["v"][i],
|
||||
)
|
||||
)
|
||||
# logger.info(f"Finnhub returns {len(klines)} pieces of data")
|
||||
except Exception as e:
|
||||
msg = str(e).lower()
|
||||
@@ -270,47 +258,48 @@ class USStockDataSource(BaseDataSource):
|
||||
logger.debug(f"Finnhub candles skipped (no access): {symbol}: {e}")
|
||||
else:
|
||||
logger.warning(f"Finnhub fetch failed: {e}")
|
||||
|
||||
|
||||
return klines
|
||||
|
||||
|
||||
def _convert_dataframe(self, df, limit: int) -> List[Dict[str, Any]]:
|
||||
"""Convert DataFrame to K-line list"""
|
||||
klines = []
|
||||
df = df.tail(limit).reset_index()
|
||||
|
||||
|
||||
# Determine the time column name (the daily line is Date, the minute level is Datetime)
|
||||
time_col = None
|
||||
if 'Datetime' in df.columns:
|
||||
time_col = 'Datetime'
|
||||
elif 'Date' in df.columns:
|
||||
time_col = 'Date'
|
||||
elif 'index' in df.columns:
|
||||
time_col = 'index'
|
||||
|
||||
if "Datetime" in df.columns:
|
||||
time_col = "Datetime"
|
||||
elif "Date" in df.columns:
|
||||
time_col = "Date"
|
||||
elif "index" in df.columns:
|
||||
time_col = "index"
|
||||
|
||||
if time_col is None:
|
||||
logger.warning(f"Unable to determine time column; available columns: {df.columns.tolist()}")
|
||||
return klines
|
||||
|
||||
|
||||
for _, row in df.iterrows():
|
||||
try:
|
||||
# Processing timestamps
|
||||
time_value = row[time_col]
|
||||
if hasattr(time_value, 'timestamp'):
|
||||
if hasattr(time_value, "timestamp"):
|
||||
ts = int(time_value.timestamp())
|
||||
else:
|
||||
continue
|
||||
|
||||
klines.append(self.format_kline(
|
||||
timestamp=ts,
|
||||
open_price=row['Open'],
|
||||
high=row['High'],
|
||||
low=row['Low'],
|
||||
close=row['Close'],
|
||||
volume=row['Volume']
|
||||
))
|
||||
|
||||
klines.append(
|
||||
self.format_kline(
|
||||
timestamp=ts,
|
||||
open_price=row["Open"],
|
||||
high=row["High"],
|
||||
low=row["Low"],
|
||||
close=row["Close"],
|
||||
volume=row["Volume"],
|
||||
)
|
||||
)
|
||||
except Exception as e:
|
||||
logger.debug(f"Failed to parse row data: {e}")
|
||||
continue
|
||||
|
||||
return klines
|
||||
|
||||
return klines
|
||||
|
||||
Reference in New Issue
Block a user