Dinger
@@ -0,0 +1,566 @@
|
||||
"""
|
||||
A-share / H-share chart K-lines — multi-tier fallback.
|
||||
|
||||
Priority order (when TWELVE_DATA_API_KEY is configured):
|
||||
ALL timeframes → Twelve Data (paid, globally stable) → Tencent daily/weekly → yfinance → AkShare
|
||||
|
||||
Without API key:
|
||||
Daily / Weekly → Tencent fqkline (fast, no key) → yfinance → AkShare
|
||||
Minute / Hour → yfinance → AkShare (Eastmoney, fragile overseas)
|
||||
|
||||
Tencent ``fqkline`` only reliably supports day/week/month.
|
||||
yfinance supports CN (.SS/.SZ) and HK (.HK) at all common intervals.
|
||||
Twelve Data (https://twelvedata.com) supports XSHG/XSHE/XHKG at all intervals.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import time
|
||||
from datetime import datetime, timedelta
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
import pandas as pd
|
||||
import requests
|
||||
|
||||
from app.utils.logger import get_logger
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
_MAX_ATTEMPTS = 3
|
||||
_BACKOFF_BASE_SEC = 1.5
|
||||
_BACKOFF_CAP_SEC = 12.0
|
||||
|
||||
_TRANSIENT_ERR_MARKERS = (
|
||||
"remote end closed connection",
|
||||
"connection aborted",
|
||||
"connection reset",
|
||||
"timed out",
|
||||
"timeout",
|
||||
"max retries exceeded",
|
||||
"temporarily unavailable",
|
||||
"broken pipe",
|
||||
"eof occurred",
|
||||
"remote disconnected",
|
||||
"chunkedencodingerror",
|
||||
"incompleteread",
|
||||
"rate",
|
||||
"too many requests",
|
||||
"429",
|
||||
)
|
||||
|
||||
|
||||
def _is_transient(exc: BaseException) -> bool:
|
||||
return any(m in str(exc).lower() for m in _TRANSIENT_ERR_MARKERS)
|
||||
|
||||
|
||||
_CHART_TF_ALIASES = {
|
||||
"1w": "1W",
|
||||
"1d": "1D",
|
||||
"1h": "1H",
|
||||
"4h": "4H",
|
||||
"d": "1D",
|
||||
"day": "1D",
|
||||
"w": "1W",
|
||||
"week": "1W",
|
||||
"wk": "1W",
|
||||
"60m": "1H",
|
||||
"240m": "4H",
|
||||
"1day": "1D",
|
||||
"1week": "1W",
|
||||
}
|
||||
|
||||
|
||||
def normalize_chart_timeframe(timeframe: str) -> str:
|
||||
t = (timeframe or "1D").strip()
|
||||
if not t:
|
||||
return "1D"
|
||||
key = t.lower()
|
||||
if key in _CHART_TF_ALIASES:
|
||||
return _CHART_TF_ALIASES[key]
|
||||
return t
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# AkShare code converters
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def ak_a_code_from_tencent(tencent_code: str) -> str:
|
||||
c = (tencent_code or "").strip().lower()
|
||||
if len(c) >= 8 and c[:2] in ("sh", "sz"):
|
||||
return c[2:]
|
||||
return c
|
||||
|
||||
|
||||
def ak_hk_code_from_tencent(tencent_code: str) -> str:
|
||||
c = (tencent_code or "").strip().upper().replace(".HK", "")
|
||||
if c.startswith("HK"):
|
||||
num = c[2:]
|
||||
else:
|
||||
num = c
|
||||
if num.isdigit():
|
||||
return num.zfill(5)
|
||||
return num
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Twelve Data (paid, globally reliable — https://twelvedata.com)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _get_twelve_data_api_key() -> str:
|
||||
try:
|
||||
from app.utils.config_loader import load_addon_config
|
||||
key = load_addon_config().get("twelve_data", {}).get("api_key", "")
|
||||
if key:
|
||||
return key
|
||||
except Exception:
|
||||
pass
|
||||
return (os.getenv("TWELVE_DATA_API_KEY") or "").strip()
|
||||
|
||||
|
||||
_TD_INTERVAL_MAP = {
|
||||
"1m": "1min",
|
||||
"5m": "5min",
|
||||
"15m": "15min",
|
||||
"30m": "30min",
|
||||
"1H": "1h",
|
||||
"4H": "4h",
|
||||
"1D": "1day",
|
||||
"1W": "1week",
|
||||
}
|
||||
|
||||
|
||||
def _td_symbol_and_exchange(tencent_code: str, is_hk: bool) -> tuple[str, str]:
|
||||
"""Convert Tencent code to Twelve Data (symbol, exchange)."""
|
||||
c = (tencent_code or "").strip().upper()
|
||||
if is_hk:
|
||||
num = c.replace("HK", "")
|
||||
if num.isdigit():
|
||||
num = str(int(num)).zfill(4)
|
||||
return num, "XHKG"
|
||||
digits = c.lstrip("SHSZ")
|
||||
if c.startswith("SH") or digits.startswith("6"):
|
||||
return digits, "XSHG"
|
||||
return digits, "XSHE"
|
||||
|
||||
|
||||
def fetch_twelvedata_klines(
|
||||
*,
|
||||
is_hk: bool,
|
||||
tencent_code: str,
|
||||
timeframe: str,
|
||||
limit: int,
|
||||
before_time: Optional[int],
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""Fetch K-lines from Twelve Data REST API. Requires TWELVE_DATA_API_KEY."""
|
||||
api_key = _get_twelve_data_api_key()
|
||||
if not api_key:
|
||||
return []
|
||||
|
||||
interval = _TD_INTERVAL_MAP.get(timeframe)
|
||||
if not interval:
|
||||
return []
|
||||
|
||||
symbol, exchange = _td_symbol_and_exchange(tencent_code, is_hk)
|
||||
params: Dict[str, Any] = {
|
||||
"symbol": symbol,
|
||||
"exchange": exchange,
|
||||
"interval": interval,
|
||||
"outputsize": min(int(limit), 5000),
|
||||
"apikey": api_key,
|
||||
"format": "JSON",
|
||||
"dp": "4",
|
||||
}
|
||||
if before_time:
|
||||
end_dt = datetime.fromtimestamp(int(before_time))
|
||||
params["end_date"] = end_dt.strftime("%Y-%m-%d %H:%M:%S")
|
||||
|
||||
url = "https://api.twelvedata.com/time_series"
|
||||
|
||||
for attempt in range(_MAX_ATTEMPTS):
|
||||
try:
|
||||
resp = requests.get(url, params=params, timeout=20)
|
||||
data = resp.json()
|
||||
break
|
||||
except Exception as e:
|
||||
if attempt + 1 < _MAX_ATTEMPTS and _is_transient(e):
|
||||
delay = min(_BACKOFF_CAP_SEC, _BACKOFF_BASE_SEC * (2 ** attempt))
|
||||
logger.debug(
|
||||
"TwelveData transient error %s/%s tf=%s (attempt %s/%s): %s",
|
||||
symbol, exchange, timeframe, attempt + 1, _MAX_ATTEMPTS, e,
|
||||
)
|
||||
time.sleep(delay)
|
||||
continue
|
||||
logger.warning("TwelveData request failed %s/%s tf=%s: %s", symbol, exchange, timeframe, e)
|
||||
return []
|
||||
else:
|
||||
return []
|
||||
|
||||
if data.get("status") != "ok" or "values" not in data:
|
||||
code = data.get("code", "")
|
||||
msg = data.get("message", str(data))
|
||||
if code == 429 or "API credits" in msg or "minute limit" in msg:
|
||||
logger.warning("TwelveData rate limit for %s/%s: %s", symbol, exchange, msg)
|
||||
else:
|
||||
logger.warning("TwelveData error %s/%s tf=%s: %s", symbol, exchange, timeframe, msg)
|
||||
return []
|
||||
|
||||
out: List[Dict[str, Any]] = []
|
||||
for v in data["values"]:
|
||||
try:
|
||||
dt_str = v.get("datetime", "")
|
||||
for fmt in ("%Y-%m-%d %H:%M:%S", "%Y-%m-%d"):
|
||||
try:
|
||||
ts = int(datetime.strptime(dt_str, fmt).timestamp())
|
||||
break
|
||||
except ValueError:
|
||||
continue
|
||||
else:
|
||||
continue
|
||||
o = float(v["open"])
|
||||
h = float(v["high"])
|
||||
low = float(v["low"])
|
||||
c = float(v["close"])
|
||||
vol = float(v.get("volume") or 0)
|
||||
if o == 0 and c == 0:
|
||||
continue
|
||||
out.append({
|
||||
"time": ts,
|
||||
"open": round(o, 4),
|
||||
"high": round(h, 4),
|
||||
"low": round(low, 4),
|
||||
"close": round(c, 4),
|
||||
"volume": round(vol, 2),
|
||||
})
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
out.sort(key=lambda x: x["time"])
|
||||
logger.debug("TwelveData returned %d bars for %s/%s tf=%s", len(out), symbol, exchange, timeframe)
|
||||
return out
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# yfinance helpers (globally accessible — Yahoo CDN)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def yf_symbol_from_tencent(tencent_code: str, is_hk: bool) -> str:
|
||||
"""Convert Tencent-style code (SH600519 / SZ000001 / HK00700) to yfinance ticker."""
|
||||
c = (tencent_code or "").strip().upper()
|
||||
if is_hk:
|
||||
num = c.replace("HK", "")
|
||||
if num.isdigit():
|
||||
return str(int(num)).zfill(4) + ".HK"
|
||||
return num + ".HK"
|
||||
if c.startswith("SH"):
|
||||
return c[2:] + ".SS"
|
||||
if c.startswith("SZ"):
|
||||
return c[2:] + ".SZ"
|
||||
digits = c.lstrip("SHSZ")
|
||||
if digits.startswith("6"):
|
||||
return digits + ".SS"
|
||||
return digits + ".SZ"
|
||||
|
||||
|
||||
_YF_INTERVAL_MAP = {
|
||||
"1m": "1m",
|
||||
"5m": "5m",
|
||||
"15m": "15m",
|
||||
"30m": "30m",
|
||||
"1H": "1h",
|
||||
"4H": "1h",
|
||||
"1D": "1d",
|
||||
"1W": "1wk",
|
||||
}
|
||||
|
||||
_YF_DAYS_MAP = {
|
||||
"1m": lambda lim: min(7, max(2, (lim // 240) + 2)),
|
||||
"5m": lambda lim: min(60, max(3, (lim // 48) + 3)),
|
||||
"15m": lambda lim: min(60, max(3, (lim // 16) + 3)),
|
||||
"30m": lambda lim: min(60, max(5, (lim // 8) + 5)),
|
||||
"1H": lambda lim: min(730, max(8, (lim // 4) + 8)),
|
||||
"4H": lambda lim: min(730, max(20, lim + 10)),
|
||||
"1D": lambda lim: min(3650, lim + 10),
|
||||
"1W": lambda lim: min(3650, lim * 7 + 30),
|
||||
}
|
||||
|
||||
|
||||
def _bars_from_yfinance_df(df: Any) -> List[Dict[str, Any]]:
|
||||
"""Convert a yfinance DataFrame (with DatetimeIndex or Date/Datetime column) to bar dicts."""
|
||||
if df is None or getattr(df, "empty", True):
|
||||
return []
|
||||
df = df.reset_index()
|
||||
time_col = None
|
||||
for candidate in ("Datetime", "Date", "index"):
|
||||
if candidate in df.columns:
|
||||
time_col = candidate
|
||||
break
|
||||
if time_col is None:
|
||||
return []
|
||||
out: List[Dict[str, Any]] = []
|
||||
for _, row in df.iterrows():
|
||||
try:
|
||||
tv = row[time_col]
|
||||
if hasattr(tv, "timestamp"):
|
||||
ts = int(tv.timestamp())
|
||||
else:
|
||||
continue
|
||||
o, h, low, c, v = (
|
||||
float(row["Open"]),
|
||||
float(row["High"]),
|
||||
float(row["Low"]),
|
||||
float(row["Close"]),
|
||||
float(row["Volume"]),
|
||||
)
|
||||
if o == 0 and c == 0:
|
||||
continue
|
||||
out.append({
|
||||
"time": ts,
|
||||
"open": round(o, 4),
|
||||
"high": round(h, 4),
|
||||
"low": round(low, 4),
|
||||
"close": round(c, 4),
|
||||
"volume": round(v, 2),
|
||||
})
|
||||
except Exception:
|
||||
continue
|
||||
out.sort(key=lambda x: x["time"])
|
||||
return out
|
||||
|
||||
|
||||
def fetch_yfinance_klines(
|
||||
*,
|
||||
is_hk: bool,
|
||||
tencent_code: str,
|
||||
timeframe: str,
|
||||
limit: int,
|
||||
before_time: Optional[int],
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""Fetch K-lines via yfinance for CN/HK stocks. Globally accessible, no API key needed."""
|
||||
try:
|
||||
import yfinance as yf
|
||||
except ImportError:
|
||||
logger.debug("yfinance not installed; skipping yfinance K-lines")
|
||||
return []
|
||||
|
||||
interval = _YF_INTERVAL_MAP.get(timeframe)
|
||||
if not interval:
|
||||
return []
|
||||
|
||||
yf_sym = yf_symbol_from_tencent(tencent_code, is_hk)
|
||||
effective_limit = limit * 4 if timeframe == "4H" else limit
|
||||
days_func = _YF_DAYS_MAP.get(timeframe, lambda x: x + 10)
|
||||
days = days_func(effective_limit)
|
||||
|
||||
end = datetime.fromtimestamp(int(before_time)) if before_time else datetime.now()
|
||||
start = end - timedelta(days=days)
|
||||
|
||||
df: Any = None
|
||||
for attempt in range(_MAX_ATTEMPTS):
|
||||
try:
|
||||
ticker = yf.Ticker(yf_sym)
|
||||
df = ticker.history(
|
||||
start=start.strftime("%Y-%m-%d"),
|
||||
end=(end + timedelta(days=1)).strftime("%Y-%m-%d"),
|
||||
interval=interval,
|
||||
)
|
||||
break
|
||||
except Exception as e:
|
||||
if attempt + 1 < _MAX_ATTEMPTS and _is_transient(e):
|
||||
delay = min(_BACKOFF_CAP_SEC, _BACKOFF_BASE_SEC * (2 ** attempt))
|
||||
logger.debug(
|
||||
"yfinance transient error %s tf=%s (attempt %s/%s), retry in %.1fs: %s",
|
||||
yf_sym, timeframe, attempt + 1, _MAX_ATTEMPTS, delay, e,
|
||||
)
|
||||
time.sleep(delay)
|
||||
continue
|
||||
logger.warning("yfinance K-line failed %s tf=%s: %s", yf_sym, timeframe, e)
|
||||
return []
|
||||
|
||||
bars = _bars_from_yfinance_df(df)
|
||||
if timeframe == "4H" and bars:
|
||||
bars = _merge_every_n_sorted_bars(bars, 4)
|
||||
logger.debug("yfinance returned %d bars for %s tf=%s", len(bars), yf_sym, timeframe)
|
||||
return bars
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# AkShare helpers (Eastmoney — unreliable from overseas, used as last resort)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _minute_period_str(timeframe: str) -> Optional[str]:
|
||||
return {"1m": "1", "5m": "5", "15m": "15", "30m": "30", "1H": "60", "4H": "60"}.get(timeframe)
|
||||
|
||||
|
||||
def _min_bar_window(timeframe: str, limit: int, before_time: Optional[int]) -> tuple[str, str]:
|
||||
_ = (timeframe, limit)
|
||||
end = datetime.fromtimestamp(int(before_time)) if before_time else datetime.now()
|
||||
start = end - timedelta(days=16)
|
||||
fmt = "%Y-%m-%d %H:%M:%S"
|
||||
return start.strftime(fmt), end.strftime(fmt)
|
||||
|
||||
|
||||
def _bars_from_ak_min_df(df: Any) -> List[Dict[str, Any]]:
|
||||
if df is None or getattr(df, "empty", True):
|
||||
return []
|
||||
cols = [str(x) for x in df.columns]
|
||||
time_c = "时间" if "时间" in cols else (cols[0] if len(cols) > 5 else None)
|
||||
if not time_c:
|
||||
return []
|
||||
|
||||
def _pick(name_zh: str, idx: int) -> str:
|
||||
return name_zh if name_zh in cols else (cols[idx] if len(cols) > idx else "")
|
||||
|
||||
c_open = _pick("开盘", 1)
|
||||
c_close = _pick("收盘", 2)
|
||||
c_high = _pick("最高", 3)
|
||||
c_low = _pick("最低", 4)
|
||||
c_vol = _pick("成交量", 5)
|
||||
if not all((c_open, c_close, c_high, c_low, c_vol)):
|
||||
return []
|
||||
|
||||
out: List[Dict[str, Any]] = []
|
||||
for _, row in df.iterrows():
|
||||
try:
|
||||
t = pd.Timestamp(row[time_c])
|
||||
ts = int(t.timestamp())
|
||||
o, c, h, low, v = float(row[c_open]), float(row[c_close]), float(row[c_high]), float(row[c_low]), float(row[c_vol])
|
||||
out.append({
|
||||
"time": ts,
|
||||
"open": round(o, 4),
|
||||
"high": round(h, 4),
|
||||
"low": round(low, 4),
|
||||
"close": round(c, 4),
|
||||
"volume": round(v, 2),
|
||||
})
|
||||
except Exception:
|
||||
continue
|
||||
out.sort(key=lambda x: x["time"])
|
||||
return out
|
||||
|
||||
|
||||
def _merge_every_n_sorted_bars(bars: List[Dict[str, Any]], n: int) -> List[Dict[str, Any]]:
|
||||
if n <= 1 or len(bars) < n:
|
||||
return bars
|
||||
out: List[Dict[str, Any]] = []
|
||||
i = 0
|
||||
while i + n <= len(bars):
|
||||
chunk = bars[i : i + n]
|
||||
out.append({
|
||||
"time": chunk[0]["time"],
|
||||
"open": chunk[0]["open"],
|
||||
"high": max(b["high"] for b in chunk),
|
||||
"low": min(b["low"] for b in chunk),
|
||||
"close": chunk[-1]["close"],
|
||||
"volume": round(sum(b["volume"] for b in chunk), 2),
|
||||
})
|
||||
i += n
|
||||
return out
|
||||
|
||||
|
||||
def fetch_akshare_minute_klines(
|
||||
*,
|
||||
is_hk: bool,
|
||||
tencent_code: str,
|
||||
timeframe: str,
|
||||
limit: int,
|
||||
before_time: Optional[int],
|
||||
) -> List[Dict[str, Any]]:
|
||||
p = _minute_period_str(timeframe)
|
||||
if p is None:
|
||||
return []
|
||||
try:
|
||||
import akshare as ak # type: ignore
|
||||
except ImportError:
|
||||
logger.debug("akshare not installed; skipping AkShare minute K-lines")
|
||||
return []
|
||||
|
||||
sym = ak_hk_code_from_tencent(tencent_code) if is_hk else ak_a_code_from_tencent(tencent_code)
|
||||
sd, ed = _min_bar_window(timeframe, limit, before_time)
|
||||
adj = "" if p == "1" else "qfq"
|
||||
|
||||
df: Any = None
|
||||
for attempt in range(_MAX_ATTEMPTS):
|
||||
try:
|
||||
if is_hk:
|
||||
df = ak.stock_hk_hist_min_em(symbol=sym, period=p, adjust=adj, start_date=sd, end_date=ed)
|
||||
else:
|
||||
df = ak.stock_zh_a_hist_min_em(symbol=sym, start_date=sd, end_date=ed, period=p, adjust=adj)
|
||||
break
|
||||
except Exception as e:
|
||||
if attempt + 1 < _MAX_ATTEMPTS and _is_transient(e):
|
||||
delay = min(_BACKOFF_CAP_SEC, _BACKOFF_BASE_SEC * (2 ** attempt))
|
||||
logger.debug(
|
||||
"AkShare minute transient error %s tf=%s sym=%s (attempt %s/%s): %s",
|
||||
tencent_code, timeframe, sym, attempt + 1, _MAX_ATTEMPTS, e,
|
||||
)
|
||||
time.sleep(delay)
|
||||
continue
|
||||
logger.warning("AkShare minute K-line failed %s tf=%s sym=%s: %s", tencent_code, timeframe, sym, e)
|
||||
return []
|
||||
|
||||
bars = _bars_from_ak_min_df(df)
|
||||
if timeframe == "4H" and bars:
|
||||
bars = _merge_every_n_sorted_bars(bars, 4)
|
||||
return bars
|
||||
|
||||
|
||||
def fetch_akshare_weekly_klines(
|
||||
*,
|
||||
is_hk: bool,
|
||||
tencent_code: str,
|
||||
limit: int,
|
||||
before_time: Optional[int],
|
||||
) -> List[Dict[str, Any]]:
|
||||
try:
|
||||
import akshare as ak # type: ignore
|
||||
except ImportError:
|
||||
return []
|
||||
|
||||
sym = ak_hk_code_from_tencent(tencent_code) if is_hk else ak_a_code_from_tencent(tencent_code)
|
||||
end = datetime.fromtimestamp(int(before_time)) if before_time else datetime.now()
|
||||
start = end - timedelta(days=max(int(limit or 300), 1) * 14 + 400)
|
||||
start_s = start.strftime("%Y%m%d")
|
||||
end_s = end.strftime("%Y%m%d")
|
||||
|
||||
df: Any = None
|
||||
for attempt in range(_MAX_ATTEMPTS):
|
||||
try:
|
||||
if is_hk:
|
||||
df = ak.stock_hk_hist(symbol=sym, period="weekly", start_date=start_s, end_date=end_s, adjust="qfq")
|
||||
else:
|
||||
df = ak.stock_zh_a_hist(symbol=sym, period="weekly", start_date=start_s, end_date=end_s, adjust="qfq")
|
||||
break
|
||||
except Exception as e:
|
||||
if attempt + 1 < _MAX_ATTEMPTS and _is_transient(e):
|
||||
delay = min(_BACKOFF_CAP_SEC, _BACKOFF_BASE_SEC * (2 ** attempt))
|
||||
logger.debug(
|
||||
"AkShare weekly transient error sym=%s (attempt %s/%s): %s",
|
||||
sym, attempt + 1, _MAX_ATTEMPTS, e,
|
||||
)
|
||||
time.sleep(delay)
|
||||
continue
|
||||
logger.warning("AkShare weekly K-line failed sym=%s: %s", sym, e)
|
||||
return []
|
||||
|
||||
if df is None or getattr(df, "empty", True) or "日期" not in df.columns:
|
||||
return []
|
||||
out: List[Dict[str, Any]] = []
|
||||
for _, row in df.iterrows():
|
||||
try:
|
||||
t = pd.Timestamp(row["日期"])
|
||||
ts = int(t.timestamp())
|
||||
o, c, h, low = float(row["开盘"]), float(row["收盘"]), float(row["最高"]), float(row["最低"])
|
||||
v = float(row["成交量"])
|
||||
out.append({
|
||||
"time": ts,
|
||||
"open": round(o, 4),
|
||||
"high": round(h, 4),
|
||||
"low": round(low, 4),
|
||||
"close": round(c, 4),
|
||||
"volume": round(v, 2),
|
||||
})
|
||||
except Exception:
|
||||
continue
|
||||
out.sort(key=lambda x: x["time"])
|
||||
return out
|
||||
@@ -0,0 +1,292 @@
|
||||
"""
|
||||
A-share / HK share fundamentals — multi-tier fallback.
|
||||
|
||||
Priority (when TWELVE_DATA_API_KEY configured):
|
||||
Twelve Data /statistics + /profile → AkShare (Eastmoney, fragile overseas)
|
||||
|
||||
Without API key:
|
||||
AkShare only (may fail from overseas servers)
|
||||
|
||||
Keys are aligned with MarketDataCollector expectations (pe_ratio, pb_ratio, etc.).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
import time
|
||||
from typing import Any, Dict, Optional
|
||||
|
||||
import requests
|
||||
|
||||
from app.data_sources.asia_stock_kline import (
|
||||
_get_twelve_data_api_key,
|
||||
_td_symbol_and_exchange,
|
||||
ak_a_code_from_tencent,
|
||||
ak_hk_code_from_tencent,
|
||||
)
|
||||
from app.utils.logger import get_logger
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
_TD_TIMEOUT = 15
|
||||
_TD_MAX_ATTEMPTS = 2
|
||||
_TD_BACKOFF_SEC = 2.0
|
||||
|
||||
|
||||
def _float_clean(x: Any) -> Optional[float]:
|
||||
if x is None or x == "":
|
||||
return None
|
||||
try:
|
||||
v = float(x)
|
||||
if math.isnan(v) or math.isinf(v):
|
||||
return None
|
||||
return v
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Twelve Data fundamentals (globally stable, paid)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _td_request(endpoint: str, symbol: str, exchange: str) -> Optional[Dict[str, Any]]:
|
||||
"""Generic Twelve Data GET with retry."""
|
||||
api_key = _get_twelve_data_api_key()
|
||||
if not api_key:
|
||||
return None
|
||||
url = f"https://api.twelvedata.com{endpoint}"
|
||||
params = {"symbol": symbol, "exchange": exchange, "apikey": api_key}
|
||||
for attempt in range(_TD_MAX_ATTEMPTS):
|
||||
try:
|
||||
resp = requests.get(url, params=params, timeout=_TD_TIMEOUT)
|
||||
data = resp.json()
|
||||
if data.get("status") == "error":
|
||||
code = data.get("code", "")
|
||||
msg = (data.get("message") or "")[:120]
|
||||
if code == 429 or "API credits" in msg or "minute limit" in msg:
|
||||
logger.warning("TwelveData rate limit on %s %s/%s: %s", endpoint, symbol, exchange, msg)
|
||||
else:
|
||||
logger.debug("TwelveData %s error %s/%s: %s", endpoint, symbol, exchange, msg)
|
||||
return None
|
||||
return data
|
||||
except Exception as e:
|
||||
if attempt + 1 < _TD_MAX_ATTEMPTS:
|
||||
time.sleep(_TD_BACKOFF_SEC)
|
||||
continue
|
||||
logger.warning("TwelveData %s request failed %s/%s: %s", endpoint, symbol, exchange, e)
|
||||
return None
|
||||
|
||||
|
||||
def fetch_twelvedata_fundamental(tencent_code: str, is_hk: bool) -> Dict[str, Any]:
|
||||
"""Fetch PE/PB/PS/PEG/ROE/margin/market_cap/52w from Twelve Data /statistics."""
|
||||
symbol, exchange = _td_symbol_and_exchange(tencent_code, is_hk)
|
||||
data = _td_request("/statistics", symbol, exchange)
|
||||
if not data or "statistics" not in data:
|
||||
return {}
|
||||
|
||||
stats = data["statistics"]
|
||||
result: Dict[str, Any] = {"source": "twelvedata"}
|
||||
|
||||
vm = stats.get("valuations_metrics") or {}
|
||||
result["market_cap"] = _float_clean(vm.get("market_capitalization"))
|
||||
result["pe_ratio"] = _float_clean(vm.get("trailing_pe"))
|
||||
result["forward_pe"] = _float_clean(vm.get("forward_pe"))
|
||||
result["pb_ratio"] = _float_clean(vm.get("price_to_book_mrq"))
|
||||
result["ps_ratio"] = _float_clean(vm.get("price_to_sales_ttm"))
|
||||
result["peg"] = _float_clean(vm.get("peg_ratio"))
|
||||
result["enterprise_value"] = _float_clean(vm.get("enterprise_value"))
|
||||
|
||||
fin = stats.get("financials") or {}
|
||||
result["profit_margin"] = _float_clean(fin.get("profit_margin"))
|
||||
result["gross_margin"] = _float_clean(fin.get("gross_margin"))
|
||||
result["operating_margin"] = _float_clean(fin.get("operating_margin"))
|
||||
result["roe"] = _float_clean(fin.get("return_on_equity_ttm"))
|
||||
result["roa"] = _float_clean(fin.get("return_on_assets_ttm"))
|
||||
|
||||
ss = stats.get("stock_statistics") or {}
|
||||
result["total_shares"] = _float_clean(ss.get("shares_outstanding"))
|
||||
result["float_shares"] = _float_clean(ss.get("float_shares"))
|
||||
|
||||
sp = stats.get("stock_price_summary") or {}
|
||||
result["52w_high"] = _float_clean(sp.get("fifty_two_week_high"))
|
||||
result["52w_low"] = _float_clean(sp.get("fifty_two_week_low"))
|
||||
result["beta"] = _float_clean(sp.get("beta"))
|
||||
|
||||
div = stats.get("dividends_and_splits") or {}
|
||||
result["dividend_yield"] = _float_clean(div.get("trailing_annual_dividend_yield"))
|
||||
result["dividend_rate"] = _float_clean(div.get("trailing_annual_dividend_rate"))
|
||||
|
||||
non_null = sum(1 for v in result.values() if v is not None and v != "twelvedata")
|
||||
logger.debug("TwelveData /statistics %s/%s: %d non-null fields", symbol, exchange, non_null)
|
||||
return result
|
||||
|
||||
|
||||
def fetch_twelvedata_profile(tencent_code: str, is_hk: bool) -> Dict[str, Any]:
|
||||
"""Fetch company info from Twelve Data /profile."""
|
||||
symbol, exchange = _td_symbol_and_exchange(tencent_code, is_hk)
|
||||
data = _td_request("/profile", symbol, exchange)
|
||||
if not data or not data.get("name"):
|
||||
return {}
|
||||
|
||||
out: Dict[str, Any] = {"source": "twelvedata"}
|
||||
for src, dst in (
|
||||
("name", "name"),
|
||||
("industry", "industry"),
|
||||
("sector", "sector"),
|
||||
("website", "website"),
|
||||
("description", "description"),
|
||||
("employees", "employees"),
|
||||
("name", "full_name"),
|
||||
):
|
||||
v = data.get(src)
|
||||
if v is not None and str(v).strip():
|
||||
out[dst] = str(v).strip() if isinstance(v, str) else v
|
||||
|
||||
country = data.get("country")
|
||||
if country:
|
||||
out["country"] = country
|
||||
|
||||
logger.debug("TwelveData /profile %s/%s: name=%s industry=%s", symbol, exchange, out.get("name"), out.get("industry"))
|
||||
return out
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# AkShare fundamentals (Eastmoney — fragile overseas, used as fallback)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _eastmoney_a_em_symbol(tencent_code: str) -> str:
|
||||
c = ak_a_code_from_tencent(tencent_code)
|
||||
c = (c or "").zfill(6)
|
||||
if c.startswith("6"):
|
||||
return "SH" + c
|
||||
return "SZ" + c
|
||||
|
||||
|
||||
def _individual_info_map(symbol_6: str) -> Dict[str, Any]:
|
||||
out: Dict[str, Any] = {}
|
||||
try:
|
||||
import akshare as ak # type: ignore
|
||||
df = ak.stock_individual_info_em(symbol=symbol_6)
|
||||
except Exception as e:
|
||||
logger.debug("stock_individual_info_em failed %s: %s", symbol_6, e)
|
||||
return out
|
||||
if df is None or getattr(df, "empty", True) or len(df.columns) < 2:
|
||||
return out
|
||||
kcol, vcol = df.columns[0], df.columns[1]
|
||||
for _, row in df.iterrows():
|
||||
try:
|
||||
k = str(row[kcol]).strip()
|
||||
if k:
|
||||
out[k] = row[vcol]
|
||||
except Exception:
|
||||
continue
|
||||
return out
|
||||
|
||||
|
||||
def fetch_cn_fundamental_akshare(tencent_code: str) -> Dict[str, Any]:
|
||||
"""PE/PB/PS, market cap, ROE proxy, EPS for A-share (best-effort)."""
|
||||
sym6 = ak_a_code_from_tencent(tencent_code)
|
||||
if not sym6:
|
||||
return {}
|
||||
result: Dict[str, Any] = {"source": "akshare_em"}
|
||||
info = _individual_info_map(sym6)
|
||||
if info:
|
||||
result["market_cap"] = _float_clean(info.get("总市值"))
|
||||
result["float_market_cap"] = _float_clean(info.get("流通市值"))
|
||||
ind = info.get("行业")
|
||||
if ind is not None and str(ind).strip():
|
||||
result["industry"] = str(ind).strip()
|
||||
result["total_shares"] = _float_clean(info.get("总股本"))
|
||||
result["float_shares"] = _float_clean(info.get("流通股"))
|
||||
|
||||
em_sym = _eastmoney_a_em_symbol(tencent_code)
|
||||
try:
|
||||
import akshare as ak # type: ignore
|
||||
vdf = ak.stock_zh_valuation_comparison_em(symbol=em_sym)
|
||||
except Exception as e:
|
||||
logger.debug("stock_zh_valuation_comparison_em failed %s: %s", em_sym, e)
|
||||
vdf = None
|
||||
|
||||
if vdf is not None and not vdf.empty and "代码" in vdf.columns:
|
||||
hit = vdf[vdf["代码"].astype(str).str.replace(".0", "", regex=False).str.zfill(6) == sym6.zfill(6)]
|
||||
if not hit.empty:
|
||||
r = hit.iloc[0]
|
||||
pe = _float_clean(r.get("市盈率-TTM"))
|
||||
if pe is not None:
|
||||
result["pe_ratio"] = pe
|
||||
pb = _float_clean(r.get("市净率-MRQ"))
|
||||
if pb is not None:
|
||||
result["pb_ratio"] = pb
|
||||
ps = _float_clean(r.get("市销率-TTM"))
|
||||
if ps is not None:
|
||||
result["ps_ratio"] = ps
|
||||
peg = _float_clean(r.get("PEG"))
|
||||
if peg is not None:
|
||||
result["peg"] = peg
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def fetch_hk_fundamental_akshare(tencent_code: str) -> Dict[str, Any]:
|
||||
hk5 = ak_hk_code_from_tencent(tencent_code)
|
||||
if not hk5:
|
||||
return {}
|
||||
result: Dict[str, Any] = {"source": "akshare_em"}
|
||||
try:
|
||||
import akshare as ak # type: ignore
|
||||
df = ak.stock_hk_financial_indicator_em(symbol=hk5)
|
||||
except Exception as e:
|
||||
logger.debug("stock_hk_financial_indicator_em failed %s: %s", hk5, e)
|
||||
return result
|
||||
if df is None or df.empty:
|
||||
return result
|
||||
r = df.iloc[0]
|
||||
result["pe_ratio"] = _float_clean(r.get("市盈率"))
|
||||
result["pb_ratio"] = _float_clean(r.get("市净率"))
|
||||
result["eps"] = _float_clean(r.get("基本每股收益(元)"))
|
||||
result["roe"] = _float_clean(r.get("股东权益回报率(%)"))
|
||||
result["profit_margin"] = _float_clean(r.get("销售净利率(%)"))
|
||||
mcap = _float_clean(r.get("总市值(港元)")) or _float_clean(r.get("港股市值(港元)"))
|
||||
if mcap is not None:
|
||||
result["market_cap"] = mcap
|
||||
result["dividend_yield"] = _float_clean(r.get("股息率TTM(%)"))
|
||||
return result
|
||||
|
||||
|
||||
def fetch_cn_company_extras(tencent_code: str) -> Dict[str, Any]:
|
||||
sym6 = ak_a_code_from_tencent(tencent_code)
|
||||
if not sym6:
|
||||
return {}
|
||||
info = _individual_info_map(sym6)
|
||||
out: Dict[str, Any] = {}
|
||||
if info.get("行业"):
|
||||
out["industry"] = str(info["行业"]).strip()
|
||||
if info.get("上市时间"):
|
||||
out["ipo_date"] = str(info["上市时间"]).strip()
|
||||
return out
|
||||
|
||||
|
||||
def fetch_hk_company_extras(tencent_code: str) -> Dict[str, Any]:
|
||||
hk5 = ak_hk_code_from_tencent(tencent_code)
|
||||
if not hk5:
|
||||
return {}
|
||||
out: Dict[str, Any] = {}
|
||||
try:
|
||||
import akshare as ak # type: ignore
|
||||
df = ak.stock_hk_company_profile_em(symbol=hk5)
|
||||
except Exception as e:
|
||||
logger.debug("stock_hk_company_profile_em failed %s: %s", hk5, e)
|
||||
return out
|
||||
if df is None or df.empty:
|
||||
return out
|
||||
r = df.iloc[0]
|
||||
for key, col in (
|
||||
("industry", "所属行业"),
|
||||
("ipo_date", "公司成立日期"),
|
||||
("website", "公司网址"),
|
||||
("full_name", "公司名称"),
|
||||
):
|
||||
v = r.get(col)
|
||||
if v is not None and str(v).strip():
|
||||
out[key] = str(v).strip()
|
||||
return out
|
||||
@@ -0,0 +1,99 @@
|
||||
"""
|
||||
中国A股数据源 — 多层 fallback
|
||||
|
||||
有 TWELVE_DATA_API_KEY:
|
||||
所有周期 → Twelve Data（主） → 腾讯日/周线 → yfinance → AkShare
|
||||
|
||||
无 API Key:
|
||||
分钟/小时 → yfinance → AkShare
|
||||
日/周线 → 腾讯 fqkline → yfinance → AkShare
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Dict, List, Any, Optional
|
||||
|
||||
from app.data_sources.base import BaseDataSource
|
||||
from app.data_sources.tencent import normalize_cn_code, fetch_quote, parse_quote_to_ticker, fetch_kline, tencent_kline_rows_to_dicts
|
||||
from app.data_sources.asia_stock_kline import (
|
||||
normalize_chart_timeframe,
|
||||
fetch_twelvedata_klines,
|
||||
fetch_yfinance_klines,
|
||||
fetch_akshare_minute_klines,
|
||||
fetch_akshare_weekly_klines,
|
||||
)
|
||||
from app.utils.logger import get_logger
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
|
||||
class CNStockDataSource(BaseDataSource):
|
||||
"""A股数据源（TwelveData + Tencent + yfinance + AkShare）"""
|
||||
|
||||
name = "CNStock/multi-source"
|
||||
|
||||
def get_ticker(self, symbol: str) -> Dict[str, Any]:
|
||||
code = normalize_cn_code(symbol)
|
||||
parts = fetch_quote(code)
|
||||
if not parts:
|
||||
return {"last": 0, "symbol": code}
|
||||
t = parse_quote_to_ticker(parts)
|
||||
return {
|
||||
"last": t.get("last", 0),
|
||||
"change": t.get("change", 0),
|
||||
"changePercent": t.get("changePercent", 0),
|
||||
"high": t.get("high", 0),
|
||||
"low": t.get("low", 0),
|
||||
"open": t.get("open", 0),
|
||||
"previousClose": t.get("previousClose", 0),
|
||||
"name": t.get("name", ""),
|
||||
"symbol": code,
|
||||
}
|
||||
|
||||
def get_kline(
|
||||
self,
|
||||
symbol: str,
|
||||
timeframe: str,
|
||||
limit: int,
|
||||
before_time: Optional[int] = None,
|
||||
) -> List[Dict[str, Any]]:
|
||||
code = normalize_cn_code(symbol)
|
||||
tf = normalize_chart_timeframe(timeframe)
|
||||
lim = max(int(limit or 300), 1)
|
||||
|
||||
# Tier 1: Twelve Data (paid, most reliable)
|
||||
rows = fetch_twelvedata_klines(
|
||||
is_hk=False, tencent_code=code, timeframe=tf, limit=lim, before_time=before_time
|
||||
)
|
||||
if rows:
|
||||
return self.filter_and_limit(rows, limit=lim, before_time=before_time)
|
||||
|
||||
# Tier 2: Tencent for daily/weekly (fast, free)
|
||||
if tf in ("1D", "1W"):
|
||||
tf_map = {"1D": "day", "1W": "week"}
|
||||
period = tf_map.get(tf, "day")
|
||||
raw_rows = fetch_kline(code, period=period, count=lim, adj="qfq")
|
||||
out = tencent_kline_rows_to_dicts(raw_rows)
|
||||
if out:
|
||||
return self.filter_and_limit(out, limit=lim, before_time=before_time)
|
||||
|
||||
# Tier 3: yfinance (works when Yahoo not rate-limited)
|
||||
rows = fetch_yfinance_klines(
|
||||
is_hk=False, tencent_code=code, timeframe=tf, limit=lim, before_time=before_time
|
||||
)
|
||||
if rows:
|
||||
return self.filter_and_limit(rows, limit=lim, before_time=before_time)
|
||||
|
||||
# Tier 4: AkShare (fragile overseas, last resort)
|
||||
if tf in ("1m", "5m", "15m", "30m", "1H", "4H"):
|
||||
rows = fetch_akshare_minute_klines(
|
||||
is_hk=False, tencent_code=code, timeframe=tf, limit=lim, before_time=before_time
|
||||
)
|
||||
elif tf == "1W":
|
||||
rows = fetch_akshare_weekly_klines(
|
||||
is_hk=False, tencent_code=code, limit=lim, before_time=before_time
|
||||
)
|
||||
else:
|
||||
rows = []
|
||||
|
||||
return self.filter_and_limit(rows, limit=lim, before_time=before_time)
|
||||
@@ -52,6 +52,12 @@ class DataSourceFactory:
|
||||
if market == 'Crypto':
|
||||
from app.data_sources.crypto import CryptoDataSource
|
||||
return CryptoDataSource()
|
||||
elif market == 'CNStock':
|
||||
from app.data_sources.cn_stock import CNStockDataSource
|
||||
return CNStockDataSource()
|
||||
elif market == 'HKStock':
|
||||
from app.data_sources.hk_stock import HKStockDataSource
|
||||
return HKStockDataSource()
|
||||
elif market == 'USStock':
|
||||
from app.data_sources.us_stock import USStockDataSource
|
||||
return USStockDataSource()
|
||||
|
||||
@@ -0,0 +1,99 @@
|
||||
"""
|
||||
港股/H股数据源 — 多层 fallback
|
||||
|
||||
有 TWELVE_DATA_API_KEY:
|
||||
所有周期 → Twelve Data（主） → 腾讯日/周线 → yfinance → AkShare
|
||||
|
||||
无 API Key:
|
||||
分钟/小时 → yfinance → AkShare
|
||||
日/周线 → 腾讯 fqkline → yfinance → AkShare
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Dict, List, Any, Optional
|
||||
|
||||
from app.data_sources.base import BaseDataSource
|
||||
from app.data_sources.tencent import normalize_hk_code, fetch_quote, parse_quote_to_ticker, fetch_kline, tencent_kline_rows_to_dicts
|
||||
from app.data_sources.asia_stock_kline import (
|
||||
normalize_chart_timeframe,
|
||||
fetch_twelvedata_klines,
|
||||
fetch_yfinance_klines,
|
||||
fetch_akshare_minute_klines,
|
||||
fetch_akshare_weekly_klines,
|
||||
)
|
||||
from app.utils.logger import get_logger
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
|
||||
class HKStockDataSource(BaseDataSource):
|
||||
"""港股/H股数据源（TwelveData + Tencent + yfinance + AkShare）"""
|
||||
|
||||
name = "HKStock/multi-source"
|
||||
|
||||
def get_ticker(self, symbol: str) -> Dict[str, Any]:
|
||||
code = normalize_hk_code(symbol)
|
||||
parts = fetch_quote(code)
|
||||
if not parts:
|
||||
return {"last": 0, "symbol": code}
|
||||
t = parse_quote_to_ticker(parts)
|
||||
return {
|
||||
"last": t.get("last", 0),
|
||||
"change": t.get("change", 0),
|
||||
"changePercent": t.get("changePercent", 0),
|
||||
"high": t.get("high", 0),
|
||||
"low": t.get("low", 0),
|
||||
"open": t.get("open", 0),
|
||||
"previousClose": t.get("previousClose", 0),
|
||||
"name": t.get("name", ""),
|
||||
"symbol": code,
|
||||
}
|
||||
|
||||
def get_kline(
|
||||
self,
|
||||
symbol: str,
|
||||
timeframe: str,
|
||||
limit: int,
|
||||
before_time: Optional[int] = None,
|
||||
) -> List[Dict[str, Any]]:
|
||||
code = normalize_hk_code(symbol)
|
||||
tf = normalize_chart_timeframe(timeframe)
|
||||
lim = max(int(limit or 300), 1)
|
||||
|
||||
# Tier 1: Twelve Data (paid, most reliable)
|
||||
rows = fetch_twelvedata_klines(
|
||||
is_hk=True, tencent_code=code, timeframe=tf, limit=lim, before_time=before_time
|
||||
)
|
||||
if rows:
|
||||
return self.filter_and_limit(rows, limit=lim, before_time=before_time)
|
||||
|
||||
# Tier 2: Tencent for daily/weekly (fast, free)
|
||||
if tf in ("1D", "1W"):
|
||||
tf_map = {"1D": "day", "1W": "week"}
|
||||
period = tf_map.get(tf, "day")
|
||||
raw_rows = fetch_kline(code, period=period, count=lim, adj="qfq")
|
||||
out = tencent_kline_rows_to_dicts(raw_rows)
|
||||
if out:
|
||||
return self.filter_and_limit(out, limit=lim, before_time=before_time)
|
||||
|
||||
# Tier 3: yfinance (works when Yahoo not rate-limited)
|
||||
rows = fetch_yfinance_klines(
|
||||
is_hk=True, tencent_code=code, timeframe=tf, limit=lim, before_time=before_time
|
||||
)
|
||||
if rows:
|
||||
return self.filter_and_limit(rows, limit=lim, before_time=before_time)
|
||||
|
||||
# Tier 4: AkShare (fragile overseas, last resort)
|
||||
if tf in ("1m", "5m", "15m", "30m", "1H", "4H"):
|
||||
rows = fetch_akshare_minute_klines(
|
||||
is_hk=True, tencent_code=code, timeframe=tf, limit=lim, before_time=before_time
|
||||
)
|
||||
elif tf == "1W":
|
||||
rows = fetch_akshare_weekly_klines(
|
||||
is_hk=True, tencent_code=code, limit=lim, before_time=before_time
|
||||
)
|
||||
else:
|
||||
rows = []
|
||||
|
||||
return self.filter_and_limit(rows, limit=lim, before_time=before_time)
|
||||
@@ -49,19 +49,17 @@ class PolymarketDataSource:
|
||||
if cached:
|
||||
return cached
|
||||
|
||||
# 从真实API获取 - 获取多个分类的数据以确保多样性
|
||||
# 从真实API获取
|
||||
all_markets = []
|
||||
|
||||
if category and category != "all":
|
||||
# 如果指定了类别，只获取该类别的数据
|
||||
# 获取所有事件，然后按类别过滤
|
||||
markets = self._fetch_markets_from_api(category, limit * 2)
|
||||
all_markets.extend(markets)
|
||||
else:
|
||||
# 如果没有指定类别或指定了"all"，获取多个分类的数据
|
||||
categories_to_fetch = ["crypto", "politics", "economics", "sports"]
|
||||
for cat in categories_to_fetch:
|
||||
markets = self._fetch_markets_from_api(cat, limit // len(categories_to_fetch) + 10)
|
||||
all_markets.extend(markets)
|
||||
# 获取所有事件（不指定类别，避免重复请求）
|
||||
markets = self._fetch_from_gamma_api(category=None, limit=100)
|
||||
all_markets.extend(markets)
|
||||
|
||||
# 去重（按market_id）
|
||||
seen = set()
|
||||
|
||||
@@ -0,0 +1,238 @@
|
||||
"""
|
||||
Tencent market data helpers (no API key).
|
||||
|
||||
Provides:
|
||||
- Quote: https://qt.gtimg.cn/q=sh600519 / sz000001 / hk00700
|
||||
- Kline: https://web.ifzq.gtimg.cn/appstock/app/fqkline/get?param=CODE,PERIOD,,,COUNT,ADJ
|
||||
|
||||
This is used as a stable alternative when Yahoo/yfinance gets rate-limited.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
|
||||
import requests
|
||||
|
||||
from app.data_sources.rate_limiter import get_request_headers, retry_with_backoff, get_tencent_limiter
|
||||
from app.utils.logger import get_logger
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
|
||||
def normalize_cn_code(symbol: str) -> str:
|
||||
"""
|
||||
Normalize A-share symbol to Tencent code: sh600519 / sz000001.
|
||||
Accepts:
|
||||
- 600519 / 600519.SH / 600519.SS
|
||||
- 000001 / 000001.SZ
|
||||
"""
|
||||
s = (symbol or "").strip().upper()
|
||||
if not s:
|
||||
return s
|
||||
if s.endswith(".SH"):
|
||||
s = s[:-3]
|
||||
return f"SH{s}"
|
||||
if s.endswith(".SS"):
|
||||
s = s[:-3]
|
||||
return f"SH{s}"
|
||||
if s.endswith(".SZ"):
|
||||
s = s[:-3]
|
||||
return f"SZ{s}"
|
||||
|
||||
if s.isdigit() and len(s) == 6:
|
||||
return ("SH" + s) if s.startswith("6") else ("SZ" + s)
|
||||
|
||||
return s
|
||||
|
||||
|
||||
def normalize_hk_code(symbol: str) -> str:
|
||||
"""
|
||||
Normalize HK stock symbol to Tencent code: hk00700 (5 digits).
|
||||
Accepts:
|
||||
- 700 / 0700 / 00700.HK / 0700.HK
|
||||
"""
|
||||
s = (symbol or "").strip().upper()
|
||||
if not s:
|
||||
return s
|
||||
if s.endswith(".HK"):
|
||||
s = s[:-3]
|
||||
if s.isdigit():
|
||||
return "HK" + s.zfill(5)
|
||||
# If user already passed HKxxxxx
|
||||
if s.startswith("HK") and s[2:].isdigit():
|
||||
return "HK" + s[2:].zfill(5)
|
||||
return s
|
||||
|
||||
|
||||
def _lower_code(code: str) -> str:
|
||||
return (code or "").strip().lower()
|
||||
|
||||
|
||||
@retry_with_backoff(max_attempts=3, base_delay=1.2, max_delay=8.0, exceptions=(Exception,))
|
||||
def fetch_quote(code: str, timeout: int = 8) -> Optional[List[str]]:
|
||||
"""
|
||||
Returns the raw '~' split array from qt.gtimg.cn, or None.
|
||||
"""
|
||||
c = _lower_code(code)
|
||||
if not c:
|
||||
return None
|
||||
|
||||
limiter = get_tencent_limiter()
|
||||
limiter.wait()
|
||||
url = f"https://qt.gtimg.cn/q={c}"
|
||||
resp = requests.get(url, headers=get_request_headers(referer="https://qt.gtimg.cn/"), timeout=timeout)
|
||||
# Tencent quote is often GBK encoded
|
||||
try:
|
||||
resp.encoding = "gbk"
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
text = (resp.text or "").strip()
|
||||
if not text or "~" not in text:
|
||||
return None
|
||||
|
||||
# Format: v_sh600519="1~NAME~CODE~LAST~PREV~OPEN~..."
|
||||
try:
|
||||
start = text.index('="') + 2
|
||||
end = text.rindex('"')
|
||||
payload = text[start:end]
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
parts = payload.split("~")
|
||||
return parts if len(parts) > 5 else None
|
||||
|
||||
|
||||
def parse_quote_to_ticker(parts: List[str]) -> Dict[str, Any]:
|
||||
"""
|
||||
Best-effort conversion to a unified ticker dict.
|
||||
"""
|
||||
def _f(i: int, default: float = 0.0) -> float:
|
||||
try:
|
||||
v = parts[i]
|
||||
if v is None or v == "":
|
||||
return default
|
||||
return float(v)
|
||||
except Exception:
|
||||
return default
|
||||
|
||||
name = (parts[1] or "").strip() if len(parts) > 1 else ""
|
||||
symbol = (parts[2] or "").strip() if len(parts) > 2 else ""
|
||||
last_ = _f(3, 0.0)
|
||||
prev = _f(4, 0.0)
|
||||
open_ = _f(5, 0.0)
|
||||
|
||||
change = round(last_ - prev, 4) if prev else 0.0
|
||||
change_pct = round(change / prev * 100, 2) if prev else 0.0
|
||||
|
||||
# Indices are not fully consistent across markets; keep conservative.
|
||||
high = _f(33, last_) if len(parts) > 33 else last_
|
||||
low = _f(34, last_) if len(parts) > 34 else last_
|
||||
|
||||
return {
|
||||
"symbol": symbol,
|
||||
"name": name,
|
||||
"last": last_,
|
||||
"change": change,
|
||||
"changePercent": change_pct,
|
||||
"high": high,
|
||||
"low": low,
|
||||
"open": open_ or last_,
|
||||
"previousClose": prev,
|
||||
"raw": parts,
|
||||
}
|
||||
|
||||
|
||||
def parse_tencent_kline_time(ds: str) -> Optional[int]:
|
||||
"""Parse Tencent fqkline first column to Unix seconds (local parse, matches prior chart behavior)."""
|
||||
raw = str(ds or "").strip()
|
||||
if not raw:
|
||||
return None
|
||||
for fmt in ("%Y-%m-%d %H:%M:%S", "%Y-%m-%d %H:%M", "%Y-%m-%d", "%Y/%m/%d"):
|
||||
try:
|
||||
return int(datetime.strptime(raw, fmt).timestamp())
|
||||
except ValueError:
|
||||
continue
|
||||
try:
|
||||
ts = int(float(raw))
|
||||
if ts > 10**12:
|
||||
ts = int(ts / 1000)
|
||||
return ts
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def tencent_kline_rows_to_dicts(rows: List[Any]) -> List[Dict[str, Any]]:
|
||||
"""Convert raw fqkline rows to chart dicts; ignores corporate-action tail objects on HK rows."""
|
||||
out: List[Dict[str, Any]] = []
|
||||
for r in rows:
|
||||
if not isinstance(r, (list, tuple)) or len(r) < 6:
|
||||
continue
|
||||
ts = parse_tencent_kline_time(r[0])
|
||||
if ts is None:
|
||||
continue
|
||||
try:
|
||||
o, c, h, low, vol = float(r[1]), float(r[2]), float(r[3]), float(r[4]), float(r[5])
|
||||
except (TypeError, ValueError):
|
||||
continue
|
||||
out.append(
|
||||
{
|
||||
"time": ts,
|
||||
"open": round(o, 4),
|
||||
"high": round(h, 4),
|
||||
"low": round(low, 4),
|
||||
"close": round(c, 4),
|
||||
"volume": round(vol, 2),
|
||||
}
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
@retry_with_backoff(max_attempts=3, base_delay=1.2, max_delay=8.0, exceptions=(Exception,))
|
||||
def fetch_kline(code: str, period: str, count: int = 300, adj: str = "qfq", timeout: int = 10) -> List[List[str]]:
|
||||
"""
|
||||
Fetch kline arrays from Tencent.
|
||||
|
||||
period examples:
|
||||
- day, week, month (supported by Tencent fqkline)
|
||||
|
||||
Note: Minute periods (m1/m5/…) return **bad params** on this endpoint; use AkShare in ``asia_stock_kline``.
|
||||
"""
|
||||
c = _lower_code(code)
|
||||
if not c:
|
||||
return []
|
||||
|
||||
limiter = get_tencent_limiter()
|
||||
limiter.wait()
|
||||
|
||||
url = "https://web.ifzq.gtimg.cn/appstock/app/fqkline/get"
|
||||
params = {"param": f"{c},{period},,,{int(count)},{adj}"}
|
||||
resp = requests.get(url, headers=get_request_headers(referer="https://gu.qq.com/"), params=params, timeout=timeout)
|
||||
data = resp.json() if resp.text else {}
|
||||
if not isinstance(data, dict) or int(data.get("code", 0)) != 0:
|
||||
return []
|
||||
root = (data.get("data") or {}).get(c)
|
||||
if not isinstance(root, dict):
|
||||
return []
|
||||
|
||||
# Data key variants:
|
||||
# - A-share: qfqday / qfqweek / qfqm1 ...
|
||||
# - HK: day / week / m1 ...
|
||||
candidates = []
|
||||
if adj:
|
||||
candidates.append(f"{adj}{period}")
|
||||
candidates.append(period)
|
||||
|
||||
for key in candidates:
|
||||
arr = root.get(key)
|
||||
if isinstance(arr, list) and arr:
|
||||
return arr
|
||||
|
||||
# Fallback: search any key that endswith period and is a list
|
||||
for k, v in root.items():
|
||||
if isinstance(v, list) and v and str(k).lower().endswith(str(period).lower()):
|
||||
return v
|
||||
return []
|
||||
|
||||
@@ -88,7 +88,11 @@ class USStockDataSource(BaseDataSource):
|
||||
'previousClose': quote.get('pc', 0) # 昨收价
|
||||
}
|
||||
except Exception as e:
|
||||
logger.warning(f"Finnhub quote failed for {symbol}: {e}")
|
||||
msg = str(e).lower()
|
||||
if "403" in str(e) or "don't have access" in msg or "no access" in msg:
|
||||
logger.debug(f"Finnhub quote skipped (no access): {symbol}: {e}")
|
||||
else:
|
||||
logger.warning(f"Finnhub quote failed for {symbol}: {e}")
|
||||
|
||||
# 降级使用 yfinance
|
||||
try:
|
||||
@@ -260,7 +264,12 @@ class USStockDataSource(BaseDataSource):
|
||||
))
|
||||
# logger.info(f"Finnhub 返回 {len(klines)} 条数据")
|
||||
except Exception as e:
|
||||
logger.error(f"Finnhub fetch failed: {e}")
|
||||
msg = str(e).lower()
|
||||
# Free tier / plan: 403 "You don't have access to this resource" is common; avoid ERROR spam.
|
||||
if "403" in str(e) or "don't have access" in msg or "no access" in msg:
|
||||
logger.debug(f"Finnhub candles skipped (no access): {symbol}: {e}")
|
||||
else:
|
||||
logger.warning(f"Finnhub fetch failed: {e}")
|
||||
|
||||
return klines
|
||||
|
||||
|
||||
Reference in New Issue
Block a user