mirror of
https://github.com/shy3130/tick-stock-panel.git
synced 2026-11-02 10:26:13 +00:00
feat: 数据源插件化架构 + 首个插件
将可选数据源改为插件化架构: 插件代码放 backend/app/plugins/<name>/, 用户在设置页点击安装依赖, 主仓库不背运行时依赖(nodejs 等)。 架构: - loader.py 新增 _load_builtin_plugins() 扫描 plugins/ 目录, 通过 plugin.yaml 清单动态发现并注册插件(委托自检 + 优雅降级) - runtime 字段支持 node/python, 安装/卸载分别用 npm/pip - 现有 tickflow + YAML 自定义源逻辑 100% 保留, 插件是叠加层 插件 (plugins//): - 原始实现来自 @forrany 的 PR #57, 迁移到插件化架构, 署名保留 - Node 桥接 (bridge.py/mjs) + Python provider, 覆盖日K/除权/分钟/实时 - 设置页主列表直接显示: 未装灰显+安装按钮, 装完可切换/卸载 配套: - instrument_sync: 通用增强, 任何 provider 都能提供标的维表 - install/uninstall: uv 优先回退 pip, 容错坏 uv.toml + 国内镜像 - 10 例单测全过, tsc 零错误
This commit is contained in:
@@ -413,10 +413,11 @@ def get_preferences() -> dict:
|
||||
|
||||
@router.get("/data-sources")
|
||||
def list_data_sources() -> dict:
|
||||
"""列出已加载的自定义数据源。"""
|
||||
"""列出已加载的数据源 (内置 / 插件 / 用户自定义)。"""
|
||||
from app.data_providers import custom as custom_sources
|
||||
return {
|
||||
"builtin": [{"name": "tickflow", "display_name": "TickFlow", "datasets": ["daily", "adj_factor", "realtime"]}],
|
||||
"builtin": [{"name": "tickflow", "display_name": "TickFlow", "datasets": ["daily", "adj_factor", "realtime", "minute"]}],
|
||||
"plugins": custom_sources.list_plugins(),
|
||||
"custom": custom_sources.list_sources(),
|
||||
"errors": custom_sources.errors(),
|
||||
"config_dir": str(custom_sources.data_sources_dir()),
|
||||
@@ -431,6 +432,52 @@ def reload_data_sources() -> dict:
|
||||
return list_data_sources()
|
||||
|
||||
|
||||
@router.post("/plugins/{name}/install")
|
||||
def install_plugin(name: str) -> dict:
|
||||
"""安装指定插件的依赖 (npm install / pip install), 完成后重新扫描。
|
||||
|
||||
根据 plugin.yaml 的 runtime 字段决定安装方式。安装可能耗时较长 (网络下载),
|
||||
客户端需设较长超时。
|
||||
"""
|
||||
from app.data_providers import custom as custom_sources
|
||||
if not custom_sources.is_builtin(name):
|
||||
raise HTTPException(status_code=404, detail=f"插件 '{name}' 不存在")
|
||||
ok, message = custom_sources.install_plugin(name)
|
||||
# 无论成功失败都重新扫描, 刷新插件状态 (安装可能部分成功)
|
||||
custom_sources.load_all()
|
||||
result = list_data_sources()
|
||||
result["install_ok"] = ok
|
||||
result["install_message"] = message
|
||||
return result
|
||||
|
||||
|
||||
@router.delete("/plugins/{name}/install")
|
||||
def uninstall_plugin(name: str) -> dict:
|
||||
"""卸载指定插件的依赖 (删除 node_modules / pip uninstall), 完成后重新扫描。
|
||||
|
||||
如果该插件当前正被使用, 自动回退到 tickflow。
|
||||
"""
|
||||
from app.data_providers import custom as custom_sources
|
||||
from app.services import preferences
|
||||
if not custom_sources.is_builtin(name):
|
||||
raise HTTPException(status_code=404, detail=f"插件 '{name}' 不存在")
|
||||
ok, message = custom_sources.uninstall_plugin(name)
|
||||
# 卸载后若该插件正被使用, 回退 tickflow
|
||||
for getter, key, default in [
|
||||
(preferences.get_daily_data_provider, "daily_data_provider", "tickflow"),
|
||||
(preferences.get_minute_data_provider, "minute_data_provider", "tickflow"),
|
||||
(preferences.get_realtime_data_provider, "realtime_data_provider", "tickflow"),
|
||||
(preferences.get_financial_provider, "financial_data_provider", "tickflow"),
|
||||
]:
|
||||
if getter() == name:
|
||||
preferences.save({key: default})
|
||||
custom_sources.load_all()
|
||||
result = list_data_sources()
|
||||
result["uninstall_ok"] = ok
|
||||
result["uninstall_message"] = message
|
||||
return result
|
||||
|
||||
|
||||
@router.get("/data-sources/{name}")
|
||||
def get_data_source(name: str) -> dict:
|
||||
"""读取一个自定义数据源的完整配置(用于前端编辑回填)。"""
|
||||
|
||||
@@ -5,12 +5,16 @@ from app.data_providers.custom.loader import (
|
||||
errors,
|
||||
get_config_dict,
|
||||
get_provider,
|
||||
install_plugin,
|
||||
is_builtin,
|
||||
is_custom_provider,
|
||||
list_plugins,
|
||||
list_sources,
|
||||
load_all,
|
||||
names,
|
||||
provider_has_dataset,
|
||||
save_config,
|
||||
uninstall_plugin,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
@@ -19,10 +23,14 @@ __all__ = [
|
||||
"errors",
|
||||
"get_config_dict",
|
||||
"get_provider",
|
||||
"install_plugin",
|
||||
"is_builtin",
|
||||
"is_custom_provider",
|
||||
"list_plugins",
|
||||
"list_sources",
|
||||
"load_all",
|
||||
"names",
|
||||
"provider_has_dataset",
|
||||
"save_config",
|
||||
"uninstall_plugin",
|
||||
]
|
||||
|
||||
@@ -1,8 +1,11 @@
|
||||
"""Load custom data source definitions from user data files."""
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib
|
||||
import logging
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
import yaml
|
||||
@@ -16,9 +19,18 @@ logger = logging.getLogger(__name__)
|
||||
_PROVIDERS: dict[str, GenericHTTPProvider] = {}
|
||||
_LOAD_ERRORS: list[dict] = []
|
||||
|
||||
# 内置插件状态: {name: {available, status, runtime, ...}} 供设置页独立分类展示。
|
||||
# available=False 的插件不注册进 _PROVIDERS (不可切换), 但记录状态供 UI 显示安装提示。
|
||||
_PLUGIN_STATUS: dict[str, dict] = {}
|
||||
|
||||
_NAME_RE = re.compile(r"^[a-z0-9_]+$")
|
||||
|
||||
|
||||
def plugins_dir() -> Path:
|
||||
"""内置可选插件目录 (app/plugins/, 与现有包结构一致, 开发态/容器态路径统一)。"""
|
||||
return Path(__file__).resolve().parents[2] / "plugins"
|
||||
|
||||
|
||||
def data_sources_dir() -> Path:
|
||||
return settings.data_dir / "data_sources"
|
||||
|
||||
@@ -47,8 +59,12 @@ def load_all(path: Path | None = None) -> None:
|
||||
logger.warning("custom data source load failed %s: %s", file, e)
|
||||
_LOAD_ERRORS.append({"path": str(file), "errors": [str(e)]})
|
||||
|
||||
# 内置可选插件 (plugins/ 目录)。与用户 YAML 源独立, 缺依赖只记状态不报错。
|
||||
_load_builtin_plugins()
|
||||
|
||||
|
||||
def list_sources() -> list[dict]:
|
||||
"""只列出用户自定义 (YAML) 源。内置插件 (builtin=True) 由 list_plugins 独立呈现。"""
|
||||
return [
|
||||
{
|
||||
"name": provider.name,
|
||||
@@ -57,9 +73,164 @@ def list_sources() -> list[dict]:
|
||||
"path": str(provider.config.path) if provider.config.path else None,
|
||||
}
|
||||
for provider in _PROVIDERS.values()
|
||||
if not getattr(provider, "builtin", False)
|
||||
]
|
||||
|
||||
|
||||
def list_plugins() -> list[dict]:
|
||||
"""返回所有内置插件的状态 (含已装/未装), 供设置页独立分类显示。"""
|
||||
return list(_PLUGIN_STATUS.values())
|
||||
|
||||
|
||||
def plugin_manifest(name: str) -> dict | None:
|
||||
"""读取指定插件的 plugin.yaml 清单。"""
|
||||
plugin_dir = plugins_dir() / (name or "")
|
||||
manifest_path = plugin_dir / "plugin.yaml"
|
||||
if not manifest_path.exists():
|
||||
return None
|
||||
return yaml.safe_load(manifest_path.read_text(encoding="utf-8")) or {}
|
||||
|
||||
|
||||
def plugin_dir_of(name: str) -> Path:
|
||||
"""返回插件目录路径。"""
|
||||
return plugins_dir() / (name or "")
|
||||
|
||||
|
||||
def install_plugin(name: str) -> tuple[bool, str]:
|
||||
"""安装指定插件的依赖。根据 runtime 执行 npm install / pip install。
|
||||
|
||||
返回 (是否成功, 消息)。成功后调用方应 reload 重新扫描。
|
||||
依赖未找到 (npm/pip 缺失) 或命令失败时返回 False。
|
||||
"""
|
||||
manifest = plugin_manifest(name)
|
||||
if manifest is None:
|
||||
return False, f"插件 '{name}' 不存在或无 plugin.yaml"
|
||||
runtime = str(manifest.get("runtime", "none")).lower()
|
||||
pdir = plugin_dir_of(name)
|
||||
if not pdir.exists():
|
||||
return False, f"插件目录不存在: {pdir}"
|
||||
|
||||
try:
|
||||
if runtime == "node":
|
||||
npm = shutil.which("npm")
|
||||
if not npm:
|
||||
return False, "未找到 npm, 请先安装 Node.js (>=18)"
|
||||
# 在插件目录执行 npm install
|
||||
result = subprocess.run(
|
||||
[npm, "install", "--omit=dev", "--no-audit", "--no-fund"],
|
||||
cwd=str(pdir),
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=300,
|
||||
)
|
||||
elif runtime == "python":
|
||||
# Python 型插件: 优先用 uv pip install (uv 管理的 venv 无 pip 模块),
|
||||
# 回退 python -m pip。都装进当前后端虚拟环境。
|
||||
# uv 容错: 用户全局 uv.toml 配置错误时 exit 2, 回退 --no-config 重试。
|
||||
# UV_HTTP_TIMEOUT=300: akshare 等含大包(如 mini-racer 14MB), 默认 30s 不够。
|
||||
req = pdir / "requirements.txt"
|
||||
if not req.exists():
|
||||
return False, "Python 型插件需要 requirements.txt"
|
||||
uv_bin = shutil.which("uv")
|
||||
if uv_bin:
|
||||
result = subprocess.run(
|
||||
[uv_bin, "pip", "install", "-r", str(req)],
|
||||
capture_output=True, text=True, timeout=300,
|
||||
env={**__import__("os").environ, "UV_HTTP_TIMEOUT": "300"},
|
||||
)
|
||||
# exit 2 通常是配置文件解析错误, 绕过配置重试
|
||||
# --no-config 会丢镜像, 显式传国内镜像加速 (与用户 uv.toml 意图一致)
|
||||
if result.returncode == 2:
|
||||
result = subprocess.run(
|
||||
[uv_bin, "pip", "install", "--no-config",
|
||||
"--index-url", "https://pypi.tuna.tsinghua.edu.cn/simple",
|
||||
"-r", str(req)],
|
||||
capture_output=True, text=True, timeout=300,
|
||||
env={**__import__("os").environ, "UV_HTTP_TIMEOUT": "300"},
|
||||
)
|
||||
else:
|
||||
import sys
|
||||
result = subprocess.run(
|
||||
[sys.executable, "-m", "pip", "install", "-r", str(req)],
|
||||
capture_output=True, text=True, timeout=300,
|
||||
)
|
||||
else:
|
||||
return False, f"runtime={runtime} 无需安装依赖"
|
||||
except subprocess.TimeoutExpired:
|
||||
return False, "安装超时 (5分钟), 请检查网络后重试"
|
||||
except Exception as e: # noqa: BLE001
|
||||
return False, f"安装失败: {e}"
|
||||
|
||||
if result.returncode != 0:
|
||||
# 取 stderr 的第一个 error: 行(真正的错误原因), 避免把 uv 的长字段列表返回给用户
|
||||
raw = (result.stderr or result.stdout or "").strip()
|
||||
first_err = ""
|
||||
for line in raw.splitlines():
|
||||
if line.strip().startswith(("error", "Error", "Caused by")):
|
||||
first_err = line.strip()
|
||||
break
|
||||
msg = first_err or raw[-200:]
|
||||
return False, f"安装失败 (exit {result.returncode}): {msg}"
|
||||
return True, "安装成功"
|
||||
|
||||
|
||||
def uninstall_plugin(name: str) -> tuple[bool, str]:
|
||||
"""卸载指定插件的依赖。
|
||||
|
||||
node 型: 删除 node_modules 目录 (干净彻底, 下次需要重新 npm install)。
|
||||
python 型: pip uninstall (包名从 requirements.txt 推断)。
|
||||
"""
|
||||
import shutil as _shutil
|
||||
|
||||
manifest = plugin_manifest(name)
|
||||
if manifest is None:
|
||||
return False, f"插件 '{name}' 不存在或无 plugin.yaml"
|
||||
runtime = str(manifest.get("runtime", "none")).lower()
|
||||
pdir = plugin_dir_of(name)
|
||||
if not pdir.exists():
|
||||
return False, f"插件目录不存在: {pdir}"
|
||||
|
||||
if runtime == "node":
|
||||
nm = pdir / "node_modules"
|
||||
if not nm.exists():
|
||||
return True, "node_modules 不存在, 无需卸载"
|
||||
try:
|
||||
_shutil.rmtree(nm)
|
||||
return True, "已删除 node_modules"
|
||||
except Exception as e: # noqa: BLE001
|
||||
return False, f"删除 node_modules 失败: {e}"
|
||||
|
||||
if runtime == "python":
|
||||
req = pdir / "requirements.txt"
|
||||
if not req.exists():
|
||||
return False, "Python 型插件缺少 requirements.txt, 无法自动卸载"
|
||||
# 读 requirements.txt 拿包名, 逐个 pip uninstall -y
|
||||
pkgs = [l.strip().split("==")[0].split(">=")[0].strip()
|
||||
for l in req.read_text().splitlines()
|
||||
if l.strip() and not l.startswith("#")]
|
||||
if not pkgs:
|
||||
return True, "requirements.txt 无有效包名"
|
||||
uv_bin = _shutil.which("uv")
|
||||
cmd = [uv_bin, "pip", "uninstall", *pkgs] if uv_bin else None
|
||||
if cmd is None:
|
||||
import sys
|
||||
cmd = [sys.executable, "-m", "pip", "uninstall", "-y", *pkgs]
|
||||
try:
|
||||
result = subprocess.run(cmd, capture_output=True, text=True, timeout=120)
|
||||
if result.returncode != 0:
|
||||
return False, f"卸载失败: {(result.stderr or '').strip()[-300:]}"
|
||||
return True, f"已卸载 {len(pkgs)} 个包"
|
||||
except Exception as e: # noqa: BLE001
|
||||
return False, f"卸载失败: {e}"
|
||||
|
||||
return False, f"runtime={runtime} 无需卸载"
|
||||
|
||||
|
||||
def is_builtin(name: str) -> bool:
|
||||
"""判断 name 是否为内置插件 (不可被用户编辑/删除)。"""
|
||||
return (name or "").lower() in _PLUGIN_STATUS
|
||||
|
||||
|
||||
def names() -> set[str]:
|
||||
return set(_PROVIDERS)
|
||||
|
||||
@@ -91,7 +262,9 @@ def provider_has_dataset(name: str, dataset: str) -> bool:
|
||||
|
||||
|
||||
def get_config_dict(name: str) -> dict | None:
|
||||
"""读取一个已加载 custom 源的原始配置 dict(用于前端编辑回填)。"""
|
||||
"""读取一个已加载 custom 源的原始配置 dict(用于前端编辑回填)。内置插件不可编辑。"""
|
||||
if is_builtin(name):
|
||||
return None
|
||||
provider = _PROVIDERS.get((name or "").lower())
|
||||
if provider is None:
|
||||
return None
|
||||
@@ -129,6 +302,8 @@ def _config_to_dict(config: CustomSourceConfig) -> dict:
|
||||
|
||||
def save_config(name: str, config: dict) -> Path:
|
||||
"""把一份配置 dict 写成 data/data_sources/{name}.yaml, 返回写入路径。"""
|
||||
if is_builtin(name):
|
||||
raise ValueError(f"'{name}' 是内置插件, 不可编辑")
|
||||
if not _NAME_RE.match(name or ""):
|
||||
raise ValueError(f"invalid data source name: {name!r} (only lowercase a-z 0-9 _ allowed)")
|
||||
base = data_sources_dir()
|
||||
@@ -143,6 +318,8 @@ def save_config(name: str, config: dict) -> Path:
|
||||
|
||||
def delete_config(name: str) -> bool:
|
||||
"""删除 data/data_sources/{name}.yaml。返回是否真的删除了。"""
|
||||
if is_builtin(name):
|
||||
raise ValueError(f"'{name}' 是内置插件, 不可删除")
|
||||
if not _NAME_RE.match(name or ""):
|
||||
raise ValueError(f"invalid data source name: {name!r}")
|
||||
base = data_sources_dir().resolve()
|
||||
@@ -226,3 +403,97 @@ def _sanitize_dataset(ds_cfg: dict) -> dict:
|
||||
out["end_param"] = str(ds_cfg["end_param"])
|
||||
return out
|
||||
|
||||
|
||||
# ================================================================
|
||||
# 内置可选插件 (plugins/ 目录) 的发现与注册
|
||||
# ================================================================
|
||||
|
||||
def _load_builtin_plugins() -> None:
|
||||
"""扫描 plugins/ 目录下每个含 plugin.yaml 的子目录, 动态加载。
|
||||
|
||||
缺依赖时记录 "不可用" 状态, 不抛异常, 不影响主流程。
|
||||
每次调用重建 _PLUGIN_STATUS, 并把可用的插件注册进 _PROVIDERS。
|
||||
"""
|
||||
global _PLUGIN_STATUS
|
||||
_PLUGIN_STATUS = {}
|
||||
pdir = plugins_dir()
|
||||
if not pdir.exists():
|
||||
return
|
||||
for plugin_dir in sorted(pdir.iterdir()):
|
||||
if not plugin_dir.is_dir():
|
||||
continue
|
||||
manifest_path = plugin_dir / "plugin.yaml"
|
||||
if not manifest_path.exists():
|
||||
continue
|
||||
try:
|
||||
manifest = yaml.safe_load(manifest_path.read_text(encoding="utf-8")) or {}
|
||||
_register_one_plugin(manifest)
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.warning("插件 %s 清单解析失败: %s", plugin_dir.name, e)
|
||||
|
||||
|
||||
def _register_one_plugin(manifest: dict) -> None:
|
||||
"""注册单个插件: 委托自检 → 可用则动态 import entry 注册进 _PROVIDERS。"""
|
||||
name = manifest.get("name")
|
||||
if not name or not _NAME_RE.match(name):
|
||||
logger.warning("插件清单缺少合法 name: %r", name)
|
||||
return
|
||||
runtime = str(manifest.get("runtime", "none")).lower()
|
||||
# 委托检测: 调用插件自己的 check 函数 (node 型/python 型各自实现)
|
||||
available, reason = _call_check(manifest.get("check"))
|
||||
_PLUGIN_STATUS[name] = {
|
||||
"name": name,
|
||||
"display_name": manifest.get("display_name", name),
|
||||
"datasets": list(manifest.get("datasets", []) or []),
|
||||
"runtime": runtime,
|
||||
"available": available,
|
||||
"status": reason,
|
||||
"description": manifest.get("description", ""),
|
||||
"install_hint": manifest.get("install_hint", ""),
|
||||
}
|
||||
if not available:
|
||||
return # 依赖没装: 不注册, 但状态已记录供 UI 显示
|
||||
# 可用 → 动态加载 provider 类并实例化
|
||||
try:
|
||||
provider_cls = _load_entry(manifest["entry"])
|
||||
provider = provider_cls() if isinstance(provider_cls, type) else provider_cls
|
||||
provider.builtin = True # 标记为内置 (list_sources 过滤, 不可被用户编辑/删除)
|
||||
_PROVIDERS[name] = provider
|
||||
logger.info("内置插件 %s 已注册 (runtime=%s)", name, runtime)
|
||||
except Exception as e: # noqa: BLE001
|
||||
# 声称可用但 import 失败 → 标记不可用, 避免启动崩溃
|
||||
_PLUGIN_STATUS[name]["available"] = False
|
||||
_PLUGIN_STATUS[name]["status"] = f"加载失败: {e}"
|
||||
logger.warning("插件 %s provider 加载失败: %s", name, e)
|
||||
|
||||
|
||||
def _call_check(check_ref: str | None) -> tuple[bool, str]:
|
||||
"""调用插件清单里指定的可用性检测函数, 返回 (是否可用, 原因)。
|
||||
|
||||
check_ref 格式 "module.path:func_name"。无 check 字段时视为可用。
|
||||
"""
|
||||
if not check_ref:
|
||||
return True, "ok"
|
||||
try:
|
||||
func = _load_entry(check_ref)
|
||||
result = func()
|
||||
# 兼容两种返回: (bool, str) 或 bool
|
||||
if isinstance(result, tuple):
|
||||
return bool(result[0]), str(result[1])
|
||||
return bool(result), "ok" if result else "不可用"
|
||||
except Exception as e: # noqa: BLE001
|
||||
return False, str(e)
|
||||
|
||||
|
||||
def _load_entry(entry_ref: str):
|
||||
"""动态加载 'module.path:attr' 形式的引用, 返回属性对象 (类或函数)。"""
|
||||
if ":" not in entry_ref:
|
||||
raise ValueError(f"entry 格式应为 'module.path:attr', 得到: {entry_ref!r}")
|
||||
module_path, attr = entry_ref.split(":", 1)
|
||||
module = importlib.import_module(module_path)
|
||||
return getattr(module, attr)
|
||||
|
||||
|
||||
# 模块导入时即扫描一次, 保证 names()/_allowed_data_providers() 在 startup 前可用。
|
||||
_load_builtin_plugins()
|
||||
|
||||
|
||||
@@ -43,22 +43,52 @@ def _flatten_instruments(items: list[dict]) -> list[dict]:
|
||||
return rows
|
||||
|
||||
|
||||
def _fetch_instruments_via_provider() -> list[dict] | None:
|
||||
"""若当前日K数据源不是 tickflow 且该 provider 提供 get_instruments, 用它拉标的维表。
|
||||
|
||||
返回 flatten 行列表; 未命中(仍应走 tickflow)时返回 None。
|
||||
标的维表跟随日K数据源(二者天然耦合, 无独立偏好项)。
|
||||
"""
|
||||
from app.services import preferences
|
||||
|
||||
provider_name = preferences.get_daily_data_provider()
|
||||
if provider_name == "tickflow":
|
||||
return None
|
||||
from app.data_providers import custom as custom_sources
|
||||
|
||||
if not custom_sources.is_custom_provider(provider_name):
|
||||
return None
|
||||
provider = custom_sources.get_provider(provider_name)
|
||||
if not hasattr(provider, "get_instruments"):
|
||||
return None
|
||||
try:
|
||||
items = provider.get_instruments("stock") or []
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.warning("provider %s get_instruments 失败: %s", provider_name, e)
|
||||
return None
|
||||
rows = _flatten_instruments(items)
|
||||
logger.info("instruments via %s: %d stocks", provider_name, len(rows))
|
||||
return rows
|
||||
|
||||
|
||||
def sync_instruments(data_dir: Path) -> int:
|
||||
"""全量同步标的维表 → data/instruments/instruments.parquet。
|
||||
|
||||
返回写入的行数。
|
||||
"""
|
||||
tf = get_client()
|
||||
all_rows: list[dict] = []
|
||||
|
||||
for ex in _EXCHANGES:
|
||||
try:
|
||||
items = tf.exchanges.get_instruments(ex, instrument_type="stock")
|
||||
if items:
|
||||
all_rows.extend(_flatten_instruments(items))
|
||||
logger.info("instruments %s: %d stocks", ex, len(items))
|
||||
except Exception as e:
|
||||
logger.warning("get_instruments(%s) failed: %s", ex, e)
|
||||
all_rows = _fetch_instruments_via_provider()
|
||||
if all_rows is None:
|
||||
# 未命中非 tickflow provider → 走 tickflow 直连
|
||||
tf = get_client()
|
||||
all_rows = []
|
||||
for ex in _EXCHANGES:
|
||||
try:
|
||||
items = tf.exchanges.get_instruments(ex, instrument_type="stock")
|
||||
if items:
|
||||
all_rows.extend(_flatten_instruments(items))
|
||||
logger.info("instruments %s: %d stocks", ex, len(items))
|
||||
except Exception as e:
|
||||
logger.warning("get_instruments(%s) failed: %s", ex, e)
|
||||
|
||||
if not all_rows:
|
||||
return 0
|
||||
|
||||
Generated
+1834
-1834
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,118 @@
|
||||
# 数据源插件开发指南
|
||||
|
||||
数据源插件是可选的行情数据来源(、akshare 等),作为独立模块放在
|
||||
`backend/app/plugins/` 下。用户**手动安装依赖**后才可用;不安装完全不影响主功能。
|
||||
|
||||
## 快速上手
|
||||
|
||||
一个插件 = 一个目录 + 一个 `plugin.yaml` 清单:
|
||||
|
||||
```
|
||||
backend/app/plugins/<your_plugin>/
|
||||
├── plugin.yaml # 清单(必需)
|
||||
├── provider.py # Provider 实现(必需)
|
||||
├── ... # 桥接/依赖文件(按需)
|
||||
```
|
||||
|
||||
### plugin.yaml 字段
|
||||
|
||||
```yaml
|
||||
name: my_source # 唯一标识, 只允许 [a-z0-9_], 也是 provider name
|
||||
display_name: "我的数据源" # 设置页显示名
|
||||
runtime: python # 运行时类型: node | python | none
|
||||
entry: app.plugins.my_source.provider:MyProvider # provider 类的导入路径
|
||||
check: app.plugins.my_source.bridge:availability # 可用性检测函数(可选)
|
||||
datasets: [daily, adj_factor, minute, realtime] # 支持的数据集
|
||||
description: "数据源描述"
|
||||
install_hint: "pip install xxx" # 未装依赖时显示的安装提示
|
||||
```
|
||||
|
||||
### runtime 字段说明
|
||||
|
||||
| runtime | 含义 | 典型场景 |
|
||||
|---|---|---|
|
||||
| `python` | 纯 Python 依赖, `pip install` | akshare、tushare |
|
||||
| `node` | 需要 Node.js 运行时, `npm install` | |
|
||||
| `none` | 无额外依赖 | 纯 HTTP API 源 |
|
||||
|
||||
`runtime` 字段当前仅用于 UI 展示, 实际依赖检测由 `check` 函数负责。
|
||||
|
||||
### check 函数
|
||||
|
||||
插件自己负责检测依赖是否已安装。后端启动时会调用此函数:
|
||||
|
||||
```python
|
||||
# app/plugins/my_source/bridge.py
|
||||
def availability() -> tuple[bool, str]:
|
||||
"""返回 (是否可用, 原因)。不抛异常。"""
|
||||
try:
|
||||
import akshare # noqa: F401
|
||||
return True, "ok"
|
||||
except ImportError:
|
||||
return False, "未安装 akshare, 运行: pip install akshare"
|
||||
```
|
||||
|
||||
- **可用** → 插件注册进路由表, 设置页可切换
|
||||
- **不可用** → 设置页显示插件卡片但灰显, 展示 `install_hint`
|
||||
|
||||
## Provider 接口契约
|
||||
|
||||
Provider 是一个普通 Python 类(无需继承基类), 实现以下方法签名。方法签名对齐
|
||||
`GenericHTTPProvider`, 这样 services 层(kline_sync / quote_service 等)的路由逻辑
|
||||
零改动即可路由到插件。
|
||||
|
||||
```python
|
||||
class MyProvider:
|
||||
name = "my_source"
|
||||
builtin = True # 标记为内置(不可被用户编辑/删除)
|
||||
|
||||
def __init__(self):
|
||||
self.config = MyConfig() # 需有 .datasets 属性(dict, key 是数据集名)
|
||||
|
||||
def close(self) -> None:
|
||||
"""清理资源(load_all 重建注册表时会调)。"""
|
||||
|
||||
def get_daily(self, symbols, start_time, end_time, asset_type="stock", on_chunk_done=None) -> pl.DataFrame:
|
||||
"""日K: 返回 schema [symbol, date, open, high, low, close, volume, amount]"""
|
||||
|
||||
def get_adj_factors(self, symbols, start_time, end_time, asset_type="stock", on_chunk_done=None) -> pl.DataFrame:
|
||||
"""除权因子: 返回 schema [symbol, trade_date, ex_factor]"""
|
||||
|
||||
def get_minute(self, symbols, start_time, end_time, asset_type="stock", on_chunk_done=None, freq="1m") -> pl.DataFrame:
|
||||
"""分钟K: 返回 schema [symbol, datetime, open, high, low, close, volume, amount]"""
|
||||
|
||||
def get_realtime(self) -> list[dict]:
|
||||
"""全市场实时快照: 返回 list[dict], 每行含 symbol/last_price/prev_close/open/high/low/volume"""
|
||||
|
||||
def get_instruments(self, asset_type="stock") -> list[dict]:
|
||||
"""标的维表(可选): 返回 tickflow Instrument 形状的行, 供 instrument_sync 复用 flatten"""
|
||||
```
|
||||
|
||||
### config.datasets 的作用
|
||||
|
||||
`provider_has_dataset(name, dataset)` 通过 `dataset in provider.config.datasets` 判断。
|
||||
这是 services 层路由的关键: 用户在设置页选了插件, 但某数据集未声明时, 该数据集
|
||||
自动回退 TickFlow。
|
||||
|
||||
```python
|
||||
class MyConfig:
|
||||
datasets = {"daily": ..., "realtime": ...} # key 是数据集名, value 任意
|
||||
```
|
||||
|
||||
## 现有插件参考
|
||||
|
||||
- **`backend/app/plugins//`** — Node 型插件, 通过 subprocess 桥接调用
|
||||
- `bridge.py` — Python↔Node 桥接 + availability 检测
|
||||
- `bridge.mjs` — Node 端(并发池、重试、SDK 解析)
|
||||
- `provider.py` — Provider 实现(归一化、分批、错误降级)
|
||||
|
||||
## 路由机制(无需关心, 仅参考)
|
||||
|
||||
后端启动时, `loader.py` 的 `_load_builtin_plugins()` 扫描 `plugins/` 目录:
|
||||
1. 读每个子目录的 `plugin.yaml`
|
||||
2. 调 `check` 函数检测可用性
|
||||
3. 可用 → 动态 import `entry` 指向的 Provider 类 → 注册进 `_PROVIDERS`
|
||||
4. 不可用 → 记录状态, 设置页显示但不可切换
|
||||
|
||||
注册后, 插件和用户 YAML 自定义源走**完全相同的路由路径**(services 层的
|
||||
`provider_has_dataset` / `get_provider` 调用), 无需额外集成代码。
|
||||
@@ -693,6 +693,18 @@ export interface DataSourceItem {
|
||||
path?: string | null
|
||||
}
|
||||
|
||||
/** 内置可选插件数据源 (plugins/ 目录, 需手动装依赖) */
|
||||
export interface PluginDataSourceItem {
|
||||
name: string
|
||||
display_name: string
|
||||
datasets: string[]
|
||||
runtime: string // node | python | none
|
||||
available: boolean // 依赖是否已安装
|
||||
status: string // 可用性原因 (供 UI 显示)
|
||||
description: string
|
||||
install_hint: string // 未装依赖时显示的安装命令
|
||||
}
|
||||
|
||||
export interface DataSourceLoadError {
|
||||
name?: string
|
||||
path: string
|
||||
@@ -701,6 +713,7 @@ export interface DataSourceLoadError {
|
||||
|
||||
export interface DataSourcesResponse {
|
||||
builtin: DataSourceItem[]
|
||||
plugins: PluginDataSourceItem[]
|
||||
custom: DataSourceItem[]
|
||||
errors: DataSourceLoadError[]
|
||||
config_dir: string
|
||||
@@ -857,6 +870,20 @@ export const api = {
|
||||
deleteDataSource: (name: string) =>
|
||||
request<DataSourcesResponse>(`/api/settings/data-sources/${encodeURIComponent(name)}`, { method: 'DELETE' }),
|
||||
reloadDataSources: () => request<DataSourcesResponse>('/api/settings/data-sources/reload', { method: 'POST' }),
|
||||
installPlugin: (name: string) => {
|
||||
// npm install 可能耗时较长, 用 6 分钟超时
|
||||
const controller = new AbortController()
|
||||
const timer = setTimeout(() => controller.abort(), 360_000)
|
||||
return request<DataSourcesResponse & { install_ok: boolean; install_message: string }>(
|
||||
`/api/settings/plugins/${encodeURIComponent(name)}/install`,
|
||||
{ method: 'POST', signal: controller.signal },
|
||||
).finally(() => clearTimeout(timer))
|
||||
},
|
||||
uninstallPlugin: (name: string) =>
|
||||
request<DataSourcesResponse & { uninstall_ok: boolean; uninstall_message: string }>(
|
||||
`/api/settings/plugins/${encodeURIComponent(name)}/install`,
|
||||
{ method: 'DELETE' },
|
||||
),
|
||||
testDataSource: (provider: string, dataset: string, symbols?: string[]) =>
|
||||
request<DataSourceTestResult>('/api/settings/data-sources/test', {
|
||||
method: 'POST',
|
||||
|
||||
@@ -2,7 +2,7 @@ import { useState } from 'react'
|
||||
import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query'
|
||||
import { motion, AnimatePresence } from 'framer-motion'
|
||||
import { Check, Database, Plus, RefreshCw, Zap, FileWarning } from 'lucide-react'
|
||||
import { api, type DataSourceItem } from '@/lib/api'
|
||||
import { api, type DataSourceItem, type PluginDataSourceItem } from '@/lib/api'
|
||||
import { QK } from '@/lib/queryKeys'
|
||||
import { usePreferences } from '@/lib/useSharedQueries'
|
||||
import { toast } from '@/components/Toast'
|
||||
@@ -70,14 +70,50 @@ export function SettingsDataSourcesPanel() {
|
||||
onSuccess: (_data, name) => setSelected(name),
|
||||
})
|
||||
|
||||
const installMut = useMutation({
|
||||
mutationFn: (name: string) => api.installPlugin(name),
|
||||
onSuccess: (data) => {
|
||||
qc.invalidateQueries({ queryKey: QK.dataSources })
|
||||
if (data.install_ok) {
|
||||
toast('插件依赖安装成功', 'success')
|
||||
} else {
|
||||
toast(data.install_message || '安装失败', 'error')
|
||||
}
|
||||
},
|
||||
onError: (e: Error) => toast(`安装失败: ${e.message}`, 'error'),
|
||||
})
|
||||
|
||||
const uninstallMut = useMutation({
|
||||
mutationFn: (name: string) => api.uninstallPlugin(name),
|
||||
onSuccess: (data) => {
|
||||
qc.invalidateQueries({ queryKey: QK.dataSources })
|
||||
qc.invalidateQueries({ queryKey: QK.preferences })
|
||||
if (data.uninstall_ok) {
|
||||
toast(data.uninstall_message || '已卸载', 'success')
|
||||
} else {
|
||||
toast(data.uninstall_message || '卸载失败', 'error')
|
||||
}
|
||||
},
|
||||
onError: (e: Error) => toast(`卸载失败: ${e.message}`, 'error'),
|
||||
})
|
||||
|
||||
const builtin: DataSourceItem[] = sources.data?.builtin ?? []
|
||||
const pluginList: PluginDataSourceItem[] = sources.data?.plugins ?? []
|
||||
const customList: DataSourceItem[] = sources.data?.custom ?? []
|
||||
const errors = sources.data?.errors ?? []
|
||||
const activeName = prefs.data?.daily_data_provider || 'tickflow'
|
||||
|
||||
// 顶部数据源选择列表 (内置 + 自定义 + 新增)
|
||||
// 插件 name → 状态 (供卡片渲染时判断 available/installing 等)
|
||||
const pluginMap = new Map(pluginList.map(p => [p.name, p]))
|
||||
const pluginNames = new Set(pluginList.map(p => p.name))
|
||||
|
||||
// 顶部数据源选择列表 (内置 + 所有插件 + 自定义 + 新增)
|
||||
const pluginItems: DataSourceItem[] = pluginList.map(p => ({
|
||||
name: p.name, display_name: p.display_name, datasets: p.datasets,
|
||||
}))
|
||||
const allItems = [
|
||||
...builtin,
|
||||
...pluginItems,
|
||||
...customList,
|
||||
]
|
||||
|
||||
@@ -91,7 +127,10 @@ export function SettingsDataSourcesPanel() {
|
||||
<div className="flex items-center gap-2.5">
|
||||
<Database className="h-4 w-4 text-secondary" />
|
||||
<h2 className="text-sm font-medium text-foreground">数据源</h2>
|
||||
<span className="text-[10px] text-muted/40 font-mono truncate max-w-[280px]" title={sources.data?.config_dir}>
|
||||
<span
|
||||
className="text-[10px] text-muted/40 font-mono truncate hidden lg:inline max-w-[480px]"
|
||||
title={sources.data?.config_dir}
|
||||
>
|
||||
{sources.data?.config_dir}
|
||||
</span>
|
||||
</div>
|
||||
@@ -119,33 +158,84 @@ export function SettingsDataSourcesPanel() {
|
||||
{allItems.map(item => {
|
||||
const isActive = activeName === item.name
|
||||
const isSelected = selected === item.name
|
||||
const plugin = pluginMap.get(item.name)
|
||||
const pluginUnavailable = plugin && !plugin.available
|
||||
const installing = installMut.isPending && installMut.variables === item.name
|
||||
const uninstalling = uninstallMut.isPending && uninstallMut.variables === item.name
|
||||
return (
|
||||
<div
|
||||
key={item.name}
|
||||
onClick={() => {
|
||||
if (pluginUnavailable) return // 未安装的插件不可选中
|
||||
setSelected(item.name)
|
||||
if (item.name !== 'tickflow') {
|
||||
// 只有用户自定义源 (YAML) 才进编辑器; tickflow 和插件不可编辑
|
||||
if (customList.some(c => c.name === item.name)) {
|
||||
editExisting.mutate(item.name)
|
||||
}
|
||||
}}
|
||||
className={`relative cursor-pointer text-left rounded-lg border px-3.5 py-3 transition-all ${
|
||||
isSelected
|
||||
? 'border-accent/50 bg-accent/5 ring-1 ring-accent/20'
|
||||
: 'border-border/60 bg-elevated/20 hover:bg-elevated/40'
|
||||
className={`relative text-left rounded-lg border px-3.5 py-3 transition-all ${
|
||||
pluginUnavailable
|
||||
? 'border-border/40 bg-elevated/10 opacity-70'
|
||||
: isSelected
|
||||
? 'border-accent/50 bg-accent/5 ring-1 ring-accent/20 cursor-pointer'
|
||||
: 'border-border/60 bg-elevated/20 hover:bg-elevated/40 cursor-pointer'
|
||||
}`}
|
||||
>
|
||||
<div className="flex items-center gap-2 mb-1">
|
||||
<span className={`h-1.5 w-1.5 rounded-full shrink-0 ${isActive ? 'bg-accent' : 'bg-transparent border border-muted/40'}`} />
|
||||
<span className={`h-1.5 w-1.5 rounded-full shrink-0 ${
|
||||
pluginUnavailable ? 'bg-muted/30' : isActive ? 'bg-accent' : 'bg-transparent border border-muted/40'
|
||||
}`} />
|
||||
<span className={`text-sm truncate flex-1 ${isActive ? 'font-medium text-foreground' : 'text-secondary'}`}>
|
||||
{item.display_name}
|
||||
</span>
|
||||
{item.name === 'tickflow' && (
|
||||
<span className="text-[9px] text-muted/50 uppercase tracking-wider shrink-0">内置</span>
|
||||
)}
|
||||
{isActive ? (
|
||||
{pluginNames.has(item.name) && (
|
||||
<span className="text-[9px] text-muted/50 uppercase tracking-wider shrink-0">插件</span>
|
||||
)}
|
||||
{/* 右侧操作区: 插件未安装→安装按钮; 已激活→使用中; 否则→使用/卸载 */}
|
||||
{pluginUnavailable ? (
|
||||
installing ? (
|
||||
<span className="inline-flex items-center gap-1 text-[9px] text-accent shrink-0">
|
||||
<RefreshCw className="h-2.5 w-2.5 animate-spin" /> 安装中...
|
||||
</span>
|
||||
) : (
|
||||
<button
|
||||
onClick={(e) => { e.stopPropagation(); installMut.mutate(item.name) }}
|
||||
disabled={installMut.isPending}
|
||||
className="shrink-0 inline-flex items-center gap-1 rounded px-1.5 py-0.5 text-[10px] font-medium bg-accent/10 text-accent hover:bg-accent/20 transition-colors disabled:opacity-50"
|
||||
>
|
||||
<Zap className="h-2.5 w-2.5" /> 安装
|
||||
</button>
|
||||
)
|
||||
) : isActive ? (
|
||||
<span className="inline-flex items-center gap-0.5 text-[9px] text-accent shrink-0">
|
||||
<Check className="h-2.5 w-2.5" /> 使用中
|
||||
</span>
|
||||
) : plugin ? (
|
||||
/* 已安装插件: 使用 + 卸载 */
|
||||
<div className="flex items-center gap-1 shrink-0">
|
||||
<button
|
||||
onClick={(e) => { e.stopPropagation(); switchProvider.mutate(item.name) }}
|
||||
disabled={switchProvider.isPending}
|
||||
className="rounded px-1.5 py-0.5 text-[10px] font-medium bg-accent/10 text-accent hover:bg-accent/20 transition-colors disabled:opacity-50"
|
||||
>
|
||||
使用
|
||||
</button>
|
||||
{uninstalling ? (
|
||||
<RefreshCw className="h-2.5 w-2.5 animate-spin text-muted" />
|
||||
) : (
|
||||
<button
|
||||
onClick={(e) => { e.stopPropagation(); uninstallMut.mutate(item.name) }}
|
||||
disabled={uninstallMut.isPending}
|
||||
className="text-[10px] text-muted/50 hover:text-danger transition-colors disabled:opacity-40"
|
||||
title="卸载依赖"
|
||||
>
|
||||
卸载
|
||||
</button>
|
||||
)}
|
||||
</div>
|
||||
) : (
|
||||
<button
|
||||
onClick={(e) => { e.stopPropagation(); switchProvider.mutate(item.name) }}
|
||||
@@ -168,6 +258,10 @@ export function SettingsDataSourcesPanel() {
|
||||
{item.name === 'tickflow' && (
|
||||
<div className="text-[10px] text-muted/60 ml-3.5">日K · 除权 · 实时 · 分钟K</div>
|
||||
)}
|
||||
{/* 未安装插件显示安装命令提示 */}
|
||||
{pluginUnavailable && plugin?.install_hint && (
|
||||
<div className="ml-3.5 mt-1 text-[10px] text-muted/40 font-mono truncate">{plugin.install_hint}</div>
|
||||
)}
|
||||
</div>
|
||||
)
|
||||
})}
|
||||
@@ -223,7 +317,7 @@ export function SettingsDataSourcesPanel() {
|
||||
onSwitch={() => switchProvider.mutate('tickflow')}
|
||||
switching={switchProvider.isPending}
|
||||
/>
|
||||
) : (
|
||||
) : selected === '__new__' || customList.some(c => c.name === selected) ? (
|
||||
<DataSourceEditor
|
||||
key={selected}
|
||||
initial={null}
|
||||
@@ -241,7 +335,15 @@ export function SettingsDataSourcesPanel() {
|
||||
onActivate={(name) => switchProvider.mutate(name)}
|
||||
onDelete={selected !== '__new__' && selectedCustom ? () => setConfirmDelete(selected) : undefined}
|
||||
/>
|
||||
)}
|
||||
) : pluginList.find(x => x.name === selected) ? (
|
||||
/* 选中插件: 显示只读详情, 不进编辑器 */
|
||||
<PluginDetail
|
||||
plugin={pluginList.find(x => x.name === selected)!}
|
||||
isActive={activeName === selected}
|
||||
onSwitch={() => switchProvider.mutate(selected)}
|
||||
switching={switchProvider.isPending}
|
||||
/>
|
||||
) : null}
|
||||
</motion.div>
|
||||
</AnimatePresence>
|
||||
|
||||
@@ -279,6 +381,45 @@ export function SettingsDataSourcesPanel() {
|
||||
)
|
||||
}
|
||||
|
||||
function PluginDetail({ plugin, isActive, onSwitch, switching }: {
|
||||
plugin: PluginDataSourceItem
|
||||
isActive: boolean
|
||||
onSwitch: () => void
|
||||
switching: boolean
|
||||
}) {
|
||||
return (
|
||||
<section className="rounded-card border border-border bg-surface p-6">
|
||||
<div className="flex items-start gap-4 mb-5">
|
||||
<div className="h-11 w-11 rounded-xl bg-accent/10 flex items-center justify-center shrink-0">
|
||||
<Zap className="h-5 w-5 text-accent" />
|
||||
</div>
|
||||
<div className="flex-1 min-w-0">
|
||||
<div className="flex items-center gap-2 mb-1">
|
||||
<h3 className="text-base font-semibold text-foreground">{plugin.display_name}</h3>
|
||||
<span className="text-[10px] text-muted/50 uppercase tracking-wider">插件 · {plugin.runtime}</span>
|
||||
</div>
|
||||
{plugin.description && <p className="text-xs text-secondary leading-relaxed">{plugin.description}</p>}
|
||||
</div>
|
||||
</div>
|
||||
<div className="flex items-center gap-3">
|
||||
{isActive ? (
|
||||
<span className="inline-flex items-center gap-1.5 text-xs text-accent">
|
||||
<Check className="h-3.5 w-3.5" /> 当前使用中
|
||||
</span>
|
||||
) : (
|
||||
<button
|
||||
onClick={onSwitch}
|
||||
disabled={switching}
|
||||
className="px-3 py-1.5 rounded-btn bg-accent/10 text-accent hover:bg-accent/20 text-xs font-medium transition-colors disabled:opacity-50"
|
||||
>
|
||||
切换为当前数据源
|
||||
</button>
|
||||
)}
|
||||
</div>
|
||||
</section>
|
||||
)
|
||||
}
|
||||
|
||||
function TickFlowDetail({ active, onSwitch, switching }: { active: boolean; onSwitch: () => void; switching: boolean }) {
|
||||
return (
|
||||
<section className="rounded-card border border-border bg-surface p-6">
|
||||
|
||||
Reference in New Issue
Block a user