feat(data-sources): 支持外部插件安装与版本管理

Signed-off-by: shy3130 <shy3130@qq.com>
This commit is contained in:
shy3130
2026-11-01 10:47:19 +08:00
parent ba27fc932b
commit 762e04ab88
24 changed files with 1649 additions and 72 deletions
+1
View File
@@ -623,6 +623,7 @@ PORT=3018 # 服务端口
| [docs/mining.md](./docs/mining.md) | 因子与策略挖掘口径、防泄漏、任务隔离和发布边界 |
| [docs/market-phase.md](./docs/market-phase.md) | 市场情绪周期 6 阶段与概念/行业主线识别的口径与设计 |
| [docs/plugin-development.md](./docs/plugin-development.md) | 数据源插件开发规范(以 fuyao 为参考实现) |
| [外部数据源插件安装](./docs/external-data-source-packages.md) | ZIP 安装、Key 配置、版本维护和开发模板 |
| [docs/secondary-development.md](./docs/secondary-development.md) | 代码二次开发、前端插槽、后端策略接口与 AI 开发模板 |
| [docs/page-sdk.md](./docs/page-sdk.md) | 页面插件开发指南:产物契约、能力桥 ctx、打包与上传、安全边界 |
| [backend/app/strategy/prompts/strategy-guide.md](./backend/app/strategy/prompts/strategy-guide.md) | 策略开发完整规范(AI 生成与手写) |
+24 -8
View File
@@ -757,7 +757,7 @@ def save_plugin_key(req: PluginKeyIn) -> dict:
"""保存插件 API Key(先探后存, 对齐 /tickflow-key 语义)。
流程: probe_plugin_key 用候选 Key 实探 → 有效才写 secrets.json
({plugin}_api_key, 优先级高于 .env) → load_all 重扫, 插件即刻变为可切换。
({plugin}_api_key, 优先级高于 .env) → 定向刷新当前插件。
"""
from app.data_providers import custom as custom_sources
@@ -765,11 +765,27 @@ def save_plugin_key(req: PluginKeyIn) -> dict:
key = req.api_key.strip()
if not key:
return {"ok": False, "error": "key empty"}
ok, message = custom_sources.probe_plugin_key(name, key)
if not ok:
return {"ok": False, "reason": "invalid", "error": message}
secrets_store.save({f"{name}_api_key": key})
custom_sources.load_all()
from contextlib import nullcontext
from app.data_providers.custom import packages
external = packages.read_state(name) is not None
with packages.mutation() if external else nullcontext():
ok, message = custom_sources.probe_plugin_key(name, key)
if not ok:
return {"ok": False, "reason": "invalid", "error": message}
previous = secrets_store.load().get(f"{name}_api_key") if external else None
secrets_store.save({f"{name}_api_key": key})
try:
if external:
custom_sources.refresh_plugin(name, strict=True)
else:
custom_sources.refresh_plugin(name)
except ValueError:
if previous is None:
secrets_store.clear(f"{name}_api_key")
else:
secrets_store.save({f"{name}_api_key": previous})
return {"ok": False, "error": "插件初始化失败,已保留原配置"}
status = next((p for p in custom_sources.list_plugins() if p["name"] == name), None)
return {
"ok": True,
@@ -785,12 +801,12 @@ def clear_plugin_key(name: str) -> dict:
from app.data_providers import custom as custom_sources
manifest = custom_sources.plugin_manifest(name)
if manifest is None or not custom_sources.is_builtin(name):
if manifest is None:
raise HTTPException(status_code=404, detail=f"插件 '{name}' 不存在")
if not manifest.get("api_key_env"):
raise HTTPException(status_code=400, detail=f"插件 '{name}' 不支持在界面配置 Key")
secrets_store.clear(f"{name.lower()}_api_key")
custom_sources.load_all()
custom_sources.refresh_plugin(name)
status = next((p for p in custom_sources.list_plugins() if p["name"] == name), None)
return {
"ok": True,
@@ -0,0 +1,10 @@
"""External data-source package management, registered through existing L2 routes."""
from app.extensions import BACKEND_EXTENSION_API_VERSION, BackendExtensionRegistrar
EXTENSION_ID = "official.source-packages"
EXTENSION_API_VERSION = BACKEND_EXTENSION_API_VERSION
def setup(registrar: BackendExtensionRegistrar):
from .routes import router
registrar.include_router(router)
@@ -0,0 +1,87 @@
from __future__ import annotations
from contextlib import asynccontextmanager
from typing import Annotated
from urllib.parse import urlsplit
import yaml
from fastapi import APIRouter, Depends, File, Form, HTTPException, Request, UploadFile
from pydantic import BaseModel
from app.api.auth import COOKIE_NAME, _client_ip
from app.data_providers.custom import packages
from app.services import auth
def require_management(request: Request):
# Unlike read-only APIs, uploading Python is code execution. LAN access without
# a password is not enough. No API-token scope grants this permission.
if request.headers.get("authorization"):
raise HTTPException(403, "插件安装仅供面板管理会话使用")
origin = request.headers.get("origin")
if origin and urlsplit(origin).netloc != request.headers.get("host"):
raise HTTPException(403, "插件管理需要同源访问")
if request.headers.get("sec-fetch-site") == "cross-site":
raise HTTPException(403, "拒绝跨站插件管理请求")
if auth.is_configured():
if not auth.is_valid_session(request.cookies.get(COOKIE_NAME, "")):
raise HTTPException(401, "请先登录")
elif _client_ip(request) not in ("127.0.0.1", "::1", "localhost") or request.url.hostname not in ("127.0.0.1", "::1", "localhost"):
raise HTTPException(403, "请先设置访问密码并登录,或通过本机访问安装插件")
@asynccontextmanager
async def lifespan(_app):
try:
yield
finally:
from app.data_providers import custom
for plugin in custom.list_plugins():
if plugin.get("origin") == "external":
custom.get_provider(plugin["name"]).close()
router = APIRouter(lifespan=lifespan, prefix="/api/custom/source-packages", tags=["source-packages"], dependencies=[Depends(require_management)])
async def read_upload(file: UploadFile):
try:
data = await file.read(packages.MAX_UPLOAD + 1)
if len(data) > packages.MAX_UPLOAD:
raise HTTPException(413, "插件包超过 20 MiB")
return data
finally:
await file.close()
@router.post("/inspect")
async def inspect(file: Annotated[UploadFile, File()]):
data = await read_upload(file)
from starlette.concurrency import run_in_threadpool
try:
preview, _ = await run_in_threadpool(packages.inspect_archive, data)
return preview
except (ValueError, OSError, TypeError, yaml.YAMLError) as exc:
raise HTTPException(400, str(exc)) from exc
@router.post("/install")
async def install(file: Annotated[UploadFile, File()], digest: str = Form(...)):
from starlette.concurrency import run_in_threadpool
data = await read_upload(file)
try:
return await run_in_threadpool(packages.install, data, digest)
except (ValueError, OSError, TypeError, yaml.YAMLError) as exc:
raise HTTPException(400, str(exc)) from exc
class Action(BaseModel):
action: str
@router.post("/{name}/manage")
def manage(name: str, body: Action):
try:
return packages.manage(name, body.action)
except (ValueError, OSError, TypeError, yaml.YAMLError) as exc:
raise HTTPException(400, str(exc)) from exc
@@ -16,6 +16,7 @@ from app.data_providers.custom.loader import (
plugin_manifest,
probe_plugin_key,
provider_has_dataset,
refresh_plugin,
save_config,
uninstall_plugin,
)
@@ -37,6 +38,7 @@ __all__ = [
"plugin_manifest",
"probe_plugin_key",
"provider_has_dataset",
"refresh_plugin",
"save_config",
"uninstall_plugin",
]
+166 -34
View File
@@ -1,6 +1,7 @@
"""Load custom data source definitions from user data files."""
from __future__ import annotations
import hashlib
import importlib
import logging
import math
@@ -8,11 +9,13 @@ import re
import shutil
import subprocess
from pathlib import Path
from types import SimpleNamespace
import yaml
from app import secrets_store
from app.config import settings
from app.data_providers.base import ProviderRequestError
from app.data_providers.custom.config import (
DEFAULT_TIMEOUT,
MAX_TIMEOUT,
@@ -20,6 +23,7 @@ from app.data_providers.custom.config import (
config_from_dict,
load_config,
)
from app.data_providers.custom.managed import ManagedProvider
from app.data_providers.custom.provider import GenericHTTPProvider
from app.services.fs_utils import atomic_write_text
@@ -47,9 +51,12 @@ def data_sources_dir() -> Path:
def load_all(path: Path | None = None) -> None:
"""Load all custom provider YAML files into process memory."""
global _PROVIDERS, _LOAD_ERRORS
for provider in _PROVIDERS.values():
provider.close()
_PROVIDERS = {}
retained = {name: provider for name, provider in _PROVIDERS.items()
if _PLUGIN_STATUS.get(name, {}).get("origin") == "external"}
for name, provider in _PROVIDERS.items():
if name not in retained:
provider.close()
_PROVIDERS = dict(retained)
_LOAD_ERRORS = []
base = path or data_sources_dir()
@@ -88,7 +95,7 @@ def list_sources() -> list[dict]:
def list_plugins() -> list[dict]:
"""返回所有内置插件的状态 (含已装/未装), 供设置页独立分类显示。"""
return list(_PLUGIN_STATUS.values())
return [dict(item) for item in _PLUGIN_STATUS.values()]
def _plugin_key_masked(name: str, api_key_env: str) -> str:
@@ -106,7 +113,13 @@ def _plugin_key_masked(name: str, api_key_env: str) -> str:
def plugin_manifest(name: str) -> dict | None:
"""读取指定插件的 plugin.yaml 清单。"""
plugin_dir = plugins_dir() / (name or "")
if not _NAME_RE.fullmatch(name or ""):
return None
from . import packages
external = packages.manifest(name)
if external is not None:
return external
plugin_dir = plugins_dir() / name
manifest_path = plugin_dir / "plugin.yaml"
if not manifest_path.exists():
return None
@@ -133,16 +146,24 @@ def probe_plugin_key(name: str, api_key: str) -> tuple[bool, str]:
if ":" not in entry:
return False, f"插件 '{name}' entry 非法"
try:
module = importlib.import_module(entry.split(":", 1)[0])
except Exception as e:
return False, f"插件模块加载失败: {e}"
if _PLUGIN_STATUS.get(name, {}).get("origin") == "external":
from . import packages
if not packages.read_state(name).get("enabled", True):
return False, "请先启用插件"
module = packages.import_module(manifest, entry.split(":", 1)[0])
else:
module = importlib.import_module(entry.split(":", 1)[0])
except Exception: # noqa: BLE001 — do not disclose plugin exception payloads
return False, "插件模块加载失败,请检查依赖与兼容范围"
probe = getattr(module, "probe_api_key", None)
if probe is None:
return False, f"插件 '{name}' 未提供 Key 探测"
try:
return probe(api_key)
except Exception as e:
return False, f"探测失败: {e}"
ok, reason = probe(api_key)
return bool(ok), (str(reason).replace(api_key, "[已隐藏]")
if _PLUGIN_STATUS.get(name, {}).get("origin") == "external" else str(reason))
except Exception: # noqa: BLE001 — plugin exceptions may contain credentials
return False, "Key 探测失败,请检查凭据、网络与接口权限"
def install_plugin(name: str) -> tuple[bool, str]:
@@ -284,7 +305,8 @@ def uninstall_plugin(name: str) -> tuple[bool, str]:
def is_builtin(name: str) -> bool:
"""判断 name 是否为内置插件 (不可被用户编辑/删除)。"""
return (name or "").lower() in _PLUGIN_STATUS
return ((name or "").lower() in _PLUGIN_STATUS
and _PLUGIN_STATUS[(name or "").lower()].get("origin") != "external")
def names() -> set[str]:
@@ -330,7 +352,7 @@ def provider_has_dataset(name: str, dataset: str) -> bool:
def get_config_dict(name: str) -> dict | None:
"""读取一个已加载 custom 源的原始配置 dict(用于前端编辑回填)。内置插件不可编辑。"""
if is_builtin(name):
if name in _PLUGIN_STATUS:
return None
provider = _PROVIDERS.get((name or "").lower())
if provider is None:
@@ -375,7 +397,7 @@ def _config_to_dict(config: CustomSourceConfig) -> dict:
def save_config(name: str, config: dict) -> Path:
"""把一份配置 dict 写成 data/data_sources/{name}.yaml, 返回写入路径。"""
if is_builtin(name):
if name in _PLUGIN_STATUS:
raise ValueError(f"'{name}' 是内置插件, 不可编辑")
if not _NAME_RE.match(name or ""):
raise ValueError(f"invalid data source name: {name!r} (only lowercase a-z 0-9 _ allowed)")
@@ -391,7 +413,7 @@ def save_config(name: str, config: dict) -> Path:
def delete_config(name: str) -> bool:
"""删除 data/data_sources/{name}.yaml。返回是否真的删除了。"""
if is_builtin(name):
if name in _PLUGIN_STATUS:
raise ValueError(f"'{name}' 是内置插件, 不可删除")
if not _NAME_RE.match(name or ""):
raise ValueError(f"invalid data source name: {name!r}")
@@ -536,11 +558,10 @@ def _load_builtin_plugins() -> None:
每次调用重建 _PLUGIN_STATUS, 并把可用的插件注册进 _PROVIDERS。
"""
global _PLUGIN_STATUS
_PLUGIN_STATUS = {}
_PLUGIN_STATUS = {name: info for name, info in _PLUGIN_STATUS.items()
if info.get("origin") == "external"}
pdir = plugins_dir()
if not pdir.exists():
return
for plugin_dir in sorted(pdir.iterdir()):
for plugin_dir in sorted(pdir.iterdir()) if pdir.exists() else []:
if not plugin_dir.is_dir():
continue
manifest_path = plugin_dir / "plugin.yaml"
@@ -551,23 +572,16 @@ def _load_builtin_plugins() -> None:
_register_one_plugin(manifest)
except Exception as e: # noqa: BLE001
logger.warning("插件 %s 清单解析失败: %s", plugin_dir.name, e)
from . import packages
packages.discover()
def _register_one_plugin(manifest: dict) -> None:
"""注册单个插件: 委托自检 → 可用则动态 import entry 注册进 _PROVIDERS。"""
name = manifest.get("name")
if not name or not _NAME_RE.match(name):
logger.warning("插件清单缺少合法 name: %r", name)
return
# hidden: 已加载但对 UI 隐藏 (功能未完成/暂不开放), 不注册不展示
if manifest.get("hidden"):
logger.info("插件 %s 标记为 hidden, 跳过注册", name)
return
def _manifest_status(manifest, available, reason, origin="builtin"):
name = manifest["name"]
runtime = str(manifest.get("runtime", "none")).lower()
# 委托检测: 调用插件自己的 check 函数 (node 型/python 型各自实现)
available, reason = _call_check(manifest.get("check"))
_PLUGIN_STATUS[name] = {
return {
"name": name,
"origin": origin,
"display_name": manifest.get("display_name", name),
"datasets": list(manifest.get("datasets", []) or []),
"runtime": runtime,
@@ -587,14 +601,37 @@ def _register_one_plugin(manifest: dict) -> None:
# Key/账务管理页 (可选): 设置页 Key 配置区展示入口
"key_manage_url": manifest.get("key_manage_url", ""),
}
def _register_one_plugin(manifest: dict) -> None:
"""注册单个插件: 委托自检 → 可用则动态 import entry 注册进 _PROVIDERS。"""
name = manifest.get("name")
if not name or not _NAME_RE.match(name):
logger.warning("插件清单缺少合法 name: %r", name)
return
# hidden: 已加载但对 UI 隐藏 (功能未完成/暂不开放), 不注册不展示
if manifest.get("hidden"):
logger.info("插件 %s 标记为 hidden, 跳过注册", name)
return
runtime = str(manifest.get("runtime", "none")).lower()
# 委托检测: 调用插件自己的 check 函数 (node 型/python 型各自实现)
available, reason = _call_check(manifest.get("check"))
_PLUGIN_STATUS[name] = _manifest_status(manifest, available, reason)
if not available:
return # 依赖没装: 不注册, 但状态已记录供 UI 显示
old = _PROVIDERS.pop(name, None)
if old is not None:
old.close()
return # 不就绪时撤下旧实例,避免旧凭据继续生效
# 可用 → 动态加载 provider 类并实例化
try:
provider_cls = _load_entry(manifest["entry"])
provider = provider_cls() if isinstance(provider_cls, type) else provider_cls
provider.builtin = True # 标记为内置 (list_sources 过滤, 不可被用户编辑/删除)
_PROVIDERS[name] = provider
old = _PROVIDERS.get(name)
if isinstance(old, ManagedProvider):
old.replace(provider)
else:
_PROVIDERS[name] = ManagedProvider(provider)
logger.info("内置插件 %s 已注册 (runtime=%s)", name, runtime)
except Exception as e: # noqa: BLE001
# 声称可用但 import 失败 → 标记不可用, 避免启动崩溃
@@ -630,5 +667,100 @@ def _load_entry(entry_ref: str):
return getattr(module, attr)
class _UnavailableProvider:
"""Keep installed identities so preferences cannot silently switch vendors."""
def __init__(self, doc, reason):
self.name = doc["name"]
self.config = SimpleNamespace(datasets=dict.fromkeys(doc.get("datasets", [])))
self.reason = reason
def close(self):
pass
def __getattr__(self, name):
if not name.startswith(("get_", "iter_")):
raise AttributeError(name)
def unavailable(*args, **kwargs):
raise ProviderRequestError(f"数据源 {self.name} 不可用:{self.reason}")
return unavailable
def record_external_error(name):
if name in _PROVIDERS and _PLUGIN_STATUS.get(name, {}).get("origin") != "external":
logger.warning("external plugin name conflict: %s", name)
return
from .packages import DATASETS
doc = {"name": name, "datasets": sorted(DATASETS), "version": ""}
publish_external(doc, None, "插件加载失败,请检查清单与兼容范围", metadata=False)
def publish_external(doc, provider, reason, *, metadata=True):
name = doc["name"]
available = provider is not None
value = provider if available else _UnavailableProvider(doc, reason)
old = _PROVIDERS.get(name)
if isinstance(old, ManagedProvider):
old.replace(value)
else:
_PROVIDERS[name] = ManagedProvider(value)
_PLUGIN_STATUS[name] = {
**_manifest_status(doc, available, reason, origin="external"),
"version": doc.get("version", ""), "author": doc.get("author", ""),
"can_test": available and callable(getattr(provider, "test_dataset", None)),
}
if metadata:
_PROVIDERS[name]._external_signature = _external_signature(doc)
refresh_external_metadata(name)
def _external_signature(doc):
from . import packages
state = packages.read_state(doc["name"]) or {}
key = secrets_store.get_env_backed_secret(f"{doc['name']}_api_key", doc.get("api_key_env", ""))
return (doc.get("version"), state.get("enabled", True), hashlib.sha256(key.encode()).digest())
def refresh_external_metadata(name):
from . import packages
state = packages.read_state(name)
if state and name in _PLUGIN_STATUS:
_PLUGIN_STATUS[name].update({
"pending_version": state.get("pending"), "previous_version": state.get("previous"),
"enabled": state.get("enabled", True), "pending_remove": state.get("remove", False),
"update_error": state.get("error"),
})
def refresh_plugin(name, *, manifest_doc=None, strict=False):
"""Refresh one plugin without closing other providers or in-flight requests."""
from . import packages
doc = manifest_doc or plugin_manifest(name)
if doc is None:
raise ValueError("插件不存在")
state = packages.read_state(name)
if state is None:
_register_one_plugin(doc)
return
if (manifest_doc is not None and name in _PROVIDERS
and _PLUGIN_STATUS.get(name, {}).get("origin") == "external"
and getattr(_PROVIDERS[name], "_external_signature", None) == _external_signature(doc)):
refresh_external_metadata(name)
return
if not state.get("enabled", True):
publish_external(doc, None, "待重启卸载" if state.get("remove") else "已停用")
return
try:
provider, reason = packages.build_provider(doc)
if strict and provider is None:
raise ValueError("插件配置未就绪")
except Exception: # noqa: BLE001 — uploaded exceptions can contain credentials
if strict:
raise ValueError("插件初始化失败,原配置保持不变") from None
publish_external(doc, None, "插件加载失败,请检查依赖与 Provider 实现")
return
publish_external(doc, provider, reason)
# 模块导入时即扫描一次, 保证 names()/_allowed_data_providers() 在 startup 前可用。
_load_builtin_plugins()
@@ -0,0 +1,102 @@
"""Keep a provider alive until its last call / streaming iterator completes."""
from __future__ import annotations
import logging
import threading
from collections.abc import Iterator
from app.data_providers.base import ProviderRequestError
logger = logging.getLogger(__name__)
class _Stream(Iterator):
def __init__(self, iterator, release):
self.iterator, self.release = iterator, release
def __next__(self):
try:
return next(self.iterator)
except BaseException:
self.close()
raise
def close(self):
if self.release is not None:
release, self.release = self.release, None
try:
close = getattr(self.iterator, "close", None)
if close:
close()
finally:
release()
def __del__(self):
self.close()
class ManagedProvider:
"""Stable handle; swaps do not close an in-flight provider, including iter_daily."""
builtin = True
def __init__(self, provider):
self._lock = threading.RLock()
self._current = [provider, 0, False]
def _release(self, entry):
with self._lock:
entry[1] -= 1
close = entry[2] and entry[1] == 0
if close:
self._close(entry[0])
@staticmethod
def _close(provider):
try:
provider.close()
except Exception: # noqa: BLE001 — cleanup cannot invalidate the replacement
logger.warning("provider close failed", exc_info=False)
def replace(self, provider):
with self._lock:
old = self._current
self._current = [provider, 0, False]
old[2] = True
close = old[1] == 0
if close:
self._close(old[0])
def close(self):
with self._lock:
old = self._current
if old[2]:
return
old[2] = True
close = old[1] == 0
if close:
self._close(old[0])
def __getattr__(self, name):
with self._lock:
value = getattr(self._current[0], name)
if not callable(value):
return value
def invoke(*args, **kwargs):
with self._lock:
entry = self._current
if entry[2]:
raise ProviderRequestError("数据源已停止,请重新选择数据源")
method = getattr(entry[0], name)
entry[1] += 1
try:
result = method(*args, **kwargs)
except BaseException:
self._release(entry)
raise
if isinstance(result, Iterator):
return _Stream(result, lambda: self._release(entry))
self._release(entry)
return result
return invoke
@@ -0,0 +1,428 @@
"""External provider packages. Static inspection never imports uploaded code.
The existing loader owns provider instances. This module owns immutable package
versions and activation state only; it never installs into the host environment.
"""
from __future__ import annotations
import hashlib
import importlib
import importlib.metadata
import io
import json
import platform
import re
import shutil
import sqlite3
import stat
import sys
import tempfile
import types
import zipfile
from contextlib import contextmanager
from pathlib import Path, PurePosixPath
import yaml
from packaging.requirements import Requirement
from packaging.specifiers import SpecifierSet
from packaging.version import Version
from app import __version__
from app.config import settings
from app.services.fs_utils import atomic_write_text
MAX_UPLOAD = 20 * 1024 * 1024
MAX_EXPANDED = 80 * 1024 * 1024
NAME = re.compile(r"[a-z][a-z0-9_]{0,63}\Z")
VERSION = re.compile(r"\d+\.\d+\.\d+(?:[a-zA-Z0-9.+-]*)\Z")
ENTRY = re.compile(r"[a-zA-Z_]\w*(?:\.[a-zA-Z_]\w*)*:[a-zA-Z_]\w*\Z")
DATASETS = {"daily", "adj_factor", "minute", "full_minute", "realtime", "depth5", "financial"}
# Once per process: never activate a staged update during a settings reload.
_started: set[str] = set()
def root() -> Path:
return settings.data_dir / "plugins"
def directory(name: str) -> Path:
if not NAME.fullmatch(name):
raise ValueError("插件标识非法")
path = root() / name
if path.is_symlink() or root().is_symlink():
raise ValueError("插件目录不能是符号链接")
return path
@contextmanager
def mutation():
root().mkdir(parents=True, exist_ok=True)
conn = sqlite3.connect(root() / ".management.sqlite", timeout=5)
try:
conn.execute("BEGIN EXCLUSIVE")
yield
finally:
conn.rollback()
conn.close()
def read_state(name: str) -> dict | None:
if not NAME.fullmatch(name):
return None
path = directory(name) / "state.json"
if not path.exists():
return None
state = json.loads(path.read_text(encoding="utf-8"))
if state.get("schema") != 1:
raise ValueError("不支持的插件安装状态版本")
return state
def write_state(name: str, state: dict):
directory(name).mkdir(parents=True, exist_ok=True)
atomic_write_text(directory(name) / "state.json", json.dumps(state, ensure_ascii=False))
def version_dir(name: str, version: str) -> Path:
if not VERSION.fullmatch(version):
raise ValueError("插件版本非法")
path = directory(name) / "versions" / version
if path.is_symlink() or path.parent.is_symlink():
raise ValueError("插件版本目录不能是符号链接")
return path
def manifest(name: str, version: str | None = None) -> dict | None:
state = read_state(name)
if state is None:
return None
version = version or state["active"]
doc = yaml.safe_load((version_dir(name, version) / "plugin.yaml").read_text("utf-8"))
validate_manifest(doc)
if doc["name"] != name or doc["version"] != version:
raise ValueError("插件清单与安装记录不一致")
return doc
def validate_manifest(doc: dict):
if not isinstance(doc, dict):
raise ValueError("plugin.yaml 必须是对象")
if not NAME.fullmatch(str(doc.get("name", ""))):
raise ValueError("name 必须是小写字母开头的字母、数字、下划线")
if not VERSION.fullmatch(str(doc.get("version", ""))):
raise ValueError("version 必须是有效版本,例如 1.0.0")
Version(doc["version"])
if type(doc.get("plugin_api_version")) is not int or doc["plugin_api_version"] != 1:
raise ValueError("仅支持 plugin_api_version: 1")
for field in ("entry", "check"):
if (field == "entry" or doc.get(field)) and not ENTRY.fullmatch(str(doc.get(field, ""))):
raise ValueError(f"{field} 应使用包内相对模块引用,例如 provider:MyProvider")
datasets = doc.get("datasets")
if not isinstance(datasets, list) or not datasets or any(not isinstance(x, str) or x not in DATASETS for x in datasets):
raise ValueError("datasets 必须声明已支持的数据集")
for key in ("display_name", "description", "api_key_env", "homepage", "key_manage_url", "author", "billing", "billing_note", "install_hint"):
if key in doc and (not isinstance(doc[key], str) or len(doc[key]) > 4000):
raise ValueError(f"{key} 必须是文本且不超过 4000 字符")
if doc.get("api_key_env") and not re.fullmatch(r"[A-Z][A-Z0-9_]{0,127}", doc["api_key_env"]):
raise ValueError("api_key_env 必须是大写环境变量名")
for key in ("homepage", "key_manage_url"):
if doc.get(key) and not doc[key].startswith("https://"):
raise ValueError(f"{key} 必须使用 HTTPS")
if "usage_notes" in doc and (not isinstance(doc["usage_notes"], list) or any(not isinstance(x, str) for x in doc["usage_notes"])):
raise ValueError("usage_notes 必须是文本列表")
if doc.get("runtime", "none") not in ("none", "python"):
raise ValueError("外部安装暂只支持 none/python 运行时,不安装额外运行时")
if doc.get("hidden"):
raise ValueError("外部安装的插件不能隐藏卡片")
def compatibility(doc: dict) -> list[str]:
errors = []
for field, current in (("requires_app", __version__), ("requires_python", platform.python_version())):
if doc.get(field) and Version(current) not in SpecifierSet(str(doc[field])):
errors.append(f"{field}: 需要 {doc[field]},当前 {current}")
platforms = doc.get("platforms", ["any"])
current = f"{sys.platform}-{platform.machine().lower()}"
if not isinstance(platforms, list) or not all(isinstance(x, str) for x in platforms):
raise ValueError("platforms 必须是字符串列表")
if "any" not in platforms and current not in platforms:
errors.append(f"运行平台不匹配,当前 {current}")
requirements = doc.get("requires", [])
if not isinstance(requirements, list):
raise ValueError("requires 必须是依赖列表")
for item in requirements:
req = Requirement(item)
if req.url:
raise ValueError("requires 不接受 URL 或安装脚本")
if req.marker and not req.marker.evaluate():
continue
try:
installed = importlib.metadata.version(req.name)
if installed not in req.specifier:
errors.append(f"依赖 {req.name} 版本不匹配:{installed}")
except importlib.metadata.PackageNotFoundError:
errors.append(f"缺少依赖 {req.name}(不会自动修改主程序环境)")
return errors
def inspect_archive(data: bytes) -> tuple[dict, dict[str, bytes]]:
if len(data) > MAX_UPLOAD:
raise ValueError("插件包超过 20 MiB")
files = {}
folded = set()
total = 0
try:
with zipfile.ZipFile(io.BytesIO(data)) as archive:
if len(archive.infolist()) > 1000:
raise ValueError("插件包条目超过 1000")
for info in archive.infolist():
name = info.filename
parts = PurePosixPath(name).parts
if (not parts or name.startswith("/") or "\\" in name or ":" in name
or any(p in (".", "..") or p.endswith((".", " ")) for p in parts)
or PurePosixPath(name).as_posix() != name.rstrip("/")
or any(p.split(".")[0].upper() in {"CON", "PRN", "AUX", "NUL", *[f"COM{i}" for i in range(10)], *[f"LPT{i}" for i in range(10)]} for p in parts)):
raise ValueError("插件包包含非法路径")
if stat.S_IFMT(info.external_attr >> 16) not in (0, stat.S_IFREG, stat.S_IFDIR):
raise ValueError("插件包不允许符号链接或特殊文件")
if name.casefold().rstrip("/") in folded:
raise ValueError("插件包包含重复或大小写冲突的路径")
folded.add(name.casefold().rstrip("/"))
if info.is_dir():
continue
total += info.file_size
if total > MAX_EXPANDED or info.file_size > MAX_UPLOAD:
raise ValueError("插件包解压大小超限")
# Executable/native payloads need a separately verified distribution.
if PurePosixPath(name).suffix.lower() not in (".py", ".yaml", ".yml", ".json", ".txt", ".md", ".csv"):
raise ValueError("首版支持源码包;不接收原生库、wheel、脚本或可执行文件")
files[name] = archive.read(info)
except (zipfile.BadZipFile, RuntimeError, NotImplementedError) as exc:
raise ValueError("无法读取 ZIP 插件包") from exc
if "plugin.yaml" not in files or len(files["plugin.yaml"]) > 64 * 1024:
raise ValueError("包根目录需要 plugin.yaml(最大 64 KiB)")
doc = yaml.safe_load(files["plugin.yaml"])
validate_manifest(doc)
for field in ("entry", "check"):
if doc.get(field):
module = doc[field].split(":")[0].replace(".", "/")
if f"{module}.py" not in files and f"{module}/__init__.py" not in files:
raise ValueError(f"{field} 指定的模块不存在")
for name, payload in files.items():
if name.endswith(".py"):
try:
compile(payload, name, "exec")
except (SyntaxError, ValueError) as exc:
raise ValueError(f"Python 文件语法错误: {name}") from exc
errors = compatibility(doc)
from . import loader
if (loader.plugins_dir() / doc["name"]).exists() or doc["name"] == "tickflow" or any(
(loader.data_sources_dir() / f"{doc['name']}{suffix}").exists() for suffix in (".yaml", ".yml")
):
raise ValueError("插件名称与内置插件或 HTTP 数据源冲突")
state = read_state(doc["name"])
digest = hashlib.sha256(data).hexdigest()
if state and doc["version"] in state["hashes"] and state["hashes"][doc["version"]] != digest:
raise ValueError("相同版本已有不同内容,请提升版本号")
return {"manifest": doc, "digest": digest, "errors": errors,
"installed_version": state["active"] if state else None}, files
def import_module(doc: dict, module: str):
path = version_dir(doc["name"], doc["version"])
# Root-specific namespace keeps different installations and tests independent.
contents = b"".join(file.read_bytes() for file in sorted(path.rglob("*.py")))
digest = hashlib.sha256(str(path.resolve()).encode() + contents).hexdigest()[:20]
namespace = f"_tsp_provider_{digest}"
if namespace not in sys.modules:
package = types.ModuleType(namespace)
package.__path__ = [str(path)]
package.__package__ = namespace
sys.modules[namespace] = package
return importlib.import_module(f"{namespace}.{module}")
def load_entry(doc: dict, entry: str):
module, attr = entry.split(":")
return getattr(import_module(doc, module), attr)
def build_provider(doc: dict):
issues = compatibility(doc)
if issues:
raise ValueError(";".join(issues))
factory = load_entry(doc, doc["entry"])
if not callable(factory):
raise ValueError("entry 必须指向 Provider 类或工厂")
if doc.get("api_key_env"):
module = import_module(doc, doc["entry"].split(":")[0])
if not callable(getattr(module, "probe_api_key", None)):
raise ValueError("声明 API Key 的插件必须提供 probe_api_key")
from app import secrets_store
if doc.get("api_key_env") and not secrets_store.get_env_backed_secret(f"{doc['name']}_api_key", doc["api_key_env"]):
return None, "待配置 API Key"
if doc.get("check"):
check = load_entry(doc, doc["check"])()
ok, reason = check if isinstance(check, tuple) else (check, "不可用")
if not ok:
key = secrets_store.get_env_backed_secret(f"{doc['name']}_api_key", doc.get("api_key_env", ""))
return None, str(reason).replace(key, "[已隐藏]") if key else str(reason)
provider = factory()
try:
if provider.name != doc["name"] or set(provider.config.datasets) != set(doc["datasets"]):
raise ValueError("Provider 名称/能力声明与清单不一致")
required = {"daily": "get_daily", "adj_factor": "get_adj_factors", "minute": "get_minute", "realtime": "get_realtime", "depth5": "get_depth_batch", "financial": "get_financials"}
for dataset in doc["datasets"]:
methods = ("get_intraday_batch", "get_minute") if dataset == "full_minute" else (required[dataset],)
if not any(callable(getattr(provider, method, None)) for method in methods):
raise ValueError(f"缺少 {dataset} 数据方法")
if not callable(getattr(provider, "close", None)):
raise ValueError("Provider 缺少 close 方法")
except Exception:
close = getattr(provider, "close", None)
if callable(close):
close()
raise
return provider, "已就绪"
def install(data: bytes, digest: str) -> dict:
from . import loader
with mutation():
preview, files = inspect_archive(data)
if preview["digest"] != digest:
raise ValueError("插件包与预览不一致,请重新预览")
if preview["errors"]:
raise ValueError(";".join(preview["errors"]))
doc = preview["manifest"]
name, version = doc["name"], doc["version"]
state = read_state(name)
if state and state.get("remove"):
raise ValueError("插件正在等待卸载,请先重启")
if state and version == state["active"]:
return {"name": name, "message": "此版本已安装"}
if state and Version(version) <= Version(state["active"]):
raise ValueError("升级需要更高版本;退回旧版本请使用回退操作")
target = version_dir(name, version)
if target.exists() and any(not (target / filename).is_file() or (target / filename).read_bytes() != payload for filename, payload in files.items()):
raise ValueError("该版本目录已有其他内容,请提升版本号")
created = not target.exists()
if created:
target.parent.mkdir(parents=True, exist_ok=True)
staging = Path(tempfile.mkdtemp(prefix=".stage-", dir=target.parent))
try:
for filename, payload in files.items():
dest = staging / filename
dest.parent.mkdir(parents=True, exist_ok=True)
dest.write_bytes(payload)
staging.rename(target)
finally:
if staging.exists():
shutil.rmtree(staging)
if state:
state["hashes"][version] = digest
state["pending"] = version
write_state(name, state)
loader.refresh_external_metadata(name)
return {"name": name, "message": "新版本已暂存,重启后生效"}
# No live version exists: validate before publishing the installation record.
try:
provider, reason = build_provider(doc)
except Exception as exc:
raise ValueError("插件加载失败,请检查开发文档中的 Provider 契约") from exc
try:
write_state(name, {"schema": 1, "active": version, "hashes": {version: digest}, "enabled": True})
except OSError:
if provider is not None:
provider.close()
if created:
shutil.rmtree(target)
raise
_started.add(str(directory(name).resolve()))
loader.publish_external(doc, provider, reason)
return {"name": name, "message": "安装完成,请配置并选择启用能力"}
def referenced_by(name: str) -> list[str]:
from app.data_providers.capabilities import CAPABILITY_REGISTRY
from app.services import preferences
prefs = preferences.load()
return [cap["label"] for cap in CAPABILITY_REGISTRY if cap["field"] and prefs.get(cap["field"]) == name]
def manage(name: str, action: str) -> dict:
from . import loader
with mutation():
state = read_state(name)
if not state:
raise ValueError("外部插件不存在")
if action in ("disable", "remove"):
refs = referenced_by(name)
if refs:
raise ValueError(f"请先切换这些能力的数据源:{'、'.join(refs)}")
if action == "rollback":
if not state.get("previous"):
raise ValueError("没有可回退版本")
state["pending"] = state["previous"]
elif action == "cancel-update":
state.pop("pending", None)
elif action == "remove":
state["remove"] = True
state["enabled"] = False
elif action in ("enable", "disable"):
if state.get("remove"):
raise ValueError("插件正在等待卸载,请先重启")
state["enabled"] = action == "enable"
else:
raise ValueError("不支持的插件操作")
write_state(name, state)
if action in ("enable", "disable", "remove"):
loader.refresh_plugin(name)
else:
loader.refresh_external_metadata(name)
return {"name": name, "message": "操作完成" + (",重启后生效" if action in ("rollback", "remove") else "")}
def discover():
"""Apply pending versions once at cold discovery, before capability detection."""
if not root().exists():
return
from . import loader
for path in sorted(root().iterdir()):
if not path.is_dir() or not NAME.fullmatch(path.name) or path.is_symlink():
continue
name = path.name
try:
with mutation():
state = read_state(name)
if state is None:
continue
identity = str(path.resolve())
cold = identity not in _started
if cold and state.get("remove"):
shutil.rmtree(directory(name))
continue
if cold:
_started.add(identity)
if state.get("pending") and state.get("enabled", True):
pending = state["pending"]
try:
candidate = manifest(name, pending)
provider, _ = build_provider(candidate)
if provider:
provider.close()
state["previous"], state["active"] = state["active"], pending
state.pop("error", None)
except Exception: # noqa: BLE001 — retain the last working version
state["error"] = "新版本加载失败,已保留原版本"
state.pop("pending", None)
write_state(name, state)
doc = manifest(name)
# Never let external packages shadow built-in or user YAML providers.
if (loader.plugins_dir() / name).exists() or name == "tickflow" or any(
(loader.data_sources_dir() / f"{name}{suffix}").exists() for suffix in (".yaml", ".yml")
):
raise ValueError("插件名称冲突")
loader.refresh_plugin(name, manifest_doc=doc)
except Exception: # noqa: BLE001 — one malformed package must not block startup
loader.record_external_error(name)
+1
View File
@@ -32,6 +32,7 @@ dependencies = [
"apscheduler>=3.10",
# Config
"pyyaml>=6.0",
"packaging>=24.0", # 外部插件版本与依赖兼容范围
"python-dotenv>=1.0",
# AI(可选,但默认装上)
"openai>=1.66", # OpenAI 兼容适配器复用 openai SDK; 1.66+ 提供 Responses API
+3 -3
View File
@@ -457,7 +457,7 @@ def test_save_plugin_key_valid_persists_and_rescans(monkeypatch):
monkeypatch.setattr(
settings_api.secrets_store, "mask", lambda key, prefix=4, suffix=4: "abcd••••wxyz"
)
monkeypatch.setattr(custom_sources, "load_all", lambda: reloaded.append(1))
monkeypatch.setattr(custom_sources, "refresh_plugin", lambda name: reloaded.append(name))
monkeypatch.setattr(
custom_sources, "list_plugins", lambda: [{"name": "fuyao", "available": True}]
)
@@ -465,7 +465,7 @@ def test_save_plugin_key_valid_persists_and_rescans(monkeypatch):
assert out["ok"] is True
assert saved == {"fuyao_api_key": "good-key"} # 字段名与 provider.SECRETS_FIELD 一致
assert out["plugin_available"] is True
assert reloaded == [1] # 保存后重扫, 插件即刻可用
assert reloaded == ["fuyao"] # 只刷新当前插件,不关闭其他数据源
def test_clear_plugin_key(monkeypatch):
@@ -475,7 +475,7 @@ def test_clear_plugin_key(monkeypatch):
cleared: list = []
monkeypatch.setattr(custom_sources, "is_builtin", lambda n: n == "fuyao")
monkeypatch.setattr(settings_api.secrets_store, "clear", lambda *keys: cleared.extend(keys))
monkeypatch.setattr(custom_sources, "load_all", lambda: None)
monkeypatch.setattr(custom_sources, "refresh_plugin", lambda name: None)
monkeypatch.setattr(
custom_sources, "list_plugins", lambda: [{"name": "fuyao", "available": False}]
)
+322
View File
@@ -0,0 +1,322 @@
from __future__ import annotations
import io
import threading
import zipfile
from types import SimpleNamespace
import pytest
import yaml
from fastapi import FastAPI
from fastapi.testclient import TestClient
from app import secrets_store
from app.api.settings import PluginKeyIn, save_plugin_key
from app.custom.source_packages.routes import router
from app.data_providers.base import ProviderRequestError
from app.data_providers.custom import loader, packages
from app.data_providers.custom.managed import ManagedProvider
from app.services import preferences
CODE = '''from types import SimpleNamespace
from .client import price
name = "demo_source"
def probe_api_key(key):
return key == "valid-key", "ok" if key == "valid-key" else "invalid"
class Provider:
name = "demo_source"
config = SimpleNamespace(datasets={"realtime": {}})
def get_realtime(self):
return [{"symbol": "600000.SH", "last": price, "change_pct": 0.02}]
def close(self):
pass
'''
@pytest.fixture
def isolated(monkeypatch, tmp_path):
monkeypatch.setattr(packages.settings, "data_dir", tmp_path)
monkeypatch.setattr(loader, "plugins_dir", lambda: tmp_path / "builtins")
monkeypatch.setattr(loader, "_PROVIDERS", {})
monkeypatch.setattr(loader, "_PLUGIN_STATUS", {})
monkeypatch.setattr(packages, "_started", set())
monkeypatch.setattr(secrets_store, "_cache", None)
monkeypatch.setattr(secrets_store, "_cache_sig", None)
monkeypatch.setattr(preferences, "load", lambda: {})
return tmp_path
def bundle(version="1.0.0", *, code=CODE, extra=None, **changes):
doc = dict(name="demo_source", display_name="演示数据源", version=version,
plugin_api_version=1, entry="provider:Provider", datasets=["realtime"], runtime="none")
doc.update(changes)
buf = io.BytesIO()
with zipfile.ZipFile(buf, "w") as archive:
archive.writestr("plugin.yaml", yaml.safe_dump(doc))
archive.writestr("provider.py", code)
archive.writestr("client.py", f"price = {int(version.split('.')[0])}\n")
for name, value in (extra or {}).items():
archive.writestr(name, value)
return buf.getvalue()
def install(data):
preview, _ = packages.inspect_archive(data)
return packages.install(data, preview["digest"])
def cold_restart():
loader._PROVIDERS = {}
loader._PLUGIN_STATUS = {}
packages._started.clear()
packages.discover()
def test_inspection_does_not_execute_code(isolated):
data = bundle(code="raise RuntimeError('must not execute during preview')")
preview, _ = packages.inspect_archive(data)
assert preview["manifest"]["name"] == "demo_source"
assert not loader.names()
with pytest.raises(ValueError, match="加载失败"):
packages.install(data, preview["digest"])
assert packages.read_state("demo_source") is None
def test_install_routes_and_reload_keeps_instance(isolated):
install(bundle())
provider = loader.get_provider("demo_source")
assert provider.get_realtime()[0]["last"] == 1
assert loader.provider_has_dataset("demo_source", "realtime")
assert not loader.is_builtin("demo_source")
assert loader.list_sources() == []
loader.load_all()
assert loader.get_provider("demo_source") is provider
assert loader.list_plugins()[0]["origin"] == "external"
def test_key_first_probe_then_save_and_only_refresh_target(isolated):
other = SimpleNamespace(close=lambda: pytest.fail("unrelated provider closed"))
loader._PROVIDERS["other"] = other
install(bundle(api_key_env="DEMO_KEY"))
handle = loader.get_provider("demo_source")
with pytest.raises(ProviderRequestError):
handle.get_realtime()
assert not save_plugin_key(PluginKeyIn(plugin="demo_source", api_key="bad"))["ok"]
assert not secrets_store.load()
result = save_plugin_key(PluginKeyIn(plugin="demo_source", api_key="valid-key"))
assert result["ok"] and result["plugin_available"]
assert handle.get_realtime()[0]["last"] == 1
assert loader.get_provider("other") is other
assert secrets_store.load()["demo_source_api_key"] == "valid-key"
def test_upgrade_restart_rollback_and_failed_upgrade(isolated):
install(bundle())
install(bundle("2.0.0"))
loader.load_all()
assert loader.get_provider("demo_source").get_realtime()[0]["last"] == 1
assert loader.list_plugins()[0]["pending_version"] == "2.0.0"
cold_restart()
assert loader.get_provider("demo_source").get_realtime()[0]["last"] == 2
packages.manage("demo_source", "rollback")
cold_restart()
assert loader.get_provider("demo_source").get_realtime()[0]["last"] == 1
install(bundle("3.0.0", code="raise RuntimeError('broken')"))
cold_restart()
assert loader.get_provider("demo_source").get_realtime()[0]["last"] == 1
assert "失败" in loader.list_plugins()[0]["update_error"]
def test_same_version_collision_and_preview_digest(isolated):
data = bundle()
install(data)
assert "已安装" in install(data)["message"]
with pytest.raises(ValueError, match="相同版本"):
install(bundle(extra={"readme.md": "different"}))
with pytest.raises(ValueError, match="预览不一致"):
packages.install(bundle("2.0.0"), "0" * 64)
@pytest.mark.parametrize("filename", ["../escape.py", "/escape.py", "a\\b.py", "a/../b.py", "CON.py", "client.PY", "x.dll", "a./b.py"])
def test_archive_paths_and_payload_limits(isolated, filename):
with pytest.raises(ValueError):
packages.inspect_archive(bundle(extra={filename: "pass"}))
assert not (isolated.parent / "escape.py").exists()
def test_manifest_contract_and_dependency_compatibility(isolated):
for changes in ({"plugin_api_version": 9}, {"entry": "app.other:Provider"}, {"datasets": ["invented"]}):
with pytest.raises(ValueError):
packages.inspect_archive(bundle(**changes))
preview, _ = packages.inspect_archive(bundle(requires_python=">=99"))
assert preview["errors"]
with pytest.raises(ValueError, match="requires_python"):
packages.install(bundle(requires_python=">=99"), preview["digest"])
def test_name_conflict_and_yaml_mutation_rejected(isolated):
loader.plugins_dir().mkdir()
(loader.plugins_dir() / "demo_source").mkdir()
with pytest.raises(ValueError, match="冲突"):
install(bundle())
(loader.plugins_dir() / "demo_source").rmdir()
install(bundle())
with pytest.raises(ValueError):
loader.save_config("demo_source", {})
with pytest.raises(ValueError):
loader.delete_config("demo_source")
def test_disable_uninstall_blocks_all_routing_references(isolated, monkeypatch):
install(bundle())
monkeypatch.setattr(preferences, "load", lambda: {"depth5_data_provider": "demo_source"})
for action in ("disable", "remove"):
with pytest.raises(ValueError, match="先切换"):
packages.manage("demo_source", action)
monkeypatch.setattr(preferences, "load", lambda: {})
secrets_store.save({"demo_source_api_key": "keep-me"})
packages.manage("demo_source", "disable")
assert not loader.list_plugins()[0]["available"]
with pytest.raises(ProviderRequestError):
loader.get_provider("demo_source").get_realtime()
packages.manage("demo_source", "enable")
assert loader.list_plugins()[0]["available"]
packages.manage("demo_source", "remove")
cold_restart()
assert not loader.names()
assert not packages.directory("demo_source").exists()
assert secrets_store.load()["demo_source_api_key"] == "keep-me"
def test_handle_delays_close_until_stream_consumed():
closed = []
class Source:
def iter_daily(self):
yield 1
yield 2
def close(self):
closed.append(self)
old, new = Source(), Source()
handle = ManagedProvider(old)
stream = handle.iter_daily()
handle.replace(new)
assert not closed
assert list(stream) == [1, 2]
assert closed == [old]
handle.close()
assert closed == [old, new]
def test_handle_delays_close_during_concurrent_request():
started, finish = threading.Event(), threading.Event()
closed = []
class Source:
def get_realtime(self):
started.set()
assert finish.wait(3)
return [1]
def close(self):
closed.append(self)
old, new = Source(), Source()
handle = ManagedProvider(old)
worker = threading.Thread(target=handle.get_realtime)
worker.start()
assert started.wait(3)
handle.replace(new)
assert not closed
finish.set()
worker.join(3)
assert closed == [old]
def test_management_auth_and_upload_api(isolated, monkeypatch):
from app.services import auth
monkeypatch.setattr(auth, "is_configured", lambda: False)
app = FastAPI()
app.include_router(router)
with TestClient(app, base_url="http://localhost", client=("127.0.0.1", 50000)) as client:
assert client.post("/api/custom/source-packages/inspect", files={"file": ("source.zip", bundle())}, headers={"Origin": "https://evil.example"}).status_code == 403
preview = client.post("/api/custom/source-packages/inspect", files={"file": ("source.zip", bundle())})
assert preview.status_code == 200
result = client.post("/api/custom/source-packages/install", files={"file": ("source.zip", bundle())}, data={"digest": preview.json()["digest"]})
assert result.status_code == 200
assert client.post("/api/custom/source-packages/demo_source/manage", json={"action": "disable"}).status_code == 200
with TestClient(app, base_url="http://localhost", client=("192.168.1.3", 50000)) as client:
assert client.post("/api/custom/source-packages/inspect", files={"file": ("s.zip", bundle())}).status_code == 403
def test_valid_key_with_failed_initialization_restores_original(isolated):
code = CODE.replace(' def get_realtime(self):', ''' def __init__(self):
from app import secrets_store
if secrets_store.load().get("demo_source_api_key") == "valid-key":
raise RuntimeError("constructor failed")
def get_realtime(self):''')
install(bundle(code=code, api_key_env="DEMO_KEY"))
secrets_store.save({"demo_source_api_key": "old-key"})
loader.refresh_plugin("demo_source")
original = loader.get_provider("demo_source")
assert original.get_realtime()
result = save_plugin_key(PluginKeyIn(plugin="demo_source", api_key="valid-key"))
assert not result["ok"]
assert secrets_store.load()["demo_source_api_key"] == "old-key"
assert original.get_realtime()
assert loader.list_plugins()[0]["available"]
def test_disabled_plugin_does_not_execute_pending_version_on_start(isolated):
install(bundle())
packages.manage("demo_source", "disable")
install(bundle("2.0.0", code="raise RuntimeError('must not run')"))
cold_restart()
assert loader.list_plugins()[0]["pending_version"] == "2.0.0"
assert loader.list_plugins()[0]["version"] == "1.0.0"
assert not loader.list_plugins()[0]["available"]
def test_bad_package_does_not_hide_healthy_provider(isolated):
install(bundle())
path = packages.root() / "broken"
path.mkdir()
(path / "state.json").write_text('{"schema": 999}')
packages.discover()
assert loader.get_provider("demo_source").get_realtime()
assert not next(p for p in loader.list_plugins() if p["name"] == "broken")["available"]
assert loader.provider_has_dataset("broken", "daily")
with pytest.raises(ProviderRequestError):
loader.get_provider("broken").get_daily([])
def test_zip_symlink_is_rejected(isolated):
data = io.BytesIO(bundle())
with zipfile.ZipFile(data, "a") as archive:
info = zipfile.ZipInfo("link.py")
info.create_system = 3
info.external_attr = 0o120777 << 16
archive.writestr(info, "../outside")
with pytest.raises(ValueError, match="符号链接"):
packages.inspect_archive(data.getvalue())
def test_external_capability_matrix_marks_unconfigured_route_unusable(isolated):
from app.data_providers.capabilities import build_capability_matrix
install(bundle(api_key_env="DEMO_KEY"))
matrix = build_capability_matrix({"realtime_data_provider": "demo_source"})
realtime = next(row for row in matrix["capabilities"] if row["id"] == "realtime")
assert realtime["current"] == "demo_source"
assert not realtime["usable"]
assert realtime["pending"][0]["name"] == "demo_source"
def test_reload_picks_up_manually_changed_credentials(isolated):
install(bundle(api_key_env="DEMO_KEY"))
handle = loader.get_provider("demo_source")
assert not loader.list_plugins()[0]["available"]
secrets_store.save({"demo_source_api_key": "valid-key"})
loader.load_all()
assert loader.get_provider("demo_source") is handle
assert loader.list_plugins()[0]["available"]
secrets_store.clear("demo_source_api_key")
loader.load_all()
assert not loader.list_plugins()[0]["available"]
with pytest.raises(ProviderRequestError):
handle.get_realtime()
+2
View File
@@ -2532,6 +2532,7 @@ dependencies = [
{ name = "httpx" },
{ name = "numba", marker = "platform_machine != 'x86_64' or sys_platform != 'darwin'" },
{ name = "openai" },
{ name = "packaging" },
{ name = "pandas" },
{ name = "pillow" },
{ name = "platformdirs" },
@@ -2580,6 +2581,7 @@ requires-dist = [
{ name = "mypy", marker = "extra == 'dev'", specifier = ">=1.10" },
{ name = "numba", marker = "platform_machine != 'x86_64' or sys_platform != 'darwin'", specifier = ">=0.65.1" },
{ name = "openai", specifier = ">=1.66" },
{ name = "packaging", specifier = ">=24.0" },
{ name = "pandas", specifier = ">=2.2" },
{ name = "pillow", specifier = ">=10.0" },
{ name = "platformdirs", specifier = ">=4.0" },
+96
View File
@@ -0,0 +1,96 @@
# 外部数据源插件安装
外部安装是数据源插件机制的新入口:开发者发布 ZIP,用户在「设置 → 数据源 → 安装插件」安装,随后点击新卡片填写 API Key、验证并选择接管的数据能力。无需为每个源改前端或重新发布主程序。
## 使用流程
1. 点击数据源区域右上角的小盒子「安装插件」,选择或拖入 ZIP。
2. 查看名称、作者声明、版本、能力和兼容检查结果。预览不会执行包内 Python。
3. 确认信任来源后安装。Python 插件与后端同权限,可读取数据和凭据;这不是沙箱。
4. 安装后自动选中新卡片。需要 Key 时显示「待配置 API Key」,有效 Key 才保存到现有 `user_data/secrets.json`;运行时初始化失败恢复原凭据和实例。
5. 在详情区「选择启用能力」勾选需要接管的能力,确认当前源到新源的变化。仅修改勾选项;安装、配置 Key 都不会自动切换路由。
6. 插件提供 `test_dataset` 时可以逐项试拉;未提供时显示「未检测」。验证 Key 不代表全部接口都有权限,试拉可能消耗供应商额度。
主程序部署的后端收到包,而不是浏览器所在电脑。Docker 请持久挂载整个配置的数据目录。
## 更新、回退和卸载
- 使用同一入口上传更高版本。新版暂存,当前进程继续使用旧版;完整重启后生效,设置页「重新加载」不会应用待生效更新。
- 重启时新版初始化失败,保留原有效版本并显示错误。缺 Key 属于待配置状态,不视为代码初始化失败。
- 详情区支持取消待生效更新、停用/启用、回退上一版本;回退也在重启后生效。
- 停用或卸载前必须先切换引用该插件的所有能力,包括除权、五档和全量分钟。检查保存的路由配置,不依赖当前可用性。
- 卸载后当前实例停止接收新调用,正在执行的调用/流式迭代结束后关闭;文件在重启时删除。Key 保留,需清除时在卸载前使用原 Key 配置区的清除操作。
- 相同包重复安装不会重复注册。相同版本但不同内容拒绝覆盖,请提升版本。
- 外部包在 `data/plugins/<name>/versions/<version>/`,激活状态在 `state.json`。实际 `data` 根目录服从应用配置;升级主程序不会覆盖该目录,兼容性仍须重新检查。
管理接口复用面板登录会话,不向开放 API Token 授权。未设置密码时仅允许本机同源访问;局域网部署请先设置密码。多 worker 部署应在停止全部 worker 后统一维护/重启;首版不提供跨 worker 在线版本同步。
## 包结构与清单
ZIP 根目录必须直接包含 `plugin.yaml`,不要额外包一层目录。每包一个数据源,继续沿用现有 Provider 协议,不需要另加 `manifest.json`。
```text
my-source.zip
├── plugin.yaml
├── provider.py
├── client.py
└── README.md
```
```yaml
name: my_source # 小写字母开头,字母/数字/下划线,最多 64 字符
version: 1.0.0
plugin_api_version: 1
display_name: 我的数据源
runtime: none # none/python;不自动安装依赖
entry: provider:Provider # 相对于包的模块;不是 app.plugins.*
datasets: [realtime]
api_key_env: MY_SOURCE_API_KEY
requires_app: '>=0.3.24'
requires_python: '>=3.11'
platforms: [any] # 或 sys.platform-machine,如 linux-x86_64
requires: [] # 现有宿主发行包版本约束,例如 'httpx>=0.27'
```
`description`、`author`、`homepage`、`key_manage_url`、`billing`、`billing_note`、`usage_notes` 沿用插件元数据;链接须为 HTTPS。作者字段只是包内声明,不是签名认证。`check: provider:availability` 可选,返回 `(bool, reason)`;只做本地就绪检查,不进行慢网络操作。
外部插件独立命名空间加载,内部使用 `from .client import ...` 等相对导入。普通子包需要自己的 `__init__.py`。不要修改全局 `sys.path`,不要写入程序目录,不要依赖核心服务私有属性。构造函数应快速、无网络访问;取数在数据方法中执行。
`Provider.name`、`config.datasets` 与清单必须一致,并实现声明能力的方法及 `close()`。接口、单位、复权和分钟时区契约见 [插件开发指南](plugin-development.md)。已安装但缺凭据或运行失败的源保留标识并显示不可用,不因名字暂时消失而换源。
声明 `api_key_env` 的插件必须在 entry 模块实现 `probe_api_key(key) -> (bool, reason)`,使用候选 Key 试探而不是读取当前已保存的 Key。Provider 实例读取 `secrets_store.get_env_backed_secret('<name>_api_key', '<ENV>')`,不要按请求重新读取 Key,否则无法保证在途调用使用原凭据。异常和探测原因不得包含 Key、鉴权头或带凭据的 URL。
可选 `test_dataset(dataset, symbols)` 沿用现有接口,返回 `{provider, dataset, rows, columns, preview, error?}`。实现有界样本请求、超时、分页上限;不要因试拉触发全历史同步。
## 运行环境边界
首版支持源码包(Python、YAML、JSON、文本/Markdown、CSV),上限 20 MiB 压缩包、80 MiB 解压总量、1000 条目。拒绝路径穿越、符号链接、重复/大小写冲突条目、Windows 保留路径以及原生库、wheel、可执行文件。
可使用主程序已有且版本满足要求的依赖;纯 Python 辅助模块可随包发布并相对导入。不会自动运行 pip/npm 或修改宿主环境。额外 SDK 的原生库随包安装暂不开放;需要平台认证和独立运行支持后再扩展。已有主程序内置的 SDK 不受影响。
旧 `app/plugins/*/plugin.yaml` 仍按原入口加载,旧 HTTP YAML 配置和凭据保留。外部包不允许覆盖内置或 YAML 数据源名称。主程序升级若超出插件兼容范围,卡片显示不可用;请更新插件后再启用。
## 扩展边界与 API
- 数据源加载器负责外置目录发现、包入口解析、统一注册和受控实例释放;原有服务继续通过 `get_provider` / `provider_has_dataset` 取数。
- 安装管理通过 `app/custom/source_packages` 的后端扩展注册,不修改主应用路由入口。
- 前端管理位于 `src/custom/source-packages`,使用两个正式插槽:`settings.data-sources.toolbar`(`refresh`、`selectSource`)和 `settings.data-sources.detail-actions`(`plugin`、`refresh`)。这些是本次新增的已实现契约。
- 数据源列表、Key、试拉、路由 API 和查询缓存复用原实现,不建立另一套数据服务。
新增管理端点:
| 请求 | 用途 |
| --- | --- |
| `POST /api/custom/source-packages/inspect` | multipart `file`,静态预检,返回清单、摘要、兼容问题 |
| `POST /api/custom/source-packages/install` | multipart `file` + 预览的 `digest`,再次校验后安装/暂存更新 |
| `POST /api/custom/source-packages/{name}/manage` | JSON `action`: `enable` / `disable` / `rollback` / `cancel-update` / `remove` |
包文件先完整写入临时目录,再发布版本目录及原子状态文件。管理操作用跨进程文件数据库锁串行化;上传摘要只用于绑定预览和内容,不证明作者身份。Python 异常按插件隔离,但进程内插件无法隔离恶意行为、无限循环或原生崩溃。
## 开发示例与验收
仓库提供 [HTTP 数据源模板](../examples/data-source-plugin/provider.py) 及配套清单、client。它使用不可访问的示例域名;先替换实际接口并实现标准化,不会伪装成已可用数据源。
在模板目录执行 `zip -r ../example-http.zip plugin.yaml provider.py client.py`,或选中这三个文件压缩,确保 ZIP 根目录直接是文件。开发测试应使用假 Client,不需要真实 Key。
新增测试覆盖安全预检、安装取数、Key 先探后存、定向刷新、在途调用和迭代器释放、重启升级/回退、失败保留旧版、名称冲突、卸载引用检查和管理鉴权。发布时还应对目标操作系统的桌面产物和 Docker 镜像做安装冒烟;源码测试不能替代这些验证。
+5 -1
View File
@@ -9,6 +9,10 @@
> 无代码接入(纯 HTTP YAML 配置)请看 [custom-data-source.md](./custom-data-source.md),
> 两种方式遵循同一套内部数据契约。
## 外部安装
现在可通过「设置 → 数据源 → 安装插件」上传独立源码 ZIP。外部包格式、版本管理、兼容范围和模板见 [外部数据源插件安装](external-data-source-packages.md)。现有内置目录方式继续兼容。
## 快速上手
一个插件 = 一个目录 + 一个 `plugin.yaml` 清单:
@@ -64,7 +68,7 @@ TickFlow 的「先探后存」语义:
| `python` | 纯 Python 依赖, `pip install` | akshare、tushare |
| `none` | 无额外依赖 | 纯 HTTP API 源 |
`runtime` 字段当前仅用于 UI 展示, 实际依赖检测由 `check` 函数负责。
内置插件的 `runtime` 决定依赖安装方式,实际就绪检测由 `check` 函数负责。外部源码包只检查依赖兼容性,不执行 pip/npm 安装。
### check 函数
+4
View File
@@ -36,6 +36,10 @@
AI 在开始任务前必须通过代码搜索确认能力是否已经实现。找不到定义和测试时,应把示例视为设计规范,不得虚构导入路径或调用结果。
## 外部数据源包安装(已实现)
外置 Provider 源码包、版本状态与加载生命周期由原数据源加载器统一管理;上传/安装/版本管理 API 由 `app/custom/source_packages` 扩展注册。前端新增 `settings.data-sources.toolbar` 和 `settings.data-sources.detail-actions` 插槽,复用现有卡片、Key 和能力路由。完整契约及边界见 [外部数据源插件安装](external-data-source-packages.md)。
## 2. 二次开发分级
按升级风险从低到高选择实现方式:
+13
View File
@@ -0,0 +1,13 @@
"""Replace this URL and field mapping with your provider's documented contract."""
import httpx
BASE_URL = "https://example.invalid"
def fetch(key: str) -> list[dict]:
if BASE_URL.endswith(".invalid"):
raise ValueError("请先修改模板的接口地址")
with httpx.Client(timeout=15) as client:
response = client.get(f"{BASE_URL}/quotes", headers={"Authorization": f"Bearer {key}"})
response.raise_for_status()
return response.json()["data"]
+13
View File
@@ -0,0 +1,13 @@
name: example_http
version: 1.0.0
plugin_api_version: 1
display_name: HTTP 数据源模板
author: 插件开发者
runtime: none
entry: provider:Provider
requires_app: '>=0.3.24'
requires_python: '>=3.11'
platforms: [any]
datasets: [realtime]
api_key_env: EXAMPLE_HTTP_API_KEY
description: 开发模板,请先在 client.py 配置真实接口并完成字段转换和测试。
+32
View File
@@ -0,0 +1,32 @@
"""Example only: vendor data must already match the documented normalized schema."""
from types import SimpleNamespace
from app import secrets_store
from app.data_providers.base import ProviderRequestError
from .client import fetch
def probe_api_key(key):
try:
fetch(key)
return True, "接口可连接;其他能力权限需另行验证"
except Exception: # noqa: BLE001 — never return raw HTTP errors containing credentials
return False, "验证失败,请检查接口地址、Key、权限与网络"
class Provider:
name = "example_http"
def __init__(self):
self.config = SimpleNamespace(datasets={"realtime": {}})
self.key = secrets_store.get_env_backed_secret("example_http_api_key", "EXAMPLE_HTTP_API_KEY")
def get_realtime(self):
try:
return fetch(self.key)
except Exception as exc:
raise ProviderRequestError("行情请求失败,请检查供应商服务") from exc
def close(self):
pass
@@ -0,0 +1,90 @@
// @vitest-environment jsdom
import { act } from 'react'
import { createRoot, type Root } from 'react-dom/client'
import { QueryClient, QueryClientProvider } from '@tanstack/react-query'
import { beforeEach, afterEach, expect, it, vi } from 'vitest'
import { InstallDialog, PackageActions, PackageInstaller } from './extension'
import type { PluginDataSourceItem } from '@/lib/api'
const mocks = vi.hoisted(() => ({ inspect: vi.fn(), install: vi.fn(), manage: vi.fn(), route: vi.fn(), matrix: vi.fn(), toast: vi.fn(), close: vi.fn(), done: vi.fn() }))
vi.mock('@/lib/api', () => ({ api: { inspectSourcePackage: mocks.inspect, installSourcePackage: mocks.install, manageSourcePackage: mocks.manage, updateDataProviders: mocks.route, capabilityMatrix: mocks.matrix } }))
vi.mock('@/components/Toast', () => ({ toast: mocks.toast }))
let root: Root
let qc: QueryClient
const node = document.createElement('div')
document.body.appendChild(node)
const plugin = { name: 'demo', display_name: '演示源', origin: 'external', version: '1.0.0', available: true, enabled: true, status: '已就绪' } as PluginDataSourceItem
const preview = { manifest: { name: 'demo', display_name: '演示源', version: '1.0.0', datasets: ['daily'] }, digest: 'hash', errors: [], installed_version: null }
async function render(element = <InstallDialog onClose={mocks.close} onInstalled={mocks.done} />) {
root = createRoot(node)
qc = new QueryClient({ defaultOptions: { queries: { retry: false } } })
await act(async () => root.render(<QueryClientProvider client={qc}>{element}</QueryClientProvider>))
await act(async () => { await new Promise(resolve => setTimeout(resolve, 10)) })
}
const button = (label: string) => [...node.querySelectorAll('button')].find(el => el.textContent === label)!
async function click(el: HTMLElement) { await act(async () => el.click()) }
async function choose() {
const input = node.querySelector('input[type="file"]')!
Object.defineProperty(input, 'files', { configurable: true, value: [new File(['zip'], 'demo.zip')] })
await act(async () => input.dispatchEvent(new Event('change', { bubbles: true })))
}
beforeEach(() => {
Object.assign(globalThis, { IS_REACT_ACT_ENVIRONMENT: true })
Object.values(mocks).forEach(fn => fn.mockReset())
mocks.inspect.mockResolvedValue(preview)
mocks.install.mockResolvedValue({ name: 'demo', message: '安装完成' })
mocks.matrix.mockResolvedValue({ capabilities: [{ id: 'daily', label: '日K', field: 'daily_data_provider', current_display: '原数据源', candidates: [{ name: 'demo' }] }] })
mocks.route.mockResolvedValue({})
})
afterEach(() => { act(() => root?.unmount()); qc?.clear(); node.innerHTML = '' })
it('preview cannot execute install until the user confirms trust; successful install selects the card', async () => {
await render(); await choose()
expect(node.textContent).toContain('演示源')
expect(mocks.install).not.toHaveBeenCalled()
expect(button('确认安装').disabled).toBe(true)
await click(node.querySelector('input[type="checkbox"]')!)
await click(button('确认安装'))
expect(mocks.install).toHaveBeenCalledWith(expect.any(File), 'hash')
expect(mocks.done).toHaveBeenCalledWith('demo')
})
it('incompatible package is visible but cannot install', async () => {
mocks.inspect.mockResolvedValue({ ...preview, errors: ['Python 版本不兼容'] })
await render(); await choose(); await click(node.querySelector('input[type="checkbox"]')!)
expect(node.textContent).toContain('Python 版本不兼容')
expect(button('确认安装').disabled).toBe(true)
})
it('blocks escape and closing during installation; failure stays in the dialog', async () => {
let reject!: (e: Error) => void
mocks.install.mockImplementation(() => new Promise((_resolve, fail) => { reject = fail }))
await render(); await choose(); await click(node.querySelector('input[type="checkbox"]')!); await click(button('确认安装'))
await act(async () => document.dispatchEvent(new KeyboardEvent('keydown', { key: 'Escape', bubbles: true })))
expect(mocks.close).not.toHaveBeenCalled()
await act(async () => reject(new Error('加载失败')))
expect(node.querySelector('[role="alert"]')?.textContent).toBe('加载失败')
expect(mocks.done).not.toHaveBeenCalled()
})
it('reopening the installer clears the previous file and preview', async () => {
await render(<PackageInstaller refresh={mocks.done} selectSource={vi.fn()} />)
await click(button('安装插件')); await choose(); await click(button('取消'))
await click(button('安装插件'))
expect(node.textContent).not.toContain('静态兼容检查通过')
expect(button('确认安装').disabled).toBe(true)
})
it('only selected routing fields are changed', async () => {
await render(<PackageActions plugin={plugin} refresh={mocks.done} />)
await click(button('选择启用能力'))
await act(async () => { await new Promise(resolve => setTimeout(resolve, 10)) })
expect(button('确认切换所选能力').disabled).toBe(true)
expect(node.textContent).toContain('原数据源 → 演示源')
await click(node.querySelector('input[type="checkbox"]')!)
await click(button('确认切换所选能力'))
expect(mocks.route).toHaveBeenCalledWith({ daily_data_provider: 'demo' })
})
it('failed removal shows routing references without closing confirmation', async () => {
mocks.manage.mockRejectedValue(new Error('请先切换五档数据源'))
await render(<PackageActions plugin={plugin} refresh={mocks.done} />)
await click(button('卸载插件')); await click(button('确认'))
expect(node.textContent).toContain('请先切换五档数据源')
expect(node.querySelector('[role="dialog"]')).not.toBeNull()
})
@@ -0,0 +1,171 @@
import { useState } from 'react'
import { Package, X, Upload, Loader2 } from 'lucide-react'
import { useQuery } from '@tanstack/react-query'
import { api, type SourcePackagePreview } from '@/lib/api'
import { QK } from '@/lib/queryKeys'
import { Modal } from '@/components/Modal'
import { toast } from '@/components/Toast'
import type { FrontendExtension, FrontendSlotContextMap } from '@/extensions/types'
const button = 'inline-flex items-center justify-center gap-1.5 rounded-btn border border-border px-3 py-1.5 text-xs hover:bg-elevated disabled:opacity-50'
export function PackageInstaller({ refresh, selectSource }: FrontendSlotContextMap['settings.data-sources.toolbar']) {
const [open, setOpen] = useState(false)
return <>
<button
type="button"
aria-haspopup="dialog"
className="inline-flex h-8 shrink-0 items-center justify-center gap-1.5 whitespace-nowrap rounded-btn border border-accent/20 bg-accent/10 px-3 text-xs font-medium text-blue-700 transition-colors hover:border-accent/40 hover:bg-accent/15 focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-accent/60 focus-visible:ring-offset-2 focus-visible:ring-offset-surface dark:text-blue-400"
onClick={() => setOpen(true)}
>
<Package size={14} aria-hidden="true" />
安装插件
</button>
{open && <InstallDialog onClose={() => setOpen(false)} onInstalled={name => { refresh(); selectSource(name); setOpen(false) }} />}
</>
}
export function InstallDialog({ onClose, onInstalled }: { onClose: () => void; onInstalled: (name: string) => void }) {
const [file, setFile] = useState<File | null>(null)
const [preview, setPreview] = useState<SourcePackagePreview | null>(null)
const [busy, setBusy] = useState(false)
const [error, setError] = useState('')
const [trusted, setTrusted] = useState(false)
const matrix = useQuery({ queryKey: QK.capabilityMatrix, queryFn: api.capabilityMatrix })
const labels = new Map(matrix.data?.capabilities.map(c => [c.id, c.label]) ?? [])
async function inspect(candidate: File | undefined) {
if (!candidate || busy) return
setFile(null); setPreview(null); setTrusted(false); setError('')
if (candidate.size > 20 * 1024 * 1024) { setError('插件包不能超过 20 MiB'); return }
setBusy(true)
try { setPreview(await api.inspectSourcePackage(candidate)); setFile(candidate) }
catch (e) { setError(e instanceof Error ? e.message : '无法读取插件包') }
finally { setBusy(false) }
}
async function install() {
if (!file || !preview || !trusted || busy) return
setBusy(true); setError('')
try {
const result = await api.installSourcePackage(file, preview.digest)
toast(result.message, 'success'); onInstalled(result.name)
} catch (e) { setError(e instanceof Error ? e.message : '安装失败') }
finally { setBusy(false) }
}
return <Modal onClose={() => { if (!busy) onClose() }} labelledBy="source-package-title" closeOnBackdrop={!busy}
panelClassName="w-[94vw] max-w-xl max-h-[85vh] overflow-y-auto text-foreground bg-surface border border-border rounded-card shadow-xl p-5 space-y-4">
<div className="flex items-center justify-between gap-3">
<h2 id="source-package-title" className="text-[16px] leading-6 font-semibold text-foreground">安装数据源插件</h2>
<button className={button} aria-label="关闭" disabled={busy} onClick={onClose}><X size={16} /></button>
</div>
<p className="text-sm text-secondary">安装后会在数据源列表中显示卡片。配置 API Key 后,再选择需要启用的能力。</p>
<label className="flex flex-col items-center gap-3 rounded-card border border-dashed border-border p-6 text-sm text-secondary"
onDragOver={e => e.preventDefault()} onDrop={e => { e.preventDefault(); void inspect(e.dataTransfer.files[0]) }}>
<Upload size={24} /><span>选择或拖入 ZIP 插件包 · 最大 20 MiB</span>
<input aria-label="选择插件包" type="file" accept=".zip" disabled={busy} onChange={e => { void inspect(e.target.files?.[0]); e.target.value = '' }} className="max-w-full text-xs" />
</label>
{busy && <p role="status" className="flex gap-2 text-sm"><Loader2 size={16} className="animate-spin" />正在处理,请稍候…</p>}
{file && <p className="text-xs text-muted break-all">{file.name}</p>}
{preview && <div className="rounded-card border border-border p-4 space-y-2 text-sm">
<div className="font-semibold break-words">{preview.manifest.display_name || preview.manifest.name} <span className="font-mono text-muted">{preview.manifest.version}</span></div>
{preview.manifest.author && <p className="text-secondary">作者声明:{preview.manifest.author}</p>}
<p className="text-secondary break-words">{preview.manifest.description}</p>
<p>支持能力:{preview.manifest.datasets.map(d => labels.get(d) || d).join('、')}</p>
{preview.installed_version && <p className="text-warning">当前 {preview.installed_version} → {preview.manifest.version},升级重启后生效。</p>}
{preview.errors.map((item, i) => <p key={i} className="text-danger">{item}</p>)}
{!preview.errors.length && <p className="text-bear">静态兼容检查通过</p>}
</div>}
{error && <p role="alert" className="text-sm text-danger break-words">{error}</p>}
<label className="flex items-start gap-2 text-xs text-secondary"><input type="checkbox" checked={trusted} disabled={busy || !preview} onChange={e => setTrusted(e.target.checked)} />
<span>我信任此插件的来源。插件会在本机执行 Python 代码,可访问主程序的数据和凭据。</span></label>
<div className="flex justify-end gap-2">
<button className={button} disabled={busy} onClick={onClose}>取消</button>
<button className={`${button} bg-accent/10 text-accent`} disabled={busy || !preview || preview.errors.length > 0 || !trusted} onClick={() => void install()}>确认安装</button>
</div>
</Modal>
}
export function PackageActions({ plugin, refresh }: FrontendSlotContextMap['settings.data-sources.detail-actions']) {
const [busy, setBusy] = useState(false)
const [error, setError] = useState('')
const [confirm, setConfirm] = useState<string | null>(null)
const [routing, setRouting] = useState(false)
if (plugin.origin !== 'external') return null
async function run(action: string) {
setBusy(true); setError('')
try { const result = await api.manageSourcePackage(plugin.name, action); toast(result.message, 'success'); refresh(); setConfirm(null) }
catch (e) { setError(e instanceof Error ? e.message : '操作失败') }
finally { setBusy(false) }
}
return <div className="mt-4 rounded-card bg-elevated/40 p-3 text-xs space-y-3">
<p>外部安装 · 版本 {plugin.version} · {plugin.status}</p>
{plugin.update_error && <p className="text-danger">{plugin.update_error}</p>}
{plugin.pending_version && <p className="text-warning">版本 {plugin.pending_version} 已暂存,重启后生效。</p>}
{plugin.pending_remove ? <p className="text-warning">重启后移除插件文件;已保存的 Key 保留。</p> : <div className="flex flex-wrap gap-2">
{plugin.available && <button className={`${button} text-accent`} onClick={() => setRouting(true)}>选择启用能力</button>}
<button disabled={busy} className={button} onClick={() => void run(plugin.enabled === false ? 'enable' : 'disable')}>{plugin.enabled === false ? '启用插件' : '停用插件'}</button>
{plugin.previous_version && <button disabled={busy} className={button} onClick={() => setConfirm('rollback')}>回退到 {plugin.previous_version}</button>}
{plugin.pending_version && <button disabled={busy} className={button} onClick={() => void run('cancel-update')}>取消待生效更新</button>}
<button disabled={busy} className={`${button} text-danger`} onClick={() => setConfirm('remove')}>卸载插件</button>
<span className="self-center text-muted">更新版本:使用右上角“安装插件”上传新包</span>
</div>}
{error && <p role="alert" className="text-danger">{error}</p>}
{routing && <RoutingDialog plugin={plugin} onClose={() => setRouting(false)} onSaved={() => { refresh(); setRouting(false) }} />}
{confirm && <Modal onClose={() => { if (!busy) setConfirm(null) }} ariaLabel="确认插件操作" closeOnBackdrop={!busy}>
<div className="p-5 space-y-4 text-sm text-foreground">
<p>{confirm === 'remove' ? '卸载前需要先切换正在使用它的能力。插件文件重启后删除,Key 保留,可在 Key 配置中单独清除。' : '重启后切换到上一版本,已保存的 Key 和能力路由保留。'}</p>
{error && <p role="alert" className="text-danger">{error}</p>}
<div className="flex justify-end gap-2"><button className={button} disabled={busy} onClick={() => setConfirm(null)}>取消</button><button className={button} disabled={busy} onClick={() => void run(confirm)}>确认</button></div>
</div>
</Modal>}
</div>
}
function RoutingDialog({ plugin, onClose, onSaved }: { plugin: FrontendSlotContextMap['settings.data-sources.detail-actions']['plugin']; onClose: () => void; onSaved: () => void }) {
const matrix = useQuery({ queryKey: QK.capabilityMatrix, queryFn: api.capabilityMatrix })
const [selected, setSelected] = useState<string[]>([])
const [busy, setBusy] = useState(false)
const [error, setError] = useState('')
const [tests, setTests] = useState<Record<string, string>>({})
const caps = (matrix.data?.capabilities ?? []).filter(c => c.field && c.candidates.some(source => source.name === plugin.name))
async function save() {
setBusy(true); setError('')
try {
const changes = Object.fromEntries(caps.filter(c => selected.includes(c.id)).map(c => [c.field!, plugin.name]))
await api.updateDataProviders(changes)
toast('已更新所选能力的数据源', 'success'); onSaved()
} catch (e) { setError(e instanceof Error ? e.message : '切换失败') }
finally { setBusy(false) }
}
async function test(id: string) {
setBusy(true)
try {
const result = await api.testDataSource(plugin.name, id, ['600000.SH'])
setTests(previous => ({ ...previous, [id]: result.error || `试拉返回 ${result.rows} 行(不代表所有接口权限)` }))
} catch (e) { setTests(previous => ({ ...previous, [id]: e instanceof Error ? e.message : '试拉失败' })) }
finally { setBusy(false) }
}
return <Modal onClose={() => { if (!busy) onClose() }} ariaLabel="选择启用能力" closeOnBackdrop={!busy}>
<div className="p-5 space-y-4 text-foreground max-h-[80vh] overflow-y-auto">
<h3 className="text-[16px] leading-6 font-semibold text-foreground">选择启用能力</h3>
<p className="text-xs text-secondary">仅修改勾选项,其他能力保留原数据源。试拉可能消耗供应商额度。</p>
{matrix.isLoading && <p>正在读取能力…</p>}
{matrix.error && <p role="alert">能力列表读取失败,请关闭后重试。</p>}
{caps.map(cap => <div key={cap.id} className="rounded border border-border p-3 space-y-2 text-xs">
<label className="flex items-center gap-2"><input type="checkbox" disabled={busy} checked={selected.includes(cap.id)} onChange={e => setSelected(prev => e.target.checked ? [...prev, cap.id] : prev.filter(id => id !== cap.id))} />
{cap.label}:{cap.current_display} → {plugin.display_name}</label>
{plugin.can_test ? <button className={button} disabled={busy} onClick={() => void test(cap.id)}>试拉{cap.label}</button> : <p className="text-muted">此插件未提供能力试拉;Key 验证不代表接口权限已确认。</p>}
{tests[cap.id] && <p role="status">{tests[cap.id]}</p>}
</div>)}
{error && <p role="alert" className="text-danger text-xs">{error}</p>}
<div className="flex justify-end gap-2"><button className={button} disabled={busy} onClick={onClose}>取消</button><button className={button} disabled={busy || !selected.length} onClick={() => void save()}>确认切换所选能力</button></div>
</div>
</Modal>
}
export default {
id: 'official.source-packages', apiVersion: 1,
slots: [
{ name: 'settings.data-sources.toolbar', id: 'install', component: PackageInstaller },
{ name: 'settings.data-sources.detail-actions', id: 'manage', component: PackageActions },
],
} satisfies FrontendExtension
+3 -1
View File
@@ -1,3 +1,4 @@
import type { ComponentType } from 'react'
import type { FrontendSlotContextMap, FrontendSlotName } from './types'
import { getFrontendSlotRegistrations } from './registry'
import { ExtensionBoundary } from './ExtensionBoundary'
@@ -12,7 +13,8 @@ export function ExtensionSlot<K extends FrontendSlotName>({ name, context, compa
const registrations = getFrontendSlotRegistrations(name)
if (registrations.length === 0) return null
return registrations.map(registration => {
const SlotComponent = registration.component
// Registrations were selected by this exact slot name; preserve the context correlation.
const SlotComponent = registration.component as ComponentType<FrontendSlotContextMap[K]>
return (
<ExtensionBoundary
key={`${registration.extensionId}:${registration.id}`}
+11 -2
View File
@@ -1,10 +1,19 @@
import type { ComponentType, ReactNode } from 'react'
import type { LucideIcon } from 'lucide-react'
import type { PluginDataSourceItem } from '@/lib/api'
import type { BrandIconProps } from '@/components/BrandIcons'
export const FRONTEND_EXTENSION_API_VERSION = 1 as const
export interface FrontendSlotContextMap {
'settings.data-sources.toolbar': {
refresh: () => void
selectSource: (name: string) => void
}
'settings.data-sources.detail-actions': {
plugin: PluginDataSourceItem
refresh: () => void
}
'layout.navigation.extra': {
collapsed: boolean
pathname: string
@@ -32,12 +41,12 @@ export interface FrontendSlotContextMap {
export type FrontendSlotName = keyof FrontendSlotContextMap
export type FrontendSlotRegistration<K extends FrontendSlotName = FrontendSlotName> = {
export type FrontendSlotRegistration<K extends FrontendSlotName = FrontendSlotName> = K extends FrontendSlotName ? {
name: K
id: string
order?: number
component: ComponentType<FrontendSlotContextMap[K]>
}
} : never
export interface FrontendExtensionRoute {
id: string
+31
View File
@@ -2033,7 +2033,22 @@ export interface DataSourceItem {
}
/** 内置可选插件数据源 (plugins/ 目录, 需手动装依赖) */
export interface SourcePackagePreview {
manifest: { name: string; display_name?: string; version: string; author?: string; description?: string; datasets: string[] }
digest: string
errors: string[]
installed_version: string | null
}
export interface PluginDataSourceItem {
can_test?: boolean
origin?: 'builtin' | 'external'
version?: string
enabled?: boolean
pending_version?: string | null
previous_version?: string | null
pending_remove?: boolean
update_error?: string | null
name: string
display_name: string
datasets: string[]
@@ -2109,6 +2124,7 @@ export interface DataSourcesResponse {
}
export interface DataSourceTestResult {
error?: string
provider: string
dataset: string
rows: number
@@ -2707,6 +2723,21 @@ export const api = {
}),
preferences: () => request<Preferences>('/api/settings/preferences'),
inspectSourcePackage: (file: File) => {
const body = new FormData()
body.append('file', file)
return request<SourcePackagePreview>('/api/custom/source-packages/inspect', { method: 'POST', body, quiet: true })
},
installSourcePackage: (file: File, digest: string) => {
const body = new FormData()
body.append('file', file)
body.append('digest', digest)
return request<{ name: string; message: string }>('/api/custom/source-packages/install', { method: 'POST', body, quiet: true, timeoutMs: 120_000 })
},
manageSourcePackage: (name: string, action: string) =>
request<{ name: string; message: string }>(`/api/custom/source-packages/${encodeURIComponent(name)}/manage`, {
method: 'POST', body: JSON.stringify({ action }), quiet: true,
}),
dataSources: () => request<DataSourcesResponse>('/api/settings/data-sources'),
capabilityMatrix: () => request<CapabilityMatrix>('/api/settings/capability-matrix'),
dataSource: (name: string) => request<CustomSourceConfig>(`/api/settings/data-sources/${encodeURIComponent(name)}`),
+32 -23
View File
@@ -1,4 +1,5 @@
import { useState } from 'react'
import { ExtensionSlot } from '@/extensions/ExtensionSlot'
import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query'
import { motion, AnimatePresence } from 'framer-motion'
import {
@@ -441,7 +442,7 @@ function PluginKeyConfig({ plugin }: { plugin: PluginDataSourceItem }) {
) : (
<>向该数据源官方申请 API Key。</>
)}
Key 仅存本地 (secrets.json 优先, 环境变量 {plugin.api_key_env} 兜底),不会上传任何第三方,请妥善保管。
Key 保存在本地,验证和取数时发送给该数据源用于认证。环境变量可作为后备配置。
</p>
{/* 积分计费警示框 + Key/积分管理入口 (manifest billing/billing_note/key_manage_url 声明) */}
@@ -462,7 +463,7 @@ function PluginKeyConfig({ plugin }: { plugin: PluginDataSourceItem }) {
rel="noreferrer"
className="inline-flex items-center gap-1 text-xs text-accent hover:underline mb-4"
>
管理 Key / 查看积分明细 (通达信商城)
管理 Key / 查看账户用量
<ExternalLink className="h-3 w-3" />
</a>
)}
@@ -734,10 +735,10 @@ export function SettingsDataSourcesPanel({ highlight }: { highlight?: string } =
{/* ===== 下区: 数据源 (源为主视图, 配置/接入) ===== */}
<AnchorWrap highlight={highlight} anchor="data-sources">
<section className="rounded-card border border-border bg-surface p-5">
<div className="flex items-center justify-between mb-4">
<div className="mb-4 flex flex-wrap items-center justify-between gap-x-4 gap-y-3">
<div className="flex items-center gap-2.5 min-w-0">
<Database className="h-4 w-4 text-secondary shrink-0" />
<h2 className="text-sm font-medium text-foreground">数据源</h2>
<h2 className="shrink-0 text-sm font-medium text-foreground">数据源</h2>
<span
className="text-[10px] text-muted/40 font-mono truncate hidden lg:inline max-w-[480px]"
title={sources.data?.config_dir}
@@ -745,20 +746,24 @@ export function SettingsDataSourcesPanel({ highlight }: { highlight?: string } =
{sources.data?.config_dir}
</span>
</div>
<button
onClick={() => reload.mutate()}
disabled={reload.isPending}
className="inline-flex items-center gap-1.5 px-2.5 py-1 rounded-btn text-xs text-muted hover:text-foreground hover:bg-elevated transition-colors disabled:opacity-50 shrink-0"
>
<RefreshCw className={`h-3 w-3 ${reload.isPending ? 'animate-spin' : ''}`} />
重新加载
</button>
<div className="ml-auto flex shrink-0 items-center gap-2">
<ExtensionSlot name="settings.data-sources.toolbar" context={{ refresh: invalidateSources, selectSource: setSelected }} compact />
<button
type="button"
onClick={() => reload.mutate()}
disabled={reload.isPending}
className="inline-flex h-8 shrink-0 items-center justify-center gap-1.5 whitespace-nowrap rounded-btn border border-border/70 bg-surface px-3 text-xs font-medium text-secondary transition-colors hover:border-border hover:bg-elevated hover:text-foreground focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-accent/60 focus-visible:ring-offset-2 focus-visible:ring-offset-surface disabled:pointer-events-none disabled:opacity-50"
>
<RefreshCw aria-hidden="true" className={`h-3.5 w-3.5 ${reload.isPending ? 'animate-spin' : ''}`} />
重新加载
</button>
</div>
</div>
{/* 插件化说明 (置顶黄色提示条): 接入自有行情 → 把文档交给 AI */}
<div className="mb-4 flex items-start gap-2 rounded-lg border border-warning/40 bg-warning/10 px-3 py-2.5">
<Puzzle className="mt-0.5 h-3.5 w-3.5 shrink-0 text-warning" />
<div className="text-[11px] leading-relaxed text-muted">
{/* 开发说明是辅助信息, 警告色留给需要处理的异常状态。 */}
<div className="mb-4 flex items-start gap-2 rounded-btn bg-elevated/40 px-3 py-2.5">
<Puzzle className="mt-0.5 h-3.5 w-3.5 shrink-0 text-muted" />
<div className="min-w-0 break-words text-[11px] leading-relaxed text-secondary">
<span className="text-secondary">数据源已插件化</span>
,接入自有行情?把文档发给 AI 即可自动接入:
<span className="mx-0.5 rounded bg-elevated/70 px-1 py-px font-mono text-[10px] text-secondary">docs/custom-data-source.md</span>
@@ -794,8 +799,8 @@ export function SettingsDataSourcesPanel({ highlight }: { highlight?: string } =
<div
key={item.name}
onClick={() => {
// 未就绪插件仅当支持界面配 Key 时可点开(进详情配置); 其余不可选
if (pluginUnavailable && !plugin?.api_key_env) return
// 外部插件始终可打开管理详情;内置插件沿用 Key 配置入口。
if (pluginUnavailable && !plugin?.api_key_env && plugin?.origin !== 'external') return
setSelected(item.name)
// 只有用户自定义源 (YAML) 才进编辑器; tickflow 和插件不可编辑
if (customNames.has(item.name)) {
@@ -803,7 +808,7 @@ export function SettingsDataSourcesPanel({ highlight }: { highlight?: string } =
}
}}
className={`relative text-left rounded-lg border px-3.5 py-3 transition-all ${
pluginUnavailable && !plugin?.api_key_env
pluginUnavailable && !plugin?.api_key_env && plugin?.origin !== 'external'
? 'border-border/40 bg-elevated/10 opacity-70'
: isSelected
? 'border-accent/50 bg-accent/5 ring-1 ring-accent/20 cursor-pointer'
@@ -848,7 +853,7 @@ export function SettingsDataSourcesPanel({ highlight }: { highlight?: string } =
{pluginUnavailable ? (
// 安装/卸载按钮只对"纯依赖型"插件(runtime≠none 且无 api_key_env)显示;
// 带 Key 配置的插件不可用时, 卡片本身可点击进详情配 Key(如 TdxQuant 依赖已内置)。
plugin?.runtime !== 'none' && !plugin?.api_key_env && (
plugin?.origin !== 'external' && plugin?.runtime !== 'none' && !plugin?.api_key_env && (
installing ? (
<span className="inline-flex items-center gap-1 text-[10px] text-accent">
<RefreshCw className="h-2.5 w-2.5 animate-spin" /> 安装中...
@@ -872,7 +877,7 @@ export function SettingsDataSourcesPanel({ highlight }: { highlight?: string } =
>
套用
</button>
{plugin && plugin?.runtime !== 'none' && !plugin?.api_key_env && (
{plugin && plugin?.origin !== 'external' && plugin?.runtime !== 'none' && !plugin?.api_key_env && (
uninstalling ? (
<RefreshCw className="h-2.5 w-2.5 animate-spin text-muted" />
) : (
@@ -977,6 +982,7 @@ export function SettingsDataSourcesPanel({ highlight }: { highlight?: string } =
isActive={servingDatasets(selected).length > 0}
matrixCaps={matrixCaps}
servingSet={servingSetOf(selected)}
refresh={invalidateSources}
/>
) : null}
</motion.div>
@@ -1017,7 +1023,8 @@ export function SettingsDataSourcesPanel({ highlight }: { highlight?: string } =
}
/** 插件详情: 介绍 + 适配能力 + Key 配置合并为单一卡片 */
function PluginDetail({ plugin, isActive, matrixCaps, servingSet }: {
function PluginDetail({ plugin, isActive, matrixCaps, servingSet, refresh }: {
refresh: () => void
plugin: PluginDataSourceItem
isActive: boolean
matrixCaps: CapabilityRoute[]
@@ -1065,6 +1072,8 @@ function PluginDetail({ plugin, isActive, matrixCaps, servingSet }: {
</div>
</div>
<ExtensionSlot name="settings.data-sources.detail-actions" context={{ plugin, refresh }} />
{/* 主体: 操作 + Key 配置 (左) | 能力适配表 (右), 布局对齐 TickFlow 详情 */}
<div className="mt-5 pt-5 border-t border-border grid grid-cols-1 lg:grid-cols-[1fr_1.15fr] gap-6 items-start">
<div className="min-w-0">
@@ -1076,7 +1085,7 @@ function PluginDetail({ plugin, isActive, matrixCaps, servingSet }: {
)}
{/* API Key 配置 (声明了 api_key_env 的插件) */}
{plugin.api_key_env && (
{plugin.api_key_env && plugin.enabled !== false && (
<div className={plugin.available ? 'mt-4' : ''}>
<PluginKeyConfig plugin={plugin} />
</div>