mirror of
https://github.com/shy3130/tick-stock-panel.git
synced 2026-11-02 02:16:13 +00:00
feat(data-sources): 支持外部插件安装与版本管理
Signed-off-by: shy3130 <shy3130@qq.com>
This commit is contained in:
@@ -623,6 +623,7 @@ PORT=3018 # 服务端口
|
||||
| [docs/mining.md](./docs/mining.md) | 因子与策略挖掘口径、防泄漏、任务隔离和发布边界 |
|
||||
| [docs/market-phase.md](./docs/market-phase.md) | 市场情绪周期 6 阶段与概念/行业主线识别的口径与设计 |
|
||||
| [docs/plugin-development.md](./docs/plugin-development.md) | 数据源插件开发规范(以 fuyao 为参考实现) |
|
||||
| [外部数据源插件安装](./docs/external-data-source-packages.md) | ZIP 安装、Key 配置、版本维护和开发模板 |
|
||||
| [docs/secondary-development.md](./docs/secondary-development.md) | 代码二次开发、前端插槽、后端策略接口与 AI 开发模板 |
|
||||
| [docs/page-sdk.md](./docs/page-sdk.md) | 页面插件开发指南:产物契约、能力桥 ctx、打包与上传、安全边界 |
|
||||
| [backend/app/strategy/prompts/strategy-guide.md](./backend/app/strategy/prompts/strategy-guide.md) | 策略开发完整规范(AI 生成与手写) |
|
||||
|
||||
@@ -757,7 +757,7 @@ def save_plugin_key(req: PluginKeyIn) -> dict:
|
||||
"""保存插件 API Key(先探后存, 对齐 /tickflow-key 语义)。
|
||||
|
||||
流程: probe_plugin_key 用候选 Key 实探 → 有效才写 secrets.json
|
||||
({plugin}_api_key, 优先级高于 .env) → load_all 重扫, 插件即刻变为可切换。
|
||||
({plugin}_api_key, 优先级高于 .env) → 定向刷新当前插件。
|
||||
"""
|
||||
from app.data_providers import custom as custom_sources
|
||||
|
||||
@@ -765,11 +765,27 @@ def save_plugin_key(req: PluginKeyIn) -> dict:
|
||||
key = req.api_key.strip()
|
||||
if not key:
|
||||
return {"ok": False, "error": "key empty"}
|
||||
ok, message = custom_sources.probe_plugin_key(name, key)
|
||||
if not ok:
|
||||
return {"ok": False, "reason": "invalid", "error": message}
|
||||
secrets_store.save({f"{name}_api_key": key})
|
||||
custom_sources.load_all()
|
||||
from contextlib import nullcontext
|
||||
|
||||
from app.data_providers.custom import packages
|
||||
external = packages.read_state(name) is not None
|
||||
with packages.mutation() if external else nullcontext():
|
||||
ok, message = custom_sources.probe_plugin_key(name, key)
|
||||
if not ok:
|
||||
return {"ok": False, "reason": "invalid", "error": message}
|
||||
previous = secrets_store.load().get(f"{name}_api_key") if external else None
|
||||
secrets_store.save({f"{name}_api_key": key})
|
||||
try:
|
||||
if external:
|
||||
custom_sources.refresh_plugin(name, strict=True)
|
||||
else:
|
||||
custom_sources.refresh_plugin(name)
|
||||
except ValueError:
|
||||
if previous is None:
|
||||
secrets_store.clear(f"{name}_api_key")
|
||||
else:
|
||||
secrets_store.save({f"{name}_api_key": previous})
|
||||
return {"ok": False, "error": "插件初始化失败,已保留原配置"}
|
||||
status = next((p for p in custom_sources.list_plugins() if p["name"] == name), None)
|
||||
return {
|
||||
"ok": True,
|
||||
@@ -785,12 +801,12 @@ def clear_plugin_key(name: str) -> dict:
|
||||
from app.data_providers import custom as custom_sources
|
||||
|
||||
manifest = custom_sources.plugin_manifest(name)
|
||||
if manifest is None or not custom_sources.is_builtin(name):
|
||||
if manifest is None:
|
||||
raise HTTPException(status_code=404, detail=f"插件 '{name}' 不存在")
|
||||
if not manifest.get("api_key_env"):
|
||||
raise HTTPException(status_code=400, detail=f"插件 '{name}' 不支持在界面配置 Key")
|
||||
secrets_store.clear(f"{name.lower()}_api_key")
|
||||
custom_sources.load_all()
|
||||
custom_sources.refresh_plugin(name)
|
||||
status = next((p for p in custom_sources.list_plugins() if p["name"] == name), None)
|
||||
return {
|
||||
"ok": True,
|
||||
|
||||
@@ -0,0 +1,10 @@
|
||||
"""External data-source package management, registered through existing L2 routes."""
|
||||
from app.extensions import BACKEND_EXTENSION_API_VERSION, BackendExtensionRegistrar
|
||||
|
||||
EXTENSION_ID = "official.source-packages"
|
||||
EXTENSION_API_VERSION = BACKEND_EXTENSION_API_VERSION
|
||||
|
||||
|
||||
def setup(registrar: BackendExtensionRegistrar):
|
||||
from .routes import router
|
||||
registrar.include_router(router)
|
||||
@@ -0,0 +1,87 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from contextlib import asynccontextmanager
|
||||
from typing import Annotated
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
import yaml
|
||||
from fastapi import APIRouter, Depends, File, Form, HTTPException, Request, UploadFile
|
||||
from pydantic import BaseModel
|
||||
|
||||
from app.api.auth import COOKIE_NAME, _client_ip
|
||||
from app.data_providers.custom import packages
|
||||
from app.services import auth
|
||||
|
||||
|
||||
def require_management(request: Request):
|
||||
# Unlike read-only APIs, uploading Python is code execution. LAN access without
|
||||
# a password is not enough. No API-token scope grants this permission.
|
||||
if request.headers.get("authorization"):
|
||||
raise HTTPException(403, "插件安装仅供面板管理会话使用")
|
||||
origin = request.headers.get("origin")
|
||||
if origin and urlsplit(origin).netloc != request.headers.get("host"):
|
||||
raise HTTPException(403, "插件管理需要同源访问")
|
||||
if request.headers.get("sec-fetch-site") == "cross-site":
|
||||
raise HTTPException(403, "拒绝跨站插件管理请求")
|
||||
if auth.is_configured():
|
||||
if not auth.is_valid_session(request.cookies.get(COOKIE_NAME, "")):
|
||||
raise HTTPException(401, "请先登录")
|
||||
elif _client_ip(request) not in ("127.0.0.1", "::1", "localhost") or request.url.hostname not in ("127.0.0.1", "::1", "localhost"):
|
||||
raise HTTPException(403, "请先设置访问密码并登录,或通过本机访问安装插件")
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(_app):
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
from app.data_providers import custom
|
||||
for plugin in custom.list_plugins():
|
||||
if plugin.get("origin") == "external":
|
||||
custom.get_provider(plugin["name"]).close()
|
||||
|
||||
|
||||
router = APIRouter(lifespan=lifespan, prefix="/api/custom/source-packages", tags=["source-packages"], dependencies=[Depends(require_management)])
|
||||
|
||||
|
||||
async def read_upload(file: UploadFile):
|
||||
try:
|
||||
data = await file.read(packages.MAX_UPLOAD + 1)
|
||||
if len(data) > packages.MAX_UPLOAD:
|
||||
raise HTTPException(413, "插件包超过 20 MiB")
|
||||
return data
|
||||
finally:
|
||||
await file.close()
|
||||
|
||||
|
||||
@router.post("/inspect")
|
||||
async def inspect(file: Annotated[UploadFile, File()]):
|
||||
data = await read_upload(file)
|
||||
from starlette.concurrency import run_in_threadpool
|
||||
try:
|
||||
preview, _ = await run_in_threadpool(packages.inspect_archive, data)
|
||||
return preview
|
||||
except (ValueError, OSError, TypeError, yaml.YAMLError) as exc:
|
||||
raise HTTPException(400, str(exc)) from exc
|
||||
|
||||
|
||||
@router.post("/install")
|
||||
async def install(file: Annotated[UploadFile, File()], digest: str = Form(...)):
|
||||
from starlette.concurrency import run_in_threadpool
|
||||
data = await read_upload(file)
|
||||
try:
|
||||
return await run_in_threadpool(packages.install, data, digest)
|
||||
except (ValueError, OSError, TypeError, yaml.YAMLError) as exc:
|
||||
raise HTTPException(400, str(exc)) from exc
|
||||
|
||||
|
||||
class Action(BaseModel):
|
||||
action: str
|
||||
|
||||
|
||||
@router.post("/{name}/manage")
|
||||
def manage(name: str, body: Action):
|
||||
try:
|
||||
return packages.manage(name, body.action)
|
||||
except (ValueError, OSError, TypeError, yaml.YAMLError) as exc:
|
||||
raise HTTPException(400, str(exc)) from exc
|
||||
@@ -16,6 +16,7 @@ from app.data_providers.custom.loader import (
|
||||
plugin_manifest,
|
||||
probe_plugin_key,
|
||||
provider_has_dataset,
|
||||
refresh_plugin,
|
||||
save_config,
|
||||
uninstall_plugin,
|
||||
)
|
||||
@@ -37,6 +38,7 @@ __all__ = [
|
||||
"plugin_manifest",
|
||||
"probe_plugin_key",
|
||||
"provider_has_dataset",
|
||||
"refresh_plugin",
|
||||
"save_config",
|
||||
"uninstall_plugin",
|
||||
]
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
"""Load custom data source definitions from user data files."""
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import importlib
|
||||
import logging
|
||||
import math
|
||||
@@ -8,11 +9,13 @@ import re
|
||||
import shutil
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
import yaml
|
||||
|
||||
from app import secrets_store
|
||||
from app.config import settings
|
||||
from app.data_providers.base import ProviderRequestError
|
||||
from app.data_providers.custom.config import (
|
||||
DEFAULT_TIMEOUT,
|
||||
MAX_TIMEOUT,
|
||||
@@ -20,6 +23,7 @@ from app.data_providers.custom.config import (
|
||||
config_from_dict,
|
||||
load_config,
|
||||
)
|
||||
from app.data_providers.custom.managed import ManagedProvider
|
||||
from app.data_providers.custom.provider import GenericHTTPProvider
|
||||
from app.services.fs_utils import atomic_write_text
|
||||
|
||||
@@ -47,9 +51,12 @@ def data_sources_dir() -> Path:
|
||||
def load_all(path: Path | None = None) -> None:
|
||||
"""Load all custom provider YAML files into process memory."""
|
||||
global _PROVIDERS, _LOAD_ERRORS
|
||||
for provider in _PROVIDERS.values():
|
||||
provider.close()
|
||||
_PROVIDERS = {}
|
||||
retained = {name: provider for name, provider in _PROVIDERS.items()
|
||||
if _PLUGIN_STATUS.get(name, {}).get("origin") == "external"}
|
||||
for name, provider in _PROVIDERS.items():
|
||||
if name not in retained:
|
||||
provider.close()
|
||||
_PROVIDERS = dict(retained)
|
||||
_LOAD_ERRORS = []
|
||||
|
||||
base = path or data_sources_dir()
|
||||
@@ -88,7 +95,7 @@ def list_sources() -> list[dict]:
|
||||
|
||||
def list_plugins() -> list[dict]:
|
||||
"""返回所有内置插件的状态 (含已装/未装), 供设置页独立分类显示。"""
|
||||
return list(_PLUGIN_STATUS.values())
|
||||
return [dict(item) for item in _PLUGIN_STATUS.values()]
|
||||
|
||||
|
||||
def _plugin_key_masked(name: str, api_key_env: str) -> str:
|
||||
@@ -106,7 +113,13 @@ def _plugin_key_masked(name: str, api_key_env: str) -> str:
|
||||
|
||||
def plugin_manifest(name: str) -> dict | None:
|
||||
"""读取指定插件的 plugin.yaml 清单。"""
|
||||
plugin_dir = plugins_dir() / (name or "")
|
||||
if not _NAME_RE.fullmatch(name or ""):
|
||||
return None
|
||||
from . import packages
|
||||
external = packages.manifest(name)
|
||||
if external is not None:
|
||||
return external
|
||||
plugin_dir = plugins_dir() / name
|
||||
manifest_path = plugin_dir / "plugin.yaml"
|
||||
if not manifest_path.exists():
|
||||
return None
|
||||
@@ -133,16 +146,24 @@ def probe_plugin_key(name: str, api_key: str) -> tuple[bool, str]:
|
||||
if ":" not in entry:
|
||||
return False, f"插件 '{name}' entry 非法"
|
||||
try:
|
||||
module = importlib.import_module(entry.split(":", 1)[0])
|
||||
except Exception as e:
|
||||
return False, f"插件模块加载失败: {e}"
|
||||
if _PLUGIN_STATUS.get(name, {}).get("origin") == "external":
|
||||
from . import packages
|
||||
if not packages.read_state(name).get("enabled", True):
|
||||
return False, "请先启用插件"
|
||||
module = packages.import_module(manifest, entry.split(":", 1)[0])
|
||||
else:
|
||||
module = importlib.import_module(entry.split(":", 1)[0])
|
||||
except Exception: # noqa: BLE001 — do not disclose plugin exception payloads
|
||||
return False, "插件模块加载失败,请检查依赖与兼容范围"
|
||||
probe = getattr(module, "probe_api_key", None)
|
||||
if probe is None:
|
||||
return False, f"插件 '{name}' 未提供 Key 探测"
|
||||
try:
|
||||
return probe(api_key)
|
||||
except Exception as e:
|
||||
return False, f"探测失败: {e}"
|
||||
ok, reason = probe(api_key)
|
||||
return bool(ok), (str(reason).replace(api_key, "[已隐藏]")
|
||||
if _PLUGIN_STATUS.get(name, {}).get("origin") == "external" else str(reason))
|
||||
except Exception: # noqa: BLE001 — plugin exceptions may contain credentials
|
||||
return False, "Key 探测失败,请检查凭据、网络与接口权限"
|
||||
|
||||
|
||||
def install_plugin(name: str) -> tuple[bool, str]:
|
||||
@@ -284,7 +305,8 @@ def uninstall_plugin(name: str) -> tuple[bool, str]:
|
||||
|
||||
def is_builtin(name: str) -> bool:
|
||||
"""判断 name 是否为内置插件 (不可被用户编辑/删除)。"""
|
||||
return (name or "").lower() in _PLUGIN_STATUS
|
||||
return ((name or "").lower() in _PLUGIN_STATUS
|
||||
and _PLUGIN_STATUS[(name or "").lower()].get("origin") != "external")
|
||||
|
||||
|
||||
def names() -> set[str]:
|
||||
@@ -330,7 +352,7 @@ def provider_has_dataset(name: str, dataset: str) -> bool:
|
||||
|
||||
def get_config_dict(name: str) -> dict | None:
|
||||
"""读取一个已加载 custom 源的原始配置 dict(用于前端编辑回填)。内置插件不可编辑。"""
|
||||
if is_builtin(name):
|
||||
if name in _PLUGIN_STATUS:
|
||||
return None
|
||||
provider = _PROVIDERS.get((name or "").lower())
|
||||
if provider is None:
|
||||
@@ -375,7 +397,7 @@ def _config_to_dict(config: CustomSourceConfig) -> dict:
|
||||
|
||||
def save_config(name: str, config: dict) -> Path:
|
||||
"""把一份配置 dict 写成 data/data_sources/{name}.yaml, 返回写入路径。"""
|
||||
if is_builtin(name):
|
||||
if name in _PLUGIN_STATUS:
|
||||
raise ValueError(f"'{name}' 是内置插件, 不可编辑")
|
||||
if not _NAME_RE.match(name or ""):
|
||||
raise ValueError(f"invalid data source name: {name!r} (only lowercase a-z 0-9 _ allowed)")
|
||||
@@ -391,7 +413,7 @@ def save_config(name: str, config: dict) -> Path:
|
||||
|
||||
def delete_config(name: str) -> bool:
|
||||
"""删除 data/data_sources/{name}.yaml。返回是否真的删除了。"""
|
||||
if is_builtin(name):
|
||||
if name in _PLUGIN_STATUS:
|
||||
raise ValueError(f"'{name}' 是内置插件, 不可删除")
|
||||
if not _NAME_RE.match(name or ""):
|
||||
raise ValueError(f"invalid data source name: {name!r}")
|
||||
@@ -536,11 +558,10 @@ def _load_builtin_plugins() -> None:
|
||||
每次调用重建 _PLUGIN_STATUS, 并把可用的插件注册进 _PROVIDERS。
|
||||
"""
|
||||
global _PLUGIN_STATUS
|
||||
_PLUGIN_STATUS = {}
|
||||
_PLUGIN_STATUS = {name: info for name, info in _PLUGIN_STATUS.items()
|
||||
if info.get("origin") == "external"}
|
||||
pdir = plugins_dir()
|
||||
if not pdir.exists():
|
||||
return
|
||||
for plugin_dir in sorted(pdir.iterdir()):
|
||||
for plugin_dir in sorted(pdir.iterdir()) if pdir.exists() else []:
|
||||
if not plugin_dir.is_dir():
|
||||
continue
|
||||
manifest_path = plugin_dir / "plugin.yaml"
|
||||
@@ -551,23 +572,16 @@ def _load_builtin_plugins() -> None:
|
||||
_register_one_plugin(manifest)
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.warning("插件 %s 清单解析失败: %s", plugin_dir.name, e)
|
||||
from . import packages
|
||||
packages.discover()
|
||||
|
||||
|
||||
def _register_one_plugin(manifest: dict) -> None:
|
||||
"""注册单个插件: 委托自检 → 可用则动态 import entry 注册进 _PROVIDERS。"""
|
||||
name = manifest.get("name")
|
||||
if not name or not _NAME_RE.match(name):
|
||||
logger.warning("插件清单缺少合法 name: %r", name)
|
||||
return
|
||||
# hidden: 已加载但对 UI 隐藏 (功能未完成/暂不开放), 不注册不展示
|
||||
if manifest.get("hidden"):
|
||||
logger.info("插件 %s 标记为 hidden, 跳过注册", name)
|
||||
return
|
||||
def _manifest_status(manifest, available, reason, origin="builtin"):
|
||||
name = manifest["name"]
|
||||
runtime = str(manifest.get("runtime", "none")).lower()
|
||||
# 委托检测: 调用插件自己的 check 函数 (node 型/python 型各自实现)
|
||||
available, reason = _call_check(manifest.get("check"))
|
||||
_PLUGIN_STATUS[name] = {
|
||||
return {
|
||||
"name": name,
|
||||
"origin": origin,
|
||||
"display_name": manifest.get("display_name", name),
|
||||
"datasets": list(manifest.get("datasets", []) or []),
|
||||
"runtime": runtime,
|
||||
@@ -587,14 +601,37 @@ def _register_one_plugin(manifest: dict) -> None:
|
||||
# Key/账务管理页 (可选): 设置页 Key 配置区展示入口
|
||||
"key_manage_url": manifest.get("key_manage_url", ""),
|
||||
}
|
||||
|
||||
|
||||
def _register_one_plugin(manifest: dict) -> None:
|
||||
"""注册单个插件: 委托自检 → 可用则动态 import entry 注册进 _PROVIDERS。"""
|
||||
name = manifest.get("name")
|
||||
if not name or not _NAME_RE.match(name):
|
||||
logger.warning("插件清单缺少合法 name: %r", name)
|
||||
return
|
||||
# hidden: 已加载但对 UI 隐藏 (功能未完成/暂不开放), 不注册不展示
|
||||
if manifest.get("hidden"):
|
||||
logger.info("插件 %s 标记为 hidden, 跳过注册", name)
|
||||
return
|
||||
runtime = str(manifest.get("runtime", "none")).lower()
|
||||
# 委托检测: 调用插件自己的 check 函数 (node 型/python 型各自实现)
|
||||
available, reason = _call_check(manifest.get("check"))
|
||||
_PLUGIN_STATUS[name] = _manifest_status(manifest, available, reason)
|
||||
if not available:
|
||||
return # 依赖没装: 不注册, 但状态已记录供 UI 显示
|
||||
old = _PROVIDERS.pop(name, None)
|
||||
if old is not None:
|
||||
old.close()
|
||||
return # 不就绪时撤下旧实例,避免旧凭据继续生效
|
||||
# 可用 → 动态加载 provider 类并实例化
|
||||
try:
|
||||
provider_cls = _load_entry(manifest["entry"])
|
||||
provider = provider_cls() if isinstance(provider_cls, type) else provider_cls
|
||||
provider.builtin = True # 标记为内置 (list_sources 过滤, 不可被用户编辑/删除)
|
||||
_PROVIDERS[name] = provider
|
||||
old = _PROVIDERS.get(name)
|
||||
if isinstance(old, ManagedProvider):
|
||||
old.replace(provider)
|
||||
else:
|
||||
_PROVIDERS[name] = ManagedProvider(provider)
|
||||
logger.info("内置插件 %s 已注册 (runtime=%s)", name, runtime)
|
||||
except Exception as e: # noqa: BLE001
|
||||
# 声称可用但 import 失败 → 标记不可用, 避免启动崩溃
|
||||
@@ -630,5 +667,100 @@ def _load_entry(entry_ref: str):
|
||||
return getattr(module, attr)
|
||||
|
||||
|
||||
class _UnavailableProvider:
|
||||
"""Keep installed identities so preferences cannot silently switch vendors."""
|
||||
|
||||
def __init__(self, doc, reason):
|
||||
self.name = doc["name"]
|
||||
self.config = SimpleNamespace(datasets=dict.fromkeys(doc.get("datasets", [])))
|
||||
self.reason = reason
|
||||
|
||||
def close(self):
|
||||
pass
|
||||
|
||||
def __getattr__(self, name):
|
||||
if not name.startswith(("get_", "iter_")):
|
||||
raise AttributeError(name)
|
||||
def unavailable(*args, **kwargs):
|
||||
raise ProviderRequestError(f"数据源 {self.name} 不可用:{self.reason}")
|
||||
return unavailable
|
||||
|
||||
|
||||
def record_external_error(name):
|
||||
if name in _PROVIDERS and _PLUGIN_STATUS.get(name, {}).get("origin") != "external":
|
||||
logger.warning("external plugin name conflict: %s", name)
|
||||
return
|
||||
from .packages import DATASETS
|
||||
doc = {"name": name, "datasets": sorted(DATASETS), "version": ""}
|
||||
publish_external(doc, None, "插件加载失败,请检查清单与兼容范围", metadata=False)
|
||||
|
||||
|
||||
def publish_external(doc, provider, reason, *, metadata=True):
|
||||
name = doc["name"]
|
||||
available = provider is not None
|
||||
value = provider if available else _UnavailableProvider(doc, reason)
|
||||
old = _PROVIDERS.get(name)
|
||||
if isinstance(old, ManagedProvider):
|
||||
old.replace(value)
|
||||
else:
|
||||
_PROVIDERS[name] = ManagedProvider(value)
|
||||
_PLUGIN_STATUS[name] = {
|
||||
**_manifest_status(doc, available, reason, origin="external"),
|
||||
"version": doc.get("version", ""), "author": doc.get("author", ""),
|
||||
"can_test": available and callable(getattr(provider, "test_dataset", None)),
|
||||
}
|
||||
if metadata:
|
||||
_PROVIDERS[name]._external_signature = _external_signature(doc)
|
||||
refresh_external_metadata(name)
|
||||
|
||||
|
||||
def _external_signature(doc):
|
||||
from . import packages
|
||||
state = packages.read_state(doc["name"]) or {}
|
||||
key = secrets_store.get_env_backed_secret(f"{doc['name']}_api_key", doc.get("api_key_env", ""))
|
||||
return (doc.get("version"), state.get("enabled", True), hashlib.sha256(key.encode()).digest())
|
||||
|
||||
|
||||
def refresh_external_metadata(name):
|
||||
from . import packages
|
||||
state = packages.read_state(name)
|
||||
if state and name in _PLUGIN_STATUS:
|
||||
_PLUGIN_STATUS[name].update({
|
||||
"pending_version": state.get("pending"), "previous_version": state.get("previous"),
|
||||
"enabled": state.get("enabled", True), "pending_remove": state.get("remove", False),
|
||||
"update_error": state.get("error"),
|
||||
})
|
||||
|
||||
|
||||
def refresh_plugin(name, *, manifest_doc=None, strict=False):
|
||||
"""Refresh one plugin without closing other providers or in-flight requests."""
|
||||
from . import packages
|
||||
doc = manifest_doc or plugin_manifest(name)
|
||||
if doc is None:
|
||||
raise ValueError("插件不存在")
|
||||
state = packages.read_state(name)
|
||||
if state is None:
|
||||
_register_one_plugin(doc)
|
||||
return
|
||||
if (manifest_doc is not None and name in _PROVIDERS
|
||||
and _PLUGIN_STATUS.get(name, {}).get("origin") == "external"
|
||||
and getattr(_PROVIDERS[name], "_external_signature", None) == _external_signature(doc)):
|
||||
refresh_external_metadata(name)
|
||||
return
|
||||
if not state.get("enabled", True):
|
||||
publish_external(doc, None, "待重启卸载" if state.get("remove") else "已停用")
|
||||
return
|
||||
try:
|
||||
provider, reason = packages.build_provider(doc)
|
||||
if strict and provider is None:
|
||||
raise ValueError("插件配置未就绪")
|
||||
except Exception: # noqa: BLE001 — uploaded exceptions can contain credentials
|
||||
if strict:
|
||||
raise ValueError("插件初始化失败,原配置保持不变") from None
|
||||
publish_external(doc, None, "插件加载失败,请检查依赖与 Provider 实现")
|
||||
return
|
||||
publish_external(doc, provider, reason)
|
||||
|
||||
|
||||
# 模块导入时即扫描一次, 保证 names()/_allowed_data_providers() 在 startup 前可用。
|
||||
_load_builtin_plugins()
|
||||
|
||||
@@ -0,0 +1,102 @@
|
||||
"""Keep a provider alive until its last call / streaming iterator completes."""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import threading
|
||||
from collections.abc import Iterator
|
||||
|
||||
from app.data_providers.base import ProviderRequestError
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class _Stream(Iterator):
|
||||
def __init__(self, iterator, release):
|
||||
self.iterator, self.release = iterator, release
|
||||
|
||||
def __next__(self):
|
||||
try:
|
||||
return next(self.iterator)
|
||||
except BaseException:
|
||||
self.close()
|
||||
raise
|
||||
|
||||
def close(self):
|
||||
if self.release is not None:
|
||||
release, self.release = self.release, None
|
||||
try:
|
||||
close = getattr(self.iterator, "close", None)
|
||||
if close:
|
||||
close()
|
||||
finally:
|
||||
release()
|
||||
|
||||
def __del__(self):
|
||||
self.close()
|
||||
|
||||
|
||||
class ManagedProvider:
|
||||
"""Stable handle; swaps do not close an in-flight provider, including iter_daily."""
|
||||
|
||||
builtin = True
|
||||
|
||||
def __init__(self, provider):
|
||||
self._lock = threading.RLock()
|
||||
self._current = [provider, 0, False]
|
||||
|
||||
def _release(self, entry):
|
||||
with self._lock:
|
||||
entry[1] -= 1
|
||||
close = entry[2] and entry[1] == 0
|
||||
if close:
|
||||
self._close(entry[0])
|
||||
|
||||
@staticmethod
|
||||
def _close(provider):
|
||||
try:
|
||||
provider.close()
|
||||
except Exception: # noqa: BLE001 — cleanup cannot invalidate the replacement
|
||||
logger.warning("provider close failed", exc_info=False)
|
||||
|
||||
def replace(self, provider):
|
||||
with self._lock:
|
||||
old = self._current
|
||||
self._current = [provider, 0, False]
|
||||
old[2] = True
|
||||
close = old[1] == 0
|
||||
if close:
|
||||
self._close(old[0])
|
||||
|
||||
def close(self):
|
||||
with self._lock:
|
||||
old = self._current
|
||||
if old[2]:
|
||||
return
|
||||
old[2] = True
|
||||
close = old[1] == 0
|
||||
if close:
|
||||
self._close(old[0])
|
||||
|
||||
def __getattr__(self, name):
|
||||
with self._lock:
|
||||
value = getattr(self._current[0], name)
|
||||
if not callable(value):
|
||||
return value
|
||||
|
||||
def invoke(*args, **kwargs):
|
||||
with self._lock:
|
||||
entry = self._current
|
||||
if entry[2]:
|
||||
raise ProviderRequestError("数据源已停止,请重新选择数据源")
|
||||
method = getattr(entry[0], name)
|
||||
entry[1] += 1
|
||||
try:
|
||||
result = method(*args, **kwargs)
|
||||
except BaseException:
|
||||
self._release(entry)
|
||||
raise
|
||||
if isinstance(result, Iterator):
|
||||
return _Stream(result, lambda: self._release(entry))
|
||||
self._release(entry)
|
||||
return result
|
||||
return invoke
|
||||
@@ -0,0 +1,428 @@
|
||||
"""External provider packages. Static inspection never imports uploaded code.
|
||||
|
||||
The existing loader owns provider instances. This module owns immutable package
|
||||
versions and activation state only; it never installs into the host environment.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import importlib
|
||||
import importlib.metadata
|
||||
import io
|
||||
import json
|
||||
import platform
|
||||
import re
|
||||
import shutil
|
||||
import sqlite3
|
||||
import stat
|
||||
import sys
|
||||
import tempfile
|
||||
import types
|
||||
import zipfile
|
||||
from contextlib import contextmanager
|
||||
from pathlib import Path, PurePosixPath
|
||||
|
||||
import yaml
|
||||
from packaging.requirements import Requirement
|
||||
from packaging.specifiers import SpecifierSet
|
||||
from packaging.version import Version
|
||||
|
||||
from app import __version__
|
||||
from app.config import settings
|
||||
from app.services.fs_utils import atomic_write_text
|
||||
|
||||
MAX_UPLOAD = 20 * 1024 * 1024
|
||||
MAX_EXPANDED = 80 * 1024 * 1024
|
||||
NAME = re.compile(r"[a-z][a-z0-9_]{0,63}\Z")
|
||||
VERSION = re.compile(r"\d+\.\d+\.\d+(?:[a-zA-Z0-9.+-]*)\Z")
|
||||
ENTRY = re.compile(r"[a-zA-Z_]\w*(?:\.[a-zA-Z_]\w*)*:[a-zA-Z_]\w*\Z")
|
||||
DATASETS = {"daily", "adj_factor", "minute", "full_minute", "realtime", "depth5", "financial"}
|
||||
# Once per process: never activate a staged update during a settings reload.
|
||||
_started: set[str] = set()
|
||||
|
||||
|
||||
def root() -> Path:
|
||||
return settings.data_dir / "plugins"
|
||||
|
||||
|
||||
def directory(name: str) -> Path:
|
||||
if not NAME.fullmatch(name):
|
||||
raise ValueError("插件标识非法")
|
||||
path = root() / name
|
||||
if path.is_symlink() or root().is_symlink():
|
||||
raise ValueError("插件目录不能是符号链接")
|
||||
return path
|
||||
|
||||
|
||||
@contextmanager
|
||||
def mutation():
|
||||
root().mkdir(parents=True, exist_ok=True)
|
||||
conn = sqlite3.connect(root() / ".management.sqlite", timeout=5)
|
||||
try:
|
||||
conn.execute("BEGIN EXCLUSIVE")
|
||||
yield
|
||||
finally:
|
||||
conn.rollback()
|
||||
conn.close()
|
||||
|
||||
|
||||
def read_state(name: str) -> dict | None:
|
||||
if not NAME.fullmatch(name):
|
||||
return None
|
||||
path = directory(name) / "state.json"
|
||||
if not path.exists():
|
||||
return None
|
||||
state = json.loads(path.read_text(encoding="utf-8"))
|
||||
if state.get("schema") != 1:
|
||||
raise ValueError("不支持的插件安装状态版本")
|
||||
return state
|
||||
|
||||
|
||||
def write_state(name: str, state: dict):
|
||||
directory(name).mkdir(parents=True, exist_ok=True)
|
||||
atomic_write_text(directory(name) / "state.json", json.dumps(state, ensure_ascii=False))
|
||||
|
||||
|
||||
def version_dir(name: str, version: str) -> Path:
|
||||
if not VERSION.fullmatch(version):
|
||||
raise ValueError("插件版本非法")
|
||||
path = directory(name) / "versions" / version
|
||||
if path.is_symlink() or path.parent.is_symlink():
|
||||
raise ValueError("插件版本目录不能是符号链接")
|
||||
return path
|
||||
|
||||
|
||||
def manifest(name: str, version: str | None = None) -> dict | None:
|
||||
state = read_state(name)
|
||||
if state is None:
|
||||
return None
|
||||
version = version or state["active"]
|
||||
doc = yaml.safe_load((version_dir(name, version) / "plugin.yaml").read_text("utf-8"))
|
||||
validate_manifest(doc)
|
||||
if doc["name"] != name or doc["version"] != version:
|
||||
raise ValueError("插件清单与安装记录不一致")
|
||||
return doc
|
||||
|
||||
|
||||
def validate_manifest(doc: dict):
|
||||
if not isinstance(doc, dict):
|
||||
raise ValueError("plugin.yaml 必须是对象")
|
||||
if not NAME.fullmatch(str(doc.get("name", ""))):
|
||||
raise ValueError("name 必须是小写字母开头的字母、数字、下划线")
|
||||
if not VERSION.fullmatch(str(doc.get("version", ""))):
|
||||
raise ValueError("version 必须是有效版本,例如 1.0.0")
|
||||
Version(doc["version"])
|
||||
if type(doc.get("plugin_api_version")) is not int or doc["plugin_api_version"] != 1:
|
||||
raise ValueError("仅支持 plugin_api_version: 1")
|
||||
for field in ("entry", "check"):
|
||||
if (field == "entry" or doc.get(field)) and not ENTRY.fullmatch(str(doc.get(field, ""))):
|
||||
raise ValueError(f"{field} 应使用包内相对模块引用,例如 provider:MyProvider")
|
||||
datasets = doc.get("datasets")
|
||||
if not isinstance(datasets, list) or not datasets or any(not isinstance(x, str) or x not in DATASETS for x in datasets):
|
||||
raise ValueError("datasets 必须声明已支持的数据集")
|
||||
for key in ("display_name", "description", "api_key_env", "homepage", "key_manage_url", "author", "billing", "billing_note", "install_hint"):
|
||||
if key in doc and (not isinstance(doc[key], str) or len(doc[key]) > 4000):
|
||||
raise ValueError(f"{key} 必须是文本且不超过 4000 字符")
|
||||
if doc.get("api_key_env") and not re.fullmatch(r"[A-Z][A-Z0-9_]{0,127}", doc["api_key_env"]):
|
||||
raise ValueError("api_key_env 必须是大写环境变量名")
|
||||
for key in ("homepage", "key_manage_url"):
|
||||
if doc.get(key) and not doc[key].startswith("https://"):
|
||||
raise ValueError(f"{key} 必须使用 HTTPS")
|
||||
if "usage_notes" in doc and (not isinstance(doc["usage_notes"], list) or any(not isinstance(x, str) for x in doc["usage_notes"])):
|
||||
raise ValueError("usage_notes 必须是文本列表")
|
||||
if doc.get("runtime", "none") not in ("none", "python"):
|
||||
raise ValueError("外部安装暂只支持 none/python 运行时,不安装额外运行时")
|
||||
if doc.get("hidden"):
|
||||
raise ValueError("外部安装的插件不能隐藏卡片")
|
||||
|
||||
|
||||
def compatibility(doc: dict) -> list[str]:
|
||||
errors = []
|
||||
for field, current in (("requires_app", __version__), ("requires_python", platform.python_version())):
|
||||
if doc.get(field) and Version(current) not in SpecifierSet(str(doc[field])):
|
||||
errors.append(f"{field}: 需要 {doc[field]},当前 {current}")
|
||||
platforms = doc.get("platforms", ["any"])
|
||||
current = f"{sys.platform}-{platform.machine().lower()}"
|
||||
if not isinstance(platforms, list) or not all(isinstance(x, str) for x in platforms):
|
||||
raise ValueError("platforms 必须是字符串列表")
|
||||
if "any" not in platforms and current not in platforms:
|
||||
errors.append(f"运行平台不匹配,当前 {current}")
|
||||
requirements = doc.get("requires", [])
|
||||
if not isinstance(requirements, list):
|
||||
raise ValueError("requires 必须是依赖列表")
|
||||
for item in requirements:
|
||||
req = Requirement(item)
|
||||
if req.url:
|
||||
raise ValueError("requires 不接受 URL 或安装脚本")
|
||||
if req.marker and not req.marker.evaluate():
|
||||
continue
|
||||
try:
|
||||
installed = importlib.metadata.version(req.name)
|
||||
if installed not in req.specifier:
|
||||
errors.append(f"依赖 {req.name} 版本不匹配:{installed}")
|
||||
except importlib.metadata.PackageNotFoundError:
|
||||
errors.append(f"缺少依赖 {req.name}(不会自动修改主程序环境)")
|
||||
return errors
|
||||
|
||||
|
||||
def inspect_archive(data: bytes) -> tuple[dict, dict[str, bytes]]:
|
||||
if len(data) > MAX_UPLOAD:
|
||||
raise ValueError("插件包超过 20 MiB")
|
||||
files = {}
|
||||
folded = set()
|
||||
total = 0
|
||||
try:
|
||||
with zipfile.ZipFile(io.BytesIO(data)) as archive:
|
||||
if len(archive.infolist()) > 1000:
|
||||
raise ValueError("插件包条目超过 1000")
|
||||
for info in archive.infolist():
|
||||
name = info.filename
|
||||
parts = PurePosixPath(name).parts
|
||||
if (not parts or name.startswith("/") or "\\" in name or ":" in name
|
||||
or any(p in (".", "..") or p.endswith((".", " ")) for p in parts)
|
||||
or PurePosixPath(name).as_posix() != name.rstrip("/")
|
||||
or any(p.split(".")[0].upper() in {"CON", "PRN", "AUX", "NUL", *[f"COM{i}" for i in range(10)], *[f"LPT{i}" for i in range(10)]} for p in parts)):
|
||||
raise ValueError("插件包包含非法路径")
|
||||
if stat.S_IFMT(info.external_attr >> 16) not in (0, stat.S_IFREG, stat.S_IFDIR):
|
||||
raise ValueError("插件包不允许符号链接或特殊文件")
|
||||
if name.casefold().rstrip("/") in folded:
|
||||
raise ValueError("插件包包含重复或大小写冲突的路径")
|
||||
folded.add(name.casefold().rstrip("/"))
|
||||
if info.is_dir():
|
||||
continue
|
||||
total += info.file_size
|
||||
if total > MAX_EXPANDED or info.file_size > MAX_UPLOAD:
|
||||
raise ValueError("插件包解压大小超限")
|
||||
# Executable/native payloads need a separately verified distribution.
|
||||
if PurePosixPath(name).suffix.lower() not in (".py", ".yaml", ".yml", ".json", ".txt", ".md", ".csv"):
|
||||
raise ValueError("首版支持源码包;不接收原生库、wheel、脚本或可执行文件")
|
||||
files[name] = archive.read(info)
|
||||
except (zipfile.BadZipFile, RuntimeError, NotImplementedError) as exc:
|
||||
raise ValueError("无法读取 ZIP 插件包") from exc
|
||||
if "plugin.yaml" not in files or len(files["plugin.yaml"]) > 64 * 1024:
|
||||
raise ValueError("包根目录需要 plugin.yaml(最大 64 KiB)")
|
||||
doc = yaml.safe_load(files["plugin.yaml"])
|
||||
validate_manifest(doc)
|
||||
for field in ("entry", "check"):
|
||||
if doc.get(field):
|
||||
module = doc[field].split(":")[0].replace(".", "/")
|
||||
if f"{module}.py" not in files and f"{module}/__init__.py" not in files:
|
||||
raise ValueError(f"{field} 指定的模块不存在")
|
||||
for name, payload in files.items():
|
||||
if name.endswith(".py"):
|
||||
try:
|
||||
compile(payload, name, "exec")
|
||||
except (SyntaxError, ValueError) as exc:
|
||||
raise ValueError(f"Python 文件语法错误: {name}") from exc
|
||||
errors = compatibility(doc)
|
||||
from . import loader
|
||||
if (loader.plugins_dir() / doc["name"]).exists() or doc["name"] == "tickflow" or any(
|
||||
(loader.data_sources_dir() / f"{doc['name']}{suffix}").exists() for suffix in (".yaml", ".yml")
|
||||
):
|
||||
raise ValueError("插件名称与内置插件或 HTTP 数据源冲突")
|
||||
state = read_state(doc["name"])
|
||||
digest = hashlib.sha256(data).hexdigest()
|
||||
if state and doc["version"] in state["hashes"] and state["hashes"][doc["version"]] != digest:
|
||||
raise ValueError("相同版本已有不同内容,请提升版本号")
|
||||
return {"manifest": doc, "digest": digest, "errors": errors,
|
||||
"installed_version": state["active"] if state else None}, files
|
||||
|
||||
|
||||
def import_module(doc: dict, module: str):
|
||||
path = version_dir(doc["name"], doc["version"])
|
||||
# Root-specific namespace keeps different installations and tests independent.
|
||||
contents = b"".join(file.read_bytes() for file in sorted(path.rglob("*.py")))
|
||||
digest = hashlib.sha256(str(path.resolve()).encode() + contents).hexdigest()[:20]
|
||||
namespace = f"_tsp_provider_{digest}"
|
||||
if namespace not in sys.modules:
|
||||
package = types.ModuleType(namespace)
|
||||
package.__path__ = [str(path)]
|
||||
package.__package__ = namespace
|
||||
sys.modules[namespace] = package
|
||||
return importlib.import_module(f"{namespace}.{module}")
|
||||
|
||||
|
||||
def load_entry(doc: dict, entry: str):
|
||||
module, attr = entry.split(":")
|
||||
return getattr(import_module(doc, module), attr)
|
||||
|
||||
|
||||
def build_provider(doc: dict):
|
||||
issues = compatibility(doc)
|
||||
if issues:
|
||||
raise ValueError(";".join(issues))
|
||||
factory = load_entry(doc, doc["entry"])
|
||||
if not callable(factory):
|
||||
raise ValueError("entry 必须指向 Provider 类或工厂")
|
||||
if doc.get("api_key_env"):
|
||||
module = import_module(doc, doc["entry"].split(":")[0])
|
||||
if not callable(getattr(module, "probe_api_key", None)):
|
||||
raise ValueError("声明 API Key 的插件必须提供 probe_api_key")
|
||||
from app import secrets_store
|
||||
if doc.get("api_key_env") and not secrets_store.get_env_backed_secret(f"{doc['name']}_api_key", doc["api_key_env"]):
|
||||
return None, "待配置 API Key"
|
||||
if doc.get("check"):
|
||||
check = load_entry(doc, doc["check"])()
|
||||
ok, reason = check if isinstance(check, tuple) else (check, "不可用")
|
||||
if not ok:
|
||||
key = secrets_store.get_env_backed_secret(f"{doc['name']}_api_key", doc.get("api_key_env", ""))
|
||||
return None, str(reason).replace(key, "[已隐藏]") if key else str(reason)
|
||||
provider = factory()
|
||||
try:
|
||||
if provider.name != doc["name"] or set(provider.config.datasets) != set(doc["datasets"]):
|
||||
raise ValueError("Provider 名称/能力声明与清单不一致")
|
||||
required = {"daily": "get_daily", "adj_factor": "get_adj_factors", "minute": "get_minute", "realtime": "get_realtime", "depth5": "get_depth_batch", "financial": "get_financials"}
|
||||
for dataset in doc["datasets"]:
|
||||
methods = ("get_intraday_batch", "get_minute") if dataset == "full_minute" else (required[dataset],)
|
||||
if not any(callable(getattr(provider, method, None)) for method in methods):
|
||||
raise ValueError(f"缺少 {dataset} 数据方法")
|
||||
if not callable(getattr(provider, "close", None)):
|
||||
raise ValueError("Provider 缺少 close 方法")
|
||||
except Exception:
|
||||
close = getattr(provider, "close", None)
|
||||
if callable(close):
|
||||
close()
|
||||
raise
|
||||
return provider, "已就绪"
|
||||
|
||||
|
||||
def install(data: bytes, digest: str) -> dict:
|
||||
from . import loader
|
||||
with mutation():
|
||||
preview, files = inspect_archive(data)
|
||||
if preview["digest"] != digest:
|
||||
raise ValueError("插件包与预览不一致,请重新预览")
|
||||
if preview["errors"]:
|
||||
raise ValueError(";".join(preview["errors"]))
|
||||
doc = preview["manifest"]
|
||||
name, version = doc["name"], doc["version"]
|
||||
state = read_state(name)
|
||||
if state and state.get("remove"):
|
||||
raise ValueError("插件正在等待卸载,请先重启")
|
||||
if state and version == state["active"]:
|
||||
return {"name": name, "message": "此版本已安装"}
|
||||
if state and Version(version) <= Version(state["active"]):
|
||||
raise ValueError("升级需要更高版本;退回旧版本请使用回退操作")
|
||||
target = version_dir(name, version)
|
||||
if target.exists() and any(not (target / filename).is_file() or (target / filename).read_bytes() != payload for filename, payload in files.items()):
|
||||
raise ValueError("该版本目录已有其他内容,请提升版本号")
|
||||
created = not target.exists()
|
||||
if created:
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
staging = Path(tempfile.mkdtemp(prefix=".stage-", dir=target.parent))
|
||||
try:
|
||||
for filename, payload in files.items():
|
||||
dest = staging / filename
|
||||
dest.parent.mkdir(parents=True, exist_ok=True)
|
||||
dest.write_bytes(payload)
|
||||
staging.rename(target)
|
||||
finally:
|
||||
if staging.exists():
|
||||
shutil.rmtree(staging)
|
||||
if state:
|
||||
state["hashes"][version] = digest
|
||||
state["pending"] = version
|
||||
write_state(name, state)
|
||||
loader.refresh_external_metadata(name)
|
||||
return {"name": name, "message": "新版本已暂存,重启后生效"}
|
||||
# No live version exists: validate before publishing the installation record.
|
||||
try:
|
||||
provider, reason = build_provider(doc)
|
||||
except Exception as exc:
|
||||
raise ValueError("插件加载失败,请检查开发文档中的 Provider 契约") from exc
|
||||
try:
|
||||
write_state(name, {"schema": 1, "active": version, "hashes": {version: digest}, "enabled": True})
|
||||
except OSError:
|
||||
if provider is not None:
|
||||
provider.close()
|
||||
if created:
|
||||
shutil.rmtree(target)
|
||||
raise
|
||||
_started.add(str(directory(name).resolve()))
|
||||
loader.publish_external(doc, provider, reason)
|
||||
return {"name": name, "message": "安装完成,请配置并选择启用能力"}
|
||||
|
||||
|
||||
def referenced_by(name: str) -> list[str]:
|
||||
from app.data_providers.capabilities import CAPABILITY_REGISTRY
|
||||
from app.services import preferences
|
||||
prefs = preferences.load()
|
||||
return [cap["label"] for cap in CAPABILITY_REGISTRY if cap["field"] and prefs.get(cap["field"]) == name]
|
||||
|
||||
|
||||
def manage(name: str, action: str) -> dict:
|
||||
from . import loader
|
||||
with mutation():
|
||||
state = read_state(name)
|
||||
if not state:
|
||||
raise ValueError("外部插件不存在")
|
||||
if action in ("disable", "remove"):
|
||||
refs = referenced_by(name)
|
||||
if refs:
|
||||
raise ValueError(f"请先切换这些能力的数据源:{'、'.join(refs)}")
|
||||
if action == "rollback":
|
||||
if not state.get("previous"):
|
||||
raise ValueError("没有可回退版本")
|
||||
state["pending"] = state["previous"]
|
||||
elif action == "cancel-update":
|
||||
state.pop("pending", None)
|
||||
elif action == "remove":
|
||||
state["remove"] = True
|
||||
state["enabled"] = False
|
||||
elif action in ("enable", "disable"):
|
||||
if state.get("remove"):
|
||||
raise ValueError("插件正在等待卸载,请先重启")
|
||||
state["enabled"] = action == "enable"
|
||||
else:
|
||||
raise ValueError("不支持的插件操作")
|
||||
write_state(name, state)
|
||||
if action in ("enable", "disable", "remove"):
|
||||
loader.refresh_plugin(name)
|
||||
else:
|
||||
loader.refresh_external_metadata(name)
|
||||
return {"name": name, "message": "操作完成" + (",重启后生效" if action in ("rollback", "remove") else "")}
|
||||
|
||||
|
||||
def discover():
|
||||
"""Apply pending versions once at cold discovery, before capability detection."""
|
||||
if not root().exists():
|
||||
return
|
||||
from . import loader
|
||||
for path in sorted(root().iterdir()):
|
||||
if not path.is_dir() or not NAME.fullmatch(path.name) or path.is_symlink():
|
||||
continue
|
||||
name = path.name
|
||||
try:
|
||||
with mutation():
|
||||
state = read_state(name)
|
||||
if state is None:
|
||||
continue
|
||||
identity = str(path.resolve())
|
||||
cold = identity not in _started
|
||||
if cold and state.get("remove"):
|
||||
shutil.rmtree(directory(name))
|
||||
continue
|
||||
if cold:
|
||||
_started.add(identity)
|
||||
if state.get("pending") and state.get("enabled", True):
|
||||
pending = state["pending"]
|
||||
try:
|
||||
candidate = manifest(name, pending)
|
||||
provider, _ = build_provider(candidate)
|
||||
if provider:
|
||||
provider.close()
|
||||
state["previous"], state["active"] = state["active"], pending
|
||||
state.pop("error", None)
|
||||
except Exception: # noqa: BLE001 — retain the last working version
|
||||
state["error"] = "新版本加载失败,已保留原版本"
|
||||
state.pop("pending", None)
|
||||
write_state(name, state)
|
||||
doc = manifest(name)
|
||||
# Never let external packages shadow built-in or user YAML providers.
|
||||
if (loader.plugins_dir() / name).exists() or name == "tickflow" or any(
|
||||
(loader.data_sources_dir() / f"{name}{suffix}").exists() for suffix in (".yaml", ".yml")
|
||||
):
|
||||
raise ValueError("插件名称冲突")
|
||||
loader.refresh_plugin(name, manifest_doc=doc)
|
||||
except Exception: # noqa: BLE001 — one malformed package must not block startup
|
||||
loader.record_external_error(name)
|
||||
@@ -32,6 +32,7 @@ dependencies = [
|
||||
"apscheduler>=3.10",
|
||||
# Config
|
||||
"pyyaml>=6.0",
|
||||
"packaging>=24.0", # 外部插件版本与依赖兼容范围
|
||||
"python-dotenv>=1.0",
|
||||
# AI(可选,但默认装上)
|
||||
"openai>=1.66", # OpenAI 兼容适配器复用 openai SDK; 1.66+ 提供 Responses API
|
||||
|
||||
@@ -457,7 +457,7 @@ def test_save_plugin_key_valid_persists_and_rescans(monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
settings_api.secrets_store, "mask", lambda key, prefix=4, suffix=4: "abcd••••wxyz"
|
||||
)
|
||||
monkeypatch.setattr(custom_sources, "load_all", lambda: reloaded.append(1))
|
||||
monkeypatch.setattr(custom_sources, "refresh_plugin", lambda name: reloaded.append(name))
|
||||
monkeypatch.setattr(
|
||||
custom_sources, "list_plugins", lambda: [{"name": "fuyao", "available": True}]
|
||||
)
|
||||
@@ -465,7 +465,7 @@ def test_save_plugin_key_valid_persists_and_rescans(monkeypatch):
|
||||
assert out["ok"] is True
|
||||
assert saved == {"fuyao_api_key": "good-key"} # 字段名与 provider.SECRETS_FIELD 一致
|
||||
assert out["plugin_available"] is True
|
||||
assert reloaded == [1] # 保存后重扫, 插件即刻可用
|
||||
assert reloaded == ["fuyao"] # 只刷新当前插件,不关闭其他数据源
|
||||
|
||||
|
||||
def test_clear_plugin_key(monkeypatch):
|
||||
@@ -475,7 +475,7 @@ def test_clear_plugin_key(monkeypatch):
|
||||
cleared: list = []
|
||||
monkeypatch.setattr(custom_sources, "is_builtin", lambda n: n == "fuyao")
|
||||
monkeypatch.setattr(settings_api.secrets_store, "clear", lambda *keys: cleared.extend(keys))
|
||||
monkeypatch.setattr(custom_sources, "load_all", lambda: None)
|
||||
monkeypatch.setattr(custom_sources, "refresh_plugin", lambda name: None)
|
||||
monkeypatch.setattr(
|
||||
custom_sources, "list_plugins", lambda: [{"name": "fuyao", "available": False}]
|
||||
)
|
||||
|
||||
@@ -0,0 +1,322 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
import threading
|
||||
import zipfile
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import yaml
|
||||
from fastapi import FastAPI
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from app import secrets_store
|
||||
from app.api.settings import PluginKeyIn, save_plugin_key
|
||||
from app.custom.source_packages.routes import router
|
||||
from app.data_providers.base import ProviderRequestError
|
||||
from app.data_providers.custom import loader, packages
|
||||
from app.data_providers.custom.managed import ManagedProvider
|
||||
from app.services import preferences
|
||||
|
||||
CODE = '''from types import SimpleNamespace
|
||||
from .client import price
|
||||
name = "demo_source"
|
||||
def probe_api_key(key):
|
||||
return key == "valid-key", "ok" if key == "valid-key" else "invalid"
|
||||
class Provider:
|
||||
name = "demo_source"
|
||||
config = SimpleNamespace(datasets={"realtime": {}})
|
||||
def get_realtime(self):
|
||||
return [{"symbol": "600000.SH", "last": price, "change_pct": 0.02}]
|
||||
def close(self):
|
||||
pass
|
||||
'''
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def isolated(monkeypatch, tmp_path):
|
||||
monkeypatch.setattr(packages.settings, "data_dir", tmp_path)
|
||||
monkeypatch.setattr(loader, "plugins_dir", lambda: tmp_path / "builtins")
|
||||
monkeypatch.setattr(loader, "_PROVIDERS", {})
|
||||
monkeypatch.setattr(loader, "_PLUGIN_STATUS", {})
|
||||
monkeypatch.setattr(packages, "_started", set())
|
||||
monkeypatch.setattr(secrets_store, "_cache", None)
|
||||
monkeypatch.setattr(secrets_store, "_cache_sig", None)
|
||||
monkeypatch.setattr(preferences, "load", lambda: {})
|
||||
return tmp_path
|
||||
|
||||
|
||||
def bundle(version="1.0.0", *, code=CODE, extra=None, **changes):
|
||||
doc = dict(name="demo_source", display_name="演示数据源", version=version,
|
||||
plugin_api_version=1, entry="provider:Provider", datasets=["realtime"], runtime="none")
|
||||
doc.update(changes)
|
||||
buf = io.BytesIO()
|
||||
with zipfile.ZipFile(buf, "w") as archive:
|
||||
archive.writestr("plugin.yaml", yaml.safe_dump(doc))
|
||||
archive.writestr("provider.py", code)
|
||||
archive.writestr("client.py", f"price = {int(version.split('.')[0])}\n")
|
||||
for name, value in (extra or {}).items():
|
||||
archive.writestr(name, value)
|
||||
return buf.getvalue()
|
||||
|
||||
|
||||
def install(data):
|
||||
preview, _ = packages.inspect_archive(data)
|
||||
return packages.install(data, preview["digest"])
|
||||
|
||||
|
||||
def cold_restart():
|
||||
loader._PROVIDERS = {}
|
||||
loader._PLUGIN_STATUS = {}
|
||||
packages._started.clear()
|
||||
packages.discover()
|
||||
|
||||
|
||||
def test_inspection_does_not_execute_code(isolated):
|
||||
data = bundle(code="raise RuntimeError('must not execute during preview')")
|
||||
preview, _ = packages.inspect_archive(data)
|
||||
assert preview["manifest"]["name"] == "demo_source"
|
||||
assert not loader.names()
|
||||
with pytest.raises(ValueError, match="加载失败"):
|
||||
packages.install(data, preview["digest"])
|
||||
assert packages.read_state("demo_source") is None
|
||||
|
||||
|
||||
def test_install_routes_and_reload_keeps_instance(isolated):
|
||||
install(bundle())
|
||||
provider = loader.get_provider("demo_source")
|
||||
assert provider.get_realtime()[0]["last"] == 1
|
||||
assert loader.provider_has_dataset("demo_source", "realtime")
|
||||
assert not loader.is_builtin("demo_source")
|
||||
assert loader.list_sources() == []
|
||||
loader.load_all()
|
||||
assert loader.get_provider("demo_source") is provider
|
||||
assert loader.list_plugins()[0]["origin"] == "external"
|
||||
|
||||
|
||||
def test_key_first_probe_then_save_and_only_refresh_target(isolated):
|
||||
other = SimpleNamespace(close=lambda: pytest.fail("unrelated provider closed"))
|
||||
loader._PROVIDERS["other"] = other
|
||||
install(bundle(api_key_env="DEMO_KEY"))
|
||||
handle = loader.get_provider("demo_source")
|
||||
with pytest.raises(ProviderRequestError):
|
||||
handle.get_realtime()
|
||||
assert not save_plugin_key(PluginKeyIn(plugin="demo_source", api_key="bad"))["ok"]
|
||||
assert not secrets_store.load()
|
||||
result = save_plugin_key(PluginKeyIn(plugin="demo_source", api_key="valid-key"))
|
||||
assert result["ok"] and result["plugin_available"]
|
||||
assert handle.get_realtime()[0]["last"] == 1
|
||||
assert loader.get_provider("other") is other
|
||||
assert secrets_store.load()["demo_source_api_key"] == "valid-key"
|
||||
|
||||
|
||||
def test_upgrade_restart_rollback_and_failed_upgrade(isolated):
|
||||
install(bundle())
|
||||
install(bundle("2.0.0"))
|
||||
loader.load_all()
|
||||
assert loader.get_provider("demo_source").get_realtime()[0]["last"] == 1
|
||||
assert loader.list_plugins()[0]["pending_version"] == "2.0.0"
|
||||
cold_restart()
|
||||
assert loader.get_provider("demo_source").get_realtime()[0]["last"] == 2
|
||||
packages.manage("demo_source", "rollback")
|
||||
cold_restart()
|
||||
assert loader.get_provider("demo_source").get_realtime()[0]["last"] == 1
|
||||
install(bundle("3.0.0", code="raise RuntimeError('broken')"))
|
||||
cold_restart()
|
||||
assert loader.get_provider("demo_source").get_realtime()[0]["last"] == 1
|
||||
assert "失败" in loader.list_plugins()[0]["update_error"]
|
||||
|
||||
|
||||
def test_same_version_collision_and_preview_digest(isolated):
|
||||
data = bundle()
|
||||
install(data)
|
||||
assert "已安装" in install(data)["message"]
|
||||
with pytest.raises(ValueError, match="相同版本"):
|
||||
install(bundle(extra={"readme.md": "different"}))
|
||||
with pytest.raises(ValueError, match="预览不一致"):
|
||||
packages.install(bundle("2.0.0"), "0" * 64)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("filename", ["../escape.py", "/escape.py", "a\\b.py", "a/../b.py", "CON.py", "client.PY", "x.dll", "a./b.py"])
|
||||
def test_archive_paths_and_payload_limits(isolated, filename):
|
||||
with pytest.raises(ValueError):
|
||||
packages.inspect_archive(bundle(extra={filename: "pass"}))
|
||||
assert not (isolated.parent / "escape.py").exists()
|
||||
|
||||
|
||||
def test_manifest_contract_and_dependency_compatibility(isolated):
|
||||
for changes in ({"plugin_api_version": 9}, {"entry": "app.other:Provider"}, {"datasets": ["invented"]}):
|
||||
with pytest.raises(ValueError):
|
||||
packages.inspect_archive(bundle(**changes))
|
||||
preview, _ = packages.inspect_archive(bundle(requires_python=">=99"))
|
||||
assert preview["errors"]
|
||||
with pytest.raises(ValueError, match="requires_python"):
|
||||
packages.install(bundle(requires_python=">=99"), preview["digest"])
|
||||
|
||||
|
||||
def test_name_conflict_and_yaml_mutation_rejected(isolated):
|
||||
loader.plugins_dir().mkdir()
|
||||
(loader.plugins_dir() / "demo_source").mkdir()
|
||||
with pytest.raises(ValueError, match="冲突"):
|
||||
install(bundle())
|
||||
(loader.plugins_dir() / "demo_source").rmdir()
|
||||
install(bundle())
|
||||
with pytest.raises(ValueError):
|
||||
loader.save_config("demo_source", {})
|
||||
with pytest.raises(ValueError):
|
||||
loader.delete_config("demo_source")
|
||||
|
||||
|
||||
def test_disable_uninstall_blocks_all_routing_references(isolated, monkeypatch):
|
||||
install(bundle())
|
||||
monkeypatch.setattr(preferences, "load", lambda: {"depth5_data_provider": "demo_source"})
|
||||
for action in ("disable", "remove"):
|
||||
with pytest.raises(ValueError, match="先切换"):
|
||||
packages.manage("demo_source", action)
|
||||
monkeypatch.setattr(preferences, "load", lambda: {})
|
||||
secrets_store.save({"demo_source_api_key": "keep-me"})
|
||||
packages.manage("demo_source", "disable")
|
||||
assert not loader.list_plugins()[0]["available"]
|
||||
with pytest.raises(ProviderRequestError):
|
||||
loader.get_provider("demo_source").get_realtime()
|
||||
packages.manage("demo_source", "enable")
|
||||
assert loader.list_plugins()[0]["available"]
|
||||
packages.manage("demo_source", "remove")
|
||||
cold_restart()
|
||||
assert not loader.names()
|
||||
assert not packages.directory("demo_source").exists()
|
||||
assert secrets_store.load()["demo_source_api_key"] == "keep-me"
|
||||
|
||||
|
||||
def test_handle_delays_close_until_stream_consumed():
|
||||
closed = []
|
||||
class Source:
|
||||
def iter_daily(self):
|
||||
yield 1
|
||||
yield 2
|
||||
def close(self):
|
||||
closed.append(self)
|
||||
old, new = Source(), Source()
|
||||
handle = ManagedProvider(old)
|
||||
stream = handle.iter_daily()
|
||||
handle.replace(new)
|
||||
assert not closed
|
||||
assert list(stream) == [1, 2]
|
||||
assert closed == [old]
|
||||
handle.close()
|
||||
assert closed == [old, new]
|
||||
|
||||
|
||||
def test_handle_delays_close_during_concurrent_request():
|
||||
started, finish = threading.Event(), threading.Event()
|
||||
closed = []
|
||||
class Source:
|
||||
def get_realtime(self):
|
||||
started.set()
|
||||
assert finish.wait(3)
|
||||
return [1]
|
||||
def close(self):
|
||||
closed.append(self)
|
||||
old, new = Source(), Source()
|
||||
handle = ManagedProvider(old)
|
||||
worker = threading.Thread(target=handle.get_realtime)
|
||||
worker.start()
|
||||
assert started.wait(3)
|
||||
handle.replace(new)
|
||||
assert not closed
|
||||
finish.set()
|
||||
worker.join(3)
|
||||
assert closed == [old]
|
||||
|
||||
|
||||
def test_management_auth_and_upload_api(isolated, monkeypatch):
|
||||
from app.services import auth
|
||||
monkeypatch.setattr(auth, "is_configured", lambda: False)
|
||||
app = FastAPI()
|
||||
app.include_router(router)
|
||||
with TestClient(app, base_url="http://localhost", client=("127.0.0.1", 50000)) as client:
|
||||
assert client.post("/api/custom/source-packages/inspect", files={"file": ("source.zip", bundle())}, headers={"Origin": "https://evil.example"}).status_code == 403
|
||||
preview = client.post("/api/custom/source-packages/inspect", files={"file": ("source.zip", bundle())})
|
||||
assert preview.status_code == 200
|
||||
result = client.post("/api/custom/source-packages/install", files={"file": ("source.zip", bundle())}, data={"digest": preview.json()["digest"]})
|
||||
assert result.status_code == 200
|
||||
assert client.post("/api/custom/source-packages/demo_source/manage", json={"action": "disable"}).status_code == 200
|
||||
with TestClient(app, base_url="http://localhost", client=("192.168.1.3", 50000)) as client:
|
||||
assert client.post("/api/custom/source-packages/inspect", files={"file": ("s.zip", bundle())}).status_code == 403
|
||||
|
||||
|
||||
def test_valid_key_with_failed_initialization_restores_original(isolated):
|
||||
code = CODE.replace(' def get_realtime(self):', ''' def __init__(self):
|
||||
from app import secrets_store
|
||||
if secrets_store.load().get("demo_source_api_key") == "valid-key":
|
||||
raise RuntimeError("constructor failed")
|
||||
def get_realtime(self):''')
|
||||
install(bundle(code=code, api_key_env="DEMO_KEY"))
|
||||
secrets_store.save({"demo_source_api_key": "old-key"})
|
||||
loader.refresh_plugin("demo_source")
|
||||
original = loader.get_provider("demo_source")
|
||||
assert original.get_realtime()
|
||||
result = save_plugin_key(PluginKeyIn(plugin="demo_source", api_key="valid-key"))
|
||||
assert not result["ok"]
|
||||
assert secrets_store.load()["demo_source_api_key"] == "old-key"
|
||||
assert original.get_realtime()
|
||||
assert loader.list_plugins()[0]["available"]
|
||||
|
||||
|
||||
def test_disabled_plugin_does_not_execute_pending_version_on_start(isolated):
|
||||
install(bundle())
|
||||
packages.manage("demo_source", "disable")
|
||||
install(bundle("2.0.0", code="raise RuntimeError('must not run')"))
|
||||
cold_restart()
|
||||
assert loader.list_plugins()[0]["pending_version"] == "2.0.0"
|
||||
assert loader.list_plugins()[0]["version"] == "1.0.0"
|
||||
assert not loader.list_plugins()[0]["available"]
|
||||
|
||||
|
||||
def test_bad_package_does_not_hide_healthy_provider(isolated):
|
||||
install(bundle())
|
||||
path = packages.root() / "broken"
|
||||
path.mkdir()
|
||||
(path / "state.json").write_text('{"schema": 999}')
|
||||
packages.discover()
|
||||
assert loader.get_provider("demo_source").get_realtime()
|
||||
assert not next(p for p in loader.list_plugins() if p["name"] == "broken")["available"]
|
||||
assert loader.provider_has_dataset("broken", "daily")
|
||||
with pytest.raises(ProviderRequestError):
|
||||
loader.get_provider("broken").get_daily([])
|
||||
|
||||
|
||||
def test_zip_symlink_is_rejected(isolated):
|
||||
data = io.BytesIO(bundle())
|
||||
with zipfile.ZipFile(data, "a") as archive:
|
||||
info = zipfile.ZipInfo("link.py")
|
||||
info.create_system = 3
|
||||
info.external_attr = 0o120777 << 16
|
||||
archive.writestr(info, "../outside")
|
||||
with pytest.raises(ValueError, match="符号链接"):
|
||||
packages.inspect_archive(data.getvalue())
|
||||
|
||||
|
||||
def test_external_capability_matrix_marks_unconfigured_route_unusable(isolated):
|
||||
from app.data_providers.capabilities import build_capability_matrix
|
||||
install(bundle(api_key_env="DEMO_KEY"))
|
||||
matrix = build_capability_matrix({"realtime_data_provider": "demo_source"})
|
||||
realtime = next(row for row in matrix["capabilities"] if row["id"] == "realtime")
|
||||
assert realtime["current"] == "demo_source"
|
||||
assert not realtime["usable"]
|
||||
assert realtime["pending"][0]["name"] == "demo_source"
|
||||
|
||||
|
||||
def test_reload_picks_up_manually_changed_credentials(isolated):
|
||||
install(bundle(api_key_env="DEMO_KEY"))
|
||||
handle = loader.get_provider("demo_source")
|
||||
assert not loader.list_plugins()[0]["available"]
|
||||
secrets_store.save({"demo_source_api_key": "valid-key"})
|
||||
loader.load_all()
|
||||
assert loader.get_provider("demo_source") is handle
|
||||
assert loader.list_plugins()[0]["available"]
|
||||
secrets_store.clear("demo_source_api_key")
|
||||
loader.load_all()
|
||||
assert not loader.list_plugins()[0]["available"]
|
||||
with pytest.raises(ProviderRequestError):
|
||||
handle.get_realtime()
|
||||
Generated
+2
@@ -2532,6 +2532,7 @@ dependencies = [
|
||||
{ name = "httpx" },
|
||||
{ name = "numba", marker = "platform_machine != 'x86_64' or sys_platform != 'darwin'" },
|
||||
{ name = "openai" },
|
||||
{ name = "packaging" },
|
||||
{ name = "pandas" },
|
||||
{ name = "pillow" },
|
||||
{ name = "platformdirs" },
|
||||
@@ -2580,6 +2581,7 @@ requires-dist = [
|
||||
{ name = "mypy", marker = "extra == 'dev'", specifier = ">=1.10" },
|
||||
{ name = "numba", marker = "platform_machine != 'x86_64' or sys_platform != 'darwin'", specifier = ">=0.65.1" },
|
||||
{ name = "openai", specifier = ">=1.66" },
|
||||
{ name = "packaging", specifier = ">=24.0" },
|
||||
{ name = "pandas", specifier = ">=2.2" },
|
||||
{ name = "pillow", specifier = ">=10.0" },
|
||||
{ name = "platformdirs", specifier = ">=4.0" },
|
||||
|
||||
@@ -0,0 +1,96 @@
|
||||
# 外部数据源插件安装
|
||||
|
||||
外部安装是数据源插件机制的新入口:开发者发布 ZIP,用户在「设置 → 数据源 → 安装插件」安装,随后点击新卡片填写 API Key、验证并选择接管的数据能力。无需为每个源改前端或重新发布主程序。
|
||||
|
||||
## 使用流程
|
||||
|
||||
1. 点击数据源区域右上角的小盒子「安装插件」,选择或拖入 ZIP。
|
||||
2. 查看名称、作者声明、版本、能力和兼容检查结果。预览不会执行包内 Python。
|
||||
3. 确认信任来源后安装。Python 插件与后端同权限,可读取数据和凭据;这不是沙箱。
|
||||
4. 安装后自动选中新卡片。需要 Key 时显示「待配置 API Key」,有效 Key 才保存到现有 `user_data/secrets.json`;运行时初始化失败恢复原凭据和实例。
|
||||
5. 在详情区「选择启用能力」勾选需要接管的能力,确认当前源到新源的变化。仅修改勾选项;安装、配置 Key 都不会自动切换路由。
|
||||
6. 插件提供 `test_dataset` 时可以逐项试拉;未提供时显示「未检测」。验证 Key 不代表全部接口都有权限,试拉可能消耗供应商额度。
|
||||
|
||||
主程序部署的后端收到包,而不是浏览器所在电脑。Docker 请持久挂载整个配置的数据目录。
|
||||
|
||||
## 更新、回退和卸载
|
||||
|
||||
- 使用同一入口上传更高版本。新版暂存,当前进程继续使用旧版;完整重启后生效,设置页「重新加载」不会应用待生效更新。
|
||||
- 重启时新版初始化失败,保留原有效版本并显示错误。缺 Key 属于待配置状态,不视为代码初始化失败。
|
||||
- 详情区支持取消待生效更新、停用/启用、回退上一版本;回退也在重启后生效。
|
||||
- 停用或卸载前必须先切换引用该插件的所有能力,包括除权、五档和全量分钟。检查保存的路由配置,不依赖当前可用性。
|
||||
- 卸载后当前实例停止接收新调用,正在执行的调用/流式迭代结束后关闭;文件在重启时删除。Key 保留,需清除时在卸载前使用原 Key 配置区的清除操作。
|
||||
- 相同包重复安装不会重复注册。相同版本但不同内容拒绝覆盖,请提升版本。
|
||||
- 外部包在 `data/plugins/<name>/versions/<version>/`,激活状态在 `state.json`。实际 `data` 根目录服从应用配置;升级主程序不会覆盖该目录,兼容性仍须重新检查。
|
||||
|
||||
管理接口复用面板登录会话,不向开放 API Token 授权。未设置密码时仅允许本机同源访问;局域网部署请先设置密码。多 worker 部署应在停止全部 worker 后统一维护/重启;首版不提供跨 worker 在线版本同步。
|
||||
|
||||
## 包结构与清单
|
||||
|
||||
ZIP 根目录必须直接包含 `plugin.yaml`,不要额外包一层目录。每包一个数据源,继续沿用现有 Provider 协议,不需要另加 `manifest.json`。
|
||||
|
||||
```text
|
||||
my-source.zip
|
||||
├── plugin.yaml
|
||||
├── provider.py
|
||||
├── client.py
|
||||
└── README.md
|
||||
```
|
||||
|
||||
```yaml
|
||||
name: my_source # 小写字母开头,字母/数字/下划线,最多 64 字符
|
||||
version: 1.0.0
|
||||
plugin_api_version: 1
|
||||
display_name: 我的数据源
|
||||
runtime: none # none/python;不自动安装依赖
|
||||
entry: provider:Provider # 相对于包的模块;不是 app.plugins.*
|
||||
datasets: [realtime]
|
||||
api_key_env: MY_SOURCE_API_KEY
|
||||
requires_app: '>=0.3.24'
|
||||
requires_python: '>=3.11'
|
||||
platforms: [any] # 或 sys.platform-machine,如 linux-x86_64
|
||||
requires: [] # 现有宿主发行包版本约束,例如 'httpx>=0.27'
|
||||
```
|
||||
|
||||
`description`、`author`、`homepage`、`key_manage_url`、`billing`、`billing_note`、`usage_notes` 沿用插件元数据;链接须为 HTTPS。作者字段只是包内声明,不是签名认证。`check: provider:availability` 可选,返回 `(bool, reason)`;只做本地就绪检查,不进行慢网络操作。
|
||||
|
||||
外部插件独立命名空间加载,内部使用 `from .client import ...` 等相对导入。普通子包需要自己的 `__init__.py`。不要修改全局 `sys.path`,不要写入程序目录,不要依赖核心服务私有属性。构造函数应快速、无网络访问;取数在数据方法中执行。
|
||||
|
||||
`Provider.name`、`config.datasets` 与清单必须一致,并实现声明能力的方法及 `close()`。接口、单位、复权和分钟时区契约见 [插件开发指南](plugin-development.md)。已安装但缺凭据或运行失败的源保留标识并显示不可用,不因名字暂时消失而换源。
|
||||
|
||||
声明 `api_key_env` 的插件必须在 entry 模块实现 `probe_api_key(key) -> (bool, reason)`,使用候选 Key 试探而不是读取当前已保存的 Key。Provider 实例读取 `secrets_store.get_env_backed_secret('<name>_api_key', '<ENV>')`,不要按请求重新读取 Key,否则无法保证在途调用使用原凭据。异常和探测原因不得包含 Key、鉴权头或带凭据的 URL。
|
||||
|
||||
可选 `test_dataset(dataset, symbols)` 沿用现有接口,返回 `{provider, dataset, rows, columns, preview, error?}`。实现有界样本请求、超时、分页上限;不要因试拉触发全历史同步。
|
||||
|
||||
## 运行环境边界
|
||||
|
||||
首版支持源码包(Python、YAML、JSON、文本/Markdown、CSV),上限 20 MiB 压缩包、80 MiB 解压总量、1000 条目。拒绝路径穿越、符号链接、重复/大小写冲突条目、Windows 保留路径以及原生库、wheel、可执行文件。
|
||||
|
||||
可使用主程序已有且版本满足要求的依赖;纯 Python 辅助模块可随包发布并相对导入。不会自动运行 pip/npm 或修改宿主环境。额外 SDK 的原生库随包安装暂不开放;需要平台认证和独立运行支持后再扩展。已有主程序内置的 SDK 不受影响。
|
||||
|
||||
旧 `app/plugins/*/plugin.yaml` 仍按原入口加载,旧 HTTP YAML 配置和凭据保留。外部包不允许覆盖内置或 YAML 数据源名称。主程序升级若超出插件兼容范围,卡片显示不可用;请更新插件后再启用。
|
||||
|
||||
## 扩展边界与 API
|
||||
|
||||
- 数据源加载器负责外置目录发现、包入口解析、统一注册和受控实例释放;原有服务继续通过 `get_provider` / `provider_has_dataset` 取数。
|
||||
- 安装管理通过 `app/custom/source_packages` 的后端扩展注册,不修改主应用路由入口。
|
||||
- 前端管理位于 `src/custom/source-packages`,使用两个正式插槽:`settings.data-sources.toolbar`(`refresh`、`selectSource`)和 `settings.data-sources.detail-actions`(`plugin`、`refresh`)。这些是本次新增的已实现契约。
|
||||
- 数据源列表、Key、试拉、路由 API 和查询缓存复用原实现,不建立另一套数据服务。
|
||||
|
||||
新增管理端点:
|
||||
|
||||
| 请求 | 用途 |
|
||||
| --- | --- |
|
||||
| `POST /api/custom/source-packages/inspect` | multipart `file`,静态预检,返回清单、摘要、兼容问题 |
|
||||
| `POST /api/custom/source-packages/install` | multipart `file` + 预览的 `digest`,再次校验后安装/暂存更新 |
|
||||
| `POST /api/custom/source-packages/{name}/manage` | JSON `action`: `enable` / `disable` / `rollback` / `cancel-update` / `remove` |
|
||||
|
||||
包文件先完整写入临时目录,再发布版本目录及原子状态文件。管理操作用跨进程文件数据库锁串行化;上传摘要只用于绑定预览和内容,不证明作者身份。Python 异常按插件隔离,但进程内插件无法隔离恶意行为、无限循环或原生崩溃。
|
||||
|
||||
## 开发示例与验收
|
||||
|
||||
仓库提供 [HTTP 数据源模板](../examples/data-source-plugin/provider.py) 及配套清单、client。它使用不可访问的示例域名;先替换实际接口并实现标准化,不会伪装成已可用数据源。
|
||||
|
||||
在模板目录执行 `zip -r ../example-http.zip plugin.yaml provider.py client.py`,或选中这三个文件压缩,确保 ZIP 根目录直接是文件。开发测试应使用假 Client,不需要真实 Key。
|
||||
|
||||
新增测试覆盖安全预检、安装取数、Key 先探后存、定向刷新、在途调用和迭代器释放、重启升级/回退、失败保留旧版、名称冲突、卸载引用检查和管理鉴权。发布时还应对目标操作系统的桌面产物和 Docker 镜像做安装冒烟;源码测试不能替代这些验证。
|
||||
@@ -9,6 +9,10 @@
|
||||
> 无代码接入(纯 HTTP YAML 配置)请看 [custom-data-source.md](./custom-data-source.md),
|
||||
> 两种方式遵循同一套内部数据契约。
|
||||
|
||||
## 外部安装
|
||||
|
||||
现在可通过「设置 → 数据源 → 安装插件」上传独立源码 ZIP。外部包格式、版本管理、兼容范围和模板见 [外部数据源插件安装](external-data-source-packages.md)。现有内置目录方式继续兼容。
|
||||
|
||||
## 快速上手
|
||||
|
||||
一个插件 = 一个目录 + 一个 `plugin.yaml` 清单:
|
||||
@@ -64,7 +68,7 @@ TickFlow 的「先探后存」语义:
|
||||
| `python` | 纯 Python 依赖, `pip install` | akshare、tushare |
|
||||
| `none` | 无额外依赖 | 纯 HTTP API 源 |
|
||||
|
||||
`runtime` 字段当前仅用于 UI 展示, 实际依赖检测由 `check` 函数负责。
|
||||
内置插件的 `runtime` 决定依赖安装方式,实际就绪检测由 `check` 函数负责。外部源码包只检查依赖兼容性,不执行 pip/npm 安装。
|
||||
|
||||
### check 函数
|
||||
|
||||
|
||||
@@ -36,6 +36,10 @@
|
||||
|
||||
AI 在开始任务前必须通过代码搜索确认能力是否已经实现。找不到定义和测试时,应把示例视为设计规范,不得虚构导入路径或调用结果。
|
||||
|
||||
## 外部数据源包安装(已实现)
|
||||
|
||||
外置 Provider 源码包、版本状态与加载生命周期由原数据源加载器统一管理;上传/安装/版本管理 API 由 `app/custom/source_packages` 扩展注册。前端新增 `settings.data-sources.toolbar` 和 `settings.data-sources.detail-actions` 插槽,复用现有卡片、Key 和能力路由。完整契约及边界见 [外部数据源插件安装](external-data-source-packages.md)。
|
||||
|
||||
## 2. 二次开发分级
|
||||
|
||||
按升级风险从低到高选择实现方式:
|
||||
|
||||
@@ -0,0 +1,13 @@
|
||||
"""Replace this URL and field mapping with your provider's documented contract."""
|
||||
import httpx
|
||||
|
||||
BASE_URL = "https://example.invalid"
|
||||
|
||||
|
||||
def fetch(key: str) -> list[dict]:
|
||||
if BASE_URL.endswith(".invalid"):
|
||||
raise ValueError("请先修改模板的接口地址")
|
||||
with httpx.Client(timeout=15) as client:
|
||||
response = client.get(f"{BASE_URL}/quotes", headers={"Authorization": f"Bearer {key}"})
|
||||
response.raise_for_status()
|
||||
return response.json()["data"]
|
||||
@@ -0,0 +1,13 @@
|
||||
name: example_http
|
||||
version: 1.0.0
|
||||
plugin_api_version: 1
|
||||
display_name: HTTP 数据源模板
|
||||
author: 插件开发者
|
||||
runtime: none
|
||||
entry: provider:Provider
|
||||
requires_app: '>=0.3.24'
|
||||
requires_python: '>=3.11'
|
||||
platforms: [any]
|
||||
datasets: [realtime]
|
||||
api_key_env: EXAMPLE_HTTP_API_KEY
|
||||
description: 开发模板,请先在 client.py 配置真实接口并完成字段转换和测试。
|
||||
@@ -0,0 +1,32 @@
|
||||
"""Example only: vendor data must already match the documented normalized schema."""
|
||||
from types import SimpleNamespace
|
||||
|
||||
from app import secrets_store
|
||||
from app.data_providers.base import ProviderRequestError
|
||||
|
||||
from .client import fetch
|
||||
|
||||
|
||||
def probe_api_key(key):
|
||||
try:
|
||||
fetch(key)
|
||||
return True, "接口可连接;其他能力权限需另行验证"
|
||||
except Exception: # noqa: BLE001 — never return raw HTTP errors containing credentials
|
||||
return False, "验证失败,请检查接口地址、Key、权限与网络"
|
||||
|
||||
|
||||
class Provider:
|
||||
name = "example_http"
|
||||
|
||||
def __init__(self):
|
||||
self.config = SimpleNamespace(datasets={"realtime": {}})
|
||||
self.key = secrets_store.get_env_backed_secret("example_http_api_key", "EXAMPLE_HTTP_API_KEY")
|
||||
|
||||
def get_realtime(self):
|
||||
try:
|
||||
return fetch(self.key)
|
||||
except Exception as exc:
|
||||
raise ProviderRequestError("行情请求失败,请检查供应商服务") from exc
|
||||
|
||||
def close(self):
|
||||
pass
|
||||
@@ -0,0 +1,90 @@
|
||||
// @vitest-environment jsdom
|
||||
import { act } from 'react'
|
||||
import { createRoot, type Root } from 'react-dom/client'
|
||||
import { QueryClient, QueryClientProvider } from '@tanstack/react-query'
|
||||
import { beforeEach, afterEach, expect, it, vi } from 'vitest'
|
||||
import { InstallDialog, PackageActions, PackageInstaller } from './extension'
|
||||
import type { PluginDataSourceItem } from '@/lib/api'
|
||||
|
||||
const mocks = vi.hoisted(() => ({ inspect: vi.fn(), install: vi.fn(), manage: vi.fn(), route: vi.fn(), matrix: vi.fn(), toast: vi.fn(), close: vi.fn(), done: vi.fn() }))
|
||||
vi.mock('@/lib/api', () => ({ api: { inspectSourcePackage: mocks.inspect, installSourcePackage: mocks.install, manageSourcePackage: mocks.manage, updateDataProviders: mocks.route, capabilityMatrix: mocks.matrix } }))
|
||||
vi.mock('@/components/Toast', () => ({ toast: mocks.toast }))
|
||||
let root: Root
|
||||
let qc: QueryClient
|
||||
const node = document.createElement('div')
|
||||
document.body.appendChild(node)
|
||||
const plugin = { name: 'demo', display_name: '演示源', origin: 'external', version: '1.0.0', available: true, enabled: true, status: '已就绪' } as PluginDataSourceItem
|
||||
const preview = { manifest: { name: 'demo', display_name: '演示源', version: '1.0.0', datasets: ['daily'] }, digest: 'hash', errors: [], installed_version: null }
|
||||
async function render(element = <InstallDialog onClose={mocks.close} onInstalled={mocks.done} />) {
|
||||
root = createRoot(node)
|
||||
qc = new QueryClient({ defaultOptions: { queries: { retry: false } } })
|
||||
await act(async () => root.render(<QueryClientProvider client={qc}>{element}</QueryClientProvider>))
|
||||
await act(async () => { await new Promise(resolve => setTimeout(resolve, 10)) })
|
||||
}
|
||||
const button = (label: string) => [...node.querySelectorAll('button')].find(el => el.textContent === label)!
|
||||
async function click(el: HTMLElement) { await act(async () => el.click()) }
|
||||
async function choose() {
|
||||
const input = node.querySelector('input[type="file"]')!
|
||||
Object.defineProperty(input, 'files', { configurable: true, value: [new File(['zip'], 'demo.zip')] })
|
||||
await act(async () => input.dispatchEvent(new Event('change', { bubbles: true })))
|
||||
}
|
||||
beforeEach(() => {
|
||||
Object.assign(globalThis, { IS_REACT_ACT_ENVIRONMENT: true })
|
||||
Object.values(mocks).forEach(fn => fn.mockReset())
|
||||
mocks.inspect.mockResolvedValue(preview)
|
||||
mocks.install.mockResolvedValue({ name: 'demo', message: '安装完成' })
|
||||
mocks.matrix.mockResolvedValue({ capabilities: [{ id: 'daily', label: '日K', field: 'daily_data_provider', current_display: '原数据源', candidates: [{ name: 'demo' }] }] })
|
||||
mocks.route.mockResolvedValue({})
|
||||
})
|
||||
afterEach(() => { act(() => root?.unmount()); qc?.clear(); node.innerHTML = '' })
|
||||
|
||||
it('preview cannot execute install until the user confirms trust; successful install selects the card', async () => {
|
||||
await render(); await choose()
|
||||
expect(node.textContent).toContain('演示源')
|
||||
expect(mocks.install).not.toHaveBeenCalled()
|
||||
expect(button('确认安装').disabled).toBe(true)
|
||||
await click(node.querySelector('input[type="checkbox"]')!)
|
||||
await click(button('确认安装'))
|
||||
expect(mocks.install).toHaveBeenCalledWith(expect.any(File), 'hash')
|
||||
expect(mocks.done).toHaveBeenCalledWith('demo')
|
||||
})
|
||||
it('incompatible package is visible but cannot install', async () => {
|
||||
mocks.inspect.mockResolvedValue({ ...preview, errors: ['Python 版本不兼容'] })
|
||||
await render(); await choose(); await click(node.querySelector('input[type="checkbox"]')!)
|
||||
expect(node.textContent).toContain('Python 版本不兼容')
|
||||
expect(button('确认安装').disabled).toBe(true)
|
||||
})
|
||||
it('blocks escape and closing during installation; failure stays in the dialog', async () => {
|
||||
let reject!: (e: Error) => void
|
||||
mocks.install.mockImplementation(() => new Promise((_resolve, fail) => { reject = fail }))
|
||||
await render(); await choose(); await click(node.querySelector('input[type="checkbox"]')!); await click(button('确认安装'))
|
||||
await act(async () => document.dispatchEvent(new KeyboardEvent('keydown', { key: 'Escape', bubbles: true })))
|
||||
expect(mocks.close).not.toHaveBeenCalled()
|
||||
await act(async () => reject(new Error('加载失败')))
|
||||
expect(node.querySelector('[role="alert"]')?.textContent).toBe('加载失败')
|
||||
expect(mocks.done).not.toHaveBeenCalled()
|
||||
})
|
||||
it('reopening the installer clears the previous file and preview', async () => {
|
||||
await render(<PackageInstaller refresh={mocks.done} selectSource={vi.fn()} />)
|
||||
await click(button('安装插件')); await choose(); await click(button('取消'))
|
||||
await click(button('安装插件'))
|
||||
expect(node.textContent).not.toContain('静态兼容检查通过')
|
||||
expect(button('确认安装').disabled).toBe(true)
|
||||
})
|
||||
it('only selected routing fields are changed', async () => {
|
||||
await render(<PackageActions plugin={plugin} refresh={mocks.done} />)
|
||||
await click(button('选择启用能力'))
|
||||
await act(async () => { await new Promise(resolve => setTimeout(resolve, 10)) })
|
||||
expect(button('确认切换所选能力').disabled).toBe(true)
|
||||
expect(node.textContent).toContain('原数据源 → 演示源')
|
||||
await click(node.querySelector('input[type="checkbox"]')!)
|
||||
await click(button('确认切换所选能力'))
|
||||
expect(mocks.route).toHaveBeenCalledWith({ daily_data_provider: 'demo' })
|
||||
})
|
||||
it('failed removal shows routing references without closing confirmation', async () => {
|
||||
mocks.manage.mockRejectedValue(new Error('请先切换五档数据源'))
|
||||
await render(<PackageActions plugin={plugin} refresh={mocks.done} />)
|
||||
await click(button('卸载插件')); await click(button('确认'))
|
||||
expect(node.textContent).toContain('请先切换五档数据源')
|
||||
expect(node.querySelector('[role="dialog"]')).not.toBeNull()
|
||||
})
|
||||
@@ -0,0 +1,171 @@
|
||||
import { useState } from 'react'
|
||||
import { Package, X, Upload, Loader2 } from 'lucide-react'
|
||||
import { useQuery } from '@tanstack/react-query'
|
||||
import { api, type SourcePackagePreview } from '@/lib/api'
|
||||
import { QK } from '@/lib/queryKeys'
|
||||
import { Modal } from '@/components/Modal'
|
||||
import { toast } from '@/components/Toast'
|
||||
import type { FrontendExtension, FrontendSlotContextMap } from '@/extensions/types'
|
||||
|
||||
const button = 'inline-flex items-center justify-center gap-1.5 rounded-btn border border-border px-3 py-1.5 text-xs hover:bg-elevated disabled:opacity-50'
|
||||
|
||||
export function PackageInstaller({ refresh, selectSource }: FrontendSlotContextMap['settings.data-sources.toolbar']) {
|
||||
const [open, setOpen] = useState(false)
|
||||
return <>
|
||||
<button
|
||||
type="button"
|
||||
aria-haspopup="dialog"
|
||||
className="inline-flex h-8 shrink-0 items-center justify-center gap-1.5 whitespace-nowrap rounded-btn border border-accent/20 bg-accent/10 px-3 text-xs font-medium text-blue-700 transition-colors hover:border-accent/40 hover:bg-accent/15 focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-accent/60 focus-visible:ring-offset-2 focus-visible:ring-offset-surface dark:text-blue-400"
|
||||
onClick={() => setOpen(true)}
|
||||
>
|
||||
<Package size={14} aria-hidden="true" />
|
||||
安装插件
|
||||
</button>
|
||||
{open && <InstallDialog onClose={() => setOpen(false)} onInstalled={name => { refresh(); selectSource(name); setOpen(false) }} />}
|
||||
</>
|
||||
}
|
||||
|
||||
export function InstallDialog({ onClose, onInstalled }: { onClose: () => void; onInstalled: (name: string) => void }) {
|
||||
const [file, setFile] = useState<File | null>(null)
|
||||
const [preview, setPreview] = useState<SourcePackagePreview | null>(null)
|
||||
const [busy, setBusy] = useState(false)
|
||||
const [error, setError] = useState('')
|
||||
const [trusted, setTrusted] = useState(false)
|
||||
const matrix = useQuery({ queryKey: QK.capabilityMatrix, queryFn: api.capabilityMatrix })
|
||||
const labels = new Map(matrix.data?.capabilities.map(c => [c.id, c.label]) ?? [])
|
||||
async function inspect(candidate: File | undefined) {
|
||||
if (!candidate || busy) return
|
||||
setFile(null); setPreview(null); setTrusted(false); setError('')
|
||||
if (candidate.size > 20 * 1024 * 1024) { setError('插件包不能超过 20 MiB'); return }
|
||||
setBusy(true)
|
||||
try { setPreview(await api.inspectSourcePackage(candidate)); setFile(candidate) }
|
||||
catch (e) { setError(e instanceof Error ? e.message : '无法读取插件包') }
|
||||
finally { setBusy(false) }
|
||||
}
|
||||
async function install() {
|
||||
if (!file || !preview || !trusted || busy) return
|
||||
setBusy(true); setError('')
|
||||
try {
|
||||
const result = await api.installSourcePackage(file, preview.digest)
|
||||
toast(result.message, 'success'); onInstalled(result.name)
|
||||
} catch (e) { setError(e instanceof Error ? e.message : '安装失败') }
|
||||
finally { setBusy(false) }
|
||||
}
|
||||
return <Modal onClose={() => { if (!busy) onClose() }} labelledBy="source-package-title" closeOnBackdrop={!busy}
|
||||
panelClassName="w-[94vw] max-w-xl max-h-[85vh] overflow-y-auto text-foreground bg-surface border border-border rounded-card shadow-xl p-5 space-y-4">
|
||||
<div className="flex items-center justify-between gap-3">
|
||||
<h2 id="source-package-title" className="text-[16px] leading-6 font-semibold text-foreground">安装数据源插件</h2>
|
||||
<button className={button} aria-label="关闭" disabled={busy} onClick={onClose}><X size={16} /></button>
|
||||
</div>
|
||||
<p className="text-sm text-secondary">安装后会在数据源列表中显示卡片。配置 API Key 后,再选择需要启用的能力。</p>
|
||||
<label className="flex flex-col items-center gap-3 rounded-card border border-dashed border-border p-6 text-sm text-secondary"
|
||||
onDragOver={e => e.preventDefault()} onDrop={e => { e.preventDefault(); void inspect(e.dataTransfer.files[0]) }}>
|
||||
<Upload size={24} /><span>选择或拖入 ZIP 插件包 · 最大 20 MiB</span>
|
||||
<input aria-label="选择插件包" type="file" accept=".zip" disabled={busy} onChange={e => { void inspect(e.target.files?.[0]); e.target.value = '' }} className="max-w-full text-xs" />
|
||||
</label>
|
||||
{busy && <p role="status" className="flex gap-2 text-sm"><Loader2 size={16} className="animate-spin" />正在处理,请稍候…</p>}
|
||||
{file && <p className="text-xs text-muted break-all">{file.name}</p>}
|
||||
{preview && <div className="rounded-card border border-border p-4 space-y-2 text-sm">
|
||||
<div className="font-semibold break-words">{preview.manifest.display_name || preview.manifest.name} <span className="font-mono text-muted">{preview.manifest.version}</span></div>
|
||||
{preview.manifest.author && <p className="text-secondary">作者声明:{preview.manifest.author}</p>}
|
||||
<p className="text-secondary break-words">{preview.manifest.description}</p>
|
||||
<p>支持能力:{preview.manifest.datasets.map(d => labels.get(d) || d).join('、')}</p>
|
||||
{preview.installed_version && <p className="text-warning">当前 {preview.installed_version} → {preview.manifest.version},升级重启后生效。</p>}
|
||||
{preview.errors.map((item, i) => <p key={i} className="text-danger">{item}</p>)}
|
||||
{!preview.errors.length && <p className="text-bear">静态兼容检查通过</p>}
|
||||
</div>}
|
||||
{error && <p role="alert" className="text-sm text-danger break-words">{error}</p>}
|
||||
<label className="flex items-start gap-2 text-xs text-secondary"><input type="checkbox" checked={trusted} disabled={busy || !preview} onChange={e => setTrusted(e.target.checked)} />
|
||||
<span>我信任此插件的来源。插件会在本机执行 Python 代码,可访问主程序的数据和凭据。</span></label>
|
||||
<div className="flex justify-end gap-2">
|
||||
<button className={button} disabled={busy} onClick={onClose}>取消</button>
|
||||
<button className={`${button} bg-accent/10 text-accent`} disabled={busy || !preview || preview.errors.length > 0 || !trusted} onClick={() => void install()}>确认安装</button>
|
||||
</div>
|
||||
</Modal>
|
||||
}
|
||||
|
||||
export function PackageActions({ plugin, refresh }: FrontendSlotContextMap['settings.data-sources.detail-actions']) {
|
||||
const [busy, setBusy] = useState(false)
|
||||
const [error, setError] = useState('')
|
||||
const [confirm, setConfirm] = useState<string | null>(null)
|
||||
const [routing, setRouting] = useState(false)
|
||||
if (plugin.origin !== 'external') return null
|
||||
async function run(action: string) {
|
||||
setBusy(true); setError('')
|
||||
try { const result = await api.manageSourcePackage(plugin.name, action); toast(result.message, 'success'); refresh(); setConfirm(null) }
|
||||
catch (e) { setError(e instanceof Error ? e.message : '操作失败') }
|
||||
finally { setBusy(false) }
|
||||
}
|
||||
return <div className="mt-4 rounded-card bg-elevated/40 p-3 text-xs space-y-3">
|
||||
<p>外部安装 · 版本 {plugin.version} · {plugin.status}</p>
|
||||
{plugin.update_error && <p className="text-danger">{plugin.update_error}</p>}
|
||||
{plugin.pending_version && <p className="text-warning">版本 {plugin.pending_version} 已暂存,重启后生效。</p>}
|
||||
{plugin.pending_remove ? <p className="text-warning">重启后移除插件文件;已保存的 Key 保留。</p> : <div className="flex flex-wrap gap-2">
|
||||
{plugin.available && <button className={`${button} text-accent`} onClick={() => setRouting(true)}>选择启用能力</button>}
|
||||
<button disabled={busy} className={button} onClick={() => void run(plugin.enabled === false ? 'enable' : 'disable')}>{plugin.enabled === false ? '启用插件' : '停用插件'}</button>
|
||||
{plugin.previous_version && <button disabled={busy} className={button} onClick={() => setConfirm('rollback')}>回退到 {plugin.previous_version}</button>}
|
||||
{plugin.pending_version && <button disabled={busy} className={button} onClick={() => void run('cancel-update')}>取消待生效更新</button>}
|
||||
<button disabled={busy} className={`${button} text-danger`} onClick={() => setConfirm('remove')}>卸载插件</button>
|
||||
<span className="self-center text-muted">更新版本:使用右上角“安装插件”上传新包</span>
|
||||
</div>}
|
||||
{error && <p role="alert" className="text-danger">{error}</p>}
|
||||
{routing && <RoutingDialog plugin={plugin} onClose={() => setRouting(false)} onSaved={() => { refresh(); setRouting(false) }} />}
|
||||
{confirm && <Modal onClose={() => { if (!busy) setConfirm(null) }} ariaLabel="确认插件操作" closeOnBackdrop={!busy}>
|
||||
<div className="p-5 space-y-4 text-sm text-foreground">
|
||||
<p>{confirm === 'remove' ? '卸载前需要先切换正在使用它的能力。插件文件重启后删除,Key 保留,可在 Key 配置中单独清除。' : '重启后切换到上一版本,已保存的 Key 和能力路由保留。'}</p>
|
||||
{error && <p role="alert" className="text-danger">{error}</p>}
|
||||
<div className="flex justify-end gap-2"><button className={button} disabled={busy} onClick={() => setConfirm(null)}>取消</button><button className={button} disabled={busy} onClick={() => void run(confirm)}>确认</button></div>
|
||||
</div>
|
||||
</Modal>}
|
||||
</div>
|
||||
}
|
||||
|
||||
function RoutingDialog({ plugin, onClose, onSaved }: { plugin: FrontendSlotContextMap['settings.data-sources.detail-actions']['plugin']; onClose: () => void; onSaved: () => void }) {
|
||||
const matrix = useQuery({ queryKey: QK.capabilityMatrix, queryFn: api.capabilityMatrix })
|
||||
const [selected, setSelected] = useState<string[]>([])
|
||||
const [busy, setBusy] = useState(false)
|
||||
const [error, setError] = useState('')
|
||||
const [tests, setTests] = useState<Record<string, string>>({})
|
||||
const caps = (matrix.data?.capabilities ?? []).filter(c => c.field && c.candidates.some(source => source.name === plugin.name))
|
||||
async function save() {
|
||||
setBusy(true); setError('')
|
||||
try {
|
||||
const changes = Object.fromEntries(caps.filter(c => selected.includes(c.id)).map(c => [c.field!, plugin.name]))
|
||||
await api.updateDataProviders(changes)
|
||||
toast('已更新所选能力的数据源', 'success'); onSaved()
|
||||
} catch (e) { setError(e instanceof Error ? e.message : '切换失败') }
|
||||
finally { setBusy(false) }
|
||||
}
|
||||
async function test(id: string) {
|
||||
setBusy(true)
|
||||
try {
|
||||
const result = await api.testDataSource(plugin.name, id, ['600000.SH'])
|
||||
setTests(previous => ({ ...previous, [id]: result.error || `试拉返回 ${result.rows} 行(不代表所有接口权限)` }))
|
||||
} catch (e) { setTests(previous => ({ ...previous, [id]: e instanceof Error ? e.message : '试拉失败' })) }
|
||||
finally { setBusy(false) }
|
||||
}
|
||||
return <Modal onClose={() => { if (!busy) onClose() }} ariaLabel="选择启用能力" closeOnBackdrop={!busy}>
|
||||
<div className="p-5 space-y-4 text-foreground max-h-[80vh] overflow-y-auto">
|
||||
<h3 className="text-[16px] leading-6 font-semibold text-foreground">选择启用能力</h3>
|
||||
<p className="text-xs text-secondary">仅修改勾选项,其他能力保留原数据源。试拉可能消耗供应商额度。</p>
|
||||
{matrix.isLoading && <p>正在读取能力…</p>}
|
||||
{matrix.error && <p role="alert">能力列表读取失败,请关闭后重试。</p>}
|
||||
{caps.map(cap => <div key={cap.id} className="rounded border border-border p-3 space-y-2 text-xs">
|
||||
<label className="flex items-center gap-2"><input type="checkbox" disabled={busy} checked={selected.includes(cap.id)} onChange={e => setSelected(prev => e.target.checked ? [...prev, cap.id] : prev.filter(id => id !== cap.id))} />
|
||||
{cap.label}:{cap.current_display} → {plugin.display_name}</label>
|
||||
{plugin.can_test ? <button className={button} disabled={busy} onClick={() => void test(cap.id)}>试拉{cap.label}</button> : <p className="text-muted">此插件未提供能力试拉;Key 验证不代表接口权限已确认。</p>}
|
||||
{tests[cap.id] && <p role="status">{tests[cap.id]}</p>}
|
||||
</div>)}
|
||||
{error && <p role="alert" className="text-danger text-xs">{error}</p>}
|
||||
<div className="flex justify-end gap-2"><button className={button} disabled={busy} onClick={onClose}>取消</button><button className={button} disabled={busy || !selected.length} onClick={() => void save()}>确认切换所选能力</button></div>
|
||||
</div>
|
||||
</Modal>
|
||||
}
|
||||
|
||||
export default {
|
||||
id: 'official.source-packages', apiVersion: 1,
|
||||
slots: [
|
||||
{ name: 'settings.data-sources.toolbar', id: 'install', component: PackageInstaller },
|
||||
{ name: 'settings.data-sources.detail-actions', id: 'manage', component: PackageActions },
|
||||
],
|
||||
} satisfies FrontendExtension
|
||||
@@ -1,3 +1,4 @@
|
||||
import type { ComponentType } from 'react'
|
||||
import type { FrontendSlotContextMap, FrontendSlotName } from './types'
|
||||
import { getFrontendSlotRegistrations } from './registry'
|
||||
import { ExtensionBoundary } from './ExtensionBoundary'
|
||||
@@ -12,7 +13,8 @@ export function ExtensionSlot<K extends FrontendSlotName>({ name, context, compa
|
||||
const registrations = getFrontendSlotRegistrations(name)
|
||||
if (registrations.length === 0) return null
|
||||
return registrations.map(registration => {
|
||||
const SlotComponent = registration.component
|
||||
// Registrations were selected by this exact slot name; preserve the context correlation.
|
||||
const SlotComponent = registration.component as ComponentType<FrontendSlotContextMap[K]>
|
||||
return (
|
||||
<ExtensionBoundary
|
||||
key={`${registration.extensionId}:${registration.id}`}
|
||||
|
||||
@@ -1,10 +1,19 @@
|
||||
import type { ComponentType, ReactNode } from 'react'
|
||||
import type { LucideIcon } from 'lucide-react'
|
||||
import type { PluginDataSourceItem } from '@/lib/api'
|
||||
import type { BrandIconProps } from '@/components/BrandIcons'
|
||||
|
||||
export const FRONTEND_EXTENSION_API_VERSION = 1 as const
|
||||
|
||||
export interface FrontendSlotContextMap {
|
||||
'settings.data-sources.toolbar': {
|
||||
refresh: () => void
|
||||
selectSource: (name: string) => void
|
||||
}
|
||||
'settings.data-sources.detail-actions': {
|
||||
plugin: PluginDataSourceItem
|
||||
refresh: () => void
|
||||
}
|
||||
'layout.navigation.extra': {
|
||||
collapsed: boolean
|
||||
pathname: string
|
||||
@@ -32,12 +41,12 @@ export interface FrontendSlotContextMap {
|
||||
|
||||
export type FrontendSlotName = keyof FrontendSlotContextMap
|
||||
|
||||
export type FrontendSlotRegistration<K extends FrontendSlotName = FrontendSlotName> = {
|
||||
export type FrontendSlotRegistration<K extends FrontendSlotName = FrontendSlotName> = K extends FrontendSlotName ? {
|
||||
name: K
|
||||
id: string
|
||||
order?: number
|
||||
component: ComponentType<FrontendSlotContextMap[K]>
|
||||
}
|
||||
} : never
|
||||
|
||||
export interface FrontendExtensionRoute {
|
||||
id: string
|
||||
|
||||
@@ -2033,7 +2033,22 @@ export interface DataSourceItem {
|
||||
}
|
||||
|
||||
/** 内置可选插件数据源 (plugins/ 目录, 需手动装依赖) */
|
||||
export interface SourcePackagePreview {
|
||||
manifest: { name: string; display_name?: string; version: string; author?: string; description?: string; datasets: string[] }
|
||||
digest: string
|
||||
errors: string[]
|
||||
installed_version: string | null
|
||||
}
|
||||
|
||||
export interface PluginDataSourceItem {
|
||||
can_test?: boolean
|
||||
origin?: 'builtin' | 'external'
|
||||
version?: string
|
||||
enabled?: boolean
|
||||
pending_version?: string | null
|
||||
previous_version?: string | null
|
||||
pending_remove?: boolean
|
||||
update_error?: string | null
|
||||
name: string
|
||||
display_name: string
|
||||
datasets: string[]
|
||||
@@ -2109,6 +2124,7 @@ export interface DataSourcesResponse {
|
||||
}
|
||||
|
||||
export interface DataSourceTestResult {
|
||||
error?: string
|
||||
provider: string
|
||||
dataset: string
|
||||
rows: number
|
||||
@@ -2707,6 +2723,21 @@ export const api = {
|
||||
}),
|
||||
|
||||
preferences: () => request<Preferences>('/api/settings/preferences'),
|
||||
inspectSourcePackage: (file: File) => {
|
||||
const body = new FormData()
|
||||
body.append('file', file)
|
||||
return request<SourcePackagePreview>('/api/custom/source-packages/inspect', { method: 'POST', body, quiet: true })
|
||||
},
|
||||
installSourcePackage: (file: File, digest: string) => {
|
||||
const body = new FormData()
|
||||
body.append('file', file)
|
||||
body.append('digest', digest)
|
||||
return request<{ name: string; message: string }>('/api/custom/source-packages/install', { method: 'POST', body, quiet: true, timeoutMs: 120_000 })
|
||||
},
|
||||
manageSourcePackage: (name: string, action: string) =>
|
||||
request<{ name: string; message: string }>(`/api/custom/source-packages/${encodeURIComponent(name)}/manage`, {
|
||||
method: 'POST', body: JSON.stringify({ action }), quiet: true,
|
||||
}),
|
||||
dataSources: () => request<DataSourcesResponse>('/api/settings/data-sources'),
|
||||
capabilityMatrix: () => request<CapabilityMatrix>('/api/settings/capability-matrix'),
|
||||
dataSource: (name: string) => request<CustomSourceConfig>(`/api/settings/data-sources/${encodeURIComponent(name)}`),
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { useState } from 'react'
|
||||
import { ExtensionSlot } from '@/extensions/ExtensionSlot'
|
||||
import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query'
|
||||
import { motion, AnimatePresence } from 'framer-motion'
|
||||
import {
|
||||
@@ -441,7 +442,7 @@ function PluginKeyConfig({ plugin }: { plugin: PluginDataSourceItem }) {
|
||||
) : (
|
||||
<>向该数据源官方申请 API Key。</>
|
||||
)}
|
||||
Key 仅存本地 (secrets.json 优先, 环境变量 {plugin.api_key_env} 兜底),不会上传任何第三方,请妥善保管。
|
||||
Key 保存在本地,验证和取数时发送给该数据源用于认证。环境变量可作为后备配置。
|
||||
</p>
|
||||
|
||||
{/* 积分计费警示框 + Key/积分管理入口 (manifest billing/billing_note/key_manage_url 声明) */}
|
||||
@@ -462,7 +463,7 @@ function PluginKeyConfig({ plugin }: { plugin: PluginDataSourceItem }) {
|
||||
rel="noreferrer"
|
||||
className="inline-flex items-center gap-1 text-xs text-accent hover:underline mb-4"
|
||||
>
|
||||
管理 Key / 查看积分明细 (通达信商城)
|
||||
管理 Key / 查看账户用量
|
||||
<ExternalLink className="h-3 w-3" />
|
||||
</a>
|
||||
)}
|
||||
@@ -734,10 +735,10 @@ export function SettingsDataSourcesPanel({ highlight }: { highlight?: string } =
|
||||
{/* ===== 下区: 数据源 (源为主视图, 配置/接入) ===== */}
|
||||
<AnchorWrap highlight={highlight} anchor="data-sources">
|
||||
<section className="rounded-card border border-border bg-surface p-5">
|
||||
<div className="flex items-center justify-between mb-4">
|
||||
<div className="mb-4 flex flex-wrap items-center justify-between gap-x-4 gap-y-3">
|
||||
<div className="flex items-center gap-2.5 min-w-0">
|
||||
<Database className="h-4 w-4 text-secondary shrink-0" />
|
||||
<h2 className="text-sm font-medium text-foreground">数据源</h2>
|
||||
<h2 className="shrink-0 text-sm font-medium text-foreground">数据源</h2>
|
||||
<span
|
||||
className="text-[10px] text-muted/40 font-mono truncate hidden lg:inline max-w-[480px]"
|
||||
title={sources.data?.config_dir}
|
||||
@@ -745,20 +746,24 @@ export function SettingsDataSourcesPanel({ highlight }: { highlight?: string } =
|
||||
{sources.data?.config_dir}
|
||||
</span>
|
||||
</div>
|
||||
<button
|
||||
onClick={() => reload.mutate()}
|
||||
disabled={reload.isPending}
|
||||
className="inline-flex items-center gap-1.5 px-2.5 py-1 rounded-btn text-xs text-muted hover:text-foreground hover:bg-elevated transition-colors disabled:opacity-50 shrink-0"
|
||||
>
|
||||
<RefreshCw className={`h-3 w-3 ${reload.isPending ? 'animate-spin' : ''}`} />
|
||||
重新加载
|
||||
</button>
|
||||
<div className="ml-auto flex shrink-0 items-center gap-2">
|
||||
<ExtensionSlot name="settings.data-sources.toolbar" context={{ refresh: invalidateSources, selectSource: setSelected }} compact />
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => reload.mutate()}
|
||||
disabled={reload.isPending}
|
||||
className="inline-flex h-8 shrink-0 items-center justify-center gap-1.5 whitespace-nowrap rounded-btn border border-border/70 bg-surface px-3 text-xs font-medium text-secondary transition-colors hover:border-border hover:bg-elevated hover:text-foreground focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-accent/60 focus-visible:ring-offset-2 focus-visible:ring-offset-surface disabled:pointer-events-none disabled:opacity-50"
|
||||
>
|
||||
<RefreshCw aria-hidden="true" className={`h-3.5 w-3.5 ${reload.isPending ? 'animate-spin' : ''}`} />
|
||||
重新加载
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* 插件化说明 (置顶黄色提示条): 接入自有行情 → 把文档交给 AI */}
|
||||
<div className="mb-4 flex items-start gap-2 rounded-lg border border-warning/40 bg-warning/10 px-3 py-2.5">
|
||||
<Puzzle className="mt-0.5 h-3.5 w-3.5 shrink-0 text-warning" />
|
||||
<div className="text-[11px] leading-relaxed text-muted">
|
||||
{/* 开发说明是辅助信息, 警告色留给需要处理的异常状态。 */}
|
||||
<div className="mb-4 flex items-start gap-2 rounded-btn bg-elevated/40 px-3 py-2.5">
|
||||
<Puzzle className="mt-0.5 h-3.5 w-3.5 shrink-0 text-muted" />
|
||||
<div className="min-w-0 break-words text-[11px] leading-relaxed text-secondary">
|
||||
<span className="text-secondary">数据源已插件化</span>
|
||||
,接入自有行情?把文档发给 AI 即可自动接入:
|
||||
<span className="mx-0.5 rounded bg-elevated/70 px-1 py-px font-mono text-[10px] text-secondary">docs/custom-data-source.md</span>
|
||||
@@ -794,8 +799,8 @@ export function SettingsDataSourcesPanel({ highlight }: { highlight?: string } =
|
||||
<div
|
||||
key={item.name}
|
||||
onClick={() => {
|
||||
// 未就绪插件仅当支持界面配 Key 时可点开(进详情配置); 其余不可选
|
||||
if (pluginUnavailable && !plugin?.api_key_env) return
|
||||
// 外部插件始终可打开管理详情;内置插件沿用 Key 配置入口。
|
||||
if (pluginUnavailable && !plugin?.api_key_env && plugin?.origin !== 'external') return
|
||||
setSelected(item.name)
|
||||
// 只有用户自定义源 (YAML) 才进编辑器; tickflow 和插件不可编辑
|
||||
if (customNames.has(item.name)) {
|
||||
@@ -803,7 +808,7 @@ export function SettingsDataSourcesPanel({ highlight }: { highlight?: string } =
|
||||
}
|
||||
}}
|
||||
className={`relative text-left rounded-lg border px-3.5 py-3 transition-all ${
|
||||
pluginUnavailable && !plugin?.api_key_env
|
||||
pluginUnavailable && !plugin?.api_key_env && plugin?.origin !== 'external'
|
||||
? 'border-border/40 bg-elevated/10 opacity-70'
|
||||
: isSelected
|
||||
? 'border-accent/50 bg-accent/5 ring-1 ring-accent/20 cursor-pointer'
|
||||
@@ -848,7 +853,7 @@ export function SettingsDataSourcesPanel({ highlight }: { highlight?: string } =
|
||||
{pluginUnavailable ? (
|
||||
// 安装/卸载按钮只对"纯依赖型"插件(runtime≠none 且无 api_key_env)显示;
|
||||
// 带 Key 配置的插件不可用时, 卡片本身可点击进详情配 Key(如 TdxQuant 依赖已内置)。
|
||||
plugin?.runtime !== 'none' && !plugin?.api_key_env && (
|
||||
plugin?.origin !== 'external' && plugin?.runtime !== 'none' && !plugin?.api_key_env && (
|
||||
installing ? (
|
||||
<span className="inline-flex items-center gap-1 text-[10px] text-accent">
|
||||
<RefreshCw className="h-2.5 w-2.5 animate-spin" /> 安装中...
|
||||
@@ -872,7 +877,7 @@ export function SettingsDataSourcesPanel({ highlight }: { highlight?: string } =
|
||||
>
|
||||
套用
|
||||
</button>
|
||||
{plugin && plugin?.runtime !== 'none' && !plugin?.api_key_env && (
|
||||
{plugin && plugin?.origin !== 'external' && plugin?.runtime !== 'none' && !plugin?.api_key_env && (
|
||||
uninstalling ? (
|
||||
<RefreshCw className="h-2.5 w-2.5 animate-spin text-muted" />
|
||||
) : (
|
||||
@@ -977,6 +982,7 @@ export function SettingsDataSourcesPanel({ highlight }: { highlight?: string } =
|
||||
isActive={servingDatasets(selected).length > 0}
|
||||
matrixCaps={matrixCaps}
|
||||
servingSet={servingSetOf(selected)}
|
||||
refresh={invalidateSources}
|
||||
/>
|
||||
) : null}
|
||||
</motion.div>
|
||||
@@ -1017,7 +1023,8 @@ export function SettingsDataSourcesPanel({ highlight }: { highlight?: string } =
|
||||
}
|
||||
|
||||
/** 插件详情: 介绍 + 适配能力 + Key 配置合并为单一卡片 */
|
||||
function PluginDetail({ plugin, isActive, matrixCaps, servingSet }: {
|
||||
function PluginDetail({ plugin, isActive, matrixCaps, servingSet, refresh }: {
|
||||
refresh: () => void
|
||||
plugin: PluginDataSourceItem
|
||||
isActive: boolean
|
||||
matrixCaps: CapabilityRoute[]
|
||||
@@ -1065,6 +1072,8 @@ function PluginDetail({ plugin, isActive, matrixCaps, servingSet }: {
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<ExtensionSlot name="settings.data-sources.detail-actions" context={{ plugin, refresh }} />
|
||||
|
||||
{/* 主体: 操作 + Key 配置 (左) | 能力适配表 (右), 布局对齐 TickFlow 详情 */}
|
||||
<div className="mt-5 pt-5 border-t border-border grid grid-cols-1 lg:grid-cols-[1fr_1.15fr] gap-6 items-start">
|
||||
<div className="min-w-0">
|
||||
@@ -1076,7 +1085,7 @@ function PluginDetail({ plugin, isActive, matrixCaps, servingSet }: {
|
||||
)}
|
||||
|
||||
{/* API Key 配置 (声明了 api_key_env 的插件) */}
|
||||
{plugin.api_key_env && (
|
||||
{plugin.api_key_env && plugin.enabled !== false && (
|
||||
<div className={plugin.available ? 'mt-4' : ''}>
|
||||
<PluginKeyConfig plugin={plugin} />
|
||||
</div>
|
||||
|
||||
Reference in New Issue
Block a user