fix(sources): 自定义源全量分钟/分钟K amount 改为可选 + 保存先验后存

amount 按契约可空, 缺成交额的源此前被整体拒绝且保存静默失败;
现在校验不过返回 400 带原因, 不落盘。
This commit is contained in:
shy3130
2026-10-09 16:21:29 +08:00
parent fb69f382e7
commit ae9feb869c
4 changed files with 127 additions and 6 deletions
+15 -1
View File
@@ -749,10 +749,24 @@ def get_data_source(name: str) -> dict:
@router.post("/data-sources")
def save_data_source(req: CustomSourceIn) -> dict:
"""创建或更新一个自定义数据源 yaml, 保存后自动 reload。"""
"""创建或更新一个自定义数据源 yaml, 保存后自动 reload。
先验后存 (对齐插件 Key 的先探后存语义): 用与加载同一套校验试建临时
provider, 不通过直接 400 且不落盘 — 否则 YAML 写入但 load_all 校验失败,
源从列表静默消失, 前端还弹"已保存"成功提示 (用户反馈: 全量分钟缺
amount 映射被拒, 以为存上了)。
"""
from app.data_providers import custom as custom_sources
config = req.model_dump()
config["name"] = (config.get("name") or "").lower()
probe = None
try:
probe = custom_sources.create_provider(config) # 失败自抛 ValueError(含全部校验原因)
except ValueError as e:
raise HTTPException(status_code=400, detail=str(e)) from e
finally:
if probe is not None:
probe.close() # 临时探针(仅校验用), 不留连接资源
try:
custom_sources.save_config(config["name"], config)
custom_sources.load_all()
@@ -31,9 +31,12 @@ _REQUIRED = {
"daily": {"symbol", "date", "open", "high", "low", "close", "volume", "amount"},
"adj_factor": {"symbol", "trade_date", "ex_factor"},
"realtime": {"symbol", "last_price", "prev_close", "open", "high", "low", "volume"},
"minute": {"symbol", "datetime", "open", "high", "low", "close", "volume", "amount"},
"minute": {"symbol", "datetime", "open", "high", "low", "close", "volume"},
# full_minute (全量分钟) 与 minute 同形: 当日窗口批量拉取, 字段映射一致
"full_minute": {"symbol", "datetime", "open", "high", "low", "close", "volume", "amount"},
"full_minute": {"symbol", "datetime", "open", "high", "low", "close", "volume"},
# minute/full_minute 的 amount(成交额)按契约可空 (docs/plugin-development.md:
# 无法提供可靠成交额时应返回 null 不得伪造, 均价线降级显示 —), 不列为必填 —
# 否则缺成交额的源整个被 validate 拒绝, 保存后源从列表消失
# financial 字段由数据源决定, 只要求能映射出 symbol
"financial": {"symbol"},
}
+104
View File
@@ -380,3 +380,107 @@ def test_full_minute_dataset_survives_config_and_registration(tmp_path: Path, mo
custom_loader._PROVIDERS.clear()
custom_loader._PROVIDERS.update(saved_providers)
custom_loader._LOAD_ERRORS[:] = saved_errors
# ================================================================
# amount 可空契约 + 保存先验后存 (用户反馈: 自定义源配全量分钟"保存不好使")
# ================================================================
def _minute_dataset(*, drop_targets: tuple[str, ...] = ()) -> dict:
"""minute/full_minute 形态的数据集配置; drop_targets 指定不映射的内部字段。"""
field_map = {
"code": "symbol", "ts": "datetime",
"o": "open", "h": "high", "lo": "low", "c": "close", "v": "volume",
}
field_map = {k: v for k, v in field_map.items() if v not in drop_targets}
return {
"url": "https://example.test/minutes",
"method": "POST",
"response_path": "data",
"field_map": field_map,
"symbols_param": "codes",
}
def test_full_minute_without_amount_is_valid():
"""契约: 分钟 amount(成交额)可空 — 缺成交额的源合法, 均价线降级显示 —。
修复前: _REQUIRED 把 amount 列为必填 → validate 报 missing mapped fields,
整个源被 load_all 拒绝, 用户保存后源从列表消失("保存不好使")。
"""
from app.data_providers.custom import loader
provider = loader.create_provider({
"name": "fm_no_amount", "display_name": "fm",
"datasets": {"full_minute": _minute_dataset(drop_targets=("amount",))},
})
try:
assert not provider.validate(), provider.validate()
assert "full_minute" in provider.config.datasets
finally:
provider.close()
def test_minute_without_amount_is_valid():
"""minute 数据集同契约: amount 不映射也合法。"""
from app.data_providers.custom import loader
provider = loader.create_provider({
"name": "m_no_amount", "display_name": "m",
"datasets": {"minute": _minute_dataset(drop_targets=("amount",))},
})
try:
assert not provider.validate(), provider.validate()
finally:
provider.close()
def test_minute_missing_core_field_still_rejected():
"""必填集仍生效: 缺 volume(核心量字段)依旧拒绝, 不能矫枉过正。"""
from app.data_providers.custom import loader
with pytest.raises(ValueError, match="volume"):
loader.create_provider({
"name": "m_no_volume", "display_name": "m",
"datasets": {"minute": _minute_dataset(drop_targets=("volume",))},
})
def test_save_data_source_validates_before_write(monkeypatch, tmp_path):
"""保存先验后存: 校验不过 400 + 不落盘; 合法配置才写 YAML 并注册。
修复前: save_config 先写 YAML, load_all 校验失败仅记 _LOAD_ERRORS,
源从列表静默消失, 前端还弹"已保存"成功提示。
"""
from app.api.settings import save_data_source
from app.data_providers.custom import loader
monkeypatch.setattr(loader, "data_sources_dir", lambda: tmp_path)
saved_providers = dict(loader._PROVIDERS)
saved_status = dict(loader._PLUGIN_STATUS)
monkeypatch.setattr(loader, "_load_builtin_plugins", lambda: None)
try:
# 合法: full_minute 缺 amount 映射 → 保存成功且源注册
ok = CustomSourceIn(
name="fm_ok",
datasets={"full_minute": DatasetConfigIn(**_minute_dataset())},
)
save_data_source(ok)
assert (tmp_path / "fm_ok.yaml").exists()
assert loader.provider_has_dataset("fm_ok", "full_minute")
# 非法: 缺 volume → HTTP 400 带原因, 且不写 YAML
bad = CustomSourceIn(
name="fm_bad",
datasets={"full_minute": DatasetConfigIn(**_minute_dataset(drop_targets=("volume",)))},
)
with pytest.raises(HTTPException) as ei:
save_data_source(bad)
assert ei.value.status_code == 400
assert "volume" in str(ei.value.detail)
assert not (tmp_path / "fm_bad.yaml").exists()
finally:
loader._PROVIDERS.clear()
loader._PROVIDERS.update(saved_providers)
loader._PLUGIN_STATUS.clear()
loader._PLUGIN_STATUS.update(saved_status)
+3 -3
View File
@@ -11,7 +11,7 @@
| 日K | `daily` | 批量返回一组股票在指定区间内的日K |
| 除权因子 | `adj_factor` | 批量返回一组股票的复权因子 |
| 实时行情 | `realtime` | 返回全市场快照,用于盘中 enriched 增量计算 |
| 分钟K | `minute` | 返回 1m 分钟K(需映射出 symbol / datetime / OHLC / 量额) |
| 分钟K | `minute` | 返回 1m 分钟K(需映射出 symbol / datetime / OHLC / 量; amount 成交额可缺省, 缺省时均价线降级显示 —) |
| 全量分钟 | `full_minute` | 与 `minute` 同形;声明后可被路由为「全量分钟」生效源,内置服务盘中按当日窗口全市场批量落盘(仅修复轮语义,节奏下限 60s) |
| 财务数据 | `financial` | 一个配置覆盖全部财务表,请求时把表名作为参数传给上游;字段由数据源决定,仅需映射出 symbol |
@@ -295,7 +295,7 @@ cp docs/examples/custom-data-source/mock_source.yaml data/data_sources/mock_sour
prev_close = 昨收价
open / high / low = 当日 OHLC
volume = 成交量
amount = 成交额
amount = 成交额 (可缺省)
change_pct = 涨跌幅 (小数, 0.0366 = 3.66%)
change_amount = 涨跌额
amplitude = 振幅 (小数, 0.024 = 2.4%)
@@ -310,7 +310,7 @@ cp docs/examples/custom-data-source/mock_source.yaml data/data_sources/mock_sour
datetime = 北京时间墙钟 (YYYY-MM-DD HH:MM:SS)
open / high / low / close = OHLC
volume = 成交量
amount = 成交额
amount = 成交额 (可缺省: 无法提供可靠成交额时不要映射, 前端均价线降级显示 —)
=== 我的 API 文档 ===
[把你的接口文档贴这里: URL / 请求方式 / 参数 / 返回字段说明]