fix(分钟数据源): 阻止自定义源失败后回退 TickFlow

This commit is contained in:
Break Zhang
2026-10-20 01:55:08 +08:00
parent 4f9af038d8
commit 62affd5a41
9 changed files with 326 additions and 151 deletions
+2 -2
View File
@@ -12,7 +12,7 @@ from fastapi import APIRouter, HTTPException, Query, Request
from app.indicators.pipeline import compute_enriched
from app.market_time import cn_today
from app.services import index_sync, kline_sync, preferences, trading_day
from app.services import index_sync, kline_sync, preferences, trading_day # noqa: F401
from app.tickflow.capabilities import Cap
logger = logging.getLogger(__name__)
@@ -105,7 +105,7 @@ def get_index_minute(
"rows": [],
"source": "future",
}
cache_key = (symbol, day.isoformat(), preferences.get_minute_data_provider())
cache_key = (symbol, day.isoformat(), kline_sync._selected_minute_provider_name())
now = time.monotonic()
with _index_minute_cache_lock:
hit = _index_minute_cache.get(cache_key)
+5 -6
View File
@@ -81,16 +81,15 @@ def _gzip_payload(request: Request, payload: dict, *, pref_key: str) -> dict | R
def _minute_allowed(capset) -> bool:
"""是否有分钟K权限 (TickFlow Pro+ 或 custom minute 源)。"""
"""仅当前分钟数据源确实可用时开放分钟 K 接口。"""
from app.tickflow.capabilities import Cap
if capset.has(Cap.KLINE_MINUTE_BATCH):
return True
from app.services import preferences
provider = preferences.get_minute_data_provider()
provider = kline_sync._selected_minute_provider_name()
if provider == "tickflow":
return capset.has(Cap.KLINE_MINUTE_BATCH)
_, fallback, error = kline_sync._resolve_minute_provider(provider)
if error is not None:
logger.warning("minute provider resolution failed while checking access: %s", error)
return not fallback
return error is None and not fallback
@lru_cache(maxsize=8192)
+77 -51
View File
@@ -956,32 +956,41 @@ def _write_minute_partition(df: pl.DataFrame, minute_dir) -> int:
return written
def _selected_minute_provider_name() -> str:
"""识别偏好 getter 静默回退的失效自定义分钟源。"""
saved = str(preferences.load().get("minute_data_provider", "tickflow") or "tickflow").lower()
selected = preferences.get_minute_data_provider()
if saved != "tickflow" and selected == "tickflow":
return f"unavailable:{saved}"
return selected
def _resolve_minute_provider(
provider_name: str,
) -> tuple[object | None, bool, str | None]:
"""统一解析 custom minute provider, 把所有 resolver 调用纳入同一异常边界。
供 _try_custom_minute 和 sync_and_persist_minute 共用, 避免两处分别调
provider_has_dataset / get_provider 时漏掉异常边界 (Issue 2 加固项)。
供分钟拉取和落盘共用,解析失败统一返回错误,不跨源补数据。
返回 (provider, should_fallback_to_tickflow, error_msg):
- provider_name == "tickflow" 或未配 minute dataset → (None, True, None) 静默降级
- resolver 异常 (registry 损坏 / 插件失效 / provider name 不存在) → (None, True, str(e))
- 成功 → (provider, False, None)
- provider_name == "tickflow" → (None, True, None)
- 自定义源未配置 minute dataset 或解析失败 → (None, False, error_msg)
- 自定义源可用 → (provider, False, None)
上层依据 error_msg 决定是否 logger.warning (区分"未配"与"异常")。
只有用户明确选择 TickFlow 时才请求 TickFlow,避免静默跨源。
注意: provider.get_minute() 仍由调用方在自身 try 块内调用 (业务异常, 非解析异常)。
"""
if provider_name == "tickflow":
return (None, True, None)
from app.data_providers import custom as custom_sources
try:
if not custom_sources.provider_has_dataset(provider_name, "minute"):
return (None, True, None)
provider = custom_sources.get_provider(provider_name)
return (provider, False, None)
datasets = provider.config.datasets
except Exception as e: # noqa: BLE001
return (None, True, str(e))
return (None, False, str(e))
if "minute" not in datasets:
return (None, False, f"provider {provider_name} does not configure minute data")
return (provider, False, None)
def _try_custom_minute(
@@ -995,11 +1004,11 @@ def _try_custom_minute(
"""尝试从自定义分钟源拉取。返回 (df, should_fallback_to_tickflow)。
返回契约:
(None, True) → 未配自定义源 / 未配 minute dataset / 自定义源异常 → 走 TickFlow
(df, False) → 自定义源成功(含空 df) → 直接用, 不回退
(None, True) → 用户明确选择 TickFlow
(df, False) → 自定义源成功(含空 df)或失败 → 直接使用结果, 不回退
自定义源异常时返回 fallback=True。单股拉取调用方另行检查 TickFlow 原生
能力, 避免自定义源增广能力误放行无权限请求。
用户明确选择了自定义分钟源后,其请求失败或解析异常均 fail-closed;
避免自定义源增广的 batch 能力被误认为 TickFlow 原生权限,造成静默跨源请求。
resolver 异常边界由 _resolve_minute_provider 统一兜底; 业务调用
(provider.get_minute) 仍在本函数 try 块内, 与 resolver 异常分离
@@ -1009,13 +1018,23 @@ def _try_custom_minute(
实现内部以 2 参 (cur, total) 调用。这里包装一层, provider 调 2 参时补
默认 seg_label="custom" 转发给上层, 保证进度展示不降级。
"""
provider_name = preferences.get_minute_data_provider()
provider_name = _selected_minute_provider_name()
if provider_name.startswith("unavailable:"):
logger.warning("custom minute provider %s is unavailable; not falling back to TickFlow",
provider_name.removeprefix("unavailable:"))
return (pl.DataFrame(), False)
provider, fallback, err = _resolve_minute_provider(provider_name)
if fallback:
if err is not None:
logger.warning("custom minute provider %s resolution failed, falling back to TickFlow: %s",
provider_name, err)
logger.warning(
"custom minute provider %s unavailable; not falling back to TickFlow: %s",
provider_name, err,
)
return (pl.DataFrame(), False)
return (None, True)
if err is not None or provider is None:
logger.warning("custom minute provider %s has no usable minute dataset: %s", provider_name, err)
return (pl.DataFrame(), False)
# 包装 on_chunk_done: provider 调 2 参 → 补 seg_label="custom" → 转发上层 3 参
wrapped_cb: Callable[[int, int], None] | None = None
@@ -1029,17 +1048,21 @@ def _try_custom_minute(
symbols, start_time=start_time, end_time=end_time,
asset_type=asset_type, freq=freq, on_chunk_done=wrapped_cb,
)
except Exception as e:
logger.warning("custom minute provider %s call failed, falling back to TickFlow: %s",
provider_name, e)
return (None, True)
except Exception as e: # noqa: BLE001
logger.warning(
"custom minute provider %s call failed; not falling back to TickFlow: %s",
provider_name, e,
)
return (pl.DataFrame(), False)
try:
# 时区契约守卫: 插件/自定义源帧同样收口为北京墙钟 (CONTRIBUTING §3.3)
df = _enforce_minute_beijing_wallclock(df, source=provider_name)
except Exception as e:
logger.warning("custom minute provider %s datetime 契约校验失败, falling back to TickFlow: %s",
provider_name, e)
return (None, True)
except Exception as e: # noqa: BLE001
logger.warning(
"custom minute provider %s datetime 契约校验失败; not falling back to TickFlow: %s",
provider_name, e,
)
return (pl.DataFrame(), False)
return (df, False)
@@ -1077,7 +1100,7 @@ def sync_minute_batch(
asset_type=asset_type, freq="1m", on_chunk_done=on_chunk_done,
)
if not fallback:
# 自定义源成功: 遵守与 TickFlow 路径一致的 on_segment 契约。
# 自定义源已选中: 遵守与 TickFlow 路径一致的 on_segment 契约;失败时返回空帧。
# 传了 on_segment (如 sync_and_persist_minute 流式落盘) → 调 on_segment, 返回空 df;
# 未传 on_segment (如 fetch_minute_single 实时补拉) 或空 df → 原样返回 df。
df = df if df is not None else pl.DataFrame()
@@ -1176,15 +1199,19 @@ def sync_minute_batch(
def intraday_monitor_support(capset: CapabilitySet | None) -> dict[str, object]:
"""返回分时信号监控可用的数据能力和单轮标的上限。"""
provider_name = preferences.get_minute_data_provider()
provider_name = _selected_minute_provider_name()
_, fallback, error = _resolve_minute_provider(provider_name)
if error is not None:
logger.warning("minute provider resolution failed while checking monitor support: %s", error)
return {
"available": False, "source": None, "max_symbols": 0,
"reason": "当前分钟数据源不可用,请检查数据源配置",
}
if not fallback:
return {
"available": True, "source": "custom_minute", "max_symbols": 100,
"reason": "使用已配置的分钟数据插件",
}
if error is not None:
logger.warning("minute provider resolution failed while checking monitor support: %s", error)
if capset is None:
return {
"available": False, "source": None, "max_symbols": 0,
@@ -1377,23 +1404,23 @@ def _resolve_full_minute_provider(
provider_name: str,
) -> tuple[object | None, bool, str | None]:
"""解析全量分钟生效的自定义源。返回 (provider, should_use_tickflow, error_msg):
返回 (provider, should_use_tickflow, error_msg):
- provider_name == "tickflow" / 未配 full_minute dataset → (None, True, None)
- resolver 异常 (registry 损坏 / 插件失效 / 源不存在) → (None, True, str(e))
- 明确选择 tickflow → (None, True, None)
- 自定义源缺少 full_minute 或解析失败 → (None, False, error_msg)
- 成功 → (provider, False, None)
与 _resolve_minute_provider 同构, 仅数据集名不同。
"""
if provider_name == "tickflow":
return (None, True, None)
from app.data_providers import custom as custom_sources
try:
if not custom_sources.provider_has_dataset(provider_name, "full_minute"):
return (None, True, None)
provider = custom_sources.get_provider(provider_name)
return (provider, False, None)
datasets = provider.config.datasets
except Exception as e: # noqa: BLE001
return (None, True, str(e))
return (None, False, str(e))
if "full_minute" not in datasets:
return (None, False, f"provider {provider_name} does not configure full_minute data")
return (provider, False, None)
def fetch_intraday_custom_batch(
@@ -1477,8 +1504,8 @@ def fetch_minute_single(
) -> pl.DataFrame:
"""实时拉取单股单日分钟 K(不写入本地)。
优先使用当前自定义分钟源。仅当 TickFlow 原生单股分钟能力存在时才允许
回退 TickFlow; 自定义源增广只授予 batch 能力, 不会误放行该回退路径。
优先使用当前自定义分钟源。只有明确选择 TickFlow 且具备原生权限时才请求 TickFlow;
已选自定义源缺少能力或请求失败时 fail-closed,不混用来源。
"""
from datetime import datetime
# 北京时间窗口必须带时区: naive datetime 会被 .timestamp() 按服务器本地时区解释,
@@ -1486,8 +1513,7 @@ def fetch_minute_single(
start_time = datetime(trade_date.year, trade_date.month, trade_date.day, 9, 25, 0, tzinfo=CN_TZ)
end_time = datetime(trade_date.year, trade_date.month, trade_date.day, 15, 5, 0, tzinfo=CN_TZ)
# 自定义数据源分流: 与 sync_minute_batch 一致, 配了自定义分钟源时走 custom provider,
# 避免无 TickFlow Pro+ 权限的用户分时图首次打开(本地无数据)时补拉失败返回空。
# 按用户当前选择的分钟数据源路由,避免无提示跨源。
df, fallback = _try_custom_minute(
[symbol], start_time=start_time, end_time=end_time,
asset_type=asset_type, freq="1m",
@@ -1665,18 +1691,18 @@ def sync_and_persist_minute(
on_chunk_done(current, total) 每个 chunk 完成后回调。
force_full_days=True 时强制回溯 days 自然日 (不增量补, 用于个股补齐历史)。
"""
minute_provider = preferences.get_minute_data_provider()
# resolver 调用统一走 _resolve_minute_provider, 与 _try_custom_minute 共用异常边界。
# resolver 异常时视为非 custom (minute_is_custom=False), 走 capset 检查 →
# sync_minute_batch 内 _try_custom_minute 会再次 resolver 异常 → fallback TickFlow。
_, fallback, resolve_err = _resolve_minute_provider(minute_provider)
minute_is_custom = not fallback
if resolve_err is not None:
logger.warning("custom minute provider %s resolution failed at sync_and_persist_minute, treating as non-custom: %s",
minute_provider, resolve_err)
minute_provider = _selected_minute_provider_name()
# 自定义源的能力和请求失败时都 fail-closed;共享能力集可能包含第三方源增补。
if not symbols:
return 0
if not minute_is_custom and not capset.has(Cap.KLINE_MINUTE_BATCH):
_, fallback, resolve_err = _resolve_minute_provider(minute_provider)
if resolve_err is not None:
logger.warning(
"custom minute provider %s resolution failed at sync_and_persist_minute: %s",
minute_provider, resolve_err,
)
return 0
if fallback and not capset.has(Cap.KLINE_MINUTE_BATCH):
return 0
# 迁移:旧版 _normalize_minute 未转换 timestamp→datetime,导致全部 datetime 为 null
+27 -14
View File
@@ -150,17 +150,21 @@ class MinuteRefreshService:
return False
def active_provider(self) -> str:
"""全量分钟生效源名 ("tickflow" 或自定义源名, 偏好 full_minute_data_provider)。"""
"""全量分钟生效源名;失效的自定义源不得静默伪装成 TickFlow。"""
try:
return preferences.get_full_minute_data_provider()
except Exception: # noqa: BLE001 — 偏好文件异常按 TickFlow 处理
return "tickflow"
saved = str(preferences.load().get("full_minute_data_provider", "tickflow") or "tickflow").lower()
selected = preferences.get_full_minute_data_provider()
if saved != "tickflow" and selected == "tickflow":
return f"unavailable:{saved}"
return selected
except Exception: # noqa: BLE001
return "unavailable:unknown"
def _resolve_custom(self) -> tuple[object | None, str]:
"""解析自定义源。返回 (provider_or_None, effective_name):
- 偏好 tickflow / 源未声明 full_minute 数据集 / 解析异常 → (None, "tickflow")
(与 minute 数据集同纪律: 静默降级 TickFlow, 能力门控决定能否真正运行)
- 明确偏好 tickflow → (None, "tickflow")
- 自定义源失效或缺少 full_minute → (None, "unavailable:<name>")
- 成功 → (provider, name)
"""
from app.services import kline_sync
@@ -168,13 +172,17 @@ class MinuteRefreshService:
name = self.active_provider()
if name == "tickflow":
return (None, "tickflow")
provider, use_tickflow, err = kline_sync._resolve_full_minute_provider(name)
if use_tickflow:
if err is not None:
logger.warning(
"full_minute provider %s 解析失败, 本轮降级 TickFlow: %s", name, err,
)
return (None, "tickflow")
if name.startswith("unavailable:"):
logger.warning("full_minute provider %s unavailable; not falling back to TickFlow",
name.removeprefix("unavailable:"))
return (None, name)
provider, _use_tickflow, err = kline_sync._resolve_full_minute_provider(name)
if err is not None or provider is None:
logger.warning(
"full_minute provider %s unavailable; not falling back to TickFlow: %s",
name, err,
)
return (None, f"unavailable:{name}")
return (provider, name)
def _custom_supports_increment(self, provider: object) -> bool:
@@ -183,7 +191,7 @@ class MinuteRefreshService:
def repair_only(self) -> bool:
"""当前生效源只能全天修复轮 (无廉价增量端点) — 节奏下限抬到 60s。"""
provider, name = self._resolve_custom()
provider, _name = self._resolve_custom()
return provider is not None and not self._custom_supports_increment(provider)
def _effective_interval(self) -> int:
@@ -282,6 +290,11 @@ class MinuteRefreshService:
t0 = time.perf_counter()
custom, provider_name = self._resolve_custom()
if provider_name.startswith("unavailable:"):
self._state.last_error = f"full_minute provider unavailable: {provider_name.removeprefix('unavailable:')}"
self._state.last_requests = 0
self._state.last_symbols = 0
return
mode = self._select_mode()
if mode == "increment" and custom is not None and not self._custom_supports_increment(custom):
# 无廉价增量端点的源: 增量轮退化为修复轮 (全天批量幂等覆盖, 数据不丢)
@@ -113,6 +113,25 @@ def test_cache_ttl_and_provider_switch(context, monkeypatch):
assert fetch.call_count == 3
def test_unavailable_custom_source_does_not_poison_tickflow_cache(context, monkeypatch):
request, fetch = context
day = date(2026, 9, 11)
fetch.side_effect = [pl.DataFrame(), pl.DataFrame({
"datetime": [datetime(2026, 9, 11, 9, 35)], "close": [3000.0],
})]
monkeypatch.setattr(indices.preferences, "load", lambda: {"minute_data_provider": "missing_src"})
monkeypatch.setattr(indices.preferences, "get_minute_data_provider", lambda: "tickflow")
first = indices.get_index_minute(request, symbol="000001.SH", trade_date=day)
assert first["source"] == "none"
monkeypatch.setattr(indices.preferences, "load", lambda: {"minute_data_provider": "tickflow"})
second = indices.get_index_minute(request, symbol="000001.SH", trade_date=day)
assert second["source"] == "live"
assert fetch.call_count == 2
assert len(indices._index_minute_cache) == 2
def test_cache_separates_dates_and_symbols_and_stays_bounded(context, monkeypatch):
request, fetch = context
monkeypatch.setattr(indices, "_INDEX_MINUTE_CACHE_MAX", 2)
@@ -165,6 +165,7 @@ def test_free_tier_skips_tickflow_minute_fallback(monkeypatch, live):
def test_custom_minute_source_remains_available_without_tickflow_capability(monkeypatch):
provider = MagicMock()
provider.config.datasets = {"minute": object()}
provider.get_minute.return_value = _minute_rows(1)
monkeypatch.setattr("app.services.preferences.get_minute_data_provider", lambda: "custom")
monkeypatch.setattr(
@@ -187,6 +188,7 @@ def test_custom_minute_source_remains_available_without_tickflow_capability(monk
def test_failed_custom_source_does_not_fall_back_to_unsupported_tickflow(monkeypatch):
provider = MagicMock()
provider.config.datasets = {"minute": object()}
provider.get_minute.side_effect = RuntimeError("custom source unavailable")
monkeypatch.setattr("app.services.preferences.get_minute_data_provider", lambda: "custom")
monkeypatch.setattr(
+43 -7
View File
@@ -102,7 +102,7 @@ def test_gate_disabled(tmp_path, monkeypatch):
def test_gate_full_minute_custom_provider_runs(tmp_path, monkeypatch):
"""全量分钟路由到自定义源: 不再让位 — 能力口径统一 (capset 增广后放行)。"""
class _P: # noqa: D401 — fake provider
class _P:
def get_intraday_batch(self, symbols, count=300, asset_type="stock"):
return pl.DataFrame()
svc = _svc(tmp_path, monkeypatch, full_minute_provider="myfm", custom=_P())
@@ -369,9 +369,8 @@ def test_custom_provider_latest_missing_forces_full_with_minute_fallback(tmp_pat
assert svc._effective_interval() == 60
def test_custom_provider_unresolved_degrades_to_tickflow(tmp_path, monkeypatch):
"""路由指向未声明 full_minute 数据集的源 (真实 resolver 判定) → 降级
TickFlow 路径, 门控口径不变。"""
def test_custom_provider_unresolved_fails_closed(tmp_path, monkeypatch):
"""失效的 full_minute 自定义源不再静默降级并请求 TickFlow。"""
svc = _svc(tmp_path, monkeypatch, full_minute_provider="not-registered")
monkeypatch.setattr(
minute_refresh.MinuteRefreshService, "_today_coverage_lag_minutes",
@@ -388,10 +387,47 @@ def test_custom_provider_unresolved_degrades_to_tickflow(tmp_path, monkeypatch):
lambda symbols, capset, *, count=300: (modes.append("tf-full"), (_full_df(), 28))[1],
)
svc._run_round()
assert modes == ["tf-inc"]
assert modes == []
st = svc.status()
assert st["provider_effective"] == "tickflow"
assert st["last_mode"] == "increment"
assert st["provider_effective"] == "unavailable:not-registered"
assert "provider unavailable" in st["last_error"]
assert st["last_requests"] == 0
assert st["last_symbols"] == 0
def test_unloaded_full_minute_provider_does_not_fall_back_to_tickflow(tmp_path, monkeypatch):
"""偏好仍选自定义源但插件卸载后,不能借旧能力快照发 TickFlow 请求。"""
svc = _svc(tmp_path, monkeypatch, full_minute_provider="tickflow")
preferences.save({"full_minute_data_provider": "missing_src"})
calls: list[str] = []
monkeypatch.setattr(
"app.services.kline_sync.fetch_intraday_universe_increment",
lambda *args, **kwargs: (calls.append("tickflow"), (pl.DataFrame(), 1))[1],
)
svc._run_round()
assert calls == []
assert svc.status()["provider_effective"] == "unavailable:missing_src"
def test_custom_provider_without_full_minute_dataset_fails_closed(tmp_path, monkeypatch):
"""已注册但未声明 full_minute 的源不能借共享能力集合请求 TickFlow。"""
from app.data_providers import custom as custom_sources
svc = _svc(tmp_path, monkeypatch, full_minute_provider="other")
provider = type("Provider", (), {"config": type("Config", (), {"datasets": {"minute": object()}})()})()
monkeypatch.setattr(custom_sources, "get_provider", lambda name: provider)
calls: list[str] = []
monkeypatch.setattr(
"app.services.kline_sync.fetch_intraday_universe_increment",
lambda *args, **kwargs: (calls.append("tickflow"), (pl.DataFrame(), 1))[1],
)
monkeypatch.setattr(minute_refresh.MinuteRefreshService, "_today_coverage_lag_minutes", lambda self: 0.2)
svc._run_round()
assert calls == []
assert svc.status()["provider_effective"] == "unavailable:other"
assert "provider unavailable" in svc.status()["last_error"]
def test_status_reports_gate_reason_when_stopped(tmp_path, monkeypatch):
+143 -66
View File
@@ -61,6 +61,8 @@ def _setup_custom_provider(monkeypatch, provider: object, has_dataset: bool = Tr
"get_minute_data_provider",
lambda: "mock_src",
)
monkeypatch.setattr("app.data_providers.custom.names", lambda: {"mock_src"})
provider.config.datasets = {"minute": object()} if has_dataset else {}
monkeypatch.setattr(
"app.data_providers.custom.provider_has_dataset",
lambda name, ds: has_dataset,
@@ -97,45 +99,56 @@ def test_custom_minute_provider_returns_1m_k(monkeypatch):
assert kwargs.get("asset_type") == "stock"
# ---------- 测试 3: 自定义源异常 + TickFlow 也失败 → 返回空 (非 500) ----------
# ---------- 测试 3: 自定义源失败时不跨源回退 ----------
def test_custom_provider_exception_no_500(monkeypatch):
"""§4 测试 3: 自定义源抛异常 + TickFlow 也失败,
fetch_minute_single / sync_minute_batch 返回空 df。
"""
# 自定义源抛异常
def test_unavailable_selected_provider_does_not_fall_back_to_tickflow(monkeypatch):
"""保存的自定义源当前未加载时,也不能被偏好 getter 静默改成 TickFlow。"""
monkeypatch.setattr(
kline_sync.preferences, "load", lambda: {"minute_data_provider": "missing_src"},
)
monkeypatch.setattr(kline_sync.preferences, "get_minute_data_provider", lambda: "tickflow")
monkeypatch.setattr("app.data_providers.custom.names", lambda: set())
get_client_spy = MagicMock(name="get_client_spy")
monkeypatch.setattr(kline_sync, "get_client", get_client_spy)
result = kline_sync.sync_minute_batch(
["600519.SH"], start_time=datetime(2026, 1, 15, 9, 25),
end_time=datetime(2026, 1, 15, 15, 5),
)
assert result.is_empty()
get_client_spy.assert_not_called()
def test_custom_provider_exception_does_not_fall_back_to_tickflow(monkeypatch):
"""自定义分钟源已选中但请求失败时,不再悄悄改请求 TickFlow。"""
mock_provider = MagicMock()
mock_provider.get_minute.side_effect = httpx.TimeoutException("timeout")
_setup_custom_provider(monkeypatch, mock_provider, has_dataset=True)
# mock get_client 返回 mock client, 其 klines.batch raise (TickFlow 也失败)
mock_tf = MagicMock()
mock_tf.klines.batch.side_effect = Exception("tickflow fail")
monkeypatch.setattr(kline_sync, "get_client", lambda: mock_tf)
get_client_spy = MagicMock(name="get_client_spy")
monkeypatch.setattr(kline_sync, "get_client", get_client_spy)
# fetch_minute_single: 自定义源异常 → fall through → TickFlow 异常 → 返回空
df_single = kline_sync.fetch_minute_single(
"600519.SH", date(2026, 1, 15), asset_type="stock",
capset=_tickflow_minute_capset(),
)
assert isinstance(df_single, pl.DataFrame)
assert df_single.is_empty()
# sync_minute_batch: 同一路径, 返回空
df_batch = kline_sync.sync_minute_batch(
["600519.SH"],
start_time=datetime(2026, 1, 15, 9, 25, 0),
end_time=datetime(2026, 1, 15, 15, 5, 0),
asset_type="stock",
)
assert isinstance(df_batch, pl.DataFrame)
assert df_batch.is_empty()
assert isinstance(df_single, pl.DataFrame) and df_single.is_empty()
assert isinstance(df_batch, pl.DataFrame) and df_batch.is_empty()
get_client_spy.assert_not_called()
# ---------- 测试 4: 未配 minute dataset → 回退 TickFlow ----------
# ---------- 测试 4: 未配 minute dataset 时不跨源 ----------
def test_provider_without_minute_dataset_fallback(monkeypatch):
"""§4 测试 4: provider_has_dataset 返回 False → (None, True) 回退 TickFlow。"""
def test_provider_without_minute_dataset_fails_closed(monkeypatch):
"""当前源缺少 minute dataset 时不静默回退到 TickFlow。"""
mock_provider = MagicMock()
_setup_custom_provider(monkeypatch, mock_provider, has_dataset=False)
@@ -143,9 +156,9 @@ def test_provider_without_minute_dataset_fallback(monkeypatch):
["600519.SH"], None, None, asset_type="stock",
)
assert fallback is True
assert df is None
# provider.get_minute 不应被调用 (回退决策在前)
assert fallback is False
assert df is not None and df.is_empty()
# provider.get_minute 不应被调用 (能力缺失直接 fail-closed)
mock_provider.get_minute.assert_not_called()
@@ -493,7 +506,36 @@ def test_sync_minute_batch_custom_empty_df_skips_on_segment(monkeypatch):
assert df.is_empty()
# ---------- 测试 12: sync_and_persist_minute + custom provider 端到端落盘 (Issue 1) ----------
# ---------- 测试 12: 同步落盘的自定义源失败路径 ----------
def test_sync_and_persist_minute_custom_failure_does_not_call_tickflow(monkeypatch, tmp_path):
"""自定义源失败时,免费档增补的分钟能力不能触发 TickFlow 回退。"""
from app.tickflow.capabilities import Cap, CapabilityLimits, CapabilitySet
provider = MagicMock()
provider.get_minute.side_effect = httpx.TimeoutException("timeout")
_setup_custom_provider(monkeypatch, provider, has_dataset=True)
monkeypatch.setattr(kline_sync, "_cleanup_null_datetime_minute", lambda repo: None)
monkeypatch.setattr(kline_sync, "_migrate_symbol_to_date_partition", lambda repo: None)
monkeypatch.setattr(kline_sync, "_latest_minute_datetime", lambda repo: None)
monkeypatch.setattr(kline_sync.preferences, "get_minute_sync_segment_days", lambda: 20)
monkeypatch.setattr(kline_sync, "resolve_limit", lambda *a, **kw: MagicMock(batch=100, rpm=30))
write_spy = MagicMock()
monkeypatch.setattr(kline_sync, "_write_minute_partition", write_spy)
get_client_spy = MagicMock(name="get_client_spy")
monkeypatch.setattr(kline_sync, "get_client", get_client_spy)
repo = MagicMock()
repo.store.data_dir = tmp_path
repo._write_lock = Lock()
capset = CapabilitySet({Cap.KLINE_MINUTE_BATCH: CapabilityLimits()})
written = kline_sync.sync_and_persist_minute(["600519.SH"], repo, capset)
assert written == 0
write_spy.assert_not_called()
get_client_spy.assert_not_called()
def test_sync_and_persist_minute_custom_persists(monkeypatch, tmp_path):
"""Issue 1 端到端: sync_and_persist_minute + 自定义源 →
@@ -569,17 +611,16 @@ def test_sync_and_persist_minute_holds_repository_write_lock(monkeypatch, tmp_pa
assert written == expected_df.height
# ---------- 测试 13: get_provider 异常时 fall through TickFlow (Issue 2) ----------
# ---------- 测试 13: get_provider 异常时 fail-closed ----------
def test_get_provider_exception_falls_back_to_tickflow(monkeypatch):
"""Issue 2: get_provider raise ValueError →
_try_custom_minute 返回 (None, True), 无异常穿透。
"""
def test_get_provider_exception_does_not_fall_back_to_tickflow(monkeypatch):
"""选中的自定义源无法解析时 fail-closed,不向其他数据源回退。"""
monkeypatch.setattr(
kline_sync.preferences,
"get_minute_data_provider",
lambda: "mock_src",
)
monkeypatch.setattr("app.data_providers.custom.names", lambda: {"mock_src"})
monkeypatch.setattr(
"app.data_providers.custom.provider_has_dataset",
lambda name, ds: True, # provider 存在, 但 get_provider 会抛
@@ -596,21 +637,21 @@ def test_get_provider_exception_falls_back_to_tickflow(monkeypatch):
["600519.SH"], None, None, asset_type="stock",
)
assert fallback is True
assert df is None
assert fallback is False
assert df is not None and df.is_empty()
# ---------- 测试 14: provider_has_dataset 异常时 fall through (Issue 2) ----------
# ---------- 测试 14: provider_has_dataset 异常时 fail-closed ----------
def test_provider_has_dataset_exception_falls_back(monkeypatch):
"""Issue 2: provider_has_dataset raise →
_try_custom_minute 返回 (None, True), 无异常穿透。
"""
def test_provider_has_dataset_exception_does_not_fall_back(monkeypatch):
"""自定义源注册信息异常时 fail-closed,不向 TickFlow 请求数据。"""
monkeypatch.setattr(
kline_sync.preferences,
"get_minute_data_provider",
lambda: "mock_src",
)
monkeypatch.setattr("app.data_providers.custom.names", lambda: {"mock_src"})
monkeypatch.setattr("app.data_providers.custom.get_provider", lambda name: MagicMock(config=MagicMock(datasets={"minute": {}})))
def _raising_has_dataset(name, ds):
raise RuntimeError("registry corrupted")
@@ -623,8 +664,8 @@ def test_provider_has_dataset_exception_falls_back(monkeypatch):
["600519.SH"], None, None, asset_type="stock",
)
assert fallback is True
assert df is None
assert fallback is False
assert df is not None and df.is_empty()
# ---------- 测试 15-17: GenericHTTPProvider opt-in 参数传递 (Issue 3) ----------
@@ -732,67 +773,84 @@ def test_sync_and_persist_minute_resolver_exception_returns_zero(monkeypatch, tm
assert written == 0
# ---------- 测试 19: _resolve_minute_provider helper 单元测试 ----------
# ---------- 测试 19: _resolve_minute_provider 单元测试 ----------
def test_resolve_minute_provider_tickflow_returns_silent_fallback():
"""观察项加固: provider_name == "tickflow" → (None, True, None) 静默降级, 无 err。"""
"""provider_name == "tickflow" 时使用原生数据源。"""
provider, fallback, err = kline_sync._resolve_minute_provider("tickflow")
assert provider is None
assert fallback is True
assert err is None
def test_resolve_minute_provider_no_dataset_returns_silent_fallback(monkeypatch):
"""观察项加固: 配了 custom 但未配 minute dataset → (None, True, None) 静默降级。"""
def test_resolve_minute_provider_no_dataset_fails_closed(monkeypatch):
"""选中的源没有 minute dataset 时明确停用,不静默切换来源。"""
monkeypatch.setattr("app.data_providers.custom.names", lambda: {"mock_src"})
provider = MagicMock()
provider.config.datasets = {}
monkeypatch.setattr("app.data_providers.custom.get_provider", lambda name: provider)
monkeypatch.setattr(
"app.data_providers.custom.provider_has_dataset",
lambda name, ds: False, # 已注册但未配 minute
)
provider, fallback, err = kline_sync._resolve_minute_provider("mock_src")
assert provider is None
assert fallback is True
assert err is None # 未配 ≠ 异常, 不应触发 warning
assert fallback is False
assert err is not None
assert "does not configure minute" in err
def test_resolve_minute_provider_has_dataset_exception_returns_err(monkeypatch):
"""观察项加固: provider_has_dataset 抛异常 → (None, True, str(e)), 上层据此 warning。"""
def _raising(name, ds):
raise RuntimeError("registry corrupted")
monkeypatch.setattr("app.data_providers.custom.provider_has_dataset", _raising)
def test_resolve_minute_provider_unloaded_during_resolution_returns_err(monkeypatch):
"""数据源重载期间源消失时,不把它误判为未配置 minute 数据集。"""
monkeypatch.setattr("app.data_providers.custom.get_provider", lambda name: (_ for _ in ()).throw(ValueError("Custom data source not found or invalid: mock_src")))
monkeypatch.setattr("app.data_providers.custom.provider_has_dataset", lambda name, ds: False)
monkeypatch.setattr("app.data_providers.custom.names", lambda: set())
provider, fallback, err = kline_sync._resolve_minute_provider("mock_src")
assert provider is None
assert fallback is False
assert "not found or invalid" in err
def test_resolve_minute_provider_config_exception_returns_err(monkeypatch):
"""自定义源配置无法读取时记录解析错误。"""
class _BrokenProvider:
@property
def config(self):
raise RuntimeError("registry corrupted")
monkeypatch.setattr("app.data_providers.custom.get_provider", lambda name: _BrokenProvider())
provider, fallback, err = kline_sync._resolve_minute_provider("mock_src")
assert provider is None
assert fallback is True
assert fallback is False
assert err is not None
assert "registry corrupted" in err
def test_resolve_minute_provider_get_provider_exception_returns_err(monkeypatch):
"""观察项加固: provider_has_dataset 返回 True 但 get_provider 抛 → (None, True, str(e))。"""
monkeypatch.setattr(
"app.data_providers.custom.provider_has_dataset",
lambda name, ds: True,
)
"""provider_has_dataset 返回 True 但 get_provider 抛错时记录解析错误。"""
def _raising_get(name):
raise ValueError("not found")
monkeypatch.setattr("app.data_providers.custom.get_provider", _raising_get)
provider, fallback, err = kline_sync._resolve_minute_provider("mock_src")
assert provider is None
assert fallback is True
assert fallback is False
assert err is not None
assert "not found" in err
def test_resolve_minute_provider_success_returns_provider(monkeypatch):
"""观察项加固: 正常路径 → (provider, False, None)。"""
mock_provider = object() # 任意 truthy 对象即可
monkeypatch.setattr(
"app.data_providers.custom.provider_has_dataset",
lambda name, ds: True,
)
mock_provider = MagicMock()
mock_provider.config.datasets = {"minute": {}}
monkeypatch.setattr(
"app.data_providers.custom.get_provider",
lambda name: mock_provider,
)
monkeypatch.setattr(
"app.data_providers.custom.provider_has_dataset",
lambda name, ds: True,
)
provider, fallback, err = kline_sync._resolve_minute_provider("mock_src")
assert provider is mock_provider
assert fallback is False
@@ -817,8 +875,8 @@ def test_minute_allowed_resolver_exception_returns_false(monkeypatch):
assert kline_api._minute_allowed(CapabilitySet()) is False
def test_intraday_monitor_support_resolver_exception_falls_back(monkeypatch):
"""监控入口解析自定义源失败后继续按 TickFlow 能力判断。"""
def test_intraday_monitor_support_resolver_exception_fails_closed(monkeypatch):
"""所选自定义源无法解析时,分时监控不误报为 TickFlow 可用。"""
from app.tickflow.capabilities import Cap, CapabilitySet
monkeypatch.setattr(
@@ -836,8 +894,27 @@ def test_intraday_monitor_support_resolver_exception_falls_back(monkeypatch):
support = kline_sync.intraday_monitor_support(capset)
assert support["available"] is True
assert support["source"] == "minute_batch"
assert support["available"] is False
assert support["source"] is None
def test_intraday_monitor_custom_failure_does_not_request_tickflow(monkeypatch):
"""分时监控批量取数遇到自定义源故障时,不尝试请求 TickFlow。"""
from app.tickflow.capabilities import Cap, CapabilityLimits, CapabilitySet
provider = MagicMock()
provider.get_minute.side_effect = httpx.TimeoutException("timeout")
_setup_custom_provider(monkeypatch, provider, has_dataset=True)
get_client_spy = MagicMock(name="get_client_spy")
monkeypatch.setattr(kline_sync, "get_client", get_client_spy)
capset = CapabilitySet({Cap.KLINE_MINUTE_BATCH: CapabilityLimits()})
result = kline_sync.fetch_intraday_monitor_batch(
["600519.SH"], capset, now=datetime(2026, 1, 15, 10, 0),
)
assert result.is_empty()
get_client_spy.assert_not_called()
# ---------- 测试 20: sync_minute_single 拒绝指数 symbol (防污染 kline_minute) ----------
@@ -147,10 +147,13 @@ def test_guard_string_datetimes_classified_after_parse():
assert out["datetime"].to_list() == [datetime(2026, 1, 15, 9, 30)]
# ---------- 路由级: 自定义源契约违规 → 回退 TickFlow ----------
# ---------- 路由级: 自定义源契约违规 → fail-closed ----------
def _setup_custom_provider(monkeypatch, provider: object) -> None:
monkeypatch.setattr(kline_sync.preferences, "get_minute_data_provider", lambda: "mock_src")
provider.config = MagicMock(datasets={"minute": {}})
monkeypatch.setattr("app.data_providers.custom.names", lambda: {"mock_src"})
monkeypatch.setattr("app.data_providers.custom.get_provider", lambda name: provider)
monkeypatch.setattr("app.data_providers.custom.provider_has_dataset", lambda name, ds: True)
monkeypatch.setattr("app.data_providers.custom.get_provider", lambda name: provider)
@@ -170,8 +173,8 @@ def test_custom_provider_utc_frame_selfhealed(monkeypatch):
assert df["datetime"].to_list() == [datetime(2026, 1, 15, 9, 30), datetime(2026, 1, 15, 13, 0)]
def test_custom_provider_garbage_datetime_falls_back(monkeypatch):
"""插件返回无法识别口径 → fail-closed 回退 TickFlow。"""
def test_custom_provider_garbage_datetime_fails_closed(monkeypatch):
"""插件返回无法识别口径 → fail-closed,不请求其他数据源。"""
mock_provider = MagicMock()
mock_provider.get_minute = MagicMock(return_value=_minute_frame(
[datetime(2026, 1, 15, 21, 30)]))
@@ -181,5 +184,5 @@ def test_custom_provider_garbage_datetime_falls_back(monkeypatch):
["600519.SH"], datetime(2026, 1, 15, 9, 25), datetime(2026, 1, 15, 15, 5),
asset_type="stock",
)
assert fallback is True
assert df is None
assert fallback is False
assert df is not None and df.is_empty()