perf: 收窄缓存失效并优化异步任务与首屏加载

按 enriched 写入最早影响日期维护有界区间版本，保留全局发布一致性校验，使历史读取与回测缓存复用不受较晚行情更新影响；历史修订和未知范围变更仍失效。将扩展数据解析、落盘与备份/策略导入移出事件循环；串行限制扩展数据写入，并在请求取消后保持执行中的写入限额及临时文件清理。拆分 11 个扩展页面为轻量注册清单和按需页面，助手抽屉首次使用时加载；并发获取扩展后按固定顺序注册，为运行时脚本增加超时隔离。保持金融计算、业务数据格式与扩展 v1 契约不变。验证：隔离全量后端 3923 项通过、7 项跳过、1 项预期失败；前端 447 项通过，TypeScript 与生产构建通过。Ruff 无新增告警，git diff --check 通过。浏览器验证 11 个扩展路由与脚本超时隔离。性能：1000 标的、120 天、12 万行、30 次更新，历史面板读取中位数 7.19ms -> 0.14ms，计算 31 -> 1 次，结果一致。注入 250ms 解析耗时的心跳间隔中位最大值 269ms -> 11.3ms。启动必需 JS 2.33MB -> 0.65MB；10Mbps/40ms 网络与固定 API 响应下，登录页 3 次冷启动中位数 3.15s -> 1.12s。 Signed-off-by: shy3130 <shy3130@qq.com>
2026-11-02 10:26:13 +00:00 · 2026-11-01 22:32:34 +08:00
parent f443627377
commit ab2b82da37
43 changed files with 12374 additions and 11475 deletions
@@ -0,0 +1,145 @@
+"""Isolated cache and event-loop benchmark; no network or user data.
+
+Run from backend: uv run --frozen python -m scripts.benchmark_cache_async
+To compare revisions, run this same file with PYTHONPATH pointing at each backend.
+The 250 ms parser delay is deliberately synthetic: it measures event-loop
+responsiveness, not CSV parser throughput. All data is generated in a tempdir.
+"""
+from __future__ import annotations
+
+import asyncio
+import io
+import json
+import logging
+import os
+import statistics
+import tempfile
+import time
+from datetime import date, timedelta
+from pathlib import Path
+from types import SimpleNamespace
+from unittest.mock import patch
+
+
+def cache_benchmark(root: Path) -> dict:
+    import polars as pl
+
+    from app.backtest.engine import BacktestEngine
+    from app.tickflow.repository import DataStore, KlineRepository
+
+    days = [date(2026, 1, 1) + timedelta(days=i) for i in range(168)]
+    days = [d for d in days if d.weekday() < 5]
+    symbols = [f"{600000 + i}.SH" for i in range(1000)]
+    frames = [pl.DataFrame({
+        "symbol": symbols, "date": [day] * len(symbols),
+        "open": [10.0] * len(symbols), "high": [11.0] * len(symbols),
+        "low": [9.0] * len(symbols), "close": [10.0 + i / 100] * len(symbols),
+        "volume": [1000.0] * len(symbols),
+    }) for i, day in enumerate(days)]
+    history = pl.concat(frames)
+    repo = KlineRepository(DataStore(root))
+    try:
+        repo.append_enriched(history)
+        repo._enriched_history_cache = history
+        repo._enriched_history_start = days[0]
+        repo._enriched_history_generation = repo.get_matrix_data_generation()
+        if hasattr(repo, "get_enriched_generation_snapshot"):
+            repo._enriched_history_snapshot = (history, repo.get_enriched_generation_snapshot())
+        engine = BacktestEngine(repo)
+        columns = ["symbol", "date", "close"]
+        args = (None, days[0], days[-2], columns)
+        expected = history.filter(pl.col("date") <= days[-2]).select(columns).sort(["symbol", "date"])
+        assert engine.load_panel(*args).equals(expected)
+        reads, writes, hits = [], [], 0
+        for i in range(30):
+            tick = frames[-1].with_columns(pl.lit(12.0 + i / 100).alias("close"))
+            started = time.perf_counter()
+            repo.flush_live_enriched(tick)
+            writes.append((time.perf_counter() - started) * 1000)
+            started = time.perf_counter()
+            actual = engine.load_panel(*args)
+            reads.append((time.perf_counter() - started) * 1000)
+            assert actual.equals(expected)
+            cached = repo.get_enriched_range(days[0], days[-2], columns=columns)
+            if cached is not None:
+                assert cached.equals(expected)
+                hits += 1
+        return {
+            "symbols": len(symbols), "partitions": len(days), "rows": history.height,
+            "ticks": 30, "historical_cache_hits": hits,
+            "panel_computations": engine._cache.stats()["compute_count"],
+            "panel_read_median_ms": round(statistics.median(reads), 3),
+            "tick_write_median_ms": round(statistics.median(writes), 3),
+            "outputs_equal": True,
+        }
+    finally:
+        repo.db.close()
+
+
+async def async_benchmark(root: Path) -> dict:
+    import polars as pl
+    from fastapi import UploadFile
+
+    from app.api import ext_data
+    from app.services.ext_data import ExtConfig, ExtField
+
+    config = ExtConfig(id="bench", label="bench", mode="snapshot", fields=[
+        ExtField("symbol"), ExtField("value", dtype="float"),
+    ])
+    request = SimpleNamespace(app=SimpleNamespace(state=SimpleNamespace(
+        repo=SimpleNamespace(store=SimpleNamespace(data_dir=root)),
+    )))
+
+    def parse(*args, **kwargs):
+        time.sleep(0.25)
+        return pl.DataFrame({"symbol": ["600000.SH"], "value": [1.5]})
+
+    maxima = []
+    for _ in range(5):
+        intervals = []
+        stop = asyncio.Event()
+
+        async def heartbeat(stop=stop, intervals=intervals):
+            while not stop.is_set():
+                started = time.perf_counter()
+                await asyncio.sleep(0.01)
+                intervals.append((time.perf_counter() - started) * 1000)
+
+        task = asyncio.create_task(heartbeat())
+        await asyncio.sleep(0.03)
+        try:
+            with (
+                patch.object(ext_data, "_store", lambda _: SimpleNamespace(get=lambda _: config)),
+                patch.object(ext_data.pl, "read_csv", parse),
+                patch.object(ext_data, "_refresh_views", lambda _: None),
+            ):
+                file = UploadFile(io.BytesIO(b"symbol,value\n600000.SH,1.5\n"), filename="bench.csv")
+                result = await ext_data.upload_data(request, "bench", file, "2026-10-29")
+                assert result == {"status": "ok", "rows": 1, "date": "2026-10-29"}
+                await file.close()
+            await asyncio.sleep(0.03)
+        finally:
+            stop.set()
+            await task
+        maxima.append(max(intervals))
+    return {
+        "synthetic_parser_ms": 250, "heartbeat_period_ms": 10, "runs": 5,
+        "median_max_heartbeat_ms": round(statistics.median(maxima), 3),
+        "max_heartbeat_runs_ms": [round(value, 3) for value in maxima],
+    }
+
+
+def main():
+    logging.basicConfig(level=logging.ERROR)
+    with tempfile.TemporaryDirectory(prefix="tsp-hotpaths-") as temporary:
+        root = Path(temporary)
+        os.environ["DATA_DIR"] = str(root / "settings")
+        os.environ["TICKFLOW_ENV_FILE"] = str(root / "absent.env")
+        print(json.dumps({
+            "cache": cache_benchmark(root / "cache"),
+            "async": asyncio.run(async_benchmark(root / "upload")),
+        }, indent=2))
+
+
+if __name__ == "__main__":
+    main()