From 8cedb67c367fafbebfe39db93ad95e7ec48550c2 Mon Sep 17 00:00:00 2001 From: shy3130 Date: Mon, 26 Oct 2026 13:39:39 +0800 Subject: [PATCH] =?UTF-8?q?feat(strategy):=20filter=5Fhistory=20=E5=88=97?= =?UTF-8?q?=E4=BE=9D=E8=B5=96=20AST=20=E6=8E=A8=E5=AF=BC=20=E2=80=94=20?= =?UTF-8?q?=E6=B6=88=E7=81=AD=E5=9B=9E=E6=B5=8B=E5=85=A8=E9=87=8F=E5=85=9C?= =?UTF-8?q?=E5=BA=95=20(P1a)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - feature_deps: 追踪 pl.col/cols 常量引用推导输入列; 动态列名/pl.all/cs.* 等不可证明形态一律返回 None, 保持全量兜底 (fail-closed, 宁慢不错列) - engine: REQUIRED_FEATURES 缺失且存在动态过滤时加载期自动推导 - bench_feature_resolution: legacy(空依赖) vs 推导 的解析宽度对比基准 - 实测: 看板策略等 4 个无声明策略 37→3~5 指标列 (宽度比 7~12x); short_term_top10 动态列名循环正确保持兜底 Signed-off-by: shy3130 --- backend/app/strategy/engine.py | 25 +++- backend/app/strategy/feature_deps.py | 100 ++++++++++++++ backend/scripts/bench_feature_resolution.py | 120 ++++++++++++++++ backend/tests/test_feature_deps_inference.py | 137 +++++++++++++++++++ 4 files changed, 380 insertions(+), 2 deletions(-) create mode 100644 backend/app/strategy/feature_deps.py create mode 100644 backend/scripts/bench_feature_resolution.py create mode 100644 backend/tests/test_feature_deps_inference.py diff --git a/backend/app/strategy/engine.py b/backend/app/strategy/engine.py index 1e8d9bba..21e88e8f 100644 --- a/backend/app/strategy/engine.py +++ b/backend/app/strategy/engine.py @@ -233,6 +233,28 @@ class _RealtimeMatrixEntry: buffer: Any +def _resolve_required_features(meta: dict, mod, filter_history_fn) -> frozenset[str]: + """显式声明 (META/module REQUIRED_FEATURES) 优先; 缺失且存在动态 + filter_history 时, AST 推导列依赖兜底 — 推导不可证明则保持为空, + 回测层继续走全量指标计算 (fail-closed)。""" + declared = frozenset(meta.get("required_features", []) or []) | frozenset( + getattr(mod, "REQUIRED_FEATURES", []) or [] + ) + if declared or filter_history_fn is None: + return declared + from app.strategy.feature_deps import infer_filter_history_features + + inferred = infer_filter_history_features(filter_history_fn) + if inferred is None: + return declared + logger.info( + "strategy %s: filter_history AST 推导出 %d 个依赖列, 免除回测全量兜底", + meta.get("id", ""), + len(inferred), + ) + return inferred + + class StrategyEngine: """策略引擎 — 策略加载 + 执行 + 评分""" @@ -582,8 +604,7 @@ class StrategyEngine: max_hold_days=getattr(mod, "MAX_HOLD_DAYS", None), filter_fn=filter_fn, filter_history_fn=filter_history_fn, - required_features=frozenset(meta.get("required_features", []) or []) - | frozenset(getattr(mod, "REQUIRED_FEATURES", []) or []), + required_features=_resolve_required_features(meta, mod, filter_history_fn), lookback_days=int(getattr(mod, "LOOKBACK_DAYS", meta.get("lookback_days", 1)) or 1), source=source, file_path=path, diff --git a/backend/app/strategy/feature_deps.py b/backend/app/strategy/feature_deps.py new file mode 100644 index 00000000..2a99cf68 --- /dev/null +++ b/backend/app/strategy/feature_deps.py @@ -0,0 +1,100 @@ +"""filter_history 列依赖的 AST 推导 — REQUIRED_FEATURES 缺失时的自动补全。 + +背景: 声明了 filter_history (动态 Python 过滤) 但没有声明 REQUIRED_FEATURES 的 +策略, 回测无法静态推断需要哪些列, 只能退回"计算全部指标"(backtest/strategy.py +的 full_fallback, 结果正确但白白多算)。本模块对函数体做 AST 静态追踪, 收集 +列引用常量, 让绝大多数此类策略拿到精确依赖, 消灭全量兜底。 + +fail-closed 契约: 任何无法证明的形态 — 动态列名、pl.all()/polars.selectors、 +完全收集不到列引用 — 一律返回 None, 调用方保持原有全量兜底。宁可慢, +绝不漏列 (漏列会静默算错, 是金融数据的 P0 事故形态)。 +""" +from __future__ import annotations + +import ast +import inspect +import logging +import textwrap + +logger = logging.getLogger(__name__) + +# 列引用形态: <...>.col("x") / 裸 col("x") / pl.cols("a", "b")。 +_COL_FORMS = ("pl.col", "col") +_COLS_FORMS = ("pl.cols", "cols") +# 列名动态函数 (polars 表达式里按正则/前后缀选列): 出现即不可推导。 +_DYNAMIC_NAME_FUNCS = frozenset({"contains", "starts_with", "ends_with", "matches"}) + + +def _dotted(func: ast.expr) -> str | None: + """AST 调用目标 → 点链名 ("pl.col" / "col" / "cs.numeric" / None)。""" + if isinstance(func, ast.Name): + return func.id + if isinstance(func, ast.Attribute): + base = _dotted(func.value) + return f"{base}.{func.attr}" if base else func.attr + return None + + +def _is_dynamic_name_expr(expr: ast.expr) -> bool: + """col(...) 参数是否为动态列名 (非常量字符串)。""" + return not (isinstance(expr, ast.Constant) and isinstance(expr.value, str)) + + +def _collect_from_source(source: str) -> tuple[set[str], bool]: + """返回 (收集到的列名, 是否存在不可证明形态)。""" + tree = ast.parse(source) + names: set[str] = set() + uninferable = False + for node in ast.walk(tree): + if not isinstance(node, ast.Call): + continue + dotted = _dotted(node.func) + if dotted is None: + continue + # pl.all() 是列选择器 (裸 all(...) 是 python 内建, 与列无关, 忽略); + # polars.selectors (import polars.selectors as cs; cs.numeric()) 全族同样不可推导。 + if dotted == "pl.all" or dotted.startswith("cs."): + uninferable = True + elif dotted in _COL_FORMS: + if not node.args: + continue + if _is_dynamic_name_expr(node.args[0]): + # 动态列名: col(f"{p}_col") / col(name_var) — 漏掉就是错列。 + uninferable = True + else: + assert isinstance(node.args[0], ast.Constant) + names.add(node.args[0].value) + elif dotted in _COLS_FORMS: + for arg in node.args: + if _is_dynamic_name_expr(arg): + uninferable = True + else: + assert isinstance(arg, ast.Constant) + names.add(arg.value) + return names, uninferable + + +def infer_filter_history_features(fn) -> frozenset[str] | None: + """推导 filter_history 引用的输入列; None = 不可推导 (调用方保持全量兜底)。 + + 只保留已知输入列 (enriched 存储/指标/证券维表), `_` 前缀的中间列 + (函数内 with_columns 先建后用的临时列) 不是输入依赖, 剔除。 + """ + from app.indicators.pipeline import ENRICHED_STORAGE_COLS, INDICATOR_COLUMNS + + try: + # 嵌套定义的函数 getsource 带原始缩进, 直接 parse 会 IndentationError。 + source = textwrap.dedent(inspect.getsource(fn)) + except (OSError, TypeError) as e: + logger.debug("feature inference: no source for %r: %s", getattr(fn, "__name__", fn), e) + return None + try: + names, uninferable = _collect_from_source(source) + except SyntaxError: + return None # 已加载模块不应有语法错误; 防御性 fail-closed + if uninferable or not names: + # 收集不到列引用 (如 df["x"] 下标风格) 与存在动态形态同样不可证明。 + return None + known = set(ENRICHED_STORAGE_COLS) | set(INDICATOR_COLUMNS) | {"name", "total_shares", "float_shares"} + features = frozenset(n for n in names if n in known and not n.startswith("_")) + return features or None diff --git a/backend/scripts/bench_feature_resolution.py b/backend/scripts/bench_feature_resolution.py new file mode 100644 index 00000000..552f96e9 --- /dev/null +++ b/backend/scripts/bench_feature_resolution.py @@ -0,0 +1,120 @@ +"""P0/P1a 基准: 动态 filter_history 策略的依赖解析宽度对比。 + +对同一策略比较两种依赖声明下的 ResolvedFeaturePlan: + legacy — required_features 为空 (P1a 之前的行为, 触发全量指标兜底) + inferred — 引擎 AST 推导结果 (P1a 之后, 不可推导的策略无此行) + +记录解析耗时与 indicator/base 列宽。数据加载 IO 大致随列宽线性增长, +宽度比即回测数据准备阶段的近似提速上界 (非端到端回测耗时)。 + +用法: + cd backend && uv run python scripts/bench_feature_resolution.py [strategy_id ...] + 不传 id 时扫描全部 filter_history 策略。--json PATH 另存结果。 +""" +from __future__ import annotations + +import argparse +import dataclasses +import json +import logging +import sys +import time +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +# legacy 臂会故意触发全量兜底 WARNING, 基准输出只保留表格。 +logging.getLogger("app.backtest.strategy").setLevel(logging.ERROR) + +from app.backtest.strategy import StrategyDependencyResolver # noqa: E402 +from app.config import settings # noqa: E402 +from app.strategy.engine import StrategyEngine # noqa: E402 + + +def _build_engine() -> StrategyEngine: + dirs = [ + Path(__file__).resolve().parents[1] / "app" / "strategy" / "research", + settings.data_dir / "strategies" / "custom", + settings.data_dir / "strategies" / "ai", + settings.data_dir / "strategies" / "composite", + ] + return StrategyEngine(strategy_dirs=[d for d in dirs if d.exists()]) + + +def _resolve_quiet(resolver: StrategyDependencyResolver, strategy, required: frozenset): + """给定 required_features 解析 FeaturePlan, 返回 (耗时ms, plan)。""" + patched = dataclasses.replace(strategy, required_features=required) + t0 = time.perf_counter() + plan = resolver.resolve( + patched, + params={p.get("id"): p.get("default") for p in strategy.meta.get("params", [])}, + basic_filter=strategy.basic_filter or {"enabled": False}, + entry_signals=strategy.entry_signals, + exit_signals=strategy.exit_signals, + ) + elapsed_ms = (time.perf_counter() - t0) * 1000 + return elapsed_ms, plan + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("ids", nargs="*", help="策略 id; 缺省扫描全部动态策略") + parser.add_argument("--json", dest="json_path", default=None) + args = parser.parse_args() + + engine = _build_engine() + resolver = StrategyDependencyResolver() + strategies = [ + s for s in engine.list_strategies_full() if s.filter_history_fn is not None + ] if hasattr(engine, "list_strategies_full") else [] + if not strategies: + # list 接口名差异兜底: 直接遍历注册表 + strategies = [ + s for s in engine._strategies.values() if s.filter_history_fn is not None + ] + if args.ids: + strategies = [s for s in strategies if s.meta.get("id") in set(args.ids)] + + rows: list[dict] = [] + for s in strategies: + sid = s.meta.get("id", "") + declared = bool(s.required_features) + legacy_ms, legacy_plan = _resolve_quiet(resolver, s, frozenset()) + inferred_ms, inferred_plan = _resolve_quiet(resolver, s, s.required_features) + legacy_w = len(legacy_plan.indicator_columns) + inferred_w = len(inferred_plan.indicator_columns) + rows.append({ + "strategy": sid, + "declared_required_features": declared, + "legacy": { + "resolve_ms": round(legacy_ms, 2), + "indicator_cols": legacy_w, + "base_cols": len(legacy_plan.base_columns), + }, + "inferred": { + "resolve_ms": round(inferred_ms, 2), + "indicator_cols": inferred_w, + "base_cols": len(inferred_plan.base_columns), + }, + "indicator_width_ratio": round(legacy_w / inferred_w, 2) if inferred_w else None, + }) + + print(f"{'strategy':36} {'依赖':4} {'legacy列宽':>9} {'当前列宽':>8} {'宽度比':>6}") + for r in rows: + # 依赖列非空 = 显式声明或 P1a 推导; 空的唯一情形是不可证明 (保持全量兜底)。 + has_deps = "有" if r["declared_required_features"] else "无" + ratio = r["indicator_width_ratio"] + print( + f"{r['strategy']:36} {has_deps:4} " + f"{r['legacy']['indicator_cols']:>9} {r['inferred']['indicator_cols']:>8} " + f"{(f'{ratio:.1f}x') if ratio else '—':>6}" + ) + if args.json_path: + Path(args.json_path).write_text( + json.dumps({"rows": rows}, ensure_ascii=False, indent=2), encoding="utf-8" + ) + print(f"\n已写入 {args.json_path}") + + +if __name__ == "__main__": + main() diff --git a/backend/tests/test_feature_deps_inference.py b/backend/tests/test_feature_deps_inference.py new file mode 100644 index 00000000..27d684fa --- /dev/null +++ b/backend/tests/test_feature_deps_inference.py @@ -0,0 +1,137 @@ +"""filter_history 列依赖 AST 推导 — 消灭回测全量兜底的 P1 契约。 + +回归对象: REQUIRED_FEATURES 缺失 + 存在动态 filter_history 的策略。 +- 可证明形态 (col/cols 常量引用) → 引擎加载时自动推导精确依赖; +- 不可证明形态 (动态列名/pl.all/cs.*) → 保持空依赖, 回测继续全量兜底 (fail-closed)。 +""" +from __future__ import annotations + +from pathlib import Path + +import polars as pl +import polars.selectors as cs + +from app.strategy.engine import StrategyEngine +from app.strategy.feature_deps import infer_filter_history_features + +# ── 单元: 推导函数 ─────────────────────────────────────── + +def test_infer_basic_column_refs(): + def filter_history(df: pl.DataFrame, params: dict) -> pl.DataFrame: + return df.filter( + (pl.col("close") > pl.col("ma20")) + & (pl.col("kdj_k") < 25) + ) + + assert infer_filter_history_features(filter_history) == frozenset({"close", "ma20", "kdj_k"}) + + +def test_infer_cols_and_drop_intermediates(): + def filter_history(df: pl.DataFrame, params: dict) -> pl.DataFrame: + d = df.with_columns(pl.col("_rsv")) # 中间列, 不是输入依赖 + return d.select(pl.cols("ma5", "ma10")) + + assert infer_filter_history_features(filter_history) == frozenset({"ma5", "ma10"}) + + +def test_infer_dynamic_column_name_is_uninferable(): + def filter_history(df: pl.DataFrame, params: dict) -> pl.DataFrame: + col_name = str(params.get("col", "close")) + return df.filter(pl.col(col_name) > 1) + + assert infer_filter_history_features(filter_history) is None + + +def test_infer_no_column_refs_is_uninferable(): + """df["x"] 下标风格收集不到 → 不可证明 → None (绝不声称零依赖)。""" + + def filter_history(df: pl.DataFrame, params: dict) -> pl.DataFrame: + if df.is_empty(): + return df + return df.filter(df["close"] > 1) + + assert infer_filter_history_features(filter_history) is None + + +def test_infer_pl_all_selector_is_uninferable(): + def filter_history(df: pl.DataFrame, params: dict) -> pl.DataFrame: + return df.select(pl.all()) + + assert infer_filter_history_features(filter_history) is None + + +def test_infer_polars_selectors_is_uninferable(): + def filter_history(df: pl.DataFrame, params: dict) -> pl.DataFrame: + return df.with_columns((cs.numeric() * 2).name.suffix("_x")) + + assert infer_filter_history_features(filter_history) is None + + +def test_infer_unknown_names_filtered_out(): + def filter_history(df: pl.DataFrame, params: dict) -> pl.DataFrame: + return df.filter(pl.col("not_a_real_column") > pl.col("close")) + + assert infer_filter_history_features(filter_history) == frozenset({"close"}) + + +# ── 集成: 引擎加载路径 ────────────────────────────────── + +_INFERABLE = '''\ +META = {"id": "custom_inf_ok", "name": "ok", "description": "", "tags": []} + +import polars as pl + + +def filter_history(df: pl.DataFrame, params: dict) -> pl.DataFrame: + return df.filter((pl.col("close") > pl.col("ma20")) & (pl.col("kdj_k") < 25)) +''' + +_DYNAMIC = '''\ +META = {"id": "custom_inf_dyn", "name": "dyn", "description": "", "tags": []} + +import polars as pl + + +def filter_history(df: pl.DataFrame, params: dict) -> pl.DataFrame: + col_name = str(params.get("col", "close")) + return df.filter(pl.col(col_name) > 1) +''' + + +def _engine_with(tmp_path: Path, source: str): + # 引擎只 glob 各 strategy_dir 顶层的 *.py, 必须传叶子目录。 + d = tmp_path / "custom" + d.mkdir(parents=True) + (d / f"{_meta_id(source)}.py").write_text(source, encoding="utf-8") + return StrategyEngine(strategy_dirs=[d]) + + +def _meta_id(source: str) -> str: + return source.split('"id": "')[1].split('"')[0] + + +def test_engine_infers_features_when_undeclared(tmp_path): + engine = _engine_with(tmp_path, _INFERABLE) + assert not engine.load_errors() + s = engine.get("custom_inf_ok") + assert s is not None + assert s.required_features == frozenset({"close", "ma20", "kdj_k"}) + + +def test_engine_keeps_fallback_for_dynamic(tmp_path): + engine = _engine_with(tmp_path, _DYNAMIC) + assert not engine.load_errors() + s = engine.get("custom_inf_dyn") + assert s is not None + # 不可证明 → 不推导, required_features 保持空 (回测层继续全量兜底) + assert s.required_features == frozenset() + + +def test_engine_explicit_declaration_wins(tmp_path): + source = _INFERABLE.replace( + 'META = {"id": "custom_inf_ok"', + 'REQUIRED_FEATURES = ["close"]\nMETA = {"id": "custom_inf_ok"', + ) + engine = _engine_with(tmp_path, source) + assert not engine.load_errors() + assert engine.get("custom_inf_ok").required_features == frozenset({"close"})