diff --git a/backend/app/strategy/engine.py b/backend/app/strategy/engine.py index 21e88e8f..3f43bc10 100644 --- a/backend/app/strategy/engine.py +++ b/backend/app/strategy/engine.py @@ -233,22 +233,32 @@ class _RealtimeMatrixEntry: buffer: Any -def _resolve_required_features(meta: dict, mod, filter_history_fn) -> frozenset[str]: +def _resolve_required_features(meta: dict, mod, filter_history_fn, path: Path) -> frozenset[str]: """显式声明 (META/module REQUIRED_FEATURES) 优先; 缺失且存在动态 filter_history 时, AST 推导列依赖兜底 — 推导不可证明则保持为空, - 回测层继续走全量指标计算 (fail-closed)。""" + 回测层继续走全量指标计算 (fail-closed)。 + + 推导范围是整个策略文件 + 同目录 `_*.py` 依赖文件, 不只是 filter_history + 函数体: 列引用在模块级辅助函数或依赖文件里硬编码时, 只看函数体会漏列, + 回测帧缺列直接 ColumnNotFoundError。范围放宽只会多载列, 不会漏。""" declared = frozenset(meta.get("required_features", []) or []) | frozenset( getattr(mod, "REQUIRED_FEATURES", []) or [] ) if declared or filter_history_fn is None: return declared - from app.strategy.feature_deps import infer_filter_history_features + from app.strategy.feature_deps import infer_features_from_sources - inferred = infer_filter_history_features(filter_history_fn) + family = [path, *sorted(p for p in path.parent.glob("_*.py") if p != path)] + try: + sources = [p.read_text(encoding="utf-8") for p in family] + except OSError as e: + logger.debug("strategy %s: 依赖源码读取失败, 保持全量兜底: %s", meta.get("id", ""), e) + return declared + inferred = infer_features_from_sources(sources) if inferred is None: return declared logger.info( - "strategy %s: filter_history AST 推导出 %d 个依赖列, 免除回测全量兜底", + "strategy %s: 依赖源码 AST 推导出 %d 个依赖列, 免除回测全量兜底", meta.get("id", ""), len(inferred), ) @@ -604,7 +614,7 @@ class StrategyEngine: max_hold_days=getattr(mod, "MAX_HOLD_DAYS", None), filter_fn=filter_fn, filter_history_fn=filter_history_fn, - required_features=_resolve_required_features(meta, mod, filter_history_fn), + required_features=_resolve_required_features(meta, mod, filter_history_fn, path), lookback_days=int(getattr(mod, "LOOKBACK_DAYS", meta.get("lookback_days", 1)) or 1), source=source, file_path=path, diff --git a/backend/app/strategy/feature_deps.py b/backend/app/strategy/feature_deps.py index 2a99cf68..f215eea8 100644 --- a/backend/app/strategy/feature_deps.py +++ b/backend/app/strategy/feature_deps.py @@ -2,8 +2,9 @@ 背景: 声明了 filter_history (动态 Python 过滤) 但没有声明 REQUIRED_FEATURES 的 策略, 回测无法静态推断需要哪些列, 只能退回"计算全部指标"(backtest/strategy.py -的 full_fallback, 结果正确但白白多算)。本模块对函数体做 AST 静态追踪, 收集 -列引用常量, 让绝大多数此类策略拿到精确依赖, 消灭全量兜底。 +的 full_fallback, 结果正确但白白多算)。本模块对策略文件及其 `_*.py` 依赖文件 +做 AST 静态追踪, 收集列引用常量 (含模块级辅助函数), 让绝大多数此类策略拿到 +精确依赖, 消灭全量兜底。 fail-closed 契约: 任何无法证明的形态 — 动态列名、pl.all()/polars.selectors、 完全收集不到列引用 — 一律返回 None, 调用方保持原有全量兜底。宁可慢, @@ -75,25 +76,39 @@ def _collect_from_source(source: str) -> tuple[set[str], bool]: def infer_filter_history_features(fn) -> frozenset[str] | None: - """推导 filter_history 引用的输入列; None = 不可推导 (调用方保持全量兜底)。 + """推导单个 filter_history 函数引用的输入列; None = 不可推导。 - 只保留已知输入列 (enriched 存储/指标/证券维表), `_` 前缀的中间列 - (函数内 with_columns 先建后用的临时列) 不是输入依赖, 剔除。 + 只看函数自身源码 — 覆盖不了"同模块辅助函数/`_*.py` 依赖文件里硬编码列" + 的形态, 引擎路径请用 infer_features_from_sources (整家族文件一起推导)。 """ - from app.indicators.pipeline import ENRICHED_STORAGE_COLS, INDICATOR_COLUMNS - try: - # 嵌套定义的函数 getsource 带原始缩进, 直接 parse 会 IndentationError。 source = textwrap.dedent(inspect.getsource(fn)) except (OSError, TypeError) as e: logger.debug("feature inference: no source for %r: %s", getattr(fn, "__name__", fn), e) return None - try: - names, uninferable = _collect_from_source(source) - except SyntaxError: - return None # 已加载模块不应有语法错误; 防御性 fail-closed - if uninferable or not names: - # 收集不到列引用 (如 df["x"] 下标风格) 与存在动态形态同样不可证明。 + return infer_features_from_sources([source]) + + +def infer_features_from_sources(sources) -> frozenset[str] | None: + """对一组源码 (策略主文件 + `_*.py` 依赖) 整体推导输入列。 + + 家族内任一文件出现不可证明形态 (动态列名/pl.all/cs.*) 即整体返回 None — + 辅助函数漏列会让回测帧缺列, 宁可整体回退全量兜底。 + 其余 fail-closed 契约与单函数版一致。 + """ + from app.indicators.pipeline import ENRICHED_STORAGE_COLS, INDICATOR_COLUMNS + + names: set[str] = set() + for raw in sources: + try: + collected, uninferable = _collect_from_source(textwrap.dedent(raw)) + except SyntaxError: + return None # 家族文件解析失败按不可证明处理 + if uninferable: + return None + names |= collected + if not names: + # 收集不到列引用 (如 df["x"] 下标风格) 同样不可证明。 return None known = set(ENRICHED_STORAGE_COLS) | set(INDICATOR_COLUMNS) | {"name", "total_shares", "float_shares"} features = frozenset(n for n in names if n in known and not n.startswith("_")) diff --git a/backend/tests/test_feature_deps_inference.py b/backend/tests/test_feature_deps_inference.py index 27d684fa..39512f1a 100644 --- a/backend/tests/test_feature_deps_inference.py +++ b/backend/tests/test_feature_deps_inference.py @@ -135,3 +135,87 @@ def test_engine_explicit_declaration_wins(tmp_path): engine = _engine_with(tmp_path, source) assert not engine.load_errors() assert engine.get("custom_inf_ok").required_features == frozenset({"close"}) + + +# ── 家族推导: 模块级辅助函数 + `_*.py` 依赖文件 ──────────── +# 回归: 只解析 filter_history 函数体会漏掉这些位置硬编码的列 → 回测帧缺列 +# ColumnNotFoundError。范围放宽只会多载列, 不会漏。 + +_HELPER = '''\ +META = {"id": "custom_inf_helper", "name": "helper", "description": "", "tags": []} + +import polars as pl + + +def _trend_ok(df: pl.DataFrame) -> pl.Expr: + return pl.col("ma20") > pl.col("ma60") + + +def filter_history(df: pl.DataFrame, params: dict) -> pl.DataFrame: + return df.filter(_trend_ok(df) & (pl.col("close") > 1)) +''' + +_DEP_IMPORTER = '''\ +META = {"id": "custom_inf_dep", "name": "dep", "description": "", "tags": []} + +import polars as pl + +import _dep + + +def filter_history(df: pl.DataFrame, params: dict) -> pl.DataFrame: + return df.filter(_dep.momentum_ok("rsi_6") & (pl.col("close") > 1)) +''' + +_DEP_FILE = '''\ +import polars as pl + + +def momentum_ok(col_name: str) -> pl.Expr: + # 实现固定引用 rsi_6 (col_name 参数仅供调用方语义, 不参与列选择) + return pl.col("rsi_6") < 30 +''' + +_DEP_FILE_DYNAMIC = '''\ +import polars as pl + + +def momentum_ok(col_name: str) -> pl.Expr: + # 列名经参数传递 — 调用方可传任意列, AST 常量收集无法证明 + return pl.col(col_name) < 30 +''' + + +def test_engine_infers_module_level_helper_columns(tmp_path): + engine = _engine_with(tmp_path, _HELPER) + assert not engine.load_errors() + s = engine.get("custom_inf_helper") + assert s is not None + # ma20/ma60 在辅助函数里, 不在 filter_history 函数体内 — 必须仍被推导 + assert s.required_features == frozenset({"close", "ma20", "ma60"}) + + +def test_engine_infers_dependency_file_columns(tmp_path): + d = tmp_path / "custom" + d.mkdir(parents=True) + (d / "custom_inf_dep.py").write_text(_DEP_IMPORTER, encoding="utf-8") + (d / "_dep.py").write_text(_DEP_FILE, encoding="utf-8") + engine = StrategyEngine(strategy_dirs=[d]) + assert not engine.load_errors() + s = engine.get("custom_inf_dep") + assert s is not None + # rsi_6 只存在于 `_dep.py` — 家族推导必须覆盖依赖文件 + assert s.required_features == frozenset({"close", "rsi_6"}) + + +def test_engine_family_fail_closed_on_dynamic_dependency(tmp_path): + """依赖文件里出现动态列名 → 整个家族不可证明 → 保持全量兜底。""" + d = tmp_path / "custom" + d.mkdir(parents=True) + (d / "custom_inf_dep.py").write_text(_DEP_IMPORTER, encoding="utf-8") + (d / "_dep.py").write_text(_DEP_FILE_DYNAMIC, encoding="utf-8") + engine = StrategyEngine(strategy_dirs=[d]) + assert not engine.load_errors() + s = engine.get("custom_inf_dep") + assert s is not None + assert s.required_features == frozenset()