From 2de7275b8f656088200d1428f7d9697b04e0bb90 Mon Sep 17 00:00:00 2001 From: TPTBusiness Date: Mon, 27 Apr 2026 16:07:06 +0200 Subject: [PATCH] fix(auto-fixer): replace zero \$volume with price-range proxy for FX data EUR/USD synthetic data has \$volume=0 for all rows, causing any VWAP or volume-weighted factor to produce all-NaN output. Insert a guard after pd.read_hdf() that replaces zero volume with (\$high - \$low) range proxy so volume-dependent factors produce meaningful signals. Co-Authored-By: Claude Sonnet 4.6 --- .../coder/factor_coder/auto_fixer.py | 65 ++++++++++++++++--- test/qlib/test_auto_fixer.py | 34 ++++++++++ 2 files changed, 90 insertions(+), 9 deletions(-) diff --git a/rdagent/components/coder/factor_coder/auto_fixer.py b/rdagent/components/coder/factor_coder/auto_fixer.py index b9a277d5..42e45996 100644 --- a/rdagent/components/coder/factor_coder/auto_fixer.py +++ b/rdagent/components/coder/factor_coder/auto_fixer.py @@ -58,15 +58,16 @@ class FactorAutoFixer: fix_methods = [ self._fix_instrument_column_access, # First: fix df['instrument'] on MultiIndex self._fix_instrument_loc_multiindex, # Second: fix df.loc[instrument_var] on MultiIndex - self._fix_reset_index_groupby, # Third: fix groupby(level=N) after reset_index() - self._fix_groupby_mixed_levels, # Fourth: fix groupby(level=[int, str]) - self._fix_groupby_column_on_multiindex, # Fifth: fix groupby(['instrument','date']) on MultiIndex - self._fix_chained_groupby, # Sixth: fix groupby(level=N).groupby('date') chain - self._fix_rolling_ddof, # Seventh: remove unsupported ddof kwarg - self._fix_groupby_apply_to_transform, # Eighth: fix groupby patterns - self._fix_inf_nan_handling, # Ninth: add inf/nan handling - self._fix_data_range_processing, # Tenth: ensure full data range - self._fix_multiindex_groupby, # Eleventh: ensure groupby on MultiIndex + self._fix_zero_volume_proxy, # Third: replace zero $volume with range proxy + self._fix_reset_index_groupby, # Fourth: fix groupby(level=N) after reset_index() + self._fix_groupby_mixed_levels, # Fifth: fix groupby(level=[int, str]) + self._fix_groupby_column_on_multiindex, # Sixth: fix groupby(['instrument','date']) on MultiIndex + self._fix_chained_groupby, # Seventh: fix groupby(level=N).groupby('date') chain + self._fix_rolling_ddof, # Eighth: remove unsupported ddof kwarg + self._fix_groupby_apply_to_transform, # Ninth: fix groupby patterns + self._fix_inf_nan_handling, # Tenth: add inf/nan handling + self._fix_data_range_processing, # Eleventh: ensure full data range + self._fix_multiindex_groupby, # Twelfth: ensure groupby on MultiIndex ] for fix_method in fix_methods: @@ -164,6 +165,52 @@ class FactorAutoFixer: return fixed_code + def _fix_zero_volume_proxy(self, code: str) -> str: + """ + Fix: $volume is always 0 in our EUR/USD dataset (FX has no real volume). + Any factor using $volume (VWAP, volume-weighted returns, etc.) produces + all-NaN output because 0*price=0 and sum(0)/sum(0)=NaN. + + Insert a guard right after pd.read_hdf() that replaces zero volume with + the intraday price-range proxy ($high - $low) so volume-weighted factors + produce meaningful signals. + """ + if "'$volume'" not in code and '"$volume"' not in code: + return code + + # Already patched + if "volume proxy" in code: + return code + + lines = code.splitlines() + insert_after = -1 + df_var = "df" + indent = " " + + for i, line in enumerate(lines): + if "read_hdf(" in line: + m = re.match(r"(\s*)(\w+)\s*=\s*", line) + if m: + indent = m.group(1) + df_var = m.group(2) + else: + m2 = re.match(r"(\s*)", line) + indent = m2.group(1) if m2 else " " + insert_after = i + break + + if insert_after == -1: + return code + + proxy_lines = [ + f"{indent}# volume proxy: $volume is always 0 in FX data — use price-range as proxy", + f"{indent}if ({df_var}['$volume'] == 0).all():", + f"{indent} {df_var}['$volume'] = {df_var}['$high'] - {df_var}['$low']", + ] + lines = lines[: insert_after + 1] + proxy_lines + lines[insert_after + 1 :] + self.fixes_applied.append("volume_proxy: replaced zero $volume with ($high - $low)") + return "\n".join(lines) + def _fix_reset_index_groupby(self, code: str) -> str: """ Fix: groupby(level=N) on a variable created by .reset_index() fails because diff --git a/test/qlib/test_auto_fixer.py b/test/qlib/test_auto_fixer.py index 09698ef6..c3737cc2 100644 --- a/test/qlib/test_auto_fixer.py +++ b/test/qlib/test_auto_fixer.py @@ -205,6 +205,40 @@ class TestGroupbyApplyToTransform: assert ".transform(" in result +class TestZeroVolumeProxy: + def test_injects_proxy_when_volume_used(self, fixer): + code = ( + "def calc():\n" + " df = pd.read_hdf('data.h5', key='data')\n" + " df['pv'] = df['$close'] * df['$volume']\n" + " return df[['pv']]\n" + ) + result = fixer.fix(code) + assert "volume proxy" in result + assert "df['$volume'] = df['$high'] - df['$low']" in result + # Proxy must come right after read_hdf line + lines = result.splitlines() + hdf_idx = next(i for i, l in enumerate(lines) if "read_hdf" in l) + assert "volume proxy" in lines[hdf_idx + 1] + + def test_no_injection_when_volume_absent(self, fixer): + code = "df = pd.read_hdf('data.h5', key='data')\ndf['x'] = df['$close'].pct_change()\n" + result = fixer.fix(code) + assert "volume proxy" not in result + + def test_no_double_injection(self, fixer): + code = ( + "def calc():\n" + " df = pd.read_hdf('data.h5', key='data')\n" + " # volume proxy: $volume is always 0 in FX data — use price-range as proxy\n" + " if (df['$volume'] == 0).all():\n" + " df['$volume'] = df['$high'] - df['$low']\n" + " df['pv'] = df['$close'] * df['$volume']\n" + ) + result = fixer.fix(code) + assert result.count("volume proxy") == 1 + + class TestRollingDdof: def test_removes_ddof_from_rolling_args(self, fixer): result = fixer.fix("df.rolling(20, min_periods=1, ddof=1).std()")