From f4deda99b54b11870a5b10d0ea855451cfd43513 Mon Sep 17 00:00:00 2001 From: TPTBusiness Date: Sun, 26 Apr 2026 11:50:52 +0200 Subject: [PATCH] fix(auto-fixer): preserve date dimension in groupby(['instrument','date']) fix MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The previous fixer converted groupby(['instrument','date']) → groupby(level=1), stripping the date level. This caused intraday calculations (VWAP, rolling-std, cumsum) to accumulate across trading days instead of resetting daily, producing all-NaN factor output — causing 100% failure rate on intraday factors. New behaviour: capture the DataFrame variable name and emit: var.groupby([var.index.get_level_values(1), var.index.get_level_values(0).normalize()]) which groups by (instrument, day) as originally intended. Adds test/qlib/test_auto_fixer.py covering all fixer cases. Co-Authored-By: Claude Sonnet 4.6 --- .../coder/factor_coder/auto_fixer.py | 56 +++++++++++---- test/qlib/test_auto_fixer.py | 68 +++++++++++++++++++ 2 files changed, 109 insertions(+), 15 deletions(-) create mode 100644 test/qlib/test_auto_fixer.py diff --git a/rdagent/components/coder/factor_coder/auto_fixer.py b/rdagent/components/coder/factor_coder/auto_fixer.py index 5acb8997..b384989b 100644 --- a/rdagent/components/coder/factor_coder/auto_fixer.py +++ b/rdagent/components/coder/factor_coder/auto_fixer.py @@ -129,25 +129,51 @@ class FactorAutoFixer: def _fix_groupby_column_on_multiindex(self, code: str) -> str: """ - Fix: groupby(['instrument', 'date']) on a MultiIndex DataFrame fails with - KeyError because 'instrument' and 'date' are index levels, not columns. + Fix: groupby(['instrument', 'date']) on a MultiIndex (datetime, instrument) + DataFrame fails with KeyError because those are index levels, not columns. - Replace with groupby(level=1) (instrument is level 1). - Also handle groupby(['date', 'instrument']) and single groupby('instrument'). + Correct replacement preserves BOTH dimensions so intraday calculations reset + per day: + var.groupby(['instrument', 'date']) + → var.groupby([var.index.get_level_values(1), var.index.get_level_values(0).normalize()]) + + Single-column groupby(['instrument']) is correctly replaced with groupby(level=1). + Note: do NOT convert groupby('instrument') → groupby(level=1) here — that would + undo the reset_index_groupby fix which correctly emits groupby('instrument'). """ fixed_code = code - # groupby(['instrument', 'date']) or groupby(['date', 'instrument']) - # Note: do NOT convert groupby('instrument') → groupby(level=1) here — - # that would undo the reset_index_groupby fix which correctly emits groupby('instrument'). - for pat, repl in [ - (r"\.groupby\(\['instrument',\s*'date'\]\)", ".groupby(level=1)"), - (r"\.groupby\(\['date',\s*'instrument'\]\)", ".groupby(level=1)"), - (r"\.groupby\(\['instrument'\]\)", ".groupby(level=1)"), - ]: - if re.search(pat, fixed_code): - fixed_code = re.sub(pat, repl, fixed_code) - self.fixes_applied.append(f"multiindex_groupby: {pat} → {repl}") + def _replace_two_col_groupby(m: re.Match, order: str) -> str: + var = m.group(1) + if order == "instrument_date": + repl = ( + f"{var}.groupby([{var}.index.get_level_values(1), " + f"{var}.index.get_level_values(0).normalize()])" + ) + else: # date_instrument + repl = ( + f"{var}.groupby([{var}.index.get_level_values(0).normalize(), " + f"{var}.index.get_level_values(1)])" + ) + self.fixes_applied.append(f"multiindex_groupby: {m.group(0)[:60]} → two-level") + return repl + + # groupby(['instrument', 'date']) — capture variable name before .groupby + fixed_code = re.sub( + r'(\w+)\.groupby\(\[\'instrument\',\s*\'date\'\]\)', + lambda m: _replace_two_col_groupby(m, "instrument_date"), + fixed_code, + ) + # groupby(['date', 'instrument']) + fixed_code = re.sub( + r'(\w+)\.groupby\(\[\'date\',\s*\'instrument\'\]\)', + lambda m: _replace_two_col_groupby(m, "date_instrument"), + fixed_code, + ) + # single: groupby(['instrument']) → groupby(level=1) + if re.search(r"\.groupby\(\['instrument'\]\)", fixed_code): + fixed_code = re.sub(r"\.groupby\(\['instrument'\]\)", ".groupby(level=1)", fixed_code) + self.fixes_applied.append("multiindex_groupby: groupby(['instrument']) → groupby(level=1)") return fixed_code diff --git a/test/qlib/test_auto_fixer.py b/test/qlib/test_auto_fixer.py new file mode 100644 index 00000000..cd89180c --- /dev/null +++ b/test/qlib/test_auto_fixer.py @@ -0,0 +1,68 @@ +"""Tests for FactorAutoFixer — the pre-execution code patcher.""" + +import pytest + +from rdagent.components.coder.factor_coder.auto_fixer import FactorAutoFixer + + +@pytest.fixture() +def fixer(): + return FactorAutoFixer() + + +class TestResetIndexGroupby: + def test_replaces_level_groupby_on_reset_var(self, fixer): + code = "df_r = df.reset_index()\ndf_r['x'] = df_r.groupby(level=1)['$close'].mean()" + result = fixer.fix(code) + assert "groupby('instrument')" in result + + def test_does_not_touch_normal_multiindex_groupby(self, fixer): + code = "df['x'] = df.groupby(level=1)['$close'].mean()" + result = fixer.fix(code) + assert "groupby(level=1)" in result + + +class TestGroupbyMixedLevels: + def test_strips_string_from_mixed_list(self, fixer): + result = fixer.fix("df.groupby(level=[1, 'date']).apply(fn)") + assert "groupby(level=1)" in result + + def test_multiple_ints_kept(self, fixer): + result = fixer.fix("df.groupby(level=[0, 1, 'x']).apply(fn)") + assert "groupby(level=[0, 1])" in result + + +class TestGroupbyColumnOnMultiindex: + def test_instrument_date_becomes_two_level(self, fixer): + code = "df['v'] = df.groupby(['instrument', 'date'])['$volume'].cumsum()" + result = fixer.fix(code) + assert "get_level_values(1)" in result + assert "normalize()" in result + assert "level=1)" not in result.split("get_level_values")[0] + + def test_date_instrument_becomes_two_level(self, fixer): + code = "df['v'] = df.groupby(['date', 'instrument'])['$volume'].cumsum()" + result = fixer.fix(code) + assert "get_level_values(0).normalize()" in result + assert "get_level_values(1)" in result + + def test_single_instrument_becomes_level1(self, fixer): + result = fixer.fix("df.groupby(['instrument'])['x'].mean()") + assert "groupby(level=1)" in result + + def test_reset_index_not_double_fixed(self, fixer): + # After reset_index fix emits groupby('instrument'), this fixer must NOT + # convert that to groupby(level=1). + code = "df_r = df.reset_index()\ndf_r['x'] = df_r.groupby(level=1)['p'].mean()" + result = fixer.fix(code) + assert "groupby('instrument')" in result + + +class TestRollingDdof: + def test_removes_ddof_from_rolling_args(self, fixer): + result = fixer.fix("df.rolling(20, min_periods=1, ddof=1).std()") + assert "ddof" not in result + + def test_removes_ddof_from_std_args(self, fixer): + result = fixer.fix("df.rolling(20).std(ddof=1)") + assert "ddof" not in result