fix(auto-fixer): preserve date dimension in groupby(['instrument','date']) fix

The previous fixer converted groupby(['instrument','date']) → groupby(level=1),
stripping the date level. This caused intraday calculations (VWAP, rolling-std,
cumsum) to accumulate across trading days instead of resetting daily, producing
all-NaN factor output — causing 100% failure rate on intraday factors.

New behaviour: capture the DataFrame variable name and emit:
  var.groupby([var.index.get_level_values(1),
               var.index.get_level_values(0).normalize()])
which groups by (instrument, day) as originally intended.

Adds test/qlib/test_auto_fixer.py covering all fixer cases.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
TPTBusiness
2026-04-26 11:50:52 +02:00
parent bd5a5e0fd5
commit f4deda99b5
2 changed files with 109 additions and 15 deletions
@@ -129,25 +129,51 @@ class FactorAutoFixer:
def _fix_groupby_column_on_multiindex(self, code: str) -> str:
"""
Fix: groupby(['instrument', 'date']) on a MultiIndex DataFrame fails with
KeyError because 'instrument' and 'date' are index levels, not columns.
Fix: groupby(['instrument', 'date']) on a MultiIndex (datetime, instrument)
DataFrame fails with KeyError because those are index levels, not columns.
Replace with groupby(level=1) (instrument is level 1).
Also handle groupby(['date', 'instrument']) and single groupby('instrument').
Correct replacement preserves BOTH dimensions so intraday calculations reset
per day:
var.groupby(['instrument', 'date'])
→ var.groupby([var.index.get_level_values(1), var.index.get_level_values(0).normalize()])
Single-column groupby(['instrument']) is correctly replaced with groupby(level=1).
Note: do NOT convert groupby('instrument') → groupby(level=1) here — that would
undo the reset_index_groupby fix which correctly emits groupby('instrument').
"""
fixed_code = code
# groupby(['instrument', 'date']) or groupby(['date', 'instrument'])
# Note: do NOT convert groupby('instrument') → groupby(level=1) here —
# that would undo the reset_index_groupby fix which correctly emits groupby('instrument').
for pat, repl in [
(r"\.groupby\(\['instrument',\s*'date'\]\)", ".groupby(level=1)"),
(r"\.groupby\(\['date',\s*'instrument'\]\)", ".groupby(level=1)"),
(r"\.groupby\(\['instrument'\]\)", ".groupby(level=1)"),
]:
if re.search(pat, fixed_code):
fixed_code = re.sub(pat, repl, fixed_code)
self.fixes_applied.append(f"multiindex_groupby: {pat}{repl}")
def _replace_two_col_groupby(m: re.Match, order: str) -> str:
var = m.group(1)
if order == "instrument_date":
repl = (
f"{var}.groupby([{var}.index.get_level_values(1), "
f"{var}.index.get_level_values(0).normalize()])"
)
else: # date_instrument
repl = (
f"{var}.groupby([{var}.index.get_level_values(0).normalize(), "
f"{var}.index.get_level_values(1)])"
)
self.fixes_applied.append(f"multiindex_groupby: {m.group(0)[:60]} → two-level")
return repl
# groupby(['instrument', 'date']) — capture variable name before .groupby
fixed_code = re.sub(
r'(\w+)\.groupby\(\[\'instrument\',\s*\'date\'\]\)',
lambda m: _replace_two_col_groupby(m, "instrument_date"),
fixed_code,
)
# groupby(['date', 'instrument'])
fixed_code = re.sub(
r'(\w+)\.groupby\(\[\'date\',\s*\'instrument\'\]\)',
lambda m: _replace_two_col_groupby(m, "date_instrument"),
fixed_code,
)
# single: groupby(['instrument']) → groupby(level=1)
if re.search(r"\.groupby\(\['instrument'\]\)", fixed_code):
fixed_code = re.sub(r"\.groupby\(\['instrument'\]\)", ".groupby(level=1)", fixed_code)
self.fixes_applied.append("multiindex_groupby: groupby(['instrument']) → groupby(level=1)")
return fixed_code
+68
View File
@@ -0,0 +1,68 @@
"""Tests for FactorAutoFixer — the pre-execution code patcher."""
import pytest
from rdagent.components.coder.factor_coder.auto_fixer import FactorAutoFixer
@pytest.fixture()
def fixer():
return FactorAutoFixer()
class TestResetIndexGroupby:
def test_replaces_level_groupby_on_reset_var(self, fixer):
code = "df_r = df.reset_index()\ndf_r['x'] = df_r.groupby(level=1)['$close'].mean()"
result = fixer.fix(code)
assert "groupby('instrument')" in result
def test_does_not_touch_normal_multiindex_groupby(self, fixer):
code = "df['x'] = df.groupby(level=1)['$close'].mean()"
result = fixer.fix(code)
assert "groupby(level=1)" in result
class TestGroupbyMixedLevels:
def test_strips_string_from_mixed_list(self, fixer):
result = fixer.fix("df.groupby(level=[1, 'date']).apply(fn)")
assert "groupby(level=1)" in result
def test_multiple_ints_kept(self, fixer):
result = fixer.fix("df.groupby(level=[0, 1, 'x']).apply(fn)")
assert "groupby(level=[0, 1])" in result
class TestGroupbyColumnOnMultiindex:
def test_instrument_date_becomes_two_level(self, fixer):
code = "df['v'] = df.groupby(['instrument', 'date'])['$volume'].cumsum()"
result = fixer.fix(code)
assert "get_level_values(1)" in result
assert "normalize()" in result
assert "level=1)" not in result.split("get_level_values")[0]
def test_date_instrument_becomes_two_level(self, fixer):
code = "df['v'] = df.groupby(['date', 'instrument'])['$volume'].cumsum()"
result = fixer.fix(code)
assert "get_level_values(0).normalize()" in result
assert "get_level_values(1)" in result
def test_single_instrument_becomes_level1(self, fixer):
result = fixer.fix("df.groupby(['instrument'])['x'].mean()")
assert "groupby(level=1)" in result
def test_reset_index_not_double_fixed(self, fixer):
# After reset_index fix emits groupby('instrument'), this fixer must NOT
# convert that to groupby(level=1).
code = "df_r = df.reset_index()\ndf_r['x'] = df_r.groupby(level=1)['p'].mean()"
result = fixer.fix(code)
assert "groupby('instrument')" in result
class TestRollingDdof:
def test_removes_ddof_from_rolling_args(self, fixer):
result = fixer.fix("df.rolling(20, min_periods=1, ddof=1).std()")
assert "ddof" not in result
def test_removes_ddof_from_std_args(self, fixer):
result = fixer.fix("df.rolling(20).std(ddof=1)")
assert "ddof" not in result