mirror of
https://github.com/NicolasBohn/NexQuant.git
synced 2026-07-27 15:37:44 +00:00
fix(auto-fixer): preserve date dimension in groupby(['instrument','date']) fix
The previous fixer converted groupby(['instrument','date']) → groupby(level=1),
stripping the date level. This caused intraday calculations (VWAP, rolling-std,
cumsum) to accumulate across trading days instead of resetting daily, producing
all-NaN factor output — causing 100% failure rate on intraday factors.
New behaviour: capture the DataFrame variable name and emit:
var.groupby([var.index.get_level_values(1),
var.index.get_level_values(0).normalize()])
which groups by (instrument, day) as originally intended.
Adds test/qlib/test_auto_fixer.py covering all fixer cases.
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -129,25 +129,51 @@ class FactorAutoFixer:
|
||||
|
||||
def _fix_groupby_column_on_multiindex(self, code: str) -> str:
|
||||
"""
|
||||
Fix: groupby(['instrument', 'date']) on a MultiIndex DataFrame fails with
|
||||
KeyError because 'instrument' and 'date' are index levels, not columns.
|
||||
Fix: groupby(['instrument', 'date']) on a MultiIndex (datetime, instrument)
|
||||
DataFrame fails with KeyError because those are index levels, not columns.
|
||||
|
||||
Replace with groupby(level=1) (instrument is level 1).
|
||||
Also handle groupby(['date', 'instrument']) and single groupby('instrument').
|
||||
Correct replacement preserves BOTH dimensions so intraday calculations reset
|
||||
per day:
|
||||
var.groupby(['instrument', 'date'])
|
||||
→ var.groupby([var.index.get_level_values(1), var.index.get_level_values(0).normalize()])
|
||||
|
||||
Single-column groupby(['instrument']) is correctly replaced with groupby(level=1).
|
||||
Note: do NOT convert groupby('instrument') → groupby(level=1) here — that would
|
||||
undo the reset_index_groupby fix which correctly emits groupby('instrument').
|
||||
"""
|
||||
fixed_code = code
|
||||
|
||||
# groupby(['instrument', 'date']) or groupby(['date', 'instrument'])
|
||||
# Note: do NOT convert groupby('instrument') → groupby(level=1) here —
|
||||
# that would undo the reset_index_groupby fix which correctly emits groupby('instrument').
|
||||
for pat, repl in [
|
||||
(r"\.groupby\(\['instrument',\s*'date'\]\)", ".groupby(level=1)"),
|
||||
(r"\.groupby\(\['date',\s*'instrument'\]\)", ".groupby(level=1)"),
|
||||
(r"\.groupby\(\['instrument'\]\)", ".groupby(level=1)"),
|
||||
]:
|
||||
if re.search(pat, fixed_code):
|
||||
fixed_code = re.sub(pat, repl, fixed_code)
|
||||
self.fixes_applied.append(f"multiindex_groupby: {pat} → {repl}")
|
||||
def _replace_two_col_groupby(m: re.Match, order: str) -> str:
|
||||
var = m.group(1)
|
||||
if order == "instrument_date":
|
||||
repl = (
|
||||
f"{var}.groupby([{var}.index.get_level_values(1), "
|
||||
f"{var}.index.get_level_values(0).normalize()])"
|
||||
)
|
||||
else: # date_instrument
|
||||
repl = (
|
||||
f"{var}.groupby([{var}.index.get_level_values(0).normalize(), "
|
||||
f"{var}.index.get_level_values(1)])"
|
||||
)
|
||||
self.fixes_applied.append(f"multiindex_groupby: {m.group(0)[:60]} → two-level")
|
||||
return repl
|
||||
|
||||
# groupby(['instrument', 'date']) — capture variable name before .groupby
|
||||
fixed_code = re.sub(
|
||||
r'(\w+)\.groupby\(\[\'instrument\',\s*\'date\'\]\)',
|
||||
lambda m: _replace_two_col_groupby(m, "instrument_date"),
|
||||
fixed_code,
|
||||
)
|
||||
# groupby(['date', 'instrument'])
|
||||
fixed_code = re.sub(
|
||||
r'(\w+)\.groupby\(\[\'date\',\s*\'instrument\'\]\)',
|
||||
lambda m: _replace_two_col_groupby(m, "date_instrument"),
|
||||
fixed_code,
|
||||
)
|
||||
# single: groupby(['instrument']) → groupby(level=1)
|
||||
if re.search(r"\.groupby\(\['instrument'\]\)", fixed_code):
|
||||
fixed_code = re.sub(r"\.groupby\(\['instrument'\]\)", ".groupby(level=1)", fixed_code)
|
||||
self.fixes_applied.append("multiindex_groupby: groupby(['instrument']) → groupby(level=1)")
|
||||
|
||||
return fixed_code
|
||||
|
||||
|
||||
@@ -0,0 +1,68 @@
|
||||
"""Tests for FactorAutoFixer — the pre-execution code patcher."""
|
||||
|
||||
import pytest
|
||||
|
||||
from rdagent.components.coder.factor_coder.auto_fixer import FactorAutoFixer
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def fixer():
|
||||
return FactorAutoFixer()
|
||||
|
||||
|
||||
class TestResetIndexGroupby:
|
||||
def test_replaces_level_groupby_on_reset_var(self, fixer):
|
||||
code = "df_r = df.reset_index()\ndf_r['x'] = df_r.groupby(level=1)['$close'].mean()"
|
||||
result = fixer.fix(code)
|
||||
assert "groupby('instrument')" in result
|
||||
|
||||
def test_does_not_touch_normal_multiindex_groupby(self, fixer):
|
||||
code = "df['x'] = df.groupby(level=1)['$close'].mean()"
|
||||
result = fixer.fix(code)
|
||||
assert "groupby(level=1)" in result
|
||||
|
||||
|
||||
class TestGroupbyMixedLevels:
|
||||
def test_strips_string_from_mixed_list(self, fixer):
|
||||
result = fixer.fix("df.groupby(level=[1, 'date']).apply(fn)")
|
||||
assert "groupby(level=1)" in result
|
||||
|
||||
def test_multiple_ints_kept(self, fixer):
|
||||
result = fixer.fix("df.groupby(level=[0, 1, 'x']).apply(fn)")
|
||||
assert "groupby(level=[0, 1])" in result
|
||||
|
||||
|
||||
class TestGroupbyColumnOnMultiindex:
|
||||
def test_instrument_date_becomes_two_level(self, fixer):
|
||||
code = "df['v'] = df.groupby(['instrument', 'date'])['$volume'].cumsum()"
|
||||
result = fixer.fix(code)
|
||||
assert "get_level_values(1)" in result
|
||||
assert "normalize()" in result
|
||||
assert "level=1)" not in result.split("get_level_values")[0]
|
||||
|
||||
def test_date_instrument_becomes_two_level(self, fixer):
|
||||
code = "df['v'] = df.groupby(['date', 'instrument'])['$volume'].cumsum()"
|
||||
result = fixer.fix(code)
|
||||
assert "get_level_values(0).normalize()" in result
|
||||
assert "get_level_values(1)" in result
|
||||
|
||||
def test_single_instrument_becomes_level1(self, fixer):
|
||||
result = fixer.fix("df.groupby(['instrument'])['x'].mean()")
|
||||
assert "groupby(level=1)" in result
|
||||
|
||||
def test_reset_index_not_double_fixed(self, fixer):
|
||||
# After reset_index fix emits groupby('instrument'), this fixer must NOT
|
||||
# convert that to groupby(level=1).
|
||||
code = "df_r = df.reset_index()\ndf_r['x'] = df_r.groupby(level=1)['p'].mean()"
|
||||
result = fixer.fix(code)
|
||||
assert "groupby('instrument')" in result
|
||||
|
||||
|
||||
class TestRollingDdof:
|
||||
def test_removes_ddof_from_rolling_args(self, fixer):
|
||||
result = fixer.fix("df.rolling(20, min_periods=1, ddof=1).std()")
|
||||
assert "ddof" not in result
|
||||
|
||||
def test_removes_ddof_from_std_args(self, fixer):
|
||||
result = fixer.fix("df.rolling(20).std(ddof=1)")
|
||||
assert "ddof" not in result
|
||||
Reference in New Issue
Block a user