fix(auto-fixer): fix df.loc[instrument] DateParseError on MultiIndex frames

When LLM iterates over instruments via get_level_values('instrument').unique()
and then does df.loc[instrument], pandas tries to parse the instrument string
('EURUSD') as a datetime against level-0 of the (datetime, instrument) index,
raising DateParseError.

Fix: detect loop variables bound to get_level_values(1) or get_level_values('instrument')
and replace DF.loc[loop_var] (read) with DF.xs(loop_var, level=1). Assignment
write-backs are left untouched to avoid complex rewrites.

Adds 4 new tests (TestInstrumentLocMultiindex) — total 23 tests, all passing.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
TPTBusiness
2026-04-27 15:31:35 +02:00
parent 8070de3ae1
commit eaebd60d93
2 changed files with 90 additions and 9 deletions
@@ -57,15 +57,16 @@ class FactorAutoFixer:
# (window=240 > 96 means zero valid bars per day). The LLM sets its own min_periods.
fix_methods = [
self._fix_instrument_column_access, # First: fix df['instrument'] on MultiIndex
self._fix_reset_index_groupby, # Second: fix groupby(level=N) after reset_index()
self._fix_groupby_mixed_levels, # Third: fix groupby(level=[int, str])
self._fix_groupby_column_on_multiindex, # Fourth: fix groupby(['instrument','date']) on MultiIndex
self._fix_chained_groupby, # Fifth: fix groupby(level=N).groupby('date') chain
self._fix_rolling_ddof, # Sixth: remove unsupported ddof kwarg
self._fix_groupby_apply_to_transform, # Seventh: fix groupby patterns
self._fix_inf_nan_handling, # Eighth: add inf/nan handling
self._fix_data_range_processing, # Ninth: ensure full data range
self._fix_multiindex_groupby, # Tenth: ensure groupby on MultiIndex
self._fix_instrument_loc_multiindex, # Second: fix df.loc[instrument_var] on MultiIndex
self._fix_reset_index_groupby, # Third: fix groupby(level=N) after reset_index()
self._fix_groupby_mixed_levels, # Fourth: fix groupby(level=[int, str])
self._fix_groupby_column_on_multiindex, # Fifth: fix groupby(['instrument','date']) on MultiIndex
self._fix_chained_groupby, # Sixth: fix groupby(level=N).groupby('date') chain
self._fix_rolling_ddof, # Seventh: remove unsupported ddof kwarg
self._fix_groupby_apply_to_transform, # Eighth: fix groupby patterns
self._fix_inf_nan_handling, # Ninth: add inf/nan handling
self._fix_data_range_processing, # Tenth: ensure full data range
self._fix_multiindex_groupby, # Eleventh: ensure groupby on MultiIndex
]
for fix_method in fix_methods:
@@ -115,6 +116,52 @@ class FactorAutoFixer:
return fixed_code
def _fix_instrument_loc_multiindex(self, code: str) -> str:
"""
Fix: df.loc[instrument_var] raises DateParseError on a (datetime, instrument)
MultiIndex because pandas tries to match the instrument string against the
datetime level (level 0).
Pattern detected: for-loops iterating over get_level_values('instrument') or
get_level_values(1) where the loop variable is then used as df.loc[loop_var].
Replacement: df.loc[instrument_var] → df.xs(instrument_var, level=1)
"""
fixed_code = code
# Find variables iterated from get_level_values('instrument') or get_level_values(1)
inst_vars = set(
re.findall(
r"for\s+(\w+)\s+in\s+.+?\.get_level_values\s*\(\s*(?:1|['\"]instrument['\"])\s*\)[^:\n]*:",
code,
)
)
if not inst_vars:
return fixed_code
for var in inst_vars:
# Replace DF.loc[var] (read) with DF.xs(var, level=1)
# Exclude write-back patterns (DF.loc[var] = ...) — leave those as-is
def _make_replacer(v: str):
def _replace(m: re.Match) -> str:
df_var = m.group(1)
self.fixes_applied.append(
f"instrument_loc: {df_var}.loc[{v}] → {df_var}.xs({v}, level=1)"
)
return f"{df_var}.xs({v}, level=1)"
return _replace
# Only match when NOT followed by ' =' (assignment)
fixed_code = re.sub(
rf"(\w+)\.loc\[\s*{re.escape(var)}\s*\](?!\s*=)",
_make_replacer(var),
fixed_code,
)
return fixed_code
def _fix_reset_index_groupby(self, code: str) -> str:
"""
Fix: groupby(level=N) on a variable created by .reset_index() fails because
+34
View File
@@ -122,6 +122,40 @@ class TestInstrumentColumnAccess:
assert "df['instrument']" not in result
class TestInstrumentLocMultiindex:
def test_loc_replaced_with_xs(self, fixer):
code = (
"for instrument in df.index.get_level_values('instrument').unique():\n"
" inst_df = df.loc[instrument].copy()\n"
)
result = fixer.fix(code)
assert "df.xs(instrument, level=1)" in result
assert "df.loc[instrument]" not in result
def test_loc_replaced_with_level1_int(self, fixer):
code = (
"for inst in df.index.get_level_values(1).unique():\n"
" data = df.loc[inst]\n"
)
result = fixer.fix(code)
assert "df.xs(inst, level=1)" in result
def test_loc_assignment_not_touched(self, fixer):
# Write-back df.loc[instrument] = ... must not be changed
code = (
"for instrument in df.index.get_level_values('instrument').unique():\n"
" df.loc[instrument] = modified\n"
)
result = fixer.fix(code)
assert "df.loc[instrument] = modified" in result
def test_non_instrument_loop_not_touched(self, fixer):
# for-loop not related to instrument levels must not be changed
code = "for date in dates:\n sub = df.loc[date]\n"
result = fixer.fix(code)
assert "df.loc[date]" in result
class TestRollingDdof:
def test_removes_ddof_from_rolling_args(self, fixer):
result = fixer.fix("df.rolling(20, min_periods=1, ddof=1).std()")