mirror of
https://github.com/NicolasBohn/NexQuant.git
synced 2026-07-27 15:37:44 +00:00
fix(auto-fixer): fix df.loc[instrument] DateParseError on MultiIndex frames
When LLM iterates over instruments via get_level_values('instrument').unique()
and then does df.loc[instrument], pandas tries to parse the instrument string
('EURUSD') as a datetime against level-0 of the (datetime, instrument) index,
raising DateParseError.
Fix: detect loop variables bound to get_level_values(1) or get_level_values('instrument')
and replace DF.loc[loop_var] (read) with DF.xs(loop_var, level=1). Assignment
write-backs are left untouched to avoid complex rewrites.
Adds 4 new tests (TestInstrumentLocMultiindex) — total 23 tests, all passing.
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -57,15 +57,16 @@ class FactorAutoFixer:
|
||||
# (window=240 > 96 means zero valid bars per day). The LLM sets its own min_periods.
|
||||
fix_methods = [
|
||||
self._fix_instrument_column_access, # First: fix df['instrument'] on MultiIndex
|
||||
self._fix_reset_index_groupby, # Second: fix groupby(level=N) after reset_index()
|
||||
self._fix_groupby_mixed_levels, # Third: fix groupby(level=[int, str])
|
||||
self._fix_groupby_column_on_multiindex, # Fourth: fix groupby(['instrument','date']) on MultiIndex
|
||||
self._fix_chained_groupby, # Fifth: fix groupby(level=N).groupby('date') chain
|
||||
self._fix_rolling_ddof, # Sixth: remove unsupported ddof kwarg
|
||||
self._fix_groupby_apply_to_transform, # Seventh: fix groupby patterns
|
||||
self._fix_inf_nan_handling, # Eighth: add inf/nan handling
|
||||
self._fix_data_range_processing, # Ninth: ensure full data range
|
||||
self._fix_multiindex_groupby, # Tenth: ensure groupby on MultiIndex
|
||||
self._fix_instrument_loc_multiindex, # Second: fix df.loc[instrument_var] on MultiIndex
|
||||
self._fix_reset_index_groupby, # Third: fix groupby(level=N) after reset_index()
|
||||
self._fix_groupby_mixed_levels, # Fourth: fix groupby(level=[int, str])
|
||||
self._fix_groupby_column_on_multiindex, # Fifth: fix groupby(['instrument','date']) on MultiIndex
|
||||
self._fix_chained_groupby, # Sixth: fix groupby(level=N).groupby('date') chain
|
||||
self._fix_rolling_ddof, # Seventh: remove unsupported ddof kwarg
|
||||
self._fix_groupby_apply_to_transform, # Eighth: fix groupby patterns
|
||||
self._fix_inf_nan_handling, # Ninth: add inf/nan handling
|
||||
self._fix_data_range_processing, # Tenth: ensure full data range
|
||||
self._fix_multiindex_groupby, # Eleventh: ensure groupby on MultiIndex
|
||||
]
|
||||
|
||||
for fix_method in fix_methods:
|
||||
@@ -115,6 +116,52 @@ class FactorAutoFixer:
|
||||
|
||||
return fixed_code
|
||||
|
||||
def _fix_instrument_loc_multiindex(self, code: str) -> str:
|
||||
"""
|
||||
Fix: df.loc[instrument_var] raises DateParseError on a (datetime, instrument)
|
||||
MultiIndex because pandas tries to match the instrument string against the
|
||||
datetime level (level 0).
|
||||
|
||||
Pattern detected: for-loops iterating over get_level_values('instrument') or
|
||||
get_level_values(1) where the loop variable is then used as df.loc[loop_var].
|
||||
|
||||
Replacement: df.loc[instrument_var] → df.xs(instrument_var, level=1)
|
||||
"""
|
||||
fixed_code = code
|
||||
|
||||
# Find variables iterated from get_level_values('instrument') or get_level_values(1)
|
||||
inst_vars = set(
|
||||
re.findall(
|
||||
r"for\s+(\w+)\s+in\s+.+?\.get_level_values\s*\(\s*(?:1|['\"]instrument['\"])\s*\)[^:\n]*:",
|
||||
code,
|
||||
)
|
||||
)
|
||||
|
||||
if not inst_vars:
|
||||
return fixed_code
|
||||
|
||||
for var in inst_vars:
|
||||
# Replace DF.loc[var] (read) with DF.xs(var, level=1)
|
||||
# Exclude write-back patterns (DF.loc[var] = ...) — leave those as-is
|
||||
def _make_replacer(v: str):
|
||||
def _replace(m: re.Match) -> str:
|
||||
df_var = m.group(1)
|
||||
self.fixes_applied.append(
|
||||
f"instrument_loc: {df_var}.loc[{v}] → {df_var}.xs({v}, level=1)"
|
||||
)
|
||||
return f"{df_var}.xs({v}, level=1)"
|
||||
|
||||
return _replace
|
||||
|
||||
# Only match when NOT followed by ' =' (assignment)
|
||||
fixed_code = re.sub(
|
||||
rf"(\w+)\.loc\[\s*{re.escape(var)}\s*\](?!\s*=)",
|
||||
_make_replacer(var),
|
||||
fixed_code,
|
||||
)
|
||||
|
||||
return fixed_code
|
||||
|
||||
def _fix_reset_index_groupby(self, code: str) -> str:
|
||||
"""
|
||||
Fix: groupby(level=N) on a variable created by .reset_index() fails because
|
||||
|
||||
@@ -122,6 +122,40 @@ class TestInstrumentColumnAccess:
|
||||
assert "df['instrument']" not in result
|
||||
|
||||
|
||||
class TestInstrumentLocMultiindex:
|
||||
def test_loc_replaced_with_xs(self, fixer):
|
||||
code = (
|
||||
"for instrument in df.index.get_level_values('instrument').unique():\n"
|
||||
" inst_df = df.loc[instrument].copy()\n"
|
||||
)
|
||||
result = fixer.fix(code)
|
||||
assert "df.xs(instrument, level=1)" in result
|
||||
assert "df.loc[instrument]" not in result
|
||||
|
||||
def test_loc_replaced_with_level1_int(self, fixer):
|
||||
code = (
|
||||
"for inst in df.index.get_level_values(1).unique():\n"
|
||||
" data = df.loc[inst]\n"
|
||||
)
|
||||
result = fixer.fix(code)
|
||||
assert "df.xs(inst, level=1)" in result
|
||||
|
||||
def test_loc_assignment_not_touched(self, fixer):
|
||||
# Write-back df.loc[instrument] = ... must not be changed
|
||||
code = (
|
||||
"for instrument in df.index.get_level_values('instrument').unique():\n"
|
||||
" df.loc[instrument] = modified\n"
|
||||
)
|
||||
result = fixer.fix(code)
|
||||
assert "df.loc[instrument] = modified" in result
|
||||
|
||||
def test_non_instrument_loop_not_touched(self, fixer):
|
||||
# for-loop not related to instrument levels must not be changed
|
||||
code = "for date in dates:\n sub = df.loc[date]\n"
|
||||
result = fixer.fix(code)
|
||||
assert "df.loc[date]" in result
|
||||
|
||||
|
||||
class TestRollingDdof:
|
||||
def test_removes_ddof_from_rolling_args(self, fixer):
|
||||
result = fixer.fix("df.rolling(20, min_periods=1, ddof=1).std()")
|
||||
|
||||
Reference in New Issue
Block a user