From eaebd60d93d404954176617b8ba6fb06350568fb Mon Sep 17 00:00:00 2001 From: TPTBusiness Date: Mon, 27 Apr 2026 15:31:35 +0200 Subject: [PATCH] fix(auto-fixer): fix df.loc[instrument] DateParseError on MultiIndex frames MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When LLM iterates over instruments via get_level_values('instrument').unique() and then does df.loc[instrument], pandas tries to parse the instrument string ('EURUSD') as a datetime against level-0 of the (datetime, instrument) index, raising DateParseError. Fix: detect loop variables bound to get_level_values(1) or get_level_values('instrument') and replace DF.loc[loop_var] (read) with DF.xs(loop_var, level=1). Assignment write-backs are left untouched to avoid complex rewrites. Adds 4 new tests (TestInstrumentLocMultiindex) — total 23 tests, all passing. Co-Authored-By: Claude Sonnet 4.6 --- .../coder/factor_coder/auto_fixer.py | 65 ++++++++++++++++--- test/qlib/test_auto_fixer.py | 34 ++++++++++ 2 files changed, 90 insertions(+), 9 deletions(-) diff --git a/rdagent/components/coder/factor_coder/auto_fixer.py b/rdagent/components/coder/factor_coder/auto_fixer.py index 48f8ab24..7aec60d9 100644 --- a/rdagent/components/coder/factor_coder/auto_fixer.py +++ b/rdagent/components/coder/factor_coder/auto_fixer.py @@ -57,15 +57,16 @@ class FactorAutoFixer: # (window=240 > 96 means zero valid bars per day). The LLM sets its own min_periods. fix_methods = [ self._fix_instrument_column_access, # First: fix df['instrument'] on MultiIndex - self._fix_reset_index_groupby, # Second: fix groupby(level=N) after reset_index() - self._fix_groupby_mixed_levels, # Third: fix groupby(level=[int, str]) - self._fix_groupby_column_on_multiindex, # Fourth: fix groupby(['instrument','date']) on MultiIndex - self._fix_chained_groupby, # Fifth: fix groupby(level=N).groupby('date') chain - self._fix_rolling_ddof, # Sixth: remove unsupported ddof kwarg - self._fix_groupby_apply_to_transform, # Seventh: fix groupby patterns - self._fix_inf_nan_handling, # Eighth: add inf/nan handling - self._fix_data_range_processing, # Ninth: ensure full data range - self._fix_multiindex_groupby, # Tenth: ensure groupby on MultiIndex + self._fix_instrument_loc_multiindex, # Second: fix df.loc[instrument_var] on MultiIndex + self._fix_reset_index_groupby, # Third: fix groupby(level=N) after reset_index() + self._fix_groupby_mixed_levels, # Fourth: fix groupby(level=[int, str]) + self._fix_groupby_column_on_multiindex, # Fifth: fix groupby(['instrument','date']) on MultiIndex + self._fix_chained_groupby, # Sixth: fix groupby(level=N).groupby('date') chain + self._fix_rolling_ddof, # Seventh: remove unsupported ddof kwarg + self._fix_groupby_apply_to_transform, # Eighth: fix groupby patterns + self._fix_inf_nan_handling, # Ninth: add inf/nan handling + self._fix_data_range_processing, # Tenth: ensure full data range + self._fix_multiindex_groupby, # Eleventh: ensure groupby on MultiIndex ] for fix_method in fix_methods: @@ -115,6 +116,52 @@ class FactorAutoFixer: return fixed_code + def _fix_instrument_loc_multiindex(self, code: str) -> str: + """ + Fix: df.loc[instrument_var] raises DateParseError on a (datetime, instrument) + MultiIndex because pandas tries to match the instrument string against the + datetime level (level 0). + + Pattern detected: for-loops iterating over get_level_values('instrument') or + get_level_values(1) where the loop variable is then used as df.loc[loop_var]. + + Replacement: df.loc[instrument_var] → df.xs(instrument_var, level=1) + """ + fixed_code = code + + # Find variables iterated from get_level_values('instrument') or get_level_values(1) + inst_vars = set( + re.findall( + r"for\s+(\w+)\s+in\s+.+?\.get_level_values\s*\(\s*(?:1|['\"]instrument['\"])\s*\)[^:\n]*:", + code, + ) + ) + + if not inst_vars: + return fixed_code + + for var in inst_vars: + # Replace DF.loc[var] (read) with DF.xs(var, level=1) + # Exclude write-back patterns (DF.loc[var] = ...) — leave those as-is + def _make_replacer(v: str): + def _replace(m: re.Match) -> str: + df_var = m.group(1) + self.fixes_applied.append( + f"instrument_loc: {df_var}.loc[{v}] → {df_var}.xs({v}, level=1)" + ) + return f"{df_var}.xs({v}, level=1)" + + return _replace + + # Only match when NOT followed by ' =' (assignment) + fixed_code = re.sub( + rf"(\w+)\.loc\[\s*{re.escape(var)}\s*\](?!\s*=)", + _make_replacer(var), + fixed_code, + ) + + return fixed_code + def _fix_reset_index_groupby(self, code: str) -> str: """ Fix: groupby(level=N) on a variable created by .reset_index() fails because diff --git a/test/qlib/test_auto_fixer.py b/test/qlib/test_auto_fixer.py index ed03afb1..4db980ef 100644 --- a/test/qlib/test_auto_fixer.py +++ b/test/qlib/test_auto_fixer.py @@ -122,6 +122,40 @@ class TestInstrumentColumnAccess: assert "df['instrument']" not in result +class TestInstrumentLocMultiindex: + def test_loc_replaced_with_xs(self, fixer): + code = ( + "for instrument in df.index.get_level_values('instrument').unique():\n" + " inst_df = df.loc[instrument].copy()\n" + ) + result = fixer.fix(code) + assert "df.xs(instrument, level=1)" in result + assert "df.loc[instrument]" not in result + + def test_loc_replaced_with_level1_int(self, fixer): + code = ( + "for inst in df.index.get_level_values(1).unique():\n" + " data = df.loc[inst]\n" + ) + result = fixer.fix(code) + assert "df.xs(inst, level=1)" in result + + def test_loc_assignment_not_touched(self, fixer): + # Write-back df.loc[instrument] = ... must not be changed + code = ( + "for instrument in df.index.get_level_values('instrument').unique():\n" + " df.loc[instrument] = modified\n" + ) + result = fixer.fix(code) + assert "df.loc[instrument] = modified" in result + + def test_non_instrument_loop_not_touched(self, fixer): + # for-loop not related to instrument levels must not be changed + code = "for date in dates:\n sub = df.loc[date]\n" + result = fixer.fix(code) + assert "df.loc[date]" in result + + class TestRollingDdof: def test_removes_ddof_from_rolling_args(self, fixer): result = fixer.fix("df.rolling(20, min_periods=1, ddof=1).std()")