From dbbe374ac8b0cefcde9145a76b4cd5c0b40b3f92 Mon Sep 17 00:00:00 2001 From: you-n-g Date: Fri, 29 Aug 2025 17:07:41 +0800 Subject: [PATCH] fix: update fallback criterion (#1210) * fix: update fallback criterion * fix: ensure evo_fb is initialized and used correctly in fallback logic * refactor: rename use_new_evo to should_use_new_evo for clarity --- rdagent/components/coder/CoSTEER/__init__.py | 12 ++++++++---- .../scenarios/data_science/dev/runner/__init__.py | 5 ++++- .../scenarios/data_science/dev/runner/prompts.yaml | 2 +- rdagent/utils/prompts.yaml | 3 ++- 4 files changed, 15 insertions(+), 7 deletions(-) diff --git a/rdagent/components/coder/CoSTEER/__init__.py b/rdagent/components/coder/CoSTEER/__init__.py index 563715ce..b74313cc 100644 --- a/rdagent/components/coder/CoSTEER/__init__.py +++ b/rdagent/components/coder/CoSTEER/__init__.py @@ -77,7 +77,7 @@ class CoSTEER(Developer[Experiment]): assert isinstance(fb, CoSTEERMultiFeedback), "feedback must be of type CoSTEERMultiFeedback" return fb - def compare_and_pick_fb(self, base_fb: CoSTEERMultiFeedback | None, new_fb: CoSTEERMultiFeedback | None) -> bool: + def should_use_new_evo(self, base_fb: CoSTEERMultiFeedback | None, new_fb: CoSTEERMultiFeedback) -> bool: """ Compare new feedback with the fallback feedback. @@ -110,14 +110,16 @@ class CoSTEER(Developer[Experiment]): fallback_evo_exp = None fallback_evo_fb = None reached_max_seconds = False + + evo_fb = None for evo_exp in self.evolve_agent.multistep_evolve(evo_exp, self.evaluator): assert isinstance(evo_exp, Experiment) # multiple inheritance evo_fb = self._get_last_fb() - fallback_decision = self.compare_and_pick_fb( + update_fallback = self.should_use_new_evo( base_fb=fallback_evo_fb, new_fb=evo_fb, ) - if fallback_decision: + if update_fallback: fallback_evo_exp = deepcopy(evo_exp) fallback_evo_fb = deepcopy(evo_fb) fallback_evo_exp.create_ws_ckp() # NOTE: creating checkpoints for saving files in the workspace to prevent inplace mutation. @@ -139,7 +141,9 @@ class CoSTEER(Developer[Experiment]): logger.info("Fallback to the fallback solution.") evo_exp = fallback_evo_exp evo_exp.recover_ws_ckp() - evo_exp = self._exp_postprocess_by_feedback(evo_exp, self._get_last_fb()) + evo_fb = fallback_evo_fb + assert evo_fb is not None # multistep_evolve should run at least once + evo_exp = self._exp_postprocess_by_feedback(evo_exp, evo_fb) except CoderError as e: e.caused_by_timeout = reached_max_seconds raise e diff --git a/rdagent/scenarios/data_science/dev/runner/__init__.py b/rdagent/scenarios/data_science/dev/runner/__init__.py index 86d7df58..073572a2 100644 --- a/rdagent/scenarios/data_science/dev/runner/__init__.py +++ b/rdagent/scenarios/data_science/dev/runner/__init__.py @@ -171,7 +171,10 @@ class DSCoSTEERRunner(CoSTEER): """ return int(self.scen.real_full_timeout() * self.settings.max_seconds_multiplier) - def compare_and_pick_fb(self, base_fb: CoSTEERMultiFeedback | None, new_fb: CoSTEERMultiFeedback | None) -> bool: + def should_use_new_evo(self, base_fb: CoSTEERMultiFeedback | None, new_fb: CoSTEERMultiFeedback) -> bool: + if not new_fb.is_acceptable(): + return False + # In data science, we only have a single feedback. # Note: new_fb should always exists as indicated by _get_last_fb() function. if base_fb is None: diff --git a/rdagent/scenarios/data_science/dev/runner/prompts.yaml b/rdagent/scenarios/data_science/dev/runner/prompts.yaml index 79995d53..dabc4665 100644 --- a/rdagent/scenarios/data_science/dev/runner/prompts.yaml +++ b/rdagent/scenarios/data_science/dev/runner/prompts.yaml @@ -170,4 +170,4 @@ DSCoSTEER: ### Validation Scores {{ former_failed_knowledge.feedback.score }} {% endfor %} - {% endif %} \ No newline at end of file + {% endif %} diff --git a/rdagent/utils/prompts.yaml b/rdagent/utils/prompts.yaml index 810c848b..071406cf 100644 --- a/rdagent/utils/prompts.yaml +++ b/rdagent/utils/prompts.yaml @@ -4,7 +4,8 @@ filter_redundant_text: 1. Training log messages should be evaluated based on their usefulness—logs that contain meaningful training metrics such as loss or accuracy reported at each epoch should be retained, while redundant messages, such as those repeatedly reporting NaN values or iteration numbers without valuable information, should be removed. 2. For warning messages, **only one occurrence of each unique message should be kept**, eliminating any duplicates. 3. Additionally, any visual progress indicators, such as ASCII-based progress bars or dynamic percentage updates, should be removed. Once these patterns are identified, you should generate appropriate regex expressions to filter them out. - 4. Lastly, indicate whether substitution is needed. If the input exceeds a token limit, the system will provide only a shortened portion of the text. + 4. Don't remove useful information that is not duplicated. + 5. Lastly, indicate whether substitution is needed in `needs_sub` field. If the input exceeds a token limit, the system will provide only a shortened portion of the text. Respond in the following JSON format and order: {