fix: update fallback criterion (#1210)

* fix: update fallback criterion

* fix: ensure evo_fb is initialized and used correctly in fallback logic

* refactor: rename use_new_evo to should_use_new_evo for clarity

rdagent/components/coder/CoSTEER/__init__.py

@@ -77,7 +77,7 @@ class CoSTEER(Developer[Experiment]):
         assert isinstance(fb, CoSTEERMultiFeedback), "feedback must be of type CoSTEERMultiFeedback"
         return fb
 
-    def compare_and_pick_fb(self, base_fb: CoSTEERMultiFeedback | None, new_fb: CoSTEERMultiFeedback | None) -> bool:
+    def should_use_new_evo(self, base_fb: CoSTEERMultiFeedback | None, new_fb: CoSTEERMultiFeedback) -> bool:
         """
         Compare new feedback with the fallback feedback.
 
@@ -110,14 +110,16 @@ class CoSTEER(Developer[Experiment]):
         fallback_evo_exp = None
         fallback_evo_fb = None
         reached_max_seconds = False
+
+        evo_fb = None
         for evo_exp in self.evolve_agent.multistep_evolve(evo_exp, self.evaluator):
             assert isinstance(evo_exp, Experiment)  # multiple inheritance
             evo_fb = self._get_last_fb()
-            fallback_decision = self.compare_and_pick_fb(
+            update_fallback = self.should_use_new_evo(
                 base_fb=fallback_evo_fb,
                 new_fb=evo_fb,
             )
-            if fallback_decision:
+            if update_fallback:
                 fallback_evo_exp = deepcopy(evo_exp)
                 fallback_evo_fb = deepcopy(evo_fb)
                 fallback_evo_exp.create_ws_ckp()  # NOTE: creating checkpoints for saving files in the workspace to prevent inplace mutation.
@@ -139,7 +141,9 @@ class CoSTEER(Developer[Experiment]):
                 logger.info("Fallback to the fallback solution.")
                 evo_exp = fallback_evo_exp
                 evo_exp.recover_ws_ckp()
-            evo_exp = self._exp_postprocess_by_feedback(evo_exp, self._get_last_fb())
+                evo_fb = fallback_evo_fb
+            assert evo_fb is not None  # multistep_evolve should run at least once
+            evo_exp = self._exp_postprocess_by_feedback(evo_exp, evo_fb)
         except CoderError as e:
             e.caused_by_timeout = reached_max_seconds
             raise e

rdagent/scenarios/data_science/dev/runner/__init__.py

@@ -171,7 +171,10 @@ class DSCoSTEERRunner(CoSTEER):
         """
         return int(self.scen.real_full_timeout() * self.settings.max_seconds_multiplier)
 
-    def compare_and_pick_fb(self, base_fb: CoSTEERMultiFeedback | None, new_fb: CoSTEERMultiFeedback | None) -> bool:
+    def should_use_new_evo(self, base_fb: CoSTEERMultiFeedback | None, new_fb: CoSTEERMultiFeedback) -> bool:
+        if not new_fb.is_acceptable():
+            return False
+
         # In data science, we only have a single feedback.
         # Note: new_fb should always exists as indicated by _get_last_fb() function.
         if base_fb is None:

rdagent/scenarios/data_science/dev/runner/prompts.yaml

@@ -170,4 +170,4 @@ DSCoSTEER:
     ### Validation Scores
     {{ former_failed_knowledge.feedback.score }}
     {% endfor %}
-    {% endif %}
+    {% endif %}

rdagent/utils/prompts.yaml

@@ -4,7 +4,8 @@ filter_redundant_text:
     1. Training log messages should be evaluated based on their usefulness—logs that contain meaningful training metrics such as loss or accuracy reported at each epoch should be retained, while redundant messages, such as those repeatedly reporting NaN values or iteration numbers without valuable information, should be removed. 
     2. For warning messages, **only one occurrence of each unique message should be kept**, eliminating any duplicates.
     3. Additionally, any visual progress indicators, such as ASCII-based progress bars or dynamic percentage updates, should be removed. Once these patterns are identified, you should generate appropriate regex expressions to filter them out.
-    4. Lastly, indicate whether substitution is needed. If the input exceeds a token limit, the system will provide only a shortened portion of the text.
+    4. Don't remove useful information that is not duplicated.
+    5. Lastly, indicate whether substitution is needed in `needs_sub` field. If the input exceeds a token limit, the system will provide only a shortened portion of the text.
 
     Respond in the following JSON format and order:
     {