feat: analyze feedback based on sota numbers (#1116)

* give sota numbers to feedback

* refine prompt

rdagent/scenarios/data_science/dev/feedback.py

@@ -55,6 +55,34 @@ class DSExperiment2Feedback(Experiment2Feedback):
         else:
             diff_edition = []
 
+        sota_exp_and_feedback = trace.experiment_and_feedback_list_after_init(return_type="sota", with_index=True)
+        sota_valid_score_df = pd.DataFrame(
+            index=[index for index, exp, fb in sota_exp_and_feedback], columns=["score", "improvement"]
+        )
+        for row_index, (index, sota_exp, sota_fb) in enumerate(sota_exp_and_feedback):
+            sota_valid_score_df.loc[index, "score"] = (
+                sota_exp.result.loc["ensemble"].iloc[0] if sota_exp.result is not None else "N/A"
+            )
+            if row_index > 0:
+                improve_percent = (
+                    (
+                        ((sota_valid_score_df.loc[index, "score"] - sota_valid_score_df.iloc[row_index - 1]["score"]))
+                        / sota_valid_score_df.iloc[row_index - 1]["score"]
+                        * 100
+                    )
+                    if self.scen.evaluation_metric_direction
+                    else (
+                        ((sota_valid_score_df.iloc[row_index - 1]["score"] - sota_valid_score_df.loc[index, "score"]))
+                        / sota_valid_score_df.iloc[row_index - 1]["score"]
+                        * 100
+                    )
+                )
+                sota_valid_score_df.loc[index, "improvement"] = f"{improve_percent:.2f}%"
+        sota_valid_score_df["improvement"] = sota_valid_score_df["score"].diff()
+        all_sota_results_desc = T("scenarios.data_science.share:describe.sota_results").r(
+            sota_valid_score_dict=sota_valid_score_df.to_dict(orient="index"),
+        )
+
         # assumption:
         # The feedback should focus on experiment **improving**.
         # Assume that all the the sota exp is based on the previous sota experiment
@@ -78,6 +106,7 @@ class DSExperiment2Feedback(Experiment2Feedback):
             diff_edition=diff_edition,
             feedback_desc=feedback_desc,
             cur_vs_sota_score=cur_vs_sota_score,
+            all_sota_results_desc=all_sota_results_desc,
         )
 
         resp_dict = json.loads(

rdagent/scenarios/data_science/dev/prompts.yaml

@@ -40,15 +40,23 @@ exp_feedback:
     - Directly compare the current `ensemble` validation score to the SOTA `ensemble` validation score. Do not focus on individual models unless anomalies are significant.
     - Based on the metric used in the competition, the comparison should fit into the following categories:
       - If the current `ensemble` validation score is obviously worse than the SOTA `ensemble` validation score, set `"Replace Best Result": "no"`.
-      - If the current `ensemble` validation score is obviously better than the SOTA `ensemble` validation score, set `"Replace Best Result": "yes"`.
-      - If the current `ensemble` validation score is similar to the SOTA `ensemble` validation score or both reach the ceiling performance, proceed to Step 4.
+      - If the current `ensemble` validation score is obviously better than the SOTA `ensemble` validation score, proceed to Step 4.
+      - If the current `ensemble` validation score is similar to the SOTA `ensemble` validation score or both reach the ceiling performance, proceed to Step 5.
     - Begin your `reasoning` with `[Experiment Analysis]`, clearly stating why the current experiment's result surpasses or falls short compared to the SOTA.
     - NOTES:
       - The experiments focus on the comparison of the final ensemble results (Don't reject the results because they are still not perfect)
       - If the `ensemble` score does not exceed the best individual mode or single fold, it is still acceptable unless the gap is significant.
     
-    Step 4: Analyze Code With Similar validation Results
-    - If the current `ensemble` validation score is similar to the SOTA `ensemble` validation score, give the decision based on the comparison between the current experiment and SOTA.
+    Step 4: Identify validation score improvement source
+    - The user also provides you all the ensemble validation scores of the SOTA experiments you can get the improve pattern of the validation score numbers.
+    - Since the current validation score is obviously better than the SOTA `ensemble` validation score, you should analyze whether the current number based on all the history numbers following the patterns:
+      - If the sota validation score has been stabilized or joggling for several rounds, you should be very careful to accept the current score as a valid improvement because it is likely that this validation score improvement is the cause of over-fit. 
+        - If the code manipulate the validation dataset or modified the validation process. please set `"Replace Best Result": "no"` and explain in `"Reasoning"` starting with `[Validation Score Manipulation]`.
+        - If the improvement is based a big innovated code change, please proceed to Step 5 to check the code quality and give decision based on both the validation score and code quality.
+      - If no much sota validation score is provided, you should proceed to Step 5 to check the code quality and give decision based on both the validation score and code quality.
+
+    Step 5: Analyze Code To get Better validation results
+    - If the current `ensemble` validation score is similar to the SOTA `ensemble` validation score or suspicious validation improvement after several trials, give the decision based on the comparison between the current experiment and SOTA.
     - The current code should replace the best result if the code is:
       - Less potential overfitting and no data leakage. The code should not modify the validation and test set distributions.
       - Using best practices and modeling techniques. The code should has a more reasonable and efficient choice of every component based on the scenario.

rdagent/scenarios/data_science/loop.py

@@ -324,6 +324,11 @@ class DataScienceRDLoop(RDLoop):
             )  # backup when upper code line is killed when running
             self.timer.add_duration(datetime.now() - start_archive_datetime)
 
+    def _check_exit_conditions_on_step(self, loop_id: Optional[int] = None, step_id: Optional[int] = None):
+        if step_id not in [self.steps.index("running"), self.steps.index("feedback")]:
+            # pass the check for running and feedbacks since they are very likely to be finished soon.
+            super()._check_exit_conditions_on_step(loop_id=loop_id, step_id=step_id)
+
     @classmethod
     def load(
         cls,

rdagent/scenarios/data_science/proposal/exp_gen/base.py

@@ -171,6 +171,7 @@ class DSTrace(Trace[DataScienceScen, KnowledgeBase]):
         search_type: Literal["all", "ancestors"] = "ancestors",
         selection: tuple[int, ...] | None = None,
         max_retrieve_num: int | None = None,
+        with_index: bool = False,
     ) -> list[tuple[DSExperiment, ExperimentFeedback]]:
         """
         Retrieve a list of experiments and feedbacks based on the return_type.
@@ -180,13 +181,13 @@ class DSTrace(Trace[DataScienceScen, KnowledgeBase]):
         has_final_component = True if DS_RD_SETTING.coder_on_whole_pipeline else False
         SOTA_exp_and_feedback_list = []
         failed_exp_and_feedback_list_after_sota = []
-        for exp, fb in search_list:
+        for index, (exp, fb) in enumerate(search_list):
             if has_final_component:
                 if fb.decision:
-                    SOTA_exp_and_feedback_list.append((exp, fb))
+                    SOTA_exp_and_feedback_list.append((index, exp, fb) if with_index else (exp, fb))
                     failed_exp_and_feedback_list_after_sota = []
                 else:
-                    failed_exp_and_feedback_list_after_sota.append((exp, fb))
+                    failed_exp_and_feedback_list_after_sota.append((index, exp, fb) if with_index else (exp, fb))
             if exp.hypothesis.component == final_component and fb:
                 has_final_component = True
         if max_retrieve_num is not None and (SOTA_exp_and_feedback_list or failed_exp_and_feedback_list_after_sota):

rdagent/scenarios/data_science/share.yaml

@@ -65,6 +65,12 @@ describe: # some template to describe some object
     {% endif %}
     {% endfor %}
     {% endif %}
+  
+  sota_results: |-
+    All validation scores of the SOTA experiments:
+    {% for index, data in sota_valid_score_dict.items() %}
+    ## Trial index: {{ index }}, validation score: {{ data.score }}, validation score improvement: {{ data.improvement }}
+    {% endfor %}
 
 scen:  # customizable
   role: |-

rdagent/utils/workflow/loop.py

@@ -164,7 +164,7 @@ class LoopBase:
             self._pbar.close()
             del self._pbar
 
-    def _check_exit_conditions_on_step(self) -> None:
+    def _check_exit_conditions_on_step(self, loop_id: Optional[int] = None, step_id: Optional[int] = None) -> None:
         """Check if the loop should continue or terminate.
 
         Raises
@@ -270,7 +270,7 @@ class LoopBase:
                             step_index=next_step,
                             step_name=self.steps[next_step],
                         )
-                        self._check_exit_conditions_on_step()
+                        self._check_exit_conditions_on_step(loop_id=li, step_id=si)
                     else:
                         logger.warning(f"Step forward {si} of loop {li} is skipped.")