chore: avoid more traces during merge (#1108)

* avoid more traces during merge

* restore change

* remove duplicate experiments

* add dropped pool

* use scenario

* remove drop_pool

* use max_sota_retrieved_num

rdagent/scenarios/data_science/proposal/exp_gen/merge.py

@@ -6,7 +6,7 @@ from typing import Dict, Tuple
 
 from rdagent.app.data_science.conf import DS_RD_SETTING
 from rdagent.components.coder.data_science.pipeline.exp import PipelineTask
-from rdagent.core.proposal import ExpGen
+from rdagent.core.proposal import ExperimentFeedback, ExpGen
 from rdagent.log import rdagent_logger as logger
 from rdagent.log.timer import RD_Agent_TIMER_wrapper, RDAgentTimer
 from rdagent.oai.llm_utils import APIBackend
@@ -159,29 +159,27 @@ class ExpGen2Hypothesis(DSProposalV2ExpGen):
             sota_exp_desc = ""
             eda_output = None
 
-        trace_fbs = []
+        trace_fbs: list[tuple[DSExperiment, ExperimentFeedback]] = []
         # find the best exp to merge
         leaves: list[int] = trace.get_leaves()
+        max_sota_retrieved_num_per_trace = max(DS_RD_SETTING.max_sota_retrieved_num * 2 // len(leaves), 4)
         for leaf in leaves:
             if leaf == trace.current_selection[0]:
                 continue
 
-            trace_fbs.append(
+            trace_fbs.extend(
                 trace.experiment_and_feedback_list_after_init(
                     return_type="sota",
                     search_type="ancestors",
                     selection=(leaf,),
+                    max_retrieve_num=max_sota_retrieved_num_per_trace,
                 )
             )
 
-        num_to_slice = 20
-        if sum(len(fb_list) for fb_list in trace_fbs) > num_to_slice:
-            success_fb_trace_count = sum(1 for fb_list in trace_fbs if fb_list)
-            success_fb_list = [
-                fb for fb_list in trace_fbs for fb in fb_list[-(num_to_slice // success_fb_trace_count) :]
-            ]
-        else:
-            success_fb_list = [fb for fb_list in trace_fbs for fb in fb_list]
+        success_fb_list = list(set(trace_fbs))
+        logger.info(
+            f"Merge Hypothesis: select {len(success_fb_list)} from {len(trace_fbs)} SOTA experiments found in {len(leaves)} traces"
+        )
 
         if len(success_fb_list) > 0:
             exp_to_merge_fb_desc = T("scenarios.data_science.proposal.exp_gen.merge:trace").r(

rdagent/scenarios/data_science/proposal/exp_gen/select/prompts.yaml

@@ -1,6 +1,7 @@
 auto_sota_selector:
   system: |-
-    You are an expert Kaggle competitor. You are given a list of SOTA experiments and feedbacks for a Kaggle competition.
+    You are an expert Kaggle competitor. You are given a list of SOTA experiments and feedbacks for a Kaggle competition in the following scenario:
+    {{ scenario }}
 
     You are tasked with reviewing the list of SOTA experiments and feedbacks, and selecting the most promising experiment to submit.
 
@@ -60,14 +61,11 @@ auto_sota_selector:
 
     Your response should be short and concise, strictly adhere to the following JSON format:
 
-
     {
       "selected_SOTA_idx": [Experiment No.](positive integer),
       "explanation": "A brief explanation text for your selection."
     }
 
-
-
     If you cannot make a selection, like no SOTA experiments and feedbacks, return 
       {
         "selected_SOTA_idx": None,

rdagent/scenarios/data_science/proposal/exp_gen/select/submit.py

@@ -76,7 +76,7 @@ class AutoSOTAexpSelector(SOTAexpSelector):
                 # multiple trace case, collect the latest SOTA experiments from each trace
                 new_sota_exp_fb_list: list[tuple[DSExperiment, ExperimentFeedback]] = []
                 # calculate the number of SOTA experiments to retrieve from each trace, prevent it from becoming zero
-                max_sota_retrieved_num_per_trace = max(DS_RD_SETTING.max_sota_retrieved_num // len(leaves), 2)
+                max_sota_retrieved_num_per_trace = max(DS_RD_SETTING.max_sota_retrieved_num // len(leaves), 5)
                 # recall, due to the integer division, the final number of SOTA experiments to retrieve may be different
                 for leaf in leaves:
                     sota_exp_fb_list_per_trace = trace.experiment_and_feedback_list_after_init(
@@ -91,7 +91,7 @@ class AutoSOTAexpSelector(SOTAexpSelector):
 
                     new_sota_exp_fb_list.extend(sota_exp_fb_list_per_trace)
 
-                sota_exp_fb_list = new_sota_exp_fb_list
+                sota_exp_fb_list = list(set(new_sota_exp_fb_list))
 
                 if len(sota_exp_fb_list) == 0:
                     logger.info("Auto SOTA selector: No SOTA in trace yet")
@@ -102,8 +102,14 @@ class AutoSOTAexpSelector(SOTAexpSelector):
                     return sota_exp_fb_list[0][0]
                 else:
                     logger.info(
-                        f"Auto SOTA selector: {len(sota_exp_fb_list)} SOTA experiments found in all traces, calling LLM to select the best one"
+                        f"Auto SOTA selector: select {len(sota_exp_fb_list)} of {len(new_sota_exp_fb_list)} SOTA experiments found in all traces, calling LLM to select the best one"
                     )
+                    if len(sota_exp_fb_list) > DS_RD_SETTING.max_sota_retrieved_num:
+                        sota_exp_fb_list = sorted(
+                            sota_exp_fb_list,
+                            key=lambda exp_fb: pd.DataFrame(exp_fb[0].result).loc["ensemble"].iloc[0],
+                            reverse=not trace.scen.metric_direction,
+                        )[-DS_RD_SETTING.max_sota_retrieved_num :]
 
             for i, (exp, ef) in enumerate(sota_exp_fb_list):
                 if exp: