diff --git a/rdagent/scenarios/data_science/proposal/exp_gen/merge.py b/rdagent/scenarios/data_science/proposal/exp_gen/merge.py index f1cda06d..29eb54d7 100644 --- a/rdagent/scenarios/data_science/proposal/exp_gen/merge.py +++ b/rdagent/scenarios/data_science/proposal/exp_gen/merge.py @@ -6,7 +6,7 @@ from typing import Dict, Tuple from rdagent.app.data_science.conf import DS_RD_SETTING from rdagent.components.coder.data_science.pipeline.exp import PipelineTask -from rdagent.core.proposal import ExpGen +from rdagent.core.proposal import ExperimentFeedback, ExpGen from rdagent.log import rdagent_logger as logger from rdagent.log.timer import RD_Agent_TIMER_wrapper, RDAgentTimer from rdagent.oai.llm_utils import APIBackend @@ -159,29 +159,27 @@ class ExpGen2Hypothesis(DSProposalV2ExpGen): sota_exp_desc = "" eda_output = None - trace_fbs = [] + trace_fbs: list[tuple[DSExperiment, ExperimentFeedback]] = [] # find the best exp to merge leaves: list[int] = trace.get_leaves() + max_sota_retrieved_num_per_trace = max(DS_RD_SETTING.max_sota_retrieved_num * 2 // len(leaves), 4) for leaf in leaves: if leaf == trace.current_selection[0]: continue - trace_fbs.append( + trace_fbs.extend( trace.experiment_and_feedback_list_after_init( return_type="sota", search_type="ancestors", selection=(leaf,), + max_retrieve_num=max_sota_retrieved_num_per_trace, ) ) - num_to_slice = 20 - if sum(len(fb_list) for fb_list in trace_fbs) > num_to_slice: - success_fb_trace_count = sum(1 for fb_list in trace_fbs if fb_list) - success_fb_list = [ - fb for fb_list in trace_fbs for fb in fb_list[-(num_to_slice // success_fb_trace_count) :] - ] - else: - success_fb_list = [fb for fb_list in trace_fbs for fb in fb_list] + success_fb_list = list(set(trace_fbs)) + logger.info( + f"Merge Hypothesis: select {len(success_fb_list)} from {len(trace_fbs)} SOTA experiments found in {len(leaves)} traces" + ) if len(success_fb_list) > 0: exp_to_merge_fb_desc = T("scenarios.data_science.proposal.exp_gen.merge:trace").r( diff --git a/rdagent/scenarios/data_science/proposal/exp_gen/select/prompts.yaml b/rdagent/scenarios/data_science/proposal/exp_gen/select/prompts.yaml index 240ba779..8aa6baef 100644 --- a/rdagent/scenarios/data_science/proposal/exp_gen/select/prompts.yaml +++ b/rdagent/scenarios/data_science/proposal/exp_gen/select/prompts.yaml @@ -1,6 +1,7 @@ auto_sota_selector: system: |- - You are an expert Kaggle competitor. You are given a list of SOTA experiments and feedbacks for a Kaggle competition. + You are an expert Kaggle competitor. You are given a list of SOTA experiments and feedbacks for a Kaggle competition in the following scenario: + {{ scenario }} You are tasked with reviewing the list of SOTA experiments and feedbacks, and selecting the most promising experiment to submit. @@ -60,14 +61,11 @@ auto_sota_selector: Your response should be short and concise, strictly adhere to the following JSON format: - { "selected_SOTA_idx": [Experiment No.](positive integer), "explanation": "A brief explanation text for your selection." } - - If you cannot make a selection, like no SOTA experiments and feedbacks, return { "selected_SOTA_idx": None, diff --git a/rdagent/scenarios/data_science/proposal/exp_gen/select/submit.py b/rdagent/scenarios/data_science/proposal/exp_gen/select/submit.py index f478d7c2..253fde6c 100644 --- a/rdagent/scenarios/data_science/proposal/exp_gen/select/submit.py +++ b/rdagent/scenarios/data_science/proposal/exp_gen/select/submit.py @@ -76,7 +76,7 @@ class AutoSOTAexpSelector(SOTAexpSelector): # multiple trace case, collect the latest SOTA experiments from each trace new_sota_exp_fb_list: list[tuple[DSExperiment, ExperimentFeedback]] = [] # calculate the number of SOTA experiments to retrieve from each trace, prevent it from becoming zero - max_sota_retrieved_num_per_trace = max(DS_RD_SETTING.max_sota_retrieved_num // len(leaves), 2) + max_sota_retrieved_num_per_trace = max(DS_RD_SETTING.max_sota_retrieved_num // len(leaves), 5) # recall, due to the integer division, the final number of SOTA experiments to retrieve may be different for leaf in leaves: sota_exp_fb_list_per_trace = trace.experiment_and_feedback_list_after_init( @@ -91,7 +91,7 @@ class AutoSOTAexpSelector(SOTAexpSelector): new_sota_exp_fb_list.extend(sota_exp_fb_list_per_trace) - sota_exp_fb_list = new_sota_exp_fb_list + sota_exp_fb_list = list(set(new_sota_exp_fb_list)) if len(sota_exp_fb_list) == 0: logger.info("Auto SOTA selector: No SOTA in trace yet") @@ -102,8 +102,14 @@ class AutoSOTAexpSelector(SOTAexpSelector): return sota_exp_fb_list[0][0] else: logger.info( - f"Auto SOTA selector: {len(sota_exp_fb_list)} SOTA experiments found in all traces, calling LLM to select the best one" + f"Auto SOTA selector: select {len(sota_exp_fb_list)} of {len(new_sota_exp_fb_list)} SOTA experiments found in all traces, calling LLM to select the best one" ) + if len(sota_exp_fb_list) > DS_RD_SETTING.max_sota_retrieved_num: + sota_exp_fb_list = sorted( + sota_exp_fb_list, + key=lambda exp_fb: pd.DataFrame(exp_fb[0].result).loc["ensemble"].iloc[0], + reverse=not trace.scen.metric_direction, + )[-DS_RD_SETTING.max_sota_retrieved_num :] for i, (exp, ef) in enumerate(sota_exp_fb_list): if exp: