diff --git a/rdagent/core/proposal.py b/rdagent/core/proposal.py index d78a1f95..d73ffea2 100644 --- a/rdagent/core/proposal.py +++ b/rdagent/core/proposal.py @@ -57,12 +57,10 @@ class ExperimentFeedback(Feedback): *, code_change_summary: str | None = None, decision: bool, - refine_decision: bool = False, eda_improvement: str | None = None, exception: Exception | None = None, ) -> None: self.decision = decision - self.refine_decision = refine_decision self.eda_improvement = eda_improvement self.reason = reason # Exception is not None means failing to generate runnable experiments due to exception. @@ -100,13 +98,11 @@ class HypothesisFeedback(ExperimentFeedback): *, code_change_summary: str | None = None, decision: bool, - refine_decision: bool = False, eda_improvement: str | None = None, ) -> None: super().__init__( reason, decision=decision, - refine_decision=refine_decision, code_change_summary=code_change_summary, eda_improvement=eda_improvement, ) diff --git a/rdagent/scenarios/data_science/dev/feedback.py b/rdagent/scenarios/data_science/dev/feedback.py index 6fef4ec9..094372f7 100644 --- a/rdagent/scenarios/data_science/dev/feedback.py +++ b/rdagent/scenarios/data_science/dev/feedback.py @@ -102,7 +102,6 @@ class DSExperiment2Feedback(Experiment2Feedback): if evaluation_not_aligned else convert2bool(dict_get_with_warning(resp_dict, "Replace Best Result", "no")) ), - refine_decision=convert2bool(dict_get_with_warning(resp_dict, "Refine Decision", "no")), eda_improvement=dict_get_with_warning(resp_dict, "EDA Improvement", "no"), # EDA improvement suggestion ) diff --git a/rdagent/scenarios/data_science/proposal/exp_gen/base.py b/rdagent/scenarios/data_science/proposal/exp_gen/base.py index 442528ca..464272cd 100644 --- a/rdagent/scenarios/data_science/proposal/exp_gen/base.py +++ b/rdagent/scenarios/data_science/proposal/exp_gen/base.py @@ -226,7 +226,7 @@ class DSTrace(Trace[DataScienceScen, KnowledgeBase]): # the sota exp should be accepted decision and all required components are completed. if ef.decision: return exp, ef - return None, None + return None def sota_experiment( self, diff --git a/rdagent/scenarios/data_science/proposal/exp_gen/proposal.py b/rdagent/scenarios/data_science/proposal/exp_gen/proposal.py index f638b76f..eea0d05a 100644 --- a/rdagent/scenarios/data_science/proposal/exp_gen/proposal.py +++ b/rdagent/scenarios/data_science/proposal/exp_gen/proposal.py @@ -21,7 +21,6 @@ from rdagent.scenarios.data_science.experiment.experiment import DSExperiment from rdagent.scenarios.data_science.proposal.exp_gen.base import DSHypothesis, DSTrace from rdagent.scenarios.data_science.proposal.exp_gen.draft import DSDraftExpGen from rdagent.scenarios.data_science.proposal.exp_gen.idea_pool import DSIdea -from rdagent.scenarios.data_science.proposal.exp_gen.refine import DSRefineExpGen from rdagent.utils.agent.tpl import T from rdagent.utils.repo.diff import generate_diff_from_dict from rdagent.utils.workflow import wait_retry @@ -721,7 +720,7 @@ class DSProposalV2ExpGen(ExpGen): hypotheses: list[DSHypothesis], pipeline: bool, failed_exp_feedback_list_desc: str, - sota_exp_fb: ExperimentFeedback | None = None, + fb_to_sota_exp: ExperimentFeedback | None = None, ) -> DSExperiment: if pipeline: component_info = get_component("Pipeline") @@ -740,7 +739,7 @@ class DSProposalV2ExpGen(ExpGen): sota_exp_desc=sota_exp_desc, hypotheses=hypotheses, failed_exp_and_feedback_list_desc=failed_exp_feedback_list_desc, - eda_improvement=sota_exp_fb.eda_improvement if sota_exp_fb else None, + eda_improvement=fb_to_sota_exp.eda_improvement if fb_to_sota_exp else None, ) response = APIBackend().build_messages_and_create_chat_completion( user_prompt=user_prompt, @@ -824,7 +823,11 @@ class DSProposalV2ExpGen(ExpGen): ] ) - sota_exp, sota_exp_fb = trace.sota_experiment_fb() + if sota_exp_fb := trace.sota_experiment_fb() is None: + sota_exp, fb_to_sota_exp = None, None + else: + sota_exp, fb_to_sota_exp = sota_exp_fb + if not isinstance(sota_exp, DSExperiment): eda_output = None else: @@ -922,5 +925,5 @@ class DSProposalV2ExpGen(ExpGen): ), pipeline=pipeline, failed_exp_feedback_list_desc=failed_exp_feedback_list_desc, - sota_exp_fb=sota_exp_fb, + fb_to_sota_exp=fb_to_sota_exp, ) diff --git a/rdagent/scenarios/data_science/proposal/exp_gen/refine.py b/rdagent/scenarios/data_science/proposal/exp_gen/refine.py deleted file mode 100644 index 4c33392a..00000000 --- a/rdagent/scenarios/data_science/proposal/exp_gen/refine.py +++ /dev/null @@ -1,105 +0,0 @@ -import json -from typing import Any, Dict, List, Optional, Tuple - -from rdagent.app.data_science.conf import DS_RD_SETTING -from rdagent.core.proposal import ExpGen -from rdagent.log import rdagent_logger as logger -from rdagent.oai.llm_utils import APIBackend -from rdagent.scenarios.data_science.experiment.experiment import DSExperiment -from rdagent.scenarios.data_science.proposal.exp_gen.base import DSHypothesis, DSTrace -from rdagent.scenarios.data_science.proposal.exp_gen.utils import ( - CodingSketch, - get_component, -) -from rdagent.utils.agent.tpl import T - - -class DSRefineExpGen(ExpGen): - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self.support_function_calling = APIBackend().support_function_calling() - - def task_gen( - self, - component_desc: str, - scenario_desc: str, - sota_exp_desc: str, - sota_exp: DSExperiment, - hypothesis: DSHypothesis, - exp_and_feedback_list_desc: str, - ) -> DSExperiment: - component_info = get_component("Pipeline") - data_folder_info = self.scen.processed_data_folder_description - sys_prompt = T(".prompts_refine:task_gen.system").r( - task_output_format=component_info["task_output_format"] if not self.support_function_calling else None, - component_desc=component_desc, - ) - user_prompt = T(".prompts_refine:task_gen.user").r( - scenario_desc=scenario_desc, - data_folder_info=data_folder_info, - sota_exp_desc=sota_exp_desc, - hypothesis=hypothesis, - exp_and_feedback_list_desc=exp_and_feedback_list_desc, - ) - response = APIBackend().build_messages_and_create_chat_completion( - user_prompt=user_prompt, - system_prompt=sys_prompt, - response_format=CodingSketch if self.support_function_calling else {"type": "json_object"}, - json_target_type=Dict[str, str | Dict[str, str]] if not self.support_function_calling else None, - ) - task_dict = json.loads(response) - task_design = ( - task_dict.get("task_design", {}) if not self.support_function_calling else task_dict.get("sketch", {}) - ) - logger.info(f"Task design:\n{task_design}") - task_name = hypothesis.component - description = ( - task_design - if isinstance(task_design, str) - else task_design.get("description", f"{component_info['target_name']} description not provided") - ) - task_class = component_info["task_class"] - task = task_class( - name=task_name, - description=description, - ) - exp = DSExperiment(pending_tasks_list=[[task]], hypothesis=hypothesis) - if sota_exp is not None: - exp.experiment_workspace.inject_code_from_file_dict(sota_exp.experiment_workspace) - return exp - - def gen(self, trace: DSTrace) -> DSExperiment: - # Step 0: Prepare - pipeline = DS_RD_SETTING.coder_on_whole_pipeline - component_desc = T("scenarios.data_science.share:component_description_in_pipeline").r() - sota_exp, sota_fb = trace.sota_experiment_fb() - if not isinstance(sota_exp, DSExperiment): - eda_output = None - else: - eda_output = sota_exp.experiment_workspace.file_dict.get("EDA.md", None) - scenario_desc = self.scen.get_scenario_all_desc(trace=trace, eda_output=eda_output) - sota_exp_desc = T("scenarios.data_science.share:describe.exp").r( - exp=sota_exp, heading="Best of previous exploration of the scenario" - ) - exp_feedback_list_desc = T("scenarios.data_science.share:describe.trace").r( - exp_and_feedback_list=trace.experiment_and_feedback_list_after_init(return_type="all"), - type="all", - pipeline=pipeline, - ) - - # Step 1: Generate a Pseudo Hypothesis for Refinement - hypothesis = DSHypothesis( - component="Model", - hypothesis="The current pipeline is note effective in terms of efficiency or hyperparameters. Refinement or Adjustment should be made.", - reason=sota_fb.reason, - ) - - # Step 2: Design Task to Refine SOTA - return self.task_gen( - component_desc=component_desc, - scenario_desc=scenario_desc, - sota_exp_desc=sota_exp_desc, - sota_exp=sota_exp, - hypothesis=hypothesis, - exp_and_feedback_list_desc=exp_feedback_list_desc, - )