From 0284b735969fcd1355df9be6eb342b8e57cb889b Mon Sep 17 00:00:00 2001 From: Xu Yang Date: Tue, 5 Aug 2025 18:16:10 +0800 Subject: [PATCH] feat: refactor CoSTEER classes to use DSCoSTEER and update max seconds handling (#1156) * feat: refactor CoSTEER classes to use DSCoSTEER and update max seconds handling * remove useless line * enable time_ratio_limit_to_enable_hyperparameter_tuning --- rdagent/app/kaggle/conf.py | 4 ++-- rdagent/components/coder/CoSTEER/__init__.py | 17 +++++++++++------ .../coder/data_science/ensemble/__init__.py | 5 ++--- .../coder/data_science/feature/__init__.py | 7 ++----- .../coder/data_science/model/__init__.py | 6 ++---- .../coder/data_science/pipeline/__init__.py | 5 ++--- .../data_science/raw_data_loader/__init__.py | 7 ++----- .../coder/data_science/share/ds_costeer.py | 9 +++++++++ .../coder/data_science/workflow/__init__.py | 8 ++------ rdagent/core/conf.py | 1 + rdagent/core/experiment.py | 5 ++++- .../data_science/dev/runner/__init__.py | 7 ++++++- .../scenarios/data_science/dev/runner/eval.py | 15 ++++++++------- 13 files changed, 53 insertions(+), 43 deletions(-) create mode 100644 rdagent/components/coder/data_science/share/ds_costeer.py diff --git a/rdagent/app/kaggle/conf.py b/rdagent/app/kaggle/conf.py index 77e81a02..6bb88ab7 100644 --- a/rdagent/app/kaggle/conf.py +++ b/rdagent/app/kaggle/conf.py @@ -75,8 +75,8 @@ class KaggleBasePropSetting(ExtendedBaseSettings): mini_case: bool = False """Enable mini-case study for experiments""" - time_ratio_limit_to_enable_hyperparameter_tuning: int | None = None - """Time ratio limit to enable hyperparameter tuning, if None, hyperparameter tuning is always enabled in the first evolution.""" + time_ratio_limit_to_enable_hyperparameter_tuning: float = 1 + """Time ratio limit to enable hyperparameter tuning, if not change, hyperparameter tuning is always enabled in the first evolution.""" KAGGLE_IMPLEMENT_SETTING = KaggleBasePropSetting() diff --git a/rdagent/components/coder/CoSTEER/__init__.py b/rdagent/components/coder/CoSTEER/__init__.py index fa123c61..692e09ff 100644 --- a/rdagent/components/coder/CoSTEER/__init__.py +++ b/rdagent/components/coder/CoSTEER/__init__.py @@ -1,4 +1,3 @@ -import pickle from copy import deepcopy from datetime import datetime from pathlib import Path @@ -26,7 +25,6 @@ class CoSTEER(Developer[Experiment]): es: EvolvingStrategy, *args, evolving_version: int = 2, - max_seconds: int | None = None, with_knowledge: bool = True, knowledge_self_gen: bool = True, max_loop: int | None = None, @@ -36,7 +34,6 @@ class CoSTEER(Developer[Experiment]): self.settings = settings self.max_loop = settings.max_loop if max_loop is None else max_loop - self.max_seconds = max_seconds self.knowledge_base_path = ( Path(settings.knowledge_base_path) if settings.knowledge_base_path is not None else None ) @@ -67,6 +64,13 @@ class CoSTEER(Developer[Experiment]): ) ) + def get_develop_max_seconds(self) -> int | None: + """ + Get the maximum seconds for the develop task. + Sub classes might override this method to provide a different value. + """ + return None + def _get_last_fb(self) -> CoSTEERMultiFeedback: fb = self.evolve_agent.evolving_trace[-1].feedback assert fb is not None, "feedback is None" @@ -76,6 +80,7 @@ class CoSTEER(Developer[Experiment]): def develop(self, exp: Experiment) -> Experiment: # init intermediate items + max_seconds = self.get_develop_max_seconds() evo_exp = EvolvingItem.from_experiment(exp) self.evolve_agent = RAGEvoAgent[EvolvingItem]( @@ -102,8 +107,8 @@ class CoSTEER(Developer[Experiment]): logger.log_object(evo_exp.sub_workspace_list, tag="evolving code") for sw in evo_exp.sub_workspace_list: logger.info(f"evolving workspace: {sw}") - if self.max_seconds is not None and (datetime.now() - start_datetime).seconds > self.max_seconds: - logger.info(f"Reached max time limit {self.max_seconds} seconds, stop evolving") + if max_seconds is not None and (datetime.now() - start_datetime).seconds > max_seconds: + logger.info(f"Reached max time limit {max_seconds} seconds, stop evolving") reached_max_seconds = True break if RD_Agent_TIMER_wrapper.timer.started and RD_Agent_TIMER_wrapper.timer.is_timeout(): @@ -140,7 +145,7 @@ class CoSTEER(Developer[Experiment]): failed_feedbacks = [ f"- feedback{index + 1:02d}:\n - execution: {f.execution}\n - return_checking: {f.return_checking}\n - code: {f.code}" for index, f in enumerate(feedback) - if f is not None and not f.final_decision + if f is not None and not f.is_acceptable() ] if len(failed_feedbacks) == len(feedback): diff --git a/rdagent/components/coder/data_science/ensemble/__init__.py b/rdagent/components/coder/data_science/ensemble/__init__.py index 9fad71dc..b4985b9d 100644 --- a/rdagent/components/coder/data_science/ensemble/__init__.py +++ b/rdagent/components/coder/data_science/ensemble/__init__.py @@ -16,7 +16,6 @@ from pathlib import Path from jinja2 import Environment, StrictUndefined from rdagent.app.data_science.conf import DS_RD_SETTING -from rdagent.components.coder.CoSTEER import CoSTEER from rdagent.components.coder.CoSTEER.evaluators import ( CoSTEERMultiEvaluator, CoSTEERSingleFeedback, @@ -30,6 +29,7 @@ from rdagent.components.coder.CoSTEER.knowledge_management import ( from rdagent.components.coder.data_science.conf import DSCoderCoSTEERSettings from rdagent.components.coder.data_science.ensemble.eval import EnsembleCoSTEEREvaluator from rdagent.components.coder.data_science.ensemble.exp import EnsembleTask +from rdagent.components.coder.data_science.share.ds_costeer import DSCoSTEER from rdagent.core.exception import CoderError from rdagent.core.experiment import FBWorkspace from rdagent.core.scenario import Scenario @@ -141,7 +141,7 @@ class EnsembleMultiProcessEvolvingStrategy(MultiProcessEvolvingStrategy): return evo -class EnsembleCoSTEER(CoSTEER): +class EnsembleCoSTEER(DSCoSTEER): def __init__( self, scen: Scenario, @@ -160,6 +160,5 @@ class EnsembleCoSTEER(CoSTEER): evolving_version=2, scen=scen, max_loop=DS_RD_SETTING.coder_max_loop, - max_seconds=scen.real_debug_timeout() * settings.max_seconds_multiplier, **kwargs, ) diff --git a/rdagent/components/coder/data_science/feature/__init__.py b/rdagent/components/coder/data_science/feature/__init__.py index f469eba1..dbb0cd62 100644 --- a/rdagent/components/coder/data_science/feature/__init__.py +++ b/rdagent/components/coder/data_science/feature/__init__.py @@ -1,9 +1,6 @@ -import json from pathlib import Path -from typing import Dict from rdagent.app.data_science.conf import DS_RD_SETTING -from rdagent.components.coder.CoSTEER import CoSTEER from rdagent.components.coder.CoSTEER.evaluators import ( CoSTEERMultiEvaluator, CoSTEERSingleFeedback, @@ -17,6 +14,7 @@ from rdagent.components.coder.CoSTEER.knowledge_management import ( from rdagent.components.coder.data_science.conf import DSCoderCoSTEERSettings from rdagent.components.coder.data_science.feature.eval import FeatureCoSTEEREvaluator from rdagent.components.coder.data_science.feature.exp import FeatureTask +from rdagent.components.coder.data_science.share.ds_costeer import DSCoSTEER from rdagent.core.exception import CoderError from rdagent.core.experiment import FBWorkspace from rdagent.core.scenario import Scenario @@ -117,7 +115,7 @@ class FeatureMultiProcessEvolvingStrategy(MultiProcessEvolvingStrategy): return evo -class FeatureCoSTEER(CoSTEER): +class FeatureCoSTEER(DSCoSTEER): def __init__( self, scen: Scenario, @@ -138,6 +136,5 @@ class FeatureCoSTEER(CoSTEER): evolving_version=2, scen=scen, max_loop=DS_RD_SETTING.coder_max_loop, - max_seconds=scen.real_debug_timeout() * settings.max_seconds_multiplier, **kwargs, ) diff --git a/rdagent/components/coder/data_science/model/__init__.py b/rdagent/components/coder/data_science/model/__init__.py index e37492be..2ee2e00f 100644 --- a/rdagent/components/coder/data_science/model/__init__.py +++ b/rdagent/components/coder/data_science/model/__init__.py @@ -1,8 +1,6 @@ from pathlib import Path -from typing import Dict from rdagent.app.data_science.conf import DS_RD_SETTING -from rdagent.components.coder.CoSTEER import CoSTEER from rdagent.components.coder.CoSTEER.evaluators import ( CoSTEERMultiEvaluator, CoSTEERSingleFeedback, @@ -18,6 +16,7 @@ from rdagent.components.coder.data_science.model.eval import ( ModelGeneralCaseSpecEvaluator, ) from rdagent.components.coder.data_science.model.exp import ModelTask +from rdagent.components.coder.data_science.share.ds_costeer import DSCoSTEER from rdagent.core.exception import CoderError from rdagent.core.experiment import FBWorkspace from rdagent.core.scenario import Scenario @@ -148,7 +147,7 @@ class ModelMultiProcessEvolvingStrategy(MultiProcessEvolvingStrategy): return evo -class ModelCoSTEER(CoSTEER): +class ModelCoSTEER(DSCoSTEER): def __init__( self, scen: Scenario, @@ -170,6 +169,5 @@ class ModelCoSTEER(CoSTEER): evolving_version=2, scen=scen, max_loop=DS_RD_SETTING.coder_max_loop, - max_seconds=scen.real_debug_timeout() * settings.max_seconds_multiplier, **kwargs, ) diff --git a/rdagent/components/coder/data_science/pipeline/__init__.py b/rdagent/components/coder/data_science/pipeline/__init__.py index 8a5f27d2..3dd89a7b 100644 --- a/rdagent/components/coder/data_science/pipeline/__init__.py +++ b/rdagent/components/coder/data_science/pipeline/__init__.py @@ -25,7 +25,6 @@ File structure from pathlib import Path from rdagent.app.data_science.conf import DS_RD_SETTING -from rdagent.components.coder.CoSTEER import CoSTEER from rdagent.components.coder.CoSTEER.evaluators import ( CoSTEERMultiEvaluator, CoSTEERSingleFeedback, @@ -39,6 +38,7 @@ from rdagent.components.coder.CoSTEER.knowledge_management import ( from rdagent.components.coder.data_science.conf import DSCoderCoSTEERSettings from rdagent.components.coder.data_science.pipeline.eval import PipelineCoSTEEREvaluator from rdagent.components.coder.data_science.pipeline.exp import PipelineTask +from rdagent.components.coder.data_science.share.ds_costeer import DSCoSTEER from rdagent.components.coder.data_science.share.eval import ModelDumpEvaluator from rdagent.core.exception import CoderError from rdagent.core.experiment import FBWorkspace @@ -130,7 +130,7 @@ class PipelineMultiProcessEvolvingStrategy(MultiProcessEvolvingStrategy): return evo -class PipelineCoSTEER(CoSTEER): +class PipelineCoSTEER(DSCoSTEER): def __init__( self, scen: Scenario, @@ -159,6 +159,5 @@ class PipelineCoSTEER(CoSTEER): evolving_version=2, scen=scen, max_loop=DS_RD_SETTING.coder_max_loop, - max_seconds=scen.real_debug_timeout() * settings.max_seconds_multiplier, **kwargs, ) diff --git a/rdagent/components/coder/data_science/raw_data_loader/__init__.py b/rdagent/components/coder/data_science/raw_data_loader/__init__.py index 18ff403a..f3f0ff13 100644 --- a/rdagent/components/coder/data_science/raw_data_loader/__init__.py +++ b/rdagent/components/coder/data_science/raw_data_loader/__init__.py @@ -22,13 +22,10 @@ File structure - Each coder could be tested. """ -import json import re from pathlib import Path -from typing import Dict from rdagent.app.data_science.conf import DS_RD_SETTING -from rdagent.components.coder.CoSTEER import CoSTEER from rdagent.components.coder.CoSTEER.evaluators import ( CoSTEERMultiEvaluator, CoSTEERSingleFeedback, @@ -47,6 +44,7 @@ from rdagent.components.coder.data_science.raw_data_loader.eval import ( DataLoaderCoSTEEREvaluator, ) from rdagent.components.coder.data_science.raw_data_loader.exp import DataLoaderTask +from rdagent.components.coder.data_science.share.ds_costeer import DSCoSTEER from rdagent.core.exception import CoderError from rdagent.core.experiment import FBWorkspace from rdagent.core.scenario import Scenario @@ -197,7 +195,7 @@ class DataLoaderMultiProcessEvolvingStrategy(MultiProcessEvolvingStrategy): return evo -class DataLoaderCoSTEER(CoSTEER): +class DataLoaderCoSTEER(DSCoSTEER): def __init__( self, scen: Scenario, @@ -218,7 +216,6 @@ class DataLoaderCoSTEER(CoSTEER): evolving_version=2, scen=scen, max_loop=DS_RD_SETTING.coder_max_loop, - max_seconds=scen.real_debug_timeout() * settings.max_seconds_multiplier, **kwargs, ) diff --git a/rdagent/components/coder/data_science/share/ds_costeer.py b/rdagent/components/coder/data_science/share/ds_costeer.py new file mode 100644 index 00000000..515b0309 --- /dev/null +++ b/rdagent/components/coder/data_science/share/ds_costeer.py @@ -0,0 +1,9 @@ +from rdagent.components.coder.CoSTEER import CoSTEER + + +class DSCoSTEER(CoSTEER): + def get_develop_max_seconds(self) -> int | None: + """ + The coder uses the scenario's real debug timeout as the maximum seconds for development. + """ + return int(self.scen.real_debug_timeout() * self.settings.max_seconds_multiplier) diff --git a/rdagent/components/coder/data_science/workflow/__init__.py b/rdagent/components/coder/data_science/workflow/__init__.py index f11f7786..a6ba4bde 100644 --- a/rdagent/components/coder/data_science/workflow/__init__.py +++ b/rdagent/components/coder/data_science/workflow/__init__.py @@ -1,8 +1,4 @@ -import json -from typing import Dict - from rdagent.app.data_science.conf import DS_RD_SETTING -from rdagent.components.coder.CoSTEER import CoSTEER from rdagent.components.coder.CoSTEER.evaluators import ( CoSTEERMultiEvaluator, CoSTEERSingleFeedback, @@ -14,6 +10,7 @@ from rdagent.components.coder.CoSTEER.knowledge_management import ( CoSTEERQueriedKnowledge, ) from rdagent.components.coder.data_science.conf import DSCoderCoSTEERSettings +from rdagent.components.coder.data_science.share.ds_costeer import DSCoSTEER from rdagent.components.coder.data_science.workflow.eval import ( WorkflowGeneralCaseSpecEvaluator, ) @@ -111,7 +108,7 @@ class WorkflowMultiProcessEvolvingStrategy(MultiProcessEvolvingStrategy): return evo -class WorkflowCoSTEER(CoSTEER): +class WorkflowCoSTEER(DSCoSTEER): def __init__( self, scen: Scenario, @@ -131,6 +128,5 @@ class WorkflowCoSTEER(CoSTEER): evolving_version=2, scen=scen, max_loop=DS_RD_SETTING.coder_max_loop, - max_seconds=scen.real_debug_timeout() * settings.max_seconds_multiplier, **kwargs, ) diff --git a/rdagent/core/conf.py b/rdagent/core/conf.py index b1efe7b6..e0be428a 100644 --- a/rdagent/core/conf.py +++ b/rdagent/core/conf.py @@ -56,6 +56,7 @@ class RDAgentSettings(ExtendedBaseSettings): # workspace conf workspace_path: Path = Path.cwd() / "git_ignore_folder" / "RD-Agent_workspace" workspace_ckp_size_limit: int = 0 + workspace_ckp_white_list_names: list[str] | None = None """ the checkpoint for the workspace is a zip file. 0 (or any value <=0) means *no* size limit for files in workspace checkpoints diff --git a/rdagent/core/experiment.py b/rdagent/core/experiment.py index 326d5f62..12b2abf9 100644 --- a/rdagent/core/experiment.py +++ b/rdagent/core/experiment.py @@ -318,7 +318,10 @@ class FBWorkspace(Workspace): zf.writestr(zi, str(file_path.readlink())) elif file_path.is_file(): size_limit = RD_AGENT_SETTINGS.workspace_ckp_size_limit - if size_limit <= 0 or file_path.stat().st_size <= size_limit: + if ( + RD_AGENT_SETTINGS.workspace_ckp_white_list_names is not None + and file_path.name in RD_AGENT_SETTINGS.workspace_ckp_white_list_names + ) or (size_limit <= 0 or file_path.stat().st_size <= size_limit): zf.write(file_path, file_path.relative_to(self.workspace_path)) self.ws_ckp = buf.getvalue() diff --git a/rdagent/scenarios/data_science/dev/runner/__init__.py b/rdagent/scenarios/data_science/dev/runner/__init__.py index 1966bbb2..3b3a6961 100644 --- a/rdagent/scenarios/data_science/dev/runner/__init__.py +++ b/rdagent/scenarios/data_science/dev/runner/__init__.py @@ -162,10 +162,15 @@ class DSCoSTEERRunner(CoSTEER): evolving_version=2, scen=scen, max_loop=DS_RD_SETTING.runner_max_loop, - max_seconds=scen.real_full_timeout() * settings.max_seconds_multiplier, **kwargs, ) + def get_develop_max_seconds(self) -> int | None: + """ + The coder uses the scenario's real debug timeout as the maximum seconds for development. + """ + return int(self.scen.real_full_timeout() * self.settings.max_seconds_multiplier) + def develop(self, exp): bak_sub_tasks = exp.pending_tasks_list exp.sub_tasks = [ diff --git a/rdagent/scenarios/data_science/dev/runner/eval.py b/rdagent/scenarios/data_science/dev/runner/eval.py index 9121f073..bc9e84e0 100644 --- a/rdagent/scenarios/data_science/dev/runner/eval.py +++ b/rdagent/scenarios/data_science/dev/runner/eval.py @@ -76,13 +76,6 @@ class DSRunnerEvaluator(CoSTEEREvaluator): queried_knowledge: QueriedKnowledge = None, **kwargs, ) -> DSRunnerFeedback: - # Only enalbe hyperparameter tuning on the first evaluation. - # Avoid too much time cunsumming. - if len(queried_knowledge.task_to_former_failed_traces[target_task.get_task_information()][0]) == 0: - enable_hyperparameter_tuning_check = True - else: - enable_hyperparameter_tuning_check = False - env = get_ds_env( extra_volumes={ f"{DS_RD_SETTING.local_data_path}/{self.scen.competition}": T( @@ -168,6 +161,14 @@ class DSRunnerEvaluator(CoSTEEREvaluator): stdout += f"\n### Submission check:\n{submission_check_out}\nIf Submission check returns a 'Submission is valid' or similar message, despite some warning messages, you should still consider the submission as valid and give a positive final decision. " time_spent_ratio = implementation.running_info.running_time / env.conf.running_timeout_period + # Only enable hyperparameter tuning on the first evaluation. + # Avoid too much time consuming. + enable_hyperparameter_tuning_check = False + if len(queried_knowledge.task_to_former_failed_traces[target_task.get_task_information()][0]) == 0 and ( + time_spent_ratio < DS_RD_SETTING.time_ratio_limit_to_enable_hyperparameter_tuning + ): + enable_hyperparameter_tuning_check = True + if ( DS_RD_SETTING.time_ratio_limit_to_enable_hyperparameter_tuning is not None and time_spent_ratio > DS_RD_SETTING.time_ratio_limit_to_enable_hyperparameter_tuning