diff --git a/rdagent/app/kaggle/conf.py b/rdagent/app/kaggle/conf.py index 698cde23..e19a49b0 100644 --- a/rdagent/app/kaggle/conf.py +++ b/rdagent/app/kaggle/conf.py @@ -76,10 +76,18 @@ class KaggleBasePropSetting(ExtendedBaseSettings): """Enable mini-case study for experiments""" time_ratio_limit_to_enable_hyperparameter_tuning: float = 1 - """Runner time ratio limit to enable hyperparameter tuning, if not change, hyperparameter tuning is always enabled in the first evolution.""" + """ + Runner time ratio limit to enable hyperparameter tuning, if not change, hyperparameter tuning is always enabled in the first evolution. + """ res_time_ratio_limit_to_enable_hyperparameter_tuning: float = 1 - """Overall rest time ratio limit to enable hyperparameter tuning, if not change, hyperparameter tuning is always enabled in the first evolution.""" + """ + Overall rest time ratio limit to enable hyperparameter tuning, if not change, hyperparameter tuning is always enabled in the first evolution. + `1` indicate we enable hyperparameter tuning when we have 100% residual time. (so hyperparameter tuning is always enabled) + """ + + only_first_loop_enable_hyperparameter_tuning: bool = True + """Enable hyperparameter tuning feedback only in the first loop of evaluation.""" only_enable_tuning_in_merge: bool = False """Enable hyperparameter tuning only in the merge stage""" diff --git a/rdagent/components/coder/CoSTEER/__init__.py b/rdagent/components/coder/CoSTEER/__init__.py index b7da40a5..563715ce 100644 --- a/rdagent/components/coder/CoSTEER/__init__.py +++ b/rdagent/components/coder/CoSTEER/__init__.py @@ -125,7 +125,7 @@ class CoSTEER(Developer[Experiment]): logger.log_object(evo_exp.sub_workspace_list, tag="evolving code") for sw in evo_exp.sub_workspace_list: logger.info(f"evolving workspace: {sw}") - if max_seconds is not None and (datetime.now() - start_datetime).seconds > max_seconds: + if max_seconds is not None and (datetime.now() - start_datetime).total_seconds() > max_seconds: logger.info(f"Reached max time limit {max_seconds} seconds, stop evolving") reached_max_seconds = True break diff --git a/rdagent/scenarios/data_science/dev/runner/eval.py b/rdagent/scenarios/data_science/dev/runner/eval.py index 39e4c39f..b98c4d7b 100644 --- a/rdagent/scenarios/data_science/dev/runner/eval.py +++ b/rdagent/scenarios/data_science/dev/runner/eval.py @@ -164,7 +164,10 @@ class DSRunnerEvaluator(CoSTEEREvaluator): # Whether to enable hyperparameter tuning check # 1. This is the first loop of evaluation. - c1 = len(queried_knowledge.task_to_former_failed_traces[target_task.get_task_information()][0]) == 0 + if DS_RD_SETTING.only_first_loop_enable_hyperparameter_tuning: + c1 = len(queried_knowledge.task_to_former_failed_traces[target_task.get_task_information()][0]) == 0 + else: + c1 = True # 2. The current time spent on runner is less than the time limit ratio for runner timeout. time_spent_ratio = implementation.running_info.running_time / env.conf.running_timeout_period diff --git a/rdagent/utils/workflow/tracking.py b/rdagent/utils/workflow/tracking.py index a1ad36b6..598b16a7 100644 --- a/rdagent/utils/workflow/tracking.py +++ b/rdagent/utils/workflow/tracking.py @@ -85,7 +85,7 @@ class WorkflowTracker: if self.loop_base.timer.started: remain_time = self.loop_base.timer.remain_time() assert remain_time is not None - mlflow.log_metric("remain_time", remain_time.seconds) + mlflow.log_metric("remain_time", remain_time.total_seconds()) mlflow.log_metric( "remain_percent", remain_time / self.loop_base.timer.all_duration * 100,