feat: add option to enable hyperparameter tuning only in first eval loop (#1211)

* feat: add option to enable hyperparameter tuning only in first eval loop

* fix: use total_seconds() for accurate time calculations in evolution and tracking

rdagent/app/kaggle/conf.py

@@ -76,10 +76,18 @@ class KaggleBasePropSetting(ExtendedBaseSettings):
     """Enable mini-case study for experiments"""
 
     time_ratio_limit_to_enable_hyperparameter_tuning: float = 1
-    """Runner time ratio limit to enable hyperparameter tuning, if not change, hyperparameter tuning is always enabled in the first evolution."""
+    """
+    Runner time ratio limit to enable hyperparameter tuning, if not change, hyperparameter tuning is always enabled in the first evolution.
+    """
 
     res_time_ratio_limit_to_enable_hyperparameter_tuning: float = 1
-    """Overall rest time ratio limit to enable hyperparameter tuning, if not change, hyperparameter tuning is always enabled in the first evolution."""
+    """
+    Overall rest time ratio limit to enable hyperparameter tuning, if not change, hyperparameter tuning is always enabled in the first evolution.
+    `1` indicate we enable hyperparameter tuning when we have 100% residual time. (so hyperparameter tuning is always enabled)
+    """
+
+    only_first_loop_enable_hyperparameter_tuning: bool = True
+    """Enable hyperparameter tuning feedback only in the first loop of evaluation."""
 
     only_enable_tuning_in_merge: bool = False
     """Enable hyperparameter tuning only in the merge stage"""

rdagent/components/coder/CoSTEER/__init__.py

@@ -125,7 +125,7 @@ class CoSTEER(Developer[Experiment]):
             logger.log_object(evo_exp.sub_workspace_list, tag="evolving code")
             for sw in evo_exp.sub_workspace_list:
                 logger.info(f"evolving workspace: {sw}")
-            if max_seconds is not None and (datetime.now() - start_datetime).seconds > max_seconds:
+            if max_seconds is not None and (datetime.now() - start_datetime).total_seconds() > max_seconds:
                 logger.info(f"Reached max time limit {max_seconds} seconds, stop evolving")
                 reached_max_seconds = True
                 break

rdagent/scenarios/data_science/dev/runner/eval.py

@@ -164,7 +164,10 @@ class DSRunnerEvaluator(CoSTEEREvaluator):
 
         # Whether to enable hyperparameter tuning check
         # 1. This is the first loop of evaluation.
-        c1 = len(queried_knowledge.task_to_former_failed_traces[target_task.get_task_information()][0]) == 0
+        if DS_RD_SETTING.only_first_loop_enable_hyperparameter_tuning:
+            c1 = len(queried_knowledge.task_to_former_failed_traces[target_task.get_task_information()][0]) == 0
+        else:
+            c1 = True
 
         # 2. The current time spent on runner is less than the time limit ratio for runner timeout.
         time_spent_ratio = implementation.running_info.running_time / env.conf.running_timeout_period

rdagent/utils/workflow/tracking.py

@@ -85,7 +85,7 @@ class WorkflowTracker:
             if self.loop_base.timer.started:
                 remain_time = self.loop_base.timer.remain_time()
                 assert remain_time is not None
-                mlflow.log_metric("remain_time", remain_time.seconds)
+                mlflow.log_metric("remain_time", remain_time.total_seconds())
                 mlflow.log_metric(
                     "remain_percent",
                     remain_time / self.loop_base.timer.all_duration * 100,