feat: add time ratio limit for hyperparameter tuning in Kaggle settin… (#1135)

* feat: add time ratio limit for hyperparameter tuning in Kaggle settings and update evaluator logic

* add recommend time limit

* remove 25% hard line

* small fix

rdagent/app/data_science/conf.py

@@ -32,8 +32,12 @@ class DataScienceBasePropSetting(KaggleBasePropSetting):
     ## Coding Related
     coding_fail_reanalyze_threshold: int = 3
 
+    debug_recommend_timeout: int = 600
+    """The recommend time limit for running on debugging data"""
     debug_timeout: int = 600
     """The timeout limit for running on debugging data"""
+    full_recommend_timeout: int = 3600
+    """The recommend time limit for running on full data"""
     full_timeout: int = 3600
     """The timeout limit for running on full data"""
 

rdagent/app/kaggle/conf.py

@@ -75,5 +75,8 @@ class KaggleBasePropSetting(ExtendedBaseSettings):
     mini_case: bool = False
     """Enable mini-case study for experiments"""
 
+    time_ratio_limit_to_enable_hyperparameter_tuning: int | None = None
+    """Time ratio limit to enable hyperparameter tuning, if None, hyperparameter tuning is always enabled in the first evolution."""
+
 
 KAGGLE_IMPLEMENT_SETTING = KaggleBasePropSetting()

rdagent/scenarios/data_science/dev/runner/eval.py

@@ -133,6 +133,16 @@ class DSCoSTEERCoSTEEREvaluator(CoSTEEREvaluator):
             submission_check_out, submission_ret_code = test_eval.valid(self.scen.competition, implementation)
             stdout += f"\nSubmission check:\n{submission_check_out}\nIf Submission check returns a 'Submission is valid' or similar message, despite some warning messages, you should still consider the submission as valid and give a positive final decision. "
 
+        time_spent_ratio = implementation.running_info.running_time / env.conf.running_timeout_period
+        if (
+            DS_RD_SETTING.time_ratio_limit_to_enable_hyperparameter_tuning is not None
+            and time_spent_ratio > DS_RD_SETTING.time_ratio_limit_to_enable_hyperparameter_tuning
+        ):
+            enable_hyperparameter_tuning_check = False
+            logger.info(
+                f"Time spent ratio {time_spent_ratio:.2f} exceeds the limit {DS_RD_SETTING.time_ratio_limit_to_enable_hyperparameter_tuning}, hyperparameter tuning is disabled."
+            )
+
         system_prompt = T(".prompts:DSCoSTEER_eval.system").r(
             scenario=self.scen.get_scenario_all_desc(eda_output=implementation.file_dict.get("EDA.md", None)),
             is_sub_enabled=test_eval.is_sub_enabled(self.scen.competition),
@@ -144,7 +154,7 @@ class DSCoSTEERCoSTEEREvaluator(CoSTEEREvaluator):
             stdout=shrink_text(stdout),
             time_spent=f"{implementation.running_info.running_time:.2f} seconds",
             timeout=f"{env.conf.running_timeout_period} seconds",
-            percent_of_timeout_used=f"{(implementation.running_info.running_time / env.conf.running_timeout_period) * 100:.2f}%",
+            percent_of_timeout_used=f"{time_spent_ratio * 100:.2f}%",
         )
 
         feedback = build_cls_from_json_with_retry(

rdagent/scenarios/data_science/dev/runner/prompts.yaml

@@ -33,8 +33,8 @@ DSCoSTEER_eval:
     # Evaluation 2: Hyperparameter
     ## Evaluation Description
     The user will provide you the time spent on the whole code execution and the timeout of the code execution. You should decide whether the hyperparameter is reasonable based on the time.
-    If the code uses only a very small portion (below 25%) of the allowed time, and hyperparameters like `n_estimators` or `epochs` have low values, with early stopping not being triggered and possible signs of underfitting, you should suggest increasing these hyperparameters.
-    You should also notice other resources utilization hyper-parameters.
+    For example, if the code uses only a very small portion of the allowed time, and hyperparameters like `n_estimators` or `epochs` have low values, with early stopping not being triggered and possible signs of underfitting, you should suggest increasing these hyperparameters.
+    You should also notice other resources utilization hyper-parameters,
     For example, if you are using a GPU with large memory, and the batch size is set very low, you should suggest increasing the batch size if it is not reasonable.
 
     ## Evaluation Guidelines

rdagent/scenarios/data_science/scen/__init__.py

@@ -127,6 +127,17 @@ class DataScienceScen(Scenario):
             else DS_RD_SETTING.debug_timeout
         )
 
+    def recommend_debug_timeout(self):
+        return (
+            DS_RD_SETTING.debug_recommend_timeout
+            * min(
+                DS_RD_SETTING.coder_longer_timeout_multiplier_upper,
+                self.timeout_increase_count * DS_RD_SETTING.timeout_increase_stage + 1,
+            )
+            if self.longer_time_limit_required and DS_RD_SETTING.allow_longer_timeout
+            else DS_RD_SETTING.debug_timeout
+        )
+
     def real_full_timeout(self):
         return (
             DS_RD_SETTING.full_timeout
@@ -138,6 +149,17 @@ class DataScienceScen(Scenario):
             else DS_RD_SETTING.full_timeout
         )
 
+    def recommend_full_timeout(self):
+        return (
+            DS_RD_SETTING.full_recommend_timeout
+            * min(
+                DS_RD_SETTING.runner_longer_timeout_multiplier_upper,
+                self.timeout_increase_count * DS_RD_SETTING.timeout_increase_stage + 1,
+            )
+            if self.longer_time_limit_required and DS_RD_SETTING.allow_longer_timeout
+            else DS_RD_SETTING.full_recommend_timeout
+        )
+
     def increase_timeout(self):
         """Increase the timeout multiplier for the scenario."""
         self.timeout_increase_count += 1
@@ -172,9 +194,11 @@ class DataScienceScen(Scenario):
             raw_description=self.raw_description,
             use_raw_description=DS_RD_SETTING.use_raw_description,
             time_limit=None,
+            recommend_time_limit=None,
             eda_output=None,
             sample_data_by_LLM=None,
             debug_time_limit=None,
+            recommend_debug_time_limit=None,
             runtime_environment=self.get_runtime_environment(),
         )
 
@@ -191,9 +215,11 @@ class DataScienceScen(Scenario):
             raw_description=self.raw_description,
             use_raw_description=DS_RD_SETTING.use_raw_description,
             time_limit=f"{self.real_full_timeout() / 60 / 60 : .2f} hours",
+            recommend_time_limit=f"{self.recommend_full_timeout() / 60 / 60 : .2f} hours",
             eda_output=eda_output,
             sample_data_by_LLM=DS_RD_SETTING.sample_data_by_LLM,
-            debug_time_limit=f"{self.real_debug_timeout() / 60 / 60 : .2f} hours",
+            debug_time_limit=f"{self.real_debug_timeout() / 60 : .2f} minutes",
+            recommend_debug_time_limit=f"{self.recommend_debug_timeout() / 60 : .2f} minutes",
             runtime_environment=self.get_runtime_environment(),
         )
 

rdagent/scenarios/data_science/scen/prompts.yaml

@@ -36,12 +36,20 @@ scenario_description: |-
 
   {% if time_limit %}
   ====== Time Limit On Full Code Execution ======
-  Your full code's execution is limited to **{{ time_limit }}**. After this time limit, your code will be terminated. But remember your main target is to achieve the best performance and you have several times to modify your code. So please be bold to make the best use of all the time limit and don't be too conservative.
+  Your full code's execution is limited to **{{ time_limit }}**. After this time limit, your code will be terminated and all time and resources are wasted. Always make sure your code will not run longer than this time limit.
   During this time limit, you have all the resources available to you. Please fully leverage all the computational resources(CPUs and GPUs) to achieve the best performance like choose a powerful model, use a large batch size, enable data sampler with big parallel.
   {% if sample_data_by_LLM is not none and sample_data_by_LLM is true %}
   ====== Time Limit On Debug Mode Code Execution ======
   Your are also required to include a debug mode in your code, the debug code's execution is limited to **{{ debug_time_limit }}**. You should make sure 10 percent of the data training one epoch can be finished within this time limit. If not, your should propose a new debug strategy in your task.
-  {% endif %}{% endif %}
+  {% endif %}
+  ====== Recommend Time Spent On Full Code Execution ======
+  You should always prioritize performance over time spent since some tasks requires very little time to run the code to achieve the best performance while some tasks might need a lot of time to train one or more large models.
+  We recommend you to spend less than **{{ recommend_time_limit }}** on the full code execution to boost efficiency. This is a recommended time limit, you can spend more time on the code execution if you think it is necessary. But always remember to not exceed the time limit.
+  {% if sample_data_by_LLM is not none and sample_data_by_LLM is true %}
+  ====== Recommend Time Spent On Debug Mode Code Execution ======
+  We recommend you to spend less than **{{ recommend_debug_time_limit }}** on the debug mode code execution to boost efficiency. This is a recommended time limit, you can spend more time on the debug mode code execution if you think it is necessary. But always remember to not exceed the time limit.
+  {% endif %}
+  {% endif %}
 
   {% if runtime_environment is not none %}
   ====== Runtime Environment ======