diff --git a/rdagent/app/data_science/conf.py b/rdagent/app/data_science/conf.py index 16f5dad7..b9ebcc3f 100644 --- a/rdagent/app/data_science/conf.py +++ b/rdagent/app/data_science/conf.py @@ -126,6 +126,7 @@ class DataScienceBasePropSetting(KaggleBasePropSetting): coder_longer_timeout_multiplier_upper: int = 3 runner_longer_timeout_multiplier_upper: int = 2 timeout_increase_stage: float = 0.3 + show_hard_limit: bool = True #### hypothesis critique and rewrite enable_hypo_critique_rewrite: bool = True diff --git a/rdagent/scenarios/data_science/scen/__init__.py b/rdagent/scenarios/data_science/scen/__init__.py index fe2158b4..e076cd3b 100644 --- a/rdagent/scenarios/data_science/scen/__init__.py +++ b/rdagent/scenarios/data_science/scen/__init__.py @@ -180,7 +180,6 @@ class DataScienceScen(Scenario): time_limit=None, recommend_time_limit=None, eda_output=None, - sample_data_by_LLM=None, debug_time_limit=None, recommend_debug_time_limit=None, runtime_environment=self.get_runtime_environment(), @@ -198,12 +197,17 @@ class DataScienceScen(Scenario): metric_direction=self.metric_direction, raw_description=self.raw_description, use_raw_description=DS_RD_SETTING.use_raw_description, - time_limit=f"{self.real_full_timeout() / 60 / 60 : .2f} hours", - recommend_time_limit=f"{self.recommend_full_timeout() / 60 / 60 : .2f} hours", + time_limit=f"{self.real_full_timeout() / 60 / 60 : .2f} hours" if DS_RD_SETTING.show_hard_limit else None, + recommend_time_limit=( + f"{self.recommend_full_timeout() / 60 / 60 : .2f} hours" if DS_RD_SETTING.sample_data_by_LLM else None + ), eda_output=eda_output, - sample_data_by_LLM=DS_RD_SETTING.sample_data_by_LLM, - debug_time_limit=f"{self.real_debug_timeout() / 60 : .2f} minutes", - recommend_debug_time_limit=f"{self.recommend_debug_timeout() / 60 : .2f} minutes", + debug_time_limit=( + f"{self.real_debug_timeout() / 60 : .2f} minutes" if DS_RD_SETTING.show_hard_limit else None + ), + recommend_debug_time_limit=( + f"{self.recommend_debug_timeout() / 60 : .2f} minutes" if DS_RD_SETTING.sample_data_by_LLM else None + ), runtime_environment=self.get_runtime_environment(), ) diff --git a/rdagent/scenarios/data_science/scen/prompts.yaml b/rdagent/scenarios/data_science/scen/prompts.yaml index 0ea5e64f..5d2a8716 100644 --- a/rdagent/scenarios/data_science/scen/prompts.yaml +++ b/rdagent/scenarios/data_science/scen/prompts.yaml @@ -34,21 +34,20 @@ scenario_description: |- {{ evaluation }} {% endif %} - {% if time_limit %} + {% if time_limit is not none %} ====== Time Limit On Full Code Execution ====== Your full code's execution is limited to **{{ time_limit }}**. After this time limit, your code will be terminated and all time and resources are wasted. Always make sure your code will not run longer than this time limit. During this time limit, you have all the resources available to you. Please fully leverage all the computational resources(CPUs and GPUs) to achieve the best performance like choose a powerful model, use a large batch size, enable data sampler with big parallel. - {% if sample_data_by_LLM is not none and sample_data_by_LLM is true %} + {% endif %}{% if debug_time_limit is not none%} ====== Time Limit On Debug Mode Code Execution ====== Your are also required to include a debug mode in your code, the debug code's execution is limited to **{{ debug_time_limit }}**. You should make sure 10 percent of the data training one epoch can be finished within this time limit. If not, your should propose a new debug strategy in your task. - {% endif %} + {% endif %}{% if recommend_time_limit is not none %} ====== Recommend Time Spent On Full Code Execution ====== You should always prioritize performance over time spent since some tasks requires very little time to run the code to achieve the best performance while some tasks might need a lot of time to train one or more large models. - We recommend you to spend less than **{{ recommend_time_limit }}** on the full code execution to boost efficiency. This is a recommended time limit, you can spend more time on the code execution if you think it is necessary. But always remember to not exceed the time limit. - {% if sample_data_by_LLM is not none and sample_data_by_LLM is true %} + We recommend you to spend less than **{{ recommend_time_limit }}** on the full code execution to boost efficiency. This is a recommended time limit, you can spend more time on the code execution if you think it is very necessary. But your code could be terminated sometime after this recommend time limit. + {% endif %}{% if recommend_debug_time_limit is not none %} ====== Recommend Time Spent On Debug Mode Code Execution ====== - We recommend you to spend less than **{{ recommend_debug_time_limit }}** on the debug mode code execution to boost efficiency. This is a recommended time limit, you can spend more time on the debug mode code execution if you think it is necessary. But always remember to not exceed the time limit. - {% endif %} + We recommend you to spend less than **{{ recommend_debug_time_limit }}** on the debug mode code execution to boost efficiency. This is a recommended time limit, you can spend more time on the debug mode code execution if you think it is very necessary. But your code could be terminated sometime after this recommend time limit. {% endif %} {% if runtime_environment is not none %} diff --git a/rdagent/utils/__init__.py b/rdagent/utils/__init__.py index 7716be93..a89dbd04 100644 --- a/rdagent/utils/__init__.py +++ b/rdagent/utils/__init__.py @@ -127,7 +127,7 @@ def filter_redundant_text(stdout: str) -> str: for line in filtered_stdout_lines: lines_to_count[line] = lines_to_count.get(line, 0) + 1 filtered_stdout = "\n".join( - [line for line in filtered_stdout_lines if lines_to_count[line] <= len(filtered_stdout_lines) // 10] + [line for line in filtered_stdout_lines if lines_to_count[line] <= max(len(filtered_stdout_lines) // 10, 10)] ) # Iteratively ask the LLM for additional filtering patterns (up to 3 rounds)