feat: add coder check and give more time (#1127)

* feat: introduce max_seconds_multiplier for timeout management across components

* avoid using assert in test

* hot fix a very small bug

* runner multiply twice

* revert feedback change

* add a switch to longer timeout

rdagent/components/coder/CoSTEER/__init__.py

@@ -28,6 +28,7 @@ class CoSTEER(Developer[Experiment]):
         es: EvolvingStrategy,
         evolving_version: int,
         *args,
+        max_seconds: int | None = None,
         with_knowledge: bool = True,
         with_feedback: bool = True,
         knowledge_self_gen: bool = True,
@@ -37,7 +38,7 @@ class CoSTEER(Developer[Experiment]):
     ) -> None:
         super().__init__(*args, **kwargs)
         self.max_loop = settings.max_loop if max_loop is None else max_loop
-        self.max_seconds = settings.max_seconds
+        self.max_seconds = max_seconds
         self.knowledge_base_path = (
             Path(settings.knowledge_base_path) if settings.knowledge_base_path is not None else None
         )
@@ -105,7 +106,7 @@ class CoSTEER(Developer[Experiment]):
             logger.log_object(evo_exp.sub_workspace_list, tag="evolving code")
             for sw in evo_exp.sub_workspace_list:
                 logger.info(f"evolving workspace: {sw}")
-            if (datetime.now() - start_datetime).seconds > self.max_seconds:
+            if self.max_seconds is not None and (datetime.now() - start_datetime).seconds > self.max_seconds:
                 logger.info(f"Reached max time limit {self.max_seconds} seconds, stop evolving")
                 break
             if RD_Agent_TIMER_wrapper.timer.started and RD_Agent_TIMER_wrapper.timer.is_timeout():

rdagent/components/coder/CoSTEER/config.py

@@ -33,7 +33,7 @@ class CoSTEERSettings(ExtendedBaseSettings):
     new_knowledge_base_path: Union[str, None] = None
     """Path to the new knowledge base"""
 
-    max_seconds: int = 10**6
+    max_seconds_multiplier: int = 10**6
 
 
 CoSTEER_SETTINGS = CoSTEERSettings()

rdagent/components/coder/data_science/conf.py

@@ -19,7 +19,7 @@ class DSCoderCoSTEERSettings(CoSTEERSettings):
     class Config:
         env_prefix = "DS_Coder_CoSTEER_"
 
-    max_seconds: int = DS_RD_SETTING.debug_timeout * 4
+    max_seconds_multiplier: int = 4
     env_type: str = "docker"
     # TODO: extract a function for env and conf.
 
@@ -28,7 +28,7 @@ class DSCoderCoSTEERSettings(CoSTEERSettings):
     Extra evaluators
 
     The evaluator follows the following assumptions:
-    - It runs after previous evaluator (So the running results are alreadly there)
+    - It runs after previous evaluator (So the running results are already there)
 
     It is not a complete feature due to it is only implemented in DS Pipeline & Coder.
 

rdagent/components/coder/data_science/ensemble/__init__.py

@@ -160,5 +160,6 @@ class EnsembleCoSTEER(CoSTEER):
             evolving_version=2,
             scen=scen,
             max_loop=DS_RD_SETTING.coder_max_loop,
+            max_seconds=scen.real_debug_timeout() * settings.max_seconds_multiplier,
             **kwargs,
         )

rdagent/components/coder/data_science/ensemble/eval.py

@@ -47,7 +47,10 @@ class EnsembleCoSTEEREvaluator(CoSTEEREvaluator):
                 final_decision=False,
             )
 
-        env = get_ds_env(extra_volumes={self.scen.debug_path: T("scenarios.data_science.share:scen.input_path").r()})
+        env = get_ds_env(
+            extra_volumes={self.scen.debug_path: T("scenarios.data_science.share:scen.input_path").r()},
+            running_timeout_period=self.scen.real_debug_timeout(),
+        )
 
         fname = "test/ensemble_test.txt"
         test_code = (DIRNAME / "eval_tests" / "ensemble_test.txt").read_text()

rdagent/components/coder/data_science/feature/__init__.py

@@ -138,5 +138,6 @@ class FeatureCoSTEER(CoSTEER):
             evolving_version=2,
             scen=scen,
             max_loop=DS_RD_SETTING.coder_max_loop,
+            max_seconds=scen.real_debug_timeout() * settings.max_seconds_multiplier,
             **kwargs,
         )

rdagent/components/coder/data_science/feature/eval.py

@@ -43,7 +43,10 @@ class FeatureCoSTEEREvaluator(CoSTEEREvaluator):
                 final_decision=False,
             )
 
-        env = get_ds_env(extra_volumes={self.scen.debug_path: T("scenarios.data_science.share:scen.input_path").r()})
+        env = get_ds_env(
+            extra_volumes={self.scen.debug_path: T("scenarios.data_science.share:scen.input_path").r()},
+            running_timeout_period=self.scen.real_debug_timeout(),
+        )
 
         # TODO: do we need to clean the generated temporary content?
         fname = "test/feature_test.py"

rdagent/components/coder/data_science/model/__init__.py

@@ -170,5 +170,6 @@ class ModelCoSTEER(CoSTEER):
             evolving_version=2,
             scen=scen,
             max_loop=DS_RD_SETTING.coder_max_loop,
+            max_seconds=scen.real_debug_timeout() * settings.max_seconds_multiplier,
             **kwargs,
         )

rdagent/components/coder/data_science/model/eval.py

@@ -57,7 +57,10 @@ class ModelGeneralCaseSpecEvaluator(CoSTEEREvaluator):
                 final_decision=False,
             )
 
-        env = get_ds_env(extra_volumes={self.scen.debug_path: T("scenarios.data_science.share:scen.input_path").r()})
+        env = get_ds_env(
+            extra_volumes={self.scen.debug_path: T("scenarios.data_science.share:scen.input_path").r()},
+            running_timeout_period=self.scen.real_debug_timeout(),
+        )
 
         if_model_removed = False
 

rdagent/components/coder/data_science/pipeline/__init__.py

@@ -161,5 +161,6 @@ class PipelineCoSTEER(CoSTEER):
             evolving_version=2,
             scen=scen,
             max_loop=DS_RD_SETTING.coder_max_loop,
+            max_seconds=scen.real_debug_timeout() * settings.max_seconds_multiplier,
             **kwargs,
         )

rdagent/components/coder/data_science/pipeline/eval.py

@@ -53,7 +53,10 @@ class PipelineCoSTEEREvaluator(CoSTEEREvaluator):
                 final_decision=False,
             )
 
-        env = get_ds_env(extra_volumes={self.scen.debug_path: T("scenarios.data_science.share:scen.input_path").r()})
+        env = get_ds_env(
+            extra_volumes={self.scen.debug_path: T("scenarios.data_science.share:scen.input_path").r()},
+            running_timeout_period=self.scen.real_debug_timeout(),
+        )
 
         stdout = ""
         implementation.execute(env=env, entry=get_clear_ws_cmd())
@@ -97,6 +100,11 @@ class PipelineCoSTEEREvaluator(CoSTEEREvaluator):
             else:
                 stdout += "Debug mode did not provide debug_time or estimated_time, it's a buggy implementation.\n"
 
+            test_eval = get_test_eval()
+            if test_eval.enabled(self.scen.competition):
+                submission_check_out, submission_ret_code = test_eval.valid(self.scen.competition, implementation)
+                stdout += f"\n### Submission check:\n{submission_check_out}\nIf Submission check returns a 'Submission is valid' or similar message, despite some warning messages, you should still consider the submission as valid and give a positive final decision. "
+
         score_fp = implementation.workspace_path / "scores.csv"
         score_ret_code = 0
         score_check_text = ""
@@ -153,6 +161,7 @@ class PipelineCoSTEEREvaluator(CoSTEEREvaluator):
         system_prompt = T(".prompts:pipeline_eval.system").r(
             is_sub_enabled=test_eval.is_sub_enabled(self.scen.competition),
             debug_mode=DS_RD_SETTING.sample_data_by_LLM,
+            mle_check=(DS_RD_SETTING.sample_data_by_LLM and test_eval.is_sub_enabled(self.scen.competition)),
         )
         user_prompt = T(".prompts:pipeline_eval.user").r(
             scenario=self.scen.get_scenario_all_desc(eda_output=eda_output),

rdagent/components/coder/data_science/pipeline/prompts.yaml

@@ -259,7 +259,14 @@ pipeline_eval:
       - Data sampling should only be applied in debug mode. Always use the full data in the full run.
       - The label classes number should be the same as the full run even in debug mode.
     - If the code passes this step: Finalize evaluation.
-    - If the code does not pass this step: Clearly document the debug mode compliance issues and reject the implementation.
+    - If the code does not pass this step: Clearly document the debug mode compliance issues and reject the implementation.{% endif %}
+
+    {% if mle_check %}
+    ### Step 5: Test format check
+    - The user has done a format check for your submission. Since you didn't sample any test data, your debug mode output should be the same format as the full run.
+    - The user will put the check result in the "Submission check" section of the execution output.
+    - If the submission check returns a 'Submission is valid' or similar message, despite some warning messages, you should give the conclusion that the code executed successfully. If no other code related issues are found, set the "final_decision" to true.
+    - If the submission check returns an error message, you should set the "final_decision" to false and clearly document the issues in the "return_checking" field.
     {% endif %}
 
     ## Output Format

rdagent/components/coder/data_science/raw_data_loader/__init__.py

@@ -218,6 +218,7 @@ class DataLoaderCoSTEER(CoSTEER):
             evolving_version=2,
             scen=scen,
             max_loop=DS_RD_SETTING.coder_max_loop,
+            max_seconds=scen.real_debug_timeout() * settings.max_seconds_multiplier,
             **kwargs,
         )
 
@@ -230,7 +231,7 @@ class DataLoaderCoSTEER(CoSTEER):
                     "scenarios.data_science.share:scen.input_path"
                 ).r()
             },
-            running_timeout_period=DS_RD_SETTING.full_timeout,
+            running_timeout_period=self.scen.real_full_timeout(),
         )
 
         stdout = new_exp.experiment_workspace.execute(env=env, entry=f"python test/data_loader_test.py")

rdagent/components/coder/data_science/raw_data_loader/eval.py

@@ -46,7 +46,10 @@ class DataLoaderCoSTEEREvaluator(CoSTEEREvaluator):
                 final_decision=False,
             )
 
-        env = get_ds_env(extra_volumes={self.scen.debug_path: T("scenarios.data_science.share:scen.input_path").r()})
+        env = get_ds_env(
+            extra_volumes={self.scen.debug_path: T("scenarios.data_science.share:scen.input_path").r()},
+            running_timeout_period=self.scen.real_debug_timeout(),
+        )
 
         # TODO: do we need to clean the generated temporary content?
         fname = "test/data_loader_test.py"

rdagent/components/coder/data_science/share/eval.py

@@ -55,7 +55,7 @@ class ModelDumpEvaluator(CoSTEEREvaluator):
         env = get_ds_env(
             extra_volumes={data_source_path: T("scenarios.data_science.share:scen.input_path").r()},
             running_timeout_period=(
-                DS_RD_SETTING.full_timeout if self.data_type == "full" else DS_RD_SETTING.debug_timeout
+                self.scen.real_full_timeout() if self.data_type == "full" else self.scen.real_debug_timeout()
             ),
         )
 

rdagent/components/coder/data_science/workflow/__init__.py

@@ -131,5 +131,6 @@ class WorkflowCoSTEER(CoSTEER):
             evolving_version=2,
             scen=scen,
             max_loop=DS_RD_SETTING.coder_max_loop,
+            max_seconds=scen.real_debug_timeout() * settings.max_seconds_multiplier,
             **kwargs,
         )

rdagent/components/coder/data_science/workflow/eval.py

@@ -55,7 +55,10 @@ class WorkflowGeneralCaseSpecEvaluator(CoSTEEREvaluator):
                 final_decision=False,
             )
 
-        env = get_ds_env(extra_volumes={self.scen.debug_path: T("scenarios.data_science.share:scen.input_path").r()})
+        env = get_ds_env(
+            extra_volumes={self.scen.debug_path: T("scenarios.data_science.share:scen.input_path").r()},
+            running_timeout_period=self.scen.real_debug_timeout(),
+        )
 
         # # DockerEnv for MLEBench submission validation
         # mle_de_conf = MLEBDockerConf()