feat: custom data (#810)

* custom data

* fix: simplify competition check and log local description file

* no sample data

* feat: add test evaluation module with error handling support

* fix: update eval path to use eval_sub_dir and add valid_check TODO

* refactor: add MLETestEval check to conditionally run grading steps

* avoid blank stdout

* valid in testeval

* rename test.csv to avoid conflict

* Support Disabling sample submission

* refactoring

* fix: remove DS_KAGGLE_DATA and update prompt instructions

* add try for grade

* ignore submission

* fix: remove tee from eval command and warn about pipeline exit code detection

* optional to use raw description

* support old data

* add execution result to stdout

* add metric to raw description

* custom data explain

* add debug_path

* rst update

---------

Co-authored-by: Young <afe.young@gmail.com>

rdagent/components/coder/data_science/conf.py

@@ -27,7 +27,9 @@ class DSCoderCoSTEERSettings(CoSTEERSettings):
 def get_ds_env(
     conf_type: Literal["kaggle", "mlebench"] = "kaggle",
     extra_volumes: dict = {},
-    running_timeout_period: int = DS_RD_SETTING.debug_timeout,
+    running_timeout_period: int = (
+        DS_RD_SETTING.debug_timeout if DS_RD_SETTING.sample_data else DS_RD_SETTING.full_timeout
+    ),
 ) -> Env:
     """
     Retrieve the appropriate environment configuration based on the env_type setting.

rdagent/components/coder/data_science/ensemble/eval.py

@@ -47,13 +47,7 @@ class EnsembleCoSTEEREvaluator(CoSTEEREvaluator):
                 final_decision=False,
             )
 
-        env = get_ds_env(
-            extra_volumes={
-                f"{DS_RD_SETTING.local_data_path}/sample/{self.scen.competition}": T(
-                    "scenarios.data_science.share:scen.input_path"
-                ).r()
-            }
-        )
+        env = get_ds_env(extra_volumes={self.scen.debug_path: T("scenarios.data_science.share:scen.input_path").r()})
 
         fname = "test/ensemble_test.txt"
         test_code = (DIRNAME / "eval_tests" / "ensemble_test.txt").read_text()

rdagent/components/coder/data_science/feature/eval.py

@@ -43,13 +43,7 @@ class FeatureCoSTEEREvaluator(CoSTEEREvaluator):
                 final_decision=False,
             )
 
-        env = get_ds_env(
-            extra_volumes={
-                f"{DS_RD_SETTING.local_data_path}/sample/{self.scen.competition}": T(
-                    "scenarios.data_science.share:scen.input_path"
-                ).r()
-            }
-        )
+        env = get_ds_env(extra_volumes={self.scen.debug_path: T("scenarios.data_science.share:scen.input_path").r()})
 
         # TODO: do we need to clean the generated temporary content?
         fname = "test/feature_test.py"

rdagent/components/coder/data_science/model/eval.py

@@ -57,13 +57,7 @@ class ModelGeneralCaseSpecEvaluator(CoSTEEREvaluator):
                 final_decision=False,
             )
 
-        env = get_ds_env(
-            extra_volumes={
-                f"{DS_RD_SETTING.local_data_path}/sample/{self.scen.competition}": T(
-                    "scenarios.data_science.share:scen.input_path"
-                ).r()
-            }
-        )
+        env = get_ds_env(extra_volumes={self.scen.debug_path: T("scenarios.data_science.share:scen.input_path").r()})
 
         if_model_removed = False
 

rdagent/components/coder/data_science/pipeline/eval.py

@@ -18,6 +18,7 @@ from rdagent.components.coder.CoSTEER.knowledge_management import (
 from rdagent.components.coder.data_science.conf import get_clear_ws_cmd, get_ds_env
 from rdagent.components.coder.data_science.utils import remove_eda_part
 from rdagent.core.experiment import FBWorkspace, Task
+from rdagent.scenarios.data_science.test_eval import get_test_eval
 from rdagent.utils.agent.tpl import T
 from rdagent.utils.agent.workflow import build_cls_from_json_with_retry
 
@@ -52,18 +53,13 @@ class PipelineCoSTEEREvaluator(CoSTEEREvaluator):
                 final_decision=False,
             )
 
-        env = get_ds_env(
-            extra_volumes={
-                f"{DS_RD_SETTING.local_data_path}/sample/{self.scen.competition}": T(
-                    "scenarios.data_science.share:scen.input_path"
-                ).r()
-            }
-        )
+        env = get_ds_env(extra_volumes={self.scen.debug_path: T("scenarios.data_science.share:scen.input_path").r()})
 
         # Clean the scores.csv & submission.csv.
         implementation.execute(env=env, entry=get_clear_ws_cmd())
-        stdout, execute_ret_code = implementation.execute_ret_code(env=env, entry=f"python main.py")
+        stdout, execute_ret_code = implementation.execute_ret_code(env=env, entry=f"python -m coverage run main.py")
         stdout = remove_eda_part(stdout)
+        stdout += f"The code executed {'successfully' if execute_ret_code == 0 else 'failed'}."
 
         score_fp = implementation.workspace_path / "scores.csv"
         score_ret_code = 0
@@ -101,35 +97,40 @@ class PipelineCoSTEEREvaluator(CoSTEEREvaluator):
                 score_check_text += f"\n[Error] in checking the scores.csv file: {e}\nscores.csv's content:\n-----\n{score_fp.read_text()}\n-----"
                 score_ret_code = 1
 
-        # Check submission file
-        base_check_code = T(".eval_tests.submission_format_test", ftype="txt").r()
-        implementation.inject_files(**{"test/submission_format_test.py": base_check_code})
-        # stdout += "----Submission Check 1-----\n"
-        submission_check_out, submission_ret_code = implementation.execute_ret_code(
-            env=env, entry="python test/submission_format_test.py"
-        )
-        if DS_RD_SETTING.rule_base_eval:
-            if execute_ret_code == 0 and score_ret_code == 0 and submission_ret_code == 0:
-                return PipelineSingleFeedback(
-                    execution=stdout,
-                    return_checking=score_check_text + "\n" + submission_check_out,
-                    code="Code evaluation is not available.",
-                    final_decision=True,
-                )
-            else:
-                return PipelineSingleFeedback(
-                    execution=stdout,
-                    return_checking=score_check_text + "\n" + submission_check_out,
-                    code="Code evaluation is not available.",
-                    final_decision=False,
-                )
-        stdout += "\n" + submission_check_out
+        test_eval = get_test_eval()
+        if not test_eval.is_sub_enabled(self.scen.competition):
+            submission_ret_code = 0
+        else:
+            # Check submission file
+            base_check_code = T(".eval_tests.submission_format_test", ftype="txt").r()
+            implementation.inject_files(**{"test/submission_format_test.py": base_check_code})
+            # stdout += "----Submission Check 1-----\n"
+            submission_check_out, submission_ret_code = implementation.execute_ret_code(
+                env=env, entry="python test/submission_format_test.py"
+            )
+            if DS_RD_SETTING.rule_base_eval:
+                if execute_ret_code == 0 and score_ret_code == 0 and submission_ret_code == 0:
+                    return PipelineSingleFeedback(
+                        execution=stdout,
+                        return_checking=score_check_text + "\n" + submission_check_out,
+                        code="Code evaluation is not available.",
+                        final_decision=True,
+                    )
+                else:
+                    return PipelineSingleFeedback(
+                        execution=stdout,
+                        return_checking=score_check_text + "\n" + submission_check_out,
+                        code="Code evaluation is not available.",
+                        final_decision=False,
+                    )
+            stdout += "\n" + submission_check_out
 
         eda_output = implementation.file_dict.get("EDA.md", None)
 
         system_prompt = T(".prompts:pipeline_eval.system").r(
             scenario=self.scen.get_scenario_all_desc(eda_output=eda_output),
             task_desc=target_task.get_task_information(),
+            is_sub_enabled=test_eval.is_sub_enabled(self.scen.competition),
             spec=T("scenarios.data_science.share:component_spec.Pipeline").r(),
         )
         user_prompt = T(".prompts:pipeline_eval.user").r(

rdagent/components/coder/data_science/pipeline/prompts.yaml

@@ -111,7 +111,8 @@ pipeline_eval:
 
     The details on how to structure the code are given in the specification:
     {{ spec }}
-
+    
+    {% if is_sub_enabled %}
     ## Evaluation Scope
     Your focus is to check whether the workflow code:
     Step 1: Executes successfully, correctly generating a final submission.
@@ -146,6 +147,26 @@ pipeline_eval:
         "final_decision": <true/false>
     }
     ```
+    {% else %}
+    ## Evaluation Scope
+    Your focus is to check whether the workflow code executes successfully.
+
+    You will be given the execution output (`stdout`) to determine correctness.  
+
+    [Note] 
+    1. Model performance is NOT a concern in this evaluation—only correct execution and formatting matter.
+
+    Please respond with your feedback in the following JSON format and order
+    ```json
+    {
+        "execution": "Describe whether the code executed successfully. Include any errors or issues encountered, and append all error messages and full traceback details without summarizing or omitting any information.",
+        "return_checking": "Describe the expected file to be generated.",
+        "code": "Provide feedback on code quality, readability, and adherence to the given specifications.",
+        "final_decision": <true/false>
+    }
+    ```
+    {% endif %}
+# NOTE: when is_sub_enabled == False, we don't have any checking about the return. So it is just placeholder currently
 
   user: |-
     --------- code generated by user ---------

rdagent/components/coder/data_science/raw_data_loader/eval.py

@@ -46,13 +46,7 @@ class DataLoaderCoSTEEREvaluator(CoSTEEREvaluator):
                 final_decision=False,
             )
 
-        env = get_ds_env(
-            extra_volumes={
-                f"{DS_RD_SETTING.local_data_path}/sample/{self.scen.competition}": T(
-                    "scenarios.data_science.share:scen.input_path"
-                ).r()
-            }
-        )
+        env = get_ds_env(extra_volumes={self.scen.debug_path: T("scenarios.data_science.share:scen.input_path").r()})
 
         # TODO: do we need to clean the generated temporary content?
         fname = "test/data_loader_test.py"

rdagent/components/coder/data_science/workflow/eval.py

@@ -55,13 +55,7 @@ class WorkflowGeneralCaseSpecEvaluator(CoSTEEREvaluator):
                 final_decision=False,
             )
 
-        env = get_ds_env(
-            extra_volumes={
-                f"{DS_RD_SETTING.local_data_path}/sample/{self.scen.competition}": T(
-                    "scenarios.data_science.share:scen.input_path"
-                ).r()
-            }
-        )
+        env = get_ds_env(extra_volumes={self.scen.debug_path: T("scenarios.data_science.share:scen.input_path").r()})
 
         # # DockerEnv for MLEBench submission validation
         # mle_de_conf = MLEBDockerConf()