feat: add sample submission file check (#1053)

* check sample submission & add  package constraint

* add trace.log into clear

* change default

* simplify

* clear CI workspace before running

* move to CI

* use sudo to clean workspace

* move prepare out of global var

---------

Co-authored-by: Xu Yang <xuyang1@microsoft.com>

rdagent/components/coder/data_science/conf.py

@@ -54,6 +54,7 @@ def get_ds_env(
         raise ValueError(f"Unknown env type: {conf.env_type}")
     env.conf.extra_volumes = extra_volumes
     env.conf.running_timeout_period = running_timeout_period
+    env.prepare()
     return env
 
 
@@ -63,7 +64,7 @@ def get_clear_ws_cmd(stage: Literal["before_training", "before_inference"] = "be
     """
     assert stage in ["before_training", "before_inference"], f"Unknown stage: {stage}"
     if DS_RD_SETTING.enable_model_dump and stage == "before_training":
-        cmd = "rm -r submission.csv scores.csv models"
+        cmd = "rm -r submission.csv scores.csv models trace.log"
     else:
-        cmd = "rm submission.csv scores.csv"
+        cmd = "rm submission.csv scores.csv trace.log"
     return cmd

rdagent/components/coder/data_science/pipeline/eval.py

@@ -59,9 +59,28 @@ class PipelineCoSTEEREvaluator(CoSTEEREvaluator):
         implementation.execute(env=env, entry=get_clear_ws_cmd())
         if DS_RD_SETTING.sample_data_by_LLM:
             # Because coder runs on full data, we need to run debug mode in advance to save time
-            result = implementation.run(env=env, entry=f"python -m coverage run main.py --debug")
+            result = implementation.run(
+                env=env, entry=f"strace -e trace=file -f -o trace.log python -m coverage run main.py --debug"
+            )
         else:
-            result = implementation.run(env=env, entry=f"python -m coverage run main.py")
+            result = implementation.run(
+                env=env, entry=f"strace -e trace=file -f -o trace.log python -m coverage run main.py"
+            )
+
+        sample_submission_check = True
+        test_eval = get_test_eval()
+        if (sample_submission_file_name := test_eval.get_sample_submission_name(self.scen.competition)) is not None:
+            # check whether code ever opens the sample submission file
+            if (implementation.workspace_path / "trace.log").exists():
+                opened_trace_lines = [
+                    line
+                    for line in (implementation.workspace_path / "trace.log").read_text().splitlines()
+                    if "openat" in line and sample_submission_file_name in line
+                ]
+                if len(opened_trace_lines) > 0:
+                    stdout += f"Code opened the sample submission file '{sample_submission_file_name}' during execution.\n Reject the implementation!\n"
+                    sample_submission_check = False
+
         result.stdout = remove_eda_part(result.stdout)
         if result.exit_code != 0:
             stdout += f"Code failed to run. Please check the stdout:\n Following the stdout of the debug mode run:\n{result.stdout.strip()}\n"
@@ -114,7 +133,6 @@ class PipelineCoSTEEREvaluator(CoSTEEREvaluator):
                 score_check_text += f"\n[Error] in checking the scores.csv file: {e}\nscores.csv's content:\n-----\n{score_fp.read_text()}\n-----"
                 score_ret_code = 1
 
-        test_eval = get_test_eval()
         if not test_eval.is_sub_enabled(self.scen.competition):
             submission_ret_code = 0
         else:
@@ -149,10 +167,15 @@ class PipelineCoSTEEREvaluator(CoSTEEREvaluator):
             user_prompt=user_prompt,
             init_kwargs_update_func=PipelineSingleFeedback.val_and_update_init_dict,
         )
-        if score_ret_code != 0:
+        if score_ret_code != 0 and wfb.final_decision is True:
             wfb.final_decision = False
             wfb.return_checking += "\n" + score_check_text
-        if submission_ret_code != 0:
+        if submission_ret_code != 0 and wfb.final_decision is True:
             wfb.final_decision = False
             wfb.return_checking += "\nSubmission file check failed."
+        if sample_submission_check is False and wfb.final_decision is True:
+            wfb.final_decision = False
+            wfb.return_checking += (
+                "\nSample submission file check failed. Code should not open the sample submission file."
+            )
         return wfb

rdagent/components/coder/data_science/pipeline/eval_tests/submission_format_test.txt

@@ -31,7 +31,7 @@ find . | grep -i sample | grep -i submission | grep -v sample_submission.csv | g
 input_dir = Path('{% include "scenarios.data_science.share:scen.input_path" %}')
 sample_submission_files = list(input_dir.glob("*sample_submission*.csv")) + list(
     input_dir.glob("*sampleSubmission*.csv")
-)
+) + list(input_dir.glob("*randomPredictions*.tsv"))
 
 if not sample_submission_files:
     print(f'Error: No sample submission file found in {% include "scenarios.data_science.share:scen.input_path" %}')