chore: add inference mode for model dump (#1182)

* chore: add inference mode for model dump
* check runtime environment
* update opened_trace_lines
---------

Co-authored-by: you-n-g <you-n-g@users.noreply.github.com>

rdagent/components/coder/data_science/share/eval.py

@@ -1,3 +1,4 @@
+import re
 from pathlib import Path
 from typing import Literal
 
@@ -77,13 +78,40 @@ class ModelDumpEvaluator(CoSTEEREvaluator):
         implementation.execute(env=env, entry=get_clear_ws_cmd(stage="before_inference"))
 
         # Execute the main script
-        stdout = remove_eda_part(implementation.execute(env=env, entry="python main.py"))
+        stdout = remove_eda_part(
+            implementation.execute(env=env, entry="strace -e trace=file -f -o trace.log python main.py --inference")
+        )
 
         # walk model_folder and list the files
         model_folder_files = [
             str(file.relative_to(implementation.workspace_path)) for file in model_folder.iterdir() if file.is_file()
         ]
 
+        opened_trace_lines = None
+        if (implementation.workspace_path / "trace.log").exists():
+            input_path = T("scenarios.data_science.share:scen.input_path").r()
+            abs_input_path = str(Path(input_path).resolve())
+            # matching path in string like `openat(AT_FDCWD, "/home/user/project/main.py", O_RDONLY) = 5`
+            path_regex = re.compile(r'openat\(.+?,\s*"([^"]+)"')
+            log_content = (implementation.workspace_path / "trace.log").read_text()
+
+            opened_files = set()
+            for line in log_content.splitlines():
+                if "openat" not in line or (abs_input_path not in line and input_path not in line):
+                    continue
+
+                match = path_regex.search(line)
+                if match:
+                    full_path = Path(match.group(1)).resolve()
+                    if str(full_path).startswith(abs_input_path):
+                        opened_files.add(Path(data_source_path).resolve() / full_path.relative_to(abs_input_path))
+
+            from rdagent.scenarios.data_science.scen.utils import FileTreeGenerator
+
+            tree_gen = FileTreeGenerator(allowed_paths=opened_files)  # pass opened files filter
+            opened_trace_lines = tree_gen.generate_tree(Path(data_source_path).resolve())
+            # Limitation: training and test are expected to be different files.
+
         # this will assert the generation of necessary files
         for f in ["submission.csv", "scores.csv"]:
             if not (implementation.workspace_path / f).exists():
@@ -125,6 +153,7 @@ class ModelDumpEvaluator(CoSTEEREvaluator):
             model_folder_files=model_folder_files,
             scores_content_before=scores_content_before,
             scores_content_after=scores_content_after,
+            opened_trace_lines=opened_trace_lines,
         )
 
         csfb = build_cls_from_json_with_retry(

rdagent/components/coder/data_science/share/prompts.yaml

@@ -1,9 +1,15 @@
 dump_model_coder:
   guideline: |-
+    Your code will be executed in a inference mode with following command: 
+    ```bash
+    python main.py --inference
+    ```
     Please dump the model in a "models/" subfolder in the first running, and the script rerun performs inference without needing to retrain the model when running the code again.
+    In inference Mode, the script MUST NOT load any training data. 
     If there are parameters generated from the training data that might be needed for inference on test data, please save them in the "models/" subfolder as well.
     If no test set is provided, reserve a portion of the data as your test set and save the generated test files in the models/ subfolder for use in submission and inference.
     Make sure that the required files, like submission.csv and scores.csv, are created without model training step through loading the saved model and test data file directly.
+    
 
 dump_model_eval:
   system: |-
@@ -20,8 +26,9 @@ dump_model_eval:
     Focus on these aspects:
     - Check if the code saves the model in the "models/" subfolder.
     - Check if the code saves the test data in the "models/" subfolder when there is no test data specified.
-    - Ensure that when the code is rerun, it skips the training process and loads the model from the "models/" subfolder for direct inference.
+    - Ensure that when the code is rerun in inference mode, it skips the training process and loads the model from the "models/" subfolder for direct inference.
       - Verify that there is no training activity in the output.
+      - Verify that the script does not load the original training data.
     - Ensure that even if you skip the model training by loading saved models, the files like scores.csv and submission.csv are still correctly created.
     - The model's performance should remain consistent and not vary unreasonably between training and inference.
 
@@ -42,6 +49,9 @@ dump_model_eval:
     ------------ The stdout from running the code ------------ 
     {{stdout}}
 
+    ------------ File opened by the code ------------
+    {{opened_trace_lines}}
+
     ------------ The file list in "models/" subfolder ------------
     {% for f in model_folder_files %}
     - {{ f }}