chore: add inference mode for model dump (#1182)

* chore: add inference mode for model dump
* check runtime environment
* update opened_trace_lines
---------

Co-authored-by: you-n-g <you-n-g@users.noreply.github.com>

rdagent/components/coder/data_science/share/eval.py

@@ -1,3 +1,4 @@
+import re
 from pathlib import Path
 from typing import Literal
 
@@ -77,13 +78,40 @@ class ModelDumpEvaluator(CoSTEEREvaluator):
         implementation.execute(env=env, entry=get_clear_ws_cmd(stage="before_inference"))
 
         # Execute the main script
-        stdout = remove_eda_part(implementation.execute(env=env, entry="python main.py"))
+        stdout = remove_eda_part(
+            implementation.execute(env=env, entry="strace -e trace=file -f -o trace.log python main.py --inference")
+        )
 
         # walk model_folder and list the files
         model_folder_files = [
             str(file.relative_to(implementation.workspace_path)) for file in model_folder.iterdir() if file.is_file()
         ]
 
+        opened_trace_lines = None
+        if (implementation.workspace_path / "trace.log").exists():
+            input_path = T("scenarios.data_science.share:scen.input_path").r()
+            abs_input_path = str(Path(input_path).resolve())
+            # matching path in string like `openat(AT_FDCWD, "/home/user/project/main.py", O_RDONLY) = 5`
+            path_regex = re.compile(r'openat\(.+?,\s*"([^"]+)"')
+            log_content = (implementation.workspace_path / "trace.log").read_text()
+
+            opened_files = set()
+            for line in log_content.splitlines():
+                if "openat" not in line or (abs_input_path not in line and input_path not in line):
+                    continue
+
+                match = path_regex.search(line)
+                if match:
+                    full_path = Path(match.group(1)).resolve()
+                    if str(full_path).startswith(abs_input_path):
+                        opened_files.add(Path(data_source_path).resolve() / full_path.relative_to(abs_input_path))
+
+            from rdagent.scenarios.data_science.scen.utils import FileTreeGenerator
+
+            tree_gen = FileTreeGenerator(allowed_paths=opened_files)  # pass opened files filter
+            opened_trace_lines = tree_gen.generate_tree(Path(data_source_path).resolve())
+            # Limitation: training and test are expected to be different files.
+
         # this will assert the generation of necessary files
         for f in ["submission.csv", "scores.csv"]:
             if not (implementation.workspace_path / f).exists():
@@ -125,6 +153,7 @@ class ModelDumpEvaluator(CoSTEEREvaluator):
             model_folder_files=model_folder_files,
             scores_content_before=scores_content_before,
             scores_content_after=scores_content_after,
+            opened_trace_lines=opened_trace_lines,
         )
 
         csfb = build_cls_from_json_with_retry(

rdagent/components/coder/data_science/share/prompts.yaml

@@ -1,9 +1,15 @@
 dump_model_coder:
   guideline: |-
+    Your code will be executed in a inference mode with following command: 
+    ```bash
+    python main.py --inference
+    ```
     Please dump the model in a "models/" subfolder in the first running, and the script rerun performs inference without needing to retrain the model when running the code again.
+    In inference Mode, the script MUST NOT load any training data. 
     If there are parameters generated from the training data that might be needed for inference on test data, please save them in the "models/" subfolder as well.
     If no test set is provided, reserve a portion of the data as your test set and save the generated test files in the models/ subfolder for use in submission and inference.
     Make sure that the required files, like submission.csv and scores.csv, are created without model training step through loading the saved model and test data file directly.
+    
 
 dump_model_eval:
   system: |-
@@ -20,8 +26,9 @@ dump_model_eval:
     Focus on these aspects:
     - Check if the code saves the model in the "models/" subfolder.
     - Check if the code saves the test data in the "models/" subfolder when there is no test data specified.
-    - Ensure that when the code is rerun, it skips the training process and loads the model from the "models/" subfolder for direct inference.
+    - Ensure that when the code is rerun in inference mode, it skips the training process and loads the model from the "models/" subfolder for direct inference.
       - Verify that there is no training activity in the output.
+      - Verify that the script does not load the original training data.
     - Ensure that even if you skip the model training by loading saved models, the files like scores.csv and submission.csv are still correctly created.
     - The model's performance should remain consistent and not vary unreasonably between training and inference.
 
@@ -42,6 +49,9 @@ dump_model_eval:
     ------------ The stdout from running the code ------------ 
     {{stdout}}
 
+    ------------ File opened by the code ------------
+    {{opened_trace_lines}}
+
     ------------ The file list in "models/" subfolder ------------
     {% for f in model_folder_files %}
     - {{ f }}

rdagent/scenarios/data_science/scen/__init__.py

@@ -19,7 +19,10 @@ from rdagent.scenarios.kaggle.kaggle_crawler import (
     download_data,
     get_metric_direction,
 )
-from rdagent.scenarios.shared.get_runtime_info import get_runtime_environment_by_env
+from rdagent.scenarios.shared.get_runtime_info import (
+    check_runtime_environment,
+    get_runtime_environment_by_env,
+)
 from rdagent.utils.agent.tpl import T
 
 
@@ -28,6 +31,7 @@ class DataScienceScen(Scenario):
 
     def __init__(self, competition: str) -> None:
 
+        check_runtime_environment(get_ds_env())
         # 1) prepare data
         if not Path(f"{DS_RD_SETTING.local_data_path}/{competition}").exists():
             logger.error(f"Please prepare data for competition {competition} first.")

rdagent/scenarios/data_science/scen/utils.py

@@ -299,6 +299,7 @@ class FileTreeGenerator:
         max_lines: int = 200,
         priority_files: Set[str] = None,
         hide_base_name: bool = True,
+        allowed_paths: Set[Path] | None = None,
     ):
         """
         Initialize the file tree generator.
@@ -306,12 +307,33 @@ class FileTreeGenerator:
         Args:
             max_lines: Maximum output lines to prevent overly long output
             priority_files: File extensions to prioritize for display
+            hide_base_name: Hide the base name of the directory
+            allowed_paths: Set of allowed paths to include in the tree
+
         """
         self.max_lines = max_lines
         self.priority_files = priority_files or {".csv", ".json", ".parquet", ".md", ".txt"}
         self.lines = []
         self.line_count = 0
         self.hide_base_name = hide_base_name
+        self.allowed_paths = allowed_paths
+        self._lookup_set: Set[Path] | None = None
+
+    def _build_lookup_set(self):
+        """
+        Build the lookup set for allowed paths.
+        """
+        if self.allowed_paths is None:
+            self._lookup_set = None
+            return
+
+        self._lookup_set = set()
+        for path in self.allowed_paths:
+            self._lookup_set.add(path)
+            for parent in path.parents:
+                if str(parent) == ".":
+                    continue
+                self._lookup_set.add(parent)
 
     def generate_tree(self, path: Union[str, Path]) -> str:
         """
@@ -327,6 +349,7 @@ class FileTreeGenerator:
             FileTreeGenerationError: If tree generation fails
         """
         try:
+            self._build_lookup_set()
             path = Path(path)
             base_path = path.resolve()
             self.lines = []
@@ -408,6 +431,7 @@ class FileTreeGenerator:
             prefix: Prefix for tree formatting
             base_path: Base path for symlink detection
 
+
         Raises:
             DirectoryPermissionError: If directory access is denied
             FileTreeGenerationError: If processing fails
@@ -416,6 +440,11 @@ class FileTreeGenerator:
         try:
             # Get directory contents, filter out system files
             items = [p for p in path.iterdir() if not p.name.startswith(".") and p.name not in system_names]
+
+            # Filter by allowed paths if provided
+            if self._lookup_set is not None:
+                items = [p for p in items if p in self._lookup_set]
+
             dirs = sorted([p for p in items if p.is_dir()])
             files = sorted([p for p in items if p.is_file()])
 

rdagent/scenarios/shared/get_runtime_info.py

@@ -10,3 +10,16 @@ def get_runtime_environment_by_env(env: Env) -> str:
     implementation.inject_files(**{fname: (Path(__file__).absolute().resolve().parent / "runtime_info.py").read_text()})
     stdout = implementation.execute(env=env, entry=f"python {fname}")
     return stdout
+
+
+def check_runtime_environment(env: Env) -> str:
+    implementation = FBWorkspace()
+    # 1) Check if strace exists in env
+    strace_check = implementation.execute(env=env, entry="which strace || echo MISSING").strip()
+    if strace_check.endswith("MISSING"):
+        raise RuntimeError("`strace` not found in the target environment.")
+
+    # 2) Check if coverage module works in env
+    coverage_check = implementation.execute(env=env, entry="python -m coverage --version || echo MISSING").strip()
+    if coverage_check.endswith("MISSING"):
+        raise RuntimeError("`coverage` module not found or not runnable in the target environment.")