From 4a516949b470004ab490dd6f6d2567a20fbea12d Mon Sep 17 00:00:00 2001 From: Tim Date: Fri, 15 Aug 2025 15:34:49 +0800 Subject: [PATCH] chore: add inference mode for model dump (#1182) * chore: add inference mode for model dump * check runtime environment * update opened_trace_lines --------- Co-authored-by: you-n-g --- .../coder/data_science/share/eval.py | 31 ++++++++++++++++++- .../coder/data_science/share/prompts.yaml | 12 ++++++- .../scenarios/data_science/scen/__init__.py | 6 +++- rdagent/scenarios/data_science/scen/utils.py | 29 +++++++++++++++++ rdagent/scenarios/shared/get_runtime_info.py | 13 ++++++++ 5 files changed, 88 insertions(+), 3 deletions(-) diff --git a/rdagent/components/coder/data_science/share/eval.py b/rdagent/components/coder/data_science/share/eval.py index 019e4dee..d3b95f5f 100644 --- a/rdagent/components/coder/data_science/share/eval.py +++ b/rdagent/components/coder/data_science/share/eval.py @@ -1,3 +1,4 @@ +import re from pathlib import Path from typing import Literal @@ -77,13 +78,40 @@ class ModelDumpEvaluator(CoSTEEREvaluator): implementation.execute(env=env, entry=get_clear_ws_cmd(stage="before_inference")) # Execute the main script - stdout = remove_eda_part(implementation.execute(env=env, entry="python main.py")) + stdout = remove_eda_part( + implementation.execute(env=env, entry="strace -e trace=file -f -o trace.log python main.py --inference") + ) # walk model_folder and list the files model_folder_files = [ str(file.relative_to(implementation.workspace_path)) for file in model_folder.iterdir() if file.is_file() ] + opened_trace_lines = None + if (implementation.workspace_path / "trace.log").exists(): + input_path = T("scenarios.data_science.share:scen.input_path").r() + abs_input_path = str(Path(input_path).resolve()) + # matching path in string like `openat(AT_FDCWD, "/home/user/project/main.py", O_RDONLY) = 5` + path_regex = re.compile(r'openat\(.+?,\s*"([^"]+)"') + log_content = (implementation.workspace_path / "trace.log").read_text() + + opened_files = set() + for line in log_content.splitlines(): + if "openat" not in line or (abs_input_path not in line and input_path not in line): + continue + + match = path_regex.search(line) + if match: + full_path = Path(match.group(1)).resolve() + if str(full_path).startswith(abs_input_path): + opened_files.add(Path(data_source_path).resolve() / full_path.relative_to(abs_input_path)) + + from rdagent.scenarios.data_science.scen.utils import FileTreeGenerator + + tree_gen = FileTreeGenerator(allowed_paths=opened_files) # pass opened files filter + opened_trace_lines = tree_gen.generate_tree(Path(data_source_path).resolve()) + # Limitation: training and test are expected to be different files. + # this will assert the generation of necessary files for f in ["submission.csv", "scores.csv"]: if not (implementation.workspace_path / f).exists(): @@ -125,6 +153,7 @@ class ModelDumpEvaluator(CoSTEEREvaluator): model_folder_files=model_folder_files, scores_content_before=scores_content_before, scores_content_after=scores_content_after, + opened_trace_lines=opened_trace_lines, ) csfb = build_cls_from_json_with_retry( diff --git a/rdagent/components/coder/data_science/share/prompts.yaml b/rdagent/components/coder/data_science/share/prompts.yaml index 3657c160..db77b937 100644 --- a/rdagent/components/coder/data_science/share/prompts.yaml +++ b/rdagent/components/coder/data_science/share/prompts.yaml @@ -1,9 +1,15 @@ dump_model_coder: guideline: |- + Your code will be executed in a inference mode with following command: + ```bash + python main.py --inference + ``` Please dump the model in a "models/" subfolder in the first running, and the script rerun performs inference without needing to retrain the model when running the code again. + In inference Mode, the script MUST NOT load any training data. If there are parameters generated from the training data that might be needed for inference on test data, please save them in the "models/" subfolder as well. If no test set is provided, reserve a portion of the data as your test set and save the generated test files in the models/ subfolder for use in submission and inference. Make sure that the required files, like submission.csv and scores.csv, are created without model training step through loading the saved model and test data file directly. + dump_model_eval: system: |- @@ -20,8 +26,9 @@ dump_model_eval: Focus on these aspects: - Check if the code saves the model in the "models/" subfolder. - Check if the code saves the test data in the "models/" subfolder when there is no test data specified. - - Ensure that when the code is rerun, it skips the training process and loads the model from the "models/" subfolder for direct inference. + - Ensure that when the code is rerun in inference mode, it skips the training process and loads the model from the "models/" subfolder for direct inference. - Verify that there is no training activity in the output. + - Verify that the script does not load the original training data. - Ensure that even if you skip the model training by loading saved models, the files like scores.csv and submission.csv are still correctly created. - The model's performance should remain consistent and not vary unreasonably between training and inference. @@ -42,6 +49,9 @@ dump_model_eval: ------------ The stdout from running the code ------------ {{stdout}} + ------------ File opened by the code ------------ + {{opened_trace_lines}} + ------------ The file list in "models/" subfolder ------------ {% for f in model_folder_files %} - {{ f }} diff --git a/rdagent/scenarios/data_science/scen/__init__.py b/rdagent/scenarios/data_science/scen/__init__.py index 28368ddf..ce1bd58d 100644 --- a/rdagent/scenarios/data_science/scen/__init__.py +++ b/rdagent/scenarios/data_science/scen/__init__.py @@ -19,7 +19,10 @@ from rdagent.scenarios.kaggle.kaggle_crawler import ( download_data, get_metric_direction, ) -from rdagent.scenarios.shared.get_runtime_info import get_runtime_environment_by_env +from rdagent.scenarios.shared.get_runtime_info import ( + check_runtime_environment, + get_runtime_environment_by_env, +) from rdagent.utils.agent.tpl import T @@ -28,6 +31,7 @@ class DataScienceScen(Scenario): def __init__(self, competition: str) -> None: + check_runtime_environment(get_ds_env()) # 1) prepare data if not Path(f"{DS_RD_SETTING.local_data_path}/{competition}").exists(): logger.error(f"Please prepare data for competition {competition} first.") diff --git a/rdagent/scenarios/data_science/scen/utils.py b/rdagent/scenarios/data_science/scen/utils.py index 8d563a42..719518ec 100644 --- a/rdagent/scenarios/data_science/scen/utils.py +++ b/rdagent/scenarios/data_science/scen/utils.py @@ -299,6 +299,7 @@ class FileTreeGenerator: max_lines: int = 200, priority_files: Set[str] = None, hide_base_name: bool = True, + allowed_paths: Set[Path] | None = None, ): """ Initialize the file tree generator. @@ -306,12 +307,33 @@ class FileTreeGenerator: Args: max_lines: Maximum output lines to prevent overly long output priority_files: File extensions to prioritize for display + hide_base_name: Hide the base name of the directory + allowed_paths: Set of allowed paths to include in the tree + """ self.max_lines = max_lines self.priority_files = priority_files or {".csv", ".json", ".parquet", ".md", ".txt"} self.lines = [] self.line_count = 0 self.hide_base_name = hide_base_name + self.allowed_paths = allowed_paths + self._lookup_set: Set[Path] | None = None + + def _build_lookup_set(self): + """ + Build the lookup set for allowed paths. + """ + if self.allowed_paths is None: + self._lookup_set = None + return + + self._lookup_set = set() + for path in self.allowed_paths: + self._lookup_set.add(path) + for parent in path.parents: + if str(parent) == ".": + continue + self._lookup_set.add(parent) def generate_tree(self, path: Union[str, Path]) -> str: """ @@ -327,6 +349,7 @@ class FileTreeGenerator: FileTreeGenerationError: If tree generation fails """ try: + self._build_lookup_set() path = Path(path) base_path = path.resolve() self.lines = [] @@ -408,6 +431,7 @@ class FileTreeGenerator: prefix: Prefix for tree formatting base_path: Base path for symlink detection + Raises: DirectoryPermissionError: If directory access is denied FileTreeGenerationError: If processing fails @@ -416,6 +440,11 @@ class FileTreeGenerator: try: # Get directory contents, filter out system files items = [p for p in path.iterdir() if not p.name.startswith(".") and p.name not in system_names] + + # Filter by allowed paths if provided + if self._lookup_set is not None: + items = [p for p in items if p in self._lookup_set] + dirs = sorted([p for p in items if p.is_dir()]) files = sorted([p for p in items if p.is_file()]) diff --git a/rdagent/scenarios/shared/get_runtime_info.py b/rdagent/scenarios/shared/get_runtime_info.py index 83d5836a..3e6cb7c6 100644 --- a/rdagent/scenarios/shared/get_runtime_info.py +++ b/rdagent/scenarios/shared/get_runtime_info.py @@ -10,3 +10,16 @@ def get_runtime_environment_by_env(env: Env) -> str: implementation.inject_files(**{fname: (Path(__file__).absolute().resolve().parent / "runtime_info.py").read_text()}) stdout = implementation.execute(env=env, entry=f"python {fname}") return stdout + + +def check_runtime_environment(env: Env) -> str: + implementation = FBWorkspace() + # 1) Check if strace exists in env + strace_check = implementation.execute(env=env, entry="which strace || echo MISSING").strip() + if strace_check.endswith("MISSING"): + raise RuntimeError("`strace` not found in the target environment.") + + # 2) Check if coverage module works in env + coverage_check = implementation.execute(env=env, entry="python -m coverage --version || echo MISSING").strip() + if coverage_check.endswith("MISSING"): + raise RuntimeError("`coverage` module not found or not runnable in the target environment.")