diff --git a/rdagent/scenarios/data_science/proposal/exp_gen/base.py b/rdagent/scenarios/data_science/proposal/exp_gen/base.py index 8a42a136..a44b8e1b 100644 --- a/rdagent/scenarios/data_science/proposal/exp_gen/base.py +++ b/rdagent/scenarios/data_science/proposal/exp_gen/base.py @@ -47,7 +47,6 @@ class DSHypothesis(Hypothesis): class DSTrace(Trace[DataScienceScen, KnowledgeBase]): - def __init__(self, scen: DataScienceScen, knowledge_base: KnowledgeBase | None = None) -> None: self.scen: DataScienceScen = scen self.hist: list[tuple[DSExperiment, ExperimentFeedback]] = [] @@ -60,6 +59,7 @@ class DSTrace(Trace[DataScienceScen, KnowledgeBase]): # () represents no parent; (1,) presents one parent; (1, 2) represents two parents. self.knowledge_base = knowledge_base + self.current_selection: tuple[int, ...] = (-1,) self.sota_exp_to_submit: DSExperiment | None = None # grab the global best exp to submit @@ -214,7 +214,6 @@ class DSTrace(Trace[DataScienceScen, KnowledgeBase]): selection: tuple[int, ...] | None = None, ) -> tuple[DSExperiment, ExperimentFeedback] | None: """ - Returns ------- Experiment or None diff --git a/rdagent/scenarios/data_science/scen/utils.py b/rdagent/scenarios/data_science/scen/utils.py index 893ef1f7..3113dcdb 100644 --- a/rdagent/scenarios/data_science/scen/utils.py +++ b/rdagent/scenarios/data_science/scen/utils.py @@ -226,6 +226,8 @@ from pandas.api.types import is_numeric_dtype code_files = {".py", ".sh", ".yaml", ".yml", ".md", ".html", ".xml", ".log", ".rst"} # we treat these files as text (rather than binary) files plaintext_files = {".txt", ".csv", ".json", ".tsv"} | code_files +# system-generated directories/files to filter out +system_names = {"__MACOSX", ".DS_Store", "Thumbs.db"} def get_file_len_size(f: Path) -> tuple[int, str]: @@ -245,7 +247,7 @@ def file_tree(path: Path, depth=0) -> str: """Generate a tree structure of files in a directory""" result = [] - files = [p for p in Path(path).iterdir() if not p.is_dir()] + files = [p for p in Path(path).iterdir() if not p.is_dir() and p.name not in system_names] max_n = 4 if len(files) > 30 else 8 for p in sorted(files)[:max_n]: @@ -253,7 +255,11 @@ def file_tree(path: Path, depth=0) -> str: if len(files) > max_n: result.append(f"{' '*depth*4}... and {len(files)-max_n} other files") - dirs = [p for p in Path(path).iterdir() if p.is_dir() or (p.is_symlink() and p.resolve().is_dir())] + dirs = [ + p + for p in Path(path).iterdir() + if (p.is_dir() or (p.is_symlink() and p.resolve().is_dir())) and p.name not in system_names + ] # Calculate base_path (the top-level resolved absolute directory) base_path = Path(path).resolve() @@ -283,6 +289,10 @@ def file_tree(path: Path, depth=0) -> str: def _walk(path: Path): """Recursively walk a directory (analogous to os.walk but for pathlib.Path)""" for p in sorted(Path(path).iterdir()): + # Filter out system-generated directories/files + if p.name in system_names: + continue + if p.is_dir(): # If this is a symlinked dir to a parent/ancestor, do not expand it if p.is_symlink():