fix: filter system metadata dirs and init missing DSTrace attribute (#946)

* fix: filter system metadata dirs and init missing DSTrace attribute

* style:pre-commit

rdagent/scenarios/data_science/proposal/exp_gen/base.py

@@ -47,7 +47,6 @@ class DSHypothesis(Hypothesis):
 
 
 class DSTrace(Trace[DataScienceScen, KnowledgeBase]):
-
     def __init__(self, scen: DataScienceScen, knowledge_base: KnowledgeBase | None = None) -> None:
         self.scen: DataScienceScen = scen
         self.hist: list[tuple[DSExperiment, ExperimentFeedback]] = []
@@ -60,6 +59,7 @@ class DSTrace(Trace[DataScienceScen, KnowledgeBase]):
         # () represents no parent; (1,) presents one parent; (1, 2) represents two parents.
 
         self.knowledge_base = knowledge_base
+        self.current_selection: tuple[int, ...] = (-1,)
 
         self.sota_exp_to_submit: DSExperiment | None = None  # grab the global best exp to submit
 
@@ -214,7 +214,6 @@ class DSTrace(Trace[DataScienceScen, KnowledgeBase]):
         selection: tuple[int, ...] | None = None,
     ) -> tuple[DSExperiment, ExperimentFeedback] | None:
         """
-
         Returns
         -------
         Experiment or None

rdagent/scenarios/data_science/scen/utils.py

@@ -226,6 +226,8 @@ from pandas.api.types import is_numeric_dtype
 code_files = {".py", ".sh", ".yaml", ".yml", ".md", ".html", ".xml", ".log", ".rst"}
 # we treat these files as text (rather than binary) files
 plaintext_files = {".txt", ".csv", ".json", ".tsv"} | code_files
+# system-generated directories/files to filter out
+system_names = {"__MACOSX", ".DS_Store", "Thumbs.db"}
 
 
 def get_file_len_size(f: Path) -> tuple[int, str]:
@@ -245,7 +247,7 @@ def file_tree(path: Path, depth=0) -> str:
     """Generate a tree structure of files in a directory"""
     result = []
 
-    files = [p for p in Path(path).iterdir() if not p.is_dir()]
+    files = [p for p in Path(path).iterdir() if not p.is_dir() and p.name not in system_names]
 
     max_n = 4 if len(files) > 30 else 8
     for p in sorted(files)[:max_n]:
@@ -253,7 +255,11 @@ def file_tree(path: Path, depth=0) -> str:
     if len(files) > max_n:
         result.append(f"{' '*depth*4}... and {len(files)-max_n} other files")
 
-    dirs = [p for p in Path(path).iterdir() if p.is_dir() or (p.is_symlink() and p.resolve().is_dir())]
+    dirs = [
+        p
+        for p in Path(path).iterdir()
+        if (p.is_dir() or (p.is_symlink() and p.resolve().is_dir())) and p.name not in system_names
+    ]
 
     # Calculate base_path (the top-level resolved absolute directory)
     base_path = Path(path).resolve()
@@ -283,6 +289,10 @@ def file_tree(path: Path, depth=0) -> str:
 def _walk(path: Path):
     """Recursively walk a directory (analogous to os.walk but for pathlib.Path)"""
     for p in sorted(Path(path).iterdir()):
+        # Filter out system-generated directories/files
+        if p.name in system_names:
+            continue
+
         if p.is_dir():
             # If this is a symlinked dir to a parent/ancestor, do not expand it
             if p.is_symlink():