From 73cfa26d33f18e692e43a649492d344f4087c32f Mon Sep 17 00:00:00 2001 From: Tim Date: Mon, 17 Feb 2025 18:44:26 +0800 Subject: [PATCH] print head for txt files and shuffle when sample (#603) --- rdagent/scenarios/data_science/debug/data.py | 2 ++ rdagent/scenarios/data_science/scen/__init__.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/rdagent/scenarios/data_science/debug/data.py b/rdagent/scenarios/data_science/debug/data.py index 59e495ad..d5ea6279 100644 --- a/rdagent/scenarios/data_science/debug/data.py +++ b/rdagent/scenarios/data_science/debug/data.py @@ -4,6 +4,7 @@ import shutil from collections import Counter, defaultdict from pathlib import Path +import numpy as np import pandas as pd from tqdm import tqdm @@ -287,6 +288,7 @@ def create_debug_data( # Use a greedy strategy to select groups so that the total number of files is as close as possible to num_to_keep total_files = 0 + np.random.shuffle(not_used_files) for nf in not_used_files: if total_files > num_to_keep: break diff --git a/rdagent/scenarios/data_science/scen/__init__.py b/rdagent/scenarios/data_science/scen/__init__.py index 25273e4f..f53c3c1e 100644 --- a/rdagent/scenarios/data_science/scen/__init__.py +++ b/rdagent/scenarios/data_science/scen/__init__.py @@ -179,7 +179,7 @@ def describe_data_folder(folder_path, indent=0, max_files=2, partial_expand_subf # Print the folder and its contents for file_type, count in files_count.items(): - if count > max_files and file_type not in ["csv", "md"]: + if count > max_files and file_type not in ["csv", "md", "txt"]: result.append(" " * indent + f"{count} {file_type}s:") for file, size, path in files_details[file_type]: result.append(" " * (indent + 2) + f"- {file} ({size} bytes)")