gavindiaz/NexQuant
1
0
Fork 0
mirror of https://github.com/NicolasBohn/NexQuant.git synced 2026-07-28 16:07:46 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: add the shape of the CSV to the dataset description (#561)

Browse Source 
* add csv's shape info to data description

* fix CI
This commit is contained in:
Yuante Li
2025-02-07 18:12:18 +08:00
committed by GitHub
parent 98a0801a32
commit 732839e4bd
1 changed files with 6 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
rdagent/scenarios/data_science/scen/__init__.py
+6 -5
View File
@@ -97,7 +97,6 @@ def describe_data_folder(folder_path, indent=0, max_files=2, partial_expand_subf
dirs.sort()
if not dirs:
for file in files:
print(file)
file_path = os.path.join(root, file)
file_type = os.path.splitext(file)[1][1:]
file_size = os.path.getsize(file_path)
@@ -186,14 +185,16 @@ def describe_data_folder(folder_path, indent=0, max_files=2, partial_expand_subf
result.append(" " * (indent + 2) + "... (file limit reached)")
else:
for file, size, path in files_details[file_type]:
result.append(" " * indent + f"- {file} ({size} bytes)")
if file_type == "csv":
df = pd.read_csv(path)
result.append(
" " * indent + f"- {file} ({size} bytes, with {df.shape[0]} rows and {df.shape[1]} columns)"
)
result.append(" " * (indent + 2) + f"- Head of {file}:")
csv_head = read_csv_head(path, indent + 4)
# if len(csv_head) > 300:
# csv_head = " ".join(csv_head.strip().split())
# csv_head = csv_head[:300] + "\n" + " " * (indent + 4) + "... (truncated)"
result.append(csv_head)
continue
result.append(" " * indent + f"- {file} ({size} bytes)")
if file_type == "md":
result.append(" " * (indent + 2) + f"- Content of {file}:")
if file == "description.md":