gavindiaz/NexQuant
1
0
Fork 0
mirror of https://github.com/NicolasBohn/NexQuant.git synced 2026-07-28 07:57:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: archive python and csv files in workspace to maintain results (#814)

Browse Source 
* archive workspace also

* remove non python csv and md files in workspace to avoid big workspace dump

* FIX ci
This commit is contained in:
Xu Yang
2025-04-21 02:34:29 -06:00
committed by GitHub
parent 19bb2e740e
commit 4509d92ea7
1 changed files with 32 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files
rdagent/app/data_science/loop.py
+32 -6
View File
@@ -22,6 +22,7 @@ from rdagent.components.coder.data_science.workflow import WorkflowCoSTEER
from rdagent.components.coder.data_science.workflow.exp import WorkflowTask
from rdagent.components.workflow.conf import BasePropSetting
from rdagent.components.workflow.rd_loop import RDLoop
from rdagent.core.conf import RD_AGENT_SETTINGS
from rdagent.core.exception import CoderError, RunnerError
from rdagent.core.proposal import ExperimentFeedback
from rdagent.core.scenario import Scenario
@@ -175,8 +176,8 @@ class DataScienceRDLoop(RDLoop):
and Path(DS_RD_SETTING.log_archive_path).is_dir()
):
start_archive_datetime = datetime.now()
logger.info(f"Archiving log folder after loop {self.loop_idx}")
tar_path = (
logger.info(f"Archiving log and workspace folder after loop {self.loop_idx}")
mid_log_tar_path = (
Path(
DS_RD_SETTING.log_archive_temp_path
if DS_RD_SETTING.log_archive_temp_path
@@ -184,12 +185,37 @@ class DataScienceRDLoop(RDLoop):
)
/ "mid_log.tar"
)
subprocess.run(["tar", "-cf", str(tar_path), "-C", (Path().cwd() / "log"), "."], check=True)
mid_workspace_tar_path = (
Path(
DS_RD_SETTING.log_archive_temp_path
if DS_RD_SETTING.log_archive_temp_path
else DS_RD_SETTING.log_archive_path
)
/ "mid_workspace.tar"
)
subprocess.run(["tar", "-cf", str(mid_log_tar_path), "-C", (Path().cwd() / "log"), "."], check=True)
# remove all files and folders in the workspace except for .py, .md, and .csv files to avoid large workspace dump
for workspace_id in Path(RD_AGENT_SETTINGS.workspace_path).iterdir():
for file_and_folder in workspace_id.iterdir():
if file_and_folder.is_dir():
shutil.rmtree(file_and_folder)
elif file_and_folder.is_file() and file_and_folder.suffix not in [".py", ".md", ".csv"]:
file_and_folder.unlink()
subprocess.run(
["tar", "-cf", str(mid_workspace_tar_path), "-C", (RD_AGENT_SETTINGS.workspace_path), "."], check=True
)
if DS_RD_SETTING.log_archive_temp_path is not None:
shutil.move(tar_path, Path(DS_RD_SETTING.log_archive_path) / "mid_log.tar")
tar_path = Path(DS_RD_SETTING.log_archive_path) / "mid_log.tar"
shutil.move(mid_log_tar_path, Path(DS_RD_SETTING.log_archive_path) / "mid_log.tar")
mid_log_tar_path = Path(DS_RD_SETTING.log_archive_path) / "mid_log.tar"
shutil.move(mid_workspace_tar_path, Path(DS_RD_SETTING.log_archive_path) / "mid_workspace.tar")
mid_workspace_tar_path = Path(DS_RD_SETTING.log_archive_path) / "mid_workspace.tar"
shutil.copy(
tar_path, Path(DS_RD_SETTING.log_archive_path) / "mid_log_bak.tar"
mid_log_tar_path, Path(DS_RD_SETTING.log_archive_path) / "mid_log_bak.tar"
) # backup when upper code line is killed when running
shutil.copy(
mid_workspace_tar_path, Path(DS_RD_SETTING.log_archive_path) / "mid_workspace_bak.tar"
) # backup when upper code line is killed when running
self.timer.add_duration(datetime.now() - start_archive_datetime)