diff --git a/rdagent/log/mle_summary.py b/rdagent/log/mle_summary.py index 383c4393..43954185 100644 --- a/rdagent/log/mle_summary.py +++ b/rdagent/log/mle_summary.py @@ -8,6 +8,7 @@ import pandas as pd from rdagent.app.data_science.conf import DS_RD_SETTING from rdagent.log.storage import FileStorage +from rdagent.scenarios.data_science.experiment.experiment import DSExperiment from rdagent.utils.env import DockerEnv, MLEBDockerConf mle_de_conf = MLEBDockerConf() @@ -31,16 +32,18 @@ def save_grade_info(log_trace_path: Path): competition = msg.content if "running" in msg.tag: - msg.content.experiment_workspace.execute( - env=de, - entry=f"bash -c 'mlebench grade-sample submission.csv {competition} --data-dir /mle/data > mle_score.txt 2>&1'", - ) - msg.content.experiment_workspace.execute(env=de, entry="chmod 777 mle_score.txt") + if isinstance(msg.content, DSExperiment): + msg.content.experiment_workspace.execute( + env=de, + entry=f"bash -c 'mlebench grade-sample submission.csv {competition} --data-dir /mle/data > mle_score.txt 2>&1'", + ) + msg.content.experiment_workspace.execute(env=de, entry="chmod 777 mle_score.txt") def save_all_grade_info(log_folder): for log_trace_path in log_folder.iterdir(): - save_grade_info(log_trace_path) + if log_trace_path.is_dir(): + save_grade_info(log_trace_path) def summarize_folder(log_folder: Path): @@ -62,24 +65,26 @@ def summarize_folder(log_folder: Path): if "direct_exp_gen" in msg.tag: loop_num += 1 if "running" in msg.tag: - - submission_path = msg.content.experiment_workspace.workspace_path / "submission.csv" - if submission_path.exists(): - made_submission_num += 1 - scores_path = msg.content.experiment_workspace.workspace_path / "scores.csv" - valid_scores[loop_num - 1] = pd.read_csv(scores_path, index_col=0) - grade_output_path = msg.content.experiment_workspace.workspace_path / "mle_score.txt" - if not grade_output_path.exists(): - raise FileNotFoundError(f"mle_score.txt in {grade_output_path} not found, genarate it first!") - grade_output = extract_mle_json(grade_output_path.read_text()) - if grade_output["score"] is not None: - test_scores[loop_num - 1] = grade_output["score"] - if grade_output["any_medal"]: - medal = ( - "gold" - if grade_output["gold_medal"] - else "silver" if grade_output["silver_medal"] else "bronze" + if isinstance(msg.content, DSExperiment): + submission_path = msg.content.experiment_workspace.workspace_path / "submission.csv" + if submission_path.exists(): + made_submission_num += 1 + scores_path = msg.content.experiment_workspace.workspace_path / "scores.csv" + valid_scores[loop_num - 1] = pd.read_csv(scores_path, index_col=0) + grade_output_path = msg.content.experiment_workspace.workspace_path / "mle_score.txt" + if not grade_output_path.exists(): + raise FileNotFoundError( + f"mle_score.txt in {grade_output_path} not found, genarate it first!" ) + grade_output = extract_mle_json(grade_output_path.read_text()) + if grade_output["score"] is not None: + test_scores[loop_num - 1] = grade_output["score"] + if grade_output["any_medal"]: + medal = ( + "gold" + if grade_output["gold_medal"] + else "silver" if grade_output["silver_medal"] else "bronze" + ) if "feedback" in msg.tag and "evolving" not in msg.tag: if bool(msg.content): diff --git a/rdagent/utils/workflow.py b/rdagent/utils/workflow.py index 11305916..c8a9156e 100644 --- a/rdagent/utils/workflow.py +++ b/rdagent/utils/workflow.py @@ -109,8 +109,8 @@ class LoopBase: step_n -= 1 li, si = self.loop_idx, self.step_idx - name = self.steps[si] + logger.info(f"Start Loop {li}, Step {si}: {name}") with logger.tag(f"Loop_{li}.{name}"): start = datetime.datetime.now(datetime.timezone.utc) func = getattr(self, name)