From f59e7bd486771125f7b9e01df819a0af209f3dab Mon Sep 17 00:00:00 2001 From: Tim Date: Mon, 7 Jul 2025 17:08:55 +0800 Subject: [PATCH] chore: sort trace by loop_id (#977) * last as sota to submit * chore: sort trace by loop_id * Update rdagent/log/ui/utils.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * reformat * update _log_path_hash_func * compare sota_exp_to_submit * fix ui storage * move sota_exp_to_submit setting to record(from direct_exp_gen) * save sota_exp_to_submit --------- Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> Co-authored-by: Bowen Xian --- rdagent/log/mle_summary.py | 13 +++++++------ rdagent/log/ui/storage.py | 8 ++++++-- rdagent/log/ui/utils.py | 18 ++++++++++++++---- rdagent/scenarios/data_science/loop.py | 8 +++++--- rdagent/scenarios/kaggle/kaggle_crawler.py | 1 + 5 files changed, 33 insertions(+), 15 deletions(-) diff --git a/rdagent/log/mle_summary.py b/rdagent/log/mle_summary.py index aa6232b9..3b170b5e 100644 --- a/rdagent/log/mle_summary.py +++ b/rdagent/log/mle_summary.py @@ -1,4 +1,5 @@ import pickle +import traceback from collections import defaultdict from pathlib import Path @@ -40,7 +41,7 @@ def save_grade_info(log_trace_path: Path): mle_score_str, tag=f"{msg.tag}.mle_score.pid", save_type="pkl", timestamp=msg.timestamp ) except Exception as e: - print(f"Error in {log_trace_path}: {e}") + print(f"Error in {log_trace_path}: {e}", traceback.format_exc()) def save_all_grade_info(log_folder): @@ -49,7 +50,7 @@ def save_all_grade_info(log_folder): try: save_grade_info(log_trace_path) except NoTestEvalError as e: - print(f"Error in {log_trace_path}: {e}") + print(f"Error in {log_trace_path}: {e}", traceback.format_exc()) def _get_loop_and_fn_after_hours(log_folder: Path, hours: int): @@ -108,11 +109,11 @@ def summarize_folder(log_folder: Path, hours: int | None = None): if hours: stop_li, stop_fn = _get_loop_and_fn_after_hours(log_trace_path, hours) - - for msg in FileStorage(log_trace_path).iter_msg(): # messages in log trace - loop_id, fn = extract_loopid_func_name(msg.tag) + msgs = [(msg, extract_loopid_func_name(msg.tag)) for msg in FileStorage(log_trace_path).iter_msg()] + msgs = [(msg, int(loop_id) if loop_id else loop_id, fn) for msg, (loop_id, fn) in msgs] + msgs.sort(key=lambda m: m[1] if m[1] else -1) # sort by loop id + for msg, loop_id, fn in msgs: # messages in log trace if loop_id: - loop_id = int(loop_id) loop_num = max(loop_id + 1, loop_num) if hours and loop_id == stop_li and fn == stop_fn: break diff --git a/rdagent/log/ui/storage.py b/rdagent/log/ui/storage.py index bff1c48b..de652fd6 100644 --- a/rdagent/log/ui/storage.py +++ b/rdagent/log/ui/storage.py @@ -292,8 +292,12 @@ class WebStorage(Storage): from rdagent.core.experiment import Experiment if isinstance(obj, Experiment): - if obj.result is not None: - result_str = obj.result.to_json() + try: + result = obj.result + except AttributeError: # compatibility with old versions + result = obj.__dict__["result"] + if result is not None: + result_str = result.to_json() data = { "id": id, "msg": { diff --git a/rdagent/log/ui/utils.py b/rdagent/log/ui/utils.py index 071cfe70..bb0594a1 100644 --- a/rdagent/log/ui/utils.py +++ b/rdagent/log/ui/utils.py @@ -163,10 +163,20 @@ def get_sota_exp_stat(log_path: Path): return None sota_loop_id = None - for i, ef in enumerate(final_trace.hist): - if ef[0] == sota_exp: - sota_loop_id = i - break + exp_paths = [ + (i, int(match[1])) + for i in log_path.rglob(f"*/running/*/*.pkl") + if (match := re.search(r".*Loop_(\d+).*", str(i))) + ] + if len(exp_paths) == 0: + return None + exp_paths.sort(key=lambda x: x[1], reverse=True) + for exp_path, loop_id in exp_paths: + with open(exp_path, "rb") as f: + trace = pickle.load(f) + if trace.experiment_workspace.all_codes == sota_exp.experiment_workspace.all_codes: + sota_loop_id = loop_id + break sota_mle_score_paths = [i for i in log_path.rglob(f"Loop_{sota_loop_id}/running/mle_score/**/*.pkl")] if len(sota_mle_score_paths) == 0: diff --git a/rdagent/scenarios/data_science/loop.py b/rdagent/scenarios/data_science/loop.py index f43cabfd..49f0538d 100644 --- a/rdagent/scenarios/data_science/loop.py +++ b/rdagent/scenarios/data_science/loop.py @@ -144,9 +144,6 @@ class DataScienceRDLoop(RDLoop): super(RDLoop, self).__init__() async def direct_exp_gen(self, prev_out: dict[str, Any]): - # set the SOTA experiment to submit - sota_exp_to_submit = self.sota_exp_selector.get_sota_exp_to_submit(self.trace) - self.trace.set_sota_exp_to_submit(sota_exp_to_submit) # set the checkpoint to start from selection = self.ckp_selector.get_selection(self.trace) @@ -275,6 +272,11 @@ class DataScienceRDLoop(RDLoop): logger.log_object(self.trace, tag="trace before restart") self.trace = DSTrace(scen=self.trace.scen, knowledge_base=self.trace.knowledge_base) + # set the SOTA experiment to submit + sota_exp_to_submit = self.sota_exp_selector.get_sota_exp_to_submit(self.trace) + self.trace.set_sota_exp_to_submit(sota_exp_to_submit) + logger.log_object(sota_exp_to_submit, tag="sota_exp_to_submit") + logger.log_object(self.trace, tag="trace") logger.log_object(self.trace.sota_experiment(), tag="SOTA experiment") diff --git a/rdagent/scenarios/kaggle/kaggle_crawler.py b/rdagent/scenarios/kaggle/kaggle_crawler.py index 94989577..0d6c04eb 100644 --- a/rdagent/scenarios/kaggle/kaggle_crawler.py +++ b/rdagent/scenarios/kaggle/kaggle_crawler.py @@ -221,6 +221,7 @@ def get_metric_direction(competition: str) -> bool: """ leaderboard = leaderboard_scores(competition) if float(leaderboard[0]) == float(leaderboard[-1]): + # This happens when top scores are all the same, such as 0 or 1. return float(leaderboard[0]) >= 0.5 return float(leaderboard[0]) > float(leaderboard[-1])