chore: sort trace by loop_id (#977)

* last as sota to submit

* chore: sort trace by loop_id

* Update rdagent/log/ui/utils.py

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

* reformat

* update _log_path_hash_func

* compare sota_exp_to_submit

* fix ui storage

* move sota_exp_to_submit setting to record(from direct_exp_gen)

* save sota_exp_to_submit

---------

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
Co-authored-by: Bowen Xian <xianbowen@outlook.com>

rdagent/log/mle_summary.py

@@ -1,4 +1,5 @@
 import pickle
+import traceback
 from collections import defaultdict
 from pathlib import Path
 
@@ -40,7 +41,7 @@ def save_grade_info(log_trace_path: Path):
                         mle_score_str, tag=f"{msg.tag}.mle_score.pid", save_type="pkl", timestamp=msg.timestamp
                     )
                 except Exception as e:
-                    print(f"Error in {log_trace_path}: {e}")
+                    print(f"Error in {log_trace_path}: {e}", traceback.format_exc())
 
 
 def save_all_grade_info(log_folder):
@@ -49,7 +50,7 @@ def save_all_grade_info(log_folder):
             try:
                 save_grade_info(log_trace_path)
             except NoTestEvalError as e:
-                print(f"Error in {log_trace_path}: {e}")
+                print(f"Error in {log_trace_path}: {e}", traceback.format_exc())
 
 
 def _get_loop_and_fn_after_hours(log_folder: Path, hours: int):
@@ -108,11 +109,11 @@ def summarize_folder(log_folder: Path, hours: int | None = None):
 
         if hours:
             stop_li, stop_fn = _get_loop_and_fn_after_hours(log_trace_path, hours)
-
-        for msg in FileStorage(log_trace_path).iter_msg():  # messages in log trace
-            loop_id, fn = extract_loopid_func_name(msg.tag)
+        msgs = [(msg, extract_loopid_func_name(msg.tag)) for msg in FileStorage(log_trace_path).iter_msg()]
+        msgs = [(msg, int(loop_id) if loop_id else loop_id, fn) for msg, (loop_id, fn) in msgs]
+        msgs.sort(key=lambda m: m[1] if m[1] else -1)  # sort by loop id
+        for msg, loop_id, fn in msgs:  # messages in log trace
             if loop_id:
-                loop_id = int(loop_id)
                 loop_num = max(loop_id + 1, loop_num)
             if hours and loop_id == stop_li and fn == stop_fn:
                 break

rdagent/log/ui/storage.py

@@ -292,8 +292,12 @@ class WebStorage(Storage):
             from rdagent.core.experiment import Experiment
 
             if isinstance(obj, Experiment):
-                if obj.result is not None:
-                    result_str = obj.result.to_json()
+                try:
+                    result = obj.result
+                except AttributeError:  # compatibility with old versions
+                    result = obj.__dict__["result"]
+                if result is not None:
+                    result_str = result.to_json()
                     data = {
                         "id": id,
                         "msg": {

rdagent/log/ui/utils.py

@@ -163,10 +163,20 @@ def get_sota_exp_stat(log_path: Path):
         return None
 
     sota_loop_id = None
-    for i, ef in enumerate(final_trace.hist):
-        if ef[0] == sota_exp:
-            sota_loop_id = i
-            break
+    exp_paths = [
+        (i, int(match[1]))
+        for i in log_path.rglob(f"*/running/*/*.pkl")
+        if (match := re.search(r".*Loop_(\d+).*", str(i)))
+    ]
+    if len(exp_paths) == 0:
+        return None
+    exp_paths.sort(key=lambda x: x[1], reverse=True)
+    for exp_path, loop_id in exp_paths:
+        with open(exp_path, "rb") as f:
+            trace = pickle.load(f)
+            if trace.experiment_workspace.all_codes == sota_exp.experiment_workspace.all_codes:
+                sota_loop_id = loop_id
+                break
 
     sota_mle_score_paths = [i for i in log_path.rglob(f"Loop_{sota_loop_id}/running/mle_score/**/*.pkl")]
     if len(sota_mle_score_paths) == 0:

rdagent/scenarios/data_science/loop.py

@@ -144,9 +144,6 @@ class DataScienceRDLoop(RDLoop):
         super(RDLoop, self).__init__()
 
     async def direct_exp_gen(self, prev_out: dict[str, Any]):
-        # set the SOTA experiment to submit
-        sota_exp_to_submit = self.sota_exp_selector.get_sota_exp_to_submit(self.trace)
-        self.trace.set_sota_exp_to_submit(sota_exp_to_submit)
 
         # set the checkpoint to start from
         selection = self.ckp_selector.get_selection(self.trace)
@@ -275,6 +272,11 @@ class DataScienceRDLoop(RDLoop):
                         logger.log_object(self.trace, tag="trace before restart")
                         self.trace = DSTrace(scen=self.trace.scen, knowledge_base=self.trace.knowledge_base)
 
+        # set the SOTA experiment to submit
+        sota_exp_to_submit = self.sota_exp_selector.get_sota_exp_to_submit(self.trace)
+        self.trace.set_sota_exp_to_submit(sota_exp_to_submit)
+        logger.log_object(sota_exp_to_submit, tag="sota_exp_to_submit")
+
         logger.log_object(self.trace, tag="trace")
         logger.log_object(self.trace.sota_experiment(), tag="SOTA experiment")
 

rdagent/scenarios/kaggle/kaggle_crawler.py

@@ -221,6 +221,7 @@ def get_metric_direction(competition: str) -> bool:
     """
     leaderboard = leaderboard_scores(competition)
     if float(leaderboard[0]) == float(leaderboard[-1]):
+        # This happens when top scores are all the same, such as 0 or 1.
         return float(leaderboard[0]) >= 0.5
     return float(leaderboard[0]) > float(leaderboard[-1])