From 43703d45d24a92c2d6bc3caba3baff1654aaf2ae Mon Sep 17 00:00:00 2001
From: Yuante Li <104308117+WinstonLiyt@users.noreply.github.com>
Date: Wed, 12 Feb 2025 18:57:08 +0800
Subject: [PATCH] feat: add baseline score stat (#590)

* add baseline score stat

* fix ci

* fix

* fix
---
 rdagent/log/mle_summary.py | 16 ++++++++++++++++
 rdagent/log/ui/dsapp.py    | 32 ++++++++++++++++++++++++++++++--
 2 files changed, 46 insertions(+), 2 deletions(-)

diff --git a/rdagent/log/mle_summary.py b/rdagent/log/mle_summary.py
index 6f576566..ae1d5814 100644
--- a/rdagent/log/mle_summary.py
+++ b/rdagent/log/mle_summary.py
@@ -67,9 +67,14 @@ def summarize_folder(log_folder: Path):
         gold_num = 0
         test_scores = {}
         valid_scores = {}
+        bronze_threshold = 0.0
+        silver_threshold = 0.0
+        gold_threshold = 0.0
+        median_threshold = 0.0
         success_loop_num = 0
 
         sota_exp_stat = ""
+        sota_exp_score = None
         grade_output = None
         for msg in FileStorage(log_trace_path).iter_msg():  # messages in log trace
             if msg.tag and "llm" not in msg.tag and "session" not in msg.tag:
@@ -107,6 +112,10 @@ def summarize_folder(log_folder: Path):
                                     silver_num += 1
                                 if grade_output["gold_medal"]:
                                     gold_num += 1
+                                bronze_threshold = grade_output["bronze_threshold"]
+                                silver_threshold = grade_output["silver_threshold"]
+                                gold_threshold = grade_output["gold_threshold"]
+                                median_threshold = grade_output["median_threshold"]
 
                 if "feedback" in msg.tag and "evolving" not in msg.tag:
                     if isinstance(msg.content, ExperimentFeedback) and bool(msg.content):
@@ -125,6 +134,8 @@ def summarize_folder(log_folder: Path):
                                 sota_exp_stat = "valid_submission"
                             elif grade_output["submission_exists"]:
                                 sota_exp_stat = "made_submission"
+                            if grade_output["score"] is not None:
+                                sota_exp_score = grade_output["score"]
 
         stat[log_trace_path.name].update(
             {
@@ -140,6 +151,11 @@ def summarize_folder(log_folder: Path):
                 "valid_scores": valid_scores,
                 "success_loop_num": success_loop_num,
                 "sota_exp_stat": sota_exp_stat,
+                "sota_exp_score": sota_exp_score,
+                "bronze_threshold": bronze_threshold,
+                "silver_threshold": silver_threshold,
+                "gold_threshold": gold_threshold,
+                "median_threshold": median_threshold,
             }
         )
     if (log_folder / "summary.pkl").exists():
diff --git a/rdagent/log/ui/dsapp.py b/rdagent/log/ui/dsapp.py
index e9ae8566..a14bd048 100644
--- a/rdagent/log/ui/dsapp.py
+++ b/rdagent/log/ui/dsapp.py
@@ -345,9 +345,22 @@ def all_summarize_win():
             "Gold",
             "Any Medal",
             "SOTA Exp",
+            "Ours - Base",
+            "SOTA Exp Score",
+            "Baseline Score",
+            "Bronze Threshold",
+            "Silver Threshold",
+            "Gold Threshold",
+            "Medium Threshold",
         ],
         index=summary.keys(),
     )
+
+    # Read baseline results
+    baseline_result_path = ""
+    if Path(baseline_result_path).exists():
+        baseline_df = pd.read_csv(baseline_result_path)
+
     for k, v in summary.items():
         loop_num = v["loop_num"]
         base_df.loc[k, "Competition"] = v["competition"]
@@ -376,8 +389,23 @@ def all_summarize_win():
             base_df.loc[k, "Gold"] = f"{v['gold_num']} ({round(v['gold_num'] / loop_num * 100, 2)}%)"
             base_df.loc[k, "Any Medal"] = f"{v['get_medal_num']} ({round(v['get_medal_num'] / loop_num * 100, 2)}%)"
 
-            if "sota_exp_stat" in v:
-                base_df.loc[k, "SOTA Exp"] = v["sota_exp_stat"]
+            baseline_score = None
+            if Path(baseline_result_path).exists():
+                baseline_score = baseline_df.loc[baseline_df["competition_id"] == v["competition"], "score"].item()
+
+            base_df.loc[k, "SOTA Exp"] = v.get("sota_exp_stat", None)
+            if (
+                baseline_score is not None
+                and not pd.isna(baseline_score)
+                and not pd.isna(v.get("sota_exp_score", None))
+            ):
+                base_df.loc[k, "Ours - Base"] = v.get("sota_exp_score", 0.0) - baseline_score
+            base_df.loc[k, "SOTA Exp Score"] = v.get("sota_exp_score", None)
+            base_df.loc[k, "Baseline Score"] = baseline_score
+            base_df.loc[k, "Bronze Threshold"] = v.get("bronze_threshold", None)
+            base_df.loc[k, "Silver Threshold"] = v.get("silver_threshold", None)
+            base_df.loc[k, "Gold Threshold"] = v.get("gold_threshold", None)
+            base_df.loc[k, "Medium Threshold"] = v.get("median_threshold", None)
 
     base_df["SOTA Exp"].replace("", pd.NA, inplace=True)
     st.dataframe(base_df)