feat: summary and UI update (#581)

* summary and UI update

* fix CI

* fix

* fix CI

rdagent/log/mle_summary.py

@@ -69,6 +69,8 @@ def summarize_folder(log_folder: Path):
         valid_scores = {}
         success_loop_num = 0
 
+        sota_exp_stat = ""
+        grade_output = None
         for msg in FileStorage(log_trace_path).iter_msg():  # messages in log trace
             if msg.tag and "llm" not in msg.tag and "session" not in msg.tag:
                 if "competition" in msg.tag:
@@ -110,6 +112,20 @@ def summarize_folder(log_folder: Path):
                     if isinstance(msg.content, ExperimentFeedback) and bool(msg.content):
                         success_loop_num += 1
 
+                        if grade_output:  # sota exp's grade output
+                            if grade_output["gold_medal"]:
+                                sota_exp_stat = "gold"
+                            elif grade_output["silver_medal"]:
+                                sota_exp_stat = "silver"
+                            elif grade_output["bronze_medal"]:
+                                sota_exp_stat = "bronze"
+                            elif grade_output["above_median"]:
+                                sota_exp_stat = "above_median"
+                            elif grade_output["valid_submission"]:
+                                sota_exp_stat = "valid_submission"
+                            elif grade_output["submission_exists"]:
+                                sota_exp_stat = "made_submission"
+
         stat[log_trace_path.name].update(
             {
                 "loop_num": loop_num,
@@ -123,6 +139,7 @@ def summarize_folder(log_folder: Path):
                 "test_scores": test_scores,
                 "valid_scores": valid_scores,
                 "success_loop_num": success_loop_num,
+                "sota_exp_stat": sota_exp_stat,
             }
         )
     if (log_folder / "summary.pkl").exists():

rdagent/log/ui/dsapp.py

@@ -338,11 +338,13 @@ def all_summarize_win():
             "Successful Final Decision",
             "Made Submission",
             "Valid Submission",
+            "V/M",
             "Above Median",
             "Bronze",
             "Silver",
             "Gold",
             "Any Medal",
+            "SOTA Exp",
         ],
         index=summary.keys(),
     )
@@ -362,6 +364,10 @@ def all_summarize_win():
             base_df.loc[k, "Valid Submission"] = (
                 f"{v['valid_submission_num']} ({round(v['valid_submission_num'] / loop_num * 100, 2)}%)"
             )
+            if v["made_submission_num"] != 0:
+                base_df.loc[k, "V/M"] = f"{round(v['valid_submission_num'] / v['made_submission_num'] * 100, 2)}%"
+            else:
+                base_df.loc[k, "V/M"] = "N/A"
             base_df.loc[k, "Above Median"] = (
                 f"{v['above_median_num']} ({round(v['above_median_num'] / loop_num * 100, 2)}%)"
             )
@@ -370,6 +376,10 @@ def all_summarize_win():
             base_df.loc[k, "Gold"] = f"{v['gold_num']} ({round(v['gold_num'] / loop_num * 100, 2)}%)"
             base_df.loc[k, "Any Medal"] = f"{v['get_medal_num']} ({round(v['get_medal_num'] / loop_num * 100, 2)}%)"
 
+            if "sota_exp_stat" in v:
+                base_df.loc[k, "SOTA Exp"] = v["sota_exp_stat"]
+
+    base_df["SOTA Exp"].replace("", pd.NA, inplace=True)
     st.dataframe(base_df)
     total_stat = (
         (
@@ -390,7 +400,26 @@ def all_summarize_win():
         * 100
     )
     total_stat.name = "总体统计(%)"
-    st.dataframe(total_stat.round(2))
+
+    # SOTA Exp 统计
+    se_counts = base_df["SOTA Exp"].value_counts(dropna=True)
+    se_counts.loc["made_submission"] = se_counts.sum()
+    se_counts.loc["Any Medal"] = se_counts.get("gold", 0) + se_counts.get("silver", 0) + se_counts.get("bronze", 0)
+    se_counts.loc["above_median"] = se_counts.get("above_median", 0) + se_counts.get("Any Medal", 0)
+    se_counts.loc["valid_submission"] = se_counts.get("valid_submission", 0) + se_counts.get("above_median", 0)
+
+    sota_exp_stat = pd.Series(index=total_stat.index, dtype=int, name="SOTA Exp 统计(%)")
+    sota_exp_stat.loc["Made Submission"] = se_counts.get("made_submission", 0)
+    sota_exp_stat.loc["Valid Submission"] = se_counts.get("valid_submission", 0)
+    sota_exp_stat.loc["Above Median"] = se_counts.get("above_median", 0)
+    sota_exp_stat.loc["Bronze"] = se_counts.get("bronze", 0)
+    sota_exp_stat.loc["Silver"] = se_counts.get("silver", 0)
+    sota_exp_stat.loc["Gold"] = se_counts.get("gold", 0)
+    sota_exp_stat.loc["Any Medal"] = se_counts.get("Any Medal", 0)
+    sota_exp_stat = sota_exp_stat / base_df.shape[0] * 100
+
+    stat_df = pd.concat([total_stat, sota_exp_stat], axis=1)
+    st.dataframe(stat_df.round(2))
 
     # write curve
     for k, v in summary.items():