diff --git a/rdagent/log/ui/ds_summary.py b/rdagent/log/ui/ds_summary.py index 7df2e67b..a865c5bb 100755 --- a/rdagent/log/ui/ds_summary.py +++ b/rdagent/log/ui/ds_summary.py @@ -18,6 +18,7 @@ from rdagent.log.ui.utils import ( curve_figure, get_statistics_df, get_summary_df, + lite_curve_figure, percent_df, ) from rdagent.scenarios.kaggle.kaggle_crawler import get_metric_direction @@ -47,42 +48,47 @@ def days_summarize_win(): def curves_win(summary: dict): - for k, v in summary.items(): - with st.container(border=True): - st.markdown(f"**:blue[{k}] - :violet[{v['competition']}]**") - try: - tscores = {k: v for k, v in v["test_scores"].items()} - tscores = pd.Series(tscores) - vscores = {} - for k, vs in v["valid_scores"].items(): - if not vs.index.is_unique: - st.warning( - f"Loop {k}'s valid scores index are not unique, only the last one will be kept to show." - ) - st.write(vs) - vscores[k] = vs[~vs.index.duplicated(keep="last")].iloc[:, 0] - if len(vscores) > 0: - metric_name = list(vscores.values())[0].name - else: - metric_name = "None" - vscores = pd.DataFrame(vscores) - if "ensemble" in vscores.index: - ensemble_row = vscores.loc[["ensemble"]] - vscores = pd.concat([ensemble_row, vscores.drop("ensemble")]) - vscores = vscores.T - vscores["test"] = tscores - vscores.index = [f"L{i}" for i in vscores.index] - vscores.columns.name = metric_name + # draw curves + cbwin1, cbwin2 = st.columns(2) + if cbwin1.toggle("Show Curves", key="show_curves"): + for k, v in summary.items(): + with st.container(border=True): + st.markdown(f"**:blue[{k}] - :violet[{v['competition']}]**") + try: + tscores = {k: v for k, v in v["test_scores"].items()} + tscores = pd.Series(tscores) + vscores = {} + for k, vs in v["valid_scores"].items(): + if not vs.index.is_unique: + st.warning( + f"Loop {k}'s valid scores index are not unique, only the last one will be kept to show." + ) + st.write(vs) + vscores[k] = vs[~vs.index.duplicated(keep="last")].iloc[:, 0] + if len(vscores) > 0: + metric_name = list(vscores.values())[0].name + else: + metric_name = "None" + vscores = pd.DataFrame(vscores) + if "ensemble" in vscores.index: + ensemble_row = vscores.loc[["ensemble"]] + vscores = pd.concat([ensemble_row, vscores.drop("ensemble")]) + vscores = vscores.T + vscores["test"] = tscores + vscores.index = [f"L{i}" for i in vscores.index] + vscores.columns.name = metric_name - st.plotly_chart(curve_figure(vscores)) - except Exception as e: - import traceback + st.plotly_chart(curve_figure(vscores)) + except Exception as e: + import traceback - st.markdown("- Error: " + str(e)) - st.code(traceback.format_exc()) - st.markdown("- Valid Scores: ") - # st.write({k: type(v) for k, v in v["valid_scores"].items()}) - st.json(v["valid_scores"]) + st.markdown("- Error: " + str(e)) + st.code(traceback.format_exc()) + st.markdown("- Valid Scores: ") + # st.write({k: type(v) for k, v in v["valid_scores"].items()}) + st.json(v["valid_scores"]) + if cbwin2.toggle("Show Curves (Lite)", key="show_curves_lite"): + st.pyplot(lite_curve_figure(summary)) def all_summarize_win(): @@ -233,8 +239,7 @@ def all_summarize_win(): # write curve st.subheader("Curves", divider="rainbow") - if st.toggle("Show Curves", key="show_curves"): - curves_win(summary) + curves_win(summary) with st.container(border=True): diff --git a/rdagent/log/ui/utils.py b/rdagent/log/ui/utils.py index 361523df..2482d974 100644 --- a/rdagent/log/ui/utils.py +++ b/rdagent/log/ui/utils.py @@ -10,6 +10,7 @@ import networkx as nx import pandas as pd import plotly.graph_objects as go import typer +from matplotlib import pyplot as plt from rdagent.app.data_science.loop import DataScienceRDLoop from rdagent.core.proposal import Trace @@ -607,6 +608,72 @@ def curve_figure(scores: pd.DataFrame) -> go.Figure: return fig +def lite_curve_figure(summary): + cols = 3 # 每行几个图，可调整 + rows = math.ceil(len(summary) / cols) + + fig, axes = plt.subplots(rows, cols, figsize=(6 * cols, 4.5 * rows), squeeze=False) + axes = axes.flatten() # 💡 扁平化 axes 结构，确保 ax.plot 不报错 + colors = {"Bronze": "#cd7f32", "Silver": "#c0c0c0", "Gold": "#ffd700", "Median": "gray"} + + for idx, competition in enumerate(summary.keys()): + data = summary[competition] + test_scores_df = pd.DataFrame.from_dict(data["test_scores"], orient="index", columns=["Test Score"]) + test_scores_df.index.name = "Loop" + valid_scores_dict = data["valid_scores"] + + # 提取 ensemble 验证分数 + ensemble_scores = {} + for loop_id, df in valid_scores_dict.items(): + if "ensemble" in df.index: + ensemble_scores[loop_id] = df.loc["ensemble"].iloc[0] + + ensemble_valid_df = pd.DataFrame.from_dict(ensemble_scores, orient="index", columns=["Ensemble Valid Score"]) + ensemble_valid_df.index.name = "Loop" + + combined_df = pd.merge(ensemble_valid_df, test_scores_df, left_index=True, right_index=True, how="outer") + combined_df.sort_index(inplace=True) + + bronze_threshold = data["bronze_threshold"] + silver_threshold = data["silver_threshold"] + gold_threshold = data["gold_threshold"] + sota_loop_id = data["sota_loop_id_new"] + + # 当前 subplot + ax = axes[idx] + ax.plot(combined_df.index, combined_df["Ensemble Valid Score"], marker="o", markersize=4, label="Valid Score") + ax.plot(combined_df.index, combined_df["Test Score"], marker="s", markersize=4, label="Test Score") + ax.axhline(y=bronze_threshold, color=colors["Bronze"], linestyle="--", linewidth=2) + ax.axhline(y=silver_threshold, color=colors["Silver"], linestyle="--", linewidth=2) + ax.axhline(y=gold_threshold, color=colors["Gold"], linestyle="--", linewidth=2) + + # 标记 SOTA loop + if sota_loop_id is not None and sota_loop_id in combined_df.index: + ax.axvline(x=sota_loop_id, color="red", linestyle=":", linewidth=2, alpha=0.7) + # 添加文本标注 + ax.text( + sota_loop_id, + ax.get_ylim()[1] * 0.95, + f"L{sota_loop_id}", + ha="center", + va="top", + bbox=dict(boxstyle="round,pad=0.3", facecolor="red", alpha=0.3), + ) + + ax.set_title(f"{competition}") + ax.set_xlabel("Loop") + ax.set_ylabel("Score") + ax.grid(True) + ax.legend() + + # 删除多余 subplot（如果有） + for j in range(len(summary), len(axes)): + fig.delaxes(axes[j]) + + plt.tight_layout() + return fig + + def trace_figure(trace: Trace, merge_loops: list = []): G = nx.DiGraph()