diff --git a/rdagent/log/ui/ds_summary.py b/rdagent/log/ui/ds_summary.py index 3b03b117..5ee9317a 100755 --- a/rdagent/log/ui/ds_summary.py +++ b/rdagent/log/ui/ds_summary.py @@ -1,3 +1,7 @@ +""" +Please refer to rdagent/log/ui/utils.py:get_summary_df for more detailed documents about metrics +""" + import re from pathlib import Path @@ -113,7 +117,10 @@ def all_summarize_win(): return folder selected_folders = st.multiselect( - "Show these folders", state.log_folders, state.log_folders, format_func=shorten_folder_name + "Show these folders", + state.log_folders, + state.log_folders, + format_func=shorten_folder_name, ) for lf in selected_folders: if not (Path(lf) / "summary.pkl").exists(): @@ -166,7 +173,13 @@ def all_summarize_win(): base_df = st.data_editor( base_df.style.apply( lambda col: col.map(lambda val: "background-color: #F0F8FF"), - subset=["Baseline Score", "Bronze Threshold", "Silver Threshold", "Gold Threshold", "Medium Threshold"], + subset=[ + "Baseline Score", + "Bronze Threshold", + "Silver Threshold", + "Gold Threshold", + "Medium Threshold", + ], axis=0, ) .apply( @@ -226,10 +239,18 @@ def all_summarize_win(): mean_value = Loop_counts.mean() median_value = Loop_counts.median() fig.add_vline( - x=mean_value, line_color="orange", annotation_text="Mean", annotation_position="top right", line_width=3 + x=mean_value, + line_color="orange", + annotation_text="Mean", + annotation_position="top right", + line_width=3, ) fig.add_vline( - x=median_value, line_color="red", annotation_text="Median", annotation_position="top right", line_width=3 + x=median_value, + line_color="red", + annotation_text="Median", + annotation_position="top right", + line_width=3, ) st.plotly_chart(fig) diff --git a/rdagent/log/ui/utils.py b/rdagent/log/ui/utils.py index 2f25f8b2..c2029bfa 100644 --- a/rdagent/log/ui/utils.py +++ b/rdagent/log/ui/utils.py @@ -216,6 +216,25 @@ def _log_folders_summary_hash_func(log_folders: list[str], hours: int | None = N @cache_with_pickle(_log_folders_summary_hash_func, force=True) def get_summary_df(log_folders: list[str], hours: int | None = None) -> tuple[dict, pd.DataFrame]: + """Process experiment logs and generate summary DataFrame. + + Several key metrics that need explanation: + + * Successful Final Decision: Percentage of experiment loops where code executed correctly + and produced expected output, as determined by evaluation feedback + + * Best Result: The highest achievement level reached by any experiment throughout the entire + process, ranging from lowest to highest: made_submission, valid_submission, above_median, + bronze, silver, gold + + * SOTA Exp: Version found by working backward from the last attempt to find the most recent + successful experiment + + * SOTA Exp (_to_submit): Version selected by LLM from all successful experiments for + competition submission, considering not only scores but also generalization ability + and overfitting risk, totally decided by LLM + + """ summarys = {} if hours is None: sn = "summary.pkl" @@ -260,7 +279,7 @@ def get_summary_df(log_folders: list[str], hours: int | None = None) -> tuple[di else: v["sota_exp_score_valid"] = None v["sota_exp_stat_new"] = get_sota_exp_stat(Path(lf) / k) - # 调整实验名字 + # change experiment name if "amlt" in lf: summary[f"{lf[lf.rfind('amlt')+5:].split('/')[0]} - {k}"] = v elif "ep" in lf: @@ -375,7 +394,10 @@ def get_summary_df(log_folders: list[str], hours: int | None = None) -> tuple[di base_df["SOTA Exp"] = base_df["SOTA Exp"].replace("", pd.NA) - base_df.loc[base_df["SOTA Exp Score (valid)"].apply(lambda x: isinstance(x, str)), "SOTA Exp Score (valid)"] = 0.0 + base_df.loc[ + base_df["SOTA Exp Score (valid)"].apply(lambda x: isinstance(x, str)), + "SOTA Exp Score (valid)", + ] = 0.0 base_df = base_df.astype( { "Total Loops": int,