From bad433ac470d979a72bd0b60ce182e049c551310 Mon Sep 17 00:00:00 2001 From: XianBW <36835909+XianBW@users.noreply.github.com> Date: Thu, 7 Aug 2025 15:02:46 +0800 Subject: [PATCH] add parent & root node in trace page (#1164) --- rdagent/log/ui/ds_summary.py | 92 ++++++++++++----------------------- rdagent/log/ui/ds_trace.py | 93 ++++++++++++++++++++++++++---------- 2 files changed, 99 insertions(+), 86 deletions(-) diff --git a/rdagent/log/ui/ds_summary.py b/rdagent/log/ui/ds_summary.py index a865c5bb..81ce6c81 100755 --- a/rdagent/log/ui/ds_summary.py +++ b/rdagent/log/ui/ds_summary.py @@ -154,50 +154,7 @@ def all_summarize_win(): base_df["Select"] = base_df.index.isin(best_idxs.values) base_df = st.data_editor( - base_df.style.apply( - lambda col: col.map(lambda val: "background-color: #F0F8FF"), - subset=[ - "Baseline Score", - "Bronze Threshold", - "Silver Threshold", - "Gold Threshold", - "Medium Threshold", - ], - axis=0, - ) - .apply( - lambda col: col.map(lambda val: "background-color: #FFFFE0"), - subset=[ - "Ours - Base", - "Ours vs Base", - "Ours vs Bronze", - "Ours vs Silver", - "Ours vs Gold", - ], - axis=0, - ) - .apply( - lambda col: col.map(lambda val: "background-color: #E6E6FA"), - subset=[ - "Script Time", - "Exec Time", - "Exp Gen", - "Coding", - "Running", - ], - axis=0, - ) - .apply( - lambda col: col.map(lambda val: "background-color: #F0FFF0"), - subset=[ - "Best Result", - "SOTA Exp (to_submit)", - "SOTA LID (to_submit)", - "SOTA Exp Score (to_submit)", - "SOTA Exp Score (valid, to_submit)", - ], - axis=0, - ), + base_df, column_config={ "Select": st.column_config.CheckboxColumn("Select", help="Stat this trace.", disabled=False), }, @@ -218,32 +175,43 @@ def all_summarize_win(): st.text(markdown_table) with stat_win_right: Loop_counts = base_df["Total Loops"] - fig = px.histogram(Loop_counts, nbins=10, title="Total Loops Histogram (nbins=10)") + + # Create histogram + fig = px.histogram( + Loop_counts, nbins=15, title="Distribution of Total Loops", color_discrete_sequence=["#3498db"] + ) + fig.update_layout(title_font_size=16, title_font_color="#2c3e50") + + # Calculate statistics mean_value = Loop_counts.mean() median_value = Loop_counts.median() - fig.add_vline( - x=mean_value, - line_color="orange", - annotation_text="Mean", - annotation_position="top right", - line_width=3, + + # Add mean and median lines + fig.add_vline(x=mean_value, line_color="#e74c3c", line_width=3) + fig.add_vline(x=median_value, line_color="#f39c12", line_width=3) + + fig.add_annotation( + x=0.02, + y=0.95, + xref="paper", + yref="paper", + text=f"Mean: {mean_value:.1f}
Median: {median_value:.1f}", + showarrow=False, + bgcolor="rgba(255,255,255,0.9)", + bordercolor="rgba(128,128,128,0.5)", + borderwidth=1, + font=dict(size=12, color="#333333"), ) - fig.add_vline( - x=median_value, - line_color="red", - annotation_text="Median", - annotation_position="top right", - line_width=3, - ) - st.plotly_chart(fig) + + st.plotly_chart(fig, use_container_width=True) # write curve st.subheader("Curves", divider="rainbow") curves_win(summary) -with st.container(border=True): - if st.toggle("近3天平均", key="show_3days"): - days_summarize_win() +# with st.container(border=True): +# if st.toggle("近3天平均", key="show_3days"): +# days_summarize_win() with st.container(border=True): all_summarize_win() diff --git a/rdagent/log/ui/ds_trace.py b/rdagent/log/ui/ds_trace.py index f59274eb..aaccf8d4 100644 --- a/rdagent/log/ui/ds_trace.py +++ b/rdagent/log/ui/ds_trace.py @@ -662,7 +662,6 @@ def summarize_win(): info5.metric( "LLM Filter Calls", llm_filter_call, - delta=-round(llm_filter_call / llm_call, 5), help=timedelta_to_str(filter_call_duration), ) @@ -688,11 +687,9 @@ def summarize_win(): help=f"{timeout_stats['running']['timeout']}/{timeout_stats['running']['total']}", ) + final_trace = list(FileStorage(state.log_folder / state.log_path).iter_msg(tag="record.trace"))[-1].content if show_trace_dag: st.markdown("### Trace DAG") - final_trace_loop_id = max_id - while "record" not in state.data[final_trace_loop_id]: - final_trace_loop_id -= 1 merge_loops = [] for loop_id in state.llm_data.keys(): if "direct_exp_gen" not in state.llm_data[loop_id]: @@ -701,15 +698,32 @@ def summarize_win(): i["obj"]["uri"] for i in state.llm_data[loop_id]["direct_exp_gen"]["no_tag"] if "uri" in i["obj"] ]: merge_loops.append(loop_id) - st.pyplot(trace_figure(state.data[final_trace_loop_id]["record"]["trace"], merge_loops)) + st.pyplot(trace_figure(final_trace, merge_loops)) + + # Find all root nodes (for grouping loops by trace) + root_nodes = {} + parent_nodes = {} + for node in range(len(final_trace.hist)): + parents = final_trace.get_parents(node) + root_nodes[node] = parents[0] + parent_nodes[node] = parents[-2] if len(parents) > 1 else None + root_nodes = {final_trace.idx2loop_id[n]: final_trace.idx2loop_id[r] for n, r in root_nodes.items()} + parent_nodes = { + final_trace.idx2loop_id[n]: final_trace.idx2loop_id[r] if r is not None else r + for n, r in parent_nodes.items() + } + + # Generate Summary Table df = pd.DataFrame( columns=[ + "Root N", + "Parent N", "Component", "Hypothesis", "Reason", "Others", - "Running Score (valid)", - "Running Score (test)", + "Run Score (valid)", + "Run Score (test)", "Feedback", "e-loops(c)", "e-loops(r)", @@ -726,6 +740,8 @@ def summarize_win(): sota_loop_id = state.sota_info[1] if state.sota_info else None for loop in range(min_id, max_id + 1): loop_data = state.data[loop] + df.loc[loop, "Parent N"] = parent_nodes.get(loop, None) + df.loc[loop, "Root N"] = root_nodes.get(loop, None) df.loc[loop, "Component"] = loop_data["direct_exp_gen"]["no_tag"].hypothesis.component df.loc[loop, "Hypothesis"] = loop_data["direct_exp_gen"]["no_tag"].hypothesis.hypothesis df.loc[loop, "Reason"] = loop_data["direct_exp_gen"]["no_tag"].hypothesis.reason @@ -766,10 +782,10 @@ def summarize_win(): running_result = loop_data["running"]["no_tag"].result except AttributeError as e: # Compatible with old versions running_result = loop_data["running"]["no_tag"].__dict__["result"] - df.loc[loop, "Running Score (valid)"] = str(round(running_result.loc["ensemble"].iloc[0], 5)) + df.loc[loop, "Run Score (valid)"] = str(round(running_result.loc["ensemble"].iloc[0], 5)) valid_results[loop] = running_result except: - df.loc[loop, "Running Score (valid)"] = "❌" + df.loc[loop, "Run Score (valid)"] = "❌" if "mle_score" not in state.data[loop]: if "mle_score" in loop_data["running"]: mle_score_txt = loop_data["running"]["mle_score"] @@ -787,12 +803,10 @@ def summarize_win(): else "🥉" if state.data[loop]["mle_score"]["bronze_medal"] else "" ) ) - df.loc[loop, "Running Score (test)"] = ( - f"{medal_emoji} {state.data[loop]['mle_score']['score']}" - ) + df.loc[loop, "Run Score (test)"] = f"{medal_emoji} {state.data[loop]['mle_score']['score']}" else: state.data[loop]["mle_score"] = mle_score_txt - df.loc[loop, "Running Score (test)"] = "❌" + df.loc[loop, "Run Score (test)"] = "❌" else: mle_score_path = ( replace_ep_path(loop_data["running"]["no_tag"].experiment_workspace.workspace_path) @@ -811,15 +825,15 @@ def summarize_win(): else "🥉" if state.data[loop]["mle_score"]["bronze_medal"] else "" ) ) - df.loc[loop, "Running Score (test)"] = ( + df.loc[loop, "Run Score (test)"] = ( f"{medal_emoji} {state.data[loop]['mle_score']['score']}" ) else: state.data[loop]["mle_score"] = mle_score_txt - df.loc[loop, "Running Score (test)"] = "❌" + df.loc[loop, "Run Score (test)"] = "❌" except Exception as e: state.data[loop]["mle_score"] = str(e) - df.loc[loop, "Running Score (test)"] = "❌" + df.loc[loop, "Run Score (test)"] = "❌" else: if isinstance(state.data[loop]["mle_score"], dict): medal_emoji = ( @@ -831,13 +845,13 @@ def summarize_win(): else "🥉" if state.data[loop]["mle_score"]["bronze_medal"] else "" ) ) - df.loc[loop, "Running Score (test)"] = f"{medal_emoji} {state.data[loop]['mle_score']['score']}" + df.loc[loop, "Run Score (test)"] = f"{medal_emoji} {state.data[loop]['mle_score']['score']}" else: - df.loc[loop, "Running Score (test)"] = "❌" + df.loc[loop, "Run Score (test)"] = "❌" else: - df.loc[loop, "Running Score (valid)"] = "N/A" - df.loc[loop, "Running Score (test)"] = "N/A" + df.loc[loop, "Run Score (valid)"] = "N/A" + df.loc[loop, "Run Score (test)"] = "N/A" if "coding" in loop_data: if len([i for i in loop_data["coding"].keys() if isinstance(i, int)]) == 0: @@ -859,7 +873,38 @@ def summarize_win(): if only_success: df = df[df["Feedback"] == "✅"] - st.dataframe(df[df.columns[~df.columns.isin(["Hypothesis", "Reason", "Others"])]]) + + # Add color styling based on root_nodes + def style_dataframe_by_root(df, root_nodes): + # Create a color map for different root nodes - using colors that work well in both light and dark modes + unique_roots = list(set(root_nodes.values())) + colors = [ + "rgba(255, 99, 132, 0.3)", + "rgba(54, 162, 235, 0.3)", + "rgba(75, 192, 75, 0.3)", + "rgba(255, 159, 64, 0.3)", + "rgba(153, 102, 255, 0.2)", + "rgba(255, 205, 86, 0.2)", + "rgba(199, 199, 199, 0.2)", + "rgba(83, 102, 255, 0.2)", + ] + root_color_map = {root: colors[i % len(colors)] for i, root in enumerate(unique_roots)} + + # Create styling function + def apply_color(row): + loop_id = row.name + if loop_id in root_nodes: + root_id = root_nodes[loop_id] + color = root_color_map.get(root_id, "rgba(128, 128, 128, 0.1)") + return [f"background-color: {color}"] * len(row) + return [""] * len(row) + + return df.style.apply(apply_color, axis=1) + + styled_df = style_dataframe_by_root( + df[df.columns[~df.columns.isin(["Hypothesis", "Reason", "Others"])]], root_nodes + ) + st.dataframe(styled_df) # timeline figure if state.times: @@ -882,7 +927,7 @@ def summarize_win(): ensemble_row = vscores.loc[["ensemble"]] vscores = pd.concat([ensemble_row, vscores.drop("ensemble")]) vscores = vscores.T - test_scores = df["Running Score (test)"].str.replace(r"[🥇🥈🥉]\s*", "", regex=True) + test_scores = df["Run Score (test)"].str.replace(r"[🥇🥈🥉]\s*", "", regex=True) vscores["test"] = test_scores vscores.index = [f"L{i}" for i in vscores.index] vscores.columns.name = metric_name @@ -902,7 +947,7 @@ def summarize_win(): def comp_stat_func(x: pd.DataFrame): total_num = x.shape[0] - valid_num = x[x["Running Score (test)"] != "N/A"].shape[0] + valid_num = x[x["Run Score (test)"] != "N/A"].shape[0] success_num = x[x["Feedback"] == "✅"].shape[0] avg_e_loops = x["e-loops(c)"].mean() return pd.Series( @@ -920,7 +965,7 @@ def summarize_win(): # component statistics comp_df = ( - df.loc[:, ["Component", "Running Score (test)", "Feedback", "e-loops(c)"]] + df.loc[:, ["Component", "Run Score (test)", "Feedback", "e-loops(c)"]] .groupby("Component") .apply(comp_stat_func, include_groups=False) )