diff --git a/rdagent/log/ui/app.py b/rdagent/log/ui/app.py new file mode 100644 index 00000000..f72d62e5 --- /dev/null +++ b/rdagent/log/ui/app.py @@ -0,0 +1,1133 @@ +import argparse +import re +import textwrap +from collections import defaultdict +from datetime import datetime, timezone +from importlib.resources import files as rfiles +from pathlib import Path +from typing import Callable, Type + +import pandas as pd +import plotly.express as px +import plotly.graph_objects as go +import streamlit as st +from plotly.subplots import make_subplots +from streamlit import session_state as state +from streamlit_theme import st_theme + +from rdagent.components.coder.factor_coder.evaluators import FactorSingleFeedback # nosec +from rdagent.components.coder.factor_coder.factor import FactorFBWorkspace, FactorTask +from rdagent.components.coder.model_coder.evaluators import ModelSingleFeedback # nosec +from rdagent.components.coder.model_coder.model import ModelFBWorkspace, ModelTask +from rdagent.core.proposal import Hypothesis, HypothesisFeedback +from rdagent.core.scenario import Scenario +from rdagent.log.base import Message +from rdagent.log.storage import FileStorage +from rdagent.log.ui.qlib_report_figure import report_figure +from rdagent.scenarios.general_model.scenario import GeneralModelScenario +from rdagent.scenarios.kaggle.experiment.scenario import KGScenario +from rdagent.scenarios.qlib.experiment.factor_experiment import QlibFactorScenario +from rdagent.scenarios.qlib.experiment.factor_from_report_experiment import ( + QlibFactorFromReportScenario, +) +from rdagent.scenarios.qlib.experiment.model_experiment import ( + QlibModelExperiment, + QlibModelScenario, +) +from rdagent.scenarios.qlib.experiment.quant_experiment import QlibQuantScenario + +st.set_page_config(layout="wide", page_title="RD-Agent", page_icon="🎓", initial_sidebar_state="expanded") + + +# 获取log_path参数 +parser = argparse.ArgumentParser(description="RD-Agent Streamlit App") +parser.add_argument("--log_dir", type=str, help="Path to the log directory") +parser.add_argument("--debug", action="store_true", help="Enable debug mode") +args = parser.parse_args() +if args.log_dir: + main_log_path = Path(args.log_dir) + if not main_log_path.exists(): + st.error(f"Log dir `{main_log_path}` does not exist!") + st.stop() +else: + main_log_path = None + + +QLIB_SELECTED_METRICS = [ + "IC", + "1day.excess_return_with_cost.annualized_return", + "1day.excess_return_with_cost.information_ratio", + "1day.excess_return_with_cost.max_drawdown", +] + +SIMILAR_SCENARIOS = ( + QlibModelScenario, + QlibFactorScenario, + QlibFactorFromReportScenario, + QlibQuantScenario, + KGScenario, +) + + +def filter_log_folders(main_log_path): + """ + Filter and return the log folders relative to the main log path. + """ + folders = [folder.relative_to(main_log_path) for folder in main_log_path.iterdir() if folder.is_dir()] + folders = sorted(folders, key=lambda x: x.name) + return folders + + +if "log_path" not in state: + if main_log_path: + state.log_path = filter_log_folders(main_log_path)[0] + else: + state.log_path = None + st.toast(":red[**Please Set Log Path!**]", icon="⚠️") + +if "scenario" not in state: + state.scenario = None + +if "fs" not in state: + state.fs = None + +if "msgs" not in state: + state.msgs = defaultdict(lambda: defaultdict(list)) + +if "last_msg" not in state: + state.last_msg = None + +if "current_tags" not in state: + state.current_tags = [] + +if "lround" not in state: + state.lround = 0 # RD Loop Round + +if "erounds" not in state: + state.erounds = defaultdict(int) # Evolving Rounds in each RD Loop + +if "e_decisions" not in state: + state.e_decisions = defaultdict(lambda: defaultdict(tuple)) + +# Summary Info +if "hypotheses" not in state: + # Hypotheses in each RD Loop + state.hypotheses = defaultdict(None) + +if "h_decisions" not in state: + state.h_decisions = defaultdict(bool) + +if "metric_series" not in state: + state.metric_series = [] + +if "all_metric_series" not in state: + state.all_metric_series = [] + +# Factor Task Baseline +if "alpha_baseline_metrics" not in state: + state.alpha_baseline_metrics = None + + +def should_display(msg: Message): + for t in state.excluded_tags + ["debug_tpl", "debug_llm"]: + if t in msg.tag.split("."): + return False + + if type(msg.content).__name__ in state.excluded_types: + return False + + return True + + +def get_msgs_until(end_func: Callable[[Message], bool] = lambda _: True): + if state.fs: + while True: + try: + msg = next(state.fs) + if should_display(msg): + tags = msg.tag.split(".") + if "hypothesis generation" in msg.tag: + state.lround += 1 + + # new scenario gen this tags, old version UI not have these tags. + msg.tag = re.sub(r"\.evo_loop_\d+", "", msg.tag) + msg.tag = re.sub(r"Loop_\d+\.[^.]+", "", msg.tag) + msg.tag = re.sub(r"\.\.", ".", msg.tag) + + # remove old redundant tags + msg.tag = re.sub(r"init\.", "", msg.tag) + msg.tag = re.sub(r"r\.", "", msg.tag) + msg.tag = re.sub(r"d\.", "", msg.tag) + msg.tag = re.sub(r"ef\.", "", msg.tag) + + msg.tag = msg.tag.strip(".") + + if "evolving code" not in state.current_tags and "evolving code" in tags: + state.erounds[state.lround] += 1 + + state.current_tags = tags + state.last_msg = msg + + # Update Summary Info + if "runner result" in tags: + # factor baseline exp metrics + if ( + isinstance(state.scenario, (QlibFactorScenario, QlibQuantScenario)) + and state.alpha_baseline_metrics is None + ): + try: + sms = msg.content.based_experiments[0].result + except AttributeError: + sms = msg.content.based_experiments[0].__dict__["result"] + sms = sms.loc[QLIB_SELECTED_METRICS] + sms.name = "Alpha Base" + state.alpha_baseline_metrics = sms + + if state.lround == 1 and len(msg.content.based_experiments) > 0: + try: + sms = msg.content.based_experiments[-1].result + except AttributeError: + sms = msg.content.based_experiments[-1].__dict__["result"] + if sms is not None: + if isinstance( + state.scenario, + ( + QlibModelScenario, + QlibFactorFromReportScenario, + QlibFactorScenario, + QlibQuantScenario, + ), + ): + sms_all = sms + sms = sms.loc[QLIB_SELECTED_METRICS] + sms.name = f"Baseline" + state.metric_series.append(sms) + state.all_metric_series.append(sms_all) + + # common metrics + try: + sms = msg.content.result + except AttributeError: + sms = msg.content.__dict__["result"] + if isinstance( + state.scenario, + ( + QlibModelScenario, + QlibFactorFromReportScenario, + QlibFactorScenario, + QlibQuantScenario, + ), + ): + sms_all = sms + sms = sms.loc[QLIB_SELECTED_METRICS] + + sms.name = f"Round {state.lround}" + sms_all.name = f"Round {state.lround}" + state.metric_series.append(sms) + state.all_metric_series.append(sms_all) + elif "hypothesis generation" in tags: + state.hypotheses[state.lround] = msg.content + elif "evolving code" in tags: + msg.content = [i for i in msg.content if i] + elif "evolving feedback" in tags: + total_len = len(msg.content) + none_num = total_len - len(msg.content) + right_num = 0 + for wsf in msg.content: + if wsf.final_decision: + right_num += 1 + wrong_num = len(msg.content) - right_num + state.e_decisions[state.lround][state.erounds[state.lround]] = ( + right_num, + wrong_num, + none_num, + ) + elif "feedback" in tags and isinstance(msg.content, HypothesisFeedback): + state.h_decisions[state.lround] = msg.content.decision + + state.msgs[state.lround][msg.tag].append(msg) + + # Stop Getting Logs + if end_func(msg): + break + except StopIteration: + st.toast(":red[**No More Logs to Show!**]", icon="🛑") + break + + +def refresh(same_trace: bool = False): + if state.log_path is None: + st.toast(":red[**Please Set Log Path!**]", icon="⚠️") + return + + if main_log_path: + state.fs = FileStorage(main_log_path / state.log_path).iter_msg() + else: + state.fs = FileStorage(state.log_path).iter_msg() + + # detect scenario + if not same_trace: + get_msgs_until(lambda m: isinstance(m.content, Scenario)) + if state.last_msg is None or not isinstance(state.last_msg.content, Scenario): + st.write(state.msgs) + st.toast(":red[**No Scenario Info detected**]", icon="❗") + state.scenario = None + else: + state.scenario = state.last_msg.content + st.toast(f":green[**Scenario Info detected**] *{type(state.scenario).__name__}*", icon="✅") + + state.msgs = defaultdict(lambda: defaultdict(list)) + state.lround = 0 + state.erounds = defaultdict(int) + state.e_decisions = defaultdict(lambda: defaultdict(tuple)) + state.hypotheses = defaultdict(None) + state.h_decisions = defaultdict(bool) + state.metric_series = [] + state.all_metric_series = [] + state.last_msg = None + state.current_tags = [] + state.alpha_baseline_metrics = None + + +def evolving_feedback_window(wsf: FactorSingleFeedback | ModelSingleFeedback): + if isinstance(wsf, FactorSingleFeedback): + ffc, efc, cfc, vfc = st.tabs( + ["**Final Feedback🏁**", "Execution Feedback🖥️", "Code Feedback📄", "Value Feedback🔢"] + ) + with ffc: + st.markdown(wsf.final_feedback) + with efc: + st.code(wsf.execution_feedback, language="log") # nosec + with cfc: + st.markdown(wsf.code_feedback) + with vfc: + st.markdown(wsf.value_feedback) + elif isinstance(wsf, ModelSingleFeedback): + ffc, efc, cfc, msfc, vfc = st.tabs( + [ + "**Final Feedback🏁**", + "Execution Feedback🖥️", + "Code Feedback📄", + "Model Shape Feedback📐", + "Value Feedback🔢", + ] + ) + with ffc: + st.markdown(wsf.final_feedback) + with efc: + st.code(wsf.execution_feedback, language="log") # nosec + with cfc: + st.markdown(wsf.code_feedback) + with msfc: + st.markdown(wsf.shape_feedback) + with vfc: + st.markdown(wsf.value_feedback) + + +def display_hypotheses(hypotheses: dict[int, Hypothesis], decisions: dict[int, bool], success_only: bool = False): + name_dict = { + "hypothesis": "RD-Agent proposes the hypothesis⬇️", + "concise_justification": "because the reason⬇️", + "concise_observation": "based on the observation⬇️", + "concise_knowledge": "Knowledge⬇️ gained after practice", + } + if success_only: + shd = {k: v.__dict__ for k, v in hypotheses.items() if decisions[k]} + else: + shd = {k: v.__dict__ for k, v in hypotheses.items()} + df = pd.DataFrame(shd).T + + if "concise_observation" in df.columns and "concise_justification" in df.columns: + df["concise_observation"], df["concise_justification"] = df["concise_justification"], df["concise_observation"] + df.rename( + columns={"concise_observation": "concise_justification", "concise_justification": "concise_observation"}, + inplace=True, + ) + if "reason" in df.columns: + df.drop(["reason"], axis=1, inplace=True) + if "concise_reason" in df.columns: + df.drop(["concise_reason"], axis=1, inplace=True) + + df.columns = df.columns.map(lambda x: name_dict.get(x, x)) + for col in list(df.columns): + if all([value is None for value in df[col]]): + df.drop([col], axis=1, inplace=True) + + def style_rows(row): + if decisions[row.name]: + return ["color: green;"] * len(row) + return [""] * len(row) + + def style_columns(col): + if col.name != name_dict.get("hypothesis", "hypothesis"): + return ["font-style: italic;"] * len(col) + return ["font-weight: bold;"] * len(col) + + # st.dataframe(df.style.apply(style_rows, axis=1).apply(style_columns, axis=0)) + st.markdown(df.style.apply(style_rows, axis=1).apply(style_columns, axis=0).to_html(), unsafe_allow_html=True) + + +def metrics_window(df: pd.DataFrame, R: int, C: int, *, height: int = 300, colors: list[str] = None): + fig = make_subplots(rows=R, cols=C, subplot_titles=df.columns) + + def hypothesis_hover_text(h: Hypothesis, d: bool = False): + color = "green" if d else "black" + text = h.hypothesis + lines = textwrap.wrap(text, width=60) + return f"{'
'.join(lines)}" + + hover_texts = [ + hypothesis_hover_text(state.hypotheses[int(i[6:])], state.h_decisions[int(i[6:])]) + for i in df.index + if i != "Alpha Base" and i != "Baseline" + ] + if state.alpha_baseline_metrics is not None: + hover_texts = ["Baseline"] + hover_texts + for ci, col in enumerate(df.columns): + row = ci // C + 1 + col_num = ci % C + 1 + fig.add_trace( + go.Scatter( + x=df.index, + y=df[col], + name=col, + mode="lines+markers", + connectgaps=True, + marker=dict(size=10, color=colors[ci]) if colors else dict(size=10), + hovertext=hover_texts, + hovertemplate="%{hovertext}

%{x} Value: %{y}", + ), + row=row, + col=col_num, + ) + fig.update_layout(showlegend=False, height=height) + + if state.alpha_baseline_metrics is not None: + for i in range(1, R + 1): # 行 + for j in range(1, C + 1): # 列 + fig.update_xaxes( + tickvals=[df.index[0]] + list(df.index[1:]), + ticktext=[f'{df.index[0]}'] + list(df.index[1:]), + row=i, + col=j, + ) + st.plotly_chart(fig) + + from io import BytesIO + + buffer = BytesIO() + df.to_csv(buffer) + buffer.seek(0) + st.download_button(label="download the metrics (csv)", data=buffer, file_name="metrics.csv", mime="text/csv") + + +def summary_window(): + if isinstance(state.scenario, SIMILAR_SCENARIOS): + st.header("Summary📊", divider="rainbow", anchor="_summary") + if state.lround == 0: + return + with st.container(): + # TODO: not fixed height + with st.container(): + bc, cc = st.columns([2, 2], vertical_alignment="center") + with bc: + st.subheader("Metrics📈", anchor="_metrics") + with cc: + show_true_only = st.toggle("successful hypotheses", value=False) + + # hypotheses_c, chart_c = st.columns([2, 3]) + chart_c = st.container() + hypotheses_c = st.container() + + with hypotheses_c: + st.subheader("Hypotheses🏅", anchor="_hypotheses") + display_hypotheses(state.hypotheses, state.h_decisions, show_true_only) + + with chart_c: + if isinstance(state.scenario, QlibFactorScenario) and state.alpha_baseline_metrics is not None: + df = pd.DataFrame([state.alpha_baseline_metrics] + state.metric_series[1:]) + elif isinstance(state.scenario, QlibQuantScenario) and state.alpha_baseline_metrics is not None: + df = pd.DataFrame([state.alpha_baseline_metrics] + state.metric_series[1:]) + else: + df = pd.DataFrame(state.metric_series) + if show_true_only and len(state.hypotheses) >= len(state.metric_series): + if state.alpha_baseline_metrics is not None: + selected = ["Alpha Base"] + [ + i for i in df.index if i == "Baseline" or state.h_decisions[int(i[6:])] + ] + else: + selected = [i for i in df.index if i == "Baseline" or state.h_decisions[int(i[6:])]] + df = df.loc[selected] + if df.shape[0] == 1: + st.table(df.iloc[0]) + elif df.shape[0] > 1: + if df.shape[1] == 1: + fig = px.line(df, x=df.index, y=df.columns, markers=True) + fig.update_layout(xaxis_title="Loop Round", yaxis_title=None) + st.plotly_chart(fig) + else: + metrics_window(df, 1, 4, height=300, colors=["red", "blue", "orange", "green"]) + + elif isinstance(state.scenario, GeneralModelScenario): + with st.container(border=True): + st.subheader("Summary📊", divider="rainbow", anchor="_summary") + if len(state.msgs[state.lround]["evolving code"]) > 0: + # pass + ws: list[FactorFBWorkspace | ModelFBWorkspace] = state.msgs[state.lround]["evolving code"][-1].content + # All Tasks + + tab_names = [ + w.target_task.factor_name if isinstance(w.target_task, FactorTask) else w.target_task.name + for w in ws + ] + for j in range(len(ws)): + if state.msgs[state.lround]["evolving feedback"][-1].content[j].final_decision: + tab_names[j] += "✔️" + else: + tab_names[j] += "❌" + + wtabs = st.tabs(tab_names) + for j, w in enumerate(ws): + with wtabs[j]: + # Evolving Code + for k, v in w.file_dict.items(): + with st.expander(f":green[`{k}`]", expanded=False): + st.code(v, language="python") + + # Evolving Feedback + evolving_feedback_window(state.msgs[state.lround]["evolving feedback"][-1].content[j]) + + +def tabs_hint(): + st.markdown( + "

You can navigate through the tabs using ⬅️ ➡️ or by holding Shift and scrolling with the mouse wheel🖱️.

", + unsafe_allow_html=True, + ) + + +def tasks_window(tasks: list[FactorTask | ModelTask]): + if isinstance(tasks[0], FactorTask): + st.markdown("**Factor Tasks🚩**") + tnames = [f.factor_name for f in tasks] + if sum(len(tn) for tn in tnames) > 100: + tabs_hint() + tabs = st.tabs(tnames) + for i, ft in enumerate(tasks): + with tabs[i]: + # st.markdown(f"**Factor Name**: {ft.factor_name}") + st.markdown(f"**Description**: {ft.factor_description}") + st.latex("Formulation") + st.latex(ft.factor_formulation) + + mks = "| Variable | Description |\n| --- | --- |\n" + if isinstance(ft.variables, dict): + for v, d in ft.variables.items(): + mks += f"| ${v}$ | {d} |\n" + st.markdown(mks) + + elif isinstance(tasks[0], ModelTask): + st.markdown("**Model Tasks🚩**") + tnames = [m.name for m in tasks] + if sum(len(tn) for tn in tnames) > 100: + tabs_hint() + tabs = st.tabs(tnames) + for i, mt in enumerate(tasks): + with tabs[i]: + # st.markdown(f"**Model Name**: {mt.name}") + st.markdown(f"**Model Type**: {mt.model_type}") + st.markdown(f"**Description**: {mt.description}") + st.latex("Formulation") + st.latex(mt.formulation) + + mks = "| Variable | Description |\n| --- | --- |\n" + if mt.variables: + for v, d in mt.variables.items(): + mks += f"| ${v}$ | {d} |\n" + st.markdown(mks) + st.markdown(f"**Train Para**: {mt.training_hyperparameters}") + + +def research_window(): + with st.container(border=True): + title = "Research🔍" if isinstance(state.scenario, SIMILAR_SCENARIOS) else "Research🔍 (reader)" + st.subheader(title, divider="blue", anchor="_research") + if isinstance(state.scenario, SIMILAR_SCENARIOS): + # pdf image + if pim := state.msgs[round]["load_pdf_screenshot"]: + for i in range(min(2, len(pim))): + st.image(pim[i].content, use_container_width=True) + + # Hypothesis + if hg := state.msgs[round]["hypothesis generation"]: + st.markdown("**Hypothesis💡**") # 🧠 + h: Hypothesis = hg[0].content + st.markdown(f""" +- **Hypothesis**: {h.hypothesis} +- **Reason**: {h.reason}""") + + if eg := state.msgs[round]["experiment generation"]: + tasks_window(eg[0].content) + + elif isinstance(state.scenario, GeneralModelScenario): + # pdf image + c1, c2 = st.columns([2, 3]) + with c1: + if pim := state.msgs[0]["pdf_image"]: + for i in range(len(pim)): + st.image(pim[i].content, use_container_width=True) + + # loaded model exp + with c2: + if mem := state.msgs[0]["load_experiment"]: + me: QlibModelExperiment = mem[0].content + tasks_window(me.sub_tasks) + + +def feedback_window(): + # st.write(round) + # # Check if metric series exists and has the matching round + # if state.all_metric_series: + # for metric in state.all_metric_series: + # if metric.name == f"Round {round}": + # # Select specific metrics with cost + # selected_metrics_with_cost = { + # 'IC': float(f"{metric['IC']:.4f}"), + # 'ICIR': float(f"{metric['ICIR']:.4f}"), + # 'Rank IC': float(f"{metric['Rank IC']:.4f}"), + # 'Rank ICIR': float(f"{metric['Rank ICIR']:.4f}"), + # 'ARR': float(f"{metric['1day.excess_return_with_cost.annualized_return']:.4f}"), + # 'IR': float(f"{metric['1day.excess_return_with_cost.information_ratio']:.4f}"), + # 'MDD': float(f"{metric['1day.excess_return_with_cost.max_drawdown']:.4f}"), + # 'Sharpe': float(f"{metric['1day.excess_return_with_cost.annualized_return'] / abs(metric['1day.excess_return_with_cost.max_drawdown']):.4f}") + # } + # st.write("With Cost Metrics:") + # st.write(pd.Series(selected_metrics_with_cost)) + + # # Select specific metrics without cost + # selected_metrics_without_cost = { + # 'IC': float(f"{metric['IC']:.4f}"), + # 'ICIR': float(f"{metric['ICIR']:.4f}"), + # 'Rank IC': float(f"{metric['Rank IC']:.4f}"), + # 'Rank ICIR': float(f"{metric['Rank ICIR']:.4f}"), + # 'ARR': float(f"{metric['1day.excess_return_without_cost.annualized_return']:.4f}"), + # 'IR': float(f"{metric['1day.excess_return_without_cost.information_ratio']:.4f}"), + # 'MDD': float(f"{metric['1day.excess_return_without_cost.max_drawdown']:.4f}"), + # 'Sharpe': float(f"{metric['1day.excess_return_without_cost.annualized_return'] / abs(metric['1day.excess_return_without_cost.max_drawdown']):.4f}") + # } + # st.write("Without Cost Metrics:") + # st.write(pd.Series(selected_metrics_without_cost)) + # break + if isinstance(state.scenario, SIMILAR_SCENARIOS): + with st.container(border=True): + st.subheader("Feedback📝", divider="orange", anchor="_feedback") + + if state.lround > 0 and isinstance( + state.scenario, + (QlibModelScenario, QlibFactorScenario, QlibFactorFromReportScenario, QlibQuantScenario, KGScenario), + ): + if fbr := state.msgs[round]["runner result"]: + try: + st.write("workspace") + st.write(fbr[0].content.experiment_workspace.workspace_path) + st.write(fbr[0].content.stdout) + except Exception as e: + st.error(f"Error displaying workspace path: {str(e)}") + with st.expander("**Config⚙️**", expanded=True): + st.markdown(state.scenario.experiment_setting, unsafe_allow_html=True) + + if fb := state.msgs[round]["feedback"]: + if fbr := state.msgs[round]["Quantitative Backtesting Chart"]: + st.markdown("**Returns📈**") + fig = report_figure(fbr[0].content) + st.plotly_chart(fig) + st.markdown("**Hypothesis Feedback🔍**") + h: HypothesisFeedback = fb[0].content + st.markdown(f""" +- **Observations**: {h.observations} +- **Hypothesis Evaluation**: {h.hypothesis_evaluation} # nosec +- **New Hypothesis**: {h.new_hypothesis} +- **Decision**: {h.decision} +- **Reason**: {h.reason}""") + + if isinstance(state.scenario, KGScenario): + if fbe := state.msgs[round]["runner result"]: + submission_path = fbe[0].content.experiment_workspace.workspace_path / "submission.csv" + st.markdown( + f":green[**Exp Workspace**]: {str(fbe[0].content.experiment_workspace.workspace_path.absolute())}" + ) + try: + data = submission_path.read_bytes() + st.download_button( + label="**Download** submission.csv", + data=data, + file_name="submission.csv", + mime="text/csv", + ) + except Exception as e: + st.markdown(f":red[**Download Button Error**]: {e}") + + +def evolving_window(): + title = "Development🛠️" if isinstance(state.scenario, SIMILAR_SCENARIOS) else "Development🛠️ (evolving coder)" + st.subheader(title, divider="green", anchor="_development") + + # Evolving Status + if state.erounds[round] > 0: + st.markdown("**☑️ Evolving Status**") + es = state.e_decisions[round] + e_status_mks = "".join(f"| {ei} " for ei in range(1, state.erounds[round] + 1)) + "|\n" + e_status_mks += "|--" * state.erounds[round] + "|\n" + for ei, estatus in es.items(): + if not estatus: + estatus = (0, 0, 0) + e_status_mks += "| " + "🕙
" * estatus[2] + "✔️
" * estatus[0] + "❌
" * estatus[1] + " " + e_status_mks += "|\n" + st.markdown(e_status_mks, unsafe_allow_html=True) + + # Evolving Tabs + if state.erounds[round] > 0: + if state.erounds[round] > 1: + evolving_round = st.radio( + "**🔄️Evolving Rounds**", + horizontal=True, + options=range(1, state.erounds[round] + 1), + index=state.erounds[round] - 1, + key="show_eround", + ) + else: + evolving_round = 1 + + ws: list[FactorFBWorkspace | ModelFBWorkspace] = state.msgs[round]["evolving code"][evolving_round - 1].content + # All Tasks + + tab_names = [ + w.target_task.factor_name if isinstance(w.target_task, FactorTask) else w.target_task.name for w in ws + ] + if len(state.msgs[round]["evolving feedback"]) >= evolving_round: + for j in range(len(ws)): + if state.msgs[round]["evolving feedback"][evolving_round - 1].content[j].final_decision: + tab_names[j] += "✔️" + else: + tab_names[j] += "❌" + if sum(len(tn) for tn in tab_names) > 100: + tabs_hint() + wtabs = st.tabs(tab_names) + for j, w in enumerate(ws): + with wtabs[j]: + # Evolving Code + st.markdown(f"**Workspace Path**: {w.workspace_path}") + for k, v in w.file_dict.items(): + with st.expander(f":green[`{k}`]", expanded=True): + st.code(v, language="python") + + # Evolving Feedback + if len(state.msgs[round]["evolving feedback"]) >= evolving_round: + evolving_feedback_window(state.msgs[round]["evolving feedback"][evolving_round - 1].content[j]) + + +toc = """ +## [Scenario Description📖](#_scenario) +## [Summary📊](#_summary) +- [**Metrics📈**](#_metrics) +- [**Hypotheses🏅**](#_hypotheses) +## [RD-Loops♾️](#_rdloops) +- [**Research🔍**](#_research) +- [**Development🛠️**](#_development) +- [**Feedback📝**](#_feedback) +""" +if isinstance(state.scenario, GeneralModelScenario): + toc = """ +## [Scenario Description📖](#_scenario) +### [Summary📊](#_summary) +### [Research🔍](#_research) +### [Development🛠️](#_development) +""" +# Config Sidebar +with st.sidebar: + st.markdown("# RD-Agent🤖 [:grey[@GitHub]](https://github.com/microsoft/RD-Agent)") + st.subheader(":blue[Table of Content]", divider="blue") + st.markdown(toc) + st.subheader(":orange[Control Panel]", divider="red") + + with st.container(border=True): + if main_log_path: + lc1, lc2 = st.columns([1, 2], vertical_alignment="center") + with lc1: + st.markdown(":blue[**Log Path**]") + with lc2: + manually = st.toggle("Manual Input") + if manually: + st.text_input("log path", key="log_path", on_change=refresh, label_visibility="collapsed") + else: + folders = filter_log_folders(main_log_path) + st.selectbox(f"**Select from `{main_log_path}`**", folders, key="log_path", on_change=refresh) # nosec B608 — not SQL, Bandit false positive on "Select" in UI label + else: + st.text_input(":blue[**log path**]", key="log_path", on_change=refresh) + + c1, c2 = st.columns([1, 1], vertical_alignment="center") + with c1: + if st.button(":green[**All Loops**]", use_container_width=True): + if not state.fs: + refresh() + get_msgs_until(lambda m: False) + if st.button("**Reset**", use_container_width=True): + refresh(same_trace=True) + with c2: + if st.button(":green[Next Loop]", use_container_width=True): + if not state.fs: + refresh() + get_msgs_until(lambda m: "feedback" in m.tag and "evolving feedback" not in m.tag) + + if st.button("Next Step", use_container_width=True): + if not state.fs: + refresh() + get_msgs_until(lambda m: "evolving feedback" in m.tag) + + with st.popover(":orange[**Config⚙️**]", use_container_width=True): + st.multiselect("excluded log tags", ["llm_messages"], ["llm_messages"], key="excluded_tags") + st.multiselect("excluded log types", ["str", "dict", "list"], ["str"], key="excluded_types") + + if args.debug: + debug = st.toggle("debug", value=False) + + if debug: + if st.button("Single Step Run", use_container_width=True): + get_msgs_until() + else: + debug = False + + +# Debug Info Window +if debug: + with st.expander(":red[**Debug Info**]", expanded=True): + dcol1, dcol2 = st.columns([1, 3]) + with dcol1: + st.markdown( + f"**log path**: {state.log_path}\n\n" + f"**excluded tags**: {state.excluded_tags}\n\n" + f"**excluded types**: {state.excluded_types}\n\n" + f":blue[**message id**]: {sum(sum(len(tmsgs) for tmsgs in rmsgs.values()) for rmsgs in state.msgs.values())}\n\n" + f":blue[**round**]: {state.lround}\n\n" + f":blue[**evolving round**]: {state.erounds[state.lround]}\n\n" + ) + with dcol2: + if state.last_msg: + st.write(state.last_msg) + if isinstance(state.last_msg.content, list): + st.write(state.last_msg.content[0]) + elif isinstance(state.last_msg.content, dict): + st.write(state.last_msg.content) + elif not isinstance(state.last_msg.content, str): + try: + st.write(state.last_msg.content.__dict__) + except: + st.write(type(state.last_msg.content)) + +if state.log_path and state.fs is None: + refresh() + +# Main Window +header_c1, header_c3 = st.columns([1, 6], vertical_alignment="center") +with st.container(): + with header_c1: + st.image("https://img-prod-cms-rt-microsoft-com.akamaized.net/cms/api/am/imageFileData/RE1Mu3b?ver=5c31") + with header_c3: + st.markdown( + """ +

+ RD-Agent:
LLM-based autonomous evolving agents for industrial data-driven R&D +

+ """, + unsafe_allow_html=True, + ) + +# Project Info +with st.container(): + image_c, scen_c = st.columns([3, 3], vertical_alignment="center") + with image_c: + img_path = rfiles("rdagent.log.ui").joinpath("flow.png") + st.image(str(img_path), use_container_width=True) + with scen_c: + st.header("Scenario Description📖", divider="violet", anchor="_scenario") + if state.scenario is not None: + theme = st_theme() + if theme: + theme = theme.get("base", "light") + css = f""" + +""" + st.markdown(state.scenario.rich_style_description + css, unsafe_allow_html=True) + + +def analyze_task_completion(): + st.header("Task Completion Analysis", divider="orange") + + # Dictionary to store results for all loops + completion_stats = {} + + # Iterate through all loops + for loop_round in state.msgs.keys(): + if loop_round == 0: # Skip initialization round + continue + + max_evolving_round = state.erounds[loop_round] + if max_evolving_round == 0: + continue + + # Track tasks that pass in each evolving round + tasks_passed_by_round = {} + cumulative_passed = set() + + # For each evolving round in this loop + for e_round in range(1, max_evolving_round + 1): + if len(state.msgs[loop_round]["evolving feedback"]) >= e_round: + # Get feedback for this evolving round + feedback = state.msgs[loop_round]["evolving feedback"][e_round - 1].content + + # Count passed tasks and track their indices + passed_tasks = set() + for j, task_feedback in enumerate(feedback): + if task_feedback.final_decision: + passed_tasks.add(j) + cumulative_passed.add(j) + + # Store both individual round results and cumulative results + tasks_passed_by_round[e_round] = { + "count": len(passed_tasks), + "indices": passed_tasks, + "cumulative_count": len(cumulative_passed), + "cumulative_indices": cumulative_passed.copy(), + } + + completion_stats[loop_round] = { + "total_tasks": len(state.msgs[loop_round]["evolving feedback"][0].content), + "rounds": tasks_passed_by_round, + "max_round": max_evolving_round, + } + + # Display results + if completion_stats: + # Add an aggregate view at the top + st.subheader("🔄 Aggregate Completion Across All Loops") + + # Create summary data for comparison + summary_data = [] + total_tasks_across_loops = 0 + total_passed_r1 = 0 + total_passed_r3 = 0 + total_passed_r5 = 0 + total_passed_r10 = 0 + total_passed_final = 0 + + for loop_round, stats in completion_stats.items(): + total_tasks = stats["total_tasks"] + total_tasks_across_loops += total_tasks + + # Find data for specific rounds + r1_passed = stats["rounds"].get(1, {}).get("cumulative_count", 0) + total_passed_r1 += r1_passed + + # For round 3, use the closest round if exactly 3 doesn't exist + if 3 in stats["rounds"]: + r3_passed = stats["rounds"][3]["cumulative_count"] + elif stats["max_round"] >= 3: + max_r_below_3 = max([r for r in stats["rounds"].keys() if r <= 3]) + r3_passed = stats["rounds"][max_r_below_3]["cumulative_count"] + else: + r3_passed = stats["rounds"][stats["max_round"]]["cumulative_count"] if stats["rounds"] else 0 + total_passed_r3 += r3_passed + + # For round 5, use the closest round if exactly 5 doesn't exist + if 5 in stats["rounds"]: + r5_passed = stats["rounds"][5]["cumulative_count"] + elif stats["max_round"] >= 5: + max_r_below_5 = max([r for r in stats["rounds"].keys() if r <= 5]) + r5_passed = stats["rounds"][max_r_below_5]["cumulative_count"] + else: + r5_passed = stats["rounds"][stats["max_round"]]["cumulative_count"] if stats["rounds"] else 0 + total_passed_r5 += r5_passed + + # For round 10 + if 10 in stats["rounds"]: + r10_passed = stats["rounds"][10]["cumulative_count"] + else: + r10_passed = stats["rounds"][stats["max_round"]]["cumulative_count"] if stats["rounds"] else 0 + total_passed_r10 += r10_passed + + # Final round completion + final_passed = stats["rounds"][stats["max_round"]]["cumulative_count"] if stats["rounds"] else 0 + total_passed_final += final_passed + + # Add to summary table + summary_data.append( + { + "Loop": f"Loop {loop_round}", + "Total Tasks": total_tasks, + "Passed (Round 1)": ( + f"{r1_passed}/{total_tasks} ({r1_passed/total_tasks:.0%})" if total_tasks > 0 else "N/A" + ), + "Passed (Round 3)": ( + f"{r3_passed}/{total_tasks} ({r3_passed/total_tasks:.0%})" if total_tasks > 0 else "N/A" + ), + "Passed (Round 5)": ( + f"{r5_passed}/{total_tasks} ({r5_passed/total_tasks:.0%})" if total_tasks > 0 else "N/A" + ), + "Passed (Final)": ( + f"{final_passed}/{total_tasks} ({final_passed/total_tasks:.0%})" if total_tasks > 0 else "N/A" + ), + } + ) + + if total_tasks_across_loops > 0: + summary_data.append( + { + "Loop": "**TOTAL**", + "Total Tasks": total_tasks_across_loops, + "Passed (Round 1)": f"**{total_passed_r1}/{total_tasks_across_loops} ({total_passed_r1/total_tasks_across_loops:.0%})**", + "Passed (Round 3)": f"**{total_passed_r3}/{total_tasks_across_loops} ({total_passed_r3/total_tasks_across_loops:.0%})**", + "Passed (Round 5)": f"**{total_passed_r5}/{total_tasks_across_loops} ({total_passed_r5/total_tasks_across_loops:.0%})**", + "Passed (Final)": f"**{total_passed_final}/{total_tasks_across_loops} ({total_passed_final/total_tasks_across_loops:.0%})**", + } + ) + + st.table(pd.DataFrame(summary_data)) + + # Summary statistics + st.markdown("### 📊 Overall Completion Progress:") + col1, col2, col3, col4 = st.columns(4) + with col1: + st.metric( + label="After Round 1", + value=f"{total_passed_r1/total_tasks_across_loops:.0%}", + help=f"{total_passed_r1}/{total_tasks_across_loops} tasks", + ) + with col2: + st.metric( + label="After Round 3", + value=f"{total_passed_r3/total_tasks_across_loops:.0%}", + delta=f"{(total_passed_r3-total_passed_r1)/total_tasks_across_loops:.0%}", + help=f"{total_passed_r3}/{total_tasks_across_loops} tasks", + ) + with col3: + st.metric( + label="After Round 5", + value=f"{total_passed_r5/total_tasks_across_loops:.0%}", + delta=f"{(total_passed_r5-total_passed_r3)/total_tasks_across_loops:.0%}", + help=f"{total_passed_r5}/{total_tasks_across_loops} tasks", + ) + with col4: + st.metric( + label="Final Completion", + value=f"{total_passed_final/total_tasks_across_loops:.0%}", + delta=f"{(total_passed_final-total_passed_r5)/total_tasks_across_loops:.0%}", + help=f"{total_passed_final}/{total_tasks_across_loops} tasks", + ) + + # Show detailed results by loop + st.markdown("---") + st.subheader("Detailed Results by Loop") + + for loop_round, stats in completion_stats.items(): + with st.expander(f"Loop {loop_round} Details"): + total_tasks = stats["total_tasks"] + + # Create a results table + data = [] + for e_round in range(1, min(11, stats["max_round"] + 1)): + if e_round in stats["rounds"]: + round_data = stats["rounds"][e_round] + data.append( + { + "Evolving Round": e_round, + "Tasks Passed": f"{round_data['count']}/{total_tasks} ({round_data['count']/total_tasks:.0%})", + "Cumulative Passed": f"{round_data['cumulative_count']}/{total_tasks} ({round_data['cumulative_count']/total_tasks:.0%})", + } + ) + else: + data.append({"Evolving Round": e_round, "Tasks Passed": "N/A", "Cumulative Passed": "N/A"}) + + df = pd.DataFrame(data) + st.table(df) + + st.markdown("### Summary:") + if 1 in stats["rounds"]: + st.markdown( + f"- After round 1: **{stats['rounds'][1]['cumulative_count']}/{total_tasks}** tasks passed ({stats['rounds'][1]['cumulative_count']/total_tasks:.0%})" + ) + + if 3 in stats["rounds"]: + st.markdown( + f"- After round 3: **{stats['rounds'][3]['cumulative_count']}/{total_tasks}** tasks passed ({stats['rounds'][3]['cumulative_count']/total_tasks:.0%})" + ) + elif stats["max_round"] >= 3: + max_round_below_3 = max([r for r in stats["rounds"].keys() if r <= 3]) + st.markdown( + f"- After round 3: **{stats['rounds'][max_round_below_3]['cumulative_count']}/{total_tasks}** tasks passed ({stats['rounds'][max_round_below_3]['cumulative_count']/total_tasks:.0%})" + ) + + if 5 in stats["rounds"]: + st.markdown( + f"- After round 5: **{stats['rounds'][5]['cumulative_count']}/{total_tasks}** tasks passed ({stats['rounds'][5]['cumulative_count']/total_tasks:.0%})" + ) + elif stats["max_round"] >= 5: + max_round_below_5 = max([r for r in stats["rounds"].keys() if r <= 5]) + st.markdown( + f"- After round 5: **{stats['rounds'][max_round_below_5]['cumulative_count']}/{total_tasks}** tasks passed ({stats['rounds'][max_round_below_5]['cumulative_count']/total_tasks:.0%})" + ) + + if 10 in stats["rounds"]: + st.markdown( + f"- After round 10: **{stats['rounds'][10]['cumulative_count']}/{total_tasks}** tasks passed ({stats['rounds'][10]['cumulative_count']/total_tasks:.0%})" + ) + elif stats["max_round"] >= 1: + st.markdown( + f"- After final round ({stats['max_round']}): **{stats['rounds'][stats['max_round']]['cumulative_count']}/{total_tasks}** tasks passed ({stats['rounds'][stats['max_round']]['cumulative_count']/total_tasks:.0%})" + ) + else: + st.info("No task completion data available.") + + +if state.scenario is not None: + summary_window() + if st.toggle("show analyse_task_competition"): + analyze_task_completion() + + # R&D Loops Window + if isinstance(state.scenario, SIMILAR_SCENARIOS): + st.header("R&D Loops♾️", divider="rainbow", anchor="_rdloops") + if len(state.msgs) > 1: + r_options = list(state.msgs.keys()) + if 0 in r_options: + r_options.remove(0) + round = st.radio("**Loops**", horizontal=True, options=r_options, index=state.lround - 1) + else: + round = 1 + + rf_c, d_c = st.columns([2, 2]) + elif isinstance(state.scenario, GeneralModelScenario): + + rf_c = st.container() + d_c = st.container() + round = 0 + else: + st.error("Unknown Scenario!") + st.stop() + + with rf_c: + research_window() + feedback_window() + + with d_c.container(border=True): + evolving_window() + + +st.markdown("


", unsafe_allow_html=True) +st.markdown("#### Disclaimer") +st.markdown( + "*This content is AI-generated and may not be fully accurate or up-to-date; please verify with a professional for critical matters.*", + unsafe_allow_html=True, +) diff --git a/rdagent/log/ui/ds_trace.py b/rdagent/log/ui/ds_trace.py new file mode 100644 index 00000000..609a6916 --- /dev/null +++ b/rdagent/log/ui/ds_trace.py @@ -0,0 +1,1205 @@ +import hashlib +import json +import pickle # nosec +import random +import re +from collections import defaultdict +from datetime import time, timedelta +from pathlib import Path + +import pandas as pd +import plotly.express as px +import streamlit as st +from litellm import get_valid_models +from streamlit import session_state as state + +from rdagent.app.data_science.loop import DataScienceRDLoop +from rdagent.log.storage import FileStorage +from rdagent.log.ui.conf import UI_SETTING +from rdagent.log.ui.utils import ( + curve_figure, + get_sota_exp_stat, + load_times_info, + timeline_figure, + trace_figure, +) +from rdagent.log.utils import ( + LogColors, + extract_evoid, + extract_json, + extract_loopid_func_name, + is_valid_session, +) +from rdagent.oai.backend.litellm import LITELLM_SETTINGS +from rdagent.oai.llm_utils import APIBackend + +# Import necessary classes for the response format +from rdagent.scenarios.data_science.proposal.exp_gen.proposal import ( + CodingSketch, + HypothesisList, + ScenarioChallenges, + TraceChallenges, +) +from rdagent.utils.agent.tpl import T +from rdagent.utils.repo.diff import generate_diff_from_dict + +if "show_stdout" not in state: + state.show_stdout = False +if "show_llm_log" not in state: + state.show_llm_log = False +if "data" not in state: + state.data = defaultdict(lambda: defaultdict(dict)) +if "llm_data" not in state: + state.llm_data = defaultdict(lambda: defaultdict(lambda: defaultdict(list))) +if "log_path" not in state: + state.log_path = None +if "log_folder" not in state: + state.log_folder = Path("./log") +if "sota_info" not in state: + state.sota_info = None + +available_models = get_valid_models() +LITELLM_SETTINGS.dump_chat_cache = False +LITELLM_SETTINGS.dump_embedding_cache = False +LITELLM_SETTINGS.use_chat_cache = False +LITELLM_SETTINGS.use_embedding_cache = False + + +def convert_defaultdict_to_dict(d): + if isinstance(d, defaultdict): + d = {k: convert_defaultdict_to_dict(v) for k, v in d.items()} + return d + + +def load_data(log_path: Path): + """ + Load and normalize logged data for the UI. + + Meaning of "no_tag": + - We attempt to extract an evolution id (ei) from each message tag. + - If no ei can be extracted (i.e., the entry is not tied to a specific evolving step), + the item is stored under the "no_tag" key. + - Typical "no_tag" entries include: + * direct_exp_gen["no_tag"]: the base experiment/hypothesis for the loop + * coding["no_tag"] / running["no_tag"]: the final workspace/result for that stage + * llm_data[loop_id][function]["no_tag"]: common LLM logs without an ei + """ + data = defaultdict(lambda: defaultdict(dict)) + llm_data = defaultdict(lambda: defaultdict(lambda: defaultdict(list))) + token_costs = defaultdict(list) + + for msg in FileStorage(log_path).iter_msg(): + if not msg.tag: + continue + li, fn = extract_loopid_func_name(msg.tag) + ei = extract_evoid(msg.tag) + if li is not None: + li = int(li) + if ei is not None: + ei = int(ei) + if "debug_" in msg.tag: + if ei is not None: + llm_data[li][fn][ei].append( + { + "tag": msg.tag, + "obj": msg.content, + } + ) + else: + llm_data[li][fn]["no_tag"].append( + { + "tag": msg.tag, + "obj": msg.content, + } + ) + elif "token_cost" in msg.tag: + token_costs[li].append(msg) + elif "llm" not in msg.tag and "session" not in msg.tag and "batch embedding" not in msg.tag: + if msg.tag == "competition": + data["competition"] = msg.content + continue + if "SETTINGS" in msg.tag: + data["settings"][msg.tag] = msg.content + continue + + msg.tag = re.sub(r"\.evo_loop_\d+", "", msg.tag) + msg.tag = re.sub(r"Loop_\d+\.[^.]+\.?", "", msg.tag) + msg.tag = msg.tag.strip() + + if ei is not None: + if ei not in data[li][fn]: + data[li][fn][ei] = {} + data[li][fn][ei][msg.tag] = msg.content + else: + if msg.tag: + data[li][fn][msg.tag] = msg.content + else: + if not isinstance(msg.content, str): + data[li][fn]["no_tag"] = msg.content + + # To be compatible with old version log trace, keep this + llm_log_p = log_path / "debug_llm.pkl" + if llm_log_p.exists(): + try: + rd = pickle.loads(llm_log_p.read_bytes()) # nosec + except: + rd = [] + for d in rd: + t = d["tag"] + if "debug_exp_gen" in t: + continue + if "debug_tpl" in t and "filter_" in d["obj"]["uri"]: + continue + lid, fn = extract_loopid_func_name(t) + ei = extract_evoid(t) + if lid: + lid = int(lid) + if ei is not None: + ei = int(ei) + + if ei is not None: + llm_data[lid][fn][ei].append(d) + else: + llm_data[lid][fn]["no_tag"].append(d) + + return ( + convert_defaultdict_to_dict(data), + convert_defaultdict_to_dict(llm_data), + convert_defaultdict_to_dict(token_costs), + ) + + +if UI_SETTING.enable_cache: + load_data = st.cache_data(persist=True)(load_data) + + +def load_stdout(stdout_path: Path): + if stdout_path.exists(): + stdout = stdout_path.read_text() + else: + stdout = f"Please Set: {stdout_path}" + return stdout + + +# UI windows +def task_win(task): + with st.expander(f"**:violet[{task.name}]**", expanded=False): + st.markdown(task.description) + if hasattr(task, "package_info"): + st.markdown(f"**:blue[Package Info:]**") + st.code(task.package_info) + if hasattr(task, "architecture"): # model task + st.markdown(f""" + | Model_type | Architecture | hyperparameters | + |------------|--------------|-----------------| + | {task.model_type} | {task.architecture} | {task.hyperparameters} | + """) + + +def workspace_win(workspace, cmp_workspace=None, cmp_name="last code."): + show_files = {k: v for k, v in workspace.file_dict.items() if "test" not in k} + if len(show_files) > 0: + if cmp_workspace: + diff = generate_diff_from_dict(cmp_workspace.file_dict, show_files, "main.py") + with st.popover(f":violet[**Diff with {cmp_name}**]", use_container_width=True, icon="🔍"): + st.code("".join(diff), language="diff", wrap_lines=True, line_numbers=True) + + rtime = workspace.running_info.running_time + time_str = timedelta_to_str(timedelta(seconds=rtime) if rtime else None) or "00:00:00" + + with st.popover( + f"⏱️{time_str} 📂Files in :blue[{replace_ep_path(workspace.workspace_path)}]", use_container_width=True + ): + st.write(replace_ep_path(workspace.workspace_path)) + code_tabs = st.tabs(show_files.keys()) + for ct, codename in zip(code_tabs, show_files.keys()): + with ct: + st.code( + show_files[codename], + language=("python" if codename.endswith(".py") else "markdown"), + wrap_lines=True, + line_numbers=True, + ) + + if state.show_save_input: + st.markdown("### Save All Files to Folder") + unique_key = hashlib.md5("".join(show_files.values()).encode(), usedforsecurity=False).hexdigest() + str( + random.randint(0, 10000) + ) + target_folder = st.text_input("Enter target folder path:", key=unique_key) + + if st.button("Save Files", key=f"save_files_button_{unique_key}"): + if target_folder.strip() == "": + st.warning("Please enter a valid folder path.") + else: + target_folder_path = Path(target_folder) + target_folder_path.mkdir(parents=True, exist_ok=True) + for filename, content in workspace.file_dict.items(): + save_path = target_folder_path / filename + save_path.parent.mkdir(parents=True, exist_ok=True) + save_path.write_text(content, encoding="utf-8") + st.success(f"All files saved to: {target_folder}") + else: + st.markdown(f"No files in :blue[{replace_ep_path(workspace.workspace_path)}]") + + +# Helper functions +def show_text(text, lang=None): + """显示文本代码块""" + if lang: + st.code(text, language=lang, wrap_lines=True, line_numbers=True) + elif "\n" in text: + st.code(text, language="python", wrap_lines=True, line_numbers=True) + else: + st.code(text, language="html", wrap_lines=True) + + +def highlight_prompts_uri(uri): + """高亮 URI 的格式""" + parts = uri.split(":") + if len(parts) > 1: + return f"**{parts[0]}:**:green[**{parts[1]}**]" + return f"**{uri}**" + + +def llm_log_win(llm_d: list): + def to_str_recursive(obj): + if isinstance(obj, dict): + return {k: to_str_recursive(v) for k, v in obj.items()} + elif isinstance(obj, list): + return [to_str_recursive(v) for v in obj] + elif isinstance(obj, tuple): + return tuple(to_str_recursive(v) for v in obj) + else: + return str(obj) + + for d in llm_d: + if "debug_tpl" in d["tag"]: + uri = d["obj"]["uri"] + if "filter_redundant_text" in uri: + continue + tpl = d["obj"]["template"] + cxt = d["obj"]["context"] + rd = d["obj"]["rendered"] + with st.popover(highlight_prompts_uri(uri), icon="⚙️", use_container_width=True): + t1, t2, t3 = st.tabs([":green[**Rendered**]", ":blue[**Template**]", ":orange[**Context**]"]) + with t1: + show_text(rd) + with t2: + show_text(tpl, lang="django") + with t3: + st.json(to_str_recursive(cxt)) + elif "debug_llm" in d["tag"]: + system = d["obj"].get("system", None) + user = d["obj"]["user"] + resp = d["obj"]["resp"] + start_time = d["obj"].get("start", "") + end_time = d["obj"].get("end", "") + if start_time and end_time: + start_str = start_time.strftime("%m-%d %H:%M:%S") + end_str = end_time.strftime("%m-%d %H:%M:%S") + duration = end_time - start_time + time_info_str = ( + f"🕰️:blue[**{start_str} ~ {end_str}**] ⏳:violet[**{round(duration.total_seconds(), 2)}s**]" + ) + else: + time_info_str = "" + with st.expander(f"**LLM** {time_info_str}", icon="🤖", expanded=False): + t1, t2, t3, t4 = st.tabs( + [":green[**Response**]", ":blue[**User**]", ":orange[**System**]", ":violet[**ChatBot**]"] + ) + with t1: + try: + rdict = json.loads(resp) + showed_keys = [] + for k, v in rdict.items(): + if k.endswith(".py") or k.endswith(".md"): + st.markdown(f":red[**{k}**]") + st.code(v, language="python", wrap_lines=True, line_numbers=True) + showed_keys.append(k) + for k in showed_keys: + rdict.pop(k) + if len(showed_keys) > 0: + st.write(":red[**Other parts (except for the code or spec) in response dict:**]") + st.json(rdict) + except: + show_text(resp) + with t2: + show_text(user) + with t3: + show_text(system or "No system prompt available") + with t4: + input_c, resp_c = st.columns(2) + key = hashlib.md5(resp.encode(), usedforsecurity=False).hexdigest() + with input_c: + btc1, btc2, btc3 = st.columns(3) + trace_model = ( + state.data.get("settings", {}) + .get("LITELLM_SETTINGS", {}) + .get("chat_model", available_models[0]) + ) + trace_reasoning_effort = ( + state.data.get("settings", {}).get("LITELLM_SETTINGS", {}).get("reasoning_effort", None) + ) + LITELLM_SETTINGS.chat_model = btc1.selectbox( + "Chat Model", + options=available_models, + index=available_models.index(trace_model), + key=key + "_chat_model", + ) + LITELLM_SETTINGS.reasoning_effort = btc2.selectbox( + "Reasoning Effort", + options=[None, "low", "medium", "high"], + index=[None, "low", "medium", "high"].index(trace_reasoning_effort), + key=key + "_reasoning_effort", + ) + rf = btc3.selectbox( + "Response Format", + options=[None, ScenarioChallenges, TraceChallenges, HypothesisList, CodingSketch], + format_func=lambda x: x.__name__ if x else "None", + key=key + "_response_format", + ) + json_mode = st.checkbox("JSON Mode", value=False, key=key + "_json_mode") + sys_p = input_c.text_area(label="system", value=system, height="content", key=key + "_system") + user_p = input_c.text_area(label="user", value=user, height="content", key=key + "_user") + with resp_c: + if st.button("Call LLM", key=key + "_call_llm"): + with st.spinner("Calling LLM..."): + try: + resp_new = APIBackend().build_messages_and_create_chat_completion( + user_prompt=user_p, + system_prompt=sys_p, + json_mode=json_mode, + response_format=rf, + ) + except Exception as e: + resp_new = f"Error: {e}" + try: # json format string + rdict = json.loads(resp_new) + st.json(rdict) + except: + try: # common string + st.code(resp_new, wrap_lines=True, line_numbers=True) + except: # response format type + st.write(resp_new) + + +def hypothesis_win(hypo): + try: + st.code(str(hypo).replace("\n", "\n\n"), wrap_lines=True) + except Exception as e: + st.write(hypo.__dict__) + + +def exp_gen_win(exp_gen_data, llm_data=None): + st.header("Exp Gen", divider="blue", anchor="exp-gen") + if state.show_llm_log and llm_data is not None: + llm_log_win(llm_data["no_tag"]) + st.subheader("💡 Hypothesis") + hypothesis_win(exp_gen_data["no_tag"].hypothesis) + + st.subheader("📋 pending_tasks") + for tasks in exp_gen_data["no_tag"].pending_tasks_list: + task_win(tasks[0]) + st.subheader("📁 Exp Workspace") + workspace_win(exp_gen_data["no_tag"].experiment_workspace) + + +def evolving_win(data, key, llm_data=None, base_workspace=None): + with st.container(border=True): + if len(data) > 1: + evo_id = st.slider("Evolving", 0, len(data) - 1, 0, key=key) + elif len(data) == 1: + evo_id = 0 + else: + st.markdown("No evolving.") + return + + if evo_id in data: + if state.show_llm_log and llm_data is not None: + llm_log_win(llm_data[evo_id]) + + # get evolving workspace + if "evolving code" in data[evo_id] and data[evo_id]["evolving code"][0] is not None: + evolving_code_workspace = data[evo_id]["evolving code"][0] + else: + evolving_code_workspace = None + + if evolving_code_workspace is not None: + st.subheader("codes") + workspace_win( + evolving_code_workspace, + cmp_workspace=data[evo_id - 1]["evolving code"][0] if evo_id > 0 else base_workspace, + cmp_name="last evolving code" if evo_id > 0 else "base workspace", + ) + fb = data[evo_id]["evolving feedback"][0] + st.subheader("evolving feedback" + ("✅" if bool(fb) else "❌")) + f1, f2, f3, f4 = st.tabs(["execution", "return_checking", "code", "others"]) # nosec + other_attributes = { + k: v for k, v in fb.__dict__.items() if k not in ["execution", "return_checking", "code"] # nosec + } + f1.code(fb.execution, wrap_lines=True) # nosec + f2.code(fb.return_checking, wrap_lines=True) + f3.code(fb.code, wrap_lines=True) + f4.json(other_attributes) + else: + st.write("data[evo_id]['evolving code'][0] is None.") + st.write(data[evo_id]) + else: + st.markdown("No evolving.") + + +def coding_win(data, base_exp, llm_data: dict | None = None): + st.header("Coding", divider="blue", anchor="coding") + if llm_data is not None: + common_llm_data = llm_data.pop("no_tag", []) + evolving_data = {k: v for k, v in data.items() if isinstance(k, int)} + task_set = set() + for v in evolving_data.values(): + for t in v: + if "Task" in t.split(".")[0]: + task_set.add(t.split(".")[0]) + if task_set: + # 新版存Task tag的Trace + for task in task_set: + st.subheader(task) + task_data = {k: {a.split(".")[1]: b for a, b in v.items() if task in a} for k, v in evolving_data.items()} + evolving_win( + task_data, + key=task, + llm_data=llm_data if llm_data else None, + base_workspace=base_exp.experiment_workspace, + ) + else: + # 旧版未存Task tag的Trace + evolving_win( + evolving_data, + key="coding", + llm_data=llm_data if llm_data else None, + base_workspace=base_exp.experiment_workspace, + ) + if state.show_llm_log: + llm_log_win(common_llm_data) + if "no_tag" in data: + st.subheader("Exp Workspace (coding final)") + workspace_win(data["no_tag"].experiment_workspace) + + +def running_win(data, base_exp, llm_data=None, last_sota_exp=None): + st.header("Running", divider="blue", anchor="running") + if llm_data is not None: + common_llm_data = llm_data.pop("no_tag", []) + evolving_win( + {k: v for k, v in data.items() if isinstance(k, int)}, + key="running", + llm_data=llm_data if llm_data else None, + base_workspace=base_exp.experiment_workspace if base_exp else None, + ) + if state.show_llm_log and llm_data is not None: + llm_log_win(common_llm_data) + if "no_tag" in data: + st.subheader("Exp Workspace (running final)") + workspace_win( + data["no_tag"].experiment_workspace, + cmp_workspace=last_sota_exp.experiment_workspace if last_sota_exp else None, + cmp_name="last SOTA(to_submit)", + ) + st.subheader("Result") + try: + st.write(data["no_tag"].result) + except AttributeError as e: # Compatible with old versions + st.write(data["no_tag"].__dict__["result"]) + mle_score_text = data.get("mle_score", "no submission to score") + mle_score = extract_json(mle_score_text) + st.subheader( + "MLE Submission Score" + + ("✅" if (isinstance(mle_score, dict) and mle_score["score"] is not None) else "❌") + ) + if isinstance(mle_score, dict): + st.json(mle_score) + else: + st.code(mle_score_text, wrap_lines=True) + + +def feedback_win(fb_data, llm_data=None): + if "no_tag" not in fb_data: + st.header("Feedback", divider="orange", anchor="feedback") + return + fb = fb_data["no_tag"] + st.header("Feedback" + ("✅" if bool(fb) else "❌"), divider="orange", anchor="feedback") + if state.show_llm_log and llm_data is not None: + llm_log_win(llm_data["no_tag"]) + try: + st.code(str(fb).replace("\n", "\n\n"), wrap_lines=True) + except Exception as e: + st.write(fb.__dict__) + if fb.exception is not None: + st.markdown(f"**:red[Exception]**: {fb.exception}") + + +def sota_win(sota_exp, trace): + st.subheader("SOTA Experiment", divider="rainbow", anchor="sota-exp") + if hasattr(trace, "sota_exp_to_submit") and trace.sota_exp_to_submit is not None: + st.markdown(":orange[trace.**sota_exp_to_submit**]") + sota_exp = trace.sota_exp_to_submit + else: + st.markdown(":orange[trace.**sota_experiment()**]") + + if sota_exp: + st.markdown(f"**SOTA Exp Hypothesis**") + hypothesis_win(sota_exp.hypothesis) + st.markdown("**Exp Workspace**") + workspace_win(sota_exp.experiment_workspace) + else: + st.markdown("No SOTA experiment.") + + +def main_win(loop_id, llm_data=None): + loop_data = state.data[loop_id] + exp_gen_win(loop_data["direct_exp_gen"], llm_data["direct_exp_gen"] if llm_data else None) + if "coding" in loop_data: + coding_win( + loop_data["coding"], + base_exp=loop_data["direct_exp_gen"]["no_tag"], + llm_data=llm_data["coding"] if llm_data else None, + ) + if "running" in loop_data: + # get last SOTA_exp_to_submit + last_sota_exp = None + if "record" in loop_data: + current_trace = loop_data["record"]["trace"] + current_selection = current_trace.get_current_selection() + if len(current_selection) > 0: # TODO: Why current_selection can be "()"? + current_idx = current_selection[0] + parent_idxs = current_trace.get_parents(current_idx) + if len(parent_idxs) >= 2 and hasattr(current_trace, "idx2loop_id"): + parent_idx = parent_idxs[-2] + parent_loop_id = current_trace.idx2loop_id[parent_idx] + if parent_loop_id in state.data: + # in some cases, the state.data is synthesized, logs does not necessarily exist + last_sota_exp = state.data[parent_loop_id]["record"].get("sota_exp_to_submit", None) + + running_win( + loop_data["running"], + base_exp=loop_data["coding"].get("no_tag", None), + llm_data=llm_data["running"] if llm_data else None, + last_sota_exp=last_sota_exp, + ) + if "feedback" in loop_data: + # Show final diff between the final workspace and the base workspace + base_workspace = loop_data["direct_exp_gen"]["no_tag"].experiment_workspace + final_workspace = None + if "running" in loop_data and "no_tag" in loop_data["running"]: + final_workspace = loop_data["running"]["no_tag"].experiment_workspace + elif "coding" in loop_data and "no_tag" in loop_data["coding"]: + final_workspace = loop_data["coding"]["no_tag"].experiment_workspace + + if final_workspace is not None and base_workspace is not None: + st.subheader("Final Diff") + workspace_win(final_workspace, cmp_workspace=base_workspace, cmp_name="base workspace") + + feedback_win(loop_data["feedback"], llm_data.get("feedback", None) if llm_data else None) + if "record" in loop_data and "SOTA experiment" in loop_data["record"]: + st.header("Record", divider="violet", anchor="record") + if state.show_llm_log and llm_data is not None and "record" in llm_data: + llm_log_win(llm_data["record"]["no_tag"]) + sota_win(loop_data["record"]["SOTA experiment"], loop_data["record"]["trace"]) + + +def replace_ep_path(p: Path): + # 替换workspace path为对应ep机器mount在ep03的path + # TODO: FIXME: 使用配置项来处理 + match = re.search(r"ep\d+", str(state.log_folder)) + if match: + ep = match.group(0) + return Path( + str(p).replace("repos/RD-Agent-Exp", f"repos/batch_ctrl/all_projects/{ep}").replace("/Data", "/data") + ) + return p + + +def get_llm_call_stats(llm_data: dict) -> tuple[int, int]: + total_llm_call = 0 + total_filter_call = 0 + total_call_duration = timedelta() + filter_call_duration = timedelta() + filter_sys_prompt = T("rdagent.utils.prompts:filter_redundant_text.system").r() + for li, loop_d in llm_data.items(): + for fn, loop_fn_d in loop_d.items(): + for k, v in loop_fn_d.items(): + for d in v: + if "debug_llm" in d["tag"]: + total_llm_call += 1 + total_call_duration += d["obj"].get("end", timedelta()) - d["obj"].get("start", timedelta()) + if "system" in d["obj"] and filter_sys_prompt == d["obj"]["system"]: + total_filter_call += 1 + filter_call_duration += d["obj"].get("end", timedelta()) - d["obj"].get( + "start", timedelta() + ) + + return total_llm_call, total_filter_call, total_call_duration, filter_call_duration + + +def get_timeout_stats(llm_data: dict): + timeout_stat = { + "coding": { + "total": 0, + "timeout": 0, + }, + "running": { + "total": 0, + "timeout": 0, + }, + } + for li, loop_d in llm_data.items(): + for fn, loop_fn_d in loop_d.items(): + for k, v in loop_fn_d.items(): + for d in v: + if "debug_tpl" in d["tag"] and "eval.user" in d["obj"]["uri"] and "stdout" in d["obj"]["context"]: # nosec + stdout = d["obj"]["context"]["stdout"] + if "The running time exceeds" in stdout: # Timeout case + timeout_stat[fn]["timeout"] += 1 + timeout_stat[fn]["total"] += 1 + + return timeout_stat + + +def timedelta_to_str(td: timedelta | None) -> str: + if isinstance(td, timedelta): + total_seconds = int(td.total_seconds()) + hours = total_seconds // 3600 + minutes = (total_seconds % 3600) // 60 + seconds = total_seconds % 60 + return f"{hours:02d}:{minutes:02d}:{seconds:02d}" + return td + + +def summarize_win(): + st.header("Summary", divider="rainbow") + with st.container(border=True): + min_id, max_id = get_state_data_range(state.data) + info0, info1, info2, info3, info4, info5, info6, info7 = st.columns(8) + show_trace_dag = info0.toggle("Show trace DAG", key="show_trace_dag") + only_success = info0.toggle("Only Success", key="only_success") + with info1.popover("LITELLM", icon="⚙️"): + st.write(state.data.get("settings", {}).get("LITELLM_SETTINGS", "No settings found.")) + with info2.popover("RD_AGENT", icon="⚙️"): + st.write(state.data.get("settings", {}).get("RD_AGENT_SETTINGS", "No settings found.")) + with info3.popover("RDLOOP", icon="⚙️"): + st.write(state.data.get("settings", {}).get("RDLOOP_SETTINGS", "No settings found.")) + + llm_call, llm_filter_call, llm_call_duration, filter_call_duration = get_llm_call_stats(state.llm_data) + info4.metric("LLM Calls", llm_call, help=timedelta_to_str(llm_call_duration)) + info5.metric( + "LLM Filter Calls", + llm_filter_call, + help=timedelta_to_str(filter_call_duration), + ) + + timeout_stats = get_timeout_stats(state.llm_data) + coding_timeout_pct = ( + round(timeout_stats["coding"]["timeout"] / timeout_stats["coding"]["total"] * 100, 2) + if timeout_stats["coding"]["total"] > 0 + else 0 + ) + info6.metric( + "Timeouts (C)", + f"{coding_timeout_pct}%", + help=f"{timeout_stats['coding']['timeout']}/{timeout_stats['coding']['total']}", + ) + running_timeout_pct = ( + round(timeout_stats["running"]["timeout"] / timeout_stats["running"]["total"] * 100, 2) + if timeout_stats["running"]["total"] > 0 + else 0 + ) + info7.metric( + "Timeouts (R)", + f"{running_timeout_pct}%", + help=f"{timeout_stats['running']['timeout']}/{timeout_stats['running']['total']}", + ) + + final_trace = list(FileStorage(state.log_folder / state.log_path).iter_msg(tag="record.trace"))[-1].content + if show_trace_dag: + st.markdown("### Trace DAG") + merge_loops = [] + for loop_id in state.llm_data.keys(): + if "direct_exp_gen" not in state.llm_data[loop_id]: + continue + if "scenarios.data_science.proposal.exp_gen.merge" in "".join( + [i["obj"]["uri"] for i in state.llm_data[loop_id]["direct_exp_gen"]["no_tag"] if "uri" in i["obj"]] + ): + merge_loops.append(loop_id) + st.pyplot(trace_figure(final_trace, merge_loops)) + + # Find all root nodes (for grouping loops by trace) + root_nodes = {} + parent_nodes = {} + for node in range(len(final_trace.hist)): + parents = final_trace.get_parents(node) + root_nodes[node] = parents[0] + parent_nodes[node] = parents[-2] if len(parents) > 1 else None + if hasattr(final_trace, "idx2loop_id"): + root_nodes = {final_trace.idx2loop_id[n]: final_trace.idx2loop_id[r] for n, r in root_nodes.items()} + parent_nodes = { + final_trace.idx2loop_id[n]: final_trace.idx2loop_id[r] if r is not None else r + for n, r in parent_nodes.items() + } + + # Generate Summary Table + df = pd.DataFrame( + columns=[ + "Root N", + "Parent N", + "Component", + "Hypothesis", + "Reason", + "Others", + "Run Score (valid)", + "Run Score (test)", + "Feedback", + "e-loops(c)", + "e-loops(r)", + "COST($)", + "Time", + "Exp Gen", + "Coding", + "Running", + ], + index=range(min_id, max_id + 1), + ) + + valid_results = {} + sota_loop_id = state.sota_info[1] if state.sota_info else None + for loop in range(min_id, max_id + 1): + loop_data = state.data[loop] + df.loc[loop, "Parent N"] = parent_nodes.get(loop, None) + df.loc[loop, "Root N"] = root_nodes.get(loop, None) + df.loc[loop, "Component"] = loop_data["direct_exp_gen"]["no_tag"].hypothesis.component + df.loc[loop, "Hypothesis"] = loop_data["direct_exp_gen"]["no_tag"].hypothesis.hypothesis + df.loc[loop, "Reason"] = loop_data["direct_exp_gen"]["no_tag"].hypothesis.reason + df.at[loop, "Others"] = { + k: v + for k, v in loop_data["direct_exp_gen"]["no_tag"].hypothesis.__dict__.items() + if k not in ["component", "hypothesis", "reason"] and v is not None + } + # In the test before 0.8.0 release, we found that when running `ui` of `data_science` (custom dataset), + # when `loop=0`, it doesn't exist in `state.token_costs.keys`, and we will get `KeyError` when running it, + # so we have fixed the problem with this dirty method for the time being. + if loop in state.token_costs: + df.loc[loop, "COST($)"] = sum(tc.content["cost"] for tc in state.token_costs[loop]) + + # Time Stats + exp_gen_time = timedelta() + coding_time = timedelta() + running_time = timedelta() + all_steps_time = timedelta() + if loop in state.times: + for step_name, step_time in state.times[loop].items(): + step_duration = step_time["end_time"] - step_time["start_time"] + if step_name == "exp_gen": + exp_gen_time += step_duration + all_steps_time += step_duration + elif step_name == "coding": + coding_time += step_duration + all_steps_time += step_duration + elif step_name == "running": + running_time += step_duration + all_steps_time += step_duration + elif step_name in ["feedback", "record"]: + all_steps_time += step_duration + df.loc[loop, "Time"] = timedelta_to_str(all_steps_time) + df.loc[loop, "Exp Gen"] = timedelta_to_str(exp_gen_time) + df.loc[loop, "Coding"] = timedelta_to_str(coding_time) + df.loc[loop, "Running"] = timedelta_to_str(running_time) + + if "running" in loop_data and "no_tag" in loop_data["running"]: + try: + try: + running_result = loop_data["running"]["no_tag"].result + except AttributeError as e: # Compatible with old versions + running_result = loop_data["running"]["no_tag"].__dict__["result"] + df.loc[loop, "Run Score (valid)"] = str(round(running_result.loc["ensemble"].iloc[0], 5)) + valid_results[loop] = running_result + except: + df.loc[loop, "Run Score (valid)"] = "❌" + if "mle_score" not in state.data[loop]: + if "mle_score" in loop_data["running"]: + mle_score_txt = loop_data["running"]["mle_score"] + state.data[loop]["mle_score"] = extract_json(mle_score_txt) + if ( + state.data[loop]["mle_score"] is not None + and state.data[loop]["mle_score"]["score"] is not None + ): + medal_emoji = ( + "🥇" + if state.data[loop]["mle_score"]["gold_medal"] + else ( + "🥈" + if state.data[loop]["mle_score"]["silver_medal"] + else "🥉" if state.data[loop]["mle_score"]["bronze_medal"] else "" + ) + ) + df.loc[loop, "Run Score (test)"] = f"{medal_emoji} {state.data[loop]['mle_score']['score']}" + else: + state.data[loop]["mle_score"] = mle_score_txt + df.loc[loop, "Run Score (test)"] = "❌" + else: + mle_score_path = ( + replace_ep_path(loop_data["running"]["no_tag"].experiment_workspace.workspace_path) + / "mle_score.txt" + ) + try: + mle_score_txt = mle_score_path.read_text() + state.data[loop]["mle_score"] = extract_json(mle_score_txt) + if state.data[loop]["mle_score"]["score"] is not None: + medal_emoji = ( + "🥇" + if state.data[loop]["mle_score"]["gold_medal"] + else ( + "🥈" + if state.data[loop]["mle_score"]["silver_medal"] + else "🥉" if state.data[loop]["mle_score"]["bronze_medal"] else "" + ) + ) + df.loc[loop, "Run Score (test)"] = ( + f"{medal_emoji} {state.data[loop]['mle_score']['score']}" + ) + else: + state.data[loop]["mle_score"] = mle_score_txt + df.loc[loop, "Run Score (test)"] = "❌" + except Exception as e: + state.data[loop]["mle_score"] = str(e) + df.loc[loop, "Run Score (test)"] = "❌" + else: + if isinstance(state.data[loop]["mle_score"], dict): + medal_emoji = ( + "🥇" + if state.data[loop]["mle_score"]["gold_medal"] + else ( + "🥈" + if state.data[loop]["mle_score"]["silver_medal"] + else "🥉" if state.data[loop]["mle_score"]["bronze_medal"] else "" + ) + ) + df.loc[loop, "Run Score (test)"] = f"{medal_emoji} {state.data[loop]['mle_score']['score']}" + else: + df.loc[loop, "Run Score (test)"] = "❌" + + else: + df.loc[loop, "Run Score (valid)"] = "N/A" + df.loc[loop, "Run Score (test)"] = "N/A" + + if "coding" in loop_data: + if len([i for i in loop_data["coding"].keys() if isinstance(i, int)]) == 0: + df.loc[loop, "e-loops(c)"] = 0 + else: + df.loc[loop, "e-loops(c)"] = max(i for i in loop_data["coding"].keys() if isinstance(i, int)) + 1 + if "running" in loop_data: + if len([i for i in loop_data["running"].keys() if isinstance(i, int)]) == 0: + df.loc[loop, "e-loops(r)"] = 0 + else: + df.loc[loop, "e-loops(r)"] = max(i for i in loop_data["running"].keys() if isinstance(i, int)) + 1 + if "feedback" in loop_data: + fb_emoji_str = ( + "✅" if "no_tag" in loop_data["feedback"] and bool(loop_data["feedback"]["no_tag"]) else "❌" + ) + if sota_loop_id == loop: + fb_emoji_str += " (💖SOTA)" + df.loc[loop, "Feedback"] = fb_emoji_str + else: + df.loc[loop, "Feedback"] = "N/A" + + if only_success: + df = df[df["Feedback"].str.contains("✅", na=False)] + + # Add color styling based on root_nodes + def style_dataframe_by_root(df, root_nodes): + # Create a color map for different root nodes - using colors that work well in both light and dark modes + unique_roots = list(set(root_nodes.values())) + colors = [ + "rgba(255, 99, 132, 0.3)", + "rgba(54, 162, 235, 0.3)", + "rgba(75, 192, 75, 0.3)", + "rgba(255, 159, 64, 0.3)", + "rgba(153, 102, 255, 0.2)", + "rgba(255, 205, 86, 0.2)", + "rgba(199, 199, 199, 0.2)", + "rgba(83, 102, 255, 0.2)", + ] + root_color_map = {root: colors[i % len(colors)] for i, root in enumerate(unique_roots)} + + # Create styling function + def apply_color(row): + loop_id = row.name + if loop_id in root_nodes: + root_id = root_nodes[loop_id] + color = root_color_map.get(root_id, "rgba(128, 128, 128, 0.1)") + return [f"background-color: {color}"] * len(row) + return [""] * len(row) + + return df.style.apply(apply_color, axis=1) + + styled_df = style_dataframe_by_root( + df[df.columns[~df.columns.isin(["Hypothesis", "Reason", "Others"])]], root_nodes + ) + st.dataframe(styled_df) + + # timeline figure + if state.times: + with st.popover("Timeline", icon="⏱️", use_container_width=True): + st.plotly_chart(timeline_figure(state.times)) + + # scores curve + vscores = {} + for k, vs in valid_results.items(): + if not vs.index.is_unique: + st.warning(f"Loop {k}'s valid scores index are not unique, only the last one will be kept to show.") + st.write(vs) + vscores[k] = vs[~vs.index.duplicated(keep="last")].iloc[:, 0] + if len(vscores) > 0: + metric_name = list(vscores.values())[0].name + else: + metric_name = "None" + vscores = pd.DataFrame(vscores) + if "ensemble" in vscores.index: + ensemble_row = vscores.loc[["ensemble"]] + vscores = pd.concat([ensemble_row, vscores.drop("ensemble")]) + vscores = vscores.T + test_scores = df["Run Score (test)"].str.replace(r"[🥇🥈🥉]\s*", "", regex=True) + vscores["test"] = test_scores + vscores.index = [f"L{i}" for i in vscores.index] + vscores.columns.name = metric_name + with st.popover("Scores Curve", icon="📈", use_container_width=True): + st.plotly_chart(curve_figure(vscores)) + + st.markdown("### Hypotheses Table") + hypotheses_df = df.iloc[:, :8].copy() + others_expanded = pd.json_normalize(hypotheses_df["Others"].fillna({})) + others_expanded.index = hypotheses_df.index + + hypotheses_df = hypotheses_df.drop("Others", axis=1) + hypotheses_df = hypotheses_df.drop("Parent N", axis=1) + hypotheses_df = pd.concat([hypotheses_df.iloc[:, :4], others_expanded, hypotheses_df.iloc[:, 4:]], axis=1) + + styled_hypotheses_table = style_dataframe_by_root(hypotheses_df, root_nodes) + st.dataframe( + styled_hypotheses_table, + row_height=100, + column_config={ + k: st.column_config.TextColumn( + k, + width=( + "small" + if k + in ["Component", "Root N", "Parent N", "Run Score (valid)", "Run Score (test)", "problem_label"] + else "medium" + ), + ) + for k in hypotheses_df.columns + }, + ) + + def comp_stat_func(x: pd.DataFrame): + total_num = x.shape[0] + valid_num = x[x["Run Score (test)"] != "N/A"].shape[0] + success_num = x[x["Feedback"] == "✅"].shape[0] + avg_e_loops = x["e-loops(c)"].mean() + return pd.Series( + { + "Loop Num": total_num, + "Valid Loop": valid_num, + "Success Loop": success_num, + "Valid Rate": round(valid_num / total_num * 100, 2), + "Success Rate": round(success_num / total_num * 100, 2), + "Avg e-loops(c)": round(avg_e_loops, 2), + } + ) + + st1, st2 = st.columns([1, 1]) + + # component statistics + comp_df = ( + df.loc[:, ["Component", "Run Score (test)", "Feedback", "e-loops(c)"]] + .groupby("Component") + .apply(comp_stat_func, include_groups=False) + ) + comp_df.loc["Total"] = comp_df.sum() + comp_df.loc["Total", "Valid Rate"] = round( + comp_df.loc["Total", "Valid Loop"] / comp_df.loc["Total", "Loop Num"] * 100, 2 + ) + comp_df.loc["Total", "Success Rate"] = round( + comp_df.loc["Total", "Success Loop"] / comp_df.loc["Total", "Loop Num"] * 100, 2 + ) + comp_df["Valid Rate"] = comp_df["Valid Rate"].apply(lambda x: f"{x}%") + comp_df["Success Rate"] = comp_df["Success Rate"].apply(lambda x: f"{x}%") + comp_df.loc["Total", "Avg e-loops(c)"] = round(df["e-loops(c)"].mean(), 2) + with st2.popover("Component Statistics", icon="📊", use_container_width=True): + st.dataframe(comp_df) + + # component time statistics + time_df = df.loc[:, ["Component", "Time", "Exp Gen", "Coding", "Running"]] + time_df = time_df.astype( + { + "Time": "timedelta64[ns]", + "Exp Gen": "timedelta64[ns]", + "Coding": "timedelta64[ns]", + "Running": "timedelta64[ns]", + } + ) + time_stat_df = time_df.groupby("Component").sum() + time_stat_df.loc["Total"] = time_stat_df.sum() + time_stat_df.loc[:, "Exp Gen(%)"] = (time_stat_df["Exp Gen"] / time_stat_df["Time"] * 100).round(2) + time_stat_df.loc[:, "Coding(%)"] = (time_stat_df["Coding"] / time_stat_df["Time"] * 100).round(2) + time_stat_df.loc[:, "Running(%)"] = (time_stat_df["Running"] / time_stat_df["Time"] * 100).round(2) + for col in ["Time", "Exp Gen", "Coding", "Running"]: + time_stat_df[col] = time_stat_df[col].map(timedelta_to_str) + with st1.popover("Time Statistics", icon="⏱️", use_container_width=True): + st.dataframe(time_stat_df) + + # COST curve + costs = df["COST($)"].astype(float) + costs.index = [f"L{i}" for i in costs.index] + cumulative_costs = costs.cumsum() + with st.popover("COST Curve", icon="💰", use_container_width=True): + fig = px.line( + x=costs.index, + y=[costs.values, cumulative_costs.values], + labels={"x": "Loop", "value": "COST($)"}, + title="COST($) per Loop & Cumulative COST($)", + markers=True, + ) + fig.update_traces(mode="lines+markers") + fig.data[0].name = "COST($) per Loop" + fig.data[1].name = "Cumulative COST($)" + st.plotly_chart(fig) + + +def stdout_win(loop_id: int): + stdout = load_stdout(state.log_folder / f"{state.log_path}.stdout") + if stdout.startswith("Please Set"): + st.toast(stdout, icon="🟡") + return + start_index = stdout.find(f"Start Loop {loop_id}") + end_index = stdout.find(f"Start Loop {loop_id + 1}") + loop_stdout = LogColors.remove_ansi_codes(stdout[start_index:end_index]) + with st.container(border=True): + st.subheader(f"Loop {loop_id} stdout") + pattern = f"Start Loop {loop_id}, " + r"Step \d+: \w+" + matches = re.finditer(pattern, loop_stdout) + step_stdouts = {} + for match in matches: + step = match.group(0) + si = match.start() + ei = loop_stdout.find(f"Start Loop {loop_id}", match.end()) + step_stdouts[step] = loop_stdout[si:ei].strip() + + for k, v in step_stdouts.items(): + with st.expander(k, expanded=False): + st.code(v, language="log", wrap_lines=True) + + +def get_folders_sorted(log_path, sort_by_time=False): + """ + Cache and return the sorted list of folders, with progress printing. + :param log_path: Log path + :param sort_by_time: Whether to sort by time, default False (sort by name) + """ + if not log_path.exists(): + st.toast(f"Path {log_path} does not exist!") + return [] + with st.spinner("Loading folder list..."): + folders = [folder for folder in log_path.iterdir() if is_valid_session(folder)] + if sort_by_time: + folders = sorted(folders, key=lambda folder: folder.stat().st_mtime, reverse=True) + else: + folders = sorted(folders, key=lambda folder: folder.name) + return [folder.name for folder in folders] + + +# UI - Sidebar +with st.sidebar: + # TODO: 只是临时的功能 + if any("log.srv" in folder for folder in state.log_folders): + day_map = {"srv": "最近(srv)", "srv2": "上一批(srv2)", "srv3": "上上批(srv3)"} + day_srv = st.radio("选择批次", ["srv", "srv2", "srv3"], format_func=lambda x: day_map[x], horizontal=True) + if day_srv == "srv": + state.log_folders = [re.sub(r"log\.srv\d*", "log.srv", folder) for folder in state.log_folders] + elif day_srv == "srv2": + state.log_folders = [re.sub(r"log\.srv\d*", "log.srv2", folder) for folder in state.log_folders] + elif day_srv == "srv3": + state.log_folders = [re.sub(r"log\.srv\d*", "log.srv3", folder) for folder in state.log_folders] + + if "log_folder" in st.query_params: + state.log_folder = Path(st.query_params["log_folder"]) + state.log_folders = [str(state.log_folder)] + else: + state.log_folder = Path( + st.radio( + f"Select :blue[**one log folder**]", + state.log_folders, + format_func=lambda x: x[x.rfind("amlt") + 5 :].split("/")[0] if "amlt" in x else x, + ) + ) + if not state.log_folder.exists(): + st.warning(f"Path {state.log_folder} does not exist!") + else: + folders = get_folders_sorted(state.log_folder, sort_by_time=False) + if "selection" in st.query_params: + default_index = ( + folders.index(st.query_params["selection"]) if st.query_params["selection"] in folders else 0 + ) + else: + default_index = 0 + state.log_path = st.selectbox( + f"Select from :blue[**{state.log_folder.absolute()}**]", folders, index=default_index # nosec B608 — not SQL, Bandit false positive on "Select" in UI label + ) + + if st.button("Refresh Data"): + if state.log_path is None: + st.toast("Please select a log path first!", icon="🟡") + st.stop() + + state.times = load_times_info(state.log_folder / state.log_path) + state.data, state.llm_data, state.token_costs = load_data(state.log_folder / state.log_path) + state.sota_info = get_sota_exp_stat(Path(state.log_folder) / state.log_path, selector="auto") + st.rerun() + st.toggle("**Show LLM Log**", key="show_llm_log") + st.toggle("*Show stdout*", key="show_stdout") + st.toggle("*Show save workspace*", key="show_save_input") + st.markdown(f""" +- [Summary](#summary) +- [Exp Gen](#exp-gen) +- [Coding](#coding) +- [Running](#running) +- [Feedback](#feedback) +- [Record](#record) + - [SOTA Experiment](#sota-exp) +""") + + +def get_state_data_range(state_data): + # we have a "competition" key in state_data + # like dict_keys(['competition', 10, 11, 12, 13, 14]) + keys = [ + k + for k in state_data.keys() + if isinstance(k, int) and "direct_exp_gen" in state_data[k] and "no_tag" in state_data[k]["direct_exp_gen"] + ] + return min(keys), max(keys) + + +# UI - Main +if "competition" in state.data: + st.title( + state.data["competition"] + + f" ([share_link](/ds_trace?log_folder={state.log_folder}&selection={state.log_path}))" + ) + summarize_win() + min_id, max_id = get_state_data_range(state.data) + if max_id > min_id: + loop_id = st.slider("Loop", min_id, max_id, min_id) + else: + loop_id = min_id + if state.show_stdout: + stdout_win(loop_id) + main_win(loop_id, state.llm_data[loop_id] if loop_id in state.llm_data else None) diff --git a/rdagent/log/ui/llm_st.py b/rdagent/log/ui/llm_st.py new file mode 100644 index 00000000..3ae8c34a --- /dev/null +++ b/rdagent/log/ui/llm_st.py @@ -0,0 +1,306 @@ +import argparse +import json +import pickle # nosec +import re +import time +from pathlib import Path + +import streamlit as st +from streamlit import session_state + +from rdagent.log.ui.conf import UI_SETTING +from rdagent.log.utils import extract_evoid, extract_loopid_func_name + +st.set_page_config(layout="wide", page_title="debug_llm", page_icon="🎓", initial_sidebar_state="expanded") + +# 获取 log_path 参数 +parser = argparse.ArgumentParser(description="RD-Agent Streamlit App") +parser.add_argument("--log_dir", type=str, help="Path to the log directory") +args = parser.parse_args() + + +def get_folders_sorted(log_path): + """缓存并返回排序后的文件夹列表，并加入进度打印""" + with st.spinner("正在加载文件夹列表..."): + folders = sorted( + (folder for folder in log_path.iterdir() if folder.is_dir() and list(folder.iterdir())), + key=lambda folder: folder.stat().st_mtime, + reverse=True, + ) + st.write(f"找到 {len(folders)} 个文件夹") + return [folder.name for folder in folders] + + +if UI_SETTING.enable_cache: + get_folders_sorted = st.cache_data(get_folders_sorted) + + +# 设置主日志路径 +main_log_path = Path(args.log_dir) if args.log_dir else Path("./log") +if not main_log_path.exists(): + st.error(f"Log dir {main_log_path} does not exist!") + st.stop() + +if "data" not in session_state: + session_state.data = [] +if "log_path" not in session_state: + session_state.log_path = None + +tlist = [] + + +def load_data(): + """加载数据到 session_state 并显示进度""" + log_file = main_log_path / session_state.log_path / "debug_llm.pkl" + try: + with st.spinner(f"正在加载数据文件 {log_file}..."): + start_time = time.time() + with open(log_file, "rb") as f: + session_state.data = pickle.load(f, encoding="utf-8") # nosec + st.success(f"数据加载完成！耗时 {time.time() - start_time:.2f} 秒") + st.session_state["current_loop"] = 1 + except Exception as e: + session_state.data = [{"error": str(e)}] + st.error(f"加载数据失败: {e}") + + +# UI - Sidebar +with st.sidebar: + st.markdown(":blue[**Log Path**]") + manually = st.toggle("Manual Input") + if manually: + st.text_input("log path", key="log_path", label_visibility="collapsed") + else: + folders = get_folders_sorted(main_log_path) + st.selectbox(f"**Select from {main_log_path.absolute()}**", folders, key="log_path") # nosec B608 — not SQL, Bandit false positive on "Select" in UI label + + if st.button("Refresh Data"): + load_data() + st.rerun() + + +# Helper functions +def show_text(text, lang=None): + """显示文本代码块""" + if lang: + st.code(text, language=lang, wrap_lines=True) + elif "\n" in text: + st.code(text, language="python", wrap_lines=True) + else: + st.code(text, language="html", wrap_lines=True) + + +def highlight_prompts_uri(uri): + """高亮 URI 的格式""" + parts = uri.split(":") + return f"**{parts[0]}:**:green[**{parts[1]}**]" + + +# Display Data +progress_text = st.empty() +progress_bar = st.progress(0) + +# 每页展示一个 Loop +LOOPS_PER_PAGE = 1 + +# 获取所有的 Loop ID +loop_groups = {} +for i, d in enumerate(session_state.data): + tag = d["tag"] + loop_id, _ = extract_loopid_func_name(tag) + if loop_id: + if loop_id not in loop_groups: + loop_groups[loop_id] = [] + loop_groups[loop_id].append(d) + +# 按 Loop ID 排序 +sorted_loop_ids = sorted(loop_groups.keys(), key=int) # 假设 Loop ID 是数字 +total_loops = len(sorted_loop_ids) +total_pages = total_loops # 每页展示一个 Loop + + +# simple display +# FIXME: Delete this simple UI if trace have tag(evo_id & loop_id) +# with st.sidebar: +# start = int(st.text_input("start", 0)) +# end = int(st.text_input("end", 100)) +# for m in session_state.data[start:end]: +# if "tpl" in m["tag"]: +# obj = m["obj"] +# uri = obj["uri"] +# tpl = obj["template"] +# cxt = obj["context"] +# rd = obj["rendered"] +# with st.expander(highlight_prompts_uri(uri), expanded=False, icon="⚙️"): +# t1, t2, t3 = st.tabs([":green[**Rendered**]", ":blue[**Template**]", ":orange[**Context**]"]) +# with t1: +# show_text(rd) +# with t2: +# show_text(tpl, lang="django") +# with t3: +# st.json(cxt) +# if "llm" in m["tag"]: +# obj = m["obj"] +# system = obj.get("system", None) +# user = obj["user"] +# resp = obj["resp"] +# with st.expander(f"**LLM**", expanded=False, icon="🤖"): +# t1, t2, t3 = st.tabs([":green[**Response**]", ":blue[**User**]", ":orange[**System**]"]) +# with t1: +# try: +# rdict = json.loads(resp) +# if "code" in rdict: +# code = rdict["code"] +# st.markdown(":red[**Code in response dict:**]") +# st.code(code, language="python", wrap_lines=True, line_numbers=True) +# rdict.pop("code") +# elif "spec" in rdict: +# spec = rdict["spec"] +# st.markdown(":red[**Spec in response dict:**]") +# st.markdown(spec) +# rdict.pop("spec") +# else: +# # show model codes +# showed_keys = [] +# for k, v in rdict.items(): +# if k.startswith("model_") and k.endswith(".py"): +# st.markdown(f":red[**{k}**]") +# st.code(v, language="python", wrap_lines=True, line_numbers=True) +# showed_keys.append(k) +# for k in showed_keys: +# rdict.pop(k) +# st.write(":red[**Other parts (except for the code or spec) in response dict:**]") +# st.json(rdict) +# except: +# st.json(resp) +# with t2: +# show_text(user) +# with t3: +# show_text(system or "No system prompt available") + + +if total_pages: + # 初始化 current_loop + if "current_loop" not in st.session_state: + st.session_state["current_loop"] = 1 + + # Loop 导航按钮 + col1, col2, col3, col4, col5 = st.sidebar.columns([1.2, 1, 2, 1, 1.2]) + + with col1: + if st.button("|<"): # 首页 + st.session_state["current_loop"] = 1 + with col2: + if st.button("<") and st.session_state["current_loop"] > 1: # 上一页 + st.session_state["current_loop"] -= 1 + with col3: + # 下拉列表显示所有 Loop + st.session_state["current_loop"] = st.selectbox( + "选择 Loop", + options=list(range(1, total_loops + 1)), + index=st.session_state["current_loop"] - 1, # 默认选中当前 Loop + label_visibility="collapsed", # 隐藏标签 + ) + with col4: + if st.button("\>") and st.session_state["current_loop"] < total_loops: # 下一页 + st.session_state["current_loop"] += 1 + with col5: + if st.button("\>|"): # 最后一页 + st.session_state["current_loop"] = total_loops + + # 获取当前 Loop + current_loop = st.session_state["current_loop"] + + # 渲染当前 Loop 数据 + loop_id = sorted_loop_ids[current_loop - 1] + progress_text = st.empty() + progress_text.text(f"正在处理 Loop {loop_id}...") + progress_bar.progress(current_loop / total_loops, text=f"Loop :green[**{current_loop}**] / {total_loops}") + + # 渲染 Loop Header + loop_anchor = f"Loop_{loop_id}" + if loop_anchor not in tlist: + tlist.append(loop_anchor) + st.header(loop_anchor, anchor=loop_anchor, divider="blue") + + # 渲染当前 Loop 的所有数据 + loop_data = loop_groups[loop_id] + for d in loop_data: + tag = d["tag"] + obj = d["obj"] + _, func_name = extract_loopid_func_name(tag) + evo_id = extract_evoid(tag) + + func_anchor = f"loop_{loop_id}.{func_name}" + if func_anchor not in tlist: + tlist.append(func_anchor) + st.header(f"in *{func_name}*", anchor=func_anchor, divider="green") + + evo_anchor = f"loop_{loop_id}.evo_step_{evo_id}" + if evo_id and evo_anchor not in tlist: + tlist.append(evo_anchor) + st.subheader(f"evo_step_{evo_id}", anchor=evo_anchor, divider="orange") + + # 根据 tag 渲染内容 + if "debug_exp_gen" in tag: + with st.expander( + f"Exp in :violet[**{obj.experiment_workspace.workspace_path}**]", expanded=False, icon="🧩" + ): + st.write(obj) + elif "debug_tpl" in tag: + uri = obj["uri"] + tpl = obj["template"] + cxt = obj["context"] + rd = obj["rendered"] + with st.expander(highlight_prompts_uri(uri), expanded=False, icon="⚙️"): + t1, t2, t3 = st.tabs([":green[**Rendered**]", ":blue[**Template**]", ":orange[**Context**]"]) + with t1: + show_text(rd) + with t2: + show_text(tpl, lang="django") + with t3: + st.json(cxt) + elif "debug_llm" in tag: + system = obj.get("system", None) + user = obj["user"] + resp = obj["resp"] + with st.expander(f"**LLM**", expanded=False, icon="🤖"): + t1, t2, t3 = st.tabs([":green[**Response**]", ":blue[**User**]", ":orange[**System**]"]) + with t1: + try: + rdict = json.loads(resp) + if "code" in rdict: + code = rdict["code"] + st.markdown(":red[**Code in response dict:**]") + st.code(code, language="python", wrap_lines=True, line_numbers=True) + rdict.pop("code") + elif "spec" in rdict: + spec = rdict["spec"] + st.markdown(":red[**Spec in response dict:**]") + st.markdown(spec) + rdict.pop("spec") + else: + # show model codes + showed_keys = [] + for k, v in rdict.items(): + if k.startswith("model_") and k.endswith(".py"): + st.markdown(f":red[**{k}**]") + st.code(v, language="python", wrap_lines=True, line_numbers=True) + showed_keys.append(k) + for k in showed_keys: + rdict.pop(k) + st.write(":red[**Other parts (except for the code or spec) in response dict:**]") + st.json(rdict) + except: + st.json(resp) + with t2: + show_text(user) + with t3: + show_text(system or "No system prompt available") + + progress_text.text("当前 Loop 数据处理完成！") + + # Sidebar TOC + with st.sidebar: + toc = "\n".join([f"- [{t}](#{t})" if t.startswith("L") else f" - [{t.split('.')[1]}](#{t})" for t in tlist]) + st.markdown(toc, unsafe_allow_html=True) diff --git a/rdagent/scenarios/data_science/proposal/exp_gen/merge.py b/rdagent/scenarios/data_science/proposal/exp_gen/merge.py index e541f159..6f943441 100644 --- a/rdagent/scenarios/data_science/proposal/exp_gen/merge.py +++ b/rdagent/scenarios/data_science/proposal/exp_gen/merge.py @@ -182,7 +182,7 @@ class ExpGen2Hypothesis(DSProposalV2ExpGen): success_fb_list = list(set(trace_fbs)) logger.info( - f"Merge Hypothesis: select {len(success_fb_list)} from {len(trace_fbs)} SOTA experiments found in {len(leaves)} traces" + f"Merge Hypothesis: select {len(success_fb_list)} from {len(trace_fbs)} SOTA experiments found in {len(leaves)} traces" # nosec B608 — not SQL, Bandit false positive on "select" in log message ) if len(success_fb_list) > 0: diff --git a/rdagent/scenarios/kaggle/developer/coder.py b/rdagent/scenarios/kaggle/developer/coder.py index 5547def7..d01b7651 100644 --- a/rdagent/scenarios/kaggle/developer/coder.py +++ b/rdagent/scenarios/kaggle/developer/coder.py @@ -38,7 +38,7 @@ class KGModelFeatureSelectionCoder(Developer[KGModelExperiment]): assert target_model_type in KG_SELECT_MAPPING if len(exp.experiment_workspace.data_description) == 1: code = ( - Environment(undefined=StrictUndefined) + Environment(undefined=StrictUndefined) # nosec B701 — renders Python code templates, not HTML; autoescape would corrupt code .from_string(DEFAULT_SELECTION_CODE) .render(feature_index_list=None) ) @@ -62,7 +62,7 @@ class KGModelFeatureSelectionCoder(Developer[KGModelExperiment]): chosen_index_to_list_index = [i - 1 for i in chosen_index] code = ( - Environment(undefined=StrictUndefined) + Environment(undefined=StrictUndefined) # nosec B701 — renders Python code templates, not HTML; autoescape would corrupt code .from_string(DEFAULT_SELECTION_CODE) .render(feature_index_list=chosen_index_to_list_index) ) diff --git a/rdagent/scenarios/qlib/experiment/utils.py b/rdagent/scenarios/qlib/experiment/utils.py index ad42a69b..11055e54 100644 --- a/rdagent/scenarios/qlib/experiment/utils.py +++ b/rdagent/scenarios/qlib/experiment/utils.py @@ -67,7 +67,7 @@ def get_file_desc(p: Path, variable_list=[]) -> str: """ p = Path(p) - JJ_TPL = Environment(undefined=StrictUndefined).from_string(""" + JJ_TPL = Environment(undefined=StrictUndefined).from_string(""" # nosec B701 — renders plain text description, not HTML; autoescape not applicable # {{file_name}} ## File Type