diff --git a/rdagent/log/ui/conf.py b/rdagent/log/ui/conf.py new file mode 100644 index 00000000..0b88098c --- /dev/null +++ b/rdagent/log/ui/conf.py @@ -0,0 +1,14 @@ +from pydantic_settings import SettingsConfigDict + +from rdagent.core.conf import ExtendedBaseSettings + + +class UIBasePropSetting(ExtendedBaseSettings): + model_config = SettingsConfigDict(env_prefix="UI_", protected_namespaces=()) + + default_log_folders: list[str] = ["./log"] + + baseline_result_path: str = "./baseline.csv" + + +UI_SETTING = UIBasePropSetting() diff --git a/rdagent/log/ui/dsapp.py b/rdagent/log/ui/dsapp.py index a14bd048..e864a3fb 100644 --- a/rdagent/log/ui/dsapp.py +++ b/rdagent/log/ui/dsapp.py @@ -1,3 +1,4 @@ +import math import re from collections import defaultdict from datetime import timedelta @@ -12,6 +13,7 @@ from streamlit import session_state as state from rdagent.app.data_science.loop import DataScienceRDLoop from rdagent.log.mle_summary import extract_mle_json, is_valid_session from rdagent.log.storage import FileStorage +from rdagent.log.ui.conf import UI_SETTING from rdagent.utils import remove_ansi_codes st.set_page_config(layout="wide", page_title="RD-Agent", page_icon="🎓", initial_sidebar_state="expanded") @@ -20,7 +22,7 @@ st.set_page_config(layout="wide", page_title="RD-Agent", page_icon="🎓", initi if "log_folder" not in state: state.log_folder = Path("./log") if "log_folders" not in state: - state.log_folders = ["./log"] + state.log_folders = UI_SETTING.default_log_folders if "log_path" not in state: state.log_path = None if "show_all_summary" not in state: @@ -67,7 +69,10 @@ def load_data(log_path: Path): # read times loop_obj_path = log_path / "__session__" / f"{li}" / "4_record" if loop_obj_path.exists(): - state.times[li] = DataScienceRDLoop.load(loop_obj_path).loop_trace[li] + try: + state.times[li] = DataScienceRDLoop.load(loop_obj_path, do_truncate=False).loop_trace[li] + except Exception as e: + pass ei = extract_evoid(msg.tag) msg.tag = re.sub(r"\.evo_loop_\d+", "", msg.tag) @@ -75,18 +80,23 @@ def load_data(log_path: Path): msg.tag = msg.tag.strip() if ei: - state.data[li][int(ei)][msg.tag] = msg.content + if int(ei) not in state.data[li][fn]: + state.data[li][fn][int(ei)] = {} + state.data[li][fn][int(ei)][msg.tag] = msg.content else: if msg.tag: state.data[li][fn][msg.tag] = msg.content else: if not isinstance(msg.content, str): - state.data[li][fn] = msg.content + state.data[li][fn]["no_tag"] = msg.content # @st.cache_data def get_folders_sorted(log_path): """缓存并返回排序后的文件夹列表，并加入进度打印""" + if not log_path.exists(): + st.toast(f"Path {log_path} does not exist!") + return [] with st.spinner("正在加载文件夹列表..."): folders = sorted( (folder for folder in log_path.iterdir() if is_valid_session(folder)), @@ -104,19 +114,29 @@ with st.sidebar: ) state.log_folders = [folder.strip() for folder in log_folder_str.split(";") if folder.strip()] + # # TODO: 只是临时的功能 + day_map = {"srv": "最近(srv)", "srv2": "上一批(srv2)", "srv3": "上上批(srv3)"} + day_srv = st.radio("选择批次", ["srv", "srv2", "srv3"], format_func=lambda x: day_map[x], horizontal=True) + if day_srv == "srv": + state.log_folders = [re.sub(r"log\.srv\d*", "log.srv", folder) for folder in state.log_folders] + elif day_srv == "srv2": + state.log_folders = [re.sub(r"log\.srv\d*", "log.srv2", folder) for folder in state.log_folders] + elif day_srv == "srv3": + state.log_folders = [re.sub(r"log\.srv\d*", "log.srv3", folder) for folder in state.log_folders] + state.log_folder = Path(st.radio(f"Select :blue[**one log folder**]", state.log_folders)) if not state.log_folder.exists(): st.warning(f"Path {state.log_folder} does not exist!") + else: + folders = get_folders_sorted(state.log_folder) + st.selectbox(f"Select from :blue[**{state.log_folder.absolute()}**]", folders, key="log_path") - folders = get_folders_sorted(state.log_folder) - st.selectbox(f"Select from :blue[**{state.log_folder.absolute()}**]", folders, key="log_path") + if st.button("Refresh Data"): + if state.log_path is None: + st.toast("Please select a log path first!", icon="🟡") + st.stop() - if st.button("Refresh Data"): - if state.log_path is None: - st.toast("Please select a log path first!", type="error") - st.stop() - - load_data(state.log_folder / state.log_path) + load_data(state.log_folder / state.log_path) st.toggle("One Trace / Log Folder Summary", key="show_all_summary") st.toggle("Show stdout", key="show_stdout") @@ -148,77 +168,106 @@ def workspace_win(data): show_files[codename], language=("python" if codename.endswith(".py") else "markdown"), wrap_lines=True, + line_numbers=True, ) else: - st.markdown("No files in the workspace") + st.markdown(f"No files in :blue[{replace_ep_path(data.workspace_path)}]") + + +def hypothesis_win(data): + st.code(str(data).replace("\n", "\n\n"), wrap_lines=True) def exp_gen_win(data): - st.header("Exp Gen", divider="blue") + st.header("Exp Gen", divider="blue", anchor="exp-gen") st.subheader("Hypothesis") - st.code(str(data.hypothesis).replace("\n", "\n\n"), wrap_lines=True) + hypothesis_win(data["no_tag"].hypothesis) st.subheader("pending_tasks") - for tasks in data.pending_tasks_list: + for tasks in data["no_tag"].pending_tasks_list: task_win(tasks[0]) - st.subheader("Exp Workspace", anchor="exp-workspace") - workspace_win(data.experiment_workspace) + st.subheader("Exp Workspace") + workspace_win(data["no_tag"].experiment_workspace) -def evolving_win(data): - st.header("Code Evolving", divider="green") - if len(data) > 1: - evo_id = st.slider("Evolving", 0, len(data) - 1, 0) - else: - evo_id = 0 - - if evo_id in data: - if data[evo_id]["evolving code"][0] is not None: - st.subheader("codes") - workspace_win(data[evo_id]["evolving code"][0]) - fb = data[evo_id]["evolving feedback"][0] - st.subheader("evolving feedback" + ("✅" if bool(fb) else "❌"), anchor="c_feedback") - f1, f2, f3 = st.tabs(["execution", "return_checking", "code"]) - f1.code(fb.execution, wrap_lines=True) - f2.code(fb.return_checking, wrap_lines=True) - f3.code(fb.code, wrap_lines=True) +def evolving_win(data, key): + with st.container(border=True): + if len(data) > 1: + evo_id = st.slider("Evolving", 0, len(data) - 1, 0, key=key) + elif len(data) == 1: + evo_id = 0 else: - st.write("data[evo_id]['evolving code'][0] is None.") - st.write(data[evo_id]) + st.markdown("No evolving.") + return + + if evo_id in data: + if data[evo_id]["evolving code"][0] is not None: + st.subheader("codes") + workspace_win(data[evo_id]["evolving code"][0]) + fb = data[evo_id]["evolving feedback"][0] + st.subheader("evolving feedback" + ("✅" if bool(fb) else "❌")) + f1, f2, f3 = st.tabs(["execution", "return_checking", "code"]) + f1.code(fb.execution, wrap_lines=True) + f2.code(fb.return_checking, wrap_lines=True) + f3.code(fb.code, wrap_lines=True) + else: + st.write("data[evo_id]['evolving code'][0] is None.") + st.write(data[evo_id]) + else: + st.markdown("No evolving.") + + +def coding_win(data): + st.header("Coding", divider="blue", anchor="coding") + evolving_data = {k: v for k, v in data.items() if isinstance(k, int)} + task_set = set() + for v in evolving_data.values(): + for t in v: + if "Task" in t.split(".")[0]: + task_set.add(t.split(".")[0]) + if task_set: + # 新版存Task tag的Trace + for task in task_set: + st.subheader(task) + task_data = {k: {a.split(".")[1]: b for a, b in v.items() if task in a} for k, v in evolving_data.items()} + evolving_win(task_data, key=task) else: - st.markdown("No evolving.") + # 旧版未存Task tag的Trace + evolving_win(evolving_data, key="coding") + if "no_tag" in data: + st.subheader("Exp Workspace (coding final)") + workspace_win(data["no_tag"].experiment_workspace) -def exp_after_coding_win(data): - st.header("Exp After Coding", divider="blue") - st.subheader("Exp Workspace", anchor="eac-exp-workspace") - workspace_win(data.experiment_workspace) - - -def exp_after_running_win(data, mle_score): - st.header("Exp After Running", divider="blue") - st.subheader("Exp Workspace", anchor="ear-exp-workspace") - workspace_win(data.experiment_workspace) - st.subheader("Result") - st.write(data.result) - st.subheader("MLE Submission Score" + ("✅" if (isinstance(mle_score, dict) and mle_score["score"]) else "❌")) - if isinstance(mle_score, dict): - st.json(mle_score) - else: - st.code(mle_score, wrap_lines=True) +def running_win(data, mle_score): + st.header("Running", divider="blue", anchor="running") + evolving_win({k: v for k, v in data.items() if isinstance(k, int)}, key="running") + if "no_tag" in data: + st.subheader("Exp Workspace (running final)") + workspace_win(data["no_tag"].experiment_workspace) + st.subheader("Result") + st.write(data["no_tag"].result) + st.subheader("MLE Submission Score" + ("✅" if (isinstance(mle_score, dict) and mle_score["score"]) else "❌")) + if isinstance(mle_score, dict): + st.json(mle_score) + else: + st.code(mle_score, wrap_lines=True) def feedback_win(data): - st.header("Feedback" + ("✅" if bool(data) else "❌"), divider="orange") - st.code(data, wrap_lines=True) + data = data["no_tag"] + st.header("Feedback" + ("✅" if bool(data) else "❌"), divider="orange", anchor="feedback") + st.code(str(data).replace("\n", "\n\n"), wrap_lines=True) if data.exception is not None: st.markdown(f"**:red[Exception]**: {data.exception}") def sota_win(data): - st.header("SOTA Experiment", divider="rainbow") + st.header("SOTA Experiment", divider="rainbow", anchor="sota-exp") if data: - st.subheader("Exp Workspace", anchor="sota-exp-workspace") + st.markdown(f"**SOTA Exp Hypothesis**") + hypothesis_win(data.hypothesis) + st.markdown("**Exp Workspace**") workspace_win(data.experiment_workspace) else: st.markdown("No SOTA experiment.") @@ -226,30 +275,23 @@ def sota_win(data): def main_win(data): exp_gen_win(data["direct_exp_gen"]) - evo_data = {k: v for k, v in data.items() if isinstance(k, int)} - evolving_win(evo_data) if "coding" in data: - exp_after_coding_win(data["coding"]) + coding_win(data["coding"]) if "running" in data: - exp_after_running_win(data["running"], data["mle_score"]) + running_win(data["running"], data["mle_score"]) if "feedback" in data: feedback_win(data["feedback"]) - sota_win(data["SOTA experiment"]) + if "record" in data and "SOTA experiment" in data["record"]: + sota_win(data["record"]["SOTA experiment"]) with st.sidebar: st.markdown( f""" - [Exp Gen](#exp-gen) - - [Hypothesis](#hypothesis) - - [pending_tasks](#pending-tasks) - - [Exp Workspace](#exp-workspace) -- [Code Evolving ({len(evo_data)})](#code-evolving) - - [codes](#codes) - - [evolving feedback](#c_feedback) -{"- [Exp After Coding](#exp-after-coding)" if "coding" in data else ""} -{"- [Exp After Running](#exp-after-running)" if "running" in data else ""} -{"- [Feedback](#feedback)" if "feedback" in data else ""} -- [SOTA Experiment](#sota-experiment) +- [Coding](#coding) +- [Running](#running) +- [Feedback](#feedback) +- [SOTA Experiment](#sota-exp) """ ) @@ -269,33 +311,43 @@ def replace_ep_path(p: Path): def summarize_data(): st.header("Summary", divider="rainbow") df = pd.DataFrame( - columns=["Component", "Running Score", "Feedback", "Time", "Start Time (UTC+8)", "End Time (UTC+8)"], + columns=["Component", "Running Score", "Feedback", "e-loops", "Time", "Start Time (UTC+8)", "End Time (UTC+8)"], index=range(len(state.data) - 1), ) for loop in range(len(state.data) - 1): loop_data = state.data[loop] - df.loc[loop, "Component"] = loop_data["direct_exp_gen"].hypothesis.component + df.loc[loop, "Component"] = loop_data["direct_exp_gen"]["no_tag"].hypothesis.component if state.times[loop]: df.loc[loop, "Time"] = str(sum((i.end - i.start for i in state.times[loop]), timedelta())).split(".")[0] df.loc[loop, "Start Time (UTC+8)"] = state.times[loop][0].start + timedelta(hours=8) df.loc[loop, "End Time (UTC+8)"] = state.times[loop][-1].end + timedelta(hours=8) - if "running" in loop_data: + if "running" in loop_data and "no_tag" in loop_data["running"]: if "mle_score" not in state.data[loop]: - mle_score_path = ( - replace_ep_path(loop_data["running"].experiment_workspace.workspace_path) / "mle_score.txt" - ) - try: - mle_score_txt = mle_score_path.read_text() + if "mle_score" in loop_data["running"]: + mle_score_txt = loop_data["running"]["mle_score"] state.data[loop]["mle_score"] = extract_mle_json(mle_score_txt) if state.data[loop]["mle_score"]["score"] is not None: df.loc[loop, "Running Score"] = str(state.data[loop]["mle_score"]["score"]) else: state.data[loop]["mle_score"] = mle_score_txt df.loc[loop, "Running Score"] = "❌" - except Exception as e: - state.data[loop]["mle_score"] = str(e) - df.loc[loop, "Running Score"] = "❌" + else: + mle_score_path = ( + replace_ep_path(loop_data["running"]["no_tag"].experiment_workspace.workspace_path) + / "mle_score.txt" + ) + try: + mle_score_txt = mle_score_path.read_text() + state.data[loop]["mle_score"] = extract_mle_json(mle_score_txt) + if state.data[loop]["mle_score"]["score"] is not None: + df.loc[loop, "Running Score"] = str(state.data[loop]["mle_score"]["score"]) + else: + state.data[loop]["mle_score"] = mle_score_txt + df.loc[loop, "Running Score"] = "❌" + except Exception as e: + state.data[loop]["mle_score"] = str(e) + df.loc[loop, "Running Score"] = "❌" else: if isinstance(state.data[loop]["mle_score"], dict): df.loc[loop, "Running Score"] = str(state.data[loop]["mle_score"]["score"]) @@ -305,30 +357,70 @@ def summarize_data(): else: df.loc[loop, "Running Score"] = "N/A" + if "coding" in loop_data: + df.loc[loop, "e-loops"] = max(i for i in loop_data["coding"].keys() if isinstance(i, int)) + 1 if "feedback" in loop_data: - df.loc[loop, "Feedback"] = "✅" if bool(loop_data["feedback"]) else "❌" + df.loc[loop, "Feedback"] = "✅" if bool(loop_data["feedback"]["no_tag"]) else "❌" else: df.loc[loop, "Feedback"] = "N/A" - st.dataframe(df) + stat_t0, stat_t1 = st.columns(2) + stat_t0.dataframe(df) + + def comp_stat_func(x: pd.DataFrame): + total_num = x.shape[0] + valid_num = x[x["Running Score"] != "N/A"].shape[0] + avg_e_loops = x["e-loops"].mean() + return pd.Series( + { + "Total": total_num, + "Valid": valid_num, + "Valid Rate": round(valid_num / total_num * 100, 2), + "Avg e-loops": round(avg_e_loops, 2), + } + ) + + comp_df = df.loc[:, ["Component", "Running Score", "e-loops"]].groupby("Component").apply(comp_stat_func) + comp_df.loc["Total"] = comp_df.sum() + comp_df.loc["Total", "Valid Rate"] = round(comp_df.loc["Total", "Valid"] / comp_df.loc["Total", "Total"] * 100, 2) + comp_df["Valid Rate"] = comp_df["Valid Rate"].apply(lambda x: f"{x}%") + comp_df.loc["Total", "Avg e-loops"] = round(df["e-loops"].mean(), 2) + stat_t1.dataframe(comp_df) -def all_summarize_win(): +def get_summary_df(log_folders: list[str]) -> tuple[dict, pd.DataFrame]: summarys = {} - for lf in state.log_folders: + for lf in log_folders: if not (Path(lf) / "summary.pkl").exists(): st.warning( - f"No summary file found in {lf}\nRun:`dotenv run -- python rdagent/log/mle_summary.py grade_summary --log_folder=`" + f"No summary file found in **{lf}**\n\nRun:`dotenv run -- python rdagent/log/mle_summary.py grade_summary --log_folder={lf}`" ) else: summarys[lf] = pd.read_pickle(Path(lf) / "summary.pkl") if len(summarys) == 0: - return + return {}, pd.DataFrame() summary = {} for lf, s in summarys.items(): for k, v in s.items(): - summary[f"{lf[lf.rfind('ep'):]}{k}"] = v + stdout_p = Path(lf) / f"{k}.stdout" + v["stdout"] = [] + if stdout_p.exists(): + # stdout = stdout_p.read_text() + stdout = "" + if "Retrying" in stdout: + v["stdout"].append("LLM Retry") + if "Traceback (most recent call last):" in stdout[-10000:]: + v["stdout"].append("Code Error") + v["stdout"] = ", ".join([i for i in v["stdout"] if i]) + + # 调整实验名字 + if "amlt" in lf: + summary[f"{lf[lf.rfind('amlt')+5:].split('/')[0]} - {k}"] = v + elif "ep" in lf: + summary[f"{lf[lf.rfind('ep'):]} - {k}"] = v + else: + summary[f"{lf} - {k}"] = v summary = {k: v for k, v in summary.items() if "competition" in v} base_df = pd.DataFrame( @@ -344,20 +436,23 @@ def all_summarize_win(): "Silver", "Gold", "Any Medal", + "Best Medal", "SOTA Exp", "Ours - Base", + "Ours vs Base", "SOTA Exp Score", "Baseline Score", "Bronze Threshold", "Silver Threshold", "Gold Threshold", "Medium Threshold", + "stdout", ], index=summary.keys(), ) # Read baseline results - baseline_result_path = "" + baseline_result_path = UI_SETTING.baseline_result_path if Path(baseline_result_path).exists(): baseline_df = pd.read_csv(baseline_result_path) @@ -368,66 +463,158 @@ def all_summarize_win(): if loop_num == 0: base_df.loc[k] = "N/A" else: - base_df.loc[k, "Successful Final Decision"] = ( - f"{v['success_loop_num']} ({round(v['success_loop_num'] / loop_num * 100, 2)}%)" - ) - base_df.loc[k, "Made Submission"] = ( - f"{v['made_submission_num']} ({round(v['made_submission_num'] / loop_num * 100, 2)}%)" - ) - base_df.loc[k, "Valid Submission"] = ( - f"{v['valid_submission_num']} ({round(v['valid_submission_num'] / loop_num * 100, 2)}%)" - ) - if v["made_submission_num"] != 0: - base_df.loc[k, "V/M"] = f"{round(v['valid_submission_num'] / v['made_submission_num'] * 100, 2)}%" - else: - base_df.loc[k, "V/M"] = "N/A" - base_df.loc[k, "Above Median"] = ( - f"{v['above_median_num']} ({round(v['above_median_num'] / loop_num * 100, 2)}%)" - ) - base_df.loc[k, "Bronze"] = f"{v['bronze_num']} ({round(v['bronze_num'] / loop_num * 100, 2)}%)" - base_df.loc[k, "Silver"] = f"{v['silver_num']} ({round(v['silver_num'] / loop_num * 100, 2)}%)" - base_df.loc[k, "Gold"] = f"{v['gold_num']} ({round(v['gold_num'] / loop_num * 100, 2)}%)" - base_df.loc[k, "Any Medal"] = f"{v['get_medal_num']} ({round(v['get_medal_num'] / loop_num * 100, 2)}%)" + base_df.loc[k, "Successful Final Decision"] = v["success_loop_num"] + base_df.loc[k, "Made Submission"] = v["made_submission_num"] + base_df.loc[k, "Valid Submission"] = v["valid_submission_num"] + base_df.loc[k, "Above Median"] = v["above_median_num"] + base_df.loc[k, "Bronze"] = v["bronze_num"] + if v["bronze_num"] > 0: + base_df.loc[k, "Best Medal"] = "bronze" + base_df.loc[k, "Silver"] = v["silver_num"] + if v["silver_num"] > 0: + base_df.loc[k, "Best Medal"] = "silver" + base_df.loc[k, "Gold"] = v["gold_num"] + if v["gold_num"] > 0: + base_df.loc[k, "Best Medal"] = "gold" + base_df.loc[k, "Any Medal"] = v["get_medal_num"] baseline_score = None if Path(baseline_result_path).exists(): baseline_score = baseline_df.loc[baseline_df["competition_id"] == v["competition"], "score"].item() base_df.loc[k, "SOTA Exp"] = v.get("sota_exp_stat", None) - if ( - baseline_score is not None - and not pd.isna(baseline_score) - and not pd.isna(v.get("sota_exp_score", None)) - ): - base_df.loc[k, "Ours - Base"] = v.get("sota_exp_score", 0.0) - baseline_score + if baseline_score is not None and v.get("sota_exp_score", None) is not None: + base_df.loc[k, "Ours - Base"] = v["sota_exp_score"] - baseline_score + try: + base_df.loc[k, "Ours vs Base"] = math.exp( + abs(math.log(v["sota_exp_score"] / baseline_score)) + ) # exp^|ln(a/b)| + except Exception as e: + base_df.loc[k, "Ours vs Base"] = None base_df.loc[k, "SOTA Exp Score"] = v.get("sota_exp_score", None) base_df.loc[k, "Baseline Score"] = baseline_score base_df.loc[k, "Bronze Threshold"] = v.get("bronze_threshold", None) base_df.loc[k, "Silver Threshold"] = v.get("silver_threshold", None) base_df.loc[k, "Gold Threshold"] = v.get("gold_threshold", None) base_df.loc[k, "Medium Threshold"] = v.get("median_threshold", None) + base_df.loc[k, "stdout"] = v["stdout"] base_df["SOTA Exp"].replace("", pd.NA, inplace=True) - st.dataframe(base_df) - total_stat = ( - ( - base_df[ - [ - "Made Submission", - "Valid Submission", - "Above Median", - "Bronze", - "Silver", - "Gold", - "Any Medal", - ] - ] - != "0 (0.0%)" - ).sum() - / base_df.shape[0] - * 100 + base_df = base_df.astype( + { + "Total Loops": int, + "Successful Final Decision": int, + "Made Submission": int, + "Valid Submission": int, + "Above Median": int, + "Bronze": int, + "Silver": int, + "Gold": int, + "Any Medal": int, + "Ours - Base": float, + "Ours vs Base": float, + "SOTA Exp Score": float, + "Baseline Score": float, + "Bronze Threshold": float, + "Silver Threshold": float, + "Gold Threshold": float, + "Medium Threshold": float, + } ) + return summary, base_df + + +def num2percent(num: int, total: int, show_origin=True) -> str: + if show_origin: + return f"{num} ({round(num / total * 100, 2)}%)" + return f"{round(num / total * 100, 2)}%" + + +def percent_df(df: pd.DataFrame, show_origin=True) -> pd.DataFrame: + base_df = df.astype("object", copy=True) + for k in base_df.index: + loop_num = int(base_df.loc[k, "Total Loops"]) + if loop_num != 0: + base_df.loc[k, "Successful Final Decision"] = num2percent( + base_df.loc[k, "Successful Final Decision"], loop_num, show_origin + ) + if base_df.loc[k, "Made Submission"] != 0: + base_df.loc[k, "V/M"] = ( + f"{round(base_df.loc[k, 'Valid Submission'] / base_df.loc[k, 'Made Submission'] * 100, 2)}%" + ) + else: + base_df.loc[k, "V/M"] = "N/A" + base_df.loc[k, "Made Submission"] = num2percent(base_df.loc[k, "Made Submission"], loop_num, show_origin) + base_df.loc[k, "Valid Submission"] = num2percent(base_df.loc[k, "Valid Submission"], loop_num, show_origin) + base_df.loc[k, "Above Median"] = num2percent(base_df.loc[k, "Above Median"], loop_num, show_origin) + base_df.loc[k, "Bronze"] = num2percent(base_df.loc[k, "Bronze"], loop_num, show_origin) + base_df.loc[k, "Silver"] = num2percent(base_df.loc[k, "Silver"], loop_num, show_origin) + base_df.loc[k, "Gold"] = num2percent(base_df.loc[k, "Gold"], loop_num, show_origin) + base_df.loc[k, "Any Medal"] = num2percent(base_df.loc[k, "Any Medal"], loop_num, show_origin) + return base_df + + +def days_summarize_win(): + lfs1 = [re.sub(r"log\.srv\d*", "log.srv", folder) for folder in state.log_folders] + lfs2 = [re.sub(r"log\.srv\d*", "log.srv2", folder) for folder in state.log_folders] + lfs3 = [re.sub(r"log\.srv\d*", "log.srv3", folder) for folder in state.log_folders] + + _, df1 = get_summary_df(lfs1) + _, df2 = get_summary_df(lfs2) + _, df3 = get_summary_df(lfs3) + + df = pd.concat([df1, df2, df3], axis=0) + + def mean_func(x: pd.DataFrame): + numeric_cols = x.select_dtypes(include=["int", "float"]).mean() + string_cols = x.select_dtypes(include=["object"]).agg(lambda col: ", ".join(col.fillna("none").astype(str))) + return pd.concat([numeric_cols, string_cols], axis=0).reindex(x.columns).drop("Competition") + + df = df.groupby("Competition").apply(mean_func) + if st.toggle("Show Percent", key="show_percent"): + st.dataframe(percent_df(df, show_origin=False)) + else: + st.dataframe(df) + + +def all_summarize_win(): + def shorten_folder_name(folder: str) -> str: + if "amlt" in folder: + return folder[folder.rfind("amlt") + 5 :].split("/")[0] + if "ep" in folder: + return folder[folder.rfind("ep") :] + return folder + + selected_folders = st.multiselect( + "Show these folders", state.log_folders, state.log_folders, format_func=shorten_folder_name + ) + summary, base_df = get_summary_df(selected_folders) + if not summary: + return + + base_df = percent_df(base_df) + st.dataframe(base_df) + st.markdown("Ours vs Base: `math.exp(abs(math.log(sota_exp_score / baseline_score)))`") + st.markdown(f"**统计的比赛数目: :red[{base_df.shape[0]}]**") + total_stat = ( + base_df[ + [ + "Made Submission", + "Valid Submission", + "Above Median", + "Bronze", + "Silver", + "Gold", + "Any Medal", + ] + ] + != "0 (0.0%)" + ).sum() total_stat.name = "总体统计(%)" + total_stat.loc["Bronze"] = base_df["Best Medal"].value_counts().get("bronze", 0) + total_stat.loc["Silver"] = base_df["Best Medal"].value_counts().get("silver", 0) + total_stat.loc["Gold"] = base_df["Best Medal"].value_counts().get("gold", 0) + total_stat = total_stat / base_df.shape[0] * 100 # SOTA Exp 统计 se_counts = base_df["SOTA Exp"].value_counts(dropna=True) @@ -447,38 +634,56 @@ def all_summarize_win(): sota_exp_stat = sota_exp_stat / base_df.shape[0] * 100 stat_df = pd.concat([total_stat, sota_exp_stat], axis=1) - st.dataframe(stat_df.round(2)) + stat_t0, stat_t1 = st.columns(2) + with stat_t0: + st.dataframe(stat_df.round(2)) + markdown_table = f""" +| xxx | {stat_df.iloc[0,1]:.1f} | {stat_df.iloc[1,1]:.1f} | {stat_df.iloc[2,1]:.1f} | {stat_df.iloc[3,1]:.1f} | {stat_df.iloc[4,1]:.1f} | {stat_df.iloc[5,1]:.1f} | {stat_df.iloc[6,1]:.1f} | +""" + st.text(markdown_table) + with stat_t1: + Loop_counts = base_df["Total Loops"] + fig = px.histogram(Loop_counts, nbins=10, title="Total Loops Histogram (nbins=10)") + mean_value = Loop_counts.mean() + median_value = Loop_counts.median() + fig.add_vline( + x=mean_value, line_color="orange", annotation_text="Mean", annotation_position="top right", line_width=3 + ) + fig.add_vline( + x=median_value, line_color="red", annotation_text="Median", annotation_position="top right", line_width=3 + ) + st.plotly_chart(fig) # write curve for k, v in summary.items(): with st.container(border=True): st.markdown(f"**:blue[{k}] - :violet[{v['competition']}]**") - vscores = {k: v.iloc[:, 0] for k, v in v["valid_scores"].items()} - tscores = {f"loop {k}": v for k, v in v["test_scores"].items()} - if len(vscores) > 0: - metric_name = list(vscores.values())[0].name - else: - metric_name = "None" - fc1, fc2 = st.columns(2) + tscores = {f"loop {k-1}": v for k, v in v["test_scores"].items()} + tdf = pd.Series(tscores, name="score") + f2 = px.line(tdf, markers=True, title="Test scores") + fc2.plotly_chart(f2, key=k) try: + vscores = {k: v.iloc[:, 0] for k, v in v["valid_scores"].items()} + + if len(vscores) > 0: + metric_name = list(vscores.values())[0].name + else: + metric_name = "None" + vdf = pd.DataFrame(vscores) vdf.columns = [f"loop {i}" for i in vdf.columns] f1 = px.line(vdf.T, markers=True, title=f"Valid scores (metric: {metric_name})") - fc1.plotly_chart(f1, key=f"{k}_v") - tdf = pd.Series(tscores, name="score") - f2 = px.line(tdf, markers=True, title="Test scores") - fc2.plotly_chart(f2, key=k) + fc1.plotly_chart(f1, key=f"{k}_v") except Exception as e: import traceback st.markdown("- Error: " + str(e)) st.code(traceback.format_exc()) st.markdown("- Valid Scores: ") - st.json(vscores) - st.markdown("- Test Scores: ") - st.json(tscores) + # st.write({k: type(v) for k, v in v["valid_scores"].items()}) + st.json(v["valid_scores"]) def stdout_win(loop_id: int): @@ -501,18 +706,24 @@ def stdout_win(loop_id: int): step_stdouts[step] = loop_stdout[si:ei].strip() for k, v in step_stdouts.items(): - expanded = True if "coding" in k else False - with st.expander(k, expanded=expanded): + with st.expander(k, expanded=False): st.code(v, language="log", wrap_lines=True) # UI - Main if state.show_all_summary: - all_summarize_win() + with st.container(border=True): + if st.toggle("近3天平均", key="show_3days"): + days_summarize_win() + with st.container(border=True): + all_summarize_win() elif "data" in state: st.title(state.data["competition"]) summarize_data() - loop_id = st.slider("Loop", 0, len(state.data) - 2, 0) + if len(state.data) > 2: + loop_id = st.slider("Loop", 0, len(state.data) - 2, 0) + else: + loop_id = 0 if state.show_stdout: stdout_win(loop_id) main_win(state.data[loop_id])