diff --git a/rdagent/log/ui/ds_summary.py b/rdagent/log/ui/ds_summary.py index 8b1f3788..aaeef645 100644 --- a/rdagent/log/ui/ds_summary.py +++ b/rdagent/log/ui/ds_summary.py @@ -14,7 +14,7 @@ from rdagent.log.ui.conf import UI_SETTING from rdagent.log.ui.ds_trace import load_times -def get_exec_time(stdout_p: Path): +def get_script_time(stdout_p: Path): with stdout_p.open("r") as f: first_line = next(f).strip() last_line = deque(f, maxlen=1).pop().strip() @@ -55,20 +55,23 @@ def get_summary_df(log_folders: list[str]) -> tuple[dict, pd.DataFrame]: for k, v in s.items(): stdout_p = Path(lf) / f"{k}.stdout" if stdout_p.exists(): - v["exec_time"] = get_exec_time(stdout_p) + v["script_time"] = get_script_time(stdout_p) else: - v["exec_time"] = None + v["script_time"] = None exp_gen_time = timedelta() coding_time = timedelta() running_time = timedelta() - if state.show_times_info: - times_info = load_times(Path(lf) / k) - for time_info in times_info.values(): - exp_gen_time += time_info[0].end - time_info[0].start + all_time = timedelta() + times_info = load_times(Path(lf) / k) + for time_info in times_info.values(): + all_time += sum((ti.end - ti.start for ti in time_info), timedelta()) + exp_gen_time += time_info[0].end - time_info[0].start + if len(time_info) > 1: coding_time += time_info[1].end - time_info[1].start - if len(time_info) > 2: - running_time += time_info[2].end - time_info[2].start + if len(time_info) > 2: + running_time += time_info[2].end - time_info[2].start + v["exec_time"] = str(all_time).split(".")[0] v["exp_gen_time"] = str(exp_gen_time).split(".")[0] v["coding_time"] = str(coding_time).split(".")[0] v["running_time"] = str(running_time).split(".")[0] @@ -84,6 +87,7 @@ def get_summary_df(log_folders: list[str]) -> tuple[dict, pd.DataFrame]: base_df = pd.DataFrame( columns=[ "Competition", + "Script Time", "Exec Time", "Exp Gen", "Coding", @@ -132,6 +136,7 @@ def get_summary_df(log_folders: list[str]) -> tuple[dict, pd.DataFrame]: for k, v in summary.items(): loop_num = v["loop_num"] base_df.loc[k, "Competition"] = v["competition"] + base_df.loc[k, "Script Time"] = v["script_time"] base_df.loc[k, "Exec Time"] = v["exec_time"] base_df.loc[k, "Exp Gen"] = v["exp_gen_time"] base_df.loc[k, "Coding"] = v["coding_time"] @@ -205,6 +210,8 @@ def get_summary_df(log_folders: list[str]) -> tuple[dict, pd.DataFrame]: def num2percent(num: int, total: int, show_origin=True) -> str: + num = int(num) + total = int(total) if show_origin: return f"{num} ({round(num / total * 100, 2)}%)" return f"{round(num / total * 100, 2)}%" @@ -212,6 +219,20 @@ def num2percent(num: int, total: int, show_origin=True) -> str: def percent_df(df: pd.DataFrame, show_origin=True) -> pd.DataFrame: base_df = df.copy(deep=True) + + # Convert columns to object dtype so we can store strings like "14 (53.85%)" without warnings + columns_to_convert = [ + "Successful Final Decision", + "Made Submission", + "Valid Submission", + "Above Median", + "Bronze", + "Silver", + "Gold", + "Any Medal", + ] + base_df[columns_to_convert] = base_df[columns_to_convert].astype(object) + for k in base_df.index: loop_num = int(base_df.loc[k, "Total Loops"]) if loop_num != 0: @@ -231,6 +252,7 @@ def percent_df(df: pd.DataFrame, show_origin=True) -> pd.DataFrame: base_df.loc[k, "Silver"] = num2percent(base_df.loc[k, "Silver"], loop_num, show_origin) base_df.loc[k, "Gold"] = num2percent(base_df.loc[k, "Gold"], loop_num, show_origin) base_df.loc[k, "Any Medal"] = num2percent(base_df.loc[k, "Any Medal"], loop_num, show_origin) + return base_df @@ -275,12 +297,13 @@ def all_summarize_win(): base_df = percent_df(base_df) base_df.insert(0, "Select", True) base_df = st.data_editor( - base_df.style.applymap( - lambda x: "background-color: #F0F8FF", + base_df.style.apply( + lambda col: col.map(lambda val: "background-color: #F0F8FF"), subset=["Baseline Score", "Bronze Threshold", "Silver Threshold", "Gold Threshold", "Medium Threshold"], + axis=0, ) - .applymap( - lambda x: "background-color: #FFFFE0", + .apply( + lambda col: col.map(lambda val: "background-color: #FFFFE0"), subset=[ "Ours - Base", "Ours vs Base", @@ -288,23 +311,27 @@ def all_summarize_win(): "Ours vs Silver", "Ours vs Gold", ], + axis=0, ) - .applymap( - lambda x: "background-color: #E6E6FA", + .apply( + lambda col: col.map(lambda val: "background-color: #E6E6FA"), subset=[ + "Script Time", "Exec Time", "Exp Gen", "Coding", "Running", ], + axis=0, ) - .applymap( - lambda x: "background-color: #F0FFF0", + .apply( + lambda col: col.map(lambda val: "background-color: #F0FFF0"), subset=[ "Best Result", "SOTA Exp", "SOTA Exp Score", ], + axis=0, ), column_config={ "Select": st.column_config.CheckboxColumn("Select", default=True, help="Stat this trace.", disabled=False), @@ -408,8 +435,6 @@ def all_summarize_win(): st.json(v["valid_scores"]) -with st.sidebar: - st.toggle("Show Times Info (Slowly)", key="show_times_info") with st.container(border=True): if st.toggle("近3天平均", key="show_3days"): days_summarize_win() diff --git a/rdagent/log/ui/ds_trace.py b/rdagent/log/ui/ds_trace.py index f0b281d5..f7637244 100644 --- a/rdagent/log/ui/ds_trace.py +++ b/rdagent/log/ui/ds_trace.py @@ -50,22 +50,16 @@ def convert_defaultdict_to_dict(d): @st.cache_data(persist=True) def load_times(log_path: Path): """加载时间数据""" - times = defaultdict(lambda: defaultdict(dict)) - for msg in FileStorage(log_path).iter_msg(): - if msg.tag and "llm" not in msg.tag and "session" not in msg.tag: - li, fn = extract_loopid_func_name(msg.tag) - if li: - li = int(li) + try: + session_path = log_path / "__session__" + max_li = max(int(p.name) for p in session_path.iterdir() if p.is_dir() and p.name.isdigit()) + max_step = max(int(p.name.split("_")[0]) for p in (session_path / str(max_li)).iterdir() if p.is_file()) + rdloop_obj_p = next((session_path / str(max_li)).glob(f"{max_step}_*")) - # read times - loop_obj_path = log_path / "__session__" / f"{li}" / "4_record" - if loop_obj_path.exists(): - try: - times[li] = DataScienceRDLoop.load(loop_obj_path, do_truncate=False).loop_trace[li] - except Exception as e: - pass - - return convert_defaultdict_to_dict(times) + rd_times = DataScienceRDLoop.load(rdloop_obj_p, do_truncate=False).loop_trace + except: + rd_times = {} + return rd_times @st.cache_data(persist=True) @@ -436,8 +430,9 @@ def summarize_data(): df.loc[loop, "Time"] = str(sum((i.end - i.start for i in state.times[loop]), timedelta())).split(".")[0] exp_gen_time = state.times[loop][0].end - state.times[loop][0].start df.loc[loop, "Exp Gen"] = str(exp_gen_time).split(".")[0] - coding_time = state.times[loop][1].end - state.times[loop][1].start - df.loc[loop, "Coding"] = str(coding_time).split(".")[0] + if len(state.times[loop]) > 1: + coding_time = state.times[loop][1].end - state.times[loop][1].start + df.loc[loop, "Coding"] = str(coding_time).split(".")[0] if len(state.times[loop]) > 2: running_time = state.times[loop][2].end - state.times[loop][2].start df.loc[loop, "Running"] = str(running_time).split(".")[0]