diff --git a/rdagent/log/mle_summary.py b/rdagent/log/mle_summary.py index 8a8a3c38..dd079053 100644 --- a/rdagent/log/mle_summary.py +++ b/rdagent/log/mle_summary.py @@ -1,6 +1,7 @@ import json import re from collections import defaultdict +from datetime import timedelta from pathlib import Path import fire @@ -61,7 +62,13 @@ def save_all_grade_info(log_folder): save_grade_info(log_trace_path) -def summarize_folder(log_folder: Path): +def summarize_folder(log_folder: Path, hours: int | None = None): + """ + Summarize the log folder and save the summary as a pickle file. + Args: + log_folder (Path): The path to the log folder (contains many log traces). + hours (int | None): The number of hours to stat. If None, stat all. + """ log_folder = Path(log_folder) stat = defaultdict(dict) for log_trace_path in log_folder.iterdir(): # One log trace @@ -88,10 +95,15 @@ def summarize_folder(log_folder: Path): sota_exp_score = None sota_exp_rank = None grade_output = None + + start_time = None for msg in FileStorage(log_trace_path).iter_msg(): # messages in log trace + if start_time and hours and msg.timestamp > start_time + timedelta(hours=hours): + break if msg.tag and "llm" not in msg.tag and "session" not in msg.tag: if "competition" in msg.tag: stat[log_trace_path.name]["competition"] = msg.content + start_time = msg.timestamp # get threshold scores workflowexp = FBWorkspace() @@ -185,10 +197,14 @@ def summarize_folder(log_folder: Path): "median_threshold": median_threshold, } ) - if (log_folder / "summary.pkl").exists(): - (log_folder / "summary.pkl").unlink() - print("Old summary file removed.") - pd.to_pickle(stat, log_folder / "summary.pkl") + + # Save the summary + save_name = f"summary_{hours}h.pkl" if hours else "summary.pkl" + save_p = log_folder / save_name + if save_p.exists(): + save_p.unlink() + print(f"Old {save_name} removed.") + pd.to_pickle(stat, save_p) # { diff --git a/rdagent/log/ui/ds_summary.py b/rdagent/log/ui/ds_summary.py index 24ea138e..8b1f3788 100644 --- a/rdagent/log/ui/ds_summary.py +++ b/rdagent/log/ui/ds_summary.py @@ -1,5 +1,7 @@ import math import re +from collections import deque +from datetime import datetime, timedelta from pathlib import Path import pandas as pd @@ -9,17 +11,41 @@ import streamlit as st from streamlit import session_state as state from rdagent.log.ui.conf import UI_SETTING +from rdagent.log.ui.ds_trace import load_times +def get_exec_time(stdout_p: Path): + with stdout_p.open("r") as f: + first_line = next(f).strip() + last_line = deque(f, maxlen=1).pop().strip() + + # Extract timestamps from the lines + first_time_match = re.search(r"(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\+\d{2}:\d{2})", first_line) + last_time_match = re.search(r"(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\+\d{2}:\d{2})", last_line) + + if first_time_match and last_time_match: + first_time = datetime.fromisoformat(first_time_match.group(1)) + last_time = datetime.fromisoformat(last_time_match.group(1)) + return pd.Timedelta(last_time - first_time) + + return None + + +# @st.cache_data(persist=True) def get_summary_df(log_folders: list[str]) -> tuple[dict, pd.DataFrame]: summarys = {} + with st.sidebar: + if st.toggle("show 24h summary", key="show_hours_summary"): + sn = "summary_24h.pkl" + else: + sn = "summary.pkl" for lf in log_folders: - if not (Path(lf) / "summary.pkl").exists(): + if not (Path(lf) / sn).exists(): st.warning( - f"No summary file found in **{lf}**\n\nRun:`dotenv run -- python rdagent/log/mle_summary.py grade_summary --log_folder={lf}`" + f"{sn} not found in **{lf}**\n\nRun:`dotenv run -- python rdagent/log/mle_summary.py grade_summary --log_folder={lf} --hours=<>`" ) else: - summarys[lf] = pd.read_pickle(Path(lf) / "summary.pkl") + summarys[lf] = pd.read_pickle(Path(lf) / sn) if len(summarys) == 0: return {}, pd.DataFrame() @@ -28,16 +54,24 @@ def get_summary_df(log_folders: list[str]) -> tuple[dict, pd.DataFrame]: for lf, s in summarys.items(): for k, v in s.items(): stdout_p = Path(lf) / f"{k}.stdout" - v["stdout"] = [] if stdout_p.exists(): - # stdout = stdout_p.read_text() - stdout = "" - if "Retrying" in stdout: - v["stdout"].append("LLM Retry") - if "Traceback (most recent call last):" in stdout[-10000:]: - v["stdout"].append("Code Error") - v["stdout"] = ", ".join([i for i in v["stdout"] if i]) + v["exec_time"] = get_exec_time(stdout_p) + else: + v["exec_time"] = None + exp_gen_time = timedelta() + coding_time = timedelta() + running_time = timedelta() + if state.show_times_info: + times_info = load_times(Path(lf) / k) + for time_info in times_info.values(): + exp_gen_time += time_info[0].end - time_info[0].start + coding_time += time_info[1].end - time_info[1].start + if len(time_info) > 2: + running_time += time_info[2].end - time_info[2].start + v["exp_gen_time"] = str(exp_gen_time).split(".")[0] + v["coding_time"] = str(coding_time).split(".")[0] + v["running_time"] = str(running_time).split(".")[0] # 调整实验名字 if "amlt" in lf: summary[f"{lf[lf.rfind('amlt')+5:].split('/')[0]} - {k}"] = v @@ -50,6 +84,10 @@ def get_summary_df(log_folders: list[str]) -> tuple[dict, pd.DataFrame]: base_df = pd.DataFrame( columns=[ "Competition", + "Exec Time", + "Exp Gen", + "Coding", + "Running", "Total Loops", "Successful Final Decision", "Made Submission", @@ -60,17 +98,19 @@ def get_summary_df(log_folders: list[str]) -> tuple[dict, pd.DataFrame]: "Silver", "Gold", "Any Medal", - "Best Medal", + "Best Result", "SOTA Exp", - "Ours - Base", - "Ours vs Base", "SOTA Exp Score", "Baseline Score", + "Ours - Base", + "Ours vs Base", + "Ours vs Bronze", + "Ours vs Silver", + "Ours vs Gold", "Bronze Threshold", "Silver Threshold", "Gold Threshold", "Medium Threshold", - "stdout", ], index=summary.keys(), ) @@ -80,26 +120,45 @@ def get_summary_df(log_folders: list[str]) -> tuple[dict, pd.DataFrame]: if Path(baseline_result_path).exists(): baseline_df = pd.read_csv(baseline_result_path) + def compare_score(s1, s2): + if s1 is None or s2 is None: + return None + try: + c_value = math.exp(abs(math.log(s1 / s2))) + except Exception as e: + c_value = None + return c_value + for k, v in summary.items(): loop_num = v["loop_num"] base_df.loc[k, "Competition"] = v["competition"] + base_df.loc[k, "Exec Time"] = v["exec_time"] + base_df.loc[k, "Exp Gen"] = v["exp_gen_time"] + base_df.loc[k, "Coding"] = v["coding_time"] + base_df.loc[k, "Running"] = v["running_time"] base_df.loc[k, "Total Loops"] = loop_num if loop_num == 0: base_df.loc[k] = "N/A" else: base_df.loc[k, "Successful Final Decision"] = v["success_loop_num"] base_df.loc[k, "Made Submission"] = v["made_submission_num"] + if v["made_submission_num"] > 0: + base_df.loc[k, "Best Result"] = "made_submission" base_df.loc[k, "Valid Submission"] = v["valid_submission_num"] + if v["valid_submission_num"] > 0: + base_df.loc[k, "Best Result"] = "valid_submission" base_df.loc[k, "Above Median"] = v["above_median_num"] + if v["above_median_num"] > 0: + base_df.loc[k, "Best Result"] = "above_median" base_df.loc[k, "Bronze"] = v["bronze_num"] if v["bronze_num"] > 0: - base_df.loc[k, "Best Medal"] = "bronze" + base_df.loc[k, "Best Result"] = "bronze" base_df.loc[k, "Silver"] = v["silver_num"] if v["silver_num"] > 0: - base_df.loc[k, "Best Medal"] = "silver" + base_df.loc[k, "Best Result"] = "silver" base_df.loc[k, "Gold"] = v["gold_num"] if v["gold_num"] > 0: - base_df.loc[k, "Best Medal"] = "gold" + base_df.loc[k, "Best Result"] = "gold" base_df.loc[k, "Any Medal"] = v["get_medal_num"] baseline_score = None @@ -109,19 +168,16 @@ def get_summary_df(log_folders: list[str]) -> tuple[dict, pd.DataFrame]: base_df.loc[k, "SOTA Exp"] = v.get("sota_exp_stat", None) if baseline_score is not None and v.get("sota_exp_score", None) is not None: base_df.loc[k, "Ours - Base"] = v["sota_exp_score"] - baseline_score - try: - base_df.loc[k, "Ours vs Base"] = math.exp( - abs(math.log(v["sota_exp_score"] / baseline_score)) - ) # exp^|ln(a/b)| - except Exception as e: - base_df.loc[k, "Ours vs Base"] = None + base_df.loc[k, "Ours vs Base"] = compare_score(v["sota_exp_score"], baseline_score) + base_df.loc[k, "Ours vs Bronze"] = compare_score(v["sota_exp_score"], v.get("bronze_threshold", None)) + base_df.loc[k, "Ours vs Silver"] = compare_score(v["sota_exp_score"], v.get("silver_threshold", None)) + base_df.loc[k, "Ours vs Gold"] = compare_score(v["sota_exp_score"], v.get("gold_threshold", None)) base_df.loc[k, "SOTA Exp Score"] = v.get("sota_exp_score", None) base_df.loc[k, "Baseline Score"] = baseline_score base_df.loc[k, "Bronze Threshold"] = v.get("bronze_threshold", None) base_df.loc[k, "Silver Threshold"] = v.get("silver_threshold", None) base_df.loc[k, "Gold Threshold"] = v.get("gold_threshold", None) base_df.loc[k, "Medium Threshold"] = v.get("median_threshold", None) - base_df.loc[k, "stdout"] = v["stdout"] base_df["SOTA Exp"] = base_df["SOTA Exp"].replace("", pd.NA) base_df = base_df.astype( @@ -155,7 +211,7 @@ def num2percent(num: int, total: int, show_origin=True) -> str: def percent_df(df: pd.DataFrame, show_origin=True) -> pd.DataFrame: - base_df = df.astype("object", copy=True) + base_df = df.copy(deep=True) for k in base_df.index: loop_num = int(base_df.loc[k, "Total Loops"]) if loop_num != 0: @@ -217,8 +273,48 @@ def all_summarize_win(): return base_df = percent_df(base_df) - st.dataframe(base_df) + base_df.insert(0, "Select", True) + base_df = st.data_editor( + base_df.style.applymap( + lambda x: "background-color: #F0F8FF", + subset=["Baseline Score", "Bronze Threshold", "Silver Threshold", "Gold Threshold", "Medium Threshold"], + ) + .applymap( + lambda x: "background-color: #FFFFE0", + subset=[ + "Ours - Base", + "Ours vs Base", + "Ours vs Bronze", + "Ours vs Silver", + "Ours vs Gold", + ], + ) + .applymap( + lambda x: "background-color: #E6E6FA", + subset=[ + "Exec Time", + "Exp Gen", + "Coding", + "Running", + ], + ) + .applymap( + lambda x: "background-color: #F0FFF0", + subset=[ + "Best Result", + "SOTA Exp", + "SOTA Exp Score", + ], + ), + column_config={ + "Select": st.column_config.CheckboxColumn("Select", default=True, help="Stat this trace.", disabled=False), + }, + disabled=(col for col in base_df.columns if col not in ["Select"]), + ) st.markdown("Ours vs Base: `math.exp(abs(math.log(sota_exp_score / baseline_score)))`") + + # 统计选择的比赛 + base_df = base_df[base_df["Select"]] st.markdown(f"**统计的比赛数目: :red[{base_df.shape[0]}]**") total_stat = ( base_df[ @@ -235,9 +331,9 @@ def all_summarize_win(): != "0 (0.0%)" ).sum() total_stat.name = "总体统计(%)" - total_stat.loc["Bronze"] = base_df["Best Medal"].value_counts().get("bronze", 0) - total_stat.loc["Silver"] = base_df["Best Medal"].value_counts().get("silver", 0) - total_stat.loc["Gold"] = base_df["Best Medal"].value_counts().get("gold", 0) + total_stat.loc["Bronze"] = base_df["Best Result"].value_counts().get("bronze", 0) + total_stat.loc["Silver"] = base_df["Best Result"].value_counts().get("silver", 0) + total_stat.loc["Gold"] = base_df["Best Result"].value_counts().get("gold", 0) total_stat = total_stat / base_df.shape[0] * 100 # SOTA Exp 统计 @@ -279,37 +375,41 @@ def all_summarize_win(): st.plotly_chart(fig) # write curve - for k, v in summary.items(): - with st.container(border=True): - st.markdown(f"**:blue[{k}] - :violet[{v['competition']}]**") - fc1, fc2 = st.columns(2) - tscores = {f"loop {k-1}": v for k, v in v["test_scores"].items()} - tdf = pd.Series(tscores, name="score") - f2 = px.line(tdf, markers=True, title="Test scores") - fc2.plotly_chart(f2, key=k) - try: - vscores = {k: v.iloc[:, 0] for k, v in v["valid_scores"].items()} + st.subheader("Curves", divider="rainbow") + if st.toggle("Show Curves", key="show_curves"): + for k, v in summary.items(): + with st.container(border=True): + st.markdown(f"**:blue[{k}] - :violet[{v['competition']}]**") + fc1, fc2 = st.columns(2) + tscores = {f"loop {k-1}": v for k, v in v["test_scores"].items()} + tdf = pd.Series(tscores, name="score") + f2 = px.line(tdf, markers=True, title="Test scores") + fc2.plotly_chart(f2, key=k) + try: + vscores = {k: v.iloc[:, 0] for k, v in v["valid_scores"].items()} - if len(vscores) > 0: - metric_name = list(vscores.values())[0].name - else: - metric_name = "None" + if len(vscores) > 0: + metric_name = list(vscores.values())[0].name + else: + metric_name = "None" - vdf = pd.DataFrame(vscores) - vdf.columns = [f"loop {i}" for i in vdf.columns] - f1 = px.line(vdf.T, markers=True, title=f"Valid scores (metric: {metric_name})") + vdf = pd.DataFrame(vscores) + vdf.columns = [f"loop {i}" for i in vdf.columns] + f1 = px.line(vdf.T, markers=True, title=f"Valid scores (metric: {metric_name})") - fc1.plotly_chart(f1, key=f"{k}_v") - except Exception as e: - import traceback + fc1.plotly_chart(f1, key=f"{k}_v") + except Exception as e: + import traceback - st.markdown("- Error: " + str(e)) - st.code(traceback.format_exc()) - st.markdown("- Valid Scores: ") - # st.write({k: type(v) for k, v in v["valid_scores"].items()}) - st.json(v["valid_scores"]) + st.markdown("- Error: " + str(e)) + st.code(traceback.format_exc()) + st.markdown("- Valid Scores: ") + # st.write({k: type(v) for k, v in v["valid_scores"].items()}) + st.json(v["valid_scores"]) +with st.sidebar: + st.toggle("Show Times Info (Slowly)", key="show_times_info") with st.container(border=True): if st.toggle("近3天平均", key="show_3days"): days_summarize_win() diff --git a/rdagent/log/ui/ds_trace.py b/rdagent/log/ui/ds_trace.py index 5744f7b6..1ec22602 100644 --- a/rdagent/log/ui/ds_trace.py +++ b/rdagent/log/ui/ds_trace.py @@ -41,9 +41,16 @@ def extract_evoid(tag): return match.group(1) if match else None +def convert_defaultdict_to_dict(d): + if isinstance(d, defaultdict): + d = {k: convert_defaultdict_to_dict(v) for k, v in d.items()} + return d + + +@st.cache_data(persist=True) def load_times(log_path: Path): """加载时间数据""" - state.times = defaultdict(lambda: defaultdict(dict)) + times = defaultdict(lambda: defaultdict(dict)) for msg in FileStorage(log_path).iter_msg(): if msg.tag and "llm" not in msg.tag and "session" not in msg.tag: li, fn = extract_loopid_func_name(msg.tag) @@ -54,18 +61,14 @@ def load_times(log_path: Path): loop_obj_path = log_path / "__session__" / f"{li}" / "4_record" if loop_obj_path.exists(): try: - state.times[li] = DataScienceRDLoop.load(loop_obj_path, do_truncate=False).loop_trace[li] + times[li] = DataScienceRDLoop.load(loop_obj_path, do_truncate=False).loop_trace[li] except Exception as e: pass - -def convert_defaultdict_to_dict(d): - if isinstance(d, defaultdict): - d = {k: convert_defaultdict_to_dict(v) for k, v in d.items()} - return d + return convert_defaultdict_to_dict(times) -@st.cache_data +@st.cache_data(persist=True) def load_data(log_path: Path): data = defaultdict(lambda: defaultdict(dict)) for msg in FileStorage(log_path).iter_msg(): @@ -402,10 +405,12 @@ def summarize_data(): df = pd.DataFrame( columns=[ "Component", - "Running Score", + "Running Score (valid)", + "Running Score (test)", "Feedback", "e-loops", "Time", + "Exp Gen", "Coding", "Running", "Start Time (UTC+8)", @@ -417,8 +422,10 @@ def summarize_data(): for loop in range(len(state.data) - 1): loop_data = state.data[loop] df.loc[loop, "Component"] = loop_data["direct_exp_gen"]["no_tag"].hypothesis.component - if state.times[loop]: + if loop in state.times and state.times[loop]: df.loc[loop, "Time"] = str(sum((i.end - i.start for i in state.times[loop]), timedelta())).split(".")[0] + exp_gen_time = state.times[loop][0].end - state.times[loop][0].start + df.loc[loop, "Exp Gen"] = str(exp_gen_time).split(".")[0] coding_time = state.times[loop][1].end - state.times[loop][1].start df.loc[loop, "Coding"] = str(coding_time).split(".")[0] if len(state.times[loop]) > 2: @@ -427,15 +434,21 @@ def summarize_data(): df.loc[loop, "Start Time (UTC+8)"] = state.times[loop][0].start + timedelta(hours=8) df.loc[loop, "End Time (UTC+8)"] = state.times[loop][-1].end + timedelta(hours=8) if "running" in loop_data and "no_tag" in loop_data["running"]: + try: + df.loc[loop, "Running Score (valid)"] = round( + loop_data["running"]["no_tag"].result.loc["ensemble"].iloc[0], 5 + ) + except: + df.loc[loop, "Running Score (valid)"] = "❌" if "mle_score" not in state.data[loop]: if "mle_score" in loop_data["running"]: mle_score_txt = loop_data["running"]["mle_score"] state.data[loop]["mle_score"] = extract_mle_json(mle_score_txt) if state.data[loop]["mle_score"]["score"] is not None: - df.loc[loop, "Running Score"] = str(state.data[loop]["mle_score"]["score"]) + df.loc[loop, "Running Score (test)"] = str(state.data[loop]["mle_score"]["score"]) else: state.data[loop]["mle_score"] = mle_score_txt - df.loc[loop, "Running Score"] = "❌" + df.loc[loop, "Running Score (test)"] = "❌" else: mle_score_path = ( replace_ep_path(loop_data["running"]["no_tag"].experiment_workspace.workspace_path) @@ -445,24 +458,28 @@ def summarize_data(): mle_score_txt = mle_score_path.read_text() state.data[loop]["mle_score"] = extract_mle_json(mle_score_txt) if state.data[loop]["mle_score"]["score"] is not None: - df.loc[loop, "Running Score"] = str(state.data[loop]["mle_score"]["score"]) + df.loc[loop, "Running Score (test)"] = str(state.data[loop]["mle_score"]["score"]) else: state.data[loop]["mle_score"] = mle_score_txt - df.loc[loop, "Running Score"] = "❌" + df.loc[loop, "Running Score (test)"] = "❌" except Exception as e: state.data[loop]["mle_score"] = str(e) - df.loc[loop, "Running Score"] = "❌" + df.loc[loop, "Running Score (test)"] = "❌" else: if isinstance(state.data[loop]["mle_score"], dict): - df.loc[loop, "Running Score"] = str(state.data[loop]["mle_score"]["score"]) + df.loc[loop, "Running Score (test)"] = str(state.data[loop]["mle_score"]["score"]) else: - df.loc[loop, "Running Score"] = "❌" + df.loc[loop, "Running Score (test)"] = "❌" else: - df.loc[loop, "Running Score"] = "N/A" + df.loc[loop, "Running Score (valid)"] = "N/A" + df.loc[loop, "Running Score (test)"] = "N/A" if "coding" in loop_data: - df.loc[loop, "e-loops"] = max(i for i in loop_data["coding"].keys() if isinstance(i, int)) + 1 + if len([i for i in loop_data["coding"].keys() if isinstance(i, int)]) == 0: + df.loc[loop, "e-loops"] = 0 + else: + df.loc[loop, "e-loops"] = max(i for i in loop_data["coding"].keys() if isinstance(i, int)) + 1 if "feedback" in loop_data: df.loc[loop, "Feedback"] = "✅" if bool(loop_data["feedback"]["no_tag"]) else "❌" else: @@ -471,13 +488,16 @@ def summarize_data(): def comp_stat_func(x: pd.DataFrame): total_num = x.shape[0] - valid_num = x[x["Running Score"] != "N/A"].shape[0] + valid_num = x[x["Running Score (test)"] != "N/A"].shape[0] + success_num = x[x["Feedback"] == "✅"].shape[0] avg_e_loops = x["e-loops"].mean() return pd.Series( { "Loop Num": total_num, "Valid Loop": valid_num, + "Success Loop": success_num, "Valid Rate": round(valid_num / total_num * 100, 2), + "Success Rate": round(success_num / total_num * 100, 2), "Avg e-loops": round(avg_e_loops, 2), } ) @@ -485,22 +505,38 @@ def summarize_data(): st1, st2 = st.columns([1, 1]) # component statistics - comp_df = df.loc[:, ["Component", "Running Score", "e-loops"]].groupby("Component").apply(comp_stat_func) + comp_df = ( + df.loc[:, ["Component", "Running Score (test)", "Feedback", "e-loops"]] + .groupby("Component") + .apply(comp_stat_func) + ) comp_df.loc["Total"] = comp_df.sum() comp_df.loc["Total", "Valid Rate"] = round( comp_df.loc["Total", "Valid Loop"] / comp_df.loc["Total", "Loop Num"] * 100, 2 ) + comp_df.loc["Total", "Success Rate"] = round( + comp_df.loc["Total", "Success Loop"] / comp_df.loc["Total", "Loop Num"] * 100, 2 + ) comp_df["Valid Rate"] = comp_df["Valid Rate"].apply(lambda x: f"{x}%") + comp_df["Success Rate"] = comp_df["Success Rate"].apply(lambda x: f"{x}%") comp_df.loc["Total", "Avg e-loops"] = round(df["e-loops"].mean(), 2) st2.markdown("### Component Statistics") st2.dataframe(comp_df) # component time statistics - time_df = df.loc[:, ["Component", "Time", "Coding", "Running"]] - time_df = time_df.astype({"Time": "timedelta64[ns]", "Coding": "timedelta64[ns]", "Running": "timedelta64[ns]"}) + time_df = df.loc[:, ["Component", "Time", "Exp Gen", "Coding", "Running"]] + time_df = time_df.astype( + { + "Time": "timedelta64[ns]", + "Exp Gen": "timedelta64[ns]", + "Coding": "timedelta64[ns]", + "Running": "timedelta64[ns]", + } + ) st1.markdown("### Time Statistics") time_stat_df = time_df.groupby("Component").sum() time_stat_df.loc["Total"] = time_stat_df.sum() + time_stat_df.loc[:, "Exp Gen(%)"] = time_stat_df["Exp Gen"] / time_stat_df["Time"] * 100 time_stat_df.loc[:, "Coding(%)"] = time_stat_df["Coding"] / time_stat_df["Time"] * 100 time_stat_df.loc[:, "Running(%)"] = time_stat_df["Running"] / time_stat_df["Time"] * 100 time_stat_df = time_stat_df.map(lambda x: str(x).split(".")[0] if pd.notnull(x) else "0:00:00") @@ -559,19 +595,30 @@ with st.sidebar: elif day_srv == "srv3": state.log_folders = [re.sub(r"log\.srv\d*", "log.srv3", folder) for folder in state.log_folders] - state.log_folder = Path(st.radio(f"Select :blue[**one log folder**]", state.log_folders)) + if "log_folder" in st.query_params: + state.log_folder = Path(st.query_params["log_folder"]) + else: + state.log_folder = Path(st.radio(f"Select :blue[**one log folder**]", state.log_folders)) if not state.log_folder.exists(): st.warning(f"Path {state.log_folder} does not exist!") else: folders = get_folders_sorted(state.log_folder) - st.selectbox(f"Select from :blue[**{state.log_folder.absolute()}**]", folders, key="log_path") + if "selection" in st.query_params: + default_index = ( + folders.index(st.query_params["selection"]) if st.query_params["selection"] in folders else 0 + ) + else: + default_index = 0 + state.log_path = st.selectbox( + f"Select from :blue[**{state.log_folder.absolute()}**]", folders, index=default_index + ) if st.button("Refresh Data"): if state.log_path is None: st.toast("Please select a log path first!", icon="🟡") st.stop() - load_times(state.log_folder / state.log_path) + state.times = load_times(state.log_folder / state.log_path) state.data, state.llm_data = load_data(state.log_folder / state.log_path) st.rerun() st.toggle("Show LLM Log", key="show_llm_log") @@ -589,6 +636,7 @@ with st.sidebar: # UI - Main if state.data["competition"]: st.title(state.data["competition"]) + st.markdown(f"[share_link](/ds_trace?log_folder={state.log_folder}&selection={state.log_path})") summarize_data() if len(state.data) > 2: loop_id = st.slider("Loop", 0, len(state.data) - 2, 0) diff --git a/rdagent/utils/env.py b/rdagent/utils/env.py index fe0e5c9f..2351bc98 100644 --- a/rdagent/utils/env.py +++ b/rdagent/utils/env.py @@ -132,9 +132,10 @@ class Env(Generic[ASpecificEnvConf]): entry, local_path, env, running_extra_volume=running_extra_volume, remove_timestamp=remove_timestamp ) end = time.time() - if end - start >= self.conf.running_timeout_period: - print( - f"[red]The running time exceeds {self.conf.running_timeout_period} seconds, so the process is killed.[/red]" + logger.info(f"Running time: {end - start} seconds") + if end - start + 1 >= self.conf.running_timeout_period: + logger.warning( + f"The running time exceeds {self.conf.running_timeout_period} seconds, so the process is killed." ) log_output += f"\n\nThe running time exceeds {self.conf.running_timeout_period} seconds, so the process is killed." return log_output, return_code