From 9df59eb7f00810f2bdafd569f50c4eaef23749e4 Mon Sep 17 00:00:00 2001 From: XianBW <36835909+XianBW@users.noreply.github.com> Date: Thu, 8 May 2025 12:24:48 +0800 Subject: [PATCH] add cache for summary, remove workspace dependency when generate summary (#854) --- rdagent/log/mle_summary.py | 17 ++++++++--------- rdagent/log/ui/ds_summary.py | 21 +++++++++------------ rdagent/log/ui/dsapp.py | 25 +++++++++++++++++-------- 3 files changed, 34 insertions(+), 29 deletions(-) diff --git a/rdagent/log/mle_summary.py b/rdagent/log/mle_summary.py index a2f1f96f..273dd066 100644 --- a/rdagent/log/mle_summary.py +++ b/rdagent/log/mle_summary.py @@ -131,21 +131,20 @@ def summarize_folder(log_folder: Path, hours: int | None = None): loop_num += 1 if "running" in msg.tag: + loop_id, _ = extract_loopid_func_name(msg.tag) + loop_id = int(loop_id) if isinstance(msg.content, DSExperiment): - submission_path = msg.content.experiment_workspace.workspace_path / "submission.csv" - if submission_path.exists(): - made_submission_num += 1 - scores_path = msg.content.experiment_workspace.workspace_path / "scores.csv" - valid_scores[loop_num - 1] = pd.read_csv(scores_path, index_col=0) + if msg.content.result is not None: + valid_scores[loop_id] = msg.content.result elif "mle_score" in msg.tag: - loop_id, _ = extract_loopid_func_name(msg.tag) - loop_id = int(loop_id) grade_output = extract_mle_json(msg.content) if grade_output: + if grade_output["submission_exists"]: + made_submission_num += 1 if grade_output["score"] is not None: - test_scores[loop_id + 1] = grade_output["score"] + test_scores[loop_id] = grade_output["score"] if is_mle: - _, test_ranks[loop_id + 1] = score_rank( + _, test_ranks[loop_id] = score_rank( stat[log_trace_path.name]["competition"], grade_output["score"] ) if grade_output["valid_submission"]: diff --git a/rdagent/log/ui/ds_summary.py b/rdagent/log/ui/ds_summary.py index abf138e3..1f550a2d 100644 --- a/rdagent/log/ui/ds_summary.py +++ b/rdagent/log/ui/ds_summary.py @@ -50,7 +50,7 @@ def get_final_sota_exp(log_path: Path): return final_sota_exp -# @st.cache_data(persist=True) +@st.cache_data(persist=True) def get_sota_exp_stat(log_path: Path): trace_paths = [i for i in log_path.rglob(f"**/trace/**/*.pkl")] if len(trace_paths) == 0: @@ -98,20 +98,12 @@ def get_sota_exp_stat(log_path: Path): return sota_exp_stat -# @st.cache_data(persist=True) +@st.cache_data(persist=True) def get_summary_df(log_folders: list[str]) -> tuple[dict, pd.DataFrame]: summarys = {} - with st.sidebar: - if st.toggle("show 24h summary", key="show_hours_summary"): - sn = "summary_24h.pkl" - else: - sn = "summary.pkl" + sn = "summary.pkl" for lf in log_folders: - if not (Path(lf) / sn).exists(): - st.warning( - f"{sn} not found in **{lf}**\n\nRun:`dotenv run -- python rdagent/log/mle_summary.py grade_summary --log_folder={lf} --hours=<>`" - ) - else: + if (Path(lf) / sn).exists(): summarys[lf] = pd.read_pickle(Path(lf) / sn) if len(summarys) == 0: @@ -397,6 +389,11 @@ def all_summarize_win(): selected_folders = st.multiselect( "Show these folders", state.log_folders, state.log_folders, format_func=shorten_folder_name ) + for lf in selected_folders: + if not (Path(lf) / "summary.pkl").exists(): + st.warning( + f"summary.pkl not found in **{lf}**\n\nRun:`dotenv run -- python rdagent/log/mle_summary.py grade_summary --log_folder={lf} --hours=<>`" + ) summary, base_df = get_summary_df(selected_folders) if not summary: return diff --git a/rdagent/log/ui/dsapp.py b/rdagent/log/ui/dsapp.py index 6b59eab7..c2dcba0c 100644 --- a/rdagent/log/ui/dsapp.py +++ b/rdagent/log/ui/dsapp.py @@ -6,11 +6,24 @@ from streamlit import session_state as state from rdagent.app.data_science.loop import DataScienceRDLoop from rdagent.log.ui.conf import UI_SETTING + +def convert_log_folder_str(lf: str) -> str: + if "/" not in lf: + return f"/data/share_folder_local/amlt/{lf.strip()}/combined_logs" + return lf.strip() + + +def extract_amlt_name(x: str) -> str: + if "amlt" not in x: + return x + return x[x.rfind("amlt") + 5 :].split("/")[0] + + # 设置主日志路径 if "log_folder" not in state: state.log_folder = Path("./log") if "log_folders" not in state: - state.log_folders = UI_SETTING.default_log_folders + state.log_folders = [convert_log_folder_str(i) for i in UI_SETTING.default_log_folders] summary_page = st.Page("ds_summary.py", title="Summary", icon="📊") trace_page = st.Page("ds_trace.py", title="Trace", icon="📈") @@ -19,12 +32,6 @@ st.set_page_config(layout="wide", page_title="RD-Agent", page_icon="🎓", initi st.navigation([summary_page, trace_page, aide_page]).run() -def convert_log_folder_str(lf: str) -> str: - if "/" not in lf: - return f"/data/share_folder_local/amlt/{lf.strip()}/combined_logs" - return lf.strip() - - # UI - Sidebar with st.sidebar: st.subheader("Pages", divider="rainbow") @@ -34,7 +41,9 @@ with st.sidebar: st.subheader("Settings", divider="rainbow") with st.form("log_folder_form", border=False): - log_folder_str = st.text_area("**Log Folders**(split by ';')", placeholder=state.log_folder) + log_folder_str = st.text_area( + "**Log Folders**(split by ';')", value=";".join(extract_amlt_name(i) for i in state.log_folders) + ) if st.form_submit_button("Confirm"): state.log_folders = [ convert_log_folder_str(folder) for folder in log_folder_str.split(";") if folder.strip()