add cache for summary, remove workspace dependency when generate summary (#854)

rdagent/log/mle_summary.py

@@ -131,21 +131,20 @@ def summarize_folder(log_folder: Path, hours: int | None = None):
                     loop_num += 1
 
                 if "running" in msg.tag:
+                    loop_id, _ = extract_loopid_func_name(msg.tag)
+                    loop_id = int(loop_id)
                     if isinstance(msg.content, DSExperiment):
-                        submission_path = msg.content.experiment_workspace.workspace_path / "submission.csv"
-                        if submission_path.exists():
-                            made_submission_num += 1
-                            scores_path = msg.content.experiment_workspace.workspace_path / "scores.csv"
-                            valid_scores[loop_num - 1] = pd.read_csv(scores_path, index_col=0)
+                        if msg.content.result is not None:
+                            valid_scores[loop_id] = msg.content.result
                     elif "mle_score" in msg.tag:
-                        loop_id, _ = extract_loopid_func_name(msg.tag)
-                        loop_id = int(loop_id)
                         grade_output = extract_mle_json(msg.content)
                         if grade_output:
+                            if grade_output["submission_exists"]:
+                                made_submission_num += 1
                             if grade_output["score"] is not None:
-                                test_scores[loop_id + 1] = grade_output["score"]
+                                test_scores[loop_id] = grade_output["score"]
                                 if is_mle:
-                                    _, test_ranks[loop_id + 1] = score_rank(
+                                    _, test_ranks[loop_id] = score_rank(
                                         stat[log_trace_path.name]["competition"], grade_output["score"]
                                     )
                             if grade_output["valid_submission"]:

rdagent/log/ui/ds_summary.py

@@ -50,7 +50,7 @@ def get_final_sota_exp(log_path: Path):
     return final_sota_exp
 
 
-# @st.cache_data(persist=True)
+@st.cache_data(persist=True)
 def get_sota_exp_stat(log_path: Path):
     trace_paths = [i for i in log_path.rglob(f"**/trace/**/*.pkl")]
     if len(trace_paths) == 0:
@@ -98,20 +98,12 @@ def get_sota_exp_stat(log_path: Path):
     return sota_exp_stat
 
 
-# @st.cache_data(persist=True)
+@st.cache_data(persist=True)
 def get_summary_df(log_folders: list[str]) -> tuple[dict, pd.DataFrame]:
     summarys = {}
-    with st.sidebar:
-        if st.toggle("show 24h summary", key="show_hours_summary"):
-            sn = "summary_24h.pkl"
-        else:
-            sn = "summary.pkl"
+    sn = "summary.pkl"
     for lf in log_folders:
-        if not (Path(lf) / sn).exists():
-            st.warning(
-                f"{sn} not found in **{lf}**\n\nRun:`dotenv run -- python rdagent/log/mle_summary.py grade_summary --log_folder={lf} --hours=<>`"
-            )
-        else:
+        if (Path(lf) / sn).exists():
             summarys[lf] = pd.read_pickle(Path(lf) / sn)
 
     if len(summarys) == 0:
@@ -397,6 +389,11 @@ def all_summarize_win():
     selected_folders = st.multiselect(
         "Show these folders", state.log_folders, state.log_folders, format_func=shorten_folder_name
     )
+    for lf in selected_folders:
+        if not (Path(lf) / "summary.pkl").exists():
+            st.warning(
+                f"summary.pkl not found in **{lf}**\n\nRun:`dotenv run -- python rdagent/log/mle_summary.py grade_summary --log_folder={lf} --hours=<>`"
+            )
     summary, base_df = get_summary_df(selected_folders)
     if not summary:
         return

rdagent/log/ui/dsapp.py

@@ -6,11 +6,24 @@ from streamlit import session_state as state
 from rdagent.app.data_science.loop import DataScienceRDLoop
 from rdagent.log.ui.conf import UI_SETTING
 
+
+def convert_log_folder_str(lf: str) -> str:
+    if "/" not in lf:
+        return f"/data/share_folder_local/amlt/{lf.strip()}/combined_logs"
+    return lf.strip()
+
+
+def extract_amlt_name(x: str) -> str:
+    if "amlt" not in x:
+        return x
+    return x[x.rfind("amlt") + 5 :].split("/")[0]
+
+
 # 设置主日志路径
 if "log_folder" not in state:
     state.log_folder = Path("./log")
 if "log_folders" not in state:
-    state.log_folders = UI_SETTING.default_log_folders
+    state.log_folders = [convert_log_folder_str(i) for i in UI_SETTING.default_log_folders]
 
 summary_page = st.Page("ds_summary.py", title="Summary", icon="📊")
 trace_page = st.Page("ds_trace.py", title="Trace", icon="📈")
@@ -19,12 +32,6 @@ st.set_page_config(layout="wide", page_title="RD-Agent", page_icon="🎓", initi
 st.navigation([summary_page, trace_page, aide_page]).run()
 
 
-def convert_log_folder_str(lf: str) -> str:
-    if "/" not in lf:
-        return f"/data/share_folder_local/amlt/{lf.strip()}/combined_logs"
-    return lf.strip()
-
-
 # UI - Sidebar
 with st.sidebar:
     st.subheader("Pages", divider="rainbow")
@@ -34,7 +41,9 @@ with st.sidebar:
 
     st.subheader("Settings", divider="rainbow")
     with st.form("log_folder_form", border=False):
-        log_folder_str = st.text_area("**Log Folders**(split by ';')", placeholder=state.log_folder)
+        log_folder_str = st.text_area(
+            "**Log Folders**(split by ';')", value=";".join(extract_amlt_name(i) for i in state.log_folders)
+        )
         if st.form_submit_button("Confirm"):
             state.log_folders = [
                 convert_log_folder_str(folder) for folder in log_folder_str.split(";") if folder.strip()