mirror of
https://github.com/NicolasBohn/NexQuant.git
synced 2026-07-28 16:07:46 +00:00
chore: split dsapp to pages & add llm_logs showing (#733)
* split dsapp to pages & add llm_logs showing * some changes * small bug * add time info * do not test UI in test_import * cache stdout
This commit is contained in:
XianBW
@@ -0,0 +1,317 @@
|
||||
import math
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
import pandas as pd
|
||||
import plotly.express as px
|
||||
import plotly.graph_objects as go
|
||||
import streamlit as st
|
||||
from streamlit import session_state as state
|
||||
|
||||
from rdagent.log.ui.conf import UI_SETTING
|
||||
|
||||
|
||||
def get_summary_df(log_folders: list[str]) -> tuple[dict, pd.DataFrame]:
|
||||
summarys = {}
|
||||
for lf in log_folders:
|
||||
if not (Path(lf) / "summary.pkl").exists():
|
||||
st.warning(
|
||||
f"No summary file found in **{lf}**\n\nRun:`dotenv run -- python rdagent/log/mle_summary.py grade_summary --log_folder={lf}`"
|
||||
)
|
||||
else:
|
||||
summarys[lf] = pd.read_pickle(Path(lf) / "summary.pkl")
|
||||
|
||||
if len(summarys) == 0:
|
||||
return {}, pd.DataFrame()
|
||||
|
||||
summary = {}
|
||||
for lf, s in summarys.items():
|
||||
for k, v in s.items():
|
||||
stdout_p = Path(lf) / f"{k}.stdout"
|
||||
v["stdout"] = []
|
||||
if stdout_p.exists():
|
||||
# stdout = stdout_p.read_text()
|
||||
stdout = ""
|
||||
if "Retrying" in stdout:
|
||||
v["stdout"].append("LLM Retry")
|
||||
if "Traceback (most recent call last):" in stdout[-10000:]:
|
||||
v["stdout"].append("Code Error")
|
||||
v["stdout"] = ", ".join([i for i in v["stdout"] if i])
|
||||
|
||||
# 调整实验名字
|
||||
if "amlt" in lf:
|
||||
summary[f"{lf[lf.rfind('amlt')+5:].split('/')[0]} - {k}"] = v
|
||||
elif "ep" in lf:
|
||||
summary[f"{lf[lf.rfind('ep'):]} - {k}"] = v
|
||||
else:
|
||||
summary[f"{lf} - {k}"] = v
|
||||
|
||||
summary = {k: v for k, v in summary.items() if "competition" in v}
|
||||
base_df = pd.DataFrame(
|
||||
columns=[
|
||||
"Competition",
|
||||
"Total Loops",
|
||||
"Successful Final Decision",
|
||||
"Made Submission",
|
||||
"Valid Submission",
|
||||
"V/M",
|
||||
"Above Median",
|
||||
"Bronze",
|
||||
"Silver",
|
||||
"Gold",
|
||||
"Any Medal",
|
||||
"Best Medal",
|
||||
"SOTA Exp",
|
||||
"Ours - Base",
|
||||
"Ours vs Base",
|
||||
"SOTA Exp Score",
|
||||
"Baseline Score",
|
||||
"Bronze Threshold",
|
||||
"Silver Threshold",
|
||||
"Gold Threshold",
|
||||
"Medium Threshold",
|
||||
"stdout",
|
||||
],
|
||||
index=summary.keys(),
|
||||
)
|
||||
|
||||
# Read baseline results
|
||||
baseline_result_path = UI_SETTING.baseline_result_path
|
||||
if Path(baseline_result_path).exists():
|
||||
baseline_df = pd.read_csv(baseline_result_path)
|
||||
|
||||
for k, v in summary.items():
|
||||
loop_num = v["loop_num"]
|
||||
base_df.loc[k, "Competition"] = v["competition"]
|
||||
base_df.loc[k, "Total Loops"] = loop_num
|
||||
if loop_num == 0:
|
||||
base_df.loc[k] = "N/A"
|
||||
else:
|
||||
base_df.loc[k, "Successful Final Decision"] = v["success_loop_num"]
|
||||
base_df.loc[k, "Made Submission"] = v["made_submission_num"]
|
||||
base_df.loc[k, "Valid Submission"] = v["valid_submission_num"]
|
||||
base_df.loc[k, "Above Median"] = v["above_median_num"]
|
||||
base_df.loc[k, "Bronze"] = v["bronze_num"]
|
||||
if v["bronze_num"] > 0:
|
||||
base_df.loc[k, "Best Medal"] = "bronze"
|
||||
base_df.loc[k, "Silver"] = v["silver_num"]
|
||||
if v["silver_num"] > 0:
|
||||
base_df.loc[k, "Best Medal"] = "silver"
|
||||
base_df.loc[k, "Gold"] = v["gold_num"]
|
||||
if v["gold_num"] > 0:
|
||||
base_df.loc[k, "Best Medal"] = "gold"
|
||||
base_df.loc[k, "Any Medal"] = v["get_medal_num"]
|
||||
|
||||
baseline_score = None
|
||||
if Path(baseline_result_path).exists():
|
||||
baseline_score = baseline_df.loc[baseline_df["competition_id"] == v["competition"], "score"].item()
|
||||
|
||||
base_df.loc[k, "SOTA Exp"] = v.get("sota_exp_stat", None)
|
||||
if baseline_score is not None and v.get("sota_exp_score", None) is not None:
|
||||
base_df.loc[k, "Ours - Base"] = v["sota_exp_score"] - baseline_score
|
||||
try:
|
||||
base_df.loc[k, "Ours vs Base"] = math.exp(
|
||||
abs(math.log(v["sota_exp_score"] / baseline_score))
|
||||
) # exp^|ln(a/b)|
|
||||
except Exception as e:
|
||||
base_df.loc[k, "Ours vs Base"] = None
|
||||
base_df.loc[k, "SOTA Exp Score"] = v.get("sota_exp_score", None)
|
||||
base_df.loc[k, "Baseline Score"] = baseline_score
|
||||
base_df.loc[k, "Bronze Threshold"] = v.get("bronze_threshold", None)
|
||||
base_df.loc[k, "Silver Threshold"] = v.get("silver_threshold", None)
|
||||
base_df.loc[k, "Gold Threshold"] = v.get("gold_threshold", None)
|
||||
base_df.loc[k, "Medium Threshold"] = v.get("median_threshold", None)
|
||||
base_df.loc[k, "stdout"] = v["stdout"]
|
||||
|
||||
base_df["SOTA Exp"] = base_df["SOTA Exp"].replace("", pd.NA)
|
||||
base_df = base_df.astype(
|
||||
{
|
||||
"Total Loops": int,
|
||||
"Successful Final Decision": int,
|
||||
"Made Submission": int,
|
||||
"Valid Submission": int,
|
||||
"Above Median": int,
|
||||
"Bronze": int,
|
||||
"Silver": int,
|
||||
"Gold": int,
|
||||
"Any Medal": int,
|
||||
"Ours - Base": float,
|
||||
"Ours vs Base": float,
|
||||
"SOTA Exp Score": float,
|
||||
"Baseline Score": float,
|
||||
"Bronze Threshold": float,
|
||||
"Silver Threshold": float,
|
||||
"Gold Threshold": float,
|
||||
"Medium Threshold": float,
|
||||
}
|
||||
)
|
||||
return summary, base_df
|
||||
|
||||
|
||||
def num2percent(num: int, total: int, show_origin=True) -> str:
|
||||
if show_origin:
|
||||
return f"{num} ({round(num / total * 100, 2)}%)"
|
||||
return f"{round(num / total * 100, 2)}%"
|
||||
|
||||
|
||||
def percent_df(df: pd.DataFrame, show_origin=True) -> pd.DataFrame:
|
||||
base_df = df.astype("object", copy=True)
|
||||
for k in base_df.index:
|
||||
loop_num = int(base_df.loc[k, "Total Loops"])
|
||||
if loop_num != 0:
|
||||
base_df.loc[k, "Successful Final Decision"] = num2percent(
|
||||
base_df.loc[k, "Successful Final Decision"], loop_num, show_origin
|
||||
)
|
||||
if base_df.loc[k, "Made Submission"] != 0:
|
||||
base_df.loc[k, "V/M"] = (
|
||||
f"{round(base_df.loc[k, 'Valid Submission'] / base_df.loc[k, 'Made Submission'] * 100, 2)}%"
|
||||
)
|
||||
else:
|
||||
base_df.loc[k, "V/M"] = "N/A"
|
||||
base_df.loc[k, "Made Submission"] = num2percent(base_df.loc[k, "Made Submission"], loop_num, show_origin)
|
||||
base_df.loc[k, "Valid Submission"] = num2percent(base_df.loc[k, "Valid Submission"], loop_num, show_origin)
|
||||
base_df.loc[k, "Above Median"] = num2percent(base_df.loc[k, "Above Median"], loop_num, show_origin)
|
||||
base_df.loc[k, "Bronze"] = num2percent(base_df.loc[k, "Bronze"], loop_num, show_origin)
|
||||
base_df.loc[k, "Silver"] = num2percent(base_df.loc[k, "Silver"], loop_num, show_origin)
|
||||
base_df.loc[k, "Gold"] = num2percent(base_df.loc[k, "Gold"], loop_num, show_origin)
|
||||
base_df.loc[k, "Any Medal"] = num2percent(base_df.loc[k, "Any Medal"], loop_num, show_origin)
|
||||
return base_df
|
||||
|
||||
|
||||
def days_summarize_win():
|
||||
lfs1 = [re.sub(r"log\.srv\d*", "log.srv", folder) for folder in state.log_folders]
|
||||
lfs2 = [re.sub(r"log\.srv\d*", "log.srv2", folder) for folder in state.log_folders]
|
||||
lfs3 = [re.sub(r"log\.srv\d*", "log.srv3", folder) for folder in state.log_folders]
|
||||
|
||||
_, df1 = get_summary_df(lfs1)
|
||||
_, df2 = get_summary_df(lfs2)
|
||||
_, df3 = get_summary_df(lfs3)
|
||||
|
||||
df = pd.concat([df1, df2, df3], axis=0)
|
||||
|
||||
def mean_func(x: pd.DataFrame):
|
||||
numeric_cols = x.select_dtypes(include=["int", "float"]).mean()
|
||||
string_cols = x.select_dtypes(include=["object"]).agg(lambda col: ", ".join(col.fillna("none").astype(str)))
|
||||
return pd.concat([numeric_cols, string_cols], axis=0).reindex(x.columns).drop("Competition")
|
||||
|
||||
df = df.groupby("Competition").apply(mean_func)
|
||||
if st.toggle("Show Percent", key="show_percent"):
|
||||
st.dataframe(percent_df(df, show_origin=False))
|
||||
else:
|
||||
st.dataframe(df)
|
||||
|
||||
|
||||
def all_summarize_win():
|
||||
def shorten_folder_name(folder: str) -> str:
|
||||
if "amlt" in folder:
|
||||
return folder[folder.rfind("amlt") + 5 :].split("/")[0]
|
||||
if "ep" in folder:
|
||||
return folder[folder.rfind("ep") :]
|
||||
return folder
|
||||
|
||||
selected_folders = st.multiselect(
|
||||
"Show these folders", state.log_folders, state.log_folders, format_func=shorten_folder_name
|
||||
)
|
||||
summary, base_df = get_summary_df(selected_folders)
|
||||
if not summary:
|
||||
return
|
||||
|
||||
base_df = percent_df(base_df)
|
||||
st.dataframe(base_df)
|
||||
st.markdown("Ours vs Base: `math.exp(abs(math.log(sota_exp_score / baseline_score)))`")
|
||||
st.markdown(f"**统计的比赛数目: :red[{base_df.shape[0]}]**")
|
||||
total_stat = (
|
||||
base_df[
|
||||
[
|
||||
"Made Submission",
|
||||
"Valid Submission",
|
||||
"Above Median",
|
||||
"Bronze",
|
||||
"Silver",
|
||||
"Gold",
|
||||
"Any Medal",
|
||||
]
|
||||
]
|
||||
!= "0 (0.0%)"
|
||||
).sum()
|
||||
total_stat.name = "总体统计(%)"
|
||||
total_stat.loc["Bronze"] = base_df["Best Medal"].value_counts().get("bronze", 0)
|
||||
total_stat.loc["Silver"] = base_df["Best Medal"].value_counts().get("silver", 0)
|
||||
total_stat.loc["Gold"] = base_df["Best Medal"].value_counts().get("gold", 0)
|
||||
total_stat = total_stat / base_df.shape[0] * 100
|
||||
|
||||
# SOTA Exp 统计
|
||||
se_counts = base_df["SOTA Exp"].value_counts(dropna=True)
|
||||
se_counts.loc["made_submission"] = se_counts.sum()
|
||||
se_counts.loc["Any Medal"] = se_counts.get("gold", 0) + se_counts.get("silver", 0) + se_counts.get("bronze", 0)
|
||||
se_counts.loc["above_median"] = se_counts.get("above_median", 0) + se_counts.get("Any Medal", 0)
|
||||
se_counts.loc["valid_submission"] = se_counts.get("valid_submission", 0) + se_counts.get("above_median", 0)
|
||||
|
||||
sota_exp_stat = pd.Series(index=total_stat.index, dtype=int, name="SOTA Exp 统计(%)")
|
||||
sota_exp_stat.loc["Made Submission"] = se_counts.get("made_submission", 0)
|
||||
sota_exp_stat.loc["Valid Submission"] = se_counts.get("valid_submission", 0)
|
||||
sota_exp_stat.loc["Above Median"] = se_counts.get("above_median", 0)
|
||||
sota_exp_stat.loc["Bronze"] = se_counts.get("bronze", 0)
|
||||
sota_exp_stat.loc["Silver"] = se_counts.get("silver", 0)
|
||||
sota_exp_stat.loc["Gold"] = se_counts.get("gold", 0)
|
||||
sota_exp_stat.loc["Any Medal"] = se_counts.get("Any Medal", 0)
|
||||
sota_exp_stat = sota_exp_stat / base_df.shape[0] * 100
|
||||
|
||||
stat_df = pd.concat([total_stat, sota_exp_stat], axis=1)
|
||||
stat_t0, stat_t1 = st.columns(2)
|
||||
with stat_t0:
|
||||
st.dataframe(stat_df.round(2))
|
||||
markdown_table = f"""
|
||||
| xxx | {stat_df.iloc[0,1]:.1f} | {stat_df.iloc[1,1]:.1f} | {stat_df.iloc[2,1]:.1f} | {stat_df.iloc[3,1]:.1f} | {stat_df.iloc[4,1]:.1f} | {stat_df.iloc[5,1]:.1f} | {stat_df.iloc[6,1]:.1f} |
|
||||
"""
|
||||
st.text(markdown_table)
|
||||
with stat_t1:
|
||||
Loop_counts = base_df["Total Loops"]
|
||||
fig = px.histogram(Loop_counts, nbins=10, title="Total Loops Histogram (nbins=10)")
|
||||
mean_value = Loop_counts.mean()
|
||||
median_value = Loop_counts.median()
|
||||
fig.add_vline(
|
||||
x=mean_value, line_color="orange", annotation_text="Mean", annotation_position="top right", line_width=3
|
||||
)
|
||||
fig.add_vline(
|
||||
x=median_value, line_color="red", annotation_text="Median", annotation_position="top right", line_width=3
|
||||
)
|
||||
st.plotly_chart(fig)
|
||||
|
||||
# write curve
|
||||
for k, v in summary.items():
|
||||
with st.container(border=True):
|
||||
st.markdown(f"**:blue[{k}] - :violet[{v['competition']}]**")
|
||||
fc1, fc2 = st.columns(2)
|
||||
tscores = {f"loop {k-1}": v for k, v in v["test_scores"].items()}
|
||||
tdf = pd.Series(tscores, name="score")
|
||||
f2 = px.line(tdf, markers=True, title="Test scores")
|
||||
fc2.plotly_chart(f2, key=k)
|
||||
try:
|
||||
vscores = {k: v.iloc[:, 0] for k, v in v["valid_scores"].items()}
|
||||
|
||||
if len(vscores) > 0:
|
||||
metric_name = list(vscores.values())[0].name
|
||||
else:
|
||||
metric_name = "None"
|
||||
|
||||
vdf = pd.DataFrame(vscores)
|
||||
vdf.columns = [f"loop {i}" for i in vdf.columns]
|
||||
f1 = px.line(vdf.T, markers=True, title=f"Valid scores (metric: {metric_name})")
|
||||
|
||||
fc1.plotly_chart(f1, key=f"{k}_v")
|
||||
except Exception as e:
|
||||
import traceback
|
||||
|
||||
st.markdown("- Error: " + str(e))
|
||||
st.code(traceback.format_exc())
|
||||
st.markdown("- Valid Scores: ")
|
||||
# st.write({k: type(v) for k, v in v["valid_scores"].items()})
|
||||
st.json(v["valid_scores"])
|
||||
|
||||
|
||||
with st.container(border=True):
|
||||
if st.toggle("近3天平均", key="show_3days"):
|
||||
days_summarize_win()
|
||||
with st.container(border=True):
|
||||
all_summarize_win()
|
||||
@@ -0,0 +1,599 @@
|
||||
import hashlib
|
||||
import json
|
||||
import pickle
|
||||
import re
|
||||
from collections import defaultdict
|
||||
from datetime import timedelta
|
||||
from pathlib import Path
|
||||
|
||||
import pandas as pd
|
||||
import streamlit as st
|
||||
from streamlit import session_state as state
|
||||
|
||||
from rdagent.app.data_science.loop import DataScienceRDLoop
|
||||
from rdagent.log.mle_summary import extract_mle_json, is_valid_session
|
||||
from rdagent.log.storage import FileStorage
|
||||
from rdagent.utils import remove_ansi_codes
|
||||
|
||||
if "show_stdout" not in state:
|
||||
state.show_stdout = False
|
||||
if "show_llm_log" not in state:
|
||||
state.show_llm_log = False
|
||||
if "data" not in state:
|
||||
state.data = defaultdict(lambda: defaultdict(dict))
|
||||
if "llm_data" not in state:
|
||||
state.llm_data = defaultdict(lambda: defaultdict(lambda: defaultdict(list)))
|
||||
if "log_path" not in state:
|
||||
state.log_path = None
|
||||
if "log_folder" not in state:
|
||||
state.log_folder = Path("./log")
|
||||
|
||||
|
||||
def extract_loopid_func_name(tag):
|
||||
"""提取 Loop ID 和函数名称"""
|
||||
match = re.search(r"Loop_(\d+)\.([^.]+)", tag)
|
||||
return match.groups() if match else (None, None)
|
||||
|
||||
|
||||
def extract_evoid(tag):
|
||||
"""提取 EVO ID"""
|
||||
match = re.search(r"\.evo_loop_(\d+)\.", tag)
|
||||
return match.group(1) if match else None
|
||||
|
||||
|
||||
def load_times(log_path: Path):
|
||||
"""加载时间数据"""
|
||||
state.times = defaultdict(lambda: defaultdict(dict))
|
||||
for msg in FileStorage(log_path).iter_msg():
|
||||
if msg.tag and "llm" not in msg.tag and "session" not in msg.tag:
|
||||
li, fn = extract_loopid_func_name(msg.tag)
|
||||
if li:
|
||||
li = int(li)
|
||||
|
||||
# read times
|
||||
loop_obj_path = log_path / "__session__" / f"{li}" / "4_record"
|
||||
if loop_obj_path.exists():
|
||||
try:
|
||||
state.times[li] = DataScienceRDLoop.load(loop_obj_path, do_truncate=False).loop_trace[li]
|
||||
except Exception as e:
|
||||
pass
|
||||
|
||||
|
||||
def convert_defaultdict_to_dict(d):
|
||||
if isinstance(d, defaultdict):
|
||||
d = {k: convert_defaultdict_to_dict(v) for k, v in d.items()}
|
||||
return d
|
||||
|
||||
|
||||
@st.cache_data
|
||||
def load_data(log_path: Path):
|
||||
data = defaultdict(lambda: defaultdict(dict))
|
||||
for msg in FileStorage(log_path).iter_msg():
|
||||
if msg.tag and "llm" not in msg.tag and "session" not in msg.tag:
|
||||
if msg.tag == "competition":
|
||||
data["competition"] = msg.content
|
||||
continue
|
||||
|
||||
li, fn = extract_loopid_func_name(msg.tag)
|
||||
if li:
|
||||
li = int(li)
|
||||
|
||||
ei = extract_evoid(msg.tag)
|
||||
msg.tag = re.sub(r"\.evo_loop_\d+", "", msg.tag)
|
||||
msg.tag = re.sub(r"Loop_\d+\.[^.]+\.?", "", msg.tag)
|
||||
msg.tag = msg.tag.strip()
|
||||
|
||||
if ei:
|
||||
if int(ei) not in data[li][fn]:
|
||||
data[li][fn][int(ei)] = {}
|
||||
data[li][fn][int(ei)][msg.tag] = msg.content
|
||||
else:
|
||||
if msg.tag:
|
||||
data[li][fn][msg.tag] = msg.content
|
||||
else:
|
||||
if not isinstance(msg.content, str):
|
||||
data[li][fn]["no_tag"] = msg.content
|
||||
|
||||
# debug_llm data
|
||||
llm_data = defaultdict(lambda: defaultdict(lambda: defaultdict(list)))
|
||||
llm_log_p = log_path / "debug_llm.pkl"
|
||||
with st.spinner("正在加载 debug_llm.pkl..."):
|
||||
rd = pickle.loads(llm_log_p.read_bytes())
|
||||
for i, d in enumerate(rd):
|
||||
t = d["tag"]
|
||||
if "debug_exp_gen" in t:
|
||||
continue
|
||||
if "debug_tpl" in t and "filter_" in d["obj"]["uri"]:
|
||||
continue
|
||||
lid, fn = extract_loopid_func_name(t)
|
||||
ei = extract_evoid(t)
|
||||
if lid:
|
||||
lid = int(lid)
|
||||
if ei:
|
||||
ei = int(ei)
|
||||
|
||||
if ei is not None:
|
||||
llm_data[lid][fn][ei].append(d)
|
||||
else:
|
||||
llm_data[lid][fn]["no_tag"].append(d)
|
||||
|
||||
return convert_defaultdict_to_dict(data), convert_defaultdict_to_dict(llm_data)
|
||||
|
||||
|
||||
@st.cache_data
|
||||
def load_stdout(stdout_path: Path):
|
||||
if stdout_path.exists():
|
||||
stdout = stdout_path.read_text()
|
||||
else:
|
||||
stdout = f"Please Set: {stdout_path}"
|
||||
return stdout
|
||||
|
||||
|
||||
# UI windows
|
||||
def task_win(data):
|
||||
with st.container(border=True):
|
||||
st.markdown(f"**:violet[{data.name}]**")
|
||||
st.markdown(data.description)
|
||||
if hasattr(data, "architecture"): # model task
|
||||
st.markdown(
|
||||
f"""
|
||||
| Model_type | Architecture | hyperparameters |
|
||||
|------------|--------------|-----------------|
|
||||
| {data.model_type} | {data.architecture} | {data.hyperparameters} |
|
||||
"""
|
||||
)
|
||||
|
||||
|
||||
def workspace_win(data, instance_id=None):
|
||||
show_files = {k: v for k, v in data.file_dict.items() if "test" not in k}
|
||||
|
||||
base_key = str(data.workspace_path)
|
||||
if instance_id is not None:
|
||||
base_key += f"_{instance_id}"
|
||||
unique_key = hashlib.md5(base_key.encode()).hexdigest()
|
||||
|
||||
if len(show_files) > 0:
|
||||
with st.expander(f"Files in :blue[{replace_ep_path(data.workspace_path)}]"):
|
||||
code_tabs = st.tabs(show_files.keys())
|
||||
for ct, codename in zip(code_tabs, show_files.keys()):
|
||||
with ct:
|
||||
st.code(
|
||||
show_files[codename],
|
||||
language=("python" if codename.endswith(".py") else "markdown"),
|
||||
wrap_lines=True,
|
||||
line_numbers=True,
|
||||
)
|
||||
|
||||
st.markdown("### Save All Files to Folder")
|
||||
target_folder = st.text_input("Enter target folder path:", key=f"save_folder_path_input_{unique_key}")
|
||||
|
||||
if st.button("Save Files", key=f"save_files_button_{unique_key}"):
|
||||
if target_folder.strip() == "":
|
||||
st.warning("Please enter a valid folder path.")
|
||||
else:
|
||||
target_folder_path = Path(target_folder)
|
||||
target_folder_path.mkdir(parents=True, exist_ok=True)
|
||||
for filename, content in data.file_dict.items():
|
||||
save_path = target_folder_path / filename
|
||||
save_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
save_path.write_text(content, encoding="utf-8")
|
||||
st.success(f"All files saved to: {target_folder}")
|
||||
else:
|
||||
st.markdown(f"No files in :blue[{replace_ep_path(data.workspace_path)}]")
|
||||
|
||||
|
||||
# Helper functions
|
||||
def show_text(text, lang=None):
|
||||
"""显示文本代码块"""
|
||||
if lang:
|
||||
st.code(text, language=lang, wrap_lines=True)
|
||||
elif "\n" in text:
|
||||
st.code(text, language="python", wrap_lines=True)
|
||||
else:
|
||||
st.code(text, language="html", wrap_lines=True)
|
||||
|
||||
|
||||
def highlight_prompts_uri(uri):
|
||||
"""高亮 URI 的格式"""
|
||||
parts = uri.split(":")
|
||||
return f"**{parts[0]}:**:green[**{parts[1]}**]"
|
||||
|
||||
|
||||
def llm_log_win(llm_d: list):
|
||||
for d in llm_d:
|
||||
if "debug_tpl" in d["tag"]:
|
||||
uri = d["obj"]["uri"]
|
||||
tpl = d["obj"]["template"]
|
||||
cxt = d["obj"]["context"]
|
||||
rd = d["obj"]["rendered"]
|
||||
with st.expander(highlight_prompts_uri(uri), expanded=False, icon="⚙️"):
|
||||
t1, t2, t3 = st.tabs([":green[**Rendered**]", ":blue[**Template**]", ":orange[**Context**]"])
|
||||
with t1:
|
||||
show_text(rd)
|
||||
with t2:
|
||||
show_text(tpl, lang="django")
|
||||
with t3:
|
||||
st.json(cxt)
|
||||
elif "debug_llm" in d["tag"]:
|
||||
system = d["obj"].get("system", None)
|
||||
user = d["obj"]["user"]
|
||||
resp = d["obj"]["resp"]
|
||||
with st.expander(f"**LLM**", expanded=False, icon="🤖"):
|
||||
t1, t2, t3 = st.tabs([":green[**Response**]", ":blue[**User**]", ":orange[**System**]"])
|
||||
with t1:
|
||||
try:
|
||||
rdict = json.loads(resp)
|
||||
if "code" in rdict:
|
||||
code = rdict["code"]
|
||||
st.markdown(":red[**Code in response dict:**]")
|
||||
st.code(code, language="python", wrap_lines=True, line_numbers=True)
|
||||
rdict.pop("code")
|
||||
elif "spec" in rdict:
|
||||
spec = rdict["spec"]
|
||||
st.markdown(":red[**Spec in response dict:**]")
|
||||
st.markdown(spec)
|
||||
rdict.pop("spec")
|
||||
else:
|
||||
# show model codes
|
||||
showed_keys = []
|
||||
for k, v in rdict.items():
|
||||
if k.startswith("model_") and k.endswith(".py"):
|
||||
st.markdown(f":red[**{k}**]")
|
||||
st.code(v, language="python", wrap_lines=True, line_numbers=True)
|
||||
showed_keys.append(k)
|
||||
for k in showed_keys:
|
||||
rdict.pop(k)
|
||||
st.write(":red[**Other parts (except for the code or spec) in response dict:**]")
|
||||
st.json(rdict)
|
||||
except:
|
||||
st.json(resp)
|
||||
with t2:
|
||||
show_text(user)
|
||||
with t3:
|
||||
show_text(system or "No system prompt available")
|
||||
|
||||
|
||||
def hypothesis_win(data):
|
||||
st.code(str(data).replace("\n", "\n\n"), wrap_lines=True)
|
||||
|
||||
|
||||
def exp_gen_win(data, llm_data=None):
|
||||
st.header("Exp Gen", divider="blue", anchor="exp-gen")
|
||||
if state.show_llm_log:
|
||||
llm_log_win(llm_data["no_tag"])
|
||||
st.subheader("Hypothesis")
|
||||
hypothesis_win(data["no_tag"].hypothesis)
|
||||
|
||||
st.subheader("pending_tasks")
|
||||
for tasks in data["no_tag"].pending_tasks_list:
|
||||
task_win(tasks[0])
|
||||
st.subheader("Exp Workspace")
|
||||
workspace_win(data["no_tag"].experiment_workspace)
|
||||
|
||||
|
||||
def evolving_win(data, key, llm_data=None):
|
||||
with st.container(border=True):
|
||||
if len(data) > 1:
|
||||
evo_id = st.slider("Evolving", 0, len(data) - 1, 0, key=key)
|
||||
elif len(data) == 1:
|
||||
evo_id = 0
|
||||
else:
|
||||
st.markdown("No evolving.")
|
||||
return
|
||||
|
||||
if evo_id in data:
|
||||
if state.show_llm_log:
|
||||
llm_log_win(llm_data[evo_id])
|
||||
if data[evo_id]["evolving code"][0] is not None:
|
||||
st.subheader("codes")
|
||||
workspace_win(data[evo_id]["evolving code"][0], instance_id=key)
|
||||
fb = data[evo_id]["evolving feedback"][0]
|
||||
st.subheader("evolving feedback" + ("✅" if bool(fb) else "❌"))
|
||||
f1, f2, f3 = st.tabs(["execution", "return_checking", "code"])
|
||||
f1.code(fb.execution, wrap_lines=True)
|
||||
f2.code(fb.return_checking, wrap_lines=True)
|
||||
f3.code(fb.code, wrap_lines=True)
|
||||
else:
|
||||
st.write("data[evo_id]['evolving code'][0] is None.")
|
||||
st.write(data[evo_id])
|
||||
else:
|
||||
st.markdown("No evolving.")
|
||||
|
||||
|
||||
def coding_win(data, llm_data: dict | None = None):
|
||||
st.header("Coding", divider="blue", anchor="coding")
|
||||
if llm_data is not None:
|
||||
common_llm_data = llm_data.pop("no_tag", [])
|
||||
evolving_data = {k: v for k, v in data.items() if isinstance(k, int)}
|
||||
task_set = set()
|
||||
for v in evolving_data.values():
|
||||
for t in v:
|
||||
if "Task" in t.split(".")[0]:
|
||||
task_set.add(t.split(".")[0])
|
||||
if task_set:
|
||||
# 新版存Task tag的Trace
|
||||
for task in task_set:
|
||||
st.subheader(task)
|
||||
task_data = {k: {a.split(".")[1]: b for a, b in v.items() if task in a} for k, v in evolving_data.items()}
|
||||
evolving_win(task_data, key=task, llm_data=llm_data if llm_data else None)
|
||||
else:
|
||||
# 旧版未存Task tag的Trace
|
||||
evolving_win(evolving_data, key="coding", llm_data=llm_data if llm_data else None)
|
||||
if state.show_llm_log:
|
||||
llm_log_win(common_llm_data)
|
||||
if "no_tag" in data:
|
||||
st.subheader("Exp Workspace (coding final)")
|
||||
workspace_win(data["no_tag"].experiment_workspace, instance_id="coding_dump")
|
||||
|
||||
|
||||
def running_win(data, mle_score, llm_data=None):
|
||||
st.header("Running", divider="blue", anchor="running")
|
||||
if llm_data is not None:
|
||||
common_llm_data = llm_data.pop("no_tag", [])
|
||||
evolving_win(
|
||||
{k: v for k, v in data.items() if isinstance(k, int)}, key="running", llm_data=llm_data if llm_data else None
|
||||
)
|
||||
if state.show_llm_log:
|
||||
llm_log_win(common_llm_data)
|
||||
if "no_tag" in data:
|
||||
st.subheader("Exp Workspace (running final)")
|
||||
workspace_win(data["no_tag"].experiment_workspace, instance_id="running_dump")
|
||||
st.subheader("Result")
|
||||
st.write(data["no_tag"].result)
|
||||
st.subheader("MLE Submission Score" + ("✅" if (isinstance(mle_score, dict) and mle_score["score"]) else "❌"))
|
||||
if isinstance(mle_score, dict):
|
||||
st.json(mle_score)
|
||||
else:
|
||||
st.code(mle_score, wrap_lines=True)
|
||||
|
||||
|
||||
def feedback_win(data, llm_data=None):
|
||||
data = data["no_tag"]
|
||||
st.header("Feedback" + ("✅" if bool(data) else "❌"), divider="orange", anchor="feedback")
|
||||
if state.show_llm_log and llm_data is not None:
|
||||
llm_log_win(llm_data["no_tag"])
|
||||
st.code(str(data).replace("\n", "\n\n"), wrap_lines=True)
|
||||
if data.exception is not None:
|
||||
st.markdown(f"**:red[Exception]**: {data.exception}")
|
||||
|
||||
|
||||
def sota_win(data):
|
||||
st.header("SOTA Experiment", divider="rainbow", anchor="sota-exp")
|
||||
if data:
|
||||
st.markdown(f"**SOTA Exp Hypothesis**")
|
||||
hypothesis_win(data.hypothesis)
|
||||
st.markdown("**Exp Workspace**")
|
||||
workspace_win(data.experiment_workspace, instance_id="sota")
|
||||
else:
|
||||
st.markdown("No SOTA experiment.")
|
||||
|
||||
|
||||
def main_win(data, llm_data=None):
|
||||
exp_gen_win(data["direct_exp_gen"], llm_data["direct_exp_gen"] if llm_data else None)
|
||||
if "coding" in data:
|
||||
coding_win(data["coding"], llm_data["coding"] if llm_data else None)
|
||||
if "running" in data:
|
||||
running_win(
|
||||
data["running"],
|
||||
data.get("mle_score", "no submission to score"),
|
||||
llm_data=llm_data["running"] if llm_data else None,
|
||||
)
|
||||
if "feedback" in data:
|
||||
feedback_win(data["feedback"], llm_data.get("feedback", None) if llm_data else None)
|
||||
if "record" in data and "SOTA experiment" in data["record"]:
|
||||
sota_win(data["record"]["SOTA experiment"])
|
||||
|
||||
|
||||
def replace_ep_path(p: Path):
|
||||
# 替换workspace path为对应ep机器mount在ep03的path
|
||||
# TODO: FIXME: 使用配置项来处理
|
||||
match = re.search(r"ep\d+", str(state.log_folder))
|
||||
if match:
|
||||
ep = match.group(0)
|
||||
return Path(
|
||||
str(p).replace("repos/RD-Agent-Exp", f"repos/batch_ctrl/all_projects/{ep}").replace("/Data", "/data")
|
||||
)
|
||||
return p
|
||||
|
||||
|
||||
def summarize_data():
|
||||
st.header("Summary", divider="rainbow")
|
||||
with st.container(border=True):
|
||||
df = pd.DataFrame(
|
||||
columns=[
|
||||
"Component",
|
||||
"Running Score",
|
||||
"Feedback",
|
||||
"e-loops",
|
||||
"Time",
|
||||
"Coding",
|
||||
"Running",
|
||||
"Start Time (UTC+8)",
|
||||
"End Time (UTC+8)",
|
||||
],
|
||||
index=range(len(state.data) - 1),
|
||||
)
|
||||
|
||||
for loop in range(len(state.data) - 1):
|
||||
loop_data = state.data[loop]
|
||||
df.loc[loop, "Component"] = loop_data["direct_exp_gen"]["no_tag"].hypothesis.component
|
||||
if state.times[loop]:
|
||||
df.loc[loop, "Time"] = str(sum((i.end - i.start for i in state.times[loop]), timedelta())).split(".")[0]
|
||||
coding_time = state.times[loop][1].end - state.times[loop][1].start
|
||||
df.loc[loop, "Coding"] = str(coding_time).split(".")[0]
|
||||
if len(state.times[loop]) > 2:
|
||||
running_time = state.times[loop][2].end - state.times[loop][2].start
|
||||
df.loc[loop, "Running"] = str(running_time).split(".")[0]
|
||||
df.loc[loop, "Start Time (UTC+8)"] = state.times[loop][0].start + timedelta(hours=8)
|
||||
df.loc[loop, "End Time (UTC+8)"] = state.times[loop][-1].end + timedelta(hours=8)
|
||||
if "running" in loop_data and "no_tag" in loop_data["running"]:
|
||||
if "mle_score" not in state.data[loop]:
|
||||
if "mle_score" in loop_data["running"]:
|
||||
mle_score_txt = loop_data["running"]["mle_score"]
|
||||
state.data[loop]["mle_score"] = extract_mle_json(mle_score_txt)
|
||||
if state.data[loop]["mle_score"]["score"] is not None:
|
||||
df.loc[loop, "Running Score"] = str(state.data[loop]["mle_score"]["score"])
|
||||
else:
|
||||
state.data[loop]["mle_score"] = mle_score_txt
|
||||
df.loc[loop, "Running Score"] = "❌"
|
||||
else:
|
||||
mle_score_path = (
|
||||
replace_ep_path(loop_data["running"]["no_tag"].experiment_workspace.workspace_path)
|
||||
/ "mle_score.txt"
|
||||
)
|
||||
try:
|
||||
mle_score_txt = mle_score_path.read_text()
|
||||
state.data[loop]["mle_score"] = extract_mle_json(mle_score_txt)
|
||||
if state.data[loop]["mle_score"]["score"] is not None:
|
||||
df.loc[loop, "Running Score"] = str(state.data[loop]["mle_score"]["score"])
|
||||
else:
|
||||
state.data[loop]["mle_score"] = mle_score_txt
|
||||
df.loc[loop, "Running Score"] = "❌"
|
||||
except Exception as e:
|
||||
state.data[loop]["mle_score"] = str(e)
|
||||
df.loc[loop, "Running Score"] = "❌"
|
||||
else:
|
||||
if isinstance(state.data[loop]["mle_score"], dict):
|
||||
df.loc[loop, "Running Score"] = str(state.data[loop]["mle_score"]["score"])
|
||||
else:
|
||||
df.loc[loop, "Running Score"] = "❌"
|
||||
|
||||
else:
|
||||
df.loc[loop, "Running Score"] = "N/A"
|
||||
|
||||
if "coding" in loop_data:
|
||||
df.loc[loop, "e-loops"] = max(i for i in loop_data["coding"].keys() if isinstance(i, int)) + 1
|
||||
if "feedback" in loop_data:
|
||||
df.loc[loop, "Feedback"] = "✅" if bool(loop_data["feedback"]["no_tag"]) else "❌"
|
||||
else:
|
||||
df.loc[loop, "Feedback"] = "N/A"
|
||||
st.dataframe(df)
|
||||
|
||||
def comp_stat_func(x: pd.DataFrame):
|
||||
total_num = x.shape[0]
|
||||
valid_num = x[x["Running Score"] != "N/A"].shape[0]
|
||||
avg_e_loops = x["e-loops"].mean()
|
||||
return pd.Series(
|
||||
{
|
||||
"Loop Num": total_num,
|
||||
"Valid Loop": valid_num,
|
||||
"Valid Rate": round(valid_num / total_num * 100, 2),
|
||||
"Avg e-loops": round(avg_e_loops, 2),
|
||||
}
|
||||
)
|
||||
|
||||
st1, st2 = st.columns([1, 1])
|
||||
|
||||
# component statistics
|
||||
comp_df = df.loc[:, ["Component", "Running Score", "e-loops"]].groupby("Component").apply(comp_stat_func)
|
||||
comp_df.loc["Total"] = comp_df.sum()
|
||||
comp_df.loc["Total", "Valid Rate"] = round(
|
||||
comp_df.loc["Total", "Valid Loop"] / comp_df.loc["Total", "Loop Num"] * 100, 2
|
||||
)
|
||||
comp_df["Valid Rate"] = comp_df["Valid Rate"].apply(lambda x: f"{x}%")
|
||||
comp_df.loc["Total", "Avg e-loops"] = round(df["e-loops"].mean(), 2)
|
||||
st2.markdown("### Component Statistics")
|
||||
st2.dataframe(comp_df)
|
||||
|
||||
# component time statistics
|
||||
time_df = df.loc[:, ["Component", "Time", "Coding", "Running"]]
|
||||
time_df = time_df.astype({"Time": "timedelta64[ns]", "Coding": "timedelta64[ns]", "Running": "timedelta64[ns]"})
|
||||
st1.markdown("### Time Statistics")
|
||||
time_stat_df = time_df.groupby("Component").sum()
|
||||
time_stat_df.loc["Total"] = time_stat_df.sum()
|
||||
time_stat_df.loc[:, "Coding(%)"] = time_stat_df["Coding"] / time_stat_df["Time"] * 100
|
||||
time_stat_df.loc[:, "Running(%)"] = time_stat_df["Running"] / time_stat_df["Time"] * 100
|
||||
time_stat_df = time_stat_df.map(lambda x: str(x).split(".")[0] if pd.notnull(x) else "0:00:00")
|
||||
st1.dataframe(time_stat_df)
|
||||
|
||||
|
||||
def stdout_win(loop_id: int):
|
||||
stdout = load_stdout(state.log_folder / f"{state.log_path}.stdout")
|
||||
if stdout.startswith("Please Set"):
|
||||
st.toast(stdout, icon="🟡")
|
||||
return
|
||||
start_index = stdout.find(f"Start Loop {loop_id}")
|
||||
end_index = stdout.find(f"Start Loop {loop_id + 1}")
|
||||
loop_stdout = remove_ansi_codes(stdout[start_index:end_index])
|
||||
with st.container(border=True):
|
||||
st.subheader(f"Loop {loop_id} stdout")
|
||||
pattern = f"Start Loop {loop_id}, " + r"Step \d+: \w+"
|
||||
matches = re.finditer(pattern, loop_stdout)
|
||||
step_stdouts = {}
|
||||
for match in matches:
|
||||
step = match.group(0)
|
||||
si = match.start()
|
||||
ei = loop_stdout.find(f"Start Loop {loop_id}", match.end())
|
||||
step_stdouts[step] = loop_stdout[si:ei].strip()
|
||||
|
||||
for k, v in step_stdouts.items():
|
||||
with st.expander(k, expanded=False):
|
||||
st.code(v, language="log", wrap_lines=True)
|
||||
|
||||
|
||||
def get_folders_sorted(log_path):
|
||||
"""缓存并返回排序后的文件夹列表，并加入进度打印"""
|
||||
if not log_path.exists():
|
||||
st.toast(f"Path {log_path} does not exist!")
|
||||
return []
|
||||
with st.spinner("正在加载文件夹列表..."):
|
||||
folders = sorted(
|
||||
(folder for folder in log_path.iterdir() if is_valid_session(folder)),
|
||||
key=lambda folder: folder.stat().st_mtime,
|
||||
reverse=True,
|
||||
)
|
||||
st.write(f"找到 {len(folders)} 个文件夹")
|
||||
return [folder.name for folder in folders]
|
||||
|
||||
|
||||
# UI - Sidebar
|
||||
with st.sidebar:
|
||||
# TODO: 只是临时的功能
|
||||
if any("log.srv" in folder for folder in state.log_folders):
|
||||
day_map = {"srv": "最近(srv)", "srv2": "上一批(srv2)", "srv3": "上上批(srv3)"}
|
||||
day_srv = st.radio("选择批次", ["srv", "srv2", "srv3"], format_func=lambda x: day_map[x], horizontal=True)
|
||||
if day_srv == "srv":
|
||||
state.log_folders = [re.sub(r"log\.srv\d*", "log.srv", folder) for folder in state.log_folders]
|
||||
elif day_srv == "srv2":
|
||||
state.log_folders = [re.sub(r"log\.srv\d*", "log.srv2", folder) for folder in state.log_folders]
|
||||
elif day_srv == "srv3":
|
||||
state.log_folders = [re.sub(r"log\.srv\d*", "log.srv3", folder) for folder in state.log_folders]
|
||||
|
||||
state.log_folder = Path(st.radio(f"Select :blue[**one log folder**]", state.log_folders))
|
||||
if not state.log_folder.exists():
|
||||
st.warning(f"Path {state.log_folder} does not exist!")
|
||||
else:
|
||||
folders = get_folders_sorted(state.log_folder)
|
||||
st.selectbox(f"Select from :blue[**{state.log_folder.absolute()}**]", folders, key="log_path")
|
||||
|
||||
if st.button("Refresh Data"):
|
||||
if state.log_path is None:
|
||||
st.toast("Please select a log path first!", icon="🟡")
|
||||
st.stop()
|
||||
|
||||
load_times(state.log_folder / state.log_path)
|
||||
state.data, state.llm_data = load_data(state.log_folder / state.log_path)
|
||||
st.rerun()
|
||||
st.toggle("Show LLM Log", key="show_llm_log")
|
||||
st.toggle("Show stdout", key="show_stdout")
|
||||
st.markdown(
|
||||
f"""
|
||||
- [Exp Gen](#exp-gen)
|
||||
- [Coding](#coding)
|
||||
- [Running](#running)
|
||||
- [Feedback](#feedback)
|
||||
- [SOTA Experiment](#sota-exp)
|
||||
"""
|
||||
)
|
||||
|
||||
# UI - Main
|
||||
if state.data["competition"]:
|
||||
st.title(state.data["competition"])
|
||||
summarize_data()
|
||||
if len(state.data) > 2:
|
||||
loop_id = st.slider("Loop", 0, len(state.data) - 2, 0)
|
||||
else:
|
||||
loop_id = 0
|
||||
if state.show_stdout:
|
||||
stdout_win(loop_id)
|
||||
main_win(state.data[loop_id], state.llm_data[loop_id])
|
||||
+14
-732
@@ -1,751 +1,33 @@
|
||||
import hashlib
|
||||
import math
|
||||
import re
|
||||
from collections import defaultdict
|
||||
from datetime import timedelta
|
||||
from pathlib import Path
|
||||
|
||||
import pandas as pd
|
||||
import plotly.express as px
|
||||
import plotly.graph_objects as go
|
||||
import streamlit as st
|
||||
from streamlit import session_state as state
|
||||
|
||||
from rdagent.app.data_science.loop import DataScienceRDLoop
|
||||
from rdagent.log.mle_summary import extract_mle_json, is_valid_session
|
||||
from rdagent.log.storage import FileStorage
|
||||
from rdagent.log.ui.conf import UI_SETTING
|
||||
from rdagent.utils import remove_ansi_codes
|
||||
|
||||
st.set_page_config(layout="wide", page_title="RD-Agent", page_icon="🎓", initial_sidebar_state="expanded")
|
||||
|
||||
# 设置主日志路径
|
||||
if "log_folder" not in state:
|
||||
state.log_folder = Path("./log")
|
||||
if "log_folders" not in state:
|
||||
state.log_folders = UI_SETTING.default_log_folders
|
||||
if "log_path" not in state:
|
||||
state.log_path = None
|
||||
if "show_all_summary" not in state:
|
||||
state.show_all_summary = True
|
||||
if "show_stdout" not in state:
|
||||
state.show_stdout = False
|
||||
|
||||
|
||||
def load_stdout():
|
||||
# FIXME: TODO: 使用配置项来指定stdout文件名
|
||||
stdout_path = state.log_folder / f"{state.log_path}.stdout"
|
||||
if stdout_path.exists():
|
||||
stdout = stdout_path.read_text()
|
||||
else:
|
||||
stdout = f"Please Set: {stdout_path}"
|
||||
return stdout
|
||||
|
||||
|
||||
def extract_loopid_func_name(tag):
|
||||
"""提取 Loop ID 和函数名称"""
|
||||
match = re.search(r"Loop_(\d+)\.([^.]+)", tag)
|
||||
return match.groups() if match else (None, None)
|
||||
|
||||
|
||||
def extract_evoid(tag):
|
||||
"""提取 EVO ID"""
|
||||
match = re.search(r"\.evo_loop_(\d+)\.", tag)
|
||||
return match.group(1) if match else None
|
||||
|
||||
|
||||
# @st.cache_data
|
||||
def load_data(log_path: Path):
|
||||
state.data = defaultdict(lambda: defaultdict(dict))
|
||||
state.times = defaultdict(lambda: defaultdict(dict))
|
||||
for msg in FileStorage(log_path).iter_msg():
|
||||
if msg.tag and "llm" not in msg.tag and "session" not in msg.tag:
|
||||
if msg.tag == "competition":
|
||||
state.data["competition"] = msg.content
|
||||
continue
|
||||
|
||||
li, fn = extract_loopid_func_name(msg.tag)
|
||||
li = int(li)
|
||||
|
||||
# read times
|
||||
loop_obj_path = log_path / "__session__" / f"{li}" / "4_record"
|
||||
if loop_obj_path.exists():
|
||||
try:
|
||||
state.times[li] = DataScienceRDLoop.load(loop_obj_path, do_truncate=False).loop_trace[li]
|
||||
except Exception as e:
|
||||
pass
|
||||
|
||||
ei = extract_evoid(msg.tag)
|
||||
msg.tag = re.sub(r"\.evo_loop_\d+", "", msg.tag)
|
||||
msg.tag = re.sub(r"Loop_\d+\.[^.]+\.?", "", msg.tag)
|
||||
msg.tag = msg.tag.strip()
|
||||
|
||||
if ei:
|
||||
if int(ei) not in state.data[li][fn]:
|
||||
state.data[li][fn][int(ei)] = {}
|
||||
state.data[li][fn][int(ei)][msg.tag] = msg.content
|
||||
else:
|
||||
if msg.tag:
|
||||
state.data[li][fn][msg.tag] = msg.content
|
||||
else:
|
||||
if not isinstance(msg.content, str):
|
||||
state.data[li][fn]["no_tag"] = msg.content
|
||||
|
||||
|
||||
# @st.cache_data
|
||||
def get_folders_sorted(log_path):
|
||||
"""缓存并返回排序后的文件夹列表，并加入进度打印"""
|
||||
if not log_path.exists():
|
||||
st.toast(f"Path {log_path} does not exist!")
|
||||
return []
|
||||
with st.spinner("正在加载文件夹列表..."):
|
||||
folders = sorted(
|
||||
(folder for folder in log_path.iterdir() if is_valid_session(folder)),
|
||||
key=lambda folder: folder.stat().st_mtime,
|
||||
reverse=True,
|
||||
)
|
||||
st.write(f"找到 {len(folders)} 个文件夹")
|
||||
return [folder.name for folder in folders]
|
||||
|
||||
summary_page = st.Page("ds_summary.py", title="Summary", icon="📊")
|
||||
trace_page = st.Page("ds_trace.py", title="Trace", icon="📈")
|
||||
st.set_page_config(layout="wide", page_title="RD-Agent", page_icon="🎓", initial_sidebar_state="expanded")
|
||||
st.navigation([summary_page, trace_page]).run()
|
||||
|
||||
# UI - Sidebar
|
||||
with st.sidebar:
|
||||
log_folder_str = st.text_area(
|
||||
"**Log Folders**(split by ';')", placeholder=state.log_folder, value=";".join(state.log_folders)
|
||||
)
|
||||
state.log_folders = [folder.strip() for folder in log_folder_str.split(";") if folder.strip()]
|
||||
st.subheader("Pages", divider="rainbow")
|
||||
st.page_link(summary_page, icon="📊")
|
||||
st.page_link(trace_page, icon="📈")
|
||||
|
||||
# # TODO: 只是临时的功能
|
||||
day_map = {"srv": "最近(srv)", "srv2": "上一批(srv2)", "srv3": "上上批(srv3)"}
|
||||
day_srv = st.radio("选择批次", ["srv", "srv2", "srv3"], format_func=lambda x: day_map[x], horizontal=True)
|
||||
if day_srv == "srv":
|
||||
state.log_folders = [re.sub(r"log\.srv\d*", "log.srv", folder) for folder in state.log_folders]
|
||||
elif day_srv == "srv2":
|
||||
state.log_folders = [re.sub(r"log\.srv\d*", "log.srv2", folder) for folder in state.log_folders]
|
||||
elif day_srv == "srv3":
|
||||
state.log_folders = [re.sub(r"log\.srv\d*", "log.srv3", folder) for folder in state.log_folders]
|
||||
|
||||
state.log_folder = Path(st.radio(f"Select :blue[**one log folder**]", state.log_folders))
|
||||
if not state.log_folder.exists():
|
||||
st.warning(f"Path {state.log_folder} does not exist!")
|
||||
else:
|
||||
folders = get_folders_sorted(state.log_folder)
|
||||
st.selectbox(f"Select from :blue[**{state.log_folder.absolute()}**]", folders, key="log_path")
|
||||
|
||||
if st.button("Refresh Data"):
|
||||
if state.log_path is None:
|
||||
st.toast("Please select a log path first!", icon="🟡")
|
||||
st.stop()
|
||||
|
||||
load_data(state.log_folder / state.log_path)
|
||||
|
||||
st.toggle("One Trace / Log Folder Summary", key="show_all_summary")
|
||||
st.toggle("Show stdout", key="show_stdout")
|
||||
|
||||
|
||||
# UI windows
|
||||
def task_win(data):
|
||||
with st.container(border=True):
|
||||
st.markdown(f"**:violet[{data.name}]**")
|
||||
st.markdown(data.description)
|
||||
if hasattr(data, "architecture"): # model task
|
||||
st.markdown(
|
||||
f"""
|
||||
| Model_type | Architecture | hyperparameters |
|
||||
|------------|--------------|-----------------|
|
||||
| {data.model_type} | {data.architecture} | {data.hyperparameters} |
|
||||
"""
|
||||
)
|
||||
|
||||
|
||||
def workspace_win(data, instance_id=None):
|
||||
show_files = {k: v for k, v in data.file_dict.items() if "test" not in k}
|
||||
|
||||
base_key = str(data.workspace_path)
|
||||
if instance_id is not None:
|
||||
base_key += f"_{instance_id}"
|
||||
unique_key = hashlib.md5(base_key.encode()).hexdigest()
|
||||
|
||||
if len(show_files) > 0:
|
||||
with st.expander(f"Files in :blue[{replace_ep_path(data.workspace_path)}]"):
|
||||
code_tabs = st.tabs(show_files.keys())
|
||||
for ct, codename in zip(code_tabs, show_files.keys()):
|
||||
with ct:
|
||||
st.code(
|
||||
show_files[codename],
|
||||
language=("python" if codename.endswith(".py") else "markdown"),
|
||||
wrap_lines=True,
|
||||
line_numbers=True,
|
||||
)
|
||||
|
||||
st.markdown("### Save All Files to Folder")
|
||||
target_folder = st.text_input("Enter target folder path:", key=f"save_folder_path_input_{unique_key}")
|
||||
|
||||
if st.button("Save Files", key=f"save_files_button_{unique_key}"):
|
||||
if target_folder.strip() == "":
|
||||
st.warning("Please enter a valid folder path.")
|
||||
else:
|
||||
target_folder_path = Path(target_folder)
|
||||
target_folder_path.mkdir(parents=True, exist_ok=True)
|
||||
for filename, content in data.file_dict.items():
|
||||
save_path = target_folder_path / filename
|
||||
save_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
save_path.write_text(content, encoding="utf-8")
|
||||
st.success(f"All files saved to: {target_folder}")
|
||||
else:
|
||||
st.markdown(f"No files in :blue[{replace_ep_path(data.workspace_path)}]")
|
||||
|
||||
|
||||
def hypothesis_win(data):
|
||||
st.code(str(data).replace("\n", "\n\n"), wrap_lines=True)
|
||||
|
||||
|
||||
def exp_gen_win(data):
|
||||
st.header("Exp Gen", divider="blue", anchor="exp-gen")
|
||||
st.subheader("Hypothesis")
|
||||
hypothesis_win(data["no_tag"].hypothesis)
|
||||
|
||||
st.subheader("pending_tasks")
|
||||
for tasks in data["no_tag"].pending_tasks_list:
|
||||
task_win(tasks[0])
|
||||
st.subheader("Exp Workspace")
|
||||
workspace_win(data["no_tag"].experiment_workspace, instance_id="exp_gen")
|
||||
|
||||
|
||||
def evolving_win(data, key):
|
||||
with st.container(border=True):
|
||||
if len(data) > 1:
|
||||
evo_id = st.slider("Evolving", 0, len(data) - 1, 0, key=key)
|
||||
elif len(data) == 1:
|
||||
evo_id = 0
|
||||
else:
|
||||
st.markdown("No evolving.")
|
||||
return
|
||||
|
||||
if evo_id in data:
|
||||
if data[evo_id]["evolving code"][0] is not None:
|
||||
st.subheader("codes")
|
||||
workspace_win(data[evo_id]["evolving code"][0], instance_id=key)
|
||||
fb = data[evo_id]["evolving feedback"][0]
|
||||
st.subheader("evolving feedback" + ("✅" if bool(fb) else "❌"))
|
||||
f1, f2, f3 = st.tabs(["execution", "return_checking", "code"])
|
||||
f1.code(fb.execution, wrap_lines=True)
|
||||
f2.code(fb.return_checking, wrap_lines=True)
|
||||
f3.code(fb.code, wrap_lines=True)
|
||||
else:
|
||||
st.write("data[evo_id]['evolving code'][0] is None.")
|
||||
st.write(data[evo_id])
|
||||
else:
|
||||
st.markdown("No evolving.")
|
||||
|
||||
|
||||
def coding_win(data):
|
||||
st.header("Coding", divider="blue", anchor="coding")
|
||||
evolving_data = {k: v for k, v in data.items() if isinstance(k, int)}
|
||||
task_set = set()
|
||||
for v in evolving_data.values():
|
||||
for t in v:
|
||||
if "Task" in t.split(".")[0]:
|
||||
task_set.add(t.split(".")[0])
|
||||
if task_set:
|
||||
# 新版存Task tag的Trace
|
||||
for task in task_set:
|
||||
st.subheader(task)
|
||||
task_data = {k: {a.split(".")[1]: b for a, b in v.items() if task in a} for k, v in evolving_data.items()}
|
||||
evolving_win(task_data, key=task)
|
||||
else:
|
||||
# 旧版未存Task tag的Trace
|
||||
evolving_win(evolving_data, key="coding")
|
||||
if "no_tag" in data:
|
||||
st.subheader("Exp Workspace (coding final)")
|
||||
workspace_win(data["no_tag"].experiment_workspace, instance_id="coding")
|
||||
|
||||
|
||||
def running_win(data, mle_score):
|
||||
st.header("Running", divider="blue", anchor="running")
|
||||
evolving_win({k: v for k, v in data.items() if isinstance(k, int)}, key="running")
|
||||
if "no_tag" in data:
|
||||
st.subheader("Exp Workspace (running final)")
|
||||
workspace_win(data["no_tag"].experiment_workspace, instance_id="running_dump")
|
||||
st.subheader("Result")
|
||||
st.write(data["no_tag"].result)
|
||||
st.subheader("MLE Submission Score" + ("✅" if (isinstance(mle_score, dict) and mle_score["score"]) else "❌"))
|
||||
if isinstance(mle_score, dict):
|
||||
st.json(mle_score)
|
||||
else:
|
||||
st.code(mle_score, wrap_lines=True)
|
||||
|
||||
|
||||
def feedback_win(data):
|
||||
data = data["no_tag"]
|
||||
st.header("Feedback" + ("✅" if bool(data) else "❌"), divider="orange", anchor="feedback")
|
||||
st.code(str(data).replace("\n", "\n\n"), wrap_lines=True)
|
||||
if data.exception is not None:
|
||||
st.markdown(f"**:red[Exception]**: {data.exception}")
|
||||
|
||||
|
||||
def sota_win(data):
|
||||
st.header("SOTA Experiment", divider="rainbow", anchor="sota-exp")
|
||||
if data:
|
||||
st.markdown(f"**SOTA Exp Hypothesis**")
|
||||
hypothesis_win(data.hypothesis)
|
||||
st.markdown("**Exp Workspace**")
|
||||
workspace_win(data.experiment_workspace, instance_id="sota")
|
||||
else:
|
||||
st.markdown("No SOTA experiment.")
|
||||
|
||||
|
||||
def main_win(data):
|
||||
exp_gen_win(data["direct_exp_gen"])
|
||||
if "coding" in data:
|
||||
coding_win(data["coding"])
|
||||
if "running" in data:
|
||||
running_win(data["running"], data["mle_score"])
|
||||
if "feedback" in data:
|
||||
feedback_win(data["feedback"])
|
||||
if "record" in data and "SOTA experiment" in data["record"]:
|
||||
sota_win(data["record"]["SOTA experiment"])
|
||||
|
||||
with st.sidebar:
|
||||
st.markdown(
|
||||
f"""
|
||||
- [Exp Gen](#exp-gen)
|
||||
- [Coding](#coding)
|
||||
- [Running](#running)
|
||||
- [Feedback](#feedback)
|
||||
- [SOTA Experiment](#sota-exp)
|
||||
"""
|
||||
st.subheader("Settings", divider="rainbow")
|
||||
with st.form("log_folder_form", border=False):
|
||||
log_folder_str = st.text_area(
|
||||
"**Log Folders**(split by ';')", placeholder=state.log_folder, value=";".join(state.log_folders)
|
||||
)
|
||||
|
||||
|
||||
def replace_ep_path(p: Path):
|
||||
# 替换workspace path为对应ep机器mount在ep03的path
|
||||
# TODO: FIXME: 使用配置项来处理
|
||||
match = re.search(r"ep\d+", str(state.log_folder))
|
||||
if match:
|
||||
ep = match.group(0)
|
||||
return Path(
|
||||
str(p).replace("repos/RD-Agent-Exp", f"repos/batch_ctrl/all_projects/{ep}").replace("/Data", "/data")
|
||||
)
|
||||
return p
|
||||
|
||||
|
||||
def summarize_data():
|
||||
st.header("Summary", divider="rainbow")
|
||||
df = pd.DataFrame(
|
||||
columns=["Component", "Running Score", "Feedback", "e-loops", "Time", "Start Time (UTC+8)", "End Time (UTC+8)"],
|
||||
index=range(len(state.data) - 1),
|
||||
)
|
||||
|
||||
for loop in range(len(state.data) - 1):
|
||||
loop_data = state.data[loop]
|
||||
df.loc[loop, "Component"] = loop_data["direct_exp_gen"]["no_tag"].hypothesis.component
|
||||
if state.times[loop]:
|
||||
df.loc[loop, "Time"] = str(sum((i.end - i.start for i in state.times[loop]), timedelta())).split(".")[0]
|
||||
df.loc[loop, "Start Time (UTC+8)"] = state.times[loop][0].start + timedelta(hours=8)
|
||||
df.loc[loop, "End Time (UTC+8)"] = state.times[loop][-1].end + timedelta(hours=8)
|
||||
if "running" in loop_data and "no_tag" in loop_data["running"]:
|
||||
if "mle_score" not in state.data[loop]:
|
||||
if "mle_score" in loop_data["running"]:
|
||||
mle_score_txt = loop_data["running"]["mle_score"]
|
||||
state.data[loop]["mle_score"] = extract_mle_json(mle_score_txt)
|
||||
if state.data[loop]["mle_score"]["score"] is not None:
|
||||
df.loc[loop, "Running Score"] = str(state.data[loop]["mle_score"]["score"])
|
||||
else:
|
||||
state.data[loop]["mle_score"] = mle_score_txt
|
||||
df.loc[loop, "Running Score"] = "❌"
|
||||
else:
|
||||
mle_score_path = (
|
||||
replace_ep_path(loop_data["running"]["no_tag"].experiment_workspace.workspace_path)
|
||||
/ "mle_score.txt"
|
||||
)
|
||||
try:
|
||||
mle_score_txt = mle_score_path.read_text()
|
||||
state.data[loop]["mle_score"] = extract_mle_json(mle_score_txt)
|
||||
if state.data[loop]["mle_score"]["score"] is not None:
|
||||
df.loc[loop, "Running Score"] = str(state.data[loop]["mle_score"]["score"])
|
||||
else:
|
||||
state.data[loop]["mle_score"] = mle_score_txt
|
||||
df.loc[loop, "Running Score"] = "❌"
|
||||
except Exception as e:
|
||||
state.data[loop]["mle_score"] = str(e)
|
||||
df.loc[loop, "Running Score"] = "❌"
|
||||
else:
|
||||
if isinstance(state.data[loop]["mle_score"], dict):
|
||||
df.loc[loop, "Running Score"] = str(state.data[loop]["mle_score"]["score"])
|
||||
else:
|
||||
df.loc[loop, "Running Score"] = "❌"
|
||||
|
||||
else:
|
||||
df.loc[loop, "Running Score"] = "N/A"
|
||||
|
||||
if "coding" in loop_data:
|
||||
df.loc[loop, "e-loops"] = max(i for i in loop_data["coding"].keys() if isinstance(i, int)) + 1
|
||||
if "feedback" in loop_data:
|
||||
df.loc[loop, "Feedback"] = "✅" if bool(loop_data["feedback"]["no_tag"]) else "❌"
|
||||
else:
|
||||
df.loc[loop, "Feedback"] = "N/A"
|
||||
stat_t0, stat_t1 = st.columns(2)
|
||||
stat_t0.dataframe(df)
|
||||
|
||||
def comp_stat_func(x: pd.DataFrame):
|
||||
total_num = x.shape[0]
|
||||
valid_num = x[x["Running Score"] != "N/A"].shape[0]
|
||||
avg_e_loops = x["e-loops"].mean()
|
||||
return pd.Series(
|
||||
{
|
||||
"Total": total_num,
|
||||
"Valid": valid_num,
|
||||
"Valid Rate": round(valid_num / total_num * 100, 2),
|
||||
"Avg e-loops": round(avg_e_loops, 2),
|
||||
}
|
||||
)
|
||||
|
||||
comp_df = df.loc[:, ["Component", "Running Score", "e-loops"]].groupby("Component").apply(comp_stat_func)
|
||||
comp_df.loc["Total"] = comp_df.sum()
|
||||
comp_df.loc["Total", "Valid Rate"] = round(comp_df.loc["Total", "Valid"] / comp_df.loc["Total", "Total"] * 100, 2)
|
||||
comp_df["Valid Rate"] = comp_df["Valid Rate"].apply(lambda x: f"{x}%")
|
||||
comp_df.loc["Total", "Avg e-loops"] = round(df["e-loops"].mean(), 2)
|
||||
stat_t1.dataframe(comp_df)
|
||||
|
||||
|
||||
def get_summary_df(log_folders: list[str]) -> tuple[dict, pd.DataFrame]:
|
||||
summarys = {}
|
||||
for lf in log_folders:
|
||||
if not (Path(lf) / "summary.pkl").exists():
|
||||
st.warning(
|
||||
f"No summary file found in **{lf}**\n\nRun:`dotenv run -- python rdagent/log/mle_summary.py grade_summary --log_folder={lf}`"
|
||||
)
|
||||
else:
|
||||
summarys[lf] = pd.read_pickle(Path(lf) / "summary.pkl")
|
||||
|
||||
if len(summarys) == 0:
|
||||
return {}, pd.DataFrame()
|
||||
|
||||
summary = {}
|
||||
for lf, s in summarys.items():
|
||||
for k, v in s.items():
|
||||
stdout_p = Path(lf) / f"{k}.stdout"
|
||||
v["stdout"] = []
|
||||
if stdout_p.exists():
|
||||
# stdout = stdout_p.read_text()
|
||||
stdout = ""
|
||||
if "Retrying" in stdout:
|
||||
v["stdout"].append("LLM Retry")
|
||||
if "Traceback (most recent call last):" in stdout[-10000:]:
|
||||
v["stdout"].append("Code Error")
|
||||
v["stdout"] = ", ".join([i for i in v["stdout"] if i])
|
||||
|
||||
# 调整实验名字
|
||||
if "amlt" in lf:
|
||||
summary[f"{lf[lf.rfind('amlt')+5:].split('/')[0]} - {k}"] = v
|
||||
elif "ep" in lf:
|
||||
summary[f"{lf[lf.rfind('ep'):]} - {k}"] = v
|
||||
else:
|
||||
summary[f"{lf} - {k}"] = v
|
||||
|
||||
summary = {k: v for k, v in summary.items() if "competition" in v}
|
||||
base_df = pd.DataFrame(
|
||||
columns=[
|
||||
"Competition",
|
||||
"Total Loops",
|
||||
"Successful Final Decision",
|
||||
"Made Submission",
|
||||
"Valid Submission",
|
||||
"V/M",
|
||||
"Above Median",
|
||||
"Bronze",
|
||||
"Silver",
|
||||
"Gold",
|
||||
"Any Medal",
|
||||
"Best Medal",
|
||||
"SOTA Exp",
|
||||
"Ours - Base",
|
||||
"Ours vs Base",
|
||||
"SOTA Exp Score",
|
||||
"Baseline Score",
|
||||
"Bronze Threshold",
|
||||
"Silver Threshold",
|
||||
"Gold Threshold",
|
||||
"Medium Threshold",
|
||||
"stdout",
|
||||
],
|
||||
index=summary.keys(),
|
||||
)
|
||||
|
||||
# Read baseline results
|
||||
baseline_result_path = UI_SETTING.baseline_result_path
|
||||
if Path(baseline_result_path).exists():
|
||||
baseline_df = pd.read_csv(baseline_result_path)
|
||||
|
||||
for k, v in summary.items():
|
||||
loop_num = v["loop_num"]
|
||||
base_df.loc[k, "Competition"] = v["competition"]
|
||||
base_df.loc[k, "Total Loops"] = loop_num
|
||||
if loop_num == 0:
|
||||
base_df.loc[k] = "N/A"
|
||||
else:
|
||||
base_df.loc[k, "Successful Final Decision"] = v["success_loop_num"]
|
||||
base_df.loc[k, "Made Submission"] = v["made_submission_num"]
|
||||
base_df.loc[k, "Valid Submission"] = v["valid_submission_num"]
|
||||
base_df.loc[k, "Above Median"] = v["above_median_num"]
|
||||
base_df.loc[k, "Bronze"] = v["bronze_num"]
|
||||
if v["bronze_num"] > 0:
|
||||
base_df.loc[k, "Best Medal"] = "bronze"
|
||||
base_df.loc[k, "Silver"] = v["silver_num"]
|
||||
if v["silver_num"] > 0:
|
||||
base_df.loc[k, "Best Medal"] = "silver"
|
||||
base_df.loc[k, "Gold"] = v["gold_num"]
|
||||
if v["gold_num"] > 0:
|
||||
base_df.loc[k, "Best Medal"] = "gold"
|
||||
base_df.loc[k, "Any Medal"] = v["get_medal_num"]
|
||||
|
||||
baseline_score = None
|
||||
if Path(baseline_result_path).exists():
|
||||
baseline_score = baseline_df.loc[baseline_df["competition_id"] == v["competition"], "score"].item()
|
||||
|
||||
base_df.loc[k, "SOTA Exp"] = v.get("sota_exp_stat", None)
|
||||
if baseline_score is not None and v.get("sota_exp_score", None) is not None:
|
||||
base_df.loc[k, "Ours - Base"] = v["sota_exp_score"] - baseline_score
|
||||
try:
|
||||
base_df.loc[k, "Ours vs Base"] = math.exp(
|
||||
abs(math.log(v["sota_exp_score"] / baseline_score))
|
||||
) # exp^|ln(a/b)|
|
||||
except Exception as e:
|
||||
base_df.loc[k, "Ours vs Base"] = None
|
||||
base_df.loc[k, "SOTA Exp Score"] = v.get("sota_exp_score", None)
|
||||
base_df.loc[k, "Baseline Score"] = baseline_score
|
||||
base_df.loc[k, "Bronze Threshold"] = v.get("bronze_threshold", None)
|
||||
base_df.loc[k, "Silver Threshold"] = v.get("silver_threshold", None)
|
||||
base_df.loc[k, "Gold Threshold"] = v.get("gold_threshold", None)
|
||||
base_df.loc[k, "Medium Threshold"] = v.get("median_threshold", None)
|
||||
base_df.loc[k, "stdout"] = v["stdout"]
|
||||
|
||||
base_df["SOTA Exp"].replace("", pd.NA, inplace=True)
|
||||
base_df = base_df.astype(
|
||||
{
|
||||
"Total Loops": int,
|
||||
"Successful Final Decision": int,
|
||||
"Made Submission": int,
|
||||
"Valid Submission": int,
|
||||
"Above Median": int,
|
||||
"Bronze": int,
|
||||
"Silver": int,
|
||||
"Gold": int,
|
||||
"Any Medal": int,
|
||||
"Ours - Base": float,
|
||||
"Ours vs Base": float,
|
||||
"SOTA Exp Score": float,
|
||||
"Baseline Score": float,
|
||||
"Bronze Threshold": float,
|
||||
"Silver Threshold": float,
|
||||
"Gold Threshold": float,
|
||||
"Medium Threshold": float,
|
||||
}
|
||||
)
|
||||
return summary, base_df
|
||||
|
||||
|
||||
def num2percent(num: int, total: int, show_origin=True) -> str:
|
||||
if show_origin:
|
||||
return f"{num} ({round(num / total * 100, 2)}%)"
|
||||
return f"{round(num / total * 100, 2)}%"
|
||||
|
||||
|
||||
def percent_df(df: pd.DataFrame, show_origin=True) -> pd.DataFrame:
|
||||
base_df = df.astype("object", copy=True)
|
||||
for k in base_df.index:
|
||||
loop_num = int(base_df.loc[k, "Total Loops"])
|
||||
if loop_num != 0:
|
||||
base_df.loc[k, "Successful Final Decision"] = num2percent(
|
||||
base_df.loc[k, "Successful Final Decision"], loop_num, show_origin
|
||||
)
|
||||
if base_df.loc[k, "Made Submission"] != 0:
|
||||
base_df.loc[k, "V/M"] = (
|
||||
f"{round(base_df.loc[k, 'Valid Submission'] / base_df.loc[k, 'Made Submission'] * 100, 2)}%"
|
||||
)
|
||||
else:
|
||||
base_df.loc[k, "V/M"] = "N/A"
|
||||
base_df.loc[k, "Made Submission"] = num2percent(base_df.loc[k, "Made Submission"], loop_num, show_origin)
|
||||
base_df.loc[k, "Valid Submission"] = num2percent(base_df.loc[k, "Valid Submission"], loop_num, show_origin)
|
||||
base_df.loc[k, "Above Median"] = num2percent(base_df.loc[k, "Above Median"], loop_num, show_origin)
|
||||
base_df.loc[k, "Bronze"] = num2percent(base_df.loc[k, "Bronze"], loop_num, show_origin)
|
||||
base_df.loc[k, "Silver"] = num2percent(base_df.loc[k, "Silver"], loop_num, show_origin)
|
||||
base_df.loc[k, "Gold"] = num2percent(base_df.loc[k, "Gold"], loop_num, show_origin)
|
||||
base_df.loc[k, "Any Medal"] = num2percent(base_df.loc[k, "Any Medal"], loop_num, show_origin)
|
||||
return base_df
|
||||
|
||||
|
||||
def days_summarize_win():
|
||||
lfs1 = [re.sub(r"log\.srv\d*", "log.srv", folder) for folder in state.log_folders]
|
||||
lfs2 = [re.sub(r"log\.srv\d*", "log.srv2", folder) for folder in state.log_folders]
|
||||
lfs3 = [re.sub(r"log\.srv\d*", "log.srv3", folder) for folder in state.log_folders]
|
||||
|
||||
_, df1 = get_summary_df(lfs1)
|
||||
_, df2 = get_summary_df(lfs2)
|
||||
_, df3 = get_summary_df(lfs3)
|
||||
|
||||
df = pd.concat([df1, df2, df3], axis=0)
|
||||
|
||||
def mean_func(x: pd.DataFrame):
|
||||
numeric_cols = x.select_dtypes(include=["int", "float"]).mean()
|
||||
string_cols = x.select_dtypes(include=["object"]).agg(lambda col: ", ".join(col.fillna("none").astype(str)))
|
||||
return pd.concat([numeric_cols, string_cols], axis=0).reindex(x.columns).drop("Competition")
|
||||
|
||||
df = df.groupby("Competition").apply(mean_func)
|
||||
if st.toggle("Show Percent", key="show_percent"):
|
||||
st.dataframe(percent_df(df, show_origin=False))
|
||||
else:
|
||||
st.dataframe(df)
|
||||
|
||||
|
||||
def all_summarize_win():
|
||||
def shorten_folder_name(folder: str) -> str:
|
||||
if "amlt" in folder:
|
||||
return folder[folder.rfind("amlt") + 5 :].split("/")[0]
|
||||
if "ep" in folder:
|
||||
return folder[folder.rfind("ep") :]
|
||||
return folder
|
||||
|
||||
selected_folders = st.multiselect(
|
||||
"Show these folders", state.log_folders, state.log_folders, format_func=shorten_folder_name
|
||||
)
|
||||
summary, base_df = get_summary_df(selected_folders)
|
||||
if not summary:
|
||||
return
|
||||
|
||||
base_df = percent_df(base_df)
|
||||
st.dataframe(base_df)
|
||||
st.markdown("Ours vs Base: `math.exp(abs(math.log(sota_exp_score / baseline_score)))`")
|
||||
st.markdown(f"**统计的比赛数目: :red[{base_df.shape[0]}]**")
|
||||
total_stat = (
|
||||
base_df[
|
||||
[
|
||||
"Made Submission",
|
||||
"Valid Submission",
|
||||
"Above Median",
|
||||
"Bronze",
|
||||
"Silver",
|
||||
"Gold",
|
||||
"Any Medal",
|
||||
]
|
||||
]
|
||||
!= "0 (0.0%)"
|
||||
).sum()
|
||||
total_stat.name = "总体统计(%)"
|
||||
total_stat.loc["Bronze"] = base_df["Best Medal"].value_counts().get("bronze", 0)
|
||||
total_stat.loc["Silver"] = base_df["Best Medal"].value_counts().get("silver", 0)
|
||||
total_stat.loc["Gold"] = base_df["Best Medal"].value_counts().get("gold", 0)
|
||||
total_stat = total_stat / base_df.shape[0] * 100
|
||||
|
||||
# SOTA Exp 统计
|
||||
se_counts = base_df["SOTA Exp"].value_counts(dropna=True)
|
||||
se_counts.loc["made_submission"] = se_counts.sum()
|
||||
se_counts.loc["Any Medal"] = se_counts.get("gold", 0) + se_counts.get("silver", 0) + se_counts.get("bronze", 0)
|
||||
se_counts.loc["above_median"] = se_counts.get("above_median", 0) + se_counts.get("Any Medal", 0)
|
||||
se_counts.loc["valid_submission"] = se_counts.get("valid_submission", 0) + se_counts.get("above_median", 0)
|
||||
|
||||
sota_exp_stat = pd.Series(index=total_stat.index, dtype=int, name="SOTA Exp 统计(%)")
|
||||
sota_exp_stat.loc["Made Submission"] = se_counts.get("made_submission", 0)
|
||||
sota_exp_stat.loc["Valid Submission"] = se_counts.get("valid_submission", 0)
|
||||
sota_exp_stat.loc["Above Median"] = se_counts.get("above_median", 0)
|
||||
sota_exp_stat.loc["Bronze"] = se_counts.get("bronze", 0)
|
||||
sota_exp_stat.loc["Silver"] = se_counts.get("silver", 0)
|
||||
sota_exp_stat.loc["Gold"] = se_counts.get("gold", 0)
|
||||
sota_exp_stat.loc["Any Medal"] = se_counts.get("Any Medal", 0)
|
||||
sota_exp_stat = sota_exp_stat / base_df.shape[0] * 100
|
||||
|
||||
stat_df = pd.concat([total_stat, sota_exp_stat], axis=1)
|
||||
stat_t0, stat_t1 = st.columns(2)
|
||||
with stat_t0:
|
||||
st.dataframe(stat_df.round(2))
|
||||
markdown_table = f"""
|
||||
| xxx | {stat_df.iloc[0,1]:.1f} | {stat_df.iloc[1,1]:.1f} | {stat_df.iloc[2,1]:.1f} | {stat_df.iloc[3,1]:.1f} | {stat_df.iloc[4,1]:.1f} | {stat_df.iloc[5,1]:.1f} | {stat_df.iloc[6,1]:.1f} |
|
||||
"""
|
||||
st.text(markdown_table)
|
||||
with stat_t1:
|
||||
Loop_counts = base_df["Total Loops"]
|
||||
fig = px.histogram(Loop_counts, nbins=10, title="Total Loops Histogram (nbins=10)")
|
||||
mean_value = Loop_counts.mean()
|
||||
median_value = Loop_counts.median()
|
||||
fig.add_vline(
|
||||
x=mean_value, line_color="orange", annotation_text="Mean", annotation_position="top right", line_width=3
|
||||
)
|
||||
fig.add_vline(
|
||||
x=median_value, line_color="red", annotation_text="Median", annotation_position="top right", line_width=3
|
||||
)
|
||||
st.plotly_chart(fig)
|
||||
|
||||
# write curve
|
||||
for k, v in summary.items():
|
||||
with st.container(border=True):
|
||||
st.markdown(f"**:blue[{k}] - :violet[{v['competition']}]**")
|
||||
fc1, fc2 = st.columns(2)
|
||||
tscores = {f"loop {k-1}": v for k, v in v["test_scores"].items()}
|
||||
tdf = pd.Series(tscores, name="score")
|
||||
f2 = px.line(tdf, markers=True, title="Test scores")
|
||||
fc2.plotly_chart(f2, key=k)
|
||||
try:
|
||||
vscores = {k: v.iloc[:, 0] for k, v in v["valid_scores"].items()}
|
||||
|
||||
if len(vscores) > 0:
|
||||
metric_name = list(vscores.values())[0].name
|
||||
else:
|
||||
metric_name = "None"
|
||||
|
||||
vdf = pd.DataFrame(vscores)
|
||||
vdf.columns = [f"loop {i}" for i in vdf.columns]
|
||||
f1 = px.line(vdf.T, markers=True, title=f"Valid scores (metric: {metric_name})")
|
||||
|
||||
fc1.plotly_chart(f1, key=f"{k}_v")
|
||||
except Exception as e:
|
||||
import traceback
|
||||
|
||||
st.markdown("- Error: " + str(e))
|
||||
st.code(traceback.format_exc())
|
||||
st.markdown("- Valid Scores: ")
|
||||
# st.write({k: type(v) for k, v in v["valid_scores"].items()})
|
||||
st.json(v["valid_scores"])
|
||||
|
||||
|
||||
def stdout_win(loop_id: int):
|
||||
stdout = load_stdout()
|
||||
if stdout.startswith("Please Set"):
|
||||
st.toast(stdout, icon="🟡")
|
||||
return
|
||||
start_index = stdout.find(f"Start Loop {loop_id}")
|
||||
end_index = stdout.find(f"Start Loop {loop_id + 1}")
|
||||
loop_stdout = remove_ansi_codes(stdout[start_index:end_index])
|
||||
with st.container(border=True):
|
||||
st.subheader(f"Loop {loop_id} stdout")
|
||||
pattern = f"Start Loop {loop_id}, " + r"Step \d+: \w+"
|
||||
matches = re.finditer(pattern, loop_stdout)
|
||||
step_stdouts = {}
|
||||
for match in matches:
|
||||
step = match.group(0)
|
||||
si = match.start()
|
||||
ei = loop_stdout.find(f"Start Loop {loop_id}", match.end())
|
||||
step_stdouts[step] = loop_stdout[si:ei].strip()
|
||||
|
||||
for k, v in step_stdouts.items():
|
||||
with st.expander(k, expanded=False):
|
||||
st.code(v, language="log", wrap_lines=True)
|
||||
|
||||
|
||||
# UI - Main
|
||||
if state.show_all_summary:
|
||||
with st.container(border=True):
|
||||
if st.toggle("近3天平均", key="show_3days"):
|
||||
days_summarize_win()
|
||||
with st.container(border=True):
|
||||
all_summarize_win()
|
||||
elif "data" in state:
|
||||
st.title(state.data["competition"])
|
||||
summarize_data()
|
||||
if len(state.data) > 2:
|
||||
loop_id = st.slider("Loop", 0, len(state.data) - 2, 0)
|
||||
else:
|
||||
loop_id = 0
|
||||
if state.show_stdout:
|
||||
stdout_win(loop_id)
|
||||
main_win(state.data[loop_id])
|
||||
if st.form_submit_button("Confirm"):
|
||||
state.log_folders = [folder.strip() for folder in log_folder_str.split(";") if folder.strip()]
|
||||
st.rerun()
|
||||
|
||||
@@ -26,7 +26,7 @@ class TestRDAgentImports(unittest.TestCase):
|
||||
if "llm_st" in fstr:
|
||||
continue
|
||||
if (
|
||||
fstr.endswith("rdagent/log/ui/app.py")
|
||||
"rdagent/log/ui/" in fstr
|
||||
or fstr.endswith("rdagent/app/cli.py")
|
||||
or fstr.endswith("rdagent/app/CI/run.py")
|
||||
or fstr.endswith("rdagent/app/utils/ape.py")
|
||||
|
||||
Reference in New Issue
Block a user