mirror of
https://github.com/NicolasBohn/NexQuant.git
synced 2026-07-27 15:37:44 +00:00
chore: organize the tool functions related to trace and summary (#889)
* provide get metric direction in kaggle_crawler.py * move utils function * move statistics logic * fix CI * fix CI * fix CI * fix CI * move some tool functions * fix CI * move curves win * add compare tool * change function name * fix CI
This commit is contained in:
XianBW
@@ -2,18 +2,19 @@ import json
|
||||
import pickle
|
||||
import re
|
||||
from collections import defaultdict
|
||||
from datetime import timedelta
|
||||
from pathlib import Path
|
||||
|
||||
import fire
|
||||
import pandas as pd
|
||||
|
||||
from rdagent.app.data_science.conf import DS_RD_SETTING
|
||||
from rdagent.app.data_science.loop import DataScienceRDLoop
|
||||
from rdagent.components.coder.data_science.conf import get_ds_env
|
||||
from rdagent.core.experiment import FBWorkspace
|
||||
from rdagent.core.proposal import ExperimentFeedback
|
||||
from rdagent.log.storage import FileStorage
|
||||
from rdagent.log.utils import (
|
||||
extract_json,
|
||||
extract_loopid_func_name,
|
||||
is_valid_session,
|
||||
)
|
||||
from rdagent.log.utils.folder import get_first_session_file_after_duration
|
||||
from rdagent.scenarios.data_science.experiment.experiment import DSExperiment
|
||||
from rdagent.scenarios.data_science.test_eval import (
|
||||
@@ -29,19 +30,6 @@ test_eval = get_test_eval()
|
||||
is_mle = isinstance(test_eval, MLETestEval)
|
||||
|
||||
|
||||
def extract_mle_json(log_content: str) -> dict | None:
|
||||
match = re.search(r"\{.*\}", log_content, re.DOTALL)
|
||||
if match:
|
||||
return json.loads(match.group(0))
|
||||
return None
|
||||
|
||||
|
||||
def extract_loopid_func_name(tag):
|
||||
"""提取 Loop ID 和函数名称"""
|
||||
match = re.search(r"Loop_(\d+)\.([^.]+)", tag)
|
||||
return match.groups() if match else (None, None)
|
||||
|
||||
|
||||
def save_grade_info(log_trace_path: Path):
|
||||
trace_storage = FileStorage(log_trace_path)
|
||||
for msg in trace_storage.iter_msg():
|
||||
@@ -61,10 +49,6 @@ def save_grade_info(log_trace_path: Path):
|
||||
print(f"Error in {log_trace_path}: {e}")
|
||||
|
||||
|
||||
def is_valid_session(p: Path) -> bool:
|
||||
return p.is_dir() and p.joinpath("__session__").exists()
|
||||
|
||||
|
||||
def save_all_grade_info(log_folder):
|
||||
for log_trace_path in log_folder.iterdir():
|
||||
if is_valid_session(log_trace_path):
|
||||
@@ -149,7 +133,7 @@ def summarize_folder(log_folder: Path, hours: int | None = None):
|
||||
env=test_eval.env,
|
||||
entry=f"mlebench grade-sample None {stat[log_trace_path.name]['competition']} --data-dir /mle/data",
|
||||
)
|
||||
grade_output = extract_mle_json(stdout)
|
||||
grade_output = extract_json(stdout)
|
||||
if grade_output:
|
||||
bronze_threshold = grade_output["bronze_threshold"]
|
||||
silver_threshold = grade_output["silver_threshold"]
|
||||
@@ -164,7 +148,7 @@ def summarize_folder(log_folder: Path, hours: int | None = None):
|
||||
if msg.content.result is not None:
|
||||
valid_scores[loop_id] = msg.content.result
|
||||
elif "mle_score" in msg.tag:
|
||||
grade_output = extract_mle_json(msg.content)
|
||||
grade_output = extract_json(msg.content)
|
||||
if grade_output:
|
||||
if grade_output["submission_exists"]:
|
||||
made_submission_num += 1
|
||||
|
||||
@@ -12,5 +12,7 @@ class UIBasePropSetting(ExtendedBaseSettings):
|
||||
|
||||
aide_path: str = "./aide"
|
||||
|
||||
amlt_path: str = "/data/share_folder_local/amlt"
|
||||
|
||||
|
||||
UI_SETTING = UIBasePropSetting()
|
||||
|
||||
+77
-441
@@ -1,8 +1,4 @@
|
||||
import math
|
||||
import pickle
|
||||
import re
|
||||
from collections import deque
|
||||
from datetime import datetime, timedelta
|
||||
from pathlib import Path
|
||||
|
||||
import pandas as pd
|
||||
@@ -11,322 +7,16 @@ import plotly.graph_objects as go
|
||||
import streamlit as st
|
||||
from streamlit import session_state as state
|
||||
|
||||
from rdagent.log.mle_summary import extract_mle_json
|
||||
from rdagent.log.ui.conf import UI_SETTING
|
||||
from rdagent.log.ui.ds_trace import load_times
|
||||
from rdagent.log.ui.utils import ALL, HIGH, LITE, MEDIUM
|
||||
from rdagent.scenarios.kaggle.kaggle_crawler import leaderboard_scores
|
||||
|
||||
|
||||
def get_metric_direction(competition: str):
|
||||
leaderboard = leaderboard_scores(competition)
|
||||
return float(leaderboard[0]) > float(leaderboard[-1])
|
||||
|
||||
|
||||
def get_script_time(stdout_p: Path):
|
||||
with stdout_p.open("r") as f:
|
||||
first_line = next(f).strip()
|
||||
last_line = deque(f, maxlen=1).pop().strip()
|
||||
|
||||
# Extract timestamps from the lines
|
||||
first_time_match = re.search(r"(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\+\d{2}:\d{2})", first_line)
|
||||
last_time_match = re.search(r"(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\+\d{2}:\d{2})", last_line)
|
||||
|
||||
if first_time_match and last_time_match:
|
||||
first_time = datetime.fromisoformat(first_time_match.group(1))
|
||||
last_time = datetime.fromisoformat(last_time_match.group(1))
|
||||
return pd.Timedelta(last_time - first_time)
|
||||
|
||||
return None
|
||||
|
||||
|
||||
@st.cache_data(persist=True)
|
||||
def get_final_sota_exp(log_path: Path):
|
||||
sota_exp_paths = [i for i in log_path.rglob(f"**/SOTA experiment/**/*.pkl")]
|
||||
if len(sota_exp_paths) == 0:
|
||||
return None
|
||||
final_sota_exp_path = max(sota_exp_paths, key=lambda x: int(re.match(r".*Loop_(\d+).*", str(x))[1]))
|
||||
with final_sota_exp_path.open("rb") as f:
|
||||
final_sota_exp = pickle.load(f)
|
||||
return final_sota_exp
|
||||
|
||||
|
||||
@st.cache_data(persist=True)
|
||||
def get_sota_exp_stat(log_path: Path):
|
||||
trace_paths = [i for i in log_path.rglob(f"**/trace/**/*.pkl")]
|
||||
if len(trace_paths) == 0:
|
||||
return None
|
||||
final_trace_path = max(trace_paths, key=lambda x: int(re.match(r".*Loop_(\d+).*", str(x))[1]))
|
||||
with final_trace_path.open("rb") as f:
|
||||
final_trace = pickle.load(f)
|
||||
|
||||
if hasattr(final_trace, "sota_exp_to_submit"):
|
||||
sota_exp = final_trace.sota_exp_to_submit
|
||||
else:
|
||||
sota_exp = final_trace.sota_experiment()
|
||||
|
||||
if sota_exp is None:
|
||||
return None
|
||||
|
||||
sota_loop_id = None
|
||||
for i, ef in enumerate(final_trace.hist):
|
||||
if ef[0] == sota_exp:
|
||||
sota_loop_id = i
|
||||
break
|
||||
|
||||
sota_mle_score_paths = [i for i in log_path.rglob(f"Loop_{sota_loop_id}/running/mle_score/**/*.pkl")]
|
||||
if len(sota_mle_score_paths) == 0:
|
||||
# sota exp is not evaluated by mle_score
|
||||
return None
|
||||
with sota_mle_score_paths[0].open("rb") as f:
|
||||
sota_mle_score = extract_mle_json(pickle.load(f))
|
||||
|
||||
sota_exp_stat = None
|
||||
if sota_mle_score: # sota exp's grade output
|
||||
if sota_mle_score["gold_medal"]:
|
||||
sota_exp_stat = "gold"
|
||||
elif sota_mle_score["silver_medal"]:
|
||||
sota_exp_stat = "silver"
|
||||
elif sota_mle_score["bronze_medal"]:
|
||||
sota_exp_stat = "bronze"
|
||||
elif sota_mle_score["above_median"]:
|
||||
sota_exp_stat = "above_median"
|
||||
elif sota_mle_score["valid_submission"]:
|
||||
sota_exp_stat = "valid_submission"
|
||||
elif sota_mle_score["submission_exists"]:
|
||||
sota_exp_stat = "made_submission"
|
||||
return sota_exp_stat
|
||||
|
||||
|
||||
@st.cache_data(persist=True)
|
||||
def get_summary_df(log_folders: list[str]) -> tuple[dict, pd.DataFrame]:
|
||||
summarys = {}
|
||||
sn = "summary.pkl"
|
||||
for lf in log_folders:
|
||||
if (Path(lf) / sn).exists():
|
||||
summarys[lf] = pd.read_pickle(Path(lf) / sn)
|
||||
|
||||
if len(summarys) == 0:
|
||||
return {}, pd.DataFrame()
|
||||
|
||||
summary = {}
|
||||
for lf, s in summarys.items():
|
||||
for k, v in s.items():
|
||||
stdout_p = Path(lf) / f"{k}.stdout"
|
||||
if stdout_p.exists():
|
||||
v["script_time"] = get_script_time(stdout_p)
|
||||
else:
|
||||
v["script_time"] = None
|
||||
|
||||
exp_gen_time = timedelta()
|
||||
coding_time = timedelta()
|
||||
running_time = timedelta()
|
||||
all_time = timedelta()
|
||||
times_info = load_times(Path(lf) / k)
|
||||
for time_info in times_info.values():
|
||||
all_time += sum((ti.end - ti.start for ti in time_info), timedelta())
|
||||
exp_gen_time += time_info[0].end - time_info[0].start
|
||||
if len(time_info) > 1:
|
||||
coding_time += time_info[1].end - time_info[1].start
|
||||
if len(time_info) > 2:
|
||||
running_time += time_info[2].end - time_info[2].start
|
||||
v["exec_time"] = str(all_time).split(".")[0]
|
||||
v["exp_gen_time"] = str(exp_gen_time).split(".")[0]
|
||||
v["coding_time"] = str(coding_time).split(".")[0]
|
||||
v["running_time"] = str(running_time).split(".")[0]
|
||||
|
||||
final_sota_exp = get_final_sota_exp(Path(lf) / k)
|
||||
if final_sota_exp is not None and final_sota_exp.result is not None:
|
||||
v["sota_exp_score_valid"] = final_sota_exp.result.loc["ensemble"].iloc[0]
|
||||
else:
|
||||
v["sota_exp_score_valid"] = None
|
||||
v["sota_exp_stat_new"] = get_sota_exp_stat(Path(lf) / k)
|
||||
# 调整实验名字
|
||||
if "amlt" in lf:
|
||||
summary[f"{lf[lf.rfind('amlt')+5:].split('/')[0]} - {k}"] = v
|
||||
elif "ep" in lf:
|
||||
summary[f"{lf[lf.rfind('ep'):]} - {k}"] = v
|
||||
else:
|
||||
summary[f"{lf} - {k}"] = v
|
||||
|
||||
summary = {k: v for k, v in summary.items() if "competition" in v}
|
||||
base_df = pd.DataFrame(
|
||||
columns=[
|
||||
"Competition",
|
||||
"Script Time",
|
||||
"Exec Time",
|
||||
"Exp Gen",
|
||||
"Coding",
|
||||
"Running",
|
||||
"Total Loops",
|
||||
"Successful Final Decision",
|
||||
"Made Submission",
|
||||
"Valid Submission",
|
||||
"V/M",
|
||||
"Above Median",
|
||||
"Bronze",
|
||||
"Silver",
|
||||
"Gold",
|
||||
"Any Medal",
|
||||
"Best Result",
|
||||
"SOTA Exp",
|
||||
"SOTA Exp (_to_submit)",
|
||||
"SOTA Exp Score (valid)",
|
||||
"SOTA Exp Score",
|
||||
"Baseline Score",
|
||||
"Ours - Base",
|
||||
"Ours vs Base",
|
||||
"Ours vs Bronze",
|
||||
"Ours vs Silver",
|
||||
"Ours vs Gold",
|
||||
"Bronze Threshold",
|
||||
"Silver Threshold",
|
||||
"Gold Threshold",
|
||||
"Medium Threshold",
|
||||
],
|
||||
index=summary.keys(),
|
||||
)
|
||||
|
||||
# Read baseline results
|
||||
baseline_result_path = UI_SETTING.baseline_result_path
|
||||
if Path(baseline_result_path).exists():
|
||||
baseline_df = pd.read_csv(baseline_result_path)
|
||||
|
||||
def compare_score(s1, s2):
|
||||
if s1 is None or s2 is None:
|
||||
return None
|
||||
try:
|
||||
c_value = math.exp(abs(math.log(s1 / s2)))
|
||||
except Exception as e:
|
||||
c_value = None
|
||||
return c_value
|
||||
|
||||
for k, v in summary.items():
|
||||
loop_num = v["loop_num"]
|
||||
base_df.loc[k, "Competition"] = v["competition"]
|
||||
base_df.loc[k, "Script Time"] = v["script_time"]
|
||||
base_df.loc[k, "Exec Time"] = v["exec_time"]
|
||||
base_df.loc[k, "Exp Gen"] = v["exp_gen_time"]
|
||||
base_df.loc[k, "Coding"] = v["coding_time"]
|
||||
base_df.loc[k, "Running"] = v["running_time"]
|
||||
base_df.loc[k, "Total Loops"] = loop_num
|
||||
if loop_num == 0:
|
||||
base_df.loc[k] = "N/A"
|
||||
else:
|
||||
base_df.loc[k, "Successful Final Decision"] = v["success_loop_num"]
|
||||
base_df.loc[k, "Made Submission"] = v["made_submission_num"]
|
||||
if v["made_submission_num"] > 0:
|
||||
base_df.loc[k, "Best Result"] = "made_submission"
|
||||
base_df.loc[k, "Valid Submission"] = v["valid_submission_num"]
|
||||
if v["valid_submission_num"] > 0:
|
||||
base_df.loc[k, "Best Result"] = "valid_submission"
|
||||
base_df.loc[k, "Above Median"] = v["above_median_num"]
|
||||
if v["above_median_num"] > 0:
|
||||
base_df.loc[k, "Best Result"] = "above_median"
|
||||
base_df.loc[k, "Bronze"] = v["bronze_num"]
|
||||
if v["bronze_num"] > 0:
|
||||
base_df.loc[k, "Best Result"] = "bronze"
|
||||
base_df.loc[k, "Silver"] = v["silver_num"]
|
||||
if v["silver_num"] > 0:
|
||||
base_df.loc[k, "Best Result"] = "silver"
|
||||
base_df.loc[k, "Gold"] = v["gold_num"]
|
||||
if v["gold_num"] > 0:
|
||||
base_df.loc[k, "Best Result"] = "gold"
|
||||
base_df.loc[k, "Any Medal"] = v["get_medal_num"]
|
||||
|
||||
baseline_score = None
|
||||
if Path(baseline_result_path).exists():
|
||||
baseline_score = baseline_df.loc[baseline_df["competition_id"] == v["competition"], "score"].item()
|
||||
|
||||
base_df.loc[k, "SOTA Exp"] = v.get("sota_exp_stat", None)
|
||||
base_df.loc[k, "SOTA Exp (_to_submit)"] = v["sota_exp_stat_new"]
|
||||
if baseline_score is not None and v.get("sota_exp_score", None) is not None:
|
||||
base_df.loc[k, "Ours - Base"] = v["sota_exp_score"] - baseline_score
|
||||
base_df.loc[k, "Ours vs Base"] = compare_score(v["sota_exp_score"], baseline_score)
|
||||
base_df.loc[k, "Ours vs Bronze"] = compare_score(v["sota_exp_score"], v.get("bronze_threshold", None))
|
||||
base_df.loc[k, "Ours vs Silver"] = compare_score(v["sota_exp_score"], v.get("silver_threshold", None))
|
||||
base_df.loc[k, "Ours vs Gold"] = compare_score(v["sota_exp_score"], v.get("gold_threshold", None))
|
||||
base_df.loc[k, "SOTA Exp Score"] = v.get("sota_exp_score", None)
|
||||
base_df.loc[k, "SOTA Exp Score (valid)"] = v.get("sota_exp_score_valid", None)
|
||||
base_df.loc[k, "Baseline Score"] = baseline_score
|
||||
base_df.loc[k, "Bronze Threshold"] = v.get("bronze_threshold", None)
|
||||
base_df.loc[k, "Silver Threshold"] = v.get("silver_threshold", None)
|
||||
base_df.loc[k, "Gold Threshold"] = v.get("gold_threshold", None)
|
||||
base_df.loc[k, "Medium Threshold"] = v.get("median_threshold", None)
|
||||
|
||||
base_df["SOTA Exp"] = base_df["SOTA Exp"].replace("", pd.NA)
|
||||
|
||||
base_df.loc[base_df["SOTA Exp Score (valid)"].apply(lambda x: isinstance(x, str)), "SOTA Exp Score (valid)"] = 0.0
|
||||
base_df = base_df.astype(
|
||||
{
|
||||
"Total Loops": int,
|
||||
"Successful Final Decision": int,
|
||||
"Made Submission": int,
|
||||
"Valid Submission": int,
|
||||
"Above Median": int,
|
||||
"Bronze": int,
|
||||
"Silver": int,
|
||||
"Gold": int,
|
||||
"Any Medal": int,
|
||||
"Ours - Base": float,
|
||||
"Ours vs Base": float,
|
||||
"SOTA Exp Score": float,
|
||||
"SOTA Exp Score (valid)": float,
|
||||
"Baseline Score": float,
|
||||
"Bronze Threshold": float,
|
||||
"Silver Threshold": float,
|
||||
"Gold Threshold": float,
|
||||
"Medium Threshold": float,
|
||||
}
|
||||
)
|
||||
return summary, base_df
|
||||
|
||||
|
||||
def num2percent(num: int, total: int, show_origin=True) -> str:
|
||||
num = int(num)
|
||||
total = int(total)
|
||||
if show_origin:
|
||||
return f"{num} ({round(num / total * 100, 2)}%)"
|
||||
return f"{round(num / total * 100, 2)}%"
|
||||
|
||||
|
||||
def percent_df(df: pd.DataFrame, show_origin=True) -> pd.DataFrame:
|
||||
base_df = df.copy(deep=True)
|
||||
|
||||
# Convert columns to object dtype so we can store strings like "14 (53.85%)" without warnings
|
||||
columns_to_convert = [
|
||||
"Successful Final Decision",
|
||||
"Made Submission",
|
||||
"Valid Submission",
|
||||
"Above Median",
|
||||
"Bronze",
|
||||
"Silver",
|
||||
"Gold",
|
||||
"Any Medal",
|
||||
]
|
||||
base_df[columns_to_convert] = base_df[columns_to_convert].astype(object)
|
||||
|
||||
for k in base_df.index:
|
||||
loop_num = int(base_df.loc[k, "Total Loops"])
|
||||
if loop_num != 0:
|
||||
base_df.loc[k, "Successful Final Decision"] = num2percent(
|
||||
base_df.loc[k, "Successful Final Decision"], loop_num, show_origin
|
||||
)
|
||||
if base_df.loc[k, "Made Submission"] != 0:
|
||||
base_df.loc[k, "V/M"] = (
|
||||
f"{round(base_df.loc[k, 'Valid Submission'] / base_df.loc[k, 'Made Submission'] * 100, 2)}%"
|
||||
)
|
||||
else:
|
||||
base_df.loc[k, "V/M"] = "N/A"
|
||||
base_df.loc[k, "Made Submission"] = num2percent(base_df.loc[k, "Made Submission"], loop_num, show_origin)
|
||||
base_df.loc[k, "Valid Submission"] = num2percent(base_df.loc[k, "Valid Submission"], loop_num, show_origin)
|
||||
base_df.loc[k, "Above Median"] = num2percent(base_df.loc[k, "Above Median"], loop_num, show_origin)
|
||||
base_df.loc[k, "Bronze"] = num2percent(base_df.loc[k, "Bronze"], loop_num, show_origin)
|
||||
base_df.loc[k, "Silver"] = num2percent(base_df.loc[k, "Silver"], loop_num, show_origin)
|
||||
base_df.loc[k, "Gold"] = num2percent(base_df.loc[k, "Gold"], loop_num, show_origin)
|
||||
base_df.loc[k, "Any Medal"] = num2percent(base_df.loc[k, "Any Medal"], loop_num, show_origin)
|
||||
|
||||
return base_df
|
||||
from rdagent.log.ui.utils import (
|
||||
ALL,
|
||||
HIGH,
|
||||
LITE,
|
||||
MEDIUM,
|
||||
get_statistics_df,
|
||||
get_summary_df,
|
||||
percent_df,
|
||||
)
|
||||
from rdagent.scenarios.kaggle.kaggle_crawler import get_metric_direction
|
||||
|
||||
|
||||
def days_summarize_win():
|
||||
@@ -352,6 +42,68 @@ def days_summarize_win():
|
||||
st.dataframe(df)
|
||||
|
||||
|
||||
def curves_win(summary: dict):
|
||||
for k, v in summary.items():
|
||||
with st.container(border=True):
|
||||
st.markdown(f"**:blue[{k}] - :violet[{v['competition']}]**")
|
||||
try:
|
||||
tscores = {f"loop {k-1}": v for k, v in v["test_scores"].items()}
|
||||
vscores = {}
|
||||
for k, vs in v["valid_scores"].items():
|
||||
if not vs.index.is_unique:
|
||||
st.warning(
|
||||
f"Loop {k}'s valid scores index are not unique, only the last one will be kept to show."
|
||||
)
|
||||
st.write(vs)
|
||||
vscores[k] = vs[~vs.index.duplicated(keep="last")].iloc[:, 0]
|
||||
|
||||
if len(vscores) > 0:
|
||||
metric_name = list(vscores.values())[0].name
|
||||
else:
|
||||
metric_name = "None"
|
||||
|
||||
tdf = pd.Series(tscores, name="score")
|
||||
vdf = pd.DataFrame(vscores)
|
||||
if "ensemble" in vdf.index:
|
||||
ensemble_row = vdf.loc[["ensemble"]]
|
||||
vdf = pd.concat([ensemble_row, vdf.drop("ensemble")])
|
||||
vdf.columns = [f"loop {i}" for i in vdf.columns]
|
||||
fig = go.Figure()
|
||||
# Add test scores trace from tdf
|
||||
fig.add_trace(
|
||||
go.Scatter(
|
||||
x=tdf.index,
|
||||
y=tdf,
|
||||
mode="lines+markers",
|
||||
name="Test scores",
|
||||
marker=dict(symbol="diamond"),
|
||||
line=dict(shape="linear", dash="dash"),
|
||||
)
|
||||
)
|
||||
# Add valid score traces from vdf (transposed to have loops on x-axis)
|
||||
for column in vdf.T.columns:
|
||||
fig.add_trace(
|
||||
go.Scatter(
|
||||
x=vdf.T.index,
|
||||
y=vdf.T[column],
|
||||
mode="lines+markers",
|
||||
name=f"{column}",
|
||||
visible=("legendonly" if column != "ensemble" else None),
|
||||
)
|
||||
)
|
||||
fig.update_layout(title=f"Test and Valid scores (metric: {metric_name})")
|
||||
|
||||
st.plotly_chart(fig)
|
||||
except Exception as e:
|
||||
import traceback
|
||||
|
||||
st.markdown("- Error: " + str(e))
|
||||
st.code(traceback.format_exc())
|
||||
st.markdown("- Valid Scores: ")
|
||||
# st.write({k: type(v) for k, v in v["valid_scores"].items()})
|
||||
st.json(v["valid_scores"])
|
||||
|
||||
|
||||
def all_summarize_win():
|
||||
def shorten_folder_name(folder: str) -> str:
|
||||
if "amlt" in folder:
|
||||
@@ -460,73 +212,15 @@ def all_summarize_win():
|
||||
# 统计选择的比赛
|
||||
base_df = base_df[base_df["Select"]]
|
||||
st.markdown(f"**统计的比赛数目: :red[{base_df.shape[0]}]**")
|
||||
total_stat = (
|
||||
base_df[
|
||||
[
|
||||
"Made Submission",
|
||||
"Valid Submission",
|
||||
"Above Median",
|
||||
"Bronze",
|
||||
"Silver",
|
||||
"Gold",
|
||||
"Any Medal",
|
||||
]
|
||||
]
|
||||
!= "0 (0.0%)"
|
||||
).sum()
|
||||
total_stat.name = "总体统计(%)"
|
||||
total_stat.loc["Bronze"] = base_df["Best Result"].value_counts().get("bronze", 0)
|
||||
total_stat.loc["Silver"] = base_df["Best Result"].value_counts().get("silver", 0)
|
||||
total_stat.loc["Gold"] = base_df["Best Result"].value_counts().get("gold", 0)
|
||||
total_stat = total_stat / base_df.shape[0] * 100
|
||||
|
||||
# SOTA Exp 统计
|
||||
se_counts = base_df["SOTA Exp"].value_counts(dropna=True)
|
||||
se_counts.loc["made_submission"] = se_counts.sum()
|
||||
se_counts.loc["Any Medal"] = se_counts.get("gold", 0) + se_counts.get("silver", 0) + se_counts.get("bronze", 0)
|
||||
se_counts.loc["above_median"] = se_counts.get("above_median", 0) + se_counts.get("Any Medal", 0)
|
||||
se_counts.loc["valid_submission"] = se_counts.get("valid_submission", 0) + se_counts.get("above_median", 0)
|
||||
|
||||
sota_exp_stat = pd.Series(index=total_stat.index, dtype=int, name="SOTA Exp 统计(%)")
|
||||
sota_exp_stat.loc["Made Submission"] = se_counts.get("made_submission", 0)
|
||||
sota_exp_stat.loc["Valid Submission"] = se_counts.get("valid_submission", 0)
|
||||
sota_exp_stat.loc["Above Median"] = se_counts.get("above_median", 0)
|
||||
sota_exp_stat.loc["Bronze"] = se_counts.get("bronze", 0)
|
||||
sota_exp_stat.loc["Silver"] = se_counts.get("silver", 0)
|
||||
sota_exp_stat.loc["Gold"] = se_counts.get("gold", 0)
|
||||
sota_exp_stat.loc["Any Medal"] = se_counts.get("Any Medal", 0)
|
||||
sota_exp_stat = sota_exp_stat / base_df.shape[0] * 100
|
||||
|
||||
# SOTA Exp (trace.sota_exp_to_submit) 统计
|
||||
se_counts_new = base_df["SOTA Exp (_to_submit)"].value_counts(dropna=True)
|
||||
se_counts_new.loc["made_submission"] = se_counts_new.sum()
|
||||
se_counts_new.loc["Any Medal"] = (
|
||||
se_counts_new.get("gold", 0) + se_counts_new.get("silver", 0) + se_counts_new.get("bronze", 0)
|
||||
)
|
||||
se_counts_new.loc["above_median"] = se_counts_new.get("above_median", 0) + se_counts_new.get("Any Medal", 0)
|
||||
se_counts_new.loc["valid_submission"] = se_counts_new.get("valid_submission", 0) + se_counts_new.get(
|
||||
"above_median", 0
|
||||
)
|
||||
|
||||
sota_exp_stat_new = pd.Series(index=total_stat.index, dtype=int, name="SOTA Exp (_to_submit) 统计(%)")
|
||||
sota_exp_stat_new.loc["Made Submission"] = se_counts_new.get("made_submission", 0)
|
||||
sota_exp_stat_new.loc["Valid Submission"] = se_counts_new.get("valid_submission", 0)
|
||||
sota_exp_stat_new.loc["Above Median"] = se_counts_new.get("above_median", 0)
|
||||
sota_exp_stat_new.loc["Bronze"] = se_counts_new.get("bronze", 0)
|
||||
sota_exp_stat_new.loc["Silver"] = se_counts_new.get("silver", 0)
|
||||
sota_exp_stat_new.loc["Gold"] = se_counts_new.get("gold", 0)
|
||||
sota_exp_stat_new.loc["Any Medal"] = se_counts_new.get("Any Medal", 0)
|
||||
sota_exp_stat_new = sota_exp_stat_new / base_df.shape[0] * 100
|
||||
|
||||
stat_df = pd.concat([total_stat, sota_exp_stat, sota_exp_stat_new], axis=1)
|
||||
stat_t0, stat_t1 = st.columns(2)
|
||||
with stat_t0:
|
||||
stat_win_left, stat_win_right = st.columns(2)
|
||||
with stat_win_left:
|
||||
stat_df = get_statistics_df(base_df)
|
||||
st.dataframe(stat_df.round(2))
|
||||
markdown_table = f"""
|
||||
| xxx | {stat_df.iloc[0,1]:.1f} | {stat_df.iloc[1,1]:.1f} | {stat_df.iloc[2,1]:.1f} | {stat_df.iloc[3,1]:.1f} | {stat_df.iloc[4,1]:.1f} | {stat_df.iloc[5,1]:.1f} | {stat_df.iloc[6,1]:.1f} |
|
||||
"""
|
||||
st.text(markdown_table)
|
||||
with stat_t1:
|
||||
with stat_win_right:
|
||||
Loop_counts = base_df["Total Loops"]
|
||||
fig = px.histogram(Loop_counts, nbins=10, title="Total Loops Histogram (nbins=10)")
|
||||
mean_value = Loop_counts.mean()
|
||||
@@ -542,65 +236,7 @@ def all_summarize_win():
|
||||
# write curve
|
||||
st.subheader("Curves", divider="rainbow")
|
||||
if st.toggle("Show Curves", key="show_curves"):
|
||||
for k, v in summary.items():
|
||||
with st.container(border=True):
|
||||
st.markdown(f"**:blue[{k}] - :violet[{v['competition']}]**")
|
||||
try:
|
||||
tscores = {f"loop {k-1}": v for k, v in v["test_scores"].items()}
|
||||
vscores = {}
|
||||
for k, vs in v["valid_scores"].items():
|
||||
if not vs.index.is_unique:
|
||||
st.warning(
|
||||
f"Loop {k}'s valid scores index are not unique, only the last one will be kept to show."
|
||||
)
|
||||
st.write(vs)
|
||||
vscores[k] = vs[~vs.index.duplicated(keep="last")].iloc[:, 0]
|
||||
|
||||
if len(vscores) > 0:
|
||||
metric_name = list(vscores.values())[0].name
|
||||
else:
|
||||
metric_name = "None"
|
||||
|
||||
tdf = pd.Series(tscores, name="score")
|
||||
vdf = pd.DataFrame(vscores)
|
||||
if "ensemble" in vdf.index:
|
||||
ensemble_row = vdf.loc[["ensemble"]]
|
||||
vdf = pd.concat([ensemble_row, vdf.drop("ensemble")])
|
||||
vdf.columns = [f"loop {i}" for i in vdf.columns]
|
||||
fig = go.Figure()
|
||||
# Add test scores trace from tdf
|
||||
fig.add_trace(
|
||||
go.Scatter(
|
||||
x=tdf.index,
|
||||
y=tdf,
|
||||
mode="lines+markers",
|
||||
name="Test scores",
|
||||
marker=dict(symbol="diamond"),
|
||||
line=dict(shape="linear", dash="dash"),
|
||||
)
|
||||
)
|
||||
# Add valid score traces from vdf (transposed to have loops on x-axis)
|
||||
for column in vdf.T.columns:
|
||||
fig.add_trace(
|
||||
go.Scatter(
|
||||
x=vdf.T.index,
|
||||
y=vdf.T[column],
|
||||
mode="lines+markers",
|
||||
name=f"{column}",
|
||||
visible=("legendonly" if column != "ensemble" else None),
|
||||
)
|
||||
)
|
||||
fig.update_layout(title=f"Test and Valid scores (metric: {metric_name})")
|
||||
|
||||
st.plotly_chart(fig)
|
||||
except Exception as e:
|
||||
import traceback
|
||||
|
||||
st.markdown("- Error: " + str(e))
|
||||
st.code(traceback.format_exc())
|
||||
st.markdown("- Valid Scores: ")
|
||||
# st.write({k: type(v) for k, v in v["valid_scores"].items()})
|
||||
st.json(v["valid_scores"])
|
||||
curves_win(summary)
|
||||
|
||||
|
||||
with st.container(border=True):
|
||||
|
||||
@@ -11,8 +11,14 @@ import streamlit as st
|
||||
from streamlit import session_state as state
|
||||
|
||||
from rdagent.app.data_science.loop import DataScienceRDLoop
|
||||
from rdagent.log.mle_summary import extract_mle_json, is_valid_session
|
||||
from rdagent.log.storage import FileStorage
|
||||
from rdagent.log.ui.utils import load_times
|
||||
from rdagent.log.utils import (
|
||||
extract_evoid,
|
||||
extract_json,
|
||||
extract_loopid_func_name,
|
||||
is_valid_session,
|
||||
)
|
||||
from rdagent.utils import remove_ansi_codes
|
||||
from rdagent.utils.repo.diff import generate_diff_from_dict
|
||||
|
||||
@@ -30,40 +36,12 @@ if "log_folder" not in state:
|
||||
state.log_folder = Path("./log")
|
||||
|
||||
|
||||
def extract_loopid_func_name(tag):
|
||||
"""提取 Loop ID 和函数名称"""
|
||||
match = re.search(r"Loop_(\d+)\.([^.]+)", tag)
|
||||
return match.groups() if match else (None, None)
|
||||
|
||||
|
||||
def extract_evoid(tag):
|
||||
"""提取 EVO ID"""
|
||||
match = re.search(r"\.evo_loop_(\d+)\.", tag)
|
||||
return match.group(1) if match else None
|
||||
|
||||
|
||||
def convert_defaultdict_to_dict(d):
|
||||
if isinstance(d, defaultdict):
|
||||
d = {k: convert_defaultdict_to_dict(v) for k, v in d.items()}
|
||||
return d
|
||||
|
||||
|
||||
@st.cache_data(persist=True)
|
||||
def load_times(log_path: Path):
|
||||
"""加载时间数据"""
|
||||
try:
|
||||
session_path = log_path / "__session__"
|
||||
max_li = max(int(p.name) for p in session_path.iterdir() if p.is_dir() and p.name.isdigit())
|
||||
max_step = max(int(p.name.split("_")[0]) for p in (session_path / str(max_li)).iterdir() if p.is_file())
|
||||
rdloop_obj_p = next((session_path / str(max_li)).glob(f"{max_step}_*"))
|
||||
|
||||
rd_times = DataScienceRDLoop.load(rdloop_obj_p, do_truncate=False).loop_trace
|
||||
except Exception as e:
|
||||
# st.toast(f"Error loading times: {e}", icon="🟡")
|
||||
rd_times = {}
|
||||
return rd_times
|
||||
|
||||
|
||||
@st.cache_data(persist=True)
|
||||
def load_data(log_path: Path):
|
||||
data = defaultdict(lambda: defaultdict(dict))
|
||||
@@ -143,7 +121,6 @@ def load_data(log_path: Path):
|
||||
return convert_defaultdict_to_dict(data), convert_defaultdict_to_dict(llm_data)
|
||||
|
||||
|
||||
@st.cache_data
|
||||
def load_stdout(stdout_path: Path):
|
||||
if stdout_path.exists():
|
||||
stdout = stdout_path.read_text()
|
||||
@@ -499,7 +476,7 @@ def summarize_data():
|
||||
if "mle_score" not in state.data[loop]:
|
||||
if "mle_score" in loop_data["running"]:
|
||||
mle_score_txt = loop_data["running"]["mle_score"]
|
||||
state.data[loop]["mle_score"] = extract_mle_json(mle_score_txt)
|
||||
state.data[loop]["mle_score"] = extract_json(mle_score_txt)
|
||||
if (
|
||||
state.data[loop]["mle_score"] is not None
|
||||
and state.data[loop]["mle_score"]["score"] is not None
|
||||
@@ -515,7 +492,7 @@ def summarize_data():
|
||||
)
|
||||
try:
|
||||
mle_score_txt = mle_score_path.read_text()
|
||||
state.data[loop]["mle_score"] = extract_mle_json(mle_score_txt)
|
||||
state.data[loop]["mle_score"] = extract_json(mle_score_txt)
|
||||
if state.data[loop]["mle_score"]["score"] is not None:
|
||||
df.loc[loop, "Running Score (test)"] = str(state.data[loop]["mle_score"]["score"])
|
||||
else:
|
||||
|
||||
@@ -8,6 +8,8 @@ from pathlib import Path
|
||||
import streamlit as st
|
||||
from streamlit import session_state
|
||||
|
||||
from rdagent.log.utils import extract_evoid, extract_loopid_func_name
|
||||
|
||||
st.set_page_config(layout="wide", page_title="debug_llm", page_icon="🎓", initial_sidebar_state="expanded")
|
||||
|
||||
# 获取 log_path 参数
|
||||
@@ -90,18 +92,6 @@ def highlight_prompts_uri(uri):
|
||||
return f"**{parts[0]}:**:green[**{parts[1]}**]"
|
||||
|
||||
|
||||
def extract_loopid_func_name(tag):
|
||||
"""提取 Loop ID 和函数名称"""
|
||||
match = re.search(r"Loop_(\d+)\.(\w+)\.", tag)
|
||||
return match.groups() if match else (None, None)
|
||||
|
||||
|
||||
def extract_evoid(tag):
|
||||
"""提取 EVO ID"""
|
||||
match = re.search(r"\.evo_loop_(\d+)\.", tag)
|
||||
return match.group(1) if match else None
|
||||
|
||||
|
||||
# Display Data
|
||||
progress_text = st.empty()
|
||||
progress_bar = st.progress(0)
|
||||
|
||||
@@ -1,3 +1,19 @@
|
||||
import math
|
||||
import pickle
|
||||
import re
|
||||
from collections import deque
|
||||
from datetime import datetime, timedelta
|
||||
from pathlib import Path
|
||||
|
||||
import pandas as pd
|
||||
import typer
|
||||
|
||||
from rdagent.app.data_science.loop import DataScienceRDLoop
|
||||
from rdagent.core.utils import cache_with_pickle
|
||||
from rdagent.log.ui.conf import UI_SETTING
|
||||
from rdagent.log.utils import extract_json
|
||||
from rdagent.oai.llm_utils import md5_hash
|
||||
|
||||
LITE = [
|
||||
"aerial-cactus-identification",
|
||||
"aptos2019-blindness-detection",
|
||||
@@ -83,3 +99,446 @@ MEDIUM = [
|
||||
]
|
||||
|
||||
ALL = HIGH + MEDIUM + LITE
|
||||
|
||||
|
||||
def get_script_time(stdout_p: Path):
|
||||
with stdout_p.open("r") as f:
|
||||
first_line = next(f).strip()
|
||||
last_line = deque(f, maxlen=1).pop().strip()
|
||||
|
||||
# Extract timestamps from the lines
|
||||
first_time_match = re.search(r"(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\+\d{2}:\d{2})", first_line)
|
||||
last_time_match = re.search(r"(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\+\d{2}:\d{2})", last_line)
|
||||
|
||||
if first_time_match and last_time_match:
|
||||
first_time = datetime.fromisoformat(first_time_match.group(1))
|
||||
last_time = datetime.fromisoformat(last_time_match.group(1))
|
||||
return pd.Timedelta(last_time - first_time)
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def _log_path_hash_func(log_path: Path):
|
||||
hash_str = str(log_path) + str(log_path.stat().st_mtime)
|
||||
session_p = log_path / "__session__"
|
||||
if session_p.exists():
|
||||
for ld in session_p.iterdir():
|
||||
if ld.is_dir():
|
||||
hash_str += str(ld.name) + str(ld.stat().st_mtime)
|
||||
else:
|
||||
hash_str += "no session now"
|
||||
return md5_hash(hash_str)
|
||||
|
||||
|
||||
@cache_with_pickle(_log_path_hash_func, force=True)
|
||||
def get_final_sota_exp(log_path: Path):
|
||||
sota_exp_paths = [i for i in log_path.rglob(f"**/SOTA experiment/**/*.pkl")]
|
||||
if len(sota_exp_paths) == 0:
|
||||
return None
|
||||
final_sota_exp_path = max(sota_exp_paths, key=lambda x: int(re.match(r".*Loop_(\d+).*", str(x))[1]))
|
||||
with final_sota_exp_path.open("rb") as f:
|
||||
final_sota_exp = pickle.load(f)
|
||||
return final_sota_exp
|
||||
|
||||
|
||||
@cache_with_pickle(_log_path_hash_func, force=True)
|
||||
def get_sota_exp_stat(log_path: Path):
|
||||
trace_paths = [i for i in log_path.rglob(f"**/trace/**/*.pkl")]
|
||||
if len(trace_paths) == 0:
|
||||
return None
|
||||
final_trace_path = max(trace_paths, key=lambda x: int(re.match(r".*Loop_(\d+).*", str(x))[1]))
|
||||
with final_trace_path.open("rb") as f:
|
||||
final_trace = pickle.load(f)
|
||||
|
||||
if hasattr(final_trace, "sota_exp_to_submit"):
|
||||
sota_exp = final_trace.sota_exp_to_submit
|
||||
else:
|
||||
sota_exp = final_trace.sota_experiment()
|
||||
|
||||
if sota_exp is None:
|
||||
return None
|
||||
|
||||
sota_loop_id = None
|
||||
for i, ef in enumerate(final_trace.hist):
|
||||
if ef[0] == sota_exp:
|
||||
sota_loop_id = i
|
||||
break
|
||||
|
||||
sota_mle_score_paths = [i for i in log_path.rglob(f"Loop_{sota_loop_id}/running/mle_score/**/*.pkl")]
|
||||
if len(sota_mle_score_paths) == 0:
|
||||
# sota exp is not evaluated by mle_score
|
||||
return None
|
||||
with sota_mle_score_paths[0].open("rb") as f:
|
||||
sota_mle_score = extract_json(pickle.load(f))
|
||||
|
||||
sota_exp_stat = None
|
||||
if sota_mle_score: # sota exp's grade output
|
||||
if sota_mle_score["gold_medal"]:
|
||||
sota_exp_stat = "gold"
|
||||
elif sota_mle_score["silver_medal"]:
|
||||
sota_exp_stat = "silver"
|
||||
elif sota_mle_score["bronze_medal"]:
|
||||
sota_exp_stat = "bronze"
|
||||
elif sota_mle_score["above_median"]:
|
||||
sota_exp_stat = "above_median"
|
||||
elif sota_mle_score["valid_submission"]:
|
||||
sota_exp_stat = "valid_submission"
|
||||
elif sota_mle_score["submission_exists"]:
|
||||
sota_exp_stat = "made_submission"
|
||||
return sota_exp_stat
|
||||
|
||||
|
||||
@cache_with_pickle(_log_path_hash_func, force=True)
|
||||
def load_times(log_path: Path):
|
||||
try:
|
||||
session_path = log_path / "__session__"
|
||||
max_li = max(int(p.name) for p in session_path.iterdir() if p.is_dir() and p.name.isdigit())
|
||||
max_step = max(int(p.name.split("_")[0]) for p in (session_path / str(max_li)).iterdir() if p.is_file())
|
||||
rdloop_obj_p = next((session_path / str(max_li)).glob(f"{max_step}_*"))
|
||||
|
||||
rd_times = DataScienceRDLoop.load(rdloop_obj_p, do_truncate=False).loop_trace
|
||||
except Exception as e:
|
||||
rd_times = {}
|
||||
return rd_times
|
||||
|
||||
|
||||
def _log_folders_summary_hash_func(log_folders: list[str], hours: int | None = None):
|
||||
hash_str = ""
|
||||
for lf in log_folders:
|
||||
summary_p = Path(lf) / (f"summary.pkl" if hours is None else f"summary_{hours}h.pkl")
|
||||
if summary_p.exists():
|
||||
hash_str += str(summary_p) + str(summary_p.stat().st_mtime)
|
||||
else:
|
||||
hash_str += f"{summary_p} not exists"
|
||||
return md5_hash(hash_str)
|
||||
|
||||
|
||||
@cache_with_pickle(_log_folders_summary_hash_func, force=True)
|
||||
def get_summary_df(log_folders: list[str], hours: int | None = None) -> tuple[dict, pd.DataFrame]:
|
||||
summarys = {}
|
||||
if hours is None:
|
||||
sn = "summary.pkl"
|
||||
else:
|
||||
sn = f"summary_{hours}h.pkl"
|
||||
for lf in log_folders:
|
||||
if (Path(lf) / sn).exists():
|
||||
summarys[lf] = pd.read_pickle(Path(lf) / sn)
|
||||
|
||||
if len(summarys) == 0:
|
||||
return {}, pd.DataFrame()
|
||||
|
||||
summary = {}
|
||||
for lf, s in summarys.items():
|
||||
for k, v in s.items():
|
||||
stdout_p = Path(lf) / f"{k}.stdout"
|
||||
if stdout_p.exists():
|
||||
v["script_time"] = get_script_time(stdout_p)
|
||||
else:
|
||||
v["script_time"] = None
|
||||
|
||||
exp_gen_time = timedelta()
|
||||
coding_time = timedelta()
|
||||
running_time = timedelta()
|
||||
all_time = timedelta()
|
||||
times_info = load_times(Path(lf) / k)
|
||||
for time_info in times_info.values():
|
||||
all_time += sum((ti.end - ti.start for ti in time_info), timedelta())
|
||||
exp_gen_time += time_info[0].end - time_info[0].start
|
||||
if len(time_info) > 1:
|
||||
coding_time += time_info[1].end - time_info[1].start
|
||||
if len(time_info) > 2:
|
||||
running_time += time_info[2].end - time_info[2].start
|
||||
v["exec_time"] = str(all_time).split(".")[0]
|
||||
v["exp_gen_time"] = str(exp_gen_time).split(".")[0]
|
||||
v["coding_time"] = str(coding_time).split(".")[0]
|
||||
v["running_time"] = str(running_time).split(".")[0]
|
||||
|
||||
final_sota_exp = get_final_sota_exp(Path(lf) / k)
|
||||
if final_sota_exp is not None and final_sota_exp.result is not None:
|
||||
v["sota_exp_score_valid"] = final_sota_exp.result.loc["ensemble"].iloc[0]
|
||||
else:
|
||||
v["sota_exp_score_valid"] = None
|
||||
v["sota_exp_stat_new"] = get_sota_exp_stat(Path(lf) / k)
|
||||
# 调整实验名字
|
||||
if "amlt" in lf:
|
||||
summary[f"{lf[lf.rfind('amlt')+5:].split('/')[0]} - {k}"] = v
|
||||
elif "ep" in lf:
|
||||
summary[f"{lf[lf.rfind('ep'):]} - {k}"] = v
|
||||
else:
|
||||
summary[f"{lf} - {k}"] = v
|
||||
|
||||
summary = {k: v for k, v in summary.items() if "competition" in v}
|
||||
base_df = pd.DataFrame(
|
||||
columns=[
|
||||
"Competition",
|
||||
"Script Time",
|
||||
"Exec Time",
|
||||
"Exp Gen",
|
||||
"Coding",
|
||||
"Running",
|
||||
"Total Loops",
|
||||
"Successful Final Decision",
|
||||
"Made Submission",
|
||||
"Valid Submission",
|
||||
"V/M",
|
||||
"Above Median",
|
||||
"Bronze",
|
||||
"Silver",
|
||||
"Gold",
|
||||
"Any Medal",
|
||||
"Best Result",
|
||||
"SOTA Exp",
|
||||
"SOTA Exp (_to_submit)",
|
||||
"SOTA Exp Score (valid)",
|
||||
"SOTA Exp Score",
|
||||
"Baseline Score",
|
||||
"Ours - Base",
|
||||
"Ours vs Base",
|
||||
"Ours vs Bronze",
|
||||
"Ours vs Silver",
|
||||
"Ours vs Gold",
|
||||
"Bronze Threshold",
|
||||
"Silver Threshold",
|
||||
"Gold Threshold",
|
||||
"Medium Threshold",
|
||||
],
|
||||
index=summary.keys(),
|
||||
)
|
||||
|
||||
# Read baseline results
|
||||
baseline_result_path = UI_SETTING.baseline_result_path
|
||||
if Path(baseline_result_path).exists():
|
||||
baseline_df = pd.read_csv(baseline_result_path)
|
||||
|
||||
def compare_score(s1, s2):
|
||||
if s1 is None or s2 is None:
|
||||
return None
|
||||
try:
|
||||
c_value = math.exp(abs(math.log(s1 / s2)))
|
||||
except Exception as e:
|
||||
c_value = None
|
||||
return c_value
|
||||
|
||||
for k, v in summary.items():
|
||||
loop_num = v["loop_num"]
|
||||
base_df.loc[k, "Competition"] = v["competition"]
|
||||
base_df.loc[k, "Script Time"] = v["script_time"]
|
||||
base_df.loc[k, "Exec Time"] = v["exec_time"]
|
||||
base_df.loc[k, "Exp Gen"] = v["exp_gen_time"]
|
||||
base_df.loc[k, "Coding"] = v["coding_time"]
|
||||
base_df.loc[k, "Running"] = v["running_time"]
|
||||
base_df.loc[k, "Total Loops"] = loop_num
|
||||
if loop_num == 0:
|
||||
base_df.loc[k] = "N/A"
|
||||
else:
|
||||
base_df.loc[k, "Successful Final Decision"] = v["success_loop_num"]
|
||||
base_df.loc[k, "Made Submission"] = v["made_submission_num"]
|
||||
if v["made_submission_num"] > 0:
|
||||
base_df.loc[k, "Best Result"] = "made_submission"
|
||||
base_df.loc[k, "Valid Submission"] = v["valid_submission_num"]
|
||||
if v["valid_submission_num"] > 0:
|
||||
base_df.loc[k, "Best Result"] = "valid_submission"
|
||||
base_df.loc[k, "Above Median"] = v["above_median_num"]
|
||||
if v["above_median_num"] > 0:
|
||||
base_df.loc[k, "Best Result"] = "above_median"
|
||||
base_df.loc[k, "Bronze"] = v["bronze_num"]
|
||||
if v["bronze_num"] > 0:
|
||||
base_df.loc[k, "Best Result"] = "bronze"
|
||||
base_df.loc[k, "Silver"] = v["silver_num"]
|
||||
if v["silver_num"] > 0:
|
||||
base_df.loc[k, "Best Result"] = "silver"
|
||||
base_df.loc[k, "Gold"] = v["gold_num"]
|
||||
if v["gold_num"] > 0:
|
||||
base_df.loc[k, "Best Result"] = "gold"
|
||||
base_df.loc[k, "Any Medal"] = v["get_medal_num"]
|
||||
|
||||
baseline_score = None
|
||||
if Path(baseline_result_path).exists():
|
||||
baseline_score = baseline_df.loc[baseline_df["competition_id"] == v["competition"], "score"].item()
|
||||
|
||||
base_df.loc[k, "SOTA Exp"] = v.get("sota_exp_stat", None)
|
||||
base_df.loc[k, "SOTA Exp (_to_submit)"] = v["sota_exp_stat_new"]
|
||||
if baseline_score is not None and v.get("sota_exp_score", None) is not None:
|
||||
base_df.loc[k, "Ours - Base"] = v["sota_exp_score"] - baseline_score
|
||||
base_df.loc[k, "Ours vs Base"] = compare_score(v["sota_exp_score"], baseline_score)
|
||||
base_df.loc[k, "Ours vs Bronze"] = compare_score(v["sota_exp_score"], v.get("bronze_threshold", None))
|
||||
base_df.loc[k, "Ours vs Silver"] = compare_score(v["sota_exp_score"], v.get("silver_threshold", None))
|
||||
base_df.loc[k, "Ours vs Gold"] = compare_score(v["sota_exp_score"], v.get("gold_threshold", None))
|
||||
base_df.loc[k, "SOTA Exp Score"] = v.get("sota_exp_score", None)
|
||||
base_df.loc[k, "SOTA Exp Score (valid)"] = v.get("sota_exp_score_valid", None)
|
||||
base_df.loc[k, "Baseline Score"] = baseline_score
|
||||
base_df.loc[k, "Bronze Threshold"] = v.get("bronze_threshold", None)
|
||||
base_df.loc[k, "Silver Threshold"] = v.get("silver_threshold", None)
|
||||
base_df.loc[k, "Gold Threshold"] = v.get("gold_threshold", None)
|
||||
base_df.loc[k, "Medium Threshold"] = v.get("median_threshold", None)
|
||||
|
||||
base_df["SOTA Exp"] = base_df["SOTA Exp"].replace("", pd.NA)
|
||||
|
||||
base_df.loc[base_df["SOTA Exp Score (valid)"].apply(lambda x: isinstance(x, str)), "SOTA Exp Score (valid)"] = 0.0
|
||||
base_df = base_df.astype(
|
||||
{
|
||||
"Total Loops": int,
|
||||
"Successful Final Decision": int,
|
||||
"Made Submission": int,
|
||||
"Valid Submission": int,
|
||||
"Above Median": int,
|
||||
"Bronze": int,
|
||||
"Silver": int,
|
||||
"Gold": int,
|
||||
"Any Medal": int,
|
||||
"Ours - Base": float,
|
||||
"Ours vs Base": float,
|
||||
"SOTA Exp Score": float,
|
||||
"SOTA Exp Score (valid)": float,
|
||||
"Baseline Score": float,
|
||||
"Bronze Threshold": float,
|
||||
"Silver Threshold": float,
|
||||
"Gold Threshold": float,
|
||||
"Medium Threshold": float,
|
||||
}
|
||||
)
|
||||
return summary, base_df
|
||||
|
||||
|
||||
def percent_df(summary_df: pd.DataFrame, show_origin=True) -> pd.DataFrame:
|
||||
"""
|
||||
Convert the summary DataFrame to a percentage format.
|
||||
"""
|
||||
new_df = summary_df.copy(deep=True)
|
||||
|
||||
# Convert columns to object dtype so we can store strings like "14 (53.85%)" without warnings
|
||||
columns_to_convert = [
|
||||
"Successful Final Decision",
|
||||
"Made Submission",
|
||||
"Valid Submission",
|
||||
"Above Median",
|
||||
"Bronze",
|
||||
"Silver",
|
||||
"Gold",
|
||||
"Any Medal",
|
||||
]
|
||||
new_df[columns_to_convert] = new_df[columns_to_convert].astype(object)
|
||||
|
||||
def num2percent(num: int, total: int, show_origin=True) -> str:
|
||||
num = int(num)
|
||||
total = int(total)
|
||||
if show_origin:
|
||||
return f"{num} ({round(num / total * 100, 2)}%)"
|
||||
return f"{round(num / total * 100, 2)}%"
|
||||
|
||||
for k in new_df.index:
|
||||
loop_num = int(new_df.loc[k, "Total Loops"])
|
||||
if loop_num != 0:
|
||||
new_df.loc[k, "Successful Final Decision"] = num2percent(
|
||||
new_df.loc[k, "Successful Final Decision"], loop_num, show_origin
|
||||
)
|
||||
if new_df.loc[k, "Made Submission"] != 0:
|
||||
new_df.loc[k, "V/M"] = (
|
||||
f"{round(new_df.loc[k, 'Valid Submission'] / new_df.loc[k, 'Made Submission'] * 100, 2)}%"
|
||||
)
|
||||
else:
|
||||
new_df.loc[k, "V/M"] = "N/A"
|
||||
new_df.loc[k, "Made Submission"] = num2percent(new_df.loc[k, "Made Submission"], loop_num, show_origin)
|
||||
new_df.loc[k, "Valid Submission"] = num2percent(new_df.loc[k, "Valid Submission"], loop_num, show_origin)
|
||||
new_df.loc[k, "Above Median"] = num2percent(new_df.loc[k, "Above Median"], loop_num, show_origin)
|
||||
new_df.loc[k, "Bronze"] = num2percent(new_df.loc[k, "Bronze"], loop_num, show_origin)
|
||||
new_df.loc[k, "Silver"] = num2percent(new_df.loc[k, "Silver"], loop_num, show_origin)
|
||||
new_df.loc[k, "Gold"] = num2percent(new_df.loc[k, "Gold"], loop_num, show_origin)
|
||||
new_df.loc[k, "Any Medal"] = num2percent(new_df.loc[k, "Any Medal"], loop_num, show_origin)
|
||||
|
||||
return new_df
|
||||
|
||||
|
||||
def get_statistics_df(summary_df: pd.DataFrame) -> pd.DataFrame:
|
||||
if summary_df["Any Medal"].dtype == int:
|
||||
check_value = 0
|
||||
else:
|
||||
sample_val = summary_df["Any Medal"].dropna().iloc[0]
|
||||
if "(" in sample_val:
|
||||
check_value = "0 (0.0%)"
|
||||
else:
|
||||
check_value = "0.0%"
|
||||
total_stat = (
|
||||
summary_df[
|
||||
[
|
||||
"Made Submission",
|
||||
"Valid Submission",
|
||||
"Above Median",
|
||||
"Bronze",
|
||||
"Silver",
|
||||
"Gold",
|
||||
"Any Medal",
|
||||
]
|
||||
]
|
||||
!= check_value
|
||||
).sum()
|
||||
total_stat.name = "总体统计(%)"
|
||||
total_stat.loc["Bronze"] = summary_df["Best Result"].value_counts().get("bronze", 0)
|
||||
total_stat.loc["Silver"] = summary_df["Best Result"].value_counts().get("silver", 0)
|
||||
total_stat.loc["Gold"] = summary_df["Best Result"].value_counts().get("gold", 0)
|
||||
total_stat = total_stat / summary_df.shape[0] * 100
|
||||
|
||||
# SOTA Exp 统计
|
||||
se_counts = summary_df["SOTA Exp"].value_counts(dropna=True)
|
||||
se_counts.loc["made_submission"] = se_counts.sum()
|
||||
se_counts.loc["Any Medal"] = se_counts.get("gold", 0) + se_counts.get("silver", 0) + se_counts.get("bronze", 0)
|
||||
se_counts.loc["above_median"] = se_counts.get("above_median", 0) + se_counts.get("Any Medal", 0)
|
||||
se_counts.loc["valid_submission"] = se_counts.get("valid_submission", 0) + se_counts.get("above_median", 0)
|
||||
|
||||
sota_exp_stat = pd.Series(index=total_stat.index, dtype=int, name="SOTA Exp 统计(%)")
|
||||
sota_exp_stat.loc["Made Submission"] = se_counts.get("made_submission", 0)
|
||||
sota_exp_stat.loc["Valid Submission"] = se_counts.get("valid_submission", 0)
|
||||
sota_exp_stat.loc["Above Median"] = se_counts.get("above_median", 0)
|
||||
sota_exp_stat.loc["Bronze"] = se_counts.get("bronze", 0)
|
||||
sota_exp_stat.loc["Silver"] = se_counts.get("silver", 0)
|
||||
sota_exp_stat.loc["Gold"] = se_counts.get("gold", 0)
|
||||
sota_exp_stat.loc["Any Medal"] = se_counts.get("Any Medal", 0)
|
||||
sota_exp_stat = sota_exp_stat / summary_df.shape[0] * 100
|
||||
|
||||
# SOTA Exp (trace.sota_exp_to_submit) 统计
|
||||
se_counts_new = summary_df["SOTA Exp (_to_submit)"].value_counts(dropna=True)
|
||||
se_counts_new.loc["made_submission"] = se_counts_new.sum()
|
||||
se_counts_new.loc["Any Medal"] = (
|
||||
se_counts_new.get("gold", 0) + se_counts_new.get("silver", 0) + se_counts_new.get("bronze", 0)
|
||||
)
|
||||
se_counts_new.loc["above_median"] = se_counts_new.get("above_median", 0) + se_counts_new.get("Any Medal", 0)
|
||||
se_counts_new.loc["valid_submission"] = se_counts_new.get("valid_submission", 0) + se_counts_new.get(
|
||||
"above_median", 0
|
||||
)
|
||||
|
||||
sota_exp_stat_new = pd.Series(index=total_stat.index, dtype=int, name="SOTA Exp (_to_submit) 统计(%)")
|
||||
sota_exp_stat_new.loc["Made Submission"] = se_counts_new.get("made_submission", 0)
|
||||
sota_exp_stat_new.loc["Valid Submission"] = se_counts_new.get("valid_submission", 0)
|
||||
sota_exp_stat_new.loc["Above Median"] = se_counts_new.get("above_median", 0)
|
||||
sota_exp_stat_new.loc["Bronze"] = se_counts_new.get("bronze", 0)
|
||||
sota_exp_stat_new.loc["Silver"] = se_counts_new.get("silver", 0)
|
||||
sota_exp_stat_new.loc["Gold"] = se_counts_new.get("gold", 0)
|
||||
sota_exp_stat_new.loc["Any Medal"] = se_counts_new.get("Any Medal", 0)
|
||||
sota_exp_stat_new = sota_exp_stat_new / summary_df.shape[0] * 100
|
||||
|
||||
stat_df = pd.concat([total_stat, sota_exp_stat, sota_exp_stat_new], axis=1)
|
||||
return stat_df
|
||||
|
||||
|
||||
def compare(
|
||||
exp_list: list[str] = typer.Option(..., "--exp-list", help="List of experiment names.", show_default=False),
|
||||
output: str = typer.Option("merge_base_df.h5", help="Output summary file name."),
|
||||
hours: int | None = typer.Option(None, help="if None, use summary.pkl, else summary_{hours}h.pkl"),
|
||||
):
|
||||
"""
|
||||
Generate summary and base dataframe for given experiment list, and save to a summary file.
|
||||
"""
|
||||
typer.secho(f"exp_list: {exp_list}", fg=typer.colors.GREEN)
|
||||
log_folders = [f"{UI_SETTING.amlt_path}/{exp}/combined_logs" for exp in exp_list]
|
||||
summary, base_df = get_summary_df(log_folders, hours=hours)
|
||||
typer.secho(f"Summary keys: {list(summary.keys())}", fg=typer.colors.CYAN)
|
||||
typer.secho("Summary DataFrame:", fg=typer.colors.MAGENTA)
|
||||
typer.secho(str(base_df), fg=typer.colors.YELLOW)
|
||||
base_df.to_hdf(output, "data")
|
||||
typer.secho(f"Summary saved to {output}", fg=typer.colors.GREEN)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
app = typer.Typer()
|
||||
app.command()(compare)
|
||||
app()
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
import inspect
|
||||
import json
|
||||
import re
|
||||
from typing import Dict, Optional, TypedDict, Union
|
||||
from pathlib import Path
|
||||
from typing import Optional, TypedDict, cast
|
||||
|
||||
|
||||
class LogColors:
|
||||
@@ -75,3 +77,26 @@ def get_caller_info() -> CallerInfo:
|
||||
"function": frame.f_code.co_name, # Get the caller's function name
|
||||
}
|
||||
return info
|
||||
|
||||
|
||||
def is_valid_session(log_path: Path) -> bool:
|
||||
return log_path.is_dir() and log_path.joinpath("__session__").exists()
|
||||
|
||||
|
||||
def extract_loopid_func_name(tag: str) -> tuple[str, str] | tuple[None, None]:
|
||||
"""extract loop id and function name from the tag in Message"""
|
||||
match = re.search(r"Loop_(\d+)\.([^.]+)", tag)
|
||||
return cast(tuple[str, str], match.groups()) if match else (None, None)
|
||||
|
||||
|
||||
def extract_evoid(tag: str) -> str | None:
|
||||
"""extract evo id from the tag in Message"""
|
||||
match = re.search(r"\.evo_loop_(\d+)\.", tag)
|
||||
return cast(str, match.group(1)) if match else None
|
||||
|
||||
|
||||
def extract_json(log_content: str) -> dict | None:
|
||||
match = re.search(r"\{.*\}", log_content, re.DOTALL)
|
||||
if match:
|
||||
return cast(dict, json.loads(match.group(0)))
|
||||
return None
|
||||
|
||||
@@ -3,6 +3,7 @@ This module provides some useful functions for working with logger folders.
|
||||
"""
|
||||
|
||||
import pickle
|
||||
from datetime import timedelta
|
||||
from pathlib import Path
|
||||
|
||||
import pandas as pd
|
||||
|
||||
@@ -1,6 +1,4 @@
|
||||
import json
|
||||
import os
|
||||
import platform
|
||||
from pathlib import Path
|
||||
from typing import Dict
|
||||
|
||||
@@ -18,7 +16,7 @@ from rdagent.scenarios.data_science.scen.utils import (
|
||||
from rdagent.scenarios.kaggle.kaggle_crawler import (
|
||||
crawl_descriptions,
|
||||
download_data,
|
||||
leaderboard_scores,
|
||||
get_metric_direction,
|
||||
)
|
||||
from rdagent.utils.agent.tpl import T
|
||||
|
||||
@@ -188,8 +186,7 @@ class KaggleScen(DataScienceScen):
|
||||
return crawl_descriptions(self.competition, DS_RD_SETTING.local_data_path)
|
||||
|
||||
def _get_direction(self):
|
||||
leaderboard = leaderboard_scores(self.competition)
|
||||
return float(leaderboard[0]) > float(leaderboard[-1])
|
||||
return get_metric_direction(self.competition)
|
||||
|
||||
@property
|
||||
def rich_style_description(self) -> str:
|
||||
|
||||
@@ -198,6 +198,14 @@ def leaderboard_scores(competition: str) -> list[float]:
|
||||
return [float(x.score) for x in ll]
|
||||
|
||||
|
||||
def get_metric_direction(competition: str) -> bool:
|
||||
"""
|
||||
Return **True** if the metric is *bigger is better*, **False** if *smaller is better*.
|
||||
"""
|
||||
leaderboard = leaderboard_scores(competition)
|
||||
return float(leaderboard[0]) > float(leaderboard[-1])
|
||||
|
||||
|
||||
def score_rank(competition: str, score: float) -> tuple[int, float]:
|
||||
"""
|
||||
Return
|
||||
|
||||
Reference in New Issue
Block a user