diff --git a/rdagent/log/mle_summary.py b/rdagent/log/mle_summary.py index d8b1fcd6..b29e186e 100644 --- a/rdagent/log/mle_summary.py +++ b/rdagent/log/mle_summary.py @@ -2,18 +2,19 @@ import json import pickle import re from collections import defaultdict -from datetime import timedelta from pathlib import Path import fire import pandas as pd -from rdagent.app.data_science.conf import DS_RD_SETTING -from rdagent.app.data_science.loop import DataScienceRDLoop -from rdagent.components.coder.data_science.conf import get_ds_env from rdagent.core.experiment import FBWorkspace from rdagent.core.proposal import ExperimentFeedback from rdagent.log.storage import FileStorage +from rdagent.log.utils import ( + extract_json, + extract_loopid_func_name, + is_valid_session, +) from rdagent.log.utils.folder import get_first_session_file_after_duration from rdagent.scenarios.data_science.experiment.experiment import DSExperiment from rdagent.scenarios.data_science.test_eval import ( @@ -29,19 +30,6 @@ test_eval = get_test_eval() is_mle = isinstance(test_eval, MLETestEval) -def extract_mle_json(log_content: str) -> dict | None: - match = re.search(r"\{.*\}", log_content, re.DOTALL) - if match: - return json.loads(match.group(0)) - return None - - -def extract_loopid_func_name(tag): - """提取 Loop ID 和函数名称""" - match = re.search(r"Loop_(\d+)\.([^.]+)", tag) - return match.groups() if match else (None, None) - - def save_grade_info(log_trace_path: Path): trace_storage = FileStorage(log_trace_path) for msg in trace_storage.iter_msg(): @@ -61,10 +49,6 @@ def save_grade_info(log_trace_path: Path): print(f"Error in {log_trace_path}: {e}") -def is_valid_session(p: Path) -> bool: - return p.is_dir() and p.joinpath("__session__").exists() - - def save_all_grade_info(log_folder): for log_trace_path in log_folder.iterdir(): if is_valid_session(log_trace_path): @@ -149,7 +133,7 @@ def summarize_folder(log_folder: Path, hours: int | None = None): env=test_eval.env, entry=f"mlebench grade-sample None {stat[log_trace_path.name]['competition']} --data-dir /mle/data", ) - grade_output = extract_mle_json(stdout) + grade_output = extract_json(stdout) if grade_output: bronze_threshold = grade_output["bronze_threshold"] silver_threshold = grade_output["silver_threshold"] @@ -164,7 +148,7 @@ def summarize_folder(log_folder: Path, hours: int | None = None): if msg.content.result is not None: valid_scores[loop_id] = msg.content.result elif "mle_score" in msg.tag: - grade_output = extract_mle_json(msg.content) + grade_output = extract_json(msg.content) if grade_output: if grade_output["submission_exists"]: made_submission_num += 1 diff --git a/rdagent/log/ui/conf.py b/rdagent/log/ui/conf.py index 8700cf50..c948d6e4 100644 --- a/rdagent/log/ui/conf.py +++ b/rdagent/log/ui/conf.py @@ -12,5 +12,7 @@ class UIBasePropSetting(ExtendedBaseSettings): aide_path: str = "./aide" + amlt_path: str = "/data/share_folder_local/amlt" + UI_SETTING = UIBasePropSetting() diff --git a/rdagent/log/ui/ds_summary.py b/rdagent/log/ui/ds_summary.py index 23a5f96a..3b03b117 100755 --- a/rdagent/log/ui/ds_summary.py +++ b/rdagent/log/ui/ds_summary.py @@ -1,8 +1,4 @@ -import math -import pickle import re -from collections import deque -from datetime import datetime, timedelta from pathlib import Path import pandas as pd @@ -11,322 +7,16 @@ import plotly.graph_objects as go import streamlit as st from streamlit import session_state as state -from rdagent.log.mle_summary import extract_mle_json -from rdagent.log.ui.conf import UI_SETTING -from rdagent.log.ui.ds_trace import load_times -from rdagent.log.ui.utils import ALL, HIGH, LITE, MEDIUM -from rdagent.scenarios.kaggle.kaggle_crawler import leaderboard_scores - - -def get_metric_direction(competition: str): - leaderboard = leaderboard_scores(competition) - return float(leaderboard[0]) > float(leaderboard[-1]) - - -def get_script_time(stdout_p: Path): - with stdout_p.open("r") as f: - first_line = next(f).strip() - last_line = deque(f, maxlen=1).pop().strip() - - # Extract timestamps from the lines - first_time_match = re.search(r"(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\+\d{2}:\d{2})", first_line) - last_time_match = re.search(r"(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\+\d{2}:\d{2})", last_line) - - if first_time_match and last_time_match: - first_time = datetime.fromisoformat(first_time_match.group(1)) - last_time = datetime.fromisoformat(last_time_match.group(1)) - return pd.Timedelta(last_time - first_time) - - return None - - -@st.cache_data(persist=True) -def get_final_sota_exp(log_path: Path): - sota_exp_paths = [i for i in log_path.rglob(f"**/SOTA experiment/**/*.pkl")] - if len(sota_exp_paths) == 0: - return None - final_sota_exp_path = max(sota_exp_paths, key=lambda x: int(re.match(r".*Loop_(\d+).*", str(x))[1])) - with final_sota_exp_path.open("rb") as f: - final_sota_exp = pickle.load(f) - return final_sota_exp - - -@st.cache_data(persist=True) -def get_sota_exp_stat(log_path: Path): - trace_paths = [i for i in log_path.rglob(f"**/trace/**/*.pkl")] - if len(trace_paths) == 0: - return None - final_trace_path = max(trace_paths, key=lambda x: int(re.match(r".*Loop_(\d+).*", str(x))[1])) - with final_trace_path.open("rb") as f: - final_trace = pickle.load(f) - - if hasattr(final_trace, "sota_exp_to_submit"): - sota_exp = final_trace.sota_exp_to_submit - else: - sota_exp = final_trace.sota_experiment() - - if sota_exp is None: - return None - - sota_loop_id = None - for i, ef in enumerate(final_trace.hist): - if ef[0] == sota_exp: - sota_loop_id = i - break - - sota_mle_score_paths = [i for i in log_path.rglob(f"Loop_{sota_loop_id}/running/mle_score/**/*.pkl")] - if len(sota_mle_score_paths) == 0: - # sota exp is not evaluated by mle_score - return None - with sota_mle_score_paths[0].open("rb") as f: - sota_mle_score = extract_mle_json(pickle.load(f)) - - sota_exp_stat = None - if sota_mle_score: # sota exp's grade output - if sota_mle_score["gold_medal"]: - sota_exp_stat = "gold" - elif sota_mle_score["silver_medal"]: - sota_exp_stat = "silver" - elif sota_mle_score["bronze_medal"]: - sota_exp_stat = "bronze" - elif sota_mle_score["above_median"]: - sota_exp_stat = "above_median" - elif sota_mle_score["valid_submission"]: - sota_exp_stat = "valid_submission" - elif sota_mle_score["submission_exists"]: - sota_exp_stat = "made_submission" - return sota_exp_stat - - -@st.cache_data(persist=True) -def get_summary_df(log_folders: list[str]) -> tuple[dict, pd.DataFrame]: - summarys = {} - sn = "summary.pkl" - for lf in log_folders: - if (Path(lf) / sn).exists(): - summarys[lf] = pd.read_pickle(Path(lf) / sn) - - if len(summarys) == 0: - return {}, pd.DataFrame() - - summary = {} - for lf, s in summarys.items(): - for k, v in s.items(): - stdout_p = Path(lf) / f"{k}.stdout" - if stdout_p.exists(): - v["script_time"] = get_script_time(stdout_p) - else: - v["script_time"] = None - - exp_gen_time = timedelta() - coding_time = timedelta() - running_time = timedelta() - all_time = timedelta() - times_info = load_times(Path(lf) / k) - for time_info in times_info.values(): - all_time += sum((ti.end - ti.start for ti in time_info), timedelta()) - exp_gen_time += time_info[0].end - time_info[0].start - if len(time_info) > 1: - coding_time += time_info[1].end - time_info[1].start - if len(time_info) > 2: - running_time += time_info[2].end - time_info[2].start - v["exec_time"] = str(all_time).split(".")[0] - v["exp_gen_time"] = str(exp_gen_time).split(".")[0] - v["coding_time"] = str(coding_time).split(".")[0] - v["running_time"] = str(running_time).split(".")[0] - - final_sota_exp = get_final_sota_exp(Path(lf) / k) - if final_sota_exp is not None and final_sota_exp.result is not None: - v["sota_exp_score_valid"] = final_sota_exp.result.loc["ensemble"].iloc[0] - else: - v["sota_exp_score_valid"] = None - v["sota_exp_stat_new"] = get_sota_exp_stat(Path(lf) / k) - # 调整实验名字 - if "amlt" in lf: - summary[f"{lf[lf.rfind('amlt')+5:].split('/')[0]} - {k}"] = v - elif "ep" in lf: - summary[f"{lf[lf.rfind('ep'):]} - {k}"] = v - else: - summary[f"{lf} - {k}"] = v - - summary = {k: v for k, v in summary.items() if "competition" in v} - base_df = pd.DataFrame( - columns=[ - "Competition", - "Script Time", - "Exec Time", - "Exp Gen", - "Coding", - "Running", - "Total Loops", - "Successful Final Decision", - "Made Submission", - "Valid Submission", - "V/M", - "Above Median", - "Bronze", - "Silver", - "Gold", - "Any Medal", - "Best Result", - "SOTA Exp", - "SOTA Exp (_to_submit)", - "SOTA Exp Score (valid)", - "SOTA Exp Score", - "Baseline Score", - "Ours - Base", - "Ours vs Base", - "Ours vs Bronze", - "Ours vs Silver", - "Ours vs Gold", - "Bronze Threshold", - "Silver Threshold", - "Gold Threshold", - "Medium Threshold", - ], - index=summary.keys(), - ) - - # Read baseline results - baseline_result_path = UI_SETTING.baseline_result_path - if Path(baseline_result_path).exists(): - baseline_df = pd.read_csv(baseline_result_path) - - def compare_score(s1, s2): - if s1 is None or s2 is None: - return None - try: - c_value = math.exp(abs(math.log(s1 / s2))) - except Exception as e: - c_value = None - return c_value - - for k, v in summary.items(): - loop_num = v["loop_num"] - base_df.loc[k, "Competition"] = v["competition"] - base_df.loc[k, "Script Time"] = v["script_time"] - base_df.loc[k, "Exec Time"] = v["exec_time"] - base_df.loc[k, "Exp Gen"] = v["exp_gen_time"] - base_df.loc[k, "Coding"] = v["coding_time"] - base_df.loc[k, "Running"] = v["running_time"] - base_df.loc[k, "Total Loops"] = loop_num - if loop_num == 0: - base_df.loc[k] = "N/A" - else: - base_df.loc[k, "Successful Final Decision"] = v["success_loop_num"] - base_df.loc[k, "Made Submission"] = v["made_submission_num"] - if v["made_submission_num"] > 0: - base_df.loc[k, "Best Result"] = "made_submission" - base_df.loc[k, "Valid Submission"] = v["valid_submission_num"] - if v["valid_submission_num"] > 0: - base_df.loc[k, "Best Result"] = "valid_submission" - base_df.loc[k, "Above Median"] = v["above_median_num"] - if v["above_median_num"] > 0: - base_df.loc[k, "Best Result"] = "above_median" - base_df.loc[k, "Bronze"] = v["bronze_num"] - if v["bronze_num"] > 0: - base_df.loc[k, "Best Result"] = "bronze" - base_df.loc[k, "Silver"] = v["silver_num"] - if v["silver_num"] > 0: - base_df.loc[k, "Best Result"] = "silver" - base_df.loc[k, "Gold"] = v["gold_num"] - if v["gold_num"] > 0: - base_df.loc[k, "Best Result"] = "gold" - base_df.loc[k, "Any Medal"] = v["get_medal_num"] - - baseline_score = None - if Path(baseline_result_path).exists(): - baseline_score = baseline_df.loc[baseline_df["competition_id"] == v["competition"], "score"].item() - - base_df.loc[k, "SOTA Exp"] = v.get("sota_exp_stat", None) - base_df.loc[k, "SOTA Exp (_to_submit)"] = v["sota_exp_stat_new"] - if baseline_score is not None and v.get("sota_exp_score", None) is not None: - base_df.loc[k, "Ours - Base"] = v["sota_exp_score"] - baseline_score - base_df.loc[k, "Ours vs Base"] = compare_score(v["sota_exp_score"], baseline_score) - base_df.loc[k, "Ours vs Bronze"] = compare_score(v["sota_exp_score"], v.get("bronze_threshold", None)) - base_df.loc[k, "Ours vs Silver"] = compare_score(v["sota_exp_score"], v.get("silver_threshold", None)) - base_df.loc[k, "Ours vs Gold"] = compare_score(v["sota_exp_score"], v.get("gold_threshold", None)) - base_df.loc[k, "SOTA Exp Score"] = v.get("sota_exp_score", None) - base_df.loc[k, "SOTA Exp Score (valid)"] = v.get("sota_exp_score_valid", None) - base_df.loc[k, "Baseline Score"] = baseline_score - base_df.loc[k, "Bronze Threshold"] = v.get("bronze_threshold", None) - base_df.loc[k, "Silver Threshold"] = v.get("silver_threshold", None) - base_df.loc[k, "Gold Threshold"] = v.get("gold_threshold", None) - base_df.loc[k, "Medium Threshold"] = v.get("median_threshold", None) - - base_df["SOTA Exp"] = base_df["SOTA Exp"].replace("", pd.NA) - - base_df.loc[base_df["SOTA Exp Score (valid)"].apply(lambda x: isinstance(x, str)), "SOTA Exp Score (valid)"] = 0.0 - base_df = base_df.astype( - { - "Total Loops": int, - "Successful Final Decision": int, - "Made Submission": int, - "Valid Submission": int, - "Above Median": int, - "Bronze": int, - "Silver": int, - "Gold": int, - "Any Medal": int, - "Ours - Base": float, - "Ours vs Base": float, - "SOTA Exp Score": float, - "SOTA Exp Score (valid)": float, - "Baseline Score": float, - "Bronze Threshold": float, - "Silver Threshold": float, - "Gold Threshold": float, - "Medium Threshold": float, - } - ) - return summary, base_df - - -def num2percent(num: int, total: int, show_origin=True) -> str: - num = int(num) - total = int(total) - if show_origin: - return f"{num} ({round(num / total * 100, 2)}%)" - return f"{round(num / total * 100, 2)}%" - - -def percent_df(df: pd.DataFrame, show_origin=True) -> pd.DataFrame: - base_df = df.copy(deep=True) - - # Convert columns to object dtype so we can store strings like "14 (53.85%)" without warnings - columns_to_convert = [ - "Successful Final Decision", - "Made Submission", - "Valid Submission", - "Above Median", - "Bronze", - "Silver", - "Gold", - "Any Medal", - ] - base_df[columns_to_convert] = base_df[columns_to_convert].astype(object) - - for k in base_df.index: - loop_num = int(base_df.loc[k, "Total Loops"]) - if loop_num != 0: - base_df.loc[k, "Successful Final Decision"] = num2percent( - base_df.loc[k, "Successful Final Decision"], loop_num, show_origin - ) - if base_df.loc[k, "Made Submission"] != 0: - base_df.loc[k, "V/M"] = ( - f"{round(base_df.loc[k, 'Valid Submission'] / base_df.loc[k, 'Made Submission'] * 100, 2)}%" - ) - else: - base_df.loc[k, "V/M"] = "N/A" - base_df.loc[k, "Made Submission"] = num2percent(base_df.loc[k, "Made Submission"], loop_num, show_origin) - base_df.loc[k, "Valid Submission"] = num2percent(base_df.loc[k, "Valid Submission"], loop_num, show_origin) - base_df.loc[k, "Above Median"] = num2percent(base_df.loc[k, "Above Median"], loop_num, show_origin) - base_df.loc[k, "Bronze"] = num2percent(base_df.loc[k, "Bronze"], loop_num, show_origin) - base_df.loc[k, "Silver"] = num2percent(base_df.loc[k, "Silver"], loop_num, show_origin) - base_df.loc[k, "Gold"] = num2percent(base_df.loc[k, "Gold"], loop_num, show_origin) - base_df.loc[k, "Any Medal"] = num2percent(base_df.loc[k, "Any Medal"], loop_num, show_origin) - - return base_df +from rdagent.log.ui.utils import ( + ALL, + HIGH, + LITE, + MEDIUM, + get_statistics_df, + get_summary_df, + percent_df, +) +from rdagent.scenarios.kaggle.kaggle_crawler import get_metric_direction def days_summarize_win(): @@ -352,6 +42,68 @@ def days_summarize_win(): st.dataframe(df) +def curves_win(summary: dict): + for k, v in summary.items(): + with st.container(border=True): + st.markdown(f"**:blue[{k}] - :violet[{v['competition']}]**") + try: + tscores = {f"loop {k-1}": v for k, v in v["test_scores"].items()} + vscores = {} + for k, vs in v["valid_scores"].items(): + if not vs.index.is_unique: + st.warning( + f"Loop {k}'s valid scores index are not unique, only the last one will be kept to show." + ) + st.write(vs) + vscores[k] = vs[~vs.index.duplicated(keep="last")].iloc[:, 0] + + if len(vscores) > 0: + metric_name = list(vscores.values())[0].name + else: + metric_name = "None" + + tdf = pd.Series(tscores, name="score") + vdf = pd.DataFrame(vscores) + if "ensemble" in vdf.index: + ensemble_row = vdf.loc[["ensemble"]] + vdf = pd.concat([ensemble_row, vdf.drop("ensemble")]) + vdf.columns = [f"loop {i}" for i in vdf.columns] + fig = go.Figure() + # Add test scores trace from tdf + fig.add_trace( + go.Scatter( + x=tdf.index, + y=tdf, + mode="lines+markers", + name="Test scores", + marker=dict(symbol="diamond"), + line=dict(shape="linear", dash="dash"), + ) + ) + # Add valid score traces from vdf (transposed to have loops on x-axis) + for column in vdf.T.columns: + fig.add_trace( + go.Scatter( + x=vdf.T.index, + y=vdf.T[column], + mode="lines+markers", + name=f"{column}", + visible=("legendonly" if column != "ensemble" else None), + ) + ) + fig.update_layout(title=f"Test and Valid scores (metric: {metric_name})") + + st.plotly_chart(fig) + except Exception as e: + import traceback + + st.markdown("- Error: " + str(e)) + st.code(traceback.format_exc()) + st.markdown("- Valid Scores: ") + # st.write({k: type(v) for k, v in v["valid_scores"].items()}) + st.json(v["valid_scores"]) + + def all_summarize_win(): def shorten_folder_name(folder: str) -> str: if "amlt" in folder: @@ -460,73 +212,15 @@ def all_summarize_win(): # 统计选择的比赛 base_df = base_df[base_df["Select"]] st.markdown(f"**统计的比赛数目: :red[{base_df.shape[0]}]**") - total_stat = ( - base_df[ - [ - "Made Submission", - "Valid Submission", - "Above Median", - "Bronze", - "Silver", - "Gold", - "Any Medal", - ] - ] - != "0 (0.0%)" - ).sum() - total_stat.name = "总体统计(%)" - total_stat.loc["Bronze"] = base_df["Best Result"].value_counts().get("bronze", 0) - total_stat.loc["Silver"] = base_df["Best Result"].value_counts().get("silver", 0) - total_stat.loc["Gold"] = base_df["Best Result"].value_counts().get("gold", 0) - total_stat = total_stat / base_df.shape[0] * 100 - - # SOTA Exp 统计 - se_counts = base_df["SOTA Exp"].value_counts(dropna=True) - se_counts.loc["made_submission"] = se_counts.sum() - se_counts.loc["Any Medal"] = se_counts.get("gold", 0) + se_counts.get("silver", 0) + se_counts.get("bronze", 0) - se_counts.loc["above_median"] = se_counts.get("above_median", 0) + se_counts.get("Any Medal", 0) - se_counts.loc["valid_submission"] = se_counts.get("valid_submission", 0) + se_counts.get("above_median", 0) - - sota_exp_stat = pd.Series(index=total_stat.index, dtype=int, name="SOTA Exp 统计(%)") - sota_exp_stat.loc["Made Submission"] = se_counts.get("made_submission", 0) - sota_exp_stat.loc["Valid Submission"] = se_counts.get("valid_submission", 0) - sota_exp_stat.loc["Above Median"] = se_counts.get("above_median", 0) - sota_exp_stat.loc["Bronze"] = se_counts.get("bronze", 0) - sota_exp_stat.loc["Silver"] = se_counts.get("silver", 0) - sota_exp_stat.loc["Gold"] = se_counts.get("gold", 0) - sota_exp_stat.loc["Any Medal"] = se_counts.get("Any Medal", 0) - sota_exp_stat = sota_exp_stat / base_df.shape[0] * 100 - - # SOTA Exp (trace.sota_exp_to_submit) 统计 - se_counts_new = base_df["SOTA Exp (_to_submit)"].value_counts(dropna=True) - se_counts_new.loc["made_submission"] = se_counts_new.sum() - se_counts_new.loc["Any Medal"] = ( - se_counts_new.get("gold", 0) + se_counts_new.get("silver", 0) + se_counts_new.get("bronze", 0) - ) - se_counts_new.loc["above_median"] = se_counts_new.get("above_median", 0) + se_counts_new.get("Any Medal", 0) - se_counts_new.loc["valid_submission"] = se_counts_new.get("valid_submission", 0) + se_counts_new.get( - "above_median", 0 - ) - - sota_exp_stat_new = pd.Series(index=total_stat.index, dtype=int, name="SOTA Exp (_to_submit) 统计(%)") - sota_exp_stat_new.loc["Made Submission"] = se_counts_new.get("made_submission", 0) - sota_exp_stat_new.loc["Valid Submission"] = se_counts_new.get("valid_submission", 0) - sota_exp_stat_new.loc["Above Median"] = se_counts_new.get("above_median", 0) - sota_exp_stat_new.loc["Bronze"] = se_counts_new.get("bronze", 0) - sota_exp_stat_new.loc["Silver"] = se_counts_new.get("silver", 0) - sota_exp_stat_new.loc["Gold"] = se_counts_new.get("gold", 0) - sota_exp_stat_new.loc["Any Medal"] = se_counts_new.get("Any Medal", 0) - sota_exp_stat_new = sota_exp_stat_new / base_df.shape[0] * 100 - - stat_df = pd.concat([total_stat, sota_exp_stat, sota_exp_stat_new], axis=1) - stat_t0, stat_t1 = st.columns(2) - with stat_t0: + stat_win_left, stat_win_right = st.columns(2) + with stat_win_left: + stat_df = get_statistics_df(base_df) st.dataframe(stat_df.round(2)) markdown_table = f""" | xxx | {stat_df.iloc[0,1]:.1f} | {stat_df.iloc[1,1]:.1f} | {stat_df.iloc[2,1]:.1f} | {stat_df.iloc[3,1]:.1f} | {stat_df.iloc[4,1]:.1f} | {stat_df.iloc[5,1]:.1f} | {stat_df.iloc[6,1]:.1f} | """ st.text(markdown_table) - with stat_t1: + with stat_win_right: Loop_counts = base_df["Total Loops"] fig = px.histogram(Loop_counts, nbins=10, title="Total Loops Histogram (nbins=10)") mean_value = Loop_counts.mean() @@ -542,65 +236,7 @@ def all_summarize_win(): # write curve st.subheader("Curves", divider="rainbow") if st.toggle("Show Curves", key="show_curves"): - for k, v in summary.items(): - with st.container(border=True): - st.markdown(f"**:blue[{k}] - :violet[{v['competition']}]**") - try: - tscores = {f"loop {k-1}": v for k, v in v["test_scores"].items()} - vscores = {} - for k, vs in v["valid_scores"].items(): - if not vs.index.is_unique: - st.warning( - f"Loop {k}'s valid scores index are not unique, only the last one will be kept to show." - ) - st.write(vs) - vscores[k] = vs[~vs.index.duplicated(keep="last")].iloc[:, 0] - - if len(vscores) > 0: - metric_name = list(vscores.values())[0].name - else: - metric_name = "None" - - tdf = pd.Series(tscores, name="score") - vdf = pd.DataFrame(vscores) - if "ensemble" in vdf.index: - ensemble_row = vdf.loc[["ensemble"]] - vdf = pd.concat([ensemble_row, vdf.drop("ensemble")]) - vdf.columns = [f"loop {i}" for i in vdf.columns] - fig = go.Figure() - # Add test scores trace from tdf - fig.add_trace( - go.Scatter( - x=tdf.index, - y=tdf, - mode="lines+markers", - name="Test scores", - marker=dict(symbol="diamond"), - line=dict(shape="linear", dash="dash"), - ) - ) - # Add valid score traces from vdf (transposed to have loops on x-axis) - for column in vdf.T.columns: - fig.add_trace( - go.Scatter( - x=vdf.T.index, - y=vdf.T[column], - mode="lines+markers", - name=f"{column}", - visible=("legendonly" if column != "ensemble" else None), - ) - ) - fig.update_layout(title=f"Test and Valid scores (metric: {metric_name})") - - st.plotly_chart(fig) - except Exception as e: - import traceback - - st.markdown("- Error: " + str(e)) - st.code(traceback.format_exc()) - st.markdown("- Valid Scores: ") - # st.write({k: type(v) for k, v in v["valid_scores"].items()}) - st.json(v["valid_scores"]) + curves_win(summary) with st.container(border=True): diff --git a/rdagent/log/ui/ds_trace.py b/rdagent/log/ui/ds_trace.py index 80fb226c..22cf4d8e 100644 --- a/rdagent/log/ui/ds_trace.py +++ b/rdagent/log/ui/ds_trace.py @@ -11,8 +11,14 @@ import streamlit as st from streamlit import session_state as state from rdagent.app.data_science.loop import DataScienceRDLoop -from rdagent.log.mle_summary import extract_mle_json, is_valid_session from rdagent.log.storage import FileStorage +from rdagent.log.ui.utils import load_times +from rdagent.log.utils import ( + extract_evoid, + extract_json, + extract_loopid_func_name, + is_valid_session, +) from rdagent.utils import remove_ansi_codes from rdagent.utils.repo.diff import generate_diff_from_dict @@ -30,40 +36,12 @@ if "log_folder" not in state: state.log_folder = Path("./log") -def extract_loopid_func_name(tag): - """提取 Loop ID 和函数名称""" - match = re.search(r"Loop_(\d+)\.([^.]+)", tag) - return match.groups() if match else (None, None) - - -def extract_evoid(tag): - """提取 EVO ID""" - match = re.search(r"\.evo_loop_(\d+)\.", tag) - return match.group(1) if match else None - - def convert_defaultdict_to_dict(d): if isinstance(d, defaultdict): d = {k: convert_defaultdict_to_dict(v) for k, v in d.items()} return d -@st.cache_data(persist=True) -def load_times(log_path: Path): - """加载时间数据""" - try: - session_path = log_path / "__session__" - max_li = max(int(p.name) for p in session_path.iterdir() if p.is_dir() and p.name.isdigit()) - max_step = max(int(p.name.split("_")[0]) for p in (session_path / str(max_li)).iterdir() if p.is_file()) - rdloop_obj_p = next((session_path / str(max_li)).glob(f"{max_step}_*")) - - rd_times = DataScienceRDLoop.load(rdloop_obj_p, do_truncate=False).loop_trace - except Exception as e: - # st.toast(f"Error loading times: {e}", icon="🟡") - rd_times = {} - return rd_times - - @st.cache_data(persist=True) def load_data(log_path: Path): data = defaultdict(lambda: defaultdict(dict)) @@ -143,7 +121,6 @@ def load_data(log_path: Path): return convert_defaultdict_to_dict(data), convert_defaultdict_to_dict(llm_data) -@st.cache_data def load_stdout(stdout_path: Path): if stdout_path.exists(): stdout = stdout_path.read_text() @@ -499,7 +476,7 @@ def summarize_data(): if "mle_score" not in state.data[loop]: if "mle_score" in loop_data["running"]: mle_score_txt = loop_data["running"]["mle_score"] - state.data[loop]["mle_score"] = extract_mle_json(mle_score_txt) + state.data[loop]["mle_score"] = extract_json(mle_score_txt) if ( state.data[loop]["mle_score"] is not None and state.data[loop]["mle_score"]["score"] is not None @@ -515,7 +492,7 @@ def summarize_data(): ) try: mle_score_txt = mle_score_path.read_text() - state.data[loop]["mle_score"] = extract_mle_json(mle_score_txt) + state.data[loop]["mle_score"] = extract_json(mle_score_txt) if state.data[loop]["mle_score"]["score"] is not None: df.loc[loop, "Running Score (test)"] = str(state.data[loop]["mle_score"]["score"]) else: diff --git a/rdagent/log/ui/llm_st.py b/rdagent/log/ui/llm_st.py index e0c270ba..7438acb1 100644 --- a/rdagent/log/ui/llm_st.py +++ b/rdagent/log/ui/llm_st.py @@ -8,6 +8,8 @@ from pathlib import Path import streamlit as st from streamlit import session_state +from rdagent.log.utils import extract_evoid, extract_loopid_func_name + st.set_page_config(layout="wide", page_title="debug_llm", page_icon="🎓", initial_sidebar_state="expanded") # 获取 log_path 参数 @@ -90,18 +92,6 @@ def highlight_prompts_uri(uri): return f"**{parts[0]}:**:green[**{parts[1]}**]" -def extract_loopid_func_name(tag): - """提取 Loop ID 和函数名称""" - match = re.search(r"Loop_(\d+)\.(\w+)\.", tag) - return match.groups() if match else (None, None) - - -def extract_evoid(tag): - """提取 EVO ID""" - match = re.search(r"\.evo_loop_(\d+)\.", tag) - return match.group(1) if match else None - - # Display Data progress_text = st.empty() progress_bar = st.progress(0) diff --git a/rdagent/log/ui/utils.py b/rdagent/log/ui/utils.py index 20f51d9d..6c5a90e7 100644 --- a/rdagent/log/ui/utils.py +++ b/rdagent/log/ui/utils.py @@ -1,3 +1,19 @@ +import math +import pickle +import re +from collections import deque +from datetime import datetime, timedelta +from pathlib import Path + +import pandas as pd +import typer + +from rdagent.app.data_science.loop import DataScienceRDLoop +from rdagent.core.utils import cache_with_pickle +from rdagent.log.ui.conf import UI_SETTING +from rdagent.log.utils import extract_json +from rdagent.oai.llm_utils import md5_hash + LITE = [ "aerial-cactus-identification", "aptos2019-blindness-detection", @@ -83,3 +99,446 @@ MEDIUM = [ ] ALL = HIGH + MEDIUM + LITE + + +def get_script_time(stdout_p: Path): + with stdout_p.open("r") as f: + first_line = next(f).strip() + last_line = deque(f, maxlen=1).pop().strip() + + # Extract timestamps from the lines + first_time_match = re.search(r"(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\+\d{2}:\d{2})", first_line) + last_time_match = re.search(r"(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\+\d{2}:\d{2})", last_line) + + if first_time_match and last_time_match: + first_time = datetime.fromisoformat(first_time_match.group(1)) + last_time = datetime.fromisoformat(last_time_match.group(1)) + return pd.Timedelta(last_time - first_time) + + return None + + +def _log_path_hash_func(log_path: Path): + hash_str = str(log_path) + str(log_path.stat().st_mtime) + session_p = log_path / "__session__" + if session_p.exists(): + for ld in session_p.iterdir(): + if ld.is_dir(): + hash_str += str(ld.name) + str(ld.stat().st_mtime) + else: + hash_str += "no session now" + return md5_hash(hash_str) + + +@cache_with_pickle(_log_path_hash_func, force=True) +def get_final_sota_exp(log_path: Path): + sota_exp_paths = [i for i in log_path.rglob(f"**/SOTA experiment/**/*.pkl")] + if len(sota_exp_paths) == 0: + return None + final_sota_exp_path = max(sota_exp_paths, key=lambda x: int(re.match(r".*Loop_(\d+).*", str(x))[1])) + with final_sota_exp_path.open("rb") as f: + final_sota_exp = pickle.load(f) + return final_sota_exp + + +@cache_with_pickle(_log_path_hash_func, force=True) +def get_sota_exp_stat(log_path: Path): + trace_paths = [i for i in log_path.rglob(f"**/trace/**/*.pkl")] + if len(trace_paths) == 0: + return None + final_trace_path = max(trace_paths, key=lambda x: int(re.match(r".*Loop_(\d+).*", str(x))[1])) + with final_trace_path.open("rb") as f: + final_trace = pickle.load(f) + + if hasattr(final_trace, "sota_exp_to_submit"): + sota_exp = final_trace.sota_exp_to_submit + else: + sota_exp = final_trace.sota_experiment() + + if sota_exp is None: + return None + + sota_loop_id = None + for i, ef in enumerate(final_trace.hist): + if ef[0] == sota_exp: + sota_loop_id = i + break + + sota_mle_score_paths = [i for i in log_path.rglob(f"Loop_{sota_loop_id}/running/mle_score/**/*.pkl")] + if len(sota_mle_score_paths) == 0: + # sota exp is not evaluated by mle_score + return None + with sota_mle_score_paths[0].open("rb") as f: + sota_mle_score = extract_json(pickle.load(f)) + + sota_exp_stat = None + if sota_mle_score: # sota exp's grade output + if sota_mle_score["gold_medal"]: + sota_exp_stat = "gold" + elif sota_mle_score["silver_medal"]: + sota_exp_stat = "silver" + elif sota_mle_score["bronze_medal"]: + sota_exp_stat = "bronze" + elif sota_mle_score["above_median"]: + sota_exp_stat = "above_median" + elif sota_mle_score["valid_submission"]: + sota_exp_stat = "valid_submission" + elif sota_mle_score["submission_exists"]: + sota_exp_stat = "made_submission" + return sota_exp_stat + + +@cache_with_pickle(_log_path_hash_func, force=True) +def load_times(log_path: Path): + try: + session_path = log_path / "__session__" + max_li = max(int(p.name) for p in session_path.iterdir() if p.is_dir() and p.name.isdigit()) + max_step = max(int(p.name.split("_")[0]) for p in (session_path / str(max_li)).iterdir() if p.is_file()) + rdloop_obj_p = next((session_path / str(max_li)).glob(f"{max_step}_*")) + + rd_times = DataScienceRDLoop.load(rdloop_obj_p, do_truncate=False).loop_trace + except Exception as e: + rd_times = {} + return rd_times + + +def _log_folders_summary_hash_func(log_folders: list[str], hours: int | None = None): + hash_str = "" + for lf in log_folders: + summary_p = Path(lf) / (f"summary.pkl" if hours is None else f"summary_{hours}h.pkl") + if summary_p.exists(): + hash_str += str(summary_p) + str(summary_p.stat().st_mtime) + else: + hash_str += f"{summary_p} not exists" + return md5_hash(hash_str) + + +@cache_with_pickle(_log_folders_summary_hash_func, force=True) +def get_summary_df(log_folders: list[str], hours: int | None = None) -> tuple[dict, pd.DataFrame]: + summarys = {} + if hours is None: + sn = "summary.pkl" + else: + sn = f"summary_{hours}h.pkl" + for lf in log_folders: + if (Path(lf) / sn).exists(): + summarys[lf] = pd.read_pickle(Path(lf) / sn) + + if len(summarys) == 0: + return {}, pd.DataFrame() + + summary = {} + for lf, s in summarys.items(): + for k, v in s.items(): + stdout_p = Path(lf) / f"{k}.stdout" + if stdout_p.exists(): + v["script_time"] = get_script_time(stdout_p) + else: + v["script_time"] = None + + exp_gen_time = timedelta() + coding_time = timedelta() + running_time = timedelta() + all_time = timedelta() + times_info = load_times(Path(lf) / k) + for time_info in times_info.values(): + all_time += sum((ti.end - ti.start for ti in time_info), timedelta()) + exp_gen_time += time_info[0].end - time_info[0].start + if len(time_info) > 1: + coding_time += time_info[1].end - time_info[1].start + if len(time_info) > 2: + running_time += time_info[2].end - time_info[2].start + v["exec_time"] = str(all_time).split(".")[0] + v["exp_gen_time"] = str(exp_gen_time).split(".")[0] + v["coding_time"] = str(coding_time).split(".")[0] + v["running_time"] = str(running_time).split(".")[0] + + final_sota_exp = get_final_sota_exp(Path(lf) / k) + if final_sota_exp is not None and final_sota_exp.result is not None: + v["sota_exp_score_valid"] = final_sota_exp.result.loc["ensemble"].iloc[0] + else: + v["sota_exp_score_valid"] = None + v["sota_exp_stat_new"] = get_sota_exp_stat(Path(lf) / k) + # 调整实验名字 + if "amlt" in lf: + summary[f"{lf[lf.rfind('amlt')+5:].split('/')[0]} - {k}"] = v + elif "ep" in lf: + summary[f"{lf[lf.rfind('ep'):]} - {k}"] = v + else: + summary[f"{lf} - {k}"] = v + + summary = {k: v for k, v in summary.items() if "competition" in v} + base_df = pd.DataFrame( + columns=[ + "Competition", + "Script Time", + "Exec Time", + "Exp Gen", + "Coding", + "Running", + "Total Loops", + "Successful Final Decision", + "Made Submission", + "Valid Submission", + "V/M", + "Above Median", + "Bronze", + "Silver", + "Gold", + "Any Medal", + "Best Result", + "SOTA Exp", + "SOTA Exp (_to_submit)", + "SOTA Exp Score (valid)", + "SOTA Exp Score", + "Baseline Score", + "Ours - Base", + "Ours vs Base", + "Ours vs Bronze", + "Ours vs Silver", + "Ours vs Gold", + "Bronze Threshold", + "Silver Threshold", + "Gold Threshold", + "Medium Threshold", + ], + index=summary.keys(), + ) + + # Read baseline results + baseline_result_path = UI_SETTING.baseline_result_path + if Path(baseline_result_path).exists(): + baseline_df = pd.read_csv(baseline_result_path) + + def compare_score(s1, s2): + if s1 is None or s2 is None: + return None + try: + c_value = math.exp(abs(math.log(s1 / s2))) + except Exception as e: + c_value = None + return c_value + + for k, v in summary.items(): + loop_num = v["loop_num"] + base_df.loc[k, "Competition"] = v["competition"] + base_df.loc[k, "Script Time"] = v["script_time"] + base_df.loc[k, "Exec Time"] = v["exec_time"] + base_df.loc[k, "Exp Gen"] = v["exp_gen_time"] + base_df.loc[k, "Coding"] = v["coding_time"] + base_df.loc[k, "Running"] = v["running_time"] + base_df.loc[k, "Total Loops"] = loop_num + if loop_num == 0: + base_df.loc[k] = "N/A" + else: + base_df.loc[k, "Successful Final Decision"] = v["success_loop_num"] + base_df.loc[k, "Made Submission"] = v["made_submission_num"] + if v["made_submission_num"] > 0: + base_df.loc[k, "Best Result"] = "made_submission" + base_df.loc[k, "Valid Submission"] = v["valid_submission_num"] + if v["valid_submission_num"] > 0: + base_df.loc[k, "Best Result"] = "valid_submission" + base_df.loc[k, "Above Median"] = v["above_median_num"] + if v["above_median_num"] > 0: + base_df.loc[k, "Best Result"] = "above_median" + base_df.loc[k, "Bronze"] = v["bronze_num"] + if v["bronze_num"] > 0: + base_df.loc[k, "Best Result"] = "bronze" + base_df.loc[k, "Silver"] = v["silver_num"] + if v["silver_num"] > 0: + base_df.loc[k, "Best Result"] = "silver" + base_df.loc[k, "Gold"] = v["gold_num"] + if v["gold_num"] > 0: + base_df.loc[k, "Best Result"] = "gold" + base_df.loc[k, "Any Medal"] = v["get_medal_num"] + + baseline_score = None + if Path(baseline_result_path).exists(): + baseline_score = baseline_df.loc[baseline_df["competition_id"] == v["competition"], "score"].item() + + base_df.loc[k, "SOTA Exp"] = v.get("sota_exp_stat", None) + base_df.loc[k, "SOTA Exp (_to_submit)"] = v["sota_exp_stat_new"] + if baseline_score is not None and v.get("sota_exp_score", None) is not None: + base_df.loc[k, "Ours - Base"] = v["sota_exp_score"] - baseline_score + base_df.loc[k, "Ours vs Base"] = compare_score(v["sota_exp_score"], baseline_score) + base_df.loc[k, "Ours vs Bronze"] = compare_score(v["sota_exp_score"], v.get("bronze_threshold", None)) + base_df.loc[k, "Ours vs Silver"] = compare_score(v["sota_exp_score"], v.get("silver_threshold", None)) + base_df.loc[k, "Ours vs Gold"] = compare_score(v["sota_exp_score"], v.get("gold_threshold", None)) + base_df.loc[k, "SOTA Exp Score"] = v.get("sota_exp_score", None) + base_df.loc[k, "SOTA Exp Score (valid)"] = v.get("sota_exp_score_valid", None) + base_df.loc[k, "Baseline Score"] = baseline_score + base_df.loc[k, "Bronze Threshold"] = v.get("bronze_threshold", None) + base_df.loc[k, "Silver Threshold"] = v.get("silver_threshold", None) + base_df.loc[k, "Gold Threshold"] = v.get("gold_threshold", None) + base_df.loc[k, "Medium Threshold"] = v.get("median_threshold", None) + + base_df["SOTA Exp"] = base_df["SOTA Exp"].replace("", pd.NA) + + base_df.loc[base_df["SOTA Exp Score (valid)"].apply(lambda x: isinstance(x, str)), "SOTA Exp Score (valid)"] = 0.0 + base_df = base_df.astype( + { + "Total Loops": int, + "Successful Final Decision": int, + "Made Submission": int, + "Valid Submission": int, + "Above Median": int, + "Bronze": int, + "Silver": int, + "Gold": int, + "Any Medal": int, + "Ours - Base": float, + "Ours vs Base": float, + "SOTA Exp Score": float, + "SOTA Exp Score (valid)": float, + "Baseline Score": float, + "Bronze Threshold": float, + "Silver Threshold": float, + "Gold Threshold": float, + "Medium Threshold": float, + } + ) + return summary, base_df + + +def percent_df(summary_df: pd.DataFrame, show_origin=True) -> pd.DataFrame: + """ + Convert the summary DataFrame to a percentage format. + """ + new_df = summary_df.copy(deep=True) + + # Convert columns to object dtype so we can store strings like "14 (53.85%)" without warnings + columns_to_convert = [ + "Successful Final Decision", + "Made Submission", + "Valid Submission", + "Above Median", + "Bronze", + "Silver", + "Gold", + "Any Medal", + ] + new_df[columns_to_convert] = new_df[columns_to_convert].astype(object) + + def num2percent(num: int, total: int, show_origin=True) -> str: + num = int(num) + total = int(total) + if show_origin: + return f"{num} ({round(num / total * 100, 2)}%)" + return f"{round(num / total * 100, 2)}%" + + for k in new_df.index: + loop_num = int(new_df.loc[k, "Total Loops"]) + if loop_num != 0: + new_df.loc[k, "Successful Final Decision"] = num2percent( + new_df.loc[k, "Successful Final Decision"], loop_num, show_origin + ) + if new_df.loc[k, "Made Submission"] != 0: + new_df.loc[k, "V/M"] = ( + f"{round(new_df.loc[k, 'Valid Submission'] / new_df.loc[k, 'Made Submission'] * 100, 2)}%" + ) + else: + new_df.loc[k, "V/M"] = "N/A" + new_df.loc[k, "Made Submission"] = num2percent(new_df.loc[k, "Made Submission"], loop_num, show_origin) + new_df.loc[k, "Valid Submission"] = num2percent(new_df.loc[k, "Valid Submission"], loop_num, show_origin) + new_df.loc[k, "Above Median"] = num2percent(new_df.loc[k, "Above Median"], loop_num, show_origin) + new_df.loc[k, "Bronze"] = num2percent(new_df.loc[k, "Bronze"], loop_num, show_origin) + new_df.loc[k, "Silver"] = num2percent(new_df.loc[k, "Silver"], loop_num, show_origin) + new_df.loc[k, "Gold"] = num2percent(new_df.loc[k, "Gold"], loop_num, show_origin) + new_df.loc[k, "Any Medal"] = num2percent(new_df.loc[k, "Any Medal"], loop_num, show_origin) + + return new_df + + +def get_statistics_df(summary_df: pd.DataFrame) -> pd.DataFrame: + if summary_df["Any Medal"].dtype == int: + check_value = 0 + else: + sample_val = summary_df["Any Medal"].dropna().iloc[0] + if "(" in sample_val: + check_value = "0 (0.0%)" + else: + check_value = "0.0%" + total_stat = ( + summary_df[ + [ + "Made Submission", + "Valid Submission", + "Above Median", + "Bronze", + "Silver", + "Gold", + "Any Medal", + ] + ] + != check_value + ).sum() + total_stat.name = "总体统计(%)" + total_stat.loc["Bronze"] = summary_df["Best Result"].value_counts().get("bronze", 0) + total_stat.loc["Silver"] = summary_df["Best Result"].value_counts().get("silver", 0) + total_stat.loc["Gold"] = summary_df["Best Result"].value_counts().get("gold", 0) + total_stat = total_stat / summary_df.shape[0] * 100 + + # SOTA Exp 统计 + se_counts = summary_df["SOTA Exp"].value_counts(dropna=True) + se_counts.loc["made_submission"] = se_counts.sum() + se_counts.loc["Any Medal"] = se_counts.get("gold", 0) + se_counts.get("silver", 0) + se_counts.get("bronze", 0) + se_counts.loc["above_median"] = se_counts.get("above_median", 0) + se_counts.get("Any Medal", 0) + se_counts.loc["valid_submission"] = se_counts.get("valid_submission", 0) + se_counts.get("above_median", 0) + + sota_exp_stat = pd.Series(index=total_stat.index, dtype=int, name="SOTA Exp 统计(%)") + sota_exp_stat.loc["Made Submission"] = se_counts.get("made_submission", 0) + sota_exp_stat.loc["Valid Submission"] = se_counts.get("valid_submission", 0) + sota_exp_stat.loc["Above Median"] = se_counts.get("above_median", 0) + sota_exp_stat.loc["Bronze"] = se_counts.get("bronze", 0) + sota_exp_stat.loc["Silver"] = se_counts.get("silver", 0) + sota_exp_stat.loc["Gold"] = se_counts.get("gold", 0) + sota_exp_stat.loc["Any Medal"] = se_counts.get("Any Medal", 0) + sota_exp_stat = sota_exp_stat / summary_df.shape[0] * 100 + + # SOTA Exp (trace.sota_exp_to_submit) 统计 + se_counts_new = summary_df["SOTA Exp (_to_submit)"].value_counts(dropna=True) + se_counts_new.loc["made_submission"] = se_counts_new.sum() + se_counts_new.loc["Any Medal"] = ( + se_counts_new.get("gold", 0) + se_counts_new.get("silver", 0) + se_counts_new.get("bronze", 0) + ) + se_counts_new.loc["above_median"] = se_counts_new.get("above_median", 0) + se_counts_new.get("Any Medal", 0) + se_counts_new.loc["valid_submission"] = se_counts_new.get("valid_submission", 0) + se_counts_new.get( + "above_median", 0 + ) + + sota_exp_stat_new = pd.Series(index=total_stat.index, dtype=int, name="SOTA Exp (_to_submit) 统计(%)") + sota_exp_stat_new.loc["Made Submission"] = se_counts_new.get("made_submission", 0) + sota_exp_stat_new.loc["Valid Submission"] = se_counts_new.get("valid_submission", 0) + sota_exp_stat_new.loc["Above Median"] = se_counts_new.get("above_median", 0) + sota_exp_stat_new.loc["Bronze"] = se_counts_new.get("bronze", 0) + sota_exp_stat_new.loc["Silver"] = se_counts_new.get("silver", 0) + sota_exp_stat_new.loc["Gold"] = se_counts_new.get("gold", 0) + sota_exp_stat_new.loc["Any Medal"] = se_counts_new.get("Any Medal", 0) + sota_exp_stat_new = sota_exp_stat_new / summary_df.shape[0] * 100 + + stat_df = pd.concat([total_stat, sota_exp_stat, sota_exp_stat_new], axis=1) + return stat_df + + +def compare( + exp_list: list[str] = typer.Option(..., "--exp-list", help="List of experiment names.", show_default=False), + output: str = typer.Option("merge_base_df.h5", help="Output summary file name."), + hours: int | None = typer.Option(None, help="if None, use summary.pkl, else summary_{hours}h.pkl"), +): + """ + Generate summary and base dataframe for given experiment list, and save to a summary file. + """ + typer.secho(f"exp_list: {exp_list}", fg=typer.colors.GREEN) + log_folders = [f"{UI_SETTING.amlt_path}/{exp}/combined_logs" for exp in exp_list] + summary, base_df = get_summary_df(log_folders, hours=hours) + typer.secho(f"Summary keys: {list(summary.keys())}", fg=typer.colors.CYAN) + typer.secho("Summary DataFrame:", fg=typer.colors.MAGENTA) + typer.secho(str(base_df), fg=typer.colors.YELLOW) + base_df.to_hdf(output, "data") + typer.secho(f"Summary saved to {output}", fg=typer.colors.GREEN) + + +if __name__ == "__main__": + app = typer.Typer() + app.command()(compare) + app() diff --git a/rdagent/log/utils/__init__.py b/rdagent/log/utils/__init__.py index b87682bc..c6f68cdb 100644 --- a/rdagent/log/utils/__init__.py +++ b/rdagent/log/utils/__init__.py @@ -1,6 +1,8 @@ import inspect +import json import re -from typing import Dict, Optional, TypedDict, Union +from pathlib import Path +from typing import Optional, TypedDict, cast class LogColors: @@ -75,3 +77,26 @@ def get_caller_info() -> CallerInfo: "function": frame.f_code.co_name, # Get the caller's function name } return info + + +def is_valid_session(log_path: Path) -> bool: + return log_path.is_dir() and log_path.joinpath("__session__").exists() + + +def extract_loopid_func_name(tag: str) -> tuple[str, str] | tuple[None, None]: + """extract loop id and function name from the tag in Message""" + match = re.search(r"Loop_(\d+)\.([^.]+)", tag) + return cast(tuple[str, str], match.groups()) if match else (None, None) + + +def extract_evoid(tag: str) -> str | None: + """extract evo id from the tag in Message""" + match = re.search(r"\.evo_loop_(\d+)\.", tag) + return cast(str, match.group(1)) if match else None + + +def extract_json(log_content: str) -> dict | None: + match = re.search(r"\{.*\}", log_content, re.DOTALL) + if match: + return cast(dict, json.loads(match.group(0))) + return None diff --git a/rdagent/log/utils/folder.py b/rdagent/log/utils/folder.py index 643e92b7..ec45c6a7 100644 --- a/rdagent/log/utils/folder.py +++ b/rdagent/log/utils/folder.py @@ -3,6 +3,7 @@ This module provides some useful functions for working with logger folders. """ import pickle +from datetime import timedelta from pathlib import Path import pandas as pd diff --git a/rdagent/scenarios/data_science/scen/__init__.py b/rdagent/scenarios/data_science/scen/__init__.py index 6e90e126..9d765053 100644 --- a/rdagent/scenarios/data_science/scen/__init__.py +++ b/rdagent/scenarios/data_science/scen/__init__.py @@ -1,6 +1,4 @@ import json -import os -import platform from pathlib import Path from typing import Dict @@ -18,7 +16,7 @@ from rdagent.scenarios.data_science.scen.utils import ( from rdagent.scenarios.kaggle.kaggle_crawler import ( crawl_descriptions, download_data, - leaderboard_scores, + get_metric_direction, ) from rdagent.utils.agent.tpl import T @@ -188,8 +186,7 @@ class KaggleScen(DataScienceScen): return crawl_descriptions(self.competition, DS_RD_SETTING.local_data_path) def _get_direction(self): - leaderboard = leaderboard_scores(self.competition) - return float(leaderboard[0]) > float(leaderboard[-1]) + return get_metric_direction(self.competition) @property def rich_style_description(self) -> str: diff --git a/rdagent/scenarios/kaggle/kaggle_crawler.py b/rdagent/scenarios/kaggle/kaggle_crawler.py index 555742f9..277f7bce 100644 --- a/rdagent/scenarios/kaggle/kaggle_crawler.py +++ b/rdagent/scenarios/kaggle/kaggle_crawler.py @@ -198,6 +198,14 @@ def leaderboard_scores(competition: str) -> list[float]: return [float(x.score) for x in ll] +def get_metric_direction(competition: str) -> bool: + """ + Return **True** if the metric is *bigger is better*, **False** if *smaller is better*. + """ + leaderboard = leaderboard_scores(competition) + return float(leaderboard[0]) > float(leaderboard[-1]) + + def score_rank(competition: str, score: float) -> tuple[int, float]: """ Return