gavindiaz/NexQuant
1
0
Fork 0
mirror of https://github.com/NicolasBohn/NexQuant.git synced 2026-07-28 07:57:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: add baseline score stat (#590)

Browse Source 
* add baseline score stat

* fix ci

* fix

* fix
This commit is contained in:
Yuante Li
2025-02-12 18:57:08 +08:00
committed by GitHub
parent f06412e628
commit 43703d45d2
2 changed files with 46 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
rdagent/log/mle_summary.py
+16
View File
@@ -67,9 +67,14 @@ def summarize_folder(log_folder: Path):
gold_num = 0
test_scores = {}
valid_scores = {}
bronze_threshold = 0.0
silver_threshold = 0.0
gold_threshold = 0.0
median_threshold = 0.0
success_loop_num = 0
sota_exp_stat = ""
sota_exp_score = None
grade_output = None
for msg in FileStorage(log_trace_path).iter_msg(): # messages in log trace
if msg.tag and "llm" not in msg.tag and "session" not in msg.tag:
@@ -107,6 +112,10 @@ def summarize_folder(log_folder: Path):
silver_num += 1
if grade_output["gold_medal"]:
gold_num += 1
bronze_threshold = grade_output["bronze_threshold"]
silver_threshold = grade_output["silver_threshold"]
gold_threshold = grade_output["gold_threshold"]
median_threshold = grade_output["median_threshold"]
if "feedback" in msg.tag and "evolving" not in msg.tag:
if isinstance(msg.content, ExperimentFeedback) and bool(msg.content):
@@ -125,6 +134,8 @@ def summarize_folder(log_folder: Path):
sota_exp_stat = "valid_submission"
elif grade_output["submission_exists"]:
sota_exp_stat = "made_submission"
if grade_output["score"] is not None:
sota_exp_score = grade_output["score"]
stat[log_trace_path.name].update(
{
@@ -140,6 +151,11 @@ def summarize_folder(log_folder: Path):
"valid_scores": valid_scores,
"success_loop_num": success_loop_num,
"sota_exp_stat": sota_exp_stat,
"sota_exp_score": sota_exp_score,
"bronze_threshold": bronze_threshold,
"silver_threshold": silver_threshold,
"gold_threshold": gold_threshold,
"median_threshold": median_threshold,
}
)
if (log_folder / "summary.pkl").exists():
rdagent/log/ui/dsapp.py
+30 -2
View File
@@ -345,9 +345,22 @@ def all_summarize_win():
"Gold",
"Any Medal",
"SOTA Exp",
"Ours - Base",
"SOTA Exp Score",
"Baseline Score",
"Bronze Threshold",
"Silver Threshold",
"Gold Threshold",
"Medium Threshold",
],
index=summary.keys(),
)
# Read baseline results
baseline_result_path = ""
if Path(baseline_result_path).exists():
baseline_df = pd.read_csv(baseline_result_path)
for k, v in summary.items():
loop_num = v["loop_num"]
base_df.loc[k, "Competition"] = v["competition"]
@@ -376,8 +389,23 @@ def all_summarize_win():
base_df.loc[k, "Gold"] = f"{v['gold_num']} ({round(v['gold_num'] / loop_num * 100, 2)}%)"
base_df.loc[k, "Any Medal"] = f"{v['get_medal_num']} ({round(v['get_medal_num'] / loop_num * 100, 2)}%)"
if "sota_exp_stat" in v:
base_df.loc[k, "SOTA Exp"] = v["sota_exp_stat"]
baseline_score = None
if Path(baseline_result_path).exists():
baseline_score = baseline_df.loc[baseline_df["competition_id"] == v["competition"], "score"].item()
base_df.loc[k, "SOTA Exp"] = v.get("sota_exp_stat", None)
if (
baseline_score is not None
and not pd.isna(baseline_score)
and not pd.isna(v.get("sota_exp_score", None))
):
base_df.loc[k, "Ours - Base"] = v.get("sota_exp_score", 0.0) - baseline_score
base_df.loc[k, "SOTA Exp Score"] = v.get("sota_exp_score", None)
base_df.loc[k, "Baseline Score"] = baseline_score
base_df.loc[k, "Bronze Threshold"] = v.get("bronze_threshold", None)
base_df.loc[k, "Silver Threshold"] = v.get("silver_threshold", None)
base_df.loc[k, "Gold Threshold"] = v.get("gold_threshold", None)
base_df.loc[k, "Medium Threshold"] = v.get("median_threshold", None)
base_df["SOTA Exp"].replace("", pd.NA, inplace=True)
st.dataframe(base_df)