From b1fce69153115751a22db4c69f9c3175aef962a8 Mon Sep 17 00:00:00 2001 From: XianBW <36835909+XianBW@users.noreply.github.com> Date: Wed, 5 Feb 2025 16:31:30 +0800 Subject: [PATCH] fix: change summary info of log folder (#552) * update stat data in log summary * show logic change * add workspace None judgement * fix CI --- rdagent/log/mle_summary.py | 37 ++++++++++++++++------- rdagent/log/ui/dsapp.py | 60 ++++++++++++++++++++++++++++---------- 2 files changed, 71 insertions(+), 26 deletions(-) diff --git a/rdagent/log/mle_summary.py b/rdagent/log/mle_summary.py index 5002fb1c..3880f8c0 100644 --- a/rdagent/log/mle_summary.py +++ b/rdagent/log/mle_summary.py @@ -19,7 +19,7 @@ de = DockerEnv(conf=mle_de_conf) de.prepare() -def extract_mle_json(log_content): +def extract_mle_json(log_content: str) -> dict | None: match = re.search(r"\{.*\}", log_content, re.DOTALL) if match: return json.loads(match.group(0)) @@ -54,9 +54,14 @@ def summarize_folder(log_folder: Path): continue loop_num = 0 made_submission_num = 0 + valid_submission_num = 0 + above_median_num = 0 + get_medal_num = 0 + bronze_num = 0 + silver_num = 0 + gold_num = 0 test_scores = {} valid_scores = {} - medal = "None" success_loop_num = 0 for msg in FileStorage(log_trace_path).iter_msg(): # messages in log trace @@ -80,14 +85,21 @@ def summarize_folder(log_folder: Path): f"mle_score.txt in {grade_output_path} not found, genarate it first!" ) grade_output = extract_mle_json(grade_output_path.read_text()) - if grade_output and grade_output["score"] is not None: - test_scores[loop_num - 1] = grade_output["score"] + if grade_output: + if grade_output["score"] is not None: + test_scores[loop_num - 1] = grade_output["score"] + if grade_output["valid_submission"]: + valid_submission_num += 1 + if grade_output["above_median"]: + above_median_num += 1 if grade_output["any_medal"]: - medal = ( - "gold" - if grade_output["gold_medal"] - else "silver" if grade_output["silver_medal"] else "bronze" - ) + get_medal_num += 1 + if grade_output["bronze_medal"]: + bronze_num += 1 + if grade_output["silver_medal"]: + silver_num += 1 + if grade_output["gold_medal"]: + gold_num += 1 if "feedback" in msg.tag and "evolving" not in msg.tag: if bool(msg.content): @@ -97,9 +109,14 @@ def summarize_folder(log_folder: Path): { "loop_num": loop_num, "made_submission_num": made_submission_num, + "valid_submission_num": valid_submission_num, + "above_median_num": above_median_num, + "get_medal_num": get_medal_num, + "bronze_num": bronze_num, + "silver_num": silver_num, + "gold_num": gold_num, "test_scores": test_scores, "valid_scores": valid_scores, - "medal": medal, "success_loop_num": success_loop_num, } ) diff --git a/rdagent/log/ui/dsapp.py b/rdagent/log/ui/dsapp.py index 63ee798b..2c69f077 100644 --- a/rdagent/log/ui/dsapp.py +++ b/rdagent/log/ui/dsapp.py @@ -144,14 +144,18 @@ def evolving_win(data): evo_id = 0 if evo_id in data: - st.subheader("codes") - workspace_win(data[evo_id]["evolving code"][0]) - fb = data[evo_id]["evolving feedback"][0] - st.subheader("evolving feedback" + ("✅" if bool(fb) else "❌"), anchor="c_feedback") - f1, f2, f3 = st.tabs(["execution", "return_checking", "code"]) - f1.code(fb.execution, wrap_lines=True) - f2.code(fb.return_checking, wrap_lines=True) - f3.code(fb.code, wrap_lines=True) + if data[evo_id]["evolving code"][0] is not None: + st.subheader("codes") + workspace_win(data[evo_id]["evolving code"][0]) + fb = data[evo_id]["evolving feedback"][0] + st.subheader("evolving feedback" + ("✅" if bool(fb) else "❌"), anchor="c_feedback") + f1, f2, f3 = st.tabs(["execution", "return_checking", "code"]) + f1.code(fb.execution, wrap_lines=True) + f2.code(fb.return_checking, wrap_lines=True) + f3.code(fb.code, wrap_lines=True) + else: + st.write("data[evo_id]['evolving code'][0] is None.") + st.write(data[evo_id]) else: st.markdown("No evolving.") @@ -268,20 +272,44 @@ def all_summarize_win(): summary = pd.read_pickle(state.log_folder / "summary.pkl") summary = {k: v for k, v in summary.items() if "competition" in v} base_df = pd.DataFrame( - columns=["Competition", "Total Loops", "Made Submission", "Successful Final Decision", "Medal"], + columns=[ + "Competition", + "Total Loops", + "Successful Final Decision", + "Made Submission", + "Valid Submission", + "Above Median", + "Bronze", + "Silver", + "Gold", + "Any Medal", + ], index=summary.keys(), ) for k, v in summary.items(): loop_num = v["loop_num"] base_df.loc[k, "Competition"] = v["competition"] base_df.loc[k, "Total Loops"] = loop_num - base_df.loc[k, "Made Submission"] = ( - f"{v['made_submission_num']} ({round(v['made_submission_num'] / loop_num * 100, 2) if loop_num != 0 else 0}%)" - ) - base_df.loc[k, "Successful Final Decision"] = ( - f"{v['success_loop_num']} ({round(v['success_loop_num'] / loop_num * 100, 2) if loop_num != 0 else 0}%)" - ) - base_df.loc[k, "Medal"] = v["medal"] + if loop_num == 0: + base_df.loc[k] = "N/A" + else: + base_df.loc[k, "Successful Final Decision"] = ( + f"{v['success_loop_num']} ({round(v['success_loop_num'] / loop_num * 100, 2)}%)" + ) + base_df.loc[k, "Made Submission"] = ( + f"{v['made_submission_num']} ({round(v['made_submission_num'] / loop_num * 100, 2)}%)" + ) + base_df.loc[k, "Valid Submission"] = ( + f"{v['valid_submission_num']} ({round(v['valid_submission_num'] / loop_num * 100, 2)}%)" + ) + base_df.loc[k, "Above Median"] = ( + f"{v['above_median_num']} ({round(v['above_median_num'] / loop_num * 100, 2)}%)" + ) + base_df.loc[k, "Bronze"] = f"{v['bronze_num']} ({round(v['bronze_num'] / loop_num * 100, 2)}%)" + base_df.loc[k, "Silver"] = f"{v['silver_num']} ({round(v['silver_num'] / loop_num * 100, 2)}%)" + base_df.loc[k, "Gold"] = f"{v['gold_num']} ({round(v['gold_num'] / loop_num * 100, 2)}%)" + base_df.loc[k, "Any Medal"] = f"{v['get_medal_num']} ({round(v['get_medal_num'] / loop_num * 100, 2)}%)" + st.dataframe(base_df) # write curve for k, v in summary.items():