From b9a36e1b25273bdc35eb9ccd0a0f4dcb1b8616af Mon Sep 17 00:00:00 2001 From: Tim Date: Tue, 15 Oct 2024 12:41:32 +0800 Subject: [PATCH] chore: more optional parameters for running benchmark analysis (#431) * set title and round * decision from multiple types * check if decision is true * reformat * remove unused file --- rdagent/app/benchmark/factor/analysis.py | 14 +++++++++----- .../coder/factor_coder/CoSTEER/evaluators.py | 17 +++-------------- 2 files changed, 12 insertions(+), 19 deletions(-) diff --git a/rdagent/app/benchmark/factor/analysis.py b/rdagent/app/benchmark/factor/analysis.py index ca4ee8fa..3ef10fe2 100644 --- a/rdagent/app/benchmark/factor/analysis.py +++ b/rdagent/app/benchmark/factor/analysis.py @@ -178,20 +178,24 @@ class Plotter: plt.rc("figure", titlesize=font_size) @staticmethod - def plot_data(data, file_name): + def plot_data(data, file_name, title): plt.figure(figsize=(10, 6)) sns.barplot(x="index", y="b", hue="a", data=data) plt.xlabel("Method") plt.ylabel("Value") - plt.title("Comparison of Different Methods") + plt.title(title) plt.savefig(file_name) -def main(path="git_ignore_folder/eval_results/res_promptV220240724-060037.pkl"): +def main( + path="git_ignore_folder/eval_results/res_promptV220240724-060037.pkl", + round=1, + title="Comparison of Different Methods", +): settings = BenchmarkSettings() benchmark = BenchmarkAnalyzer(settings) results = { - "1 round experiment": path, + f"{round} round experiment": path, } final_results = benchmark.process_results(results) final_results_df = pd.DataFrame(final_results) @@ -199,7 +203,7 @@ def main(path="git_ignore_folder/eval_results/res_promptV220240724-060037.pkl"): Plotter.change_fs(20) plot_data = final_results_df.drop(["max. accuracy", "avg. accuracy"], axis=0).T plot_data = plot_data.reset_index().melt("index", var_name="a", value_name="b") - Plotter.plot_data(plot_data, "./comparison_plot.png") + Plotter.plot_data(plot_data, "./comparison_plot.png", title) if __name__ == "__main__": diff --git a/rdagent/components/coder/factor_coder/CoSTEER/evaluators.py b/rdagent/components/coder/factor_coder/CoSTEER/evaluators.py index 214a80f8..d7cbf03b 100644 --- a/rdagent/components/coder/factor_coder/CoSTEER/evaluators.py +++ b/rdagent/components/coder/factor_coder/CoSTEER/evaluators.py @@ -195,14 +195,7 @@ class FactorOutputFormatEvaluator(FactorEvaluator): user_prompt=gen_df_info_str, system_prompt=system_prompt, json_mode=True ) resp_dict = json.loads(resp) - - if isinstance(resp_dict["output_format_decision"], str) and resp_dict[ - "output_format_decision" - ].lower() in ( - "true", - "false", - ): - resp_dict["output_format_decision"] = resp_dict["output_format_decision"].lower() == "true" + resp_dict["output_format_decision"] = str(resp_dict["output_format_decision"]).lower() in ["true", "1"] return ( resp_dict["output_format_feedback"], @@ -243,7 +236,7 @@ class FactorDatetimeDailyEvaluator(FactorEvaluator): False, ) - time_diff = gen_df.index.get_level_values("datetime").to_series().diff().dropna().unique() + time_diff = pd.to_datetime(gen_df.index.get_level_values("datetime")).to_series().diff().dropna().unique() if pd.Timedelta(minutes=1) in time_diff: return ( "The generated dataframe is not daily. The implementation is definitely wrong. Please check the implementation.", @@ -548,11 +541,7 @@ class FactorFinalDecisionEvaluator(Evaluator): final_decision = final_evaluation_dict["final_decision"] final_feedback = final_evaluation_dict["final_feedback"] - if isinstance(final_decision, str) and final_decision.lower() in ("true", "false"): - final_decision = final_decision.lower() == "true" - elif isinstance(final_decision, int) and final_decision in (0, 1): - final_decision = bool(final_decision) - + final_decision = str(final_decision).lower() in ["true", "1"] return final_decision, final_feedback except json.JSONDecodeError as e: