fix: add metric name check for valid scores (#724)

* update metric_name

* fix some bugs

* add an evaluation in workflow

* add an evalution in runner

* fix ci

* test change

* fix CI

---------

Co-authored-by: TPLin22 <tplin2@163.com>
Co-authored-by: yuanteli <1957922024@qq.com>
This commit is contained in:
XianBW
2025-03-27 19:36:57 +08:00
committed by GitHub
parent 7733841f1c
commit b13575dc69
10 changed files with 45 additions and 23 deletions
@@ -31,6 +31,8 @@ class EnsembleCoSTEEREvaluator(CoSTEEREvaluator):
) -> EnsembleEvalFeedback:
target_task_information = target_task.get_task_information()
metric_name = self.scen.metric_name
if (
queried_knowledge is not None
and target_task_information in queried_knowledge.success_task_to_knowledge_dict
@@ -55,7 +57,8 @@ class EnsembleCoSTEEREvaluator(CoSTEEREvaluator):
.render(
model_names=[
fn[:-3] for fn in implementation.file_dict.keys() if fn.startswith("model_") and "test" not in fn
]
],
metric_name=metric_name,
)
)
@@ -73,6 +76,7 @@ class EnsembleCoSTEEREvaluator(CoSTEEREvaluator):
system_prompt = T(".prompts:ensemble_eval.system").r(
task_desc=target_task_information,
test_code=test_code,
metric_name=metric_name,
code=implementation.file_dict["ensemble.py"],
workflow_stdout=workflow_stdout,
workflow_code=implementation.all_codes,