update all (#530)

This commit is contained in:
Xu Yang
2025-01-22 22:22:48 +08:00
committed by GitHub
parent 5d04badc6a
commit e3205fd064
10 changed files with 162 additions and 28 deletions
@@ -58,12 +58,22 @@ class DataLoaderCoSTEEREvaluator(CoSTEEREvaluator):
implementation.inject_files(**{fname: test_code})
stdout = implementation.execute(env=de, entry=f"python {fname}")
if "main.py" in implementation.file_dict:
workflow_stdout = implementation.execute(env=de, entry="python main.py")
else:
workflow_stdout = None
system_prompt = T(".prompts:data_loader_eval.system").r(
task_desc=target_task.get_task_information(),
test_code=test_code,
code=implementation.file_dict["load_data.py"],
workflow_stdout=workflow_stdout,
workflow_code=implementation.all_codes,
)
user_prompt = T(".prompts:data_loader_eval.user").r(
stdout=stdout,
workflow_stdout=workflow_stdout,
)
user_prompt = T(".prompts:data_loader_eval.user").r(stdout=stdout)
resp = APIBackend().build_messages_and_create_chat_completion(user_prompt, system_prompt, json_mode=True)
return DataLoaderEvalFeedback(**json.loads(resp))