feat: added running time statistics for the DS scenario experiment (#1007)

* added running time statistics for the DS scenario experiment

* update execute_ret_code to return running_time

* fix

* fix

* update describe

* add EnvResult

* update corresponding calls

* add RunningInfo class

* fix

* fix

* fix

* fix ci

* rename function name

* fix ci

* fix

* refine running_time logic

* fix ci
This commit is contained in:
Yuante Li
2025-07-02 15:11:18 +08:00
committed by GitHub
parent cbab4830c3
commit 667af3e1ea
18 changed files with 143 additions and 91 deletions
@@ -67,7 +67,9 @@ class ModelGeneralCaseSpecEvaluator(CoSTEEREvaluator):
(DIRNAME / "eval_tests" / "model_test.txt").read_text().replace("model01", target_task.name)
) # only check the model changed this time
implementation.inject_files(**{fname: test_code})
stdout, ret_code = implementation.execute_ret_code(env=env, entry=f"python {fname}")
result = implementation.run(env=env, entry=f"python {fname}")
stdout = result.stdout
ret_code = result.ret_code
if stdout is None:
raise CoderError(
@@ -113,6 +115,6 @@ class ModelGeneralCaseSpecEvaluator(CoSTEEREvaluator):
user_prompt=user_prompt,
init_kwargs_update_func=ModelSingleFeedback.val_and_update_init_dict,
)
fb.final_decision = fb.final_decision and ret_code == 0
fb.final_decision = fb.final_decision and result.ret_code == 0
return fb