feat: added running time statistics for the DS scenario experiment (#1007)

* added running time statistics for the DS scenario experiment

* update execute_ret_code to return running_time

* fix

* fix

* update describe

* add EnvResult

* update corresponding calls

* add RunningInfo class

* fix

* fix

* fix

* fix ci

* rename function name

* fix ci

* fix

* refine running_time logic

* fix ci
This commit is contained in:
Yuante Li
2025-07-02 15:11:18 +08:00
committed by GitHub
parent 92ce8a898b
commit 3d227787e8
18 changed files with 143 additions and 91 deletions
@@ -50,9 +50,9 @@ class FeatureCoSTEEREvaluator(CoSTEEREvaluator):
test_code = (DIRNAME / "eval_tests" / "feature_test.txt").read_text()
implementation.inject_files(**{fname: test_code})
stdout, ret_code = implementation.execute_ret_code(env=env, entry=f"python {fname}")
result = implementation.run(env=env, entry=f"python {fname}")
if "main.py" in implementation.file_dict and ret_code == 0:
if "main.py" in implementation.file_dict and result.ret_code == 0:
workflow_stdout = implementation.execute(env=env, entry="python main.py")
workflow_stdout = remove_eda_part(workflow_stdout)
else:
@@ -66,7 +66,7 @@ class FeatureCoSTEEREvaluator(CoSTEEREvaluator):
workflow_code=implementation.all_codes,
)
user_prompt = T(".prompts:feature_eval.user").r(
stdout=shrink_text(stdout),
stdout=shrink_text(result.stdout),
workflow_stdout=workflow_stdout,
)
@@ -76,6 +76,6 @@ class FeatureCoSTEEREvaluator(CoSTEEREvaluator):
user_prompt=user_prompt,
init_kwargs_update_func=FeatureEvalFeedback.val_and_update_init_dict,
)
fb.final_decision = fb.final_decision and ret_code == 0
fb.final_decision = fb.final_decision and result.ret_code == 0
return fb