feat: added running time statistics for the DS scenario experiment (#1007)

* added running time statistics for the DS scenario experiment

* update execute_ret_code to return running_time

* fix

* fix

* update describe

* add EnvResult

* update corresponding calls

* add RunningInfo class

* fix

* fix

* fix

* fix ci

* rename function name

* fix ci

* fix

* refine running_time logic

* fix ci

rdagent/components/coder/data_science/ensemble/eval.py

@@ -63,7 +63,9 @@ class EnsembleCoSTEEREvaluator(CoSTEEREvaluator):
         )
 
         implementation.inject_files(**{fname: test_code})
-        stdout, ret_code = implementation.execute_ret_code(env=env, entry=f"python {fname}")
+        result = implementation.run(env=env, entry=f"python {fname}")
+        stdout = result.stdout
+        ret_code = result.ret_code
 
         stdout += f"\nNOTE: the above scripts run with return code {ret_code}"
 

rdagent/components/coder/data_science/feature/eval.py

@@ -50,9 +50,9 @@ class FeatureCoSTEEREvaluator(CoSTEEREvaluator):
         test_code = (DIRNAME / "eval_tests" / "feature_test.txt").read_text()
         implementation.inject_files(**{fname: test_code})
 
-        stdout, ret_code = implementation.execute_ret_code(env=env, entry=f"python {fname}")
+        result = implementation.run(env=env, entry=f"python {fname}")
 
-        if "main.py" in implementation.file_dict and ret_code == 0:
+        if "main.py" in implementation.file_dict and result.ret_code == 0:
             workflow_stdout = implementation.execute(env=env, entry="python main.py")
             workflow_stdout = remove_eda_part(workflow_stdout)
         else:
@@ -66,7 +66,7 @@ class FeatureCoSTEEREvaluator(CoSTEEREvaluator):
             workflow_code=implementation.all_codes,
         )
         user_prompt = T(".prompts:feature_eval.user").r(
-            stdout=shrink_text(stdout),
+            stdout=shrink_text(result.stdout),
             workflow_stdout=workflow_stdout,
         )
 
@@ -76,6 +76,6 @@ class FeatureCoSTEEREvaluator(CoSTEEREvaluator):
             user_prompt=user_prompt,
             init_kwargs_update_func=FeatureEvalFeedback.val_and_update_init_dict,
         )
-        fb.final_decision = fb.final_decision and ret_code == 0
+        fb.final_decision = fb.final_decision and result.ret_code == 0
 
         return fb

rdagent/components/coder/data_science/model/eval.py

@@ -67,7 +67,9 @@ class ModelGeneralCaseSpecEvaluator(CoSTEEREvaluator):
                 (DIRNAME / "eval_tests" / "model_test.txt").read_text().replace("model01", target_task.name)
             )  # only check the model changed this time
             implementation.inject_files(**{fname: test_code})
-            stdout, ret_code = implementation.execute_ret_code(env=env, entry=f"python {fname}")
+            result = implementation.run(env=env, entry=f"python {fname}")
+            stdout = result.stdout
+            ret_code = result.ret_code
 
             if stdout is None:
                 raise CoderError(
@@ -113,6 +115,6 @@ class ModelGeneralCaseSpecEvaluator(CoSTEEREvaluator):
             user_prompt=user_prompt,
             init_kwargs_update_func=ModelSingleFeedback.val_and_update_init_dict,
         )
-        fb.final_decision = fb.final_decision and ret_code == 0
+        fb.final_decision = fb.final_decision and result.ret_code == 0
 
         return fb

rdagent/components/coder/data_science/pipeline/eval.py

@@ -57,8 +57,10 @@ class PipelineCoSTEEREvaluator(CoSTEEREvaluator):
 
         # Clean the scores.csv & submission.csv.
         implementation.execute(env=env, entry=get_clear_ws_cmd())
-        stdout, execute_ret_code = implementation.execute_ret_code(env=env, entry=f"python -m coverage run main.py")
-        stdout = remove_eda_part(stdout)
+        result = implementation.run(env=env, entry=f"python -m coverage run main.py")
+        implementation.running_info.running_time = result.running_time
+        execute_ret_code = result.ret_code
+        stdout = remove_eda_part(result.stdout)
         stdout += f"The code executed {'successfully' if execute_ret_code == 0 else 'failed'}."
 
         score_fp = implementation.workspace_path / "scores.csv"
@@ -105,9 +107,9 @@ class PipelineCoSTEEREvaluator(CoSTEEREvaluator):
             base_check_code = T(".eval_tests.submission_format_test", ftype="txt").r()
             implementation.inject_files(**{"test/submission_format_test.py": base_check_code})
             # stdout += "----Submission Check 1-----\n"
-            submission_check_out, submission_ret_code = implementation.execute_ret_code(
-                env=env, entry="python test/submission_format_test.py"
-            )
+            submission_result = implementation.run(env=env, entry="python test/submission_format_test.py")
+            submission_check_out = submission_result.stdout
+            submission_ret_code = submission_result.ret_code
             if DS_RD_SETTING.rule_base_eval:
                 if execute_ret_code == 0 and score_ret_code == 0 and submission_ret_code == 0:
                     return PipelineSingleFeedback(

rdagent/components/coder/data_science/raw_data_loader/eval.py

@@ -52,7 +52,9 @@ class DataLoaderCoSTEEREvaluator(CoSTEEREvaluator):
         fname = "test/data_loader_test.py"
         test_code = (DIRNAME / "eval_tests" / "data_loader_test.txt").read_text()
         implementation.inject_files(**{fname: test_code})
-        stdout, ret_code = implementation.execute_ret_code(env=env, entry=f"python {fname}")
+        result = implementation.run(env=env, entry=f"python {fname}")
+        stdout = result.stdout
+        ret_code = result.ret_code
         match = re.search(r"(.*?)=== Start of EDA part ===(.*)=== End of EDA part ===(.*)", stdout, re.DOTALL)
         stdout_part_1, eda_output, stdout_part_2 = match.groups() if match else (stdout, None, "")
         stdout = stdout_part_1 + stdout_part_2

rdagent/components/coder/data_science/workflow/eval.py

@@ -121,9 +121,9 @@ class WorkflowGeneralCaseSpecEvaluator(CoSTEEREvaluator):
         base_check_code = T(".eval_tests.submission_format_test", ftype="txt").r()
         implementation.inject_files(**{"test/submission_format_test.py": base_check_code})
         # stdout += "----Submission Check 1-----\n"
-        submission_check_out, submission_ret_code = implementation.execute_ret_code(
-            env=env, entry="python test/submission_format_test.py"
-        )
+        submission_result = implementation.run(env=env, entry="python test/submission_format_test.py")
+        submission_check_out = submission_result.stdout
+        submission_ret_code = submission_result.ret_code
         stdout += "\n" + submission_check_out
 
         system_prompt = T(".prompts:workflow_eval.system").r(