feat: add stdout into workspace for easier debugging (#1236)

* refactor: use get_truncated_stdout for consistent stdout handling across modules

* lint

* feat: add dump_stdout_type to DSRunnerCoSTEERSettings and use in eval

* fix: avoid circular import by moving DSRunnerEvaluator import inside method

rdagent/components/coder/data_science/ensemble/eval.py

@@ -67,7 +67,7 @@ class EnsembleCoSTEEREvaluator(CoSTEEREvaluator):
 
         implementation.inject_files(**{fname: test_code})
         result = implementation.run(env=env, entry=f"python {fname}")
-        stdout = result.stdout
+        stdout = result.get_truncated_stdout()
         ret_code = result.exit_code
 
         stdout += f"\nNOTE: the above scripts run with return code {ret_code}"

rdagent/components/coder/data_science/feature/eval.py

@@ -69,7 +69,7 @@ class FeatureCoSTEEREvaluator(CoSTEEREvaluator):
             workflow_code=implementation.all_codes,
         )
         user_prompt = T(".prompts:feature_eval.user").r(
-            stdout=shrink_text(result.stdout),
+            stdout=result.get_truncated_stdout(),
             workflow_stdout=workflow_stdout,
         )
 

rdagent/components/coder/data_science/model/eval.py

@@ -71,7 +71,7 @@ class ModelGeneralCaseSpecEvaluator(CoSTEEREvaluator):
             )  # only check the model changed this time
             implementation.inject_files(**{fname: test_code})
             result = implementation.run(env=env, entry=f"python {fname}")
-            stdout = result.stdout
+            stdout = result.get_truncated_stdout()
             ret_code = result.exit_code
 
             if stdout is None:
@@ -118,6 +118,6 @@ class ModelGeneralCaseSpecEvaluator(CoSTEEREvaluator):
             user_prompt=user_prompt,
             init_kwargs_update_func=ModelSingleFeedback.val_and_update_init_dict,
         )
-        fb.final_decision = fb.final_decision and result.exit_code == 0
+        fb.final_decision = fb.final_decision and ret_code == 0
 
         return fb

rdagent/components/coder/data_science/pipeline/eval.py

@@ -70,6 +70,7 @@ class PipelineCoSTEEREvaluator(CoSTEEREvaluator):
             result = implementation.run(
                 env=env, entry=f"strace -e trace=file -f -o trace.log python -m coverage run main.py"
             )
+        result_stdout = result.get_truncated_stdout()
 
         nb_conversion_ret_code = 0
         nb_conversion_check_text = ""
@@ -84,7 +85,7 @@ class PipelineCoSTEEREvaluator(CoSTEEREvaluator):
                 notebook_converter.convert(
                     task=target_task,
                     code=code,
-                    stdout=result.stdout,
+                    stdout=result_stdout,
                     outfile=implementation.workspace_path / "main.ipynb",
                     use_debug_flag=DS_RD_SETTING.sample_data_by_LLM,
                 )
@@ -103,16 +104,16 @@ class PipelineCoSTEEREvaluator(CoSTEEREvaluator):
                     stdout += f"Code opened the sample submission file '{sample_submission_file_name}' during execution.\n Reject the implementation!\n"
                     sample_submission_check = False
 
-        result.stdout = remove_eda_part(result.stdout)
+        result_stdout = remove_eda_part(result_stdout)
         if result.exit_code != 0:
-            stdout += f"Code failed to run. Please check the stdout:\n Following the stdout of the debug mode run:\n{result.stdout.strip()}\n"
+            stdout += f"Code failed to run. Please check the stdout:\n Following the stdout of the debug mode run:\n{result_stdout.strip()}\n"
         else:
-            stdout += f"Code ran successfully.\n Following the stdout of the debug mode run:\n{result.stdout.strip()}\n"
+            stdout += f"Code ran successfully.\n Following the stdout of the debug mode run:\n{result_stdout.strip()}\n"
         if DS_RD_SETTING.sample_data_by_LLM:
             debug_time, full_estimated_time = None, None
-            if match := re.search(r"debug_time:\s*(\d+(?:.\d+)?)", result.stdout, re.DOTALL):
+            if match := re.search(r"debug_time:\s*(\d+(?:.\d+)?)", result_stdout, re.DOTALL):
                 debug_time = float(match.group(1))
-            if match := re.search(r"estimated_time:\s*(\d+(?:.\d+)?)", result.stdout, re.DOTALL):
+            if match := re.search(r"estimated_time:\s*(\d+(?:.\d+)?)", result_stdout, re.DOTALL):
                 full_estimated_time = float(match.group(1))
             if debug_time is not None and full_estimated_time is not None:
                 stdout += f"Debug mode ran in {debug_time:.2f} seconds, estimated full run time is {full_estimated_time:.2f} seconds. The estimated time is {full_estimated_time / env.conf.running_timeout_period * 100:.2f}% the debug time."
@@ -167,7 +168,7 @@ class PipelineCoSTEEREvaluator(CoSTEEREvaluator):
             implementation.inject_files(**{"test/submission_format_test.py": base_check_code})
             # stdout += "----Submission Check 1-----\n"
             submission_result = implementation.run(env=env, entry="python test/submission_format_test.py")
-            submission_check_out = submission_result.stdout
+            submission_check_out = submission_result.get_truncated_stdout()
             submission_ret_code = submission_result.exit_code
             stdout += "\n" + submission_check_out
 

rdagent/components/coder/data_science/raw_data_loader/eval.py

@@ -56,7 +56,7 @@ class DataLoaderCoSTEEREvaluator(CoSTEEREvaluator):
         test_code = (DIRNAME / "eval_tests" / "data_loader_test.txt").read_text()
         implementation.inject_files(**{fname: test_code})
         result = implementation.run(env=env, entry=f"python {fname}")
-        stdout = result.stdout
+        stdout = result.get_truncated_stdout()
         ret_code = result.exit_code
         match = re.search(r"(.*?)=== Start of EDA part ===(.*)=== End of EDA part ===(.*)", stdout, re.DOTALL)
         stdout_part_1, eda_output, stdout_part_2 = match.groups() if match else (stdout, None, "")

rdagent/components/coder/data_science/workflow/eval.py

@@ -125,7 +125,7 @@ class WorkflowGeneralCaseSpecEvaluator(CoSTEEREvaluator):
         implementation.inject_files(**{"test/submission_format_test.py": base_check_code})
         # stdout += "----Submission Check 1-----\n"
         submission_result = implementation.run(env=env, entry="python test/submission_format_test.py")
-        submission_check_out = submission_result.stdout
+        submission_check_out = submission_result.get_truncated_stdout()
         submission_ret_code = submission_result.exit_code
         stdout += "\n" + submission_check_out