update all (#530)

rdagent/components/coder/data_science/model/eval.py

@@ -73,15 +73,23 @@ class ModelGeneralCaseSpecEvaluator(CoSTEEREvaluator):
                 "The execution output contains too many progress bars and results in the LLM's token size exceeding the limit."
             )
 
+        if "main.py" in implementation.file_dict:
+            workflow_stdout = implementation.execute(env=de, entry="python main.py")
+        else:
+            workflow_stdout = None
+
         system_prompt = T(".prompts:model_eval.system").r(
             task_desc=target_task.get_task_information(),
             test_code=test_code,
             scenario=self.scen.get_scenario_all_desc(),
             spec=implementation.file_dict["spec/model.md"],
+            workflow_stdout=workflow_stdout,
+            workflow_code=implementation.all_codes,
         )
         user_prompt = T(".prompts:model_eval.user").r(
             stdout=stdout,
             code=implementation.file_dict[f"{target_task.name}.py"],
+            workflow_stdout=workflow_stdout,
         )
         resp = APIBackend().build_messages_and_create_chat_completion(user_prompt, system_prompt, json_mode=True)
         return ModelSingleFeedback(**json.loads(resp))

rdagent/components/coder/data_science/model/prompts.yaml

@@ -116,6 +116,13 @@ model_eval:
 
         Only if there is "Model code test passed successfully." in the stdout, then the model is considered successful, or else there must be some issues with the model.
         If no stdout is provided, the model is considered to have failed due to a timeout. Please check if there are any ways to improve the model's execution speed.
+        
+        {% if workflow_stdout is not none %}
+        Your model code is also part of the whole workflow, the user also tested the whole workflow and provided you the stdout.
+        The whole workflow code is:
+        {{workflow_code}}
+        Please consider both stdout and approve the code when both the model test and the whole workflow test pass.
+        {% endif %}
     
         Please respond with your feedback in the following JSON format and order:
         ```json
@@ -128,9 +135,11 @@ model_eval:
         ```
 
     user: |-
-        --------------Code generated by user:---------------
+        ---------------Code generated by user:---------------
         {{ code }}
-        --------------stdoutput:---------------
-        '''
-        {{ stdout }}
-        '''
+        ---------------Model test stdout:---------------
+        {{stdout}}   
+        {% if workflow_stdout is not none %}
+        ---------------Whole workflow test stdout:---------------
+        {{workflow_stdout}}
+        {% endif %}