update all (#530)

rdagent/components/coder/data_science/feature/eval.py

@@ -57,12 +57,22 @@ class FeatureCoSTEEREvaluator(CoSTEEREvaluator):
 
         stdout = implementation.execute(env=de, entry=f"python {fname}")
 
+        if "main.py" in implementation.file_dict:
+            workflow_stdout = implementation.execute(env=de, entry="python main.py")
+        else:
+            workflow_stdout = None
+
         system_prompt = T(".prompts:feature_eval.system").r(
             task_desc=target_task.get_task_information(),
             test_code=test_code,
             code=implementation.file_dict["feature.py"],
+            workflow_stdout=workflow_stdout,
+            workflow_code=implementation.all_codes,
+        )
+        user_prompt = T(".prompts:feature_eval.user").r(
+            stdout=shrink_text(stdout),
+            workflow_stdout=workflow_stdout,
         )
-        user_prompt = T(".prompts:feature_eval.user").r(stdout=shrink_text(stdout))
 
         resp = APIBackend().build_messages_and_create_chat_completion(user_prompt, system_prompt, json_mode=True)
         return FeatureEvalFeedback(**json.loads(resp))

rdagent/components/coder/data_science/feature/eval_tests/feature_test.txt

@@ -8,6 +8,7 @@ Please make sure the stdout is rich enough to support informative feedback
 """
 
 import pickle
+from copy import deepcopy
 
 import numpy as np
 import pandas as pd
@@ -19,17 +20,44 @@ print(f"X.shape: {X.shape}")
 print(f"y.shape: {y.shape}" if not isinstance(y, list) else f"y(list)'s length: {len(y)}")
 print(f"X_test.shape: {X_test.shape}")
 print(f"test_ids length: {len(test_ids)}")
+X_loaded = deepcopy(X)
+y_loaded = deepcopy(y)
+X_test_loaded = deepcopy(X_test)
 X, y, X_test = feat_eng(X, y, X_test)
 
 
 def get_length(data):
     return len(data) if isinstance(data, list) else data.shape[0]
 
-assert get_length(X_test) == get_length(test_ids), (
-    f"Mismatch in length of test images and test IDs: X_test ({get_length(X_test)}) and test_ids ({get_length(test_ids)})"
-)
-assert get_length(X) == get_length(y), (
-    f"Mismatch in length of training images and labels: X ({get_length(X)}) and y ({get_length(y)})"
-)
 
-print("Feature Engineering test passed successfully. Length of test images matches length of test IDs.")
+def get_width(data):
+    return 1 if isinstance(data, list) else data.shape[1:]
+
+
+def get_column_list(data):
+    return data.columns.tolist() if isinstance(data, pd.DataFrame) else None
+
+
+assert get_length(X_test) == get_length(
+    test_ids
+), f"Mismatch in length of test images and test IDs: X_test ({get_length(X_test)}) and test_ids ({get_length(test_ids)})"
+assert get_length(X) == get_length(
+    y
+), f"Mismatch in length of training images and labels: X ({get_length(X)}) and y ({get_length(y)})"
+
+assert get_length(X) != 0, f"Training data is empty."
+assert get_length(y) != 0, f"Training labels are empty."
+assert get_length(X_test) != 0, f"Test data is empty."
+
+assert get_width(X) == get_width(
+    X_test
+), "Mismatch in width of training and test data. Width means the number of features."
+
+if isinstance(X, pd.DataFrame) and isinstance(X_test, pd.DataFrame):
+    assert get_column_list(X) == get_column_list(X_test), "Mismatch in column names of training and test data."
+
+assert sorted(X.dtypes.unique().tolist()) == sorted(
+    X_loaded.dtypes.unique().tolist()
+), f"feature engineering has produced new data types which is not allowed, data loader data types are {X_loaded.dtypes.unique().tolist()} and feature engineering data types are {X.dtypes.unique().tolist()}"
+
+print("Feature Engineering test passed successfully. All checks including length, width, and data types have been validated.")

rdagent/components/coder/data_science/feature/prompts.yaml

@@ -74,7 +74,14 @@ feature_eval:
     ```python
     {{test_code}}
     ```
-
+    
+    {% if workflow_stdout is not none %}
+    Your feature engineering code is also part of the whole workflow, the user also tested the whole workflow and provided you the stdout.
+    The whole workflow code is:
+    {{workflow_code}}
+    Please consider both stdout and approve the code when both the feature engineering test and the whole workflow test pass.
+    {% endif %}
+    
     You'll be given the stdout of your testing scripts.
     Please respond with your feedback in the following JSON format and order
     ```json
@@ -86,6 +93,9 @@ feature_eval:
     }
     ```
   user: |-
-    ```
-    {{stdout}}
-    ```
+    Feature engineering test stdout:
+    {{stdout}}   
+    {% if workflow_stdout is not none %}
+    Whole workflow test stdout:
+    {{workflow_stdout}}
+    {% endif %}