chore: rename ens_and_decision (#541)

* rename ens_and_decision

* clarify workflow prompt

* rename

rdagent/components/coder/data_science/ensemble/eval_tests/ensemble_test.txt

@@ -7,11 +7,14 @@ A qualified ensemble implementation should:
 """
 
 import numpy as np
+import pandas as pd
 from pathlib import Path
 from sklearn.model_selection import train_test_split
+import torch
+import tensorflow as tf
 from load_data import load_data
 from feature import feat_eng
-from ensemble import ens_and_decision
+from ensemble import ensemble_workflow
 
 X, y, test_X, test_ids = load_data()
 X, y, test_X = feat_eng(X, y, test_X)
@@ -46,16 +49,19 @@ for key in val_preds_dict.keys():
         print(f"Model {key} test predictions (test_preds_dict[key]) shape: {test_preds_dict[key].shape}")
 
 # Run ensemble
-final_pred = ens_and_decision(test_preds_dict, val_preds_dict, val_y)
+final_pred = ensemble_workflow(test_preds_dict, val_preds_dict, val_y)
+
+# Check type
+pred_type = type(next(iter(test_preds_dict.values())))
+assert isinstance(final_pred, pred_type), (
+    f"Type mismatch: 'final_pred' is of type {type(final_pred)}, but expected {pred_type} "
+)
 
 # Check shape
-if isinstance(final_pred, list):
+if isinstance(final_pred, (list, np.ndarray, pd.DataFrame, torch.Tensor, tf.Tensor)):
     assert len(final_pred) == len(test_X), (
-        f"Wrong output sample size: len(final_pred) ({len(final_pred)}) and len(test_X) ({len(test_X)})"
-    )
-else:
-    assert final_pred.shape[0] == test_X.shape[0], (
-        f"Wrong output sample size: final_pred.shape[0] ({final_pred.shape[0]}) and test_X.shape[0] ({test_X.shape[0]})"
+        f"Wrong output sample size: len(final_pred)={len(final_pred)} "
+        f"vs. len(test_X)={len(test_X)}"
     )
 
 # check if scores.csv is generated

rdagent/components/coder/data_science/raw_data_loader/prompts.yaml

@@ -188,7 +188,7 @@ spec:
     ensemble: |-
       Ensemble specification text adhere to the following requirements:
       1. Function Interface:
-        - Function Name: `ens_and_decision`
+        - Function Name: `ensemble_workflow`
         - Parameters:
           - `test_preds_dict` (Dict[str, DT]): A dictionary of test predictions from different models. The key is the model file name.
           - `val_preds_dict` (Dict[str, DT]): A dictionary of validation predictions from different models. The key is the model file name.
@@ -248,9 +248,14 @@ spec:
           - Integrate the following components into the workflow:
             - Data loading (`load_data.py`).
             - Feature engineering (`feature.py`).
-            - Model workflow for training and testing (`model_*.py`).
-            - Ensemble and decision-making (`ensemble.py`).
+            - Model workflow for training and testing (`model_*.py`). 
+            - Ensemble workflow that combines results from the model workflow to obtain the final prediction (`ensemble.py`).
           - Treat each component as a modular and callable Python function.
+          - The workflow script should be flexible enough to handle either a single model or multiple models, with filenames (model_*.py) that are not determined at the outset.
+            For multiple model selection, utilize Python code to identify eligible models based on filenames, for example:
+            ```python
+            available_models = [f for f in os.listdir('.') if f.startswith('model_') and 'test' not in f]
+            ```
         2. Feature Engineering
           - The feature engineering should be called only once. For example:
             `X_transformed, y_transformed, X_test_transformed = feat_eng(X, y, X_test)`
@@ -284,11 +289,11 @@ spec:
               test_X=test_X
           )
           {% endfor %}
-          final_pred = ens_and_decision(test_preds_dict, val_preds_dict, val_y)
+          final_pred = ensemble_workflow(test_preds_dict, val_preds_dict, val_y)
           {% endraw %}
 
         {% if latest_spec %}
-        5. Former Specification:
+        7. Former Specification:
           {{ latest_spec }}
           You should follow the provided specifications to improve this task.
         {% endif %}

rdagent/scenarios/kaggle/tpl_ex/aerial-cactus-identification/ensemble.py

@@ -3,7 +3,7 @@ import pandas as pd
 from sklearn.metrics import roc_auc_score
 
 
-def ens_and_decision(test_pred_l: list[np.ndarray], val_pred_l: list[np.ndarray], val_label: np.ndarray) -> np.ndarray:
+def ensemble_workflow(test_pred_l: list[np.ndarray], val_pred_l: list[np.ndarray], val_label: np.ndarray) -> np.ndarray:
     """
     Handle the following:
     1) Ensemble predictions using a simple average.

rdagent/scenarios/kaggle/tpl_ex/aerial-cactus-identification/main.py

@@ -25,9 +25,9 @@ val_pred, test_pred, _ = model_workflow(train_images, train_labels, validation_i
 
 
 # Ensemble
-from ensemble import ens_and_decision
+from ensemble import ensemble_workflow
 
-pred_binary = ens_and_decision([test_pred], [val_pred], validation_labels)
+pred_binary = ensemble_workflow([test_pred], [val_pred], validation_labels)
 
 
 # Save

rdagent/scenarios/kaggle/tpl_ex/aerial-cactus-identification/spec/ensemble.md

@@ -3,7 +3,7 @@
 - Implement a function for ensemble and decision making with the following signature:
 
 ```python
-def ens_and_decision(test_pred_l: list[np.ndarray], val_pred_l: list[np.ndarray], val_label: np.ndarray) -> np.ndarray:
+def ensemble_workflow(test_pred_l: list[np.ndarray], val_pred_l: list[np.ndarray], val_label: np.ndarray) -> np.ndarray:
     """
     Handle the following:
     1) Ensemble predictions using a simple average.