diff --git a/rdagent/components/coder/data_science/ensemble/eval_tests/ensemble_test.txt b/rdagent/components/coder/data_science/ensemble/eval_tests/ensemble_test.txt index 5a716731..7875558f 100644 --- a/rdagent/components/coder/data_science/ensemble/eval_tests/ensemble_test.txt +++ b/rdagent/components/coder/data_science/ensemble/eval_tests/ensemble_test.txt @@ -7,11 +7,14 @@ A qualified ensemble implementation should: """ import numpy as np +import pandas as pd from pathlib import Path from sklearn.model_selection import train_test_split +import torch +import tensorflow as tf from load_data import load_data from feature import feat_eng -from ensemble import ens_and_decision +from ensemble import ensemble_workflow X, y, test_X, test_ids = load_data() X, y, test_X = feat_eng(X, y, test_X) @@ -46,16 +49,19 @@ for key in val_preds_dict.keys(): print(f"Model {key} test predictions (test_preds_dict[key]) shape: {test_preds_dict[key].shape}") # Run ensemble -final_pred = ens_and_decision(test_preds_dict, val_preds_dict, val_y) +final_pred = ensemble_workflow(test_preds_dict, val_preds_dict, val_y) + +# Check type +pred_type = type(next(iter(test_preds_dict.values()))) +assert isinstance(final_pred, pred_type), ( + f"Type mismatch: 'final_pred' is of type {type(final_pred)}, but expected {pred_type} " +) # Check shape -if isinstance(final_pred, list): +if isinstance(final_pred, (list, np.ndarray, pd.DataFrame, torch.Tensor, tf.Tensor)): assert len(final_pred) == len(test_X), ( - f"Wrong output sample size: len(final_pred) ({len(final_pred)}) and len(test_X) ({len(test_X)})" - ) -else: - assert final_pred.shape[0] == test_X.shape[0], ( - f"Wrong output sample size: final_pred.shape[0] ({final_pred.shape[0]}) and test_X.shape[0] ({test_X.shape[0]})" + f"Wrong output sample size: len(final_pred)={len(final_pred)} " + f"vs. len(test_X)={len(test_X)}" ) # check if scores.csv is generated diff --git a/rdagent/components/coder/data_science/raw_data_loader/prompts.yaml b/rdagent/components/coder/data_science/raw_data_loader/prompts.yaml index be607191..cd74e447 100644 --- a/rdagent/components/coder/data_science/raw_data_loader/prompts.yaml +++ b/rdagent/components/coder/data_science/raw_data_loader/prompts.yaml @@ -188,7 +188,7 @@ spec: ensemble: |- Ensemble specification text adhere to the following requirements: 1. Function Interface: - - Function Name: `ens_and_decision` + - Function Name: `ensemble_workflow` - Parameters: - `test_preds_dict` (Dict[str, DT]): A dictionary of test predictions from different models. The key is the model file name. - `val_preds_dict` (Dict[str, DT]): A dictionary of validation predictions from different models. The key is the model file name. @@ -248,9 +248,14 @@ spec: - Integrate the following components into the workflow: - Data loading (`load_data.py`). - Feature engineering (`feature.py`). - - Model workflow for training and testing (`model_*.py`). - - Ensemble and decision-making (`ensemble.py`). + - Model workflow for training and testing (`model_*.py`). + - Ensemble workflow that combines results from the model workflow to obtain the final prediction (`ensemble.py`). - Treat each component as a modular and callable Python function. + - The workflow script should be flexible enough to handle either a single model or multiple models, with filenames (model_*.py) that are not determined at the outset. + For multiple model selection, utilize Python code to identify eligible models based on filenames, for example: + ```python + available_models = [f for f in os.listdir('.') if f.startswith('model_') and 'test' not in f] + ``` 2. Feature Engineering - The feature engineering should be called only once. For example: `X_transformed, y_transformed, X_test_transformed = feat_eng(X, y, X_test)` @@ -284,11 +289,11 @@ spec: test_X=test_X ) {% endfor %} - final_pred = ens_and_decision(test_preds_dict, val_preds_dict, val_y) + final_pred = ensemble_workflow(test_preds_dict, val_preds_dict, val_y) {% endraw %} {% if latest_spec %} - 5. Former Specification: + 7. Former Specification: {{ latest_spec }} You should follow the provided specifications to improve this task. {% endif %} diff --git a/rdagent/scenarios/kaggle/tpl_ex/aerial-cactus-identification/ensemble.py b/rdagent/scenarios/kaggle/tpl_ex/aerial-cactus-identification/ensemble.py index 76b08907..c5508cce 100644 --- a/rdagent/scenarios/kaggle/tpl_ex/aerial-cactus-identification/ensemble.py +++ b/rdagent/scenarios/kaggle/tpl_ex/aerial-cactus-identification/ensemble.py @@ -3,7 +3,7 @@ import pandas as pd from sklearn.metrics import roc_auc_score -def ens_and_decision(test_pred_l: list[np.ndarray], val_pred_l: list[np.ndarray], val_label: np.ndarray) -> np.ndarray: +def ensemble_workflow(test_pred_l: list[np.ndarray], val_pred_l: list[np.ndarray], val_label: np.ndarray) -> np.ndarray: """ Handle the following: 1) Ensemble predictions using a simple average. diff --git a/rdagent/scenarios/kaggle/tpl_ex/aerial-cactus-identification/main.py b/rdagent/scenarios/kaggle/tpl_ex/aerial-cactus-identification/main.py index d879bcb3..5e0776d0 100644 --- a/rdagent/scenarios/kaggle/tpl_ex/aerial-cactus-identification/main.py +++ b/rdagent/scenarios/kaggle/tpl_ex/aerial-cactus-identification/main.py @@ -25,9 +25,9 @@ val_pred, test_pred, _ = model_workflow(train_images, train_labels, validation_i # Ensemble -from ensemble import ens_and_decision +from ensemble import ensemble_workflow -pred_binary = ens_and_decision([test_pred], [val_pred], validation_labels) +pred_binary = ensemble_workflow([test_pred], [val_pred], validation_labels) # Save diff --git a/rdagent/scenarios/kaggle/tpl_ex/aerial-cactus-identification/spec/ensemble.md b/rdagent/scenarios/kaggle/tpl_ex/aerial-cactus-identification/spec/ensemble.md index 68326d19..2c29ad8a 100644 --- a/rdagent/scenarios/kaggle/tpl_ex/aerial-cactus-identification/spec/ensemble.md +++ b/rdagent/scenarios/kaggle/tpl_ex/aerial-cactus-identification/spec/ensemble.md @@ -3,7 +3,7 @@ - Implement a function for ensemble and decision making with the following signature: ```python -def ens_and_decision(test_pred_l: list[np.ndarray], val_pred_l: list[np.ndarray], val_label: np.ndarray) -> np.ndarray: +def ensemble_workflow(test_pred_l: list[np.ndarray], val_pred_l: list[np.ndarray], val_label: np.ndarray) -> np.ndarray: """ Handle the following: 1) Ensemble predictions using a simple average.