From 7707f60bc97613dfc43c80e71af19f71dd5482db Mon Sep 17 00:00:00 2001 From: WinstonLiyt <104308117+WinstonLiyt@users.noreply.github.com> Date: Thu, 26 Sep 2024 02:12:07 +0800 Subject: [PATCH] feat: refine the template in several Kaggle competitions (#343) * add an error catching in model runner * check forest & s3e11 & s4e8 & spaceship, change params of forest & s3e26 & spaceship * fix ci errors --------- Co-authored-by: TPLin22 --- rdagent/scenarios/kaggle/developer/runner.py | 3 + .../cross_validation.py | 99 ------------------- .../model/model_catboost.py | 29 ++++++ .../model/model_dnn.py | 78 +++++++++++++++ .../model/model_randomforest.py | 2 +- .../model/model_xgboost.py | 2 +- .../train.py | 7 ++ .../model/model_xgboost.py | 2 +- .../playground-series-s3e11_template/train.py | 2 +- .../model/model_randomforest.py | 2 +- .../playground-series-s4e8_template/train.py | 2 +- .../model/model_nn.py | 78 --------------- .../model/model_nn.py | 6 +- .../model/model_randomforest.py | 2 +- .../model/model_xgboost.py | 2 +- .../spaceship-titanic_template/train.py | 1 + 16 files changed, 129 insertions(+), 188 deletions(-) delete mode 100644 rdagent/scenarios/kaggle/experiment/forest-cover-type-prediction_template/cross_validation.py create mode 100644 rdagent/scenarios/kaggle/experiment/forest-cover-type-prediction_template/model/model_catboost.py create mode 100644 rdagent/scenarios/kaggle/experiment/forest-cover-type-prediction_template/model/model_dnn.py delete mode 100644 rdagent/scenarios/kaggle/experiment/playground-series-s4e9_template/model/model_nn.py diff --git a/rdagent/scenarios/kaggle/developer/runner.py b/rdagent/scenarios/kaggle/developer/runner.py index 0785c937..f8c4078f 100644 --- a/rdagent/scenarios/kaggle/developer/runner.py +++ b/rdagent/scenarios/kaggle/developer/runner.py @@ -119,6 +119,9 @@ class KGModelRunner(KGCachedRunner[KGModelExperiment]): result = exp.experiment_workspace.execute(run_env=env_to_use) + if result is None: + raise CoderError("No result is returned from the experiment workspace") + exp.result = result if RUNNER_SETTINGS.cache_result: self.dump_cache_result(exp, result) diff --git a/rdagent/scenarios/kaggle/experiment/forest-cover-type-prediction_template/cross_validation.py b/rdagent/scenarios/kaggle/experiment/forest-cover-type-prediction_template/cross_validation.py deleted file mode 100644 index c09c924a..00000000 --- a/rdagent/scenarios/kaggle/experiment/forest-cover-type-prediction_template/cross_validation.py +++ /dev/null @@ -1,99 +0,0 @@ -import importlib.util -import random -from pathlib import Path - -import numpy as np -import pandas as pd -from scipy import stats -from sklearn.impute import SimpleImputer -from sklearn.metrics import accuracy_score -from sklearn.model_selection import KFold - -# Set random seed for reproducibility -SEED = 42 -random.seed(SEED) -np.random.seed(SEED) -DIRNAME = Path(__file__).absolute().resolve().parent - - -def import_module_from_path(module_name, module_path): - spec = importlib.util.spec_from_file_location(module_name, module_path) - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - return module - - -# 1) Preprocess the data -data_df = pd.read_csv("/kaggle/input/train.csv") -data_df = data_df.drop(["Id"], axis=1) - -X_train = data_df.drop(["Cover_Type"], axis=1) -y_train = data_df["Cover_Type"] - 1 - -# Set up KFold -kf = KFold(n_splits=5, shuffle=True, random_state=SEED) - -# Store results -accuracies = [] - -# 3) Train and evaluate using KFold -fold_number = 1 -for train_index, valid_index in kf.split(X_train): - print(f"Starting fold {fold_number}...") - - X_train_l, X_valid_l = [], [] # Reset feature lists for each fold - X_tr, X_val = X_train.iloc[train_index], X_train.iloc[valid_index] - y_tr, y_val = y_train.iloc[train_index], y_train.iloc[valid_index] - - # Feature engineering - for f in DIRNAME.glob("feature/feat*.py"): - cls = import_module_from_path(f.stem, f).feature_engineering_cls() - cls.fit(X_tr) - X_train_f = cls.transform(X_tr) - X_valid_f = cls.transform(X_val) - - X_train_l.append(X_train_f) - X_valid_l.append(X_valid_f) - - X_tr = pd.concat(X_train_l, axis=1, keys=[f"feature_{i}" for i in range(len(X_train_l))]) - X_val = pd.concat(X_valid_l, axis=1, keys=[f"feature_{i}" for i in range(len(X_valid_l))]) - - print("Shape of X_tr: ", X_tr.shape, " Shape of X_val: ", X_val.shape) - - # Replace inf and -inf with NaN - X_tr.replace([np.inf, -np.inf], np.nan, inplace=True) - X_val.replace([np.inf, -np.inf], np.nan, inplace=True) - - # Impute missing values - imputer = SimpleImputer(strategy="mean") - X_tr = pd.DataFrame(imputer.fit_transform(X_tr), columns=X_tr.columns) - X_val = pd.DataFrame(imputer.transform(X_val), columns=X_val.columns) - - # Remove duplicate columns - X_tr = X_tr.loc[:, ~X_tr.columns.duplicated()] - X_val = X_val.loc[:, ~X_val.columns.duplicated()] - - # Train the model - model_l = [] # list[tuple[model, predict_func]] - for f in DIRNAME.glob("model/model*.py"): - m = import_module_from_path(f.stem, f) - model_l.append((m.fit(X_tr, y_tr, X_val, y_val), m.predict)) - - # Evaluate the model on the validation set - y_valid_pred_l = [] - for model, predict_func in model_l: - y_valid_pred = predict_func(model, X_val) - y_valid_pred_l.append(y_valid_pred) - - # Majority vote ensemble - y_valid_pred_ensemble = stats.mode(y_valid_pred_l, axis=0)[0].flatten() - - # Compute metrics - accuracy = accuracy_score(y_val, y_valid_pred_ensemble) - accuracies.append(accuracy) - print(f"Fold {fold_number} accuracy: {accuracy}") - - fold_number += 1 - -# Print average accuracy -print(f"Average accuracy across folds: {np.mean(accuracies)}") diff --git a/rdagent/scenarios/kaggle/experiment/forest-cover-type-prediction_template/model/model_catboost.py b/rdagent/scenarios/kaggle/experiment/forest-cover-type-prediction_template/model/model_catboost.py new file mode 100644 index 00000000..a4420efa --- /dev/null +++ b/rdagent/scenarios/kaggle/experiment/forest-cover-type-prediction_template/model/model_catboost.py @@ -0,0 +1,29 @@ +import pandas as pd +from catboost import CatBoostClassifier + + +def fit(X_train: pd.DataFrame, y_train: pd.Series, X_valid: pd.DataFrame, y_valid: pd.Series): + # Define CatBoost parameters + cat_params = { + "iterations": 5000, + "learning_rate": 0.03, + "od_wait": 1000, + "depth": 7, + "task_type": "GPU", + "l2_leaf_reg": 3, + "eval_metric": "Accuracy", + "devices": "0", + "verbose": 1000, + } + + # Initialize and train the CatBoost model + model = CatBoostClassifier(**cat_params) + model.fit(X_train, y_train, eval_set=(X_valid, y_valid)) + + return model + + +def predict(model, X: pd.DataFrame): + # Predict using the trained model + y_pred = model.predict(X) + return y_pred.reshape(-1, 1) diff --git a/rdagent/scenarios/kaggle/experiment/forest-cover-type-prediction_template/model/model_dnn.py b/rdagent/scenarios/kaggle/experiment/forest-cover-type-prediction_template/model/model_dnn.py new file mode 100644 index 00000000..13ddc8e5 --- /dev/null +++ b/rdagent/scenarios/kaggle/experiment/forest-cover-type-prediction_template/model/model_dnn.py @@ -0,0 +1,78 @@ +import numpy as np +import pandas as pd +import torch +import torch.nn as nn +import torch.optim as optim +from torch.utils.data import DataLoader, TensorDataset + + +# Define the neural network model with Batch Normalization +class NeuralNetwork(nn.Module): + def __init__(self, input_size, num_classes): + super(NeuralNetwork, self).__init__() + self.layer1 = nn.Linear(input_size, 128) + self.bn1 = nn.BatchNorm1d(128) + self.layer2 = nn.Linear(128, 64) + self.bn2 = nn.BatchNorm1d(64) + self.layer3 = nn.Linear(64, num_classes) + + def forward(self, x): + x = torch.relu(self.bn1(self.layer1(x))) + x = torch.relu(self.bn2(self.layer2(x))) + x = torch.softmax(self.layer3(x), dim=1) + return x + + +def fit(X_train: pd.DataFrame, y_train: pd.DataFrame, X_valid: pd.DataFrame, y_valid: pd.DataFrame): + # Convert data to PyTorch tensors + X_train_tensor = torch.tensor(X_train.values, dtype=torch.float32) + y_train_tensor = torch.tensor(y_train.values, dtype=torch.long) + X_valid_tensor = torch.tensor(X_valid.values, dtype=torch.float32) + y_valid_tensor = torch.tensor(y_valid.values, dtype=torch.long) + + # Create datasets and dataloaders + train_dataset = TensorDataset(X_train_tensor, y_train_tensor) + valid_dataset = TensorDataset(X_valid_tensor, y_valid_tensor) + train_loader = DataLoader(train_dataset, batch_size=32, shuffle=True) + valid_loader = DataLoader(valid_dataset, batch_size=32, shuffle=False) + + # Initialize the model, loss function and optimizer + model = NeuralNetwork(input_size=X_train.shape[1], num_classes=len(set(y_train))) + criterion = nn.CrossEntropyLoss() + optimizer = optim.Adam(model.parameters(), lr=0.001) + + # Train the model + num_epochs = 150 + for epoch in range(num_epochs): + model.train() + for X_batch, y_batch in train_loader: + optimizer.zero_grad() + outputs = model(X_batch) + loss = criterion(outputs, y_batch) + loss.backward() + optimizer.step() + + # Validate the model + model.eval() + valid_loss = 0 + correct = 0 + with torch.no_grad(): + for X_batch, y_batch in valid_loader: + outputs = model(X_batch) + valid_loss += criterion(outputs, y_batch).item() + _, predicted = torch.max(outputs, 1) + correct += (predicted == y_batch).sum().item() + + accuracy = correct / len(valid_loader.dataset) + print(f"Epoch {epoch+1}/{num_epochs}, Validation Accuracy: {accuracy:.4f}") + + return model + + +def predict(model, X): + X_tensor = torch.tensor(X.values, dtype=torch.float32) + model.eval() + with torch.no_grad(): + outputs = model(X_tensor) + _, predicted = torch.max(outputs, 1) + return predicted.numpy().reshape(-1, 1) diff --git a/rdagent/scenarios/kaggle/experiment/forest-cover-type-prediction_template/model/model_randomforest.py b/rdagent/scenarios/kaggle/experiment/forest-cover-type-prediction_template/model/model_randomforest.py index 841a5eb1..18019066 100644 --- a/rdagent/scenarios/kaggle/experiment/forest-cover-type-prediction_template/model/model_randomforest.py +++ b/rdagent/scenarios/kaggle/experiment/forest-cover-type-prediction_template/model/model_randomforest.py @@ -23,7 +23,7 @@ def fit(X_train: pd.DataFrame, y_train: pd.Series, X_valid: pd.DataFrame, y_vali Define and train the Random Forest model. Merge feature selection into the pipeline. """ # Initialize the Random Forest model - model = RandomForestClassifier(n_estimators=100, random_state=32, n_jobs=-1) + model = RandomForestClassifier(n_estimators=200, random_state=32, n_jobs=-1) # Select features (if any feature selection is needed) X_train_selected = select(X_train) diff --git a/rdagent/scenarios/kaggle/experiment/forest-cover-type-prediction_template/model/model_xgboost.py b/rdagent/scenarios/kaggle/experiment/forest-cover-type-prediction_template/model/model_xgboost.py index e95efb1a..ee9a6126 100644 --- a/rdagent/scenarios/kaggle/experiment/forest-cover-type-prediction_template/model/model_xgboost.py +++ b/rdagent/scenarios/kaggle/experiment/forest-cover-type-prediction_template/model/model_xgboost.py @@ -23,7 +23,7 @@ def fit(X_train: pd.DataFrame, y_train: pd.DataFrame, X_valid: pd.DataFrame, y_v "num_class": len(set(y_train)), # Number of classes "nthread": -1, } - num_round = 20 + num_round = 100 evallist = [(dtrain, "train"), (dvalid, "eval")] bst = xgb.train(params, dtrain, num_round, evallist) diff --git a/rdagent/scenarios/kaggle/experiment/forest-cover-type-prediction_template/train.py b/rdagent/scenarios/kaggle/experiment/forest-cover-type-prediction_template/train.py index 65086289..bd4cee93 100644 --- a/rdagent/scenarios/kaggle/experiment/forest-cover-type-prediction_template/train.py +++ b/rdagent/scenarios/kaggle/experiment/forest-cover-type-prediction_template/train.py @@ -8,6 +8,7 @@ from scipy import stats from sklearn.impute import SimpleImputer from sklearn.metrics import accuracy_score from sklearn.model_selection import KFold +from sklearn.preprocessing import StandardScaler # Set random seed for reproducibility SEED = 42 @@ -40,6 +41,7 @@ kf = KFold(n_splits=5, shuffle=True, random_state=SEED) # Store results accuracies = [] y_test_pred_l = [] +scaler = StandardScaler() # 3) Train and evaluate using KFold fold_number = 1 @@ -80,6 +82,11 @@ for train_index, valid_index in kf.split(X_train): X_val = pd.DataFrame(imputer.transform(X_val), columns=X_val.columns) X_te = pd.DataFrame(imputer.transform(X_te), columns=X_te.columns) + # Standardize the data + X_tr = pd.DataFrame(scaler.fit_transform(X_tr), columns=X_tr.columns) + X_val = pd.DataFrame(scaler.transform(X_val), columns=X_val.columns) + X_te = pd.DataFrame(scaler.transform(X_te), columns=X_te.columns) + # Remove duplicate columns X_tr = X_tr.loc[:, ~X_tr.columns.duplicated()] X_val = X_val.loc[:, ~X_val.columns.duplicated()] diff --git a/rdagent/scenarios/kaggle/experiment/playground-series-s3e11_template/model/model_xgboost.py b/rdagent/scenarios/kaggle/experiment/playground-series-s3e11_template/model/model_xgboost.py index cdb685a0..9497f128 100644 --- a/rdagent/scenarios/kaggle/experiment/playground-series-s3e11_template/model/model_xgboost.py +++ b/rdagent/scenarios/kaggle/experiment/playground-series-s3e11_template/model/model_xgboost.py @@ -39,4 +39,4 @@ def predict(model, X_test): """ X_test = select(X_test) y_pred = model.predict(X_test) - return y_pred + return y_pred.reshape(-1, 1) diff --git a/rdagent/scenarios/kaggle/experiment/playground-series-s3e11_template/train.py b/rdagent/scenarios/kaggle/experiment/playground-series-s3e11_template/train.py index 8a137df6..733bce85 100644 --- a/rdagent/scenarios/kaggle/experiment/playground-series-s3e11_template/train.py +++ b/rdagent/scenarios/kaggle/experiment/playground-series-s3e11_template/train.py @@ -85,7 +85,7 @@ for model, predict_func in model_l: # For multiclass classification, use the mode of the predictions -y_test_pred = np.mean(y_test_pred_l, axis=0) +y_test_pred = np.mean(y_test_pred_l, axis=0).ravel() submission_result = pd.DataFrame(np.expm1(y_test_pred), columns=["cost"]) diff --git a/rdagent/scenarios/kaggle/experiment/playground-series-s3e26_template/model/model_randomforest.py b/rdagent/scenarios/kaggle/experiment/playground-series-s3e26_template/model/model_randomforest.py index 33ed3eb7..31f7a24b 100644 --- a/rdagent/scenarios/kaggle/experiment/playground-series-s3e26_template/model/model_randomforest.py +++ b/rdagent/scenarios/kaggle/experiment/playground-series-s3e26_template/model/model_randomforest.py @@ -23,7 +23,7 @@ def fit(X_train: pd.DataFrame, y_train: pd.Series, X_valid: pd.DataFrame, y_vali Define and train the Random Forest model. Merge feature selection into the pipeline. """ # Initialize the Random Forest model - model = RandomForestClassifier(n_estimators=10, random_state=32, n_jobs=-1) + model = RandomForestClassifier(n_estimators=100, random_state=32, n_jobs=-1) # Select features (if any feature selection is needed) X_train_selected = select(X_train) diff --git a/rdagent/scenarios/kaggle/experiment/playground-series-s4e8_template/train.py b/rdagent/scenarios/kaggle/experiment/playground-series-s4e8_template/train.py index fae6a84a..f65a38d8 100644 --- a/rdagent/scenarios/kaggle/experiment/playground-series-s4e8_template/train.py +++ b/rdagent/scenarios/kaggle/experiment/playground-series-s4e8_template/train.py @@ -122,5 +122,5 @@ y_test_pred = (y_test_pred > 0.5).astype(int) y_test_pred_labels = np.where(y_test_pred == 1, "p", "e") # 将整数转换回 'e' 或 'p' # 8) Submit predictions for the test set -submission_result = pd.DataFrame({"id": passenger_ids, "class": y_test_pred_labels}) +submission_result = pd.DataFrame({"id": passenger_ids, "class": y_test_pred_labels.ravel()}) submission_result.to_csv("submission.csv", index=False) diff --git a/rdagent/scenarios/kaggle/experiment/playground-series-s4e9_template/model/model_nn.py b/rdagent/scenarios/kaggle/experiment/playground-series-s4e9_template/model/model_nn.py deleted file mode 100644 index 0fd122df..00000000 --- a/rdagent/scenarios/kaggle/experiment/playground-series-s4e9_template/model/model_nn.py +++ /dev/null @@ -1,78 +0,0 @@ -import numpy as np -import torch -import torch.nn as nn -import torch.nn.functional as F -from torch.utils.data import DataLoader, TensorDataset -from tqdm import tqdm - -# Check if a GPU is available -device = torch.device("cuda" if torch.cuda.is_available() else "cpu") - - -# Modified model for regression -class FeatureInteractionModel(nn.Module): - def __init__(self, num_features): - super(FeatureInteractionModel, self).__init__() - self.fc1 = nn.Linear(num_features, 128) - self.bn1 = nn.BatchNorm1d(128) - self.fc2 = nn.Linear(128, 64) - self.bn2 = nn.BatchNorm1d(64) - self.fc3 = nn.Linear(64, 1) # Output a single value for regression - self.dropout = nn.Dropout(0.3) - - def forward(self, x): - x = F.relu(self.bn1(self.fc1(x))) - x = F.relu(self.bn2(self.fc2(x))) - x = self.dropout(x) - x = self.fc3(x) # No activation for regression - return x - - -# Training function -def fit(X_train, y_train, X_valid, y_valid): - num_features = X_train.shape[1] - model = FeatureInteractionModel(num_features).to(device) - criterion = nn.MSELoss() # Use MSELoss for regression - optimizer = torch.optim.Adam(model.parameters(), lr=0.001) - - # Convert to TensorDataset and create DataLoader - train_dataset = TensorDataset( - torch.tensor(X_train.to_numpy(), dtype=torch.float32), - torch.tensor(y_train.to_numpy().reshape(-1), dtype=torch.float32), # Convert to NumPy array - ) - valid_dataset = TensorDataset( - torch.tensor(X_valid.to_numpy(), dtype=torch.float32), - torch.tensor(y_valid.to_numpy().reshape(-1), dtype=torch.float32), # Convert to NumPy array - ) - train_loader = DataLoader(train_dataset, batch_size=32, shuffle=True) - valid_loader = DataLoader(valid_dataset, batch_size=32, shuffle=False) - - # Train the model - model.train() - for epoch in range(5): - print(f"Epoch {epoch + 1}/5") - epoch_loss = 0 - for X_batch, y_batch in tqdm(train_loader, desc="Training", leave=False): - X_batch, y_batch = X_batch.to(device), y_batch.to(device) # Move data to the device - optimizer.zero_grad() - outputs = model(X_batch).squeeze(1) # Reshape outputs to [32] - loss = criterion(outputs, y_batch) # Adjust target shape - loss.backward() - optimizer.step() - epoch_loss += loss.item() - print(f"End of epoch {epoch + 1}, Avg Loss: {epoch_loss / len(train_loader):.4f}") - - return model - - -# Prediction function -def predict(model, X): - model.eval() - predictions = [] - with torch.no_grad(): - X_tensor = torch.tensor(X.values, dtype=torch.float32).to(device) # Move data to the device - for i in tqdm(range(0, len(X_tensor), 32), desc="Predicting", leave=False): - batch = X_tensor[i : i + 32] # Predict in batches - pred = model(batch).squeeze().cpu().numpy() # Move results back to CPU - predictions.extend(pred) - return np.array(predictions) # Return predicted values diff --git a/rdagent/scenarios/kaggle/experiment/spaceship-titanic_template/model/model_nn.py b/rdagent/scenarios/kaggle/experiment/spaceship-titanic_template/model/model_nn.py index 91b912ac..eb8c1d51 100644 --- a/rdagent/scenarios/kaggle/experiment/spaceship-titanic_template/model/model_nn.py +++ b/rdagent/scenarios/kaggle/experiment/spaceship-titanic_template/model/model_nn.py @@ -47,8 +47,8 @@ def fit(X_train, y_train, X_valid, y_valid): # Train the model model.train() - for epoch in range(5): - print(f"Epoch {epoch + 1}/5") + for epoch in range(100): + print(f"Epoch {epoch + 1}/100") epoch_loss = 0 for X_batch, y_batch in tqdm(train_loader, desc="Training", leave=False): X_batch, y_batch = X_batch.to(device), y_batch.to(device) # Move data to the device @@ -73,4 +73,4 @@ def predict(model, X): batch = X_tensor[i : i + 32] # Predict in batches pred = model(batch).squeeze().cpu().numpy() # Move results back to CPU predictions.extend(pred) - return np.array(predictions) # Return boolean predictions + return np.array(predictions).reshape(-1, 1) # Return predictions diff --git a/rdagent/scenarios/kaggle/experiment/spaceship-titanic_template/model/model_randomforest.py b/rdagent/scenarios/kaggle/experiment/spaceship-titanic_template/model/model_randomforest.py index 3c64a094..377683b9 100644 --- a/rdagent/scenarios/kaggle/experiment/spaceship-titanic_template/model/model_randomforest.py +++ b/rdagent/scenarios/kaggle/experiment/spaceship-titanic_template/model/model_randomforest.py @@ -51,4 +51,4 @@ def predict(model, X): y_pred_prob = model.predict_proba(X_selected)[:, 1] # Apply threshold to get boolean predictions - return y_pred_prob + return y_pred_prob.reshape(-1, 1) diff --git a/rdagent/scenarios/kaggle/experiment/spaceship-titanic_template/model/model_xgboost.py b/rdagent/scenarios/kaggle/experiment/spaceship-titanic_template/model/model_xgboost.py index 64784224..6e1adb48 100644 --- a/rdagent/scenarios/kaggle/experiment/spaceship-titanic_template/model/model_xgboost.py +++ b/rdagent/scenarios/kaggle/experiment/spaceship-titanic_template/model/model_xgboost.py @@ -37,4 +37,4 @@ def predict(model, X): X = select(X) dtest = xgb.DMatrix(X) y_pred_prob = model.predict(dtest) - return y_pred_prob + return y_pred_prob.reshape(-1, 1) diff --git a/rdagent/scenarios/kaggle/experiment/spaceship-titanic_template/train.py b/rdagent/scenarios/kaggle/experiment/spaceship-titanic_template/train.py index 9cc0bcdd..c042734f 100644 --- a/rdagent/scenarios/kaggle/experiment/spaceship-titanic_template/train.py +++ b/rdagent/scenarios/kaggle/experiment/spaceship-titanic_template/train.py @@ -118,6 +118,7 @@ for m, m_pred in model_l: y_test_pred = np.mean(y_test_pred_l, axis=0) y_test_pred = (y_test_pred > 0.5).astype(bool) +y_test_pred = y_test_pred.ravel() submission_result = pd.DataFrame({"PassengerId": passenger_ids, "Transported": y_test_pred})