From aa4803496b8000ed03b2bbc991ed5f4d3704e402 Mon Sep 17 00:00:00 2001 From: Haoran Pan <167847254+TPLin22@users.noreply.github.com> Date: Fri, 27 Sep 2024 00:40:53 +0800 Subject: [PATCH] feat: xgboost gpu accelerate (#359) * gpu accelerate * delete nn of sf-crime --- .../model/model_xgboost.py | 6 +- .../model/model_xgboost.py | 2 + .../model/model_xgboost.py | 2 + .../model/model_xgboost.py | 4 +- .../model/model_xgboost.py | 2 + .../model/model_xgboost.py | 4 +- .../sf-crime_template/model/model_nn.py | 79 ------------------- .../sf-crime_template/model/model_xgboost.py | 4 +- .../model/model_xgboost.py | 2 + 9 files changed, 21 insertions(+), 84 deletions(-) delete mode 100644 rdagent/scenarios/kaggle/experiment/sf-crime_template/model/model_nn.py diff --git a/rdagent/scenarios/kaggle/experiment/feedback-prize-english-language-learning_template/model/model_xgboost.py b/rdagent/scenarios/kaggle/experiment/feedback-prize-english-language-learning_template/model/model_xgboost.py index 07a30dc5..9c032ee2 100644 --- a/rdagent/scenarios/kaggle/experiment/feedback-prize-english-language-learning_template/model/model_xgboost.py +++ b/rdagent/scenarios/kaggle/experiment/feedback-prize-english-language-learning_template/model/model_xgboost.py @@ -21,9 +21,11 @@ def fit(X_train: pd.DataFrame, y_train: pd.DataFrame, X_valid: pd.DataFrame, y_v """Define and train the model. Merge feature_select""" X_train = select(X_train) - xgb_estimator = xgb.XGBRegressor(n_estimators=500, random_state=0, objective="reg:squarederror") + xgb_estimator = xgb.XGBRegressor( + n_estimators=500, random_state=0, objective="reg:squarederror", tree_method="gpu_hist", device="cuda" + ) - model = MultiOutputRegressor(xgb_estimator, n_jobs=2) + model = MultiOutputRegressor(xgb_estimator, n_jobs=-1) if is_sparse_df(X_train): X_train = X_train.sparse.to_coo() diff --git a/rdagent/scenarios/kaggle/experiment/forest-cover-type-prediction_template/model/model_xgboost.py b/rdagent/scenarios/kaggle/experiment/forest-cover-type-prediction_template/model/model_xgboost.py index ee9a6126..d9720c89 100644 --- a/rdagent/scenarios/kaggle/experiment/forest-cover-type-prediction_template/model/model_xgboost.py +++ b/rdagent/scenarios/kaggle/experiment/forest-cover-type-prediction_template/model/model_xgboost.py @@ -22,6 +22,8 @@ def fit(X_train: pd.DataFrame, y_train: pd.DataFrame, X_valid: pd.DataFrame, y_v "objective": "multi:softmax", # Use softmax for multi-class classification "num_class": len(set(y_train)), # Number of classes "nthread": -1, + "tree_method": "gpu_hist", + "device": "cuda", } num_round = 100 diff --git a/rdagent/scenarios/kaggle/experiment/optiver-realized-volatility-prediction_template/model/model_xgboost.py b/rdagent/scenarios/kaggle/experiment/optiver-realized-volatility-prediction_template/model/model_xgboost.py index 90f4d7a4..b5b29ce4 100644 --- a/rdagent/scenarios/kaggle/experiment/optiver-realized-volatility-prediction_template/model/model_xgboost.py +++ b/rdagent/scenarios/kaggle/experiment/optiver-realized-volatility-prediction_template/model/model_xgboost.py @@ -18,6 +18,8 @@ def fit(X_train: pd.DataFrame, y_train: pd.DataFrame, X_valid: pd.DataFrame, y_v params = { "objective": "reg:squarederror", # Use squared error for regression "nthread": -1, + "tree_method": "gpu_hist", + "device": "cuda", } num_round = 200 diff --git a/rdagent/scenarios/kaggle/experiment/playground-series-s3e26_template/model/model_xgboost.py b/rdagent/scenarios/kaggle/experiment/playground-series-s3e26_template/model/model_xgboost.py index a2abe683..b06a0839 100644 --- a/rdagent/scenarios/kaggle/experiment/playground-series-s3e26_template/model/model_xgboost.py +++ b/rdagent/scenarios/kaggle/experiment/playground-series-s3e26_template/model/model_xgboost.py @@ -24,7 +24,9 @@ def fit(X_train: pd.DataFrame, y_train: pd.DataFrame, X_valid: pd.DataFrame, y_v params = { "objective": "multi:softprob", "num_class": num_classes, - "nthred": -1, + "nthread": -1, + "tree_method": "gpu_hist", + "device": "cuda", } num_round = 100 diff --git a/rdagent/scenarios/kaggle/experiment/playground-series-s4e8_template/model/model_xgboost.py b/rdagent/scenarios/kaggle/experiment/playground-series-s4e8_template/model/model_xgboost.py index ed41dc47..c3f31ec6 100644 --- a/rdagent/scenarios/kaggle/experiment/playground-series-s4e8_template/model/model_xgboost.py +++ b/rdagent/scenarios/kaggle/experiment/playground-series-s4e8_template/model/model_xgboost.py @@ -21,6 +21,8 @@ def fit(X_train: pd.DataFrame, y_train: pd.DataFrame, X_valid: pd.DataFrame, y_v # TODO: for quick running.... params = { "nthread": -1, + "tree_method": "gpu_hist", + "device": "cuda", } num_round = 180 diff --git a/rdagent/scenarios/kaggle/experiment/playground-series-s4e9_template/model/model_xgboost.py b/rdagent/scenarios/kaggle/experiment/playground-series-s4e9_template/model/model_xgboost.py index 90f4d7a4..53fa01c3 100644 --- a/rdagent/scenarios/kaggle/experiment/playground-series-s4e9_template/model/model_xgboost.py +++ b/rdagent/scenarios/kaggle/experiment/playground-series-s4e9_template/model/model_xgboost.py @@ -18,8 +18,10 @@ def fit(X_train: pd.DataFrame, y_train: pd.DataFrame, X_valid: pd.DataFrame, y_v params = { "objective": "reg:squarederror", # Use squared error for regression "nthread": -1, + "tree_method": "gpu_hist", + "device": "cuda", } - num_round = 200 + num_round = 10 evallist = [(dtrain, "train"), (dvalid, "eval")] bst = xgb.train(params, dtrain, num_round, evallist) diff --git a/rdagent/scenarios/kaggle/experiment/sf-crime_template/model/model_nn.py b/rdagent/scenarios/kaggle/experiment/sf-crime_template/model/model_nn.py deleted file mode 100644 index 65e3eb1f..00000000 --- a/rdagent/scenarios/kaggle/experiment/sf-crime_template/model/model_nn.py +++ /dev/null @@ -1,79 +0,0 @@ -import numpy as np -import torch -import torch.nn as nn -import torch.nn.functional as F -from torch.utils.data import DataLoader, TensorDataset -from tqdm import tqdm - -# Check if a GPU is available -device = torch.device("cuda" if torch.cuda.is_available() else "cpu") - - -# Modified model for multi-class classification -class FeatureInteractionModel(nn.Module): - def __init__(self, num_features, num_classes): - super(FeatureInteractionModel, self).__init__() - self.fc1 = nn.Linear(num_features, 128) - self.bn1 = nn.BatchNorm1d(128) - self.fc2 = nn.Linear(128, 64) - self.bn2 = nn.BatchNorm1d(64) - self.fc3 = nn.Linear(64, num_classes) # Output nodes equal to num_classes - self.dropout = nn.Dropout(0.3) - - def forward(self, x): - x = F.relu(self.bn1(self.fc1(x))) - x = F.relu(self.bn2(self.fc2(x))) - x = self.dropout(x) - x = self.fc3(x) - return F.softmax(x, dim=1) # Apply softmax to get probabilities - - -# Training function -def fit(X_train, y_train, X_valid, y_valid): - num_features = X_train.shape[1] - num_classes = len(np.unique(y_train)) # Determine number of classes - model = FeatureInteractionModel(num_features, num_classes).to(device) - criterion = nn.CrossEntropyLoss() # Use CrossEntropyLoss for multi-class - optimizer = torch.optim.Adam(model.parameters(), lr=0.001) - - # Convert to TensorDataset and create DataLoader - train_dataset = TensorDataset( - torch.tensor(X_train.to_numpy(), dtype=torch.float32), - torch.tensor(y_train.to_numpy(), dtype=torch.long), # Use long for labels - ) - valid_dataset = TensorDataset( - torch.tensor(X_valid.to_numpy(), dtype=torch.float32), torch.tensor(y_valid.to_numpy(), dtype=torch.long) - ) - - train_loader = DataLoader(train_dataset, batch_size=32, shuffle=True) - valid_loader = DataLoader(valid_dataset, batch_size=32, shuffle=False) - - # Train the model - model.train() - for epoch in range(10): - print(f"Epoch {epoch + 1}/10") - epoch_loss = 0 - for X_batch, y_batch in tqdm(train_loader, desc="Training", leave=False): - X_batch, y_batch = X_batch.to(device), y_batch.to(device) - optimizer.zero_grad() - outputs = model(X_batch) - loss = criterion(outputs, y_batch) - loss.backward() - optimizer.step() - epoch_loss += loss.item() - print(f"End of epoch {epoch + 1}, Avg Loss: {epoch_loss / len(train_loader):.4f}") - - return model - - -# Prediction function -def predict(model, X): - model.eval() - probabilities = [] - with torch.no_grad(): - X_tensor = torch.tensor(X.values, dtype=torch.float32).to(device) - for i in tqdm(range(0, len(X_tensor), 32), desc="Predicting", leave=False): - batch = X_tensor[i : i + 32] - pred = model(batch) - probabilities.append(pred.cpu().numpy()) # Collect probabilities - return np.vstack(probabilities) # Return as a 2D array diff --git a/rdagent/scenarios/kaggle/experiment/sf-crime_template/model/model_xgboost.py b/rdagent/scenarios/kaggle/experiment/sf-crime_template/model/model_xgboost.py index a2abe683..3fac9a30 100644 --- a/rdagent/scenarios/kaggle/experiment/sf-crime_template/model/model_xgboost.py +++ b/rdagent/scenarios/kaggle/experiment/sf-crime_template/model/model_xgboost.py @@ -24,7 +24,9 @@ def fit(X_train: pd.DataFrame, y_train: pd.DataFrame, X_valid: pd.DataFrame, y_v params = { "objective": "multi:softprob", "num_class": num_classes, - "nthred": -1, + "nthread": -1, + "tree_method": "hist", + "device": "cuda", } num_round = 100 diff --git a/rdagent/scenarios/kaggle/experiment/spaceship-titanic_template/model/model_xgboost.py b/rdagent/scenarios/kaggle/experiment/spaceship-titanic_template/model/model_xgboost.py index f8dd425e..5dec6433 100644 --- a/rdagent/scenarios/kaggle/experiment/spaceship-titanic_template/model/model_xgboost.py +++ b/rdagent/scenarios/kaggle/experiment/spaceship-titanic_template/model/model_xgboost.py @@ -21,6 +21,8 @@ def fit(X_train: pd.DataFrame, y_train: pd.DataFrame, X_valid: pd.DataFrame, y_v # TODO: for quick running.... params = { "nthread": -1, + "tree_method": "gpu_hist", + "device": "cuda", } num_round = 100