feat: refine the template in several Kaggle competitions (#343)

* add an error catching in model runner

* check forest & s3e11 & s4e8 & spaceship, change params of forest & s3e26 & spaceship

* fix ci errors

---------

Co-authored-by: TPLin22 <tplin2@163.com>

rdagent/scenarios/kaggle/developer/runner.py

@@ -119,6 +119,9 @@ class KGModelRunner(KGCachedRunner[KGModelExperiment]):
 
         result = exp.experiment_workspace.execute(run_env=env_to_use)
 
+        if result is None:
+            raise CoderError("No result is returned from the experiment workspace")
+
         exp.result = result
         if RUNNER_SETTINGS.cache_result:
             self.dump_cache_result(exp, result)

rdagent/scenarios/kaggle/experiment/forest-cover-type-prediction_template/cross_validation.py

@@ -1,99 +0,0 @@
-import importlib.util
-import random
-from pathlib import Path
-
-import numpy as np
-import pandas as pd
-from scipy import stats
-from sklearn.impute import SimpleImputer
-from sklearn.metrics import accuracy_score
-from sklearn.model_selection import KFold
-
-# Set random seed for reproducibility
-SEED = 42
-random.seed(SEED)
-np.random.seed(SEED)
-DIRNAME = Path(__file__).absolute().resolve().parent
-
-
-def import_module_from_path(module_name, module_path):
-    spec = importlib.util.spec_from_file_location(module_name, module_path)
-    module = importlib.util.module_from_spec(spec)
-    spec.loader.exec_module(module)
-    return module
-
-
-# 1) Preprocess the data
-data_df = pd.read_csv("/kaggle/input/train.csv")
-data_df = data_df.drop(["Id"], axis=1)
-
-X_train = data_df.drop(["Cover_Type"], axis=1)
-y_train = data_df["Cover_Type"] - 1
-
-# Set up KFold
-kf = KFold(n_splits=5, shuffle=True, random_state=SEED)
-
-# Store results
-accuracies = []
-
-# 3) Train and evaluate using KFold
-fold_number = 1
-for train_index, valid_index in kf.split(X_train):
-    print(f"Starting fold {fold_number}...")
-
-    X_train_l, X_valid_l = [], []  # Reset feature lists for each fold
-    X_tr, X_val = X_train.iloc[train_index], X_train.iloc[valid_index]
-    y_tr, y_val = y_train.iloc[train_index], y_train.iloc[valid_index]
-
-    # Feature engineering
-    for f in DIRNAME.glob("feature/feat*.py"):
-        cls = import_module_from_path(f.stem, f).feature_engineering_cls()
-        cls.fit(X_tr)
-        X_train_f = cls.transform(X_tr)
-        X_valid_f = cls.transform(X_val)
-
-        X_train_l.append(X_train_f)
-        X_valid_l.append(X_valid_f)
-
-    X_tr = pd.concat(X_train_l, axis=1, keys=[f"feature_{i}" for i in range(len(X_train_l))])
-    X_val = pd.concat(X_valid_l, axis=1, keys=[f"feature_{i}" for i in range(len(X_valid_l))])
-
-    print("Shape of X_tr: ", X_tr.shape, " Shape of X_val: ", X_val.shape)
-
-    # Replace inf and -inf with NaN
-    X_tr.replace([np.inf, -np.inf], np.nan, inplace=True)
-    X_val.replace([np.inf, -np.inf], np.nan, inplace=True)
-
-    # Impute missing values
-    imputer = SimpleImputer(strategy="mean")
-    X_tr = pd.DataFrame(imputer.fit_transform(X_tr), columns=X_tr.columns)
-    X_val = pd.DataFrame(imputer.transform(X_val), columns=X_val.columns)
-
-    # Remove duplicate columns
-    X_tr = X_tr.loc[:, ~X_tr.columns.duplicated()]
-    X_val = X_val.loc[:, ~X_val.columns.duplicated()]
-
-    # Train the model
-    model_l = []  # list[tuple[model, predict_func]]
-    for f in DIRNAME.glob("model/model*.py"):
-        m = import_module_from_path(f.stem, f)
-        model_l.append((m.fit(X_tr, y_tr, X_val, y_val), m.predict))
-
-    # Evaluate the model on the validation set
-    y_valid_pred_l = []
-    for model, predict_func in model_l:
-        y_valid_pred = predict_func(model, X_val)
-        y_valid_pred_l.append(y_valid_pred)
-
-    # Majority vote ensemble
-    y_valid_pred_ensemble = stats.mode(y_valid_pred_l, axis=0)[0].flatten()
-
-    # Compute metrics
-    accuracy = accuracy_score(y_val, y_valid_pred_ensemble)
-    accuracies.append(accuracy)
-    print(f"Fold {fold_number} accuracy: {accuracy}")
-
-    fold_number += 1
-
-# Print average accuracy
-print(f"Average accuracy across folds: {np.mean(accuracies)}")

rdagent/scenarios/kaggle/experiment/forest-cover-type-prediction_template/model/model_catboost.py

@@ -0,0 +1,29 @@
+import pandas as pd
+from catboost import CatBoostClassifier
+
+
+def fit(X_train: pd.DataFrame, y_train: pd.Series, X_valid: pd.DataFrame, y_valid: pd.Series):
+    # Define CatBoost parameters
+    cat_params = {
+        "iterations": 5000,
+        "learning_rate": 0.03,
+        "od_wait": 1000,
+        "depth": 7,
+        "task_type": "GPU",
+        "l2_leaf_reg": 3,
+        "eval_metric": "Accuracy",
+        "devices": "0",
+        "verbose": 1000,
+    }
+
+    # Initialize and train the CatBoost model
+    model = CatBoostClassifier(**cat_params)
+    model.fit(X_train, y_train, eval_set=(X_valid, y_valid))
+
+    return model
+
+
+def predict(model, X: pd.DataFrame):
+    # Predict using the trained model
+    y_pred = model.predict(X)
+    return y_pred.reshape(-1, 1)

rdagent/scenarios/kaggle/experiment/forest-cover-type-prediction_template/model/model_dnn.py

@@ -0,0 +1,78 @@
+import numpy as np
+import pandas as pd
+import torch
+import torch.nn as nn
+import torch.optim as optim
+from torch.utils.data import DataLoader, TensorDataset
+
+
+# Define the neural network model with Batch Normalization
+class NeuralNetwork(nn.Module):
+    def __init__(self, input_size, num_classes):
+        super(NeuralNetwork, self).__init__()
+        self.layer1 = nn.Linear(input_size, 128)
+        self.bn1 = nn.BatchNorm1d(128)
+        self.layer2 = nn.Linear(128, 64)
+        self.bn2 = nn.BatchNorm1d(64)
+        self.layer3 = nn.Linear(64, num_classes)
+
+    def forward(self, x):
+        x = torch.relu(self.bn1(self.layer1(x)))
+        x = torch.relu(self.bn2(self.layer2(x)))
+        x = torch.softmax(self.layer3(x), dim=1)
+        return x
+
+
+def fit(X_train: pd.DataFrame, y_train: pd.DataFrame, X_valid: pd.DataFrame, y_valid: pd.DataFrame):
+    # Convert data to PyTorch tensors
+    X_train_tensor = torch.tensor(X_train.values, dtype=torch.float32)
+    y_train_tensor = torch.tensor(y_train.values, dtype=torch.long)
+    X_valid_tensor = torch.tensor(X_valid.values, dtype=torch.float32)
+    y_valid_tensor = torch.tensor(y_valid.values, dtype=torch.long)
+
+    # Create datasets and dataloaders
+    train_dataset = TensorDataset(X_train_tensor, y_train_tensor)
+    valid_dataset = TensorDataset(X_valid_tensor, y_valid_tensor)
+    train_loader = DataLoader(train_dataset, batch_size=32, shuffle=True)
+    valid_loader = DataLoader(valid_dataset, batch_size=32, shuffle=False)
+
+    # Initialize the model, loss function and optimizer
+    model = NeuralNetwork(input_size=X_train.shape[1], num_classes=len(set(y_train)))
+    criterion = nn.CrossEntropyLoss()
+    optimizer = optim.Adam(model.parameters(), lr=0.001)
+
+    # Train the model
+    num_epochs = 150
+    for epoch in range(num_epochs):
+        model.train()
+        for X_batch, y_batch in train_loader:
+            optimizer.zero_grad()
+            outputs = model(X_batch)
+            loss = criterion(outputs, y_batch)
+            loss.backward()
+            optimizer.step()
+
+        # Validate the model
+        model.eval()
+        valid_loss = 0
+        correct = 0
+        with torch.no_grad():
+            for X_batch, y_batch in valid_loader:
+                outputs = model(X_batch)
+                valid_loss += criterion(outputs, y_batch).item()
+                _, predicted = torch.max(outputs, 1)
+                correct += (predicted == y_batch).sum().item()
+
+        accuracy = correct / len(valid_loader.dataset)
+        print(f"Epoch {epoch+1}/{num_epochs}, Validation Accuracy: {accuracy:.4f}")
+
+    return model
+
+
+def predict(model, X):
+    X_tensor = torch.tensor(X.values, dtype=torch.float32)
+    model.eval()
+    with torch.no_grad():
+        outputs = model(X_tensor)
+        _, predicted = torch.max(outputs, 1)
+    return predicted.numpy().reshape(-1, 1)

rdagent/scenarios/kaggle/experiment/forest-cover-type-prediction_template/model/model_randomforest.py

@@ -23,7 +23,7 @@ def fit(X_train: pd.DataFrame, y_train: pd.Series, X_valid: pd.DataFrame, y_vali
     Define and train the Random Forest model. Merge feature selection into the pipeline.
     """
     # Initialize the Random Forest model
-    model = RandomForestClassifier(n_estimators=100, random_state=32, n_jobs=-1)
+    model = RandomForestClassifier(n_estimators=200, random_state=32, n_jobs=-1)
 
     # Select features (if any feature selection is needed)
     X_train_selected = select(X_train)

rdagent/scenarios/kaggle/experiment/forest-cover-type-prediction_template/model/model_xgboost.py

@@ -23,7 +23,7 @@ def fit(X_train: pd.DataFrame, y_train: pd.DataFrame, X_valid: pd.DataFrame, y_v
         "num_class": len(set(y_train)),  # Number of classes
         "nthread": -1,
     }
-    num_round = 20
+    num_round = 100
 
     evallist = [(dtrain, "train"), (dvalid, "eval")]
     bst = xgb.train(params, dtrain, num_round, evallist)

rdagent/scenarios/kaggle/experiment/forest-cover-type-prediction_template/train.py

@@ -8,6 +8,7 @@ from scipy import stats
 from sklearn.impute import SimpleImputer
 from sklearn.metrics import accuracy_score
 from sklearn.model_selection import KFold
+from sklearn.preprocessing import StandardScaler
 
 # Set random seed for reproducibility
 SEED = 42
@@ -40,6 +41,7 @@ kf = KFold(n_splits=5, shuffle=True, random_state=SEED)
 # Store results
 accuracies = []
 y_test_pred_l = []
+scaler = StandardScaler()
 
 # 3) Train and evaluate using KFold
 fold_number = 1
@@ -80,6 +82,11 @@ for train_index, valid_index in kf.split(X_train):
     X_val = pd.DataFrame(imputer.transform(X_val), columns=X_val.columns)
     X_te = pd.DataFrame(imputer.transform(X_te), columns=X_te.columns)
 
+    # Standardize the data
+    X_tr = pd.DataFrame(scaler.fit_transform(X_tr), columns=X_tr.columns)
+    X_val = pd.DataFrame(scaler.transform(X_val), columns=X_val.columns)
+    X_te = pd.DataFrame(scaler.transform(X_te), columns=X_te.columns)
+
     # Remove duplicate columns
     X_tr = X_tr.loc[:, ~X_tr.columns.duplicated()]
     X_val = X_val.loc[:, ~X_val.columns.duplicated()]

rdagent/scenarios/kaggle/experiment/playground-series-s3e11_template/model/model_xgboost.py

@@ -39,4 +39,4 @@ def predict(model, X_test):
     """
     X_test = select(X_test)
     y_pred = model.predict(X_test)
-    return y_pred
+    return y_pred.reshape(-1, 1)

rdagent/scenarios/kaggle/experiment/playground-series-s3e11_template/train.py

@@ -85,7 +85,7 @@ for model, predict_func in model_l:
 
 
 # For multiclass classification, use the mode of the predictions
-y_test_pred = np.mean(y_test_pred_l, axis=0)
+y_test_pred = np.mean(y_test_pred_l, axis=0).ravel()
 
 
 submission_result = pd.DataFrame(np.expm1(y_test_pred), columns=["cost"])

rdagent/scenarios/kaggle/experiment/playground-series-s3e26_template/model/model_randomforest.py

@@ -23,7 +23,7 @@ def fit(X_train: pd.DataFrame, y_train: pd.Series, X_valid: pd.DataFrame, y_vali
     Define and train the Random Forest model. Merge feature selection into the pipeline.
     """
     # Initialize the Random Forest model
-    model = RandomForestClassifier(n_estimators=10, random_state=32, n_jobs=-1)
+    model = RandomForestClassifier(n_estimators=100, random_state=32, n_jobs=-1)
 
     # Select features (if any feature selection is needed)
     X_train_selected = select(X_train)

rdagent/scenarios/kaggle/experiment/playground-series-s4e8_template/train.py

@@ -122,5 +122,5 @@ y_test_pred = (y_test_pred > 0.5).astype(int)
 y_test_pred_labels = np.where(y_test_pred == 1, "p", "e")  # 将整数转换回 'e' 或 'p'
 
 # 8) Submit predictions for the test set
-submission_result = pd.DataFrame({"id": passenger_ids, "class": y_test_pred_labels})
+submission_result = pd.DataFrame({"id": passenger_ids, "class": y_test_pred_labels.ravel()})
 submission_result.to_csv("submission.csv", index=False)

rdagent/scenarios/kaggle/experiment/playground-series-s4e9_template/model/model_nn.py

@@ -1,78 +0,0 @@
-import numpy as np
-import torch
-import torch.nn as nn
-import torch.nn.functional as F
-from torch.utils.data import DataLoader, TensorDataset
-from tqdm import tqdm
-
-# Check if a GPU is available
-device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
-
-
-# Modified model for regression
-class FeatureInteractionModel(nn.Module):
-    def __init__(self, num_features):
-        super(FeatureInteractionModel, self).__init__()
-        self.fc1 = nn.Linear(num_features, 128)
-        self.bn1 = nn.BatchNorm1d(128)
-        self.fc2 = nn.Linear(128, 64)
-        self.bn2 = nn.BatchNorm1d(64)
-        self.fc3 = nn.Linear(64, 1)  # Output a single value for regression
-        self.dropout = nn.Dropout(0.3)
-
-    def forward(self, x):
-        x = F.relu(self.bn1(self.fc1(x)))
-        x = F.relu(self.bn2(self.fc2(x)))
-        x = self.dropout(x)
-        x = self.fc3(x)  # No activation for regression
-        return x
-
-
-# Training function
-def fit(X_train, y_train, X_valid, y_valid):
-    num_features = X_train.shape[1]
-    model = FeatureInteractionModel(num_features).to(device)
-    criterion = nn.MSELoss()  # Use MSELoss for regression
-    optimizer = torch.optim.Adam(model.parameters(), lr=0.001)
-
-    # Convert to TensorDataset and create DataLoader
-    train_dataset = TensorDataset(
-        torch.tensor(X_train.to_numpy(), dtype=torch.float32),
-        torch.tensor(y_train.to_numpy().reshape(-1), dtype=torch.float32),  # Convert to NumPy array
-    )
-    valid_dataset = TensorDataset(
-        torch.tensor(X_valid.to_numpy(), dtype=torch.float32),
-        torch.tensor(y_valid.to_numpy().reshape(-1), dtype=torch.float32),  # Convert to NumPy array
-    )
-    train_loader = DataLoader(train_dataset, batch_size=32, shuffle=True)
-    valid_loader = DataLoader(valid_dataset, batch_size=32, shuffle=False)
-
-    # Train the model
-    model.train()
-    for epoch in range(5):
-        print(f"Epoch {epoch + 1}/5")
-        epoch_loss = 0
-        for X_batch, y_batch in tqdm(train_loader, desc="Training", leave=False):
-            X_batch, y_batch = X_batch.to(device), y_batch.to(device)  # Move data to the device
-            optimizer.zero_grad()
-            outputs = model(X_batch).squeeze(1)  # Reshape outputs to [32]
-            loss = criterion(outputs, y_batch)  # Adjust target shape
-            loss.backward()
-            optimizer.step()
-            epoch_loss += loss.item()
-        print(f"End of epoch {epoch + 1}, Avg Loss: {epoch_loss / len(train_loader):.4f}")
-
-    return model
-
-
-# Prediction function
-def predict(model, X):
-    model.eval()
-    predictions = []
-    with torch.no_grad():
-        X_tensor = torch.tensor(X.values, dtype=torch.float32).to(device)  # Move data to the device
-        for i in tqdm(range(0, len(X_tensor), 32), desc="Predicting", leave=False):
-            batch = X_tensor[i : i + 32]  # Predict in batches
-            pred = model(batch).squeeze().cpu().numpy()  # Move results back to CPU
-            predictions.extend(pred)
-    return np.array(predictions)  # Return predicted values

rdagent/scenarios/kaggle/experiment/spaceship-titanic_template/model/model_nn.py

@@ -47,8 +47,8 @@ def fit(X_train, y_train, X_valid, y_valid):
 
     # Train the model
     model.train()
-    for epoch in range(5):
-        print(f"Epoch {epoch + 1}/5")
+    for epoch in range(100):
+        print(f"Epoch {epoch + 1}/100")
         epoch_loss = 0
         for X_batch, y_batch in tqdm(train_loader, desc="Training", leave=False):
             X_batch, y_batch = X_batch.to(device), y_batch.to(device)  # Move data to the device
@@ -73,4 +73,4 @@ def predict(model, X):
             batch = X_tensor[i : i + 32]  # Predict in batches
             pred = model(batch).squeeze().cpu().numpy()  # Move results back to CPU
             predictions.extend(pred)
-    return np.array(predictions)  # Return boolean predictions
+    return np.array(predictions).reshape(-1, 1)  # Return predictions

rdagent/scenarios/kaggle/experiment/spaceship-titanic_template/model/model_randomforest.py

@@ -51,4 +51,4 @@ def predict(model, X):
     y_pred_prob = model.predict_proba(X_selected)[:, 1]
 
     # Apply threshold to get boolean predictions
-    return y_pred_prob
+    return y_pred_prob.reshape(-1, 1)

rdagent/scenarios/kaggle/experiment/spaceship-titanic_template/model/model_xgboost.py

@@ -37,4 +37,4 @@ def predict(model, X):
     X = select(X)
     dtest = xgb.DMatrix(X)
     y_pred_prob = model.predict(dtest)
-    return y_pred_prob
+    return y_pred_prob.reshape(-1, 1)

rdagent/scenarios/kaggle/experiment/spaceship-titanic_template/train.py

@@ -118,6 +118,7 @@ for m, m_pred in model_l:
 
 y_test_pred = np.mean(y_test_pred_l, axis=0)
 y_test_pred = (y_test_pred > 0.5).astype(bool)
+y_test_pred = y_test_pred.ravel()
 
 submission_result = pd.DataFrame({"PassengerId": passenger_ids, "Transported": y_test_pred})