feat: add more templates for kaggle (#291)

* init for forest-cover-type-prediction

* add nn model for forest-cover-type-prediction

* add cross_validation for forest-cover-type-prediction

* edit path to file

* CI issues

* CI Issue

* edit dir name

* fix a bug in s4e8 ensemble & init spaceship-titanic

* add nn model for s4e8 & spaceship-titanic

* init for s4e9

* ci issues

* ci issue

rdagent/scenarios/kaggle/experiment/forest-cover-type-prediction_template/cross_validation.py

@@ -0,0 +1,101 @@
+import importlib.util
+import random
+from pathlib import Path
+
+import numpy as np
+import pandas as pd
+from scipy import stats
+from sklearn.impute import SimpleImputer
+from sklearn.metrics import accuracy_score
+from sklearn.model_selection import KFold
+
+# Set random seed for reproducibility
+SEED = 42
+random.seed(SEED)
+np.random.seed(SEED)
+DIRNAME = Path(__file__).absolute().resolve().parent
+
+
+def import_module_from_path(module_name, module_path):
+    spec = importlib.util.spec_from_file_location(module_name, module_path)
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    return module
+
+
+# 1) Preprocess the data
+data_df = pd.read_csv(
+    "/data/userdata/v-haoranpan/RD-Agent/git_ignore_folder/data/forest-cover-type-prediction/train.csv"
+)
+data_df = data_df.drop(["Id"], axis=1)
+
+X_train = data_df.drop(["Cover_Type"], axis=1)
+y_train = data_df["Cover_Type"] - 1
+
+# Set up KFold
+kf = KFold(n_splits=5, shuffle=True, random_state=SEED)
+
+# Store results
+accuracies = []
+
+# 3) Train and evaluate using KFold
+fold_number = 1
+for train_index, valid_index in kf.split(X_train):
+    print(f"Starting fold {fold_number}...")
+
+    X_train_l, X_valid_l = [], []  # Reset feature lists for each fold
+    X_tr, X_val = X_train.iloc[train_index], X_train.iloc[valid_index]
+    y_tr, y_val = y_train.iloc[train_index], y_train.iloc[valid_index]
+
+    # Feature engineering
+    for f in DIRNAME.glob("feature/feat*.py"):
+        cls = import_module_from_path(f.stem, f).feature_engineering_cls()
+        cls.fit(X_tr)
+        X_train_f = cls.transform(X_tr)
+        X_valid_f = cls.transform(X_val)
+
+        X_train_l.append(X_train_f)
+        X_valid_l.append(X_valid_f)
+
+    X_tr = pd.concat(X_train_l, axis=1, keys=[f"feature_{i}" for i in range(len(X_train_l))])
+    X_val = pd.concat(X_valid_l, axis=1, keys=[f"feature_{i}" for i in range(len(X_valid_l))])
+
+    print("Shape of X_tr: ", X_tr.shape, " Shape of X_val: ", X_val.shape)
+
+    # Replace inf and -inf with NaN
+    X_tr.replace([np.inf, -np.inf], np.nan, inplace=True)
+    X_val.replace([np.inf, -np.inf], np.nan, inplace=True)
+
+    # Impute missing values
+    imputer = SimpleImputer(strategy="mean")
+    X_tr = pd.DataFrame(imputer.fit_transform(X_tr), columns=X_tr.columns)
+    X_val = pd.DataFrame(imputer.transform(X_val), columns=X_val.columns)
+
+    # Remove duplicate columns
+    X_tr = X_tr.loc[:, ~X_tr.columns.duplicated()]
+    X_val = X_val.loc[:, ~X_val.columns.duplicated()]
+
+    # Train the model
+    model_l = []  # list[tuple[model, predict_func]]
+    for f in DIRNAME.glob("model/model*.py"):
+        m = import_module_from_path(f.stem, f)
+        model_l.append((m.fit(X_tr, y_tr, X_val, y_val), m.predict))
+
+    # Evaluate the model on the validation set
+    y_valid_pred_l = []
+    for model, predict_func in model_l:
+        y_valid_pred = predict_func(model, X_val)
+        y_valid_pred_l.append(y_valid_pred)
+
+    # Majority vote ensemble
+    y_valid_pred_ensemble = stats.mode(y_valid_pred_l, axis=0)[0].flatten()
+
+    # Compute metrics
+    accuracy = accuracy_score(y_val, y_valid_pred_ensemble)
+    accuracies.append(accuracy)
+    print(f"Fold {fold_number} accuracy: {accuracy}")
+
+    fold_number += 1
+
+# Print average accuracy
+print(f"Average accuracy across folds: {np.mean(accuracies)}")

rdagent/scenarios/kaggle/experiment/forest-cover-type-prediction_template/fea_share_preprocess.py

@@ -0,0 +1,72 @@
+import os
+
+import numpy as np
+import pandas as pd
+from sklearn.impute import SimpleImputer
+from sklearn.model_selection import train_test_split
+
+
+def prepreprocess():
+    """
+    This method loads the data, drops the unnecessary columns, and splits it into train and validation sets.
+    """
+    # Load and preprocess the data
+    data_df = pd.read_csv("/kaggle/input/train.csv")
+    data_df = data_df.drop(["Id"], axis=1)
+
+    X = data_df.drop(["Cover_Type"], axis=1)
+    y = data_df["Cover_Type"] - 1
+
+    # Split the data into training and validation sets
+    X_train, X_valid, y_train, y_valid = train_test_split(X, y, test_size=0.20, random_state=42)
+
+    return X_train, X_valid, y_train, y_valid
+
+
+def preprocess_script():
+    """
+    This method applies the preprocessing steps to the training, validation, and test datasets.
+    """
+    if os.path.exists("X_train.pkl"):
+        X_train = pd.read_pickle("X_train.pkl")
+        X_valid = pd.read_pickle("X_valid.pkl")
+        y_train = pd.read_pickle("y_train.pkl")
+        y_valid = pd.read_pickle("y_valid.pkl")
+        X_test = pd.read_pickle("X_test.pkl")
+        ids = pd.read_pickle("ids.pkl")
+
+        return X_train, X_valid, y_train, y_valid, X_test, ids
+
+    X_train, X_valid, y_train, y_valid = prepreprocess()
+
+    # Load and preprocess the test data
+    submission_df = pd.read_csv("/kaggle/input/test.csv")
+    ids = submission_df["Id"]
+    X_test = submission_df.drop(["Id"], axis=1)
+
+    return X_train, X_valid, y_train, y_valid, X_test, ids
+
+
+def clean_and_impute_data(X_train, X_valid, X_test):
+    """
+    Handles inf and -inf values by replacing them with NaN,
+    then imputes missing values using the mean strategy.
+    Also removes duplicate columns.
+    """
+    # Replace inf and -inf with NaN
+    X_train.replace([np.inf, -np.inf], np.nan, inplace=True)
+    X_valid.replace([np.inf, -np.inf], np.nan, inplace=True)
+    X_test.replace([np.inf, -np.inf], np.nan, inplace=True)
+
+    # Impute missing values
+    imputer = SimpleImputer(strategy="mean")
+    X_train = pd.DataFrame(imputer.fit_transform(X_train), columns=X_train.columns)
+    X_valid = pd.DataFrame(imputer.transform(X_valid), columns=X_valid.columns)
+    X_test = pd.DataFrame(imputer.transform(X_test), columns=X_test.columns)
+
+    # Remove duplicate columns
+    X_train = X_train.loc[:, ~X_train.columns.duplicated()]
+    X_valid = X_valid.loc[:, ~X_valid.columns.duplicated()]
+    X_test = X_test.loc[:, ~X_test.columns.duplicated()]
+
+    return X_train, X_valid, X_test

rdagent/scenarios/kaggle/experiment/forest-cover-type-prediction_template/feature/feature.py

@@ -0,0 +1,23 @@
+import pandas as pd
+
+"""
+Here is the feature engineering code for each task, with a class that has a fit and transform method.
+Remember
+"""
+
+
+class IdentityFeature:
+    def fit(self, train_df: pd.DataFrame):
+        """
+        Fit the feature engineering model to the training data.
+        """
+        pass
+
+    def transform(self, X: pd.DataFrame):
+        """
+        Transform the input data.
+        """
+        return X
+
+
+feature_engineering_cls = IdentityFeature

rdagent/scenarios/kaggle/experiment/forest-cover-type-prediction_template/model/model_nn.py

@@ -0,0 +1,78 @@
+import numpy as np
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from torch.utils.data import DataLoader, TensorDataset
+from tqdm import tqdm
+
+# Check if a GPU is available
+device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+
+# Modified model for multi-class classification
+class HybridFeatureInteractionModel(nn.Module):
+    def __init__(self, num_features, num_classes):
+        super(HybridFeatureInteractionModel, self).__init__()
+        self.fc1 = nn.Linear(num_features, 128)
+        self.bn1 = nn.BatchNorm1d(128)
+        self.fc2 = nn.Linear(128, 64)
+        self.bn2 = nn.BatchNorm1d(64)
+        self.fc3 = nn.Linear(64, num_classes)  # Output nodes equal to num_classes
+        self.dropout = nn.Dropout(0.3)
+
+    def forward(self, x):
+        x = F.relu(self.bn1(self.fc1(x)))
+        x = F.relu(self.bn2(self.fc2(x)))
+        x = self.dropout(x)
+        x = self.fc3(x)  # No activation here, use CrossEntropyLoss
+        return x
+
+
+# Training function
+def fit(X_train, y_train, X_valid, y_valid):
+    num_features = X_train.shape[1]
+    num_classes = len(np.unique(y_train))  # Determine number of classes
+    model = HybridFeatureInteractionModel(num_features, num_classes).to(device)
+    criterion = nn.CrossEntropyLoss()  # Use CrossEntropyLoss for multi-class
+    optimizer = torch.optim.Adam(model.parameters(), lr=0.001)
+
+    # Convert to TensorDataset and create DataLoader
+    train_dataset = TensorDataset(
+        torch.tensor(X_train.to_numpy(), dtype=torch.float32), torch.tensor(y_train.to_numpy(), dtype=torch.long)
+    )
+    valid_dataset = TensorDataset(
+        torch.tensor(X_valid.to_numpy(), dtype=torch.float32), torch.tensor(y_valid.to_numpy(), dtype=torch.long)
+    )
+    train_loader = DataLoader(train_dataset, batch_size=32, shuffle=True)
+    valid_loader = DataLoader(valid_dataset, batch_size=32, shuffle=False)
+
+    # Train the model
+    model.train()
+    for epoch in range(5):  # just for quick run
+        print(f"Epoch {epoch + 1}/5")
+        epoch_loss = 0
+        for X_batch, y_batch in tqdm(train_loader, desc="Training", leave=False):
+            X_batch, y_batch = X_batch.to(device), y_batch.to(device)
+            optimizer.zero_grad()
+            outputs = model(X_batch)
+            loss = criterion(outputs, y_batch)
+            loss.backward()
+            optimizer.step()
+            epoch_loss += loss.item()
+        print(f"End of epoch {epoch + 1}, Avg Loss: {epoch_loss / len(train_loader):.4f}")
+
+    return model
+
+
+# Prediction function
+def predict(model, X):
+    model.eval()
+    predictions = []
+    with torch.no_grad():
+        X_tensor = torch.tensor(X.values, dtype=torch.float32).to(device)
+        for i in tqdm(range(0, len(X_tensor), 32), desc="Predicting", leave=False):
+            batch = X_tensor[i : i + 32]
+            pred = model(batch)
+            pred = torch.argmax(pred, dim=1).cpu().numpy()  # Use argmax to get class
+            predictions.extend(pred)
+    return np.array(predictions)

rdagent/scenarios/kaggle/experiment/forest-cover-type-prediction_template/model/model_randomforest.py

@@ -0,0 +1,53 @@
+"""
+Motivation of the model:
+The Random Forest model is chosen for its robustness and ability to handle large datasets with higher dimensionality.
+It reduces overfitting by averaging multiple decision trees and typically performs well out of the box, making it a good
+baseline model for many classification tasks.
+"""
+
+import pandas as pd
+from sklearn.ensemble import RandomForestClassifier
+from sklearn.metrics import accuracy_score
+
+
+def select(X: pd.DataFrame) -> pd.DataFrame:
+    """
+    Select relevant features. To be used in fit & predict function.
+    """
+    # For now, we assume all features are relevant. This can be expanded to feature selection logic.
+    return X
+
+
+def fit(X_train: pd.DataFrame, y_train: pd.Series, X_valid: pd.DataFrame, y_valid: pd.Series):
+    """
+    Define and train the Random Forest model. Merge feature selection into the pipeline.
+    """
+    # Initialize the Random Forest model
+    model = RandomForestClassifier(n_estimators=100, random_state=32, n_jobs=-1)
+
+    # Select features (if any feature selection is needed)
+    X_train_selected = select(X_train)
+    X_valid_selected = select(X_valid)
+
+    # Fit the model
+    model.fit(X_train_selected, y_train)
+
+    # Validate the model
+    y_valid_pred = model.predict(X_valid_selected)
+    accuracy = accuracy_score(y_valid, y_valid_pred)
+    print(f"Validation Accuracy: {accuracy:.4f}")
+
+    return model
+
+
+def predict(model, X):
+    """
+    Keep feature selection's consistency and make predictions.
+    """
+    # Select features (if any feature selection is needed)
+    X_selected = select(X)
+
+    # Predict using the trained model
+    y_pred = model.predict(X_selected)
+
+    return y_pred

rdagent/scenarios/kaggle/experiment/forest-cover-type-prediction_template/model/model_xgboost.py

@@ -0,0 +1,41 @@
+"""
+motivation  of the model
+"""
+
+import pandas as pd
+import xgboost as xgb
+
+
+def select(X: pd.DataFrame) -> pd.DataFrame:
+    # Ignore feature selection logic
+    return X
+
+
+def fit(X_train: pd.DataFrame, y_train: pd.DataFrame, X_valid: pd.DataFrame, y_valid: pd.DataFrame):
+    """Define and train the model. Merge feature_select"""
+    X_train = select(X_train)
+    X_valid = select(X_valid)
+    dtrain = xgb.DMatrix(X_train, label=y_train)
+    dvalid = xgb.DMatrix(X_valid, label=y_valid)
+
+    params = {
+        "objective": "multi:softmax",  # Use softmax for multi-class classification
+        "num_class": len(set(y_train)),  # Number of classes
+        "nthread": -1,
+    }
+    num_round = 20
+
+    evallist = [(dtrain, "train"), (dvalid, "eval")]
+    bst = xgb.train(params, dtrain, num_round, evallist)
+
+    return bst
+
+
+def predict(model, X):
+    """
+    Keep feature select's consistency.
+    """
+    X = select(X)
+    dtest = xgb.DMatrix(X)
+    y_pred = model.predict(dtest)
+    return y_pred.astype(int)

rdagent/scenarios/kaggle/experiment/forest-cover-type-prediction_template/train.py

@@ -0,0 +1,104 @@
+import importlib.util
+import random
+from pathlib import Path
+
+import numpy as np
+import pandas as pd
+from fea_share_preprocess import clean_and_impute_data, preprocess_script
+from scipy import stats
+from sklearn.metrics import accuracy_score, matthews_corrcoef
+from sklearn.preprocessing import LabelEncoder
+
+# Set random seed for reproducibility
+SEED = 42
+random.seed(SEED)
+np.random.seed(SEED)
+DIRNAME = Path(__file__).absolute().resolve().parent
+
+
+# support various method for metrics calculation
+def compute_metrics_for_classification(y_true, y_pred):
+    """Compute accuracy metric for classification."""
+    accuracy = accuracy_score(y_true, y_pred)
+    return accuracy
+
+
+def compute_metrics_for_classification(y_true, y_pred):
+    """Compute MCC for classification."""
+    mcc = matthews_corrcoef(y_true, y_pred)
+    return mcc
+
+
+def import_module_from_path(module_name, module_path):
+    spec = importlib.util.spec_from_file_location(module_name, module_path)
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    return module
+
+
+# 1) Preprocess the data
+# TODO 如果已经做过数据预处理了，不需要再做了
+X_train, X_valid, y_train, y_valid, X_test, ids = preprocess_script()
+
+# 2) Auto feature engineering
+X_train_l, X_valid_l = [], []
+X_test_l = []
+
+for f in DIRNAME.glob("feature/feat*.py"):
+    cls = import_module_from_path(f.stem, f).feature_engineering_cls()
+    cls.fit(X_train)
+    X_train_f = cls.transform(X_train)
+    X_valid_f = cls.transform(X_valid)
+    X_test_f = cls.transform(X_test)
+
+    X_train_l.append(X_train_f)
+    X_valid_l.append(X_valid_f)
+    X_test_l.append(X_test_f)
+
+X_train = pd.concat(X_train_l, axis=1, keys=[f"feature_{i}" for i in range(len(X_train_l))])
+X_valid = pd.concat(X_valid_l, axis=1, keys=[f"feature_{i}" for i in range(len(X_valid_l))])
+X_test = pd.concat(X_test_l, axis=1, keys=[f"feature_{i}" for i in range(len(X_test_l))])
+
+print(X_train.shape, X_valid.shape, X_test.shape)
+
+# Handle inf and -inf values
+X_train, X_valid, X_test = clean_and_impute_data(X_train, X_valid, X_test)
+
+# 3) Train the model
+model_l = []  # list[tuple[model, predict_func]]
+for f in DIRNAME.glob("model/model*.py"):
+    m = import_module_from_path(f.stem, f)
+    model_l.append((m.fit(X_train, y_train, X_valid, y_valid), m.predict))
+
+# 4) Evaluate the model on the validation set
+y_valid_pred_l = []
+for model, predict_func in model_l:
+    y_valid_pred = predict_func(model, X_valid)
+    y_valid_pred_l.append(y_valid_pred)
+    print(y_valid_pred)
+    print(y_valid_pred.shape)
+
+# 5) Ensemble
+# Majority vote ensemble
+y_valid_pred_ensemble = stats.mode(y_valid_pred_l, axis=0)[0].flatten()
+
+# Compute metrics
+accuracy = accuracy_score(y_valid, y_valid_pred_ensemble)
+print(f"final accuracy on valid set: {accuracy}")
+
+# 6) Save the validation metrics
+pd.Series(data=[accuracy], index=["multi-class accuracy"]).to_csv("submission_score.csv")
+
+# 7) Make predictions on the test set and save them
+y_test_pred_l = []
+for model, predict_func in model_l:
+    y_test_pred_l.append(predict_func(model, X_test))
+
+# For multiclass classification, use the mode of the predictions
+y_test_pred = stats.mode(y_test_pred_l, axis=0)[0].flatten() + 1
+
+
+submission_result = pd.DataFrame(y_test_pred, columns=["Cover_Type"])
+submission_result.insert(0, "Id", ids)
+
+submission_result.to_csv("submission.csv", index=False)

rdagent/scenarios/kaggle/experiment/playground-series-s4e8_template/cross_validation_tpl.py

@@ -1,87 +0,0 @@
-from pathlib import Path
-
-import numpy as np
-import pandas as pd
-from sklearn.model_selection import KFold
-from sklearn.preprocessing import LabelEncoder
-
-from rdagent.scenarios.kaggle.experiment.meta_tpl.fea_share_preprocess import preprocess
-
-
-def compute_metrics_for_classification(y_true, y_pred):
-    """Compute MCC for classification."""
-    from sklearn.metrics import matthews_corrcoef
-
-    return matthews_corrcoef(y_true, y_pred)
-
-
-def perform_kfold_cross_validation(X, y, n_splits=2, random_seed=42):
-    kf = KFold(n_splits=n_splits, shuffle=True, random_state=random_seed)
-    fold_metrics = []
-
-    DIRNAME = Path(__file__).absolute().resolve().parent
-
-    for fold, (train_idx, valid_idx) in enumerate(kf.split(X)):
-        X_train_fold, X_valid_fold = X.iloc[train_idx], X.iloc[valid_idx]
-        y_train_fold, y_valid_fold = y[train_idx], y[valid_idx]
-
-        # TODO: Preprocess and Feature Engineering before K-Fold CV
-
-        # Preprocess the data
-        X_train_fold = preprocess(X_train_fold)
-        X_valid_fold = preprocess(X_valid_fold)
-
-        # Feature Engineering
-        X_train_l_fold, X_valid_l_fold = [], []
-        for f in DIRNAME.glob("feat*.py"):
-            m = __import__(f.name.strip(".py"))
-            X_train_fold = m.feat_eng(X_train_fold)
-            X_valid_fold = m.feat_eng(X_valid_fold)
-
-            X_train_l_fold.append(X_train_fold)
-            X_valid_l_fold.append(X_valid_fold)
-
-        X_train_fold = pd.concat(X_train_l_fold, axis=1)
-        X_valid_fold = pd.concat(X_valid_l_fold, axis=1)
-
-        # Align features
-        X_valid_fold = X_valid_fold.reindex(columns=X_train_fold.columns, fill_value=0)
-
-        # Train and evaluate models
-        mcc_scores = []
-        model_l = []  # Reinitialize model list
-        for f in DIRNAME.glob("model*.py"):
-            m = __import__(f.name.strip(".py"))
-            model = m.fit(X_train_fold, y_train_fold, X_valid_fold, y_valid_fold)
-            y_valid_pred = m.predict(model, X_valid_fold)
-            mcc = compute_metrics_for_classification(y_valid_fold, y_valid_pred)
-            mcc_scores.append(mcc)
-            print(f"Fold {fold+1}, Model {f.name}: MCC = {mcc}")
-
-        # Store the average MCC score for this fold
-        avg_mcc = np.mean(mcc_scores)
-        fold_metrics.append(avg_mcc)
-        print(f"Fold {fold+1} average MCC: {avg_mcc}")
-
-    # Calculate the overall average MCC
-    overall_avg_mcc = np.mean(fold_metrics)
-    result_df = pd.DataFrame({"Overall Average MCC": [overall_avg_mcc]})
-    result_df.to_csv(f"path/to/playground-series-s4e8/cv_score_{f.name.strip('.py')}.csv", index=False)
-
-    print(f"Overall Average MCC across all folds: {overall_avg_mcc}")
-    return overall_avg_mcc
-
-
-# This allows the script to be run directly
-if __name__ == "__main__":
-    # Load and preprocess the data
-    data_df = pd.read_csv("path/to/playground-series-s4e8/train.csv")
-    data_df = data_df.drop(["id"], axis=1)
-
-    X = data_df.drop(["class"], axis=1)
-    y = data_df[["class"]]
-
-    label_encoder = LabelEncoder()
-    # transfrom y to 1D
-    y = label_encoder.fit_transform(y)
-    result = perform_kfold_cross_validation(X, y)

rdagent/scenarios/kaggle/experiment/playground-series-s4e8_template/model/model_nn.py

@@ -0,0 +1,76 @@
+import numpy as np
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from torch.utils.data import DataLoader, TensorDataset
+from tqdm import tqdm
+
+# Check if a GPU is available
+device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+
+# Restored three-layer model structure
+class HybridFeatureInteractionModel(nn.Module):
+    def __init__(self, num_features):
+        super(HybridFeatureInteractionModel, self).__init__()
+        self.fc1 = nn.Linear(num_features, 128)
+        self.bn1 = nn.BatchNorm1d(128)
+        self.fc2 = nn.Linear(128, 64)
+        self.bn2 = nn.BatchNorm1d(64)
+        self.fc3 = nn.Linear(64, 1)
+        self.dropout = nn.Dropout(0.3)
+
+    def forward(self, x):
+        x = F.relu(self.bn1(self.fc1(x)))
+        x = F.relu(self.bn2(self.fc2(x)))
+        x = self.dropout(x)
+        x = torch.sigmoid(self.fc3(x))
+        return x
+
+
+# Training function
+def fit(X_train, y_train, X_valid, y_valid):
+    num_features = X_train.shape[1]
+    model = HybridFeatureInteractionModel(num_features).to(device)
+    criterion = nn.BCELoss()  # Binary classification problem
+    optimizer = torch.optim.Adam(model.parameters(), lr=0.001)
+
+    # Convert to TensorDataset and create DataLoader
+    train_dataset = TensorDataset(
+        torch.tensor(X_train.to_numpy(), dtype=torch.float32), torch.tensor(y_train.reshape(-1), dtype=torch.float32)
+    )
+    valid_dataset = TensorDataset(
+        torch.tensor(X_valid.to_numpy(), dtype=torch.float32), torch.tensor(y_valid.reshape(-1), dtype=torch.float32)
+    )
+    train_loader = DataLoader(train_dataset, batch_size=32, shuffle=True)
+    valid_loader = DataLoader(valid_dataset, batch_size=32, shuffle=False)
+
+    # Train the model
+    model.train()
+    for epoch in range(5):
+        print(f"Epoch {epoch + 1}/5")
+        epoch_loss = 0
+        for X_batch, y_batch in tqdm(train_loader, desc="Training", leave=False):
+            X_batch, y_batch = X_batch.to(device), y_batch.to(device)  # Move data to the device
+            optimizer.zero_grad()
+            outputs = model(X_batch).squeeze(1)  # Reshape outputs to [32]
+            loss = criterion(outputs, y_batch)  # Adjust target shape
+            loss.backward()
+            optimizer.step()
+            epoch_loss += loss.item()
+        print(f"End of epoch {epoch + 1}, Avg Loss: {epoch_loss / len(train_loader):.4f}")
+
+    return model
+
+
+# Prediction function
+def predict(model, X):
+    model.eval()
+    predictions = []
+    with torch.no_grad():
+        X_tensor = torch.tensor(X.values, dtype=torch.float32).to(device)  # Move data to the device
+        for i in tqdm(range(0, len(X_tensor), 32), desc="Predicting", leave=False):
+            batch = X_tensor[i : i + 32]  # Predict in batches
+            pred = model(batch).squeeze().cpu().numpy()  # Move results back to CPU
+            predictions.extend(pred)
+    return np.array(predictions)  # Return boolean predictions

rdagent/scenarios/kaggle/experiment/playground-series-s4e8_template/model/model_xgboost.py

@@ -22,7 +22,7 @@ def fit(X_train: pd.DataFrame, y_train: pd.DataFrame, X_valid: pd.DataFrame, y_v
     params = {
         "nthred": -1,
     }
-    num_round = 200
+    num_round = 180
 
     evallist = [(dtrain, "train"), (dvalid, "eval")]
     bst = xgb.train(params, dtrain, num_round, evallist)

rdagent/scenarios/kaggle/experiment/playground-series-s4e8_template/train.py

@@ -16,12 +16,6 @@ DIRNAME = Path(__file__).absolute().resolve().parent
 
 
 # support various method for metrics calculation
-def compute_metrics_for_classification(y_true, y_pred):
-    """Compute accuracy metric for classification."""
-    accuracy = accuracy_score(y_true, y_pred)
-    return accuracy
-
-
 def compute_metrics_for_classification(y_true, y_pred):
     """Compute MCC for classification."""
     mcc = matthews_corrcoef(y_true, y_pred)
@@ -102,20 +96,15 @@ print("Final on validation set: ", mcc)
 pd.Series(data=[mcc], index=["MCC"]).to_csv("submission_score.csv")
 
 # 7) Make predictions on the test set and save them
-label_encoder = LabelEncoder()
-label_encoder.fit(y_train)
-y_test_pred_bool_l = []
+y_test_pred_l = []
 for m, m_pred in model_l:
-    y_test_pred_bool_l.append(
-        m_pred(m, X_test).astype(int)
-    )  # TODO Make this an ensemble. Currently it uses the last prediction
+    y_test_pred_l.append(m_pred(m, X_test))  # TODO Make this an ensemble. Currently it uses the last prediction
 
-y_test_pred = np.mean(y_test_pred_bool_l, axis=0)
+y_test_pred = np.mean(y_test_pred_l, axis=0)
 y_test_pred = (y_test_pred > 0.5).astype(int)
 
-y_test_pred_labels = label_encoder.inverse_transform(y_test_pred)  # 将整数转换回 'e' 或 'p'
-
-submission_result = pd.DataFrame({"id": passenger_ids, "class": y_test_pred_labels})
+y_test_pred_labels = np.where(y_test_pred == 1, "p", "e")  # 将整数转换回 'e' 或 'p'
 
 # 8) Submit predictions for the test set
+submission_result = pd.DataFrame({"id": passenger_ids, "class": y_test_pred_labels})
 submission_result.to_csv("submission.csv", index=False)

rdagent/scenarios/kaggle/experiment/playground-series-s4e9_template/fea_share_preprocess.py

@@ -0,0 +1,108 @@
+import os
+
+import pandas as pd
+from sklearn.compose import ColumnTransformer
+from sklearn.impute import SimpleImputer
+from sklearn.model_selection import train_test_split
+from sklearn.pipeline import Pipeline
+from sklearn.preprocessing import OneHotEncoder
+
+
+def prepreprocess():
+    """
+    This method loads the data, drops the unnecessary columns, and splits it into train and validation sets.
+    """
+    # Load and preprocess the data
+    data_df = pd.read_csv("/kaggle/input/train.csv")
+    data_df = data_df.drop(["id"], axis=1)
+
+    X = data_df.drop(["price"], axis=1)
+    y = data_df["price"]
+
+    # Split the data into training and validation sets
+    X_train, X_valid, y_train, y_valid = train_test_split(X, y, test_size=0.10, random_state=42)
+
+    return X_train, X_valid, y_train, y_valid
+
+
+def preprocess_fit(X_train: pd.DataFrame):
+    """
+    Fits the preprocessor on the training data and returns the fitted preprocessor.
+    """
+    # Identify numerical and categorical features
+    numerical_cols = [cname for cname in X_train.columns if X_train[cname].dtype in ["int64", "float64"]]
+    categorical_cols = [cname for cname in X_train.columns if X_train[cname].dtype == "object"]
+
+    # Define preprocessors for numerical and categorical features
+    categorical_transformer = Pipeline(
+        steps=[
+            ("imputer", SimpleImputer(strategy="most_frequent")),
+            ("onehot", OneHotEncoder(handle_unknown="ignore")),
+        ]
+    )
+
+    numerical_transformer = Pipeline(steps=[("imputer", SimpleImputer(strategy="mean"))])
+
+    # Combine preprocessing steps
+    preprocessor = ColumnTransformer(
+        transformers=[
+            ("cat", categorical_transformer, categorical_cols),
+            ("num", numerical_transformer, numerical_cols),
+        ]
+    )
+
+    # Fit the preprocessor on the training data
+    preprocessor.fit(X_train)
+
+    return preprocessor
+
+
+def preprocess_transform(X: pd.DataFrame, preprocessor):
+    """
+    Transforms the given DataFrame using the fitted preprocessor.
+    Ensures the processed data has consistent features across train, validation, and test sets.
+    """
+    # Transform the data using the fitted preprocessor
+    X_array = preprocessor.transform(X).toarray()
+
+    # Get feature names for the columns in the transformed data
+    categorical_cols = [cname for cname in X.columns if X[cname].dtype == "object"]
+    feature_names = preprocessor.named_transformers_["cat"]["onehot"].get_feature_names_out(
+        categorical_cols
+    ).tolist() + [cname for cname in X.columns if X[cname].dtype in ["int64", "float64"]]
+
+    # Convert arrays back to DataFrames
+    X_transformed = pd.DataFrame(X_array, columns=feature_names, index=X.index)
+
+    return X_transformed
+
+
+def preprocess_script():
+    """
+    This method applies the preprocessing steps to the training, validation, and test datasets.
+    """
+    if os.path.exists("X_train.pkl"):
+        X_train = pd.read_pickle("X_train.pkl")
+        X_valid = pd.read_pickle("X_valid.pkl")
+        y_train = pd.read_pickle("y_train.pkl")
+        y_valid = pd.read_pickle("y_valid.pkl")
+        X_test = pd.read_pickle("X_test.pkl")
+        passenger_ids = pd.read_pickle("passenger_ids.pkl")
+
+        return X_train, X_valid, y_train, y_valid, X_test, passenger_ids
+    X_train, X_valid, y_train, y_valid = prepreprocess()
+
+    # Fit the preprocessor on the training data
+    preprocessor = preprocess_fit(X_train)
+
+    # Preprocess the train, validation, and test data
+    X_train = preprocess_transform(X_train, preprocessor)
+    X_valid = preprocess_transform(X_valid, preprocessor)
+
+    # Load and preprocess the test data
+    submission_df = pd.read_csv("/kaggle/input/test.csv")
+    ids = submission_df["id"]
+    submission_df = submission_df.drop(["id"], axis=1)
+    X_test = preprocess_transform(submission_df, preprocessor)
+
+    return X_train, X_valid, y_train, y_valid, X_test, ids

rdagent/scenarios/kaggle/experiment/playground-series-s4e9_template/feature/feature.py

@@ -0,0 +1,23 @@
+import pandas as pd
+
+"""
+Here is the feature engineering code for each task, with a class that has a fit and transform method.
+Remember
+"""
+
+
+class IdentityFeature:
+    def fit(self, train_df: pd.DataFrame):
+        """
+        Fit the feature engineering model to the training data.
+        """
+        pass
+
+    def transform(self, X: pd.DataFrame):
+        """
+        Transform the input data.
+        """
+        return X
+
+
+feature_engineering_cls = IdentityFeature

rdagent/scenarios/kaggle/experiment/playground-series-s4e9_template/model/model_nn.py

@@ -0,0 +1,78 @@
+import numpy as np
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from torch.utils.data import DataLoader, TensorDataset
+from tqdm import tqdm
+
+# Check if a GPU is available
+device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+
+# Modified model for regression
+class HybridFeatureInteractionModel(nn.Module):
+    def __init__(self, num_features):
+        super(HybridFeatureInteractionModel, self).__init__()
+        self.fc1 = nn.Linear(num_features, 128)
+        self.bn1 = nn.BatchNorm1d(128)
+        self.fc2 = nn.Linear(128, 64)
+        self.bn2 = nn.BatchNorm1d(64)
+        self.fc3 = nn.Linear(64, 1)  # Output a single value for regression
+        self.dropout = nn.Dropout(0.3)
+
+    def forward(self, x):
+        x = F.relu(self.bn1(self.fc1(x)))
+        x = F.relu(self.bn2(self.fc2(x)))
+        x = self.dropout(x)
+        x = self.fc3(x)  # No activation for regression
+        return x
+
+
+# Training function
+def fit(X_train, y_train, X_valid, y_valid):
+    num_features = X_train.shape[1]
+    model = HybridFeatureInteractionModel(num_features).to(device)
+    criterion = nn.MSELoss()  # Use MSELoss for regression
+    optimizer = torch.optim.Adam(model.parameters(), lr=0.001)
+
+    # Convert to TensorDataset and create DataLoader
+    train_dataset = TensorDataset(
+        torch.tensor(X_train.to_numpy(), dtype=torch.float32),
+        torch.tensor(y_train.to_numpy().reshape(-1), dtype=torch.float32),  # Convert to NumPy array
+    )
+    valid_dataset = TensorDataset(
+        torch.tensor(X_valid.to_numpy(), dtype=torch.float32),
+        torch.tensor(y_valid.to_numpy().reshape(-1), dtype=torch.float32),  # Convert to NumPy array
+    )
+    train_loader = DataLoader(train_dataset, batch_size=32, shuffle=True)
+    valid_loader = DataLoader(valid_dataset, batch_size=32, shuffle=False)
+
+    # Train the model
+    model.train()
+    for epoch in range(5):
+        print(f"Epoch {epoch + 1}/5")
+        epoch_loss = 0
+        for X_batch, y_batch in tqdm(train_loader, desc="Training", leave=False):
+            X_batch, y_batch = X_batch.to(device), y_batch.to(device)  # Move data to the device
+            optimizer.zero_grad()
+            outputs = model(X_batch).squeeze(1)  # Reshape outputs to [32]
+            loss = criterion(outputs, y_batch)  # Adjust target shape
+            loss.backward()
+            optimizer.step()
+            epoch_loss += loss.item()
+        print(f"End of epoch {epoch + 1}, Avg Loss: {epoch_loss / len(train_loader):.4f}")
+
+    return model
+
+
+# Prediction function
+def predict(model, X):
+    model.eval()
+    predictions = []
+    with torch.no_grad():
+        X_tensor = torch.tensor(X.values, dtype=torch.float32).to(device)  # Move data to the device
+        for i in tqdm(range(0, len(X_tensor), 32), desc="Predicting", leave=False):
+            batch = X_tensor[i : i + 32]  # Predict in batches
+            pred = model(batch).squeeze().cpu().numpy()  # Move results back to CPU
+            predictions.extend(pred)
+    return np.array(predictions)  # Return predicted values

rdagent/scenarios/kaggle/experiment/playground-series-s4e9_template/model/model_randomforest.py

@@ -0,0 +1,48 @@
+import numpy as np
+import pandas as pd
+from sklearn.ensemble import RandomForestRegressor
+from sklearn.metrics import mean_squared_error
+
+
+def select(X: pd.DataFrame) -> pd.DataFrame:
+    """
+    Select relevant features. To be used in fit & predict function.
+    """
+    # For now, we assume all features are relevant. This can be expanded to feature selection logic.
+    return X
+
+
+def fit(X_train: pd.DataFrame, y_train: pd.Series, X_valid: pd.DataFrame, y_valid: pd.Series):
+    """
+    Define and train the Random Forest model. Merge feature selection into the pipeline.
+    """
+    # Initialize the Random Forest model
+    model = RandomForestRegressor(n_estimators=100, random_state=32, n_jobs=-1)
+
+    # Select features (if any feature selection is needed)
+    X_train_selected = select(X_train)
+    X_valid_selected = select(X_valid)
+
+    # Fit the model
+    model.fit(X_train_selected, y_train)
+
+    # Validate the model
+    y_valid_pred = model.predict(X_valid_selected)
+    mse = mean_squared_error(y_valid, y_valid_pred)
+    rmse = np.sqrt(mse)
+    print(f"Validation RMSE: {rmse:.4f}")
+
+    return model
+
+
+def predict(model, X):
+    """
+    Keep feature selection's consistency and make predictions.
+    """
+    # Select features (if any feature selection is needed)
+    X_selected = select(X)
+
+    # Predict using the trained model
+    y_pred = model.predict(X_selected)
+
+    return y_pred

rdagent/scenarios/kaggle/experiment/playground-series-s4e9_template/model/model_xgboost.py

@@ -0,0 +1,37 @@
+import pandas as pd
+import xgboost as xgb
+
+
+def select(X: pd.DataFrame) -> pd.DataFrame:
+    # Ignore feature selection logic
+    return X
+
+
+def fit(X_train: pd.DataFrame, y_train: pd.DataFrame, X_valid: pd.DataFrame, y_valid: pd.DataFrame):
+    """Define and train the model. Merge feature_select"""
+    X_train = select(X_train)
+    X_valid = select(X_valid)
+    dtrain = xgb.DMatrix(X_train, label=y_train)
+    dvalid = xgb.DMatrix(X_valid, label=y_valid)
+
+    # Parameters for regression
+    params = {
+        "objective": "reg:squarederror",  # Use squared error for regression
+        "nthread": -1,
+    }
+    num_round = 200
+
+    evallist = [(dtrain, "train"), (dvalid, "eval")]
+    bst = xgb.train(params, dtrain, num_round, evallist)
+
+    return bst
+
+
+def predict(model, X):
+    """
+    Keep feature select's consistency.
+    """
+    X = select(X)
+    dtest = xgb.DMatrix(X)
+    y_pred = model.predict(dtest)
+    return y_pred

rdagent/scenarios/kaggle/experiment/playground-series-s4e9_template/train.py

@@ -0,0 +1,103 @@
+import importlib.util
+import random
+from pathlib import Path
+
+import numpy as np
+import pandas as pd
+from fea_share_preprocess import preprocess_script
+from sklearn.metrics import mean_squared_error
+from sklearn.preprocessing import LabelEncoder
+
+# Set random seed for reproducibility
+SEED = 42
+random.seed(SEED)
+np.random.seed(SEED)
+DIRNAME = Path(__file__).absolute().resolve().parent
+
+
+def compute_rmse(y_true, y_pred):
+    """Compute RMSE for regression."""
+    mse = mean_squared_error(y_true, y_pred)
+    rmse = np.sqrt(mse)
+    return rmse
+
+
+def import_module_from_path(module_name, module_path):
+    spec = importlib.util.spec_from_file_location(module_name, module_path)
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    return module
+
+
+# 1) Preprocess the data
+X_train, X_valid, y_train, y_valid, X_test, ids = preprocess_script()
+
+# 2) Auto feature engineering
+X_train_l, X_valid_l = [], []
+X_test_l = []
+
+for f in DIRNAME.glob("feature/feat*.py"):
+    cls = import_module_from_path(f.stem, f).feature_engineering_cls()
+    cls.fit(X_train)
+    X_train_f = cls.transform(X_train)
+    X_valid_f = cls.transform(X_valid)
+    X_test_f = cls.transform(X_test)
+
+    X_train_l.append(X_train_f)
+    X_valid_l.append(X_valid_f)
+    X_test_l.append(X_test_f)
+
+X_train = pd.concat(X_train_l, axis=1, keys=[f"feature_{i}" for i in range(len(X_train_l))])
+X_valid = pd.concat(X_valid_l, axis=1, keys=[f"feature_{i}" for i in range(len(X_valid_l))])
+X_test = pd.concat(X_test_l, axis=1, keys=[f"feature_{i}" for i in range(len(X_test_l))])
+
+print(X_train.shape, X_valid.shape, X_test.shape)
+
+# Handle inf and -inf values
+X_train.replace([np.inf, -np.inf], np.nan, inplace=True)
+X_valid.replace([np.inf, -np.inf], np.nan, inplace=True)
+X_test.replace([np.inf, -np.inf], np.nan, inplace=True)
+
+from sklearn.impute import SimpleImputer
+
+imputer = SimpleImputer(strategy="mean")
+
+X_train = pd.DataFrame(imputer.fit_transform(X_train), columns=X_train.columns)
+X_valid = pd.DataFrame(imputer.transform(X_valid), columns=X_valid.columns)
+X_test = pd.DataFrame(imputer.transform(X_test), columns=X_test.columns)
+
+# Remove duplicate columns
+X_train = X_train.loc[:, ~X_train.columns.duplicated()]
+X_valid = X_valid.loc[:, ~X_valid.columns.duplicated()]
+X_test = X_test.loc[:, ~X_test.columns.duplicated()]
+
+# 3) Train the model
+model_l = []  # list[tuple[model, predict_func,]]
+for f in DIRNAME.glob("model/model*.py"):
+    m = import_module_from_path(f.stem, f)
+    model_l.append((m.fit(X_train, y_train, X_valid, y_valid), m.predict))
+
+# 4) Evaluate the model on the validation set
+y_valid_pred_l = []
+for model, predict_func in model_l:
+    y_valid_pred_l.append(predict_func(model, X_valid))
+
+# 5) Ensemble
+y_valid_pred = np.mean(y_valid_pred_l, axis=0)
+
+rmse = compute_rmse(y_valid, y_valid_pred)
+print("Final RMSE on validation set: ", rmse)
+
+# 6) Save the validation RMSE
+pd.Series(data=[rmse], index=["RMSE"]).to_csv("submission_score.csv")
+
+# 7) Make predictions on the test set and save them
+y_test_pred_l = []
+for m, m_pred in model_l:
+    y_test_pred_l.append(m_pred(m, X_test))
+
+y_test_pred = np.mean(y_test_pred_l, axis=0)
+
+# 8) Submit predictions for the test set
+submission_result = pd.DataFrame({"id": ids, "price": y_test_pred})
+submission_result.to_csv("submission.csv", index=False)

rdagent/scenarios/kaggle/experiment/sf-crime_template/model/model_randomforest.py

@@ -34,8 +34,8 @@ def fit(X_train: pd.DataFrame, y_train: pd.Series, X_valid: pd.DataFrame, y_vali
 
     # Validate the model
     y_valid_pred = model.predict(X_valid_selected)
-    accuracy = accuracy_score(y_valid, y_valid_pred)
-    print(f"Validation Accuracy: {accuracy:.4f}")
+    # accuracy = accuracy_score(y_valid, y_valid_pred)
+    # print(f"Validation Accuracy: {accuracy:.4f}")
 
     return model
 

rdagent/scenarios/kaggle/experiment/spaceship-titanic_template/fea_share_preprocess.py

@@ -0,0 +1,111 @@
+import os
+
+import pandas as pd
+from sklearn.compose import ColumnTransformer
+from sklearn.impute import SimpleImputer
+from sklearn.model_selection import train_test_split
+from sklearn.pipeline import Pipeline
+from sklearn.preprocessing import LabelEncoder, OneHotEncoder
+
+
+def prepreprocess():
+    """
+    This method loads the data, drops the unnecessary columns, and splits it into train and validation sets.
+    """
+    # Load and preprocess the data
+    data_df = pd.read_csv("/kaggle/input/train.csv")
+    data_df = data_df.drop(["PassengerId"], axis=1)
+
+    X = data_df.drop(["Transported"], axis=1)
+    y = data_df[["Transported"]]
+
+    label_encoder = LabelEncoder()
+    y = label_encoder.fit_transform(y)  # Convert class labels to numeric
+
+    # Split the data into training and validation sets
+    X_train, X_valid, y_train, y_valid = train_test_split(X, y, test_size=0.10, random_state=42)
+
+    return X_train, X_valid, y_train, y_valid
+
+
+def preprocess_fit(X_train: pd.DataFrame):
+    """
+    Fits the preprocessor on the training data and returns the fitted preprocessor.
+    """
+    # Identify numerical and categorical features
+    numerical_cols = [cname for cname in X_train.columns if X_train[cname].dtype in ["int64", "float64"]]
+    categorical_cols = [cname for cname in X_train.columns if X_train[cname].dtype == "object"]
+
+    # Define preprocessors for numerical and categorical features
+    categorical_transformer = Pipeline(
+        steps=[
+            ("imputer", SimpleImputer(strategy="most_frequent")),
+            ("onehot", OneHotEncoder(handle_unknown="ignore")),
+        ]
+    )
+
+    numerical_transformer = Pipeline(steps=[("imputer", SimpleImputer(strategy="mean"))])
+
+    # Combine preprocessing steps
+    preprocessor = ColumnTransformer(
+        transformers=[
+            ("cat", categorical_transformer, categorical_cols),
+            ("num", numerical_transformer, numerical_cols),
+        ]
+    )
+
+    # Fit the preprocessor on the training data
+    preprocessor.fit(X_train)
+
+    return preprocessor
+
+
+def preprocess_transform(X: pd.DataFrame, preprocessor):
+    """
+    Transforms the given DataFrame using the fitted preprocessor.
+    Ensures the processed data has consistent features across train, validation, and test sets.
+    """
+    # Transform the data using the fitted preprocessor
+    X_array = preprocessor.transform(X).toarray()
+
+    # Get feature names for the columns in the transformed data
+    categorical_cols = [cname for cname in X.columns if X[cname].dtype == "object"]
+    feature_names = preprocessor.named_transformers_["cat"]["onehot"].get_feature_names_out(
+        categorical_cols
+    ).tolist() + [cname for cname in X.columns if X[cname].dtype in ["int64", "float64"]]
+
+    # Convert arrays back to DataFrames
+    X_transformed = pd.DataFrame(X_array, columns=feature_names, index=X.index)
+
+    return X_transformed
+
+
+def preprocess_script():
+    """
+    This method applies the preprocessing steps to the training, validation, and test datasets.
+    """
+    if os.path.exists("X_train.pkl"):
+        X_train = pd.read_pickle("X_train.pkl")
+        X_valid = pd.read_pickle("X_valid.pkl")
+        y_train = pd.read_pickle("y_train.pkl")
+        y_valid = pd.read_pickle("y_valid.pkl")
+        X_test = pd.read_pickle("X_test.pkl")
+        passenger_ids = pd.read_pickle("passenger_ids.pkl")
+
+        return X_train, X_valid, y_train, y_valid, X_test, passenger_ids
+    X_train, X_valid, y_train, y_valid = prepreprocess()
+
+    # Fit the preprocessor on the training data
+    preprocessor = preprocess_fit(X_train)
+
+    # Preprocess the train, validation, and test data
+    X_train = preprocess_transform(X_train, preprocessor)
+    X_valid = preprocess_transform(X_valid, preprocessor)
+
+    # Load and preprocess the test data
+    submission_df = pd.read_csv("/kaggle/input/test.csv")
+    passenger_ids = submission_df["PassengerId"]
+    submission_df = submission_df.drop(["PassengerId"], axis=1)
+    X_test = preprocess_transform(submission_df, preprocessor)
+
+    return X_train, X_valid, y_train, y_valid, X_test, passenger_ids

rdagent/scenarios/kaggle/experiment/spaceship-titanic_template/feature/feature.py

@@ -0,0 +1,23 @@
+import pandas as pd
+
+"""
+Here is the feature engineering code for each task, with a class that has a fit and transform method.
+Remember
+"""
+
+
+class IdentityFeature:
+    def fit(self, train_df: pd.DataFrame):
+        """
+        Fit the feature engineering model to the training data.
+        """
+        pass
+
+    def transform(self, X: pd.DataFrame):
+        """
+        Transform the input data.
+        """
+        return X
+
+
+feature_engineering_cls = IdentityFeature

rdagent/scenarios/kaggle/experiment/spaceship-titanic_template/model/model_nn.py

@@ -0,0 +1,76 @@
+import numpy as np
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from torch.utils.data import DataLoader, TensorDataset
+from tqdm import tqdm
+
+# Check if a GPU is available
+device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+
+# Restored three-layer model structure
+class HybridFeatureInteractionModel(nn.Module):
+    def __init__(self, num_features):
+        super(HybridFeatureInteractionModel, self).__init__()
+        self.fc1 = nn.Linear(num_features, 128)
+        self.bn1 = nn.BatchNorm1d(128)
+        self.fc2 = nn.Linear(128, 64)
+        self.bn2 = nn.BatchNorm1d(64)
+        self.fc3 = nn.Linear(64, 1)
+        self.dropout = nn.Dropout(0.3)
+
+    def forward(self, x):
+        x = F.relu(self.bn1(self.fc1(x)))
+        x = F.relu(self.bn2(self.fc2(x)))
+        x = self.dropout(x)
+        x = torch.sigmoid(self.fc3(x))
+        return x
+
+
+# Training function
+def fit(X_train, y_train, X_valid, y_valid):
+    num_features = X_train.shape[1]
+    model = HybridFeatureInteractionModel(num_features).to(device)
+    criterion = nn.BCELoss()  # Binary classification problem
+    optimizer = torch.optim.Adam(model.parameters(), lr=0.001)
+
+    # Convert to TensorDataset and create DataLoader
+    train_dataset = TensorDataset(
+        torch.tensor(X_train.to_numpy(), dtype=torch.float32), torch.tensor(y_train.reshape(-1), dtype=torch.float32)
+    )
+    valid_dataset = TensorDataset(
+        torch.tensor(X_valid.to_numpy(), dtype=torch.float32), torch.tensor(y_valid.reshape(-1), dtype=torch.float32)
+    )
+    train_loader = DataLoader(train_dataset, batch_size=32, shuffle=True)
+    valid_loader = DataLoader(valid_dataset, batch_size=32, shuffle=False)
+
+    # Train the model
+    model.train()
+    for epoch in range(5):
+        print(f"Epoch {epoch + 1}/5")
+        epoch_loss = 0
+        for X_batch, y_batch in tqdm(train_loader, desc="Training", leave=False):
+            X_batch, y_batch = X_batch.to(device), y_batch.to(device)  # Move data to the device
+            optimizer.zero_grad()
+            outputs = model(X_batch).squeeze(1)  # Reshape outputs to [32]
+            loss = criterion(outputs, y_batch)  # Adjust target shape
+            loss.backward()
+            optimizer.step()
+            epoch_loss += loss.item()
+        print(f"End of epoch {epoch + 1}, Avg Loss: {epoch_loss / len(train_loader):.4f}")
+
+    return model
+
+
+# Prediction function
+def predict(model, X):
+    model.eval()
+    predictions = []
+    with torch.no_grad():
+        X_tensor = torch.tensor(X.values, dtype=torch.float32).to(device)  # Move data to the device
+        for i in tqdm(range(0, len(X_tensor), 32), desc="Predicting", leave=False):
+            batch = X_tensor[i : i + 32]  # Predict in batches
+            pred = model(batch).squeeze().cpu().numpy()  # Move results back to CPU
+            predictions.extend(pred)
+    return np.array(predictions)  # Return boolean predictions

rdagent/scenarios/kaggle/experiment/spaceship-titanic_template/model/model_randomforest.py

@@ -0,0 +1,54 @@
+"""
+Motivation of the model:
+The Random Forest model is chosen for its robustness and ability to handle large datasets with higher dimensionality.
+It reduces overfitting by averaging multiple decision trees and typically performs well out of the box, making it a good
+baseline model for many classification tasks.
+"""
+
+import pandas as pd
+from sklearn.ensemble import RandomForestClassifier
+from sklearn.metrics import accuracy_score
+
+
+def select(X: pd.DataFrame) -> pd.DataFrame:
+    """
+    Select relevant features. To be used in fit & predict function.
+    """
+    # For now, we assume all features are relevant. This can be expanded to feature selection logic.
+    return X
+
+
+def fit(X_train: pd.DataFrame, y_train: pd.Series, X_valid: pd.DataFrame, y_valid: pd.Series):
+    """
+    Define and train the Random Forest model. Merge feature selection into the pipeline.
+    """
+    # Initialize the Random Forest model
+    model = RandomForestClassifier(n_estimators=100, random_state=32, n_jobs=-1)
+
+    # Select features (if any feature selection is needed)
+    X_train_selected = select(X_train)
+    X_valid_selected = select(X_valid)
+
+    # Fit the model
+    model.fit(X_train_selected, y_train)
+
+    # Validate the model
+    y_valid_pred = model.predict(X_valid_selected)
+    accuracy = accuracy_score(y_valid, y_valid_pred)
+    print(f"Validation Accuracy: {accuracy:.4f}")
+
+    return model
+
+
+def predict(model, X):
+    """
+    Keep feature selection's consistency and make predictions.
+    """
+    # Select features (if any feature selection is needed)
+    X_selected = select(X)
+
+    # Predict using the trained model
+    y_pred_prob = model.predict_proba(X_selected)[:, 1]
+
+    # Apply threshold to get boolean predictions
+    return y_pred_prob

rdagent/scenarios/kaggle/experiment/spaceship-titanic_template/model/model_xgboost.py

@@ -0,0 +1,40 @@
+"""
+motivation  of the model
+"""
+
+import pandas as pd
+import xgboost as xgb
+
+
+def select(X: pd.DataFrame) -> pd.DataFrame:
+    # Ignore feature selection logic
+    return X
+
+
+def fit(X_train: pd.DataFrame, y_train: pd.DataFrame, X_valid: pd.DataFrame, y_valid: pd.DataFrame):
+    """Define and train the model. Merge feature_select"""
+    X_train = select(X_train)
+    X_valid = select(X_valid)
+    dtrain = xgb.DMatrix(X_train, label=y_train)
+    dvalid = xgb.DMatrix(X_valid, label=y_valid)
+
+    # TODO: for quick running....
+    params = {
+        "nthred": -1,
+    }
+    num_round = 180
+
+    evallist = [(dtrain, "train"), (dvalid, "eval")]
+    bst = xgb.train(params, dtrain, num_round, evallist)
+
+    return bst
+
+
+def predict(model, X):
+    """
+    Keep feature select's consistency.
+    """
+    X = select(X)
+    dtest = xgb.DMatrix(X)
+    y_pred_prob = model.predict(dtest)
+    return y_pred_prob

rdagent/scenarios/kaggle/experiment/spaceship-titanic_template/train.py

@@ -0,0 +1,109 @@
+import importlib.util
+import random
+from pathlib import Path
+
+import numpy as np
+import pandas as pd
+from fea_share_preprocess import preprocess_script
+from sklearn.metrics import accuracy_score, matthews_corrcoef
+from sklearn.preprocessing import LabelEncoder
+
+# Set random seed for reproducibility
+SEED = 42
+random.seed(SEED)
+np.random.seed(SEED)
+DIRNAME = Path(__file__).absolute().resolve().parent
+
+
+# support various method for metrics calculation
+def compute_metrics_for_classification(y_true, y_pred):
+    """Compute accuracy metric for classification."""
+    accuracy = accuracy_score(y_true, y_pred)
+    return accuracy
+
+
+def import_module_from_path(module_name, module_path):
+    spec = importlib.util.spec_from_file_location(module_name, module_path)
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    return module
+
+
+# 1) Preprocess the data
+# TODO 如果已经做过数据预处理了，不需要再做了
+X_train, X_valid, y_train, y_valid, X_test, passenger_ids = preprocess_script()
+
+# 2) Auto feature engineering
+X_train_l, X_valid_l = [], []
+X_test_l = []
+
+for f in DIRNAME.glob("feature/feat*.py"):
+    cls = import_module_from_path(f.stem, f).feature_engineering_cls()
+    cls.fit(X_train)
+    X_train_f = cls.transform(X_train)
+    X_valid_f = cls.transform(X_valid)
+    X_test_f = cls.transform(X_test)
+
+    X_train_l.append(X_train_f)
+    X_valid_l.append(X_valid_f)
+    X_test_l.append(X_test_f)
+
+X_train = pd.concat(X_train_l, axis=1, keys=[f"feature_{i}" for i in range(len(X_train_l))])
+X_valid = pd.concat(X_valid_l, axis=1, keys=[f"feature_{i}" for i in range(len(X_valid_l))])
+X_test = pd.concat(X_test_l, axis=1, keys=[f"feature_{i}" for i in range(len(X_test_l))])
+
+print(X_train.shape, X_valid.shape, X_test.shape)
+
+# Handle inf and -inf values
+X_train.replace([np.inf, -np.inf], np.nan, inplace=True)
+X_valid.replace([np.inf, -np.inf], np.nan, inplace=True)
+X_test.replace([np.inf, -np.inf], np.nan, inplace=True)
+
+from sklearn.impute import SimpleImputer
+
+imputer = SimpleImputer(strategy="mean")
+
+X_train = pd.DataFrame(imputer.fit_transform(X_train), columns=X_train.columns)
+X_valid = pd.DataFrame(imputer.transform(X_valid), columns=X_valid.columns)
+X_test = pd.DataFrame(imputer.transform(X_test), columns=X_test.columns)
+
+# Remove duplicate columns
+X_train = X_train.loc[:, ~X_train.columns.duplicated()]
+X_valid = X_valid.loc[:, ~X_valid.columns.duplicated()]
+X_test = X_test.loc[:, ~X_test.columns.duplicated()]
+
+# 3) Train the model
+model_l = []  # list[tuple[model, predict_func,]]
+for f in DIRNAME.glob("model/model*.py"):
+    m = import_module_from_path(f.stem, f)
+    model_l.append((m.fit(X_train, y_train, X_valid, y_valid), m.predict))
+
+# 4) Evaluate the model on the validation set
+y_valid_pred_l = []
+for model, predict_func in model_l:
+    y_valid_pred_l.append(predict_func(model, X_valid))
+
+# 5) Ensemble
+# TODO: ensemble method in a script
+# Average the predictions and apply a threshold to determine class labels
+y_valid_pred = np.mean(y_valid_pred_l, axis=0)
+y_valid_pred = (y_valid_pred > 0.5).astype(int)
+
+mcc = compute_metrics_for_classification(y_valid, y_valid_pred)
+print("Final on validation set: ", mcc)
+
+# 6) Save the validation accuracy
+pd.Series(data=[mcc], index=["MCC"]).to_csv("submission_score.csv")
+
+# 7) Make predictions on the test set and save them
+y_test_pred_l = []
+for m, m_pred in model_l:
+    y_test_pred_l.append(m_pred(m, X_test))  # TODO Make this an ensemble. Currently it uses the last prediction
+
+y_test_pred = np.mean(y_test_pred_l, axis=0)
+y_test_pred = (y_test_pred > 0.5).astype(bool)
+
+submission_result = pd.DataFrame({"PassengerId": passenger_ids, "Transported": y_test_pred})
+
+# 8) Submit predictions for the test set
+submission_result.to_csv("submission.csv", index=False)