diff --git a/rdagent/app/kaggle/loop.py b/rdagent/app/kaggle/loop.py index 9642e062..50f3fa71 100644 --- a/rdagent/app/kaggle/loop.py +++ b/rdagent/app/kaggle/loop.py @@ -41,7 +41,7 @@ class KaggleRDLoop(RDLoop): if PROP_SETTING.knowledge_base != "" else None ) - logger.log_object(knowledge_base, tag="knowledge_base") + logger.log_object(knowledge_base.__dict__, tag="knowledge_base") self.hypothesis_gen: HypothesisGen = import_class(PROP_SETTING.hypothesis_gen)(scen) logger.log_object(self.hypothesis_gen, tag="hypothesis generator") diff --git a/rdagent/core/knowledge_base.py b/rdagent/core/knowledge_base.py index f4cd9ab5..480fc0f9 100644 --- a/rdagent/core/knowledge_base.py +++ b/rdagent/core/knowledge_base.py @@ -14,12 +14,12 @@ class KnowledgeBase: if self.path is not None and self.path.exists(): with self.path.open("rb") as f: self.__dict__.update( - pickle.load(f).__dict__, + pickle.load(f), ) # TODO: because we need to align with init function, we need a less hacky way to do this def dump(self) -> None: if self.path is not None: self.path.parent.mkdir(parents=True, exist_ok=True) - pickle.dump(self, self.path.open("wb")) + pickle.dump(self.__dict__, self.path.open("wb")) else: logger.warning("KnowledgeBase path is not set, dump failed.") diff --git a/rdagent/scenarios/kaggle/experiment/digit-recognizer_template/fea_share_preprocess.py b/rdagent/scenarios/kaggle/experiment/digit-recognizer_template/fea_share_preprocess.py index 40b42051..f83ef601 100644 --- a/rdagent/scenarios/kaggle/experiment/digit-recognizer_template/fea_share_preprocess.py +++ b/rdagent/scenarios/kaggle/experiment/digit-recognizer_template/fea_share_preprocess.py @@ -57,11 +57,6 @@ def clean_and_impute_data(X_train, X_valid, X_test): then imputes missing values using the mean strategy. Also removes duplicate columns. """ - # Replace inf and -inf with NaNa - X_train.replace([np.inf, -np.inf], np.nan, inplace=True) - X_valid.replace([np.inf, -np.inf], np.nan, inplace=True) - X_test.replace([np.inf, -np.inf], np.nan, inplace=True) - # Impute missing values imputer = SimpleImputer(strategy="mean") X_train = pd.DataFrame(imputer.fit_transform(X_train), columns=X_train.columns) diff --git a/rdagent/scenarios/kaggle/experiment/digit-recognizer_template/model/model_nn.py b/rdagent/scenarios/kaggle/experiment/digit-recognizer_template/model/model_nn.py new file mode 100644 index 00000000..4de18357 --- /dev/null +++ b/rdagent/scenarios/kaggle/experiment/digit-recognizer_template/model/model_nn.py @@ -0,0 +1,86 @@ +import pandas as pd +import torch +import torch.nn as nn +import torch.nn.functional as F +import torch.optim as optim +from torch.utils.data import DataLoader, TensorDataset + + +# Define the neural network model with Batch Normalization +class NeuralNetwork(nn.Module): + def __init__(self, input_channels, num_classes): + super(NeuralNetwork, self).__init__() + self.conv1 = nn.Conv2d(in_channels=input_channels, out_channels=30, kernel_size=(3, 3), stride=2) + self.dropout1 = nn.Dropout(0.5) + self.conv2 = nn.Conv2d(in_channels=30, out_channels=30, kernel_size=(3, 3), stride=2) + self.dropout2 = nn.Dropout(0.5) + self.flatten = nn.Flatten() + self.fc1 = nn.Linear(30 * 6 * 6, 128) # Adjust based on your input size + self.fc2 = nn.Linear(128, num_classes) + + def forward(self, x): + x = F.relu(self.conv1(x)) + x = self.dropout1(x) + x = F.relu(self.conv2(x)) + x = self.dropout2(x) + x = self.flatten(x) + x = F.relu(self.fc1(x)) + x = F.softmax(self.fc2(x), dim=1) + return x + + +def fit(X_train: pd.DataFrame, y_train: pd.DataFrame, X_valid: pd.DataFrame, y_valid: pd.DataFrame): + # Convert data to PyTorch tensors and reshape it for convolutional layers + X_train_tensor = ( + torch.tensor(X_train.values, dtype=torch.float32).view(-1, 1, 28, 28).to(device) + ) # Reshape and move to GPU + y_train_tensor = torch.tensor(y_train.values, dtype=torch.long).to(device) + X_valid_tensor = torch.tensor(X_valid.values, dtype=torch.float32).view(-1, 1, 28, 28).to(device) + y_valid_tensor = torch.tensor(y_valid.values, dtype=torch.long).to(device) + + # Create datasets and dataloaders + train_dataset = TensorDataset(X_train_tensor, y_train_tensor) + valid_dataset = TensorDataset(X_valid_tensor, y_valid_tensor) + train_loader = DataLoader(train_dataset, batch_size=128, shuffle=True) + valid_loader = DataLoader(valid_dataset, batch_size=128, shuffle=False) + + # Initialize the model, loss function and optimizer + model = NeuralNetwork(input_channels=1, num_classes=len(set(y_train))).to(device) + criterion = nn.CrossEntropyLoss().to(device) + optimizer = optim.Adam(model.parameters(), lr=0.0005) + + # Train the model + num_epochs = 400 + for epoch in range(num_epochs): + model.train() + for X_batch, y_batch in train_loader: + optimizer.zero_grad() + outputs = model(X_batch) + loss = criterion(outputs, y_batch) + loss.backward() + optimizer.step() + + # Validate the model + model.eval() + valid_loss = 0 + correct = 0 + with torch.no_grad(): + for X_batch, y_batch in valid_loader: + outputs = model(X_batch) + valid_loss += criterion(outputs, y_batch).item() + _, predicted = torch.max(outputs, 1) + correct += (predicted == y_batch).sum().item() + + accuracy = correct / len(valid_loader.dataset) + print(f"Epoch {epoch+1}/{num_epochs}, Validation Accuracy: {accuracy:.4f}") + + return model + + +def predict(model, X): + X_tensor = torch.tensor(X.values, dtype=torch.float32).view(-1, 1, 28, 28).to(device) + model.eval() + with torch.no_grad(): + outputs = model(X_tensor) + _, predicted = torch.max(outputs, 1) + return predicted.cpu().numpy().reshape(-1, 1) diff --git a/rdagent/scenarios/kaggle/experiment/digit-recognizer_template/model/select_nn.py b/rdagent/scenarios/kaggle/experiment/digit-recognizer_template/model/select_nn.py new file mode 100644 index 00000000..d2a15dee --- /dev/null +++ b/rdagent/scenarios/kaggle/experiment/digit-recognizer_template/model/select_nn.py @@ -0,0 +1,12 @@ +import pandas as pd + + +def select(X: pd.DataFrame) -> pd.DataFrame: + """ + Select relevant features. To be used in fit & predict function. + """ + # For now, we assume all features are relevant. This can be expanded to feature selection logic. + if X.columns.nlevels == 1: + return X + X.columns = ["_".join(str(col)).strip() for col in X.columns.values] + return X diff --git a/rdagent/scenarios/kaggle/experiment/digit-recognizer_template/train.py b/rdagent/scenarios/kaggle/experiment/digit-recognizer_template/train.py index fc4cbf3d..c7572f09 100644 --- a/rdagent/scenarios/kaggle/experiment/digit-recognizer_template/train.py +++ b/rdagent/scenarios/kaggle/experiment/digit-recognizer_template/train.py @@ -83,6 +83,6 @@ ids = range(1, len(X_test) + 1) # TODO: fix selection print(X_valid_selected.columns) -y_test_pred = model_l[min_index][1](model_l[min_index][0], model_l[min_index][2].select(X_test)) +y_test_pred = model_l[min_index][1](model_l[min_index][0], model_l[min_index][2].select(X_test)).flatten() submission_result = pd.DataFrame({"ImageId": ids, "Label": y_test_pred}) submission_result.to_csv("submission.csv", index=False)