From 89e6a48b19ead1bebd97d687af41cbbb8036b10a Mon Sep 17 00:00:00 2001 From: Way2Learn <118058822+Xisen-Wang@users.noreply.github.com> Date: Sat, 28 Sep 2024 01:32:35 +0800 Subject: [PATCH] feat: Iceberge competition (#372) * Relevant files fixed * Make the entire system run * Fix CI * refine the template --------- Co-authored-by: WinstonLiye <1957922024@qq.com> --- .../fea_share_preprocess.py | 76 +++++++++++++ .../feature/feature.py | 23 ++++ .../model/model_xgboost.py | 38 +++++++ .../model/select_xgboost.py | 12 ++ .../train.py | 106 ++++++++++++++++++ test/utils/test_kaggle.py | 2 +- 6 files changed, 256 insertions(+), 1 deletion(-) create mode 100644 rdagent/scenarios/kaggle/experiment/statoil-iceberg-classifier-challenge_template/fea_share_preprocess.py create mode 100644 rdagent/scenarios/kaggle/experiment/statoil-iceberg-classifier-challenge_template/feature/feature.py create mode 100644 rdagent/scenarios/kaggle/experiment/statoil-iceberg-classifier-challenge_template/model/model_xgboost.py create mode 100644 rdagent/scenarios/kaggle/experiment/statoil-iceberg-classifier-challenge_template/model/select_xgboost.py create mode 100644 rdagent/scenarios/kaggle/experiment/statoil-iceberg-classifier-challenge_template/train.py diff --git a/rdagent/scenarios/kaggle/experiment/statoil-iceberg-classifier-challenge_template/fea_share_preprocess.py b/rdagent/scenarios/kaggle/experiment/statoil-iceberg-classifier-challenge_template/fea_share_preprocess.py new file mode 100644 index 00000000..f7be3e2d --- /dev/null +++ b/rdagent/scenarios/kaggle/experiment/statoil-iceberg-classifier-challenge_template/fea_share_preprocess.py @@ -0,0 +1,76 @@ +import os + +import numpy as np +import pandas as pd +from sklearn.model_selection import train_test_split + + +def prepreprocess(): + """ + This method loads the data, processes it, and splits it into train and validation sets. + """ + # Load the data + train = pd.read_json("/kaggle/input/train.json") + train = train.drop(columns=["id"]) + test = pd.read_json("/kaggle/input/test.json") + test_ids = test["id"] + test = test.drop(columns=["id"]) + + # Process the data + def process_data(df): + X = df.copy() + X["band_1"] = X["band_1"].apply(lambda x: np.array(x).reshape(75, 75)) + X["band_2"] = X["band_2"].apply(lambda x: np.array(x).reshape(75, 75)) + X["band_3"] = (X["band_1"] + X["band_2"]) / 2 + + # Extract features + X["band_1_mean"] = X["band_1"].apply(np.mean) + X["band_2_mean"] = X["band_2"].apply(np.mean) + X["band_3_mean"] = X["band_3"].apply(np.mean) + X["band_1_max"] = X["band_1"].apply(np.max) + X["band_2_max"] = X["band_2"].apply(np.max) + X["band_3_max"] = X["band_3"].apply(np.max) + + # Handle missing incidence angles + X["inc_angle"] = X["inc_angle"].replace("na", np.nan).astype(float) + X["inc_angle"].fillna(X["inc_angle"].mean(), inplace=True) + + return X + + X_train = process_data(train) + X_test = process_data(test) + + y_train = X_train["is_iceberg"] + X_train = X_train.drop(["is_iceberg", "band_1", "band_2", "band_3"], axis=1) + X_test = X_test.drop(["band_1", "band_2", "band_3"], axis=1) + + # Split the data into training and validation sets + X_train, X_valid, y_train, y_valid = train_test_split(X_train, y_train, test_size=0.20, random_state=42) + + return X_train, X_valid, y_train, y_valid, X_test, test_ids + + +def preprocess_script(): + """ + This method applies the preprocessing steps to the training, validation, and test datasets. + """ + if os.path.exists("X_train.pkl"): + X_train = pd.read_pickle("X_train.pkl") + X_valid = pd.read_pickle("X_valid.pkl") + y_train = pd.read_pickle("y_train.pkl") + y_valid = pd.read_pickle("y_valid.pkl") + X_test = pd.read_pickle("X_test.pkl") + test_ids = pd.read_pickle("test_ids.pkl") + return X_train, X_valid, y_train, y_valid, X_test, test_ids + + X_train, X_valid, y_train, y_valid, X_test, test_ids = prepreprocess() + + # Save preprocessed data + X_train.to_pickle("X_train.pkl") + X_valid.to_pickle("X_valid.pkl") + y_train.to_pickle("y_train.pkl") + y_valid.to_pickle("y_valid.pkl") + X_test.to_pickle("X_test.pkl") + test_ids.to_pickle("test_ids.pkl") + + return X_train, X_valid, y_train, y_valid, X_test, test_ids diff --git a/rdagent/scenarios/kaggle/experiment/statoil-iceberg-classifier-challenge_template/feature/feature.py b/rdagent/scenarios/kaggle/experiment/statoil-iceberg-classifier-challenge_template/feature/feature.py new file mode 100644 index 00000000..8ae043ac --- /dev/null +++ b/rdagent/scenarios/kaggle/experiment/statoil-iceberg-classifier-challenge_template/feature/feature.py @@ -0,0 +1,23 @@ +import pandas as pd + +""" +Here is the feature engineering code for each task, with a class that has a fit and transform method. +Remember +""" + + +class IdentityFeature: + def fit(self, train_df: pd.DataFrame): + """ + Fit the feature engineering model to the training data. + """ + pass + + def transform(self, X: pd.DataFrame): + """ + Transform the input data. + """ + return X + + +feature_engineering_cls = IdentityFeature diff --git a/rdagent/scenarios/kaggle/experiment/statoil-iceberg-classifier-challenge_template/model/model_xgboost.py b/rdagent/scenarios/kaggle/experiment/statoil-iceberg-classifier-challenge_template/model/model_xgboost.py new file mode 100644 index 00000000..2ba39ab7 --- /dev/null +++ b/rdagent/scenarios/kaggle/experiment/statoil-iceberg-classifier-challenge_template/model/model_xgboost.py @@ -0,0 +1,38 @@ +""" +motivation of the model +""" + +import numpy as np +import pandas as pd +import xgboost as xgb + + +def fit(X_train: pd.DataFrame, y_train: pd.Series, X_valid: pd.DataFrame, y_valid: pd.Series): + """Define and train the model. Merge feature_select""" + dtrain = xgb.DMatrix(X_train, label=y_train) + dvalid = xgb.DMatrix(X_valid, label=y_valid) + + params = { + "objective": "binary:logistic", + "eval_metric": "logloss", + "eta": 0.1, + "max_depth": 6, + "subsample": 0.8, + "colsample_bytree": 0.8, + "nthread": -1, + } + num_round = 200 # Increase number of rounds + + evallist = [(dtrain, "train"), (dvalid, "eval")] + bst = xgb.train(params, dtrain, num_round, evallist, early_stopping_rounds=50) + + return bst + + +def predict(model, X): + """ + Keep feature select's consistency. + """ + dtest = xgb.DMatrix(X) + y_pred_prob = model.predict(dtest) + return y_pred_prob.reshape(-1, 1) diff --git a/rdagent/scenarios/kaggle/experiment/statoil-iceberg-classifier-challenge_template/model/select_xgboost.py b/rdagent/scenarios/kaggle/experiment/statoil-iceberg-classifier-challenge_template/model/select_xgboost.py new file mode 100644 index 00000000..d2a15dee --- /dev/null +++ b/rdagent/scenarios/kaggle/experiment/statoil-iceberg-classifier-challenge_template/model/select_xgboost.py @@ -0,0 +1,12 @@ +import pandas as pd + + +def select(X: pd.DataFrame) -> pd.DataFrame: + """ + Select relevant features. To be used in fit & predict function. + """ + # For now, we assume all features are relevant. This can be expanded to feature selection logic. + if X.columns.nlevels == 1: + return X + X.columns = ["_".join(str(col)).strip() for col in X.columns.values] + return X diff --git a/rdagent/scenarios/kaggle/experiment/statoil-iceberg-classifier-challenge_template/train.py b/rdagent/scenarios/kaggle/experiment/statoil-iceberg-classifier-challenge_template/train.py new file mode 100644 index 00000000..57fcbe88 --- /dev/null +++ b/rdagent/scenarios/kaggle/experiment/statoil-iceberg-classifier-challenge_template/train.py @@ -0,0 +1,106 @@ +import importlib.util +import random +from pathlib import Path + +import numpy as np +import pandas as pd +from fea_share_preprocess import preprocess_script +from sklearn.metrics import log_loss + +# Set random seed for reproducibility +SEED = 42 +random.seed(SEED) +np.random.seed(SEED) +DIRNAME = Path(__file__).absolute().resolve().parent + + +# Support various method for metrics calculation +def compute_metrics_for_classification(y_true, y_pred): + """Compute log loss for classification.""" + return log_loss(y_true, y_pred) + + +def import_module_from_path(module_name, module_path): + spec = importlib.util.spec_from_file_location(module_name, module_path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +# 1) Preprocess the data +X_train, X_valid, y_train, y_valid, X_test, test_ids = preprocess_script() + + +# 2) Auto feature engineering +X_train_l, X_valid_l = [], [] +X_test_l = [] + +for f in DIRNAME.glob("feature/feat*.py"): + cls = import_module_from_path(f.stem, f).feature_engineering_cls() + cls.fit(X_train) + X_train_f = cls.transform(X_train) + X_valid_f = cls.transform(X_valid) + X_test_f = cls.transform(X_test) + + if X_train_f.shape[-1] == X_valid_f.shape[-1] and X_train_f.shape[-1] == X_test_f.shape[-1]: + X_train_l.append(X_train_f) + X_valid_l.append(X_valid_f) + X_test_l.append(X_test_f) + +X_train = pd.concat(X_train_l, axis=1) +X_valid = pd.concat(X_valid_l, axis=1) +X_test = pd.concat(X_test_l, axis=1) + + +# Handle inf and -inf values +X_train.replace([np.inf, -np.inf], np.nan, inplace=True) +X_valid.replace([np.inf, -np.inf], np.nan, inplace=True) +X_test.replace([np.inf, -np.inf], np.nan, inplace=True) + +from sklearn.impute import SimpleImputer + +imputer = SimpleImputer(strategy="mean") + +X_train = pd.DataFrame(imputer.fit_transform(X_train), columns=X_train.columns) +X_valid = pd.DataFrame(imputer.transform(X_valid), columns=X_valid.columns) +X_test = pd.DataFrame(imputer.transform(X_test), columns=X_test.columns) + +# Remove duplicate columns +X_train = X_train.loc[:, ~X_train.columns.duplicated()] +X_valid = X_valid.loc[:, ~X_valid.columns.duplicated()] +X_test = X_test.loc[:, ~X_test.columns.duplicated()] + +print(X_train.shape, X_valid.shape, X_test.shape) + +# 3) Train the model +model_l = [] # list[tuple[model, predict_func]] +for f in DIRNAME.glob("model/model*.py"): + select_python_path = f.with_name(f.stem.replace("model", "select") + f.suffix) + select_m = import_module_from_path(select_python_path.stem, select_python_path) + X_train_selected = select_m.select(X_train.copy()) + X_valid_selected = select_m.select(X_valid.copy()) + + m = import_module_from_path(f.stem, f) + model_l.append((m.fit(X_train_selected, y_train, X_valid_selected, y_valid), m.predict, select_m)) + +# 4) Evaluate the model on the validation set +metrics_all = [] +for model, predict_func, select_m in model_l: + X_valid_selected = select_m.select(X_valid.copy()) + y_valid_pred = predict_func(model, X_valid_selected) + metrics = compute_metrics_for_classification(y_valid, y_valid_pred) + print("Metrics: ", metrics) + metrics_all.append(metrics) + +# 5) Save the validation log loss +min_index = np.argmin(metrics_all) +pd.Series(data=[metrics_all[min_index]], index=["Log Loss"]).to_csv("submission_score.csv") + +# 6) Make predictions on the test set and save them +X_test_selected = model_l[min_index][2].select(X_test.copy()) +y_test_pred = model_l[min_index][1](model_l[min_index][0], X_test_selected) + + +# 7) Submit predictions for the test set +submission_result = pd.DataFrame({"id": test_ids, "is_iceberg": y_test_pred.ravel()}) +submission_result.to_csv("submission.csv", index=False) diff --git a/test/utils/test_kaggle.py b/test/utils/test_kaggle.py index cadb7dd1..14a8030b 100644 --- a/test/utils/test_kaggle.py +++ b/test/utils/test_kaggle.py @@ -27,7 +27,7 @@ class TestTpl(unittest.TestCase): ws.execute() success = (ws.workspace_path / "submission.csv").exists() self.assertTrue(success, "submission.csv is not generated") - ws.clear() + # ws.clear() if __name__ == "__main__":