diff --git a/rdagent/components/proposal/__init__.py b/rdagent/components/proposal/__init__.py index 846f93cf..305df219 100644 --- a/rdagent/components/proposal/__init__.py +++ b/rdagent/components/proposal/__init__.py @@ -19,7 +19,6 @@ prompt_dict = Prompts(file_path=Path(__file__).parent / "prompts.yaml") class LLMHypothesisGen(HypothesisGen): - def __init__(self, scen: Scenario): super().__init__(scen) diff --git a/rdagent/scenarios/kaggle/experiment/new-york-city-taxi-fare-prediction_template/fea_share_preprocess.py b/rdagent/scenarios/kaggle/experiment/new-york-city-taxi-fare-prediction_template/fea_share_preprocess.py new file mode 100644 index 00000000..29a9ce4f --- /dev/null +++ b/rdagent/scenarios/kaggle/experiment/new-york-city-taxi-fare-prediction_template/fea_share_preprocess.py @@ -0,0 +1,75 @@ +import os + +import numpy as np +import pandas as pd +from sklearn.impute import SimpleImputer +from sklearn.model_selection import train_test_split + +index_name = "key" +label_name = "fare_amount" + + +def prepreprocess(): + """ + This method loads the data, drops the unnecessary columns, and splits it into train and validation sets. + """ + # Load and preprocess the data + data_df = pd.read_csv("/kaggle/input/train.csv") + data_df = data_df.drop([index_name], axis=1) + + X = data_df.drop([label_name], axis=1) + y = data_df[label_name] + + # Split the data into training and validation sets + X_train, X_valid, y_train, y_valid = train_test_split(X, y, test_size=0.20, random_state=42) + + return X_train, X_valid, y_train, y_valid + + +def preprocess_script(): + """ + This method applies the preprocessing steps to the training, validation, and test datasets. + """ + if os.path.exists("/kaggle/input/X_train.pkl"): + X_train = pd.read_pickle("/kaggle/input/X_train.pkl") + X_valid = pd.read_pickle("/kaggle/input/X_valid.pkl") + y_train = pd.read_pickle("/kaggle/input/y_train.pkl") + y_valid = pd.read_pickle("/kaggle/input/y_valid.pkl") + X_test = pd.read_pickle("/kaggle/input/X_test.pkl") + others = pd.read_pickle("/kaggle/input/others.pkl") + + return X_train, X_valid, y_train, y_valid, X_test, *others + + X_train, X_valid, y_train, y_valid = prepreprocess() + + # Load and preprocess the test data + submission_df = pd.read_csv("/kaggle/input/test.csv") + ids = submission_df[index_name] + X_test = submission_df.drop([index_name], axis=1) + + return X_train, X_valid, y_train, y_valid, X_test, ids + + +def clean_and_impute_data(X_train, X_valid, X_test): + """ + Handles inf and -inf values by replacing them with NaN, + then imputes missing values using the mean strategy. + Also removes duplicate columns. + """ + # Replace inf and -inf with NaN + X_train.replace([np.inf, -np.inf], np.nan, inplace=True) + X_valid.replace([np.inf, -np.inf], np.nan, inplace=True) + X_test.replace([np.inf, -np.inf], np.nan, inplace=True) + + # Impute missing values + imputer = SimpleImputer(strategy="mean") + X_train = pd.DataFrame(imputer.fit_transform(X_train), columns=X_train.columns) + X_valid = pd.DataFrame(imputer.transform(X_valid), columns=X_valid.columns) + X_test = pd.DataFrame(imputer.transform(X_test), columns=X_test.columns) + + # Remove duplicate columns + X_train = X_train.loc[:, ~X_train.columns.duplicated()] + X_valid = X_valid.loc[:, ~X_valid.columns.duplicated()] + X_test = X_test.loc[:, ~X_test.columns.duplicated()] + + return X_train, X_valid, X_test diff --git a/rdagent/scenarios/kaggle/experiment/new-york-city-taxi-fare-prediction_template/feature/feature.py b/rdagent/scenarios/kaggle/experiment/new-york-city-taxi-fare-prediction_template/feature/feature.py new file mode 100644 index 00000000..55e33c99 --- /dev/null +++ b/rdagent/scenarios/kaggle/experiment/new-york-city-taxi-fare-prediction_template/feature/feature.py @@ -0,0 +1,30 @@ +import pandas as pd + +""" +Here is the feature engineering code for each task, with a class that has a fit and transform method. +Remember +""" + + +class DatetimeFeature: + def fit(self, train_df: pd.DataFrame): + """ + Fit the feature engineering model to the training data. + """ + pass + + def transform(self, X: pd.DataFrame): + """ + Transform the input data. + """ + X["pickup_datetime"] = pd.to_datetime(X["pickup_datetime"], format="%Y-%m-%d %H:%M:%S UTC") + X["hour"] = X.pickup_datetime.dt.hour + X["day"] = X.pickup_datetime.dt.day + X["month"] = X.pickup_datetime.dt.month + X["weekday"] = X.pickup_datetime.dt.weekday + X["year"] = X.pickup_datetime.dt.year + X.drop(columns=["pickup_datetime"], inplace=True) + return X + + +feature_engineering_cls = DatetimeFeature diff --git a/rdagent/scenarios/kaggle/experiment/new-york-city-taxi-fare-prediction_template/model/model_linear.py b/rdagent/scenarios/kaggle/experiment/new-york-city-taxi-fare-prediction_template/model/model_linear.py new file mode 100644 index 00000000..3980853e --- /dev/null +++ b/rdagent/scenarios/kaggle/experiment/new-york-city-taxi-fare-prediction_template/model/model_linear.py @@ -0,0 +1,38 @@ +""" +Motivation of the model: +The Linear Regression model is chosen for its simplicity and interpretability. It is a good starting point for regression tasks +and provides a baseline to compare more complex models against. Linear Regression assumes a linear relationship between the +features and the target variable, which can be a reasonable assumption for many problems. +""" + +import pandas as pd +from sklearn.linear_model import LinearRegression +from sklearn.metrics import mean_squared_error + + +def fit(X_train: pd.DataFrame, y_train: pd.Series, X_valid: pd.DataFrame, y_valid: pd.Series): + """ + Define and train the Linear Regression model. Merge feature selection into the pipeline. + """ + # Initialize the Linear Regression model + model = LinearRegression() + + # Fit the model + model.fit(X_train, y_train) + + # Validate the model + y_valid_pred = model.predict(X_valid) + mse = mean_squared_error(y_valid, y_valid_pred) + print(f"Validation Mean Squared Error: {mse:.4f}") + + return model + + +def predict(model, X): + """ + Keep feature selection's consistency and make predictions. + """ + # Predict using the trained model + y_pred = model.predict(X) + + return y_pred.reshape(-1, 1) diff --git a/rdagent/scenarios/kaggle/experiment/new-york-city-taxi-fare-prediction_template/model/select_lightgbm.py b/rdagent/scenarios/kaggle/experiment/new-york-city-taxi-fare-prediction_template/model/select_lightgbm.py new file mode 100644 index 00000000..f230f130 --- /dev/null +++ b/rdagent/scenarios/kaggle/experiment/new-york-city-taxi-fare-prediction_template/model/select_lightgbm.py @@ -0,0 +1,12 @@ +import pandas as pd + + +def select(X: pd.DataFrame) -> pd.DataFrame: + """ + Select relevant features. To be used in fit & predict function. + """ + # For now, we assume all features are relevant. This can be expanded to feature selection logic. + if X.columns.nlevels == 1: + return X + X.columns = ["_".join(str(i) for i in col).strip() for col in X.columns.values] + return X diff --git a/rdagent/scenarios/kaggle/experiment/new-york-city-taxi-fare-prediction_template/model/select_linear.py b/rdagent/scenarios/kaggle/experiment/new-york-city-taxi-fare-prediction_template/model/select_linear.py new file mode 100644 index 00000000..f230f130 --- /dev/null +++ b/rdagent/scenarios/kaggle/experiment/new-york-city-taxi-fare-prediction_template/model/select_linear.py @@ -0,0 +1,12 @@ +import pandas as pd + + +def select(X: pd.DataFrame) -> pd.DataFrame: + """ + Select relevant features. To be used in fit & predict function. + """ + # For now, we assume all features are relevant. This can be expanded to feature selection logic. + if X.columns.nlevels == 1: + return X + X.columns = ["_".join(str(i) for i in col).strip() for col in X.columns.values] + return X diff --git a/rdagent/scenarios/kaggle/experiment/new-york-city-taxi-fare-prediction_template/model/select_nn.py b/rdagent/scenarios/kaggle/experiment/new-york-city-taxi-fare-prediction_template/model/select_nn.py new file mode 100644 index 00000000..f230f130 --- /dev/null +++ b/rdagent/scenarios/kaggle/experiment/new-york-city-taxi-fare-prediction_template/model/select_nn.py @@ -0,0 +1,12 @@ +import pandas as pd + + +def select(X: pd.DataFrame) -> pd.DataFrame: + """ + Select relevant features. To be used in fit & predict function. + """ + # For now, we assume all features are relevant. This can be expanded to feature selection logic. + if X.columns.nlevels == 1: + return X + X.columns = ["_".join(str(i) for i in col).strip() for col in X.columns.values] + return X diff --git a/rdagent/scenarios/kaggle/experiment/new-york-city-taxi-fare-prediction_template/model/select_randomforest.py b/rdagent/scenarios/kaggle/experiment/new-york-city-taxi-fare-prediction_template/model/select_randomforest.py new file mode 100644 index 00000000..f230f130 --- /dev/null +++ b/rdagent/scenarios/kaggle/experiment/new-york-city-taxi-fare-prediction_template/model/select_randomforest.py @@ -0,0 +1,12 @@ +import pandas as pd + + +def select(X: pd.DataFrame) -> pd.DataFrame: + """ + Select relevant features. To be used in fit & predict function. + """ + # For now, we assume all features are relevant. This can be expanded to feature selection logic. + if X.columns.nlevels == 1: + return X + X.columns = ["_".join(str(i) for i in col).strip() for col in X.columns.values] + return X diff --git a/rdagent/scenarios/kaggle/experiment/new-york-city-taxi-fare-prediction_template/model/select_xgboost.py b/rdagent/scenarios/kaggle/experiment/new-york-city-taxi-fare-prediction_template/model/select_xgboost.py new file mode 100644 index 00000000..f230f130 --- /dev/null +++ b/rdagent/scenarios/kaggle/experiment/new-york-city-taxi-fare-prediction_template/model/select_xgboost.py @@ -0,0 +1,12 @@ +import pandas as pd + + +def select(X: pd.DataFrame) -> pd.DataFrame: + """ + Select relevant features. To be used in fit & predict function. + """ + # For now, we assume all features are relevant. This can be expanded to feature selection logic. + if X.columns.nlevels == 1: + return X + X.columns = ["_".join(str(i) for i in col).strip() for col in X.columns.values] + return X diff --git a/rdagent/scenarios/kaggle/experiment/new-york-city-taxi-fare-prediction_template/train.py b/rdagent/scenarios/kaggle/experiment/new-york-city-taxi-fare-prediction_template/train.py new file mode 100644 index 00000000..a148adf4 --- /dev/null +++ b/rdagent/scenarios/kaggle/experiment/new-york-city-taxi-fare-prediction_template/train.py @@ -0,0 +1,91 @@ +import importlib.util +import random +from pathlib import Path + +import numpy as np +import pandas as pd +from fea_share_preprocess import clean_and_impute_data, preprocess_script +from sklearn.metrics import matthews_corrcoef, root_mean_squared_error + +# Set random seed for reproducibility +SEED = 42 +random.seed(SEED) +np.random.seed(SEED) +DIRNAME = Path(__file__).absolute().resolve().parent + + +def compute_metrics_for_classification(y_true, y_pred): + """Compute MCC for classification.""" + mcc = matthews_corrcoef(y_true, y_pred) + return mcc + + +def import_module_from_path(module_name, module_path): + spec = importlib.util.spec_from_file_location(module_name, module_path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +# 1) Preprocess the data +X_train, X_valid, y_train, y_valid, X_test, ids = preprocess_script() + +# 2) Auto feature engineering +X_train_l, X_valid_l = [], [] +X_test_l = [] + +for f in DIRNAME.glob("feature/feat*.py"): + cls = import_module_from_path(f.stem, f).feature_engineering_cls() + cls.fit(X_train) + X_train_f = cls.transform(X_train) + X_valid_f = cls.transform(X_valid) + X_test_f = cls.transform(X_test) + + if X_train_f.shape[-1] == X_valid_f.shape[-1] and X_train_f.shape[-1] == X_test_f.shape[-1]: + X_train_l.append(X_train_f) + X_valid_l.append(X_valid_f) + X_test_l.append(X_test_f) + +X_train = pd.concat(X_train_l, axis=1, keys=[f"feature_{i}" for i in range(len(X_train_l))]) +X_valid = pd.concat(X_valid_l, axis=1, keys=[f"feature_{i}" for i in range(len(X_valid_l))]) +X_test = pd.concat(X_test_l, axis=1, keys=[f"feature_{i}" for i in range(len(X_test_l))]) + +print(X_train.shape, X_valid.shape, X_test.shape) + +# Handle inf and -inf values +X_train, X_valid, X_test = clean_and_impute_data(X_train, X_valid, X_test) + + +model_l = [] # list[tuple[model, predict_func]] +for f in DIRNAME.glob("model/model*.py"): + select_python_path = f.with_name(f.stem.replace("model", "select") + f.suffix) + select_m = import_module_from_path(select_python_path.stem, select_python_path) + X_train_selected = select_m.select(X_train.copy()) + X_valid_selected = select_m.select(X_valid.copy()) + + m = import_module_from_path(f.stem, f) + model_l.append((m.fit(X_train_selected, y_train, X_valid_selected, y_valid), m.predict, select_m)) + +# 4) Evaluate the model on the validation set +metrics_all = [] +for model, predict_func, select_m in model_l: + X_valid_selected = select_m.select(X_valid.copy()) + y_valid_pred = predict_func(model, X_valid_selected) + rmse = root_mean_squared_error(y_valid, y_valid_pred) + print(f"final root mean squared error on valid set: {rmse}") + metrics_all.append(rmse) + +# 5) Save the validation accuracy +min_index = np.argmin(metrics_all) +pd.Series(data=[metrics_all[min_index]], index=["root mean squared error"]).to_csv("submission_score.csv") + +# 6) Make predictions on the test set and save them +X_test_selected = model_l[min_index][2].select(X_test.copy()) +y_test_pred = model_l[min_index][1](model_l[min_index][0], X_test_selected).flatten() + 1 + + +# 7) Submit predictions for the test set +submission_result = pd.DataFrame(y_test_pred, columns=["fare_amount"]) +submission_result.insert(0, "key", ids) + +submission_result.to_csv("submission.csv", index=False) diff --git a/rdagent/scenarios/kaggle/experiment/tabular-playground-series-dec-2021_template/fea_share_preprocess.py b/rdagent/scenarios/kaggle/experiment/tabular-playground-series-dec-2021_template/fea_share_preprocess.py index 2469a658..c3952fbb 100644 --- a/rdagent/scenarios/kaggle/experiment/tabular-playground-series-dec-2021_template/fea_share_preprocess.py +++ b/rdagent/scenarios/kaggle/experiment/tabular-playground-series-dec-2021_template/fea_share_preprocess.py @@ -6,6 +6,25 @@ from sklearn.impute import SimpleImputer from sklearn.model_selection import train_test_split from sklearn.preprocessing import LabelEncoder +index_col_name = "key" + + +def prepreprocess(): + """ + This method loads the data, drops the unnecessary columns, and splits it into train and validation sets. + """ + # Load and preprocess the data + data_df = pd.read_csv("/kaggle/input/train.csv") + data_df = data_df.drop(["Id"], axis=1) + + X = data_df.drop(["Cover_Type"], axis=1) + y = data_df["Cover_Type"] - 1 + + # Split the data into training and validation sets + X_train, X_valid, y_train, y_valid = train_test_split(X, y, test_size=0.20, random_state=42) + + return X_train, X_valid, y_train, y_valid + def preprocess_script(): """ diff --git a/rdagent/scenarios/kaggle/experiment/workspace.py b/rdagent/scenarios/kaggle/experiment/workspace.py index c8a19172..0625dc06 100644 --- a/rdagent/scenarios/kaggle/experiment/workspace.py +++ b/rdagent/scenarios/kaggle/experiment/workspace.py @@ -26,10 +26,11 @@ pickle.dump(others, open("others.pkl", "wb")) class KGFBWorkspace(FBWorkspace): - def __init__(self, template_folder_path: Path, *args, **kwargs) -> None: + def __init__(self, template_folder_path: Path, *args, entry="python train.py", **kwargs) -> None: super().__init__(*args, **kwargs) self.inject_code_from_folder(template_folder_path) self.data_description: List[Tuple[str, int]] = [] + self.entry = entry # this is for debugging (you may want to change it into `sleep 1000`) @property def model_description(self) -> dict[str, str]: @@ -85,7 +86,7 @@ class KGFBWorkspace(FBWorkspace): execute_log = kgde.run( local_path=str(self.workspace_path), - entry=f"python train.py", + entry=self.entry, env=run_env, running_extra_volume=running_extra_volume, ) diff --git a/rdagent/scenarios/kaggle/kaggle_crawler.py b/rdagent/scenarios/kaggle/kaggle_crawler.py index 569cf53e..f2d35aa9 100644 --- a/rdagent/scenarios/kaggle/kaggle_crawler.py +++ b/rdagent/scenarios/kaggle/kaggle_crawler.py @@ -1,6 +1,7 @@ # %% import bisect import json +import shutil import subprocess import time import zipfile @@ -124,7 +125,18 @@ def download_data(competition: str, local_path: str = KAGGLE_IMPLEMENT_SETTING.l f"/bin/sh -c 'cp -r ./zip_files/{competition}/prepared/private/test.csv ./{competition}/valid.csv'", local_path=local_path, ) - + # NOTE: + # Patching: due to mle has special renaming mechanism for different competition; + # We have to switch the schema back to a uniform one; + if competition in {"new-york-city-taxi-fare-prediction"}: + cpath = Path(local_path) / f"{competition}" + labels_path = cpath / "labels.csv" + train_path = cpath / "train.csv" + if labels_path.exists(): + shutil.copy(labels_path, train_path) + else: + logger.error(f"labels.csv not found in {cpath}") + raise FileNotFoundError(f"{labels_path} does not exist") else: zipfile_path = f"{local_path}/zip_files" if not Path(f"{zipfile_path}/{competition}.zip").exists(): diff --git a/rdagent/utils/env.py b/rdagent/utils/env.py index 47b202d2..8316427c 100644 --- a/rdagent/utils/env.py +++ b/rdagent/utils/env.py @@ -190,6 +190,9 @@ class KGDockerConf(DockerConf): # } running_timeout_period: int = 600 + mem_limit: str | None = ( + "48g" # Add memory limit attribute # new-york-city-taxi-fare-prediction may need more memory + ) class MLEBDockerConf(DockerConf): @@ -206,6 +209,9 @@ class MLEBDockerConf(DockerConf): # # TODO connect to the place where the data is stored # Path("git_ignore_folder/data").resolve(): "/root/.data/" # } + mem_limit: str | None = ( + "48g" # Add memory limit attribute # new-york-city-taxi-fare-prediction may need more memory + ) # physionet.org/files/mimic-eicu-fiddle-feature/1.0.0/FIDDLE_mimic3 @@ -335,6 +341,8 @@ class DockerEnv(Env[DockerConf]): table = Table(title="Run Info", show_header=False) table.add_column("Key", style="bold cyan") table.add_column("Value", style="bold magenta") + table.add_row("Container ID", container.id) + table.add_row("Container Name", container.name) table.add_row("Entry", entry) table.add_row("Env", "\n".join(f"{k}:{v}" for k, v in env.items())) table.add_row("Volumns", "\n".join(f"{k}:{v}" for k, v in volumns.items())) diff --git a/test/utils/test_kaggle.py b/test/utils/test_kaggle.py index 14a8030b..08363814 100644 --- a/test/utils/test_kaggle.py +++ b/test/utils/test_kaggle.py @@ -7,6 +7,7 @@ from rich import print from rdagent.app.kaggle.conf import KAGGLE_IMPLEMENT_SETTING from rdagent.oai.llm_utils import APIBackend from rdagent.scenarios.kaggle.experiment.workspace import KGFBWorkspace +from rdagent.scenarios.kaggle.kaggle_crawler import download_data from rdagent.utils.agent.ret import PythonAgentOut from rdagent.utils.agent.tpl import T @@ -18,6 +19,7 @@ class TestTpl(unittest.TestCase): """ competition = KAGGLE_IMPLEMENT_SETTING.competition print(f"[bold orange]{competition}[/bold orange]") + download_data(competition) ws = KGFBWorkspace( template_folder_path=Path(__file__).parent.parent.parent / "rdagent/scenarios/kaggle/experiment"