diff --git a/rdagent/scenarios/kaggle/docker/Dockerfile b/rdagent/scenarios/kaggle/docker/Dockerfile index 26d3bc3c..50f2aa07 100644 --- a/rdagent/scenarios/kaggle/docker/Dockerfile +++ b/rdagent/scenarios/kaggle/docker/Dockerfile @@ -24,3 +24,5 @@ RUN pip install catboost RUN pip install xgboost RUN pip install sparse RUN pip install lightgbm +RUN pip install pyarrow +RUN pip install fastparquet \ No newline at end of file diff --git a/rdagent/scenarios/kaggle/experiment/meta_tpl/model/model_xgboost.py b/rdagent/scenarios/kaggle/experiment/meta_tpl/model/model_xgboost.py index 64784224..43ecd310 100644 --- a/rdagent/scenarios/kaggle/experiment/meta_tpl/model/model_xgboost.py +++ b/rdagent/scenarios/kaggle/experiment/meta_tpl/model/model_xgboost.py @@ -18,9 +18,10 @@ def fit(X_train: pd.DataFrame, y_train: pd.DataFrame, X_valid: pd.DataFrame, y_v dtrain = xgb.DMatrix(X_train, label=y_train) dvalid = xgb.DMatrix(X_valid, label=y_valid) - # TODO: for quick running.... + # Parameters for regression params = { - "nthred": -1, + "objective": "reg:squarederror", # Use squared error for regression + "nthread": -1, } num_round = 100 diff --git a/rdagent/scenarios/kaggle/experiment/optiver-realized-volatility-prediction_template/fea_share_preprocess.py b/rdagent/scenarios/kaggle/experiment/optiver-realized-volatility-prediction_template/fea_share_preprocess.py index 54c3c8a7..2368dff5 100644 --- a/rdagent/scenarios/kaggle/experiment/optiver-realized-volatility-prediction_template/fea_share_preprocess.py +++ b/rdagent/scenarios/kaggle/experiment/optiver-realized-volatility-prediction_template/fea_share_preprocess.py @@ -11,16 +11,18 @@ from sklearn.preprocessing import OrdinalEncoder def prepreprocess(): # Load the training data - train_df = pd.read_csv("/kaggle/input/optiver-realized-volatility-prediction/train.csv") + train_df = pd.read_csv("/kaggle/input/train.csv").head(1000) # Load book and trade data - book_train = pd.read_parquet("/kaggle/input/optiver-realized-volatility-prediction/book_train.parquet") - trade_train = pd.read_parquet("/kaggle/input/optiver-realized-volatility-prediction/trade_train.parquet") + book_train = pd.read_parquet("/kaggle/input/book_train.parquet").head(1000) + trade_train = pd.read_parquet("/kaggle/input/trade_train.parquet").head(1000) # Merge book and trade data with train_df merged_df = pd.merge(train_df, book_train, on=["stock_id", "time_id"], how="left") merged_df = pd.merge(merged_df, trade_train, on=["stock_id", "time_id"], how="left") + print(merged_df.head()) + # Split the data X = merged_df.drop(["target"], axis=1) y = merged_df["target"] @@ -83,8 +85,19 @@ def preprocess_script(): X_valid = preprocess_transform(X_valid, preprocessor, numerical_cols, categorical_cols) submission_df = pd.read_csv("/kaggle/input/test.csv") - ids = submission_df["id"] - submission_df = submission_df.drop(["id"], axis=1) + + ids = submission_df["row_id"] + submission_df = submission_df.drop(["row_id"], axis=1) + + # Add missing columns to submission_df + for col in X_train.columns: + if col not in submission_df.columns: + submission_df[col] = 0 # Fill with 0 or another appropriate value + X_test = preprocess_transform(submission_df, preprocessor, numerical_cols, categorical_cols) + # Handle missing values + for df in [X_train, X_valid, X_test]: + df.fillna(df.mean(), inplace=True) + return X_train, X_valid, y_train, y_valid, X_test, ids