gavindiaz/NexQuant
1
0
Fork 0
mirror of https://github.com/NicolasBohn/NexQuant.git synced 2026-07-27 23:47:46 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: implement isolated model feature selection loop (#370)

Browse Source 
* rename meta_tpl

* use a isolated coder to deal with model feature selection and refine the structure

* fix CI

* fix: fix some errors in scenario.py, proposal.py and runner.py and several complex competition scenarios(#365)

* fix several bugs in proposal and runner

* fix a bug in feedback-prize-english-language-learning

* fix some bugs and templates

* fix the bug in optiver and nlp problem

* delete unnecessary codes

* remove unnecessary codes

* complete forest and s4e8

* push

* feedback & s4e8 &  forest

* optiver finished

* s3e11 & s3e26

* s4e9 finished

* sf-crime finished

* the last one finished

---------

Co-authored-by: WinstonLiyt <104308117+WinstonLiyt@users.noreply.github.com>
Co-authored-by: WinstonLiyte <1957922024@qq.com>
This commit is contained in:
Xu Yang
2024-09-28 00:40:25 +08:00
committed by GitHub
parent 437e8d2996
commit f7c1c4fd74
72 changed files with 775 additions and 930 deletions
Download Patch File Download Diff File Expand all files Collapse all files
rdagent/app/kaggle/conf.py
+3 -2
View File
@@ -32,6 +32,9 @@ class KaggleBasePropSetting(BasePropSetting):
feature_coder: str = "rdagent.scenarios.kaggle.developer.coder.KGFactorCoSTEER"
"""Feature Coder class"""
model_feature_selection_coder: str = "rdagent.scenarios.kaggle.developer.coder.KGModelFeatureSelectionCoder"
"""Model Feature Selection Coder class"""
model_coder: str = "rdagent.scenarios.kaggle.developer.coder.KGModelCoSTEER"
"""Model Coder class"""
@@ -57,8 +60,6 @@ class KaggleBasePropSetting(BasePropSetting):
if_action_choosing_based_on_UCB: bool = False
if_using_feature_selection: bool = False
if_using_graph_rag: bool = False
if_using_vector_rag: bool = False
rdagent/app/kaggle/loop.py
+7
View File
@@ -23,6 +23,7 @@ from rdagent.scenarios.kaggle.kaggle_crawler import download_data
from rdagent.scenarios.kaggle.proposal.proposal import (
KG_ACTION_FEATURE_ENGINEERING,
KG_ACTION_FEATURE_PROCESSING,
KG_ACTION_MODEL_FEATURE_SELECTION,
KGTrace,
)
@@ -49,6 +50,10 @@ class KaggleRDLoop(RDLoop):
self.feature_coder: Developer = import_class(PROP_SETTING.feature_coder)(scen)
logger.log_object(self.feature_coder, tag="feature coder")
self.model_feature_selection_coder: Developer = import_class(PROP_SETTING.model_feature_selection_coder)(
scen
)
logger.log_object(self.model_feature_selection_coder, tag="model feature selection coder")
self.model_coder: Developer = import_class(PROP_SETTING.model_coder)(scen)
logger.log_object(self.model_coder, tag="model coder")
@@ -67,6 +72,8 @@ class KaggleRDLoop(RDLoop):
with logger.tag("d"): # develop
if prev_out["propose"].action in [KG_ACTION_FEATURE_ENGINEERING, KG_ACTION_FEATURE_PROCESSING]:
exp = self.feature_coder.develop(prev_out["exp_gen"])
elif prev_out["propose"].action == KG_ACTION_MODEL_FEATURE_SELECTION:
exp = self.model_feature_selection_coder.develop(prev_out["exp_gen"])
else:
exp = self.model_coder.develop(prev_out["exp_gen"])
logger.log_object(exp.sub_workspace_list, tag="coder result")
rdagent/components/coder/model_coder/CoSTEER/evolving_strategy.py
+3 -8
View File
@@ -21,6 +21,7 @@ from rdagent.core.evolving_framework import EvolvingStrategy
from rdagent.core.prompts import Prompts
from rdagent.core.utils import multiprocessing_wrapper
from rdagent.oai.llm_utils import APIBackend
from rdagent.scenarios.kaggle.experiment.kaggle_experiment import KG_MODEL_MAPPING
coder_prompts = Prompts(file_path=Path(__file__).parent.parent / "prompts.yaml")
@@ -41,14 +42,8 @@ class ModelCoderEvolvingStrategy(EvolvingStrategy):
current_code = ""
sota_exp_code_dict = current_exp.based_experiments[-1].experiment_workspace.code_dict
if target_task.version == 2:
model_file_mapping = {
"XGBoost": "model/model_xgboost.py",
"RandomForest": "model/model_randomforest.py",
"LightGBM": "model/model_lightgbm.py",
"NN": "model/model_nn.py",
}
if model_type in model_file_mapping:
current_code = sota_exp_code_dict.get(model_file_mapping[model_type], None)
if model_type in KG_MODEL_MAPPING:
current_code = sota_exp_code_dict.get(KG_MODEL_MAPPING[model_type], None)
elif "model.py" in sota_exp_code_dict:
current_code = sota_exp_code_dict["model.py"]
else:
rdagent/components/coder/model_coder/model_execute_template_v2.txt
+1 -1
View File
@@ -4,7 +4,7 @@ import pickle
import numpy as np
import pandas as pd
import torch
from model import fit, predict, select
from model import fit, predict
train_X = pd.DataFrame(np.random.randn(8, 30), columns=[f"{i}" for i in range(30)])
train_y = pd.Series(np.random.randint(0, 2, 8))
rdagent/components/proposal/prompts.yaml
+1 -1
View File
@@ -12,7 +12,7 @@ hypothesis_gen:
{{ hypothesis_output_format }}
user_prompt: |-
{% if hypothesis_and_feedback|length == 0 %} It is the first round of hypothesis generation. The user has no hypothesis on this scenario yet.
{% if hypothesis_and_feedback|length == 0 %}It is the first round of hypothesis generation. The user has no hypothesis on this scenario yet.
{% else %}It is not the first round, the user has made several hypothesis on this scenario and did several evaluation on them.
The former hypothesis and the corresponding feedbacks are as follows (focus on the last one & the new hypothesis that it provides and reasoning to see if you agree):
{{ hypothesis_and_feedback }}
rdagent/scenarios/kaggle/developer/coder.py
+67
View File
@@ -1,5 +1,72 @@
import json
from pathlib import Path
from jinja2 import Environment, StrictUndefined
from rdagent.components.coder.factor_coder.CoSTEER import FactorCoSTEER
from rdagent.components.coder.model_coder.CoSTEER import ModelCoSTEER
from rdagent.core.developer import Developer
from rdagent.core.prompts import Prompts
from rdagent.oai.llm_utils import APIBackend
from rdagent.scenarios.kaggle.experiment.kaggle_experiment import (
KG_SELECT_MAPPING,
KGModelExperiment,
)
KGModelCoSTEER = ModelCoSTEER
KGFactorCoSTEER = FactorCoSTEER
prompt_dict = Prompts(file_path=Path(__file__).parent.parent / "prompts.yaml")
DEFAULT_SELECTION_CODE = """
import pandas as pd
def select(X: pd.DataFrame) -> pd.DataFrame:
\"""
Select relevant features. To be used in fit & predict function.
\"""
if X.columns.nlevels == 1:
return X
{% if feature_index_list is not none %}
X = X.loc[:, X.columns.levels[0][{{feature_index_list}}].tolist()]
{% endif %}
X.columns = ["_".join(str(col)).strip() for col in X.columns.values]
return X
"""
class KGModelFeatureSelectionCoder(Developer[KGModelExperiment]):
def develop(self, exp: KGModelExperiment) -> KGModelExperiment:
target_model_type = exp.sub_tasks[0].model_type
assert target_model_type in KG_SELECT_MAPPING
if len(exp.experiment_workspace.data_description) == 1:
code = (
Environment(undefined=StrictUndefined)
.from_string(DEFAULT_SELECTION_CODE)
.render(feature_index_list=None)
)
else:
system_prompt = (
Environment(undefined=StrictUndefined)
.from_string(prompt_dict["model_feature_selection"]["system"])
.render(scenario=self.scen.get_scenario_all_desc(), model_type=exp.sub_tasks[0].model_type)
)
user_prompt = (
Environment(undefined=StrictUndefined)
.from_string(prompt_dict["model_feature_selection"]["user"])
.render(feature_groups=[desc[0] for desc in exp.experiment_workspace.data_description])
)
chosen_index = json.loads(
APIBackend().build_messages_and_create_chat_completion(
user_prompt=user_prompt, system_prompt=system_prompt, json_mode=True
)
).get("Selected Group Index", [i + 1 for i in range(len(exp.experiment_workspace.data_description))])
chosen_index_to_list_index = [i - 1 for i in chosen_index]
code = (
Environment(undefined=StrictUndefined)
.from_string(DEFAULT_SELECTION_CODE)
.render(feature_index_list=chosen_index_to_list_index)
)
exp.experiment_workspace.inject_code(**{KG_SELECT_MAPPING[target_model_type]: code})
return exp
rdagent/scenarios/kaggle/developer/feedback.py
+5 -4
View File
@@ -117,9 +117,9 @@ class KGHypothesisExperiment2Feedback(HypothesisExperiment2Feedback):
"last_hypothesis": trace.hist[-1][0] if trace.hist else None,
"last_task_and_code": last_task_and_code,
"last_result": trace.hist[-1][1].result if trace.hist else None,
"sota_task_and_code": exp.based_experiments[-1].experiment_workspace.data_description
if exp.based_experiments
else None,
"sota_task_and_code": (
exp.based_experiments[-1].experiment_workspace.data_description if exp.based_experiments else None
),
"sota_result": exp.based_experiments[-1].result if exp.based_experiments else None,
"hypothesis": hypothesis,
"exp": exp,
@@ -150,6 +150,7 @@ class KGHypothesisExperiment2Feedback(HypothesisExperiment2Feedback):
decision = convert2bool(response_json.get("Replace Best Result", "no"))
experiment_feedback = {
"current_competition": self.scen.get_competition_full_desc(),
"hypothesis_text": hypothesis_text,
"current_result": current_result,
"model_code": model_code,
@@ -163,7 +164,7 @@ class KGHypothesisExperiment2Feedback(HypothesisExperiment2Feedback):
self.scen.vector_base.add_experience_to_vector_base(experiment_feedback)
self.scen.vector_base.save()
elif self.scen.if_using_graph_rag:
trace.knowledge_base.load_from_documents([experiment_feedback], self.scen)
trace.knowledge_base.add_document(experiment_feedback, self.scen)
return HypothesisFeedback(
observations=observations,
rdagent/scenarios/kaggle/developer/runner.py
+9 -91
View File
@@ -3,17 +3,12 @@ import pickle
import shutil
from pathlib import Path
from jinja2 import Environment, StrictUndefined
from rdagent.app.kaggle.conf import KAGGLE_IMPLEMENT_SETTING
from rdagent.components.coder.factor_coder.factor import FactorTask
from rdagent.components.coder.model_coder.model import ModelTask
from rdagent.components.runner import CachedRunner
from rdagent.components.runner.conf import RUNNER_SETTINGS
from rdagent.core.exception import CoderError, FactorEmptyError, ModelEmptyError
from rdagent.core.experiment import ASpecificExp
from rdagent.core.prompts import Prompts
from rdagent.oai.llm_utils import APIBackend, md5_hash
from rdagent.oai.llm_utils import md5_hash
from rdagent.scenarios.kaggle.experiment.kaggle_experiment import (
KGFactorExperiment,
KGModelExperiment,
@@ -32,48 +27,6 @@ class KGCachedRunner(CachedRunner[ASpecificExp]):
codes = "\n".join(codes)
return md5_hash(codes)
def extract_model_task_from_code(self, code: str) -> str:
sys_prompt = (
Environment(undefined=StrictUndefined)
.from_string(prompt_dict["extract_model_task_from_code"]["system"])
.render()
)
user_prompt = (
Environment(undefined=StrictUndefined)
.from_string(prompt_dict["extract_model_task_from_code"]["user"])
.render(file_content=code)
)
model_task_description = APIBackend().build_messages_and_create_chat_completion(
user_prompt=user_prompt,
system_prompt=sys_prompt,
json_mode=True,
)
try:
response_json_analysis = json.loads(model_task_description)
task_desc = f"""name: {response_json_analysis['name']}
description: {response_json_analysis['description']}
"""
task_desc += (
f"formulation: {response_json_analysis['formulation']}\n"
if response_json_analysis.get("formulation")
else ""
)
task_desc += f"architecture: {response_json_analysis['architecture']}\n"
task_desc += (
f"variables: {json.dumps(response_json_analysis['variables'], indent=4)}\n"
if response_json_analysis.get("variables")
else ""
)
task_desc += f"hyperparameters: {json.dumps(response_json_analysis['hyperparameters'], indent=4)}\n"
task_desc += f"model_type: {response_json_analysis['model_type']}\n"
except json.JSONDecodeError:
task_desc = "Failed to parse LLM's response as JSON"
return task_desc
def init_develop(self, exp: KGFactorExperiment | KGModelExperiment) -> KGFactorExperiment | KGModelExperiment:
"""
For the initial development, the experiment serves as a benchmark for feature engineering.
@@ -89,39 +42,6 @@ class KGCachedRunner(CachedRunner[ASpecificExp]):
result = exp.experiment_workspace.execute(run_env=env_to_use)
exp.result = result
sub_task = FactorTask(
factor_name="original features", factor_description="here is the original features", factor_formulation=""
)
org_data_path = (
Path(KAGGLE_IMPLEMENT_SETTING.local_data_path) / KAGGLE_IMPLEMENT_SETTING.competition / "X_valid.pkl"
)
with open(org_data_path, "rb") as f:
org_data = pickle.load(f)
feature_shape = org_data.shape[-1]
exp.experiment_workspace.data_description.append((sub_task.get_task_information(), feature_shape))
model_map = {
"XGBoost": "model_xgboost.py",
"RandomForest": "model_randomforest.py",
"LightGBM": "model_lightgbm.py",
"NN": "model_nn.py",
}
workspace_path = exp.experiment_workspace.workspace_path / "model"
for model_name, model_file in model_map.items():
model_file_path = workspace_path / model_file
if model_file_path.exists():
model_description = (
self.extract_model_task_from_code(model_file_path.read_text())
+ f"""code: {model_file_path.read_text()}"""
)
else:
model_description = ""
exp.experiment_workspace.model_description[model_name] = model_description
if RUNNER_SETTINGS.cache_result:
self.dump_cache_result(exp, result)
@@ -135,17 +55,15 @@ class KGModelRunner(KGCachedRunner[KGModelExperiment]):
exp.based_experiments[-1] = self.init_develop(exp.based_experiments[-1])
sub_ws = exp.sub_workspace_list[0]
# TODO: There's a possibility of generating a hybrid model (lightgbm + xgboost), which results in having two items in the model_type list.
model_type = sub_ws.target_task.model_type
if sub_ws is not None:
# TODO: There's a possibility of generating a hybrid model (lightgbm + xgboost), which results in having two items in the model_type list.
model_type = sub_ws.target_task.model_type
if sub_ws.code_dict == {}:
raise ModelEmptyError("No model is implemented.")
else:
model_file_name = f"model/model_{model_type.lower()}.py"
exp.experiment_workspace.inject_code(**{model_file_name: sub_ws.code_dict["model.py"]})
model_description = sub_ws.target_task.get_task_information()
exp.experiment_workspace.model_description[model_type] = model_description
if sub_ws.code_dict == {}:
raise ModelEmptyError("No model is implemented.")
else:
model_file_name = f"model/model_{model_type.lower()}.py"
exp.experiment_workspace.inject_code(**{model_file_name: sub_ws.code_dict["model.py"]})
if RUNNER_SETTINGS.cache_result:
cache_hit, result = self.get_cache_result(exp)
rdagent/scenarios/kaggle/experiment/feedback-prize-english-language-learning_template/fea_share_preprocess.py
-1
View File
@@ -3,7 +3,6 @@ import re
import numpy as np # linear algebra
import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.model_selection import train_test_split
rdagent/scenarios/kaggle/experiment/feedback-prize-english-language-learning_template/model/model_randomforest.py
+2 -16
View File
@@ -3,14 +3,6 @@ import pandas as pd
from sklearn.ensemble import RandomForestRegressor
def select(X: pd.DataFrame) -> pd.DataFrame:
"""
Select relevant features. To be used in fit & predict function.
"""
# For now, we assume all features are relevant. This can be expanded to feature selection logic.
return X
def fit(X_train: pd.DataFrame, y_train: pd.Series, X_valid: pd.DataFrame, y_valid: pd.Series):
"""
Define and train the Random Forest model. Merge feature selection into the pipeline.
@@ -18,11 +10,8 @@ def fit(X_train: pd.DataFrame, y_train: pd.Series, X_valid: pd.DataFrame, y_vali
# Initialize the Random Forest model
model = RandomForestRegressor(n_estimators=100, random_state=32, n_jobs=-1)
# Select features (if any feature selection is needed)
X_train_selected = select(X_train)
# Fit the model
model.fit(X_train_selected, y_train)
model.fit(X_train, y_train)
return model
@@ -31,10 +20,7 @@ def predict(model, X):
"""
Keep feature selection's consistency and make predictions.
"""
# Select features (if any feature selection is needed)
X_selected = select(X)
# Predict using the trained model
y_pred = model.predict(X_selected)
y_pred = model.predict(X)
return y_pred
rdagent/scenarios/kaggle/experiment/feedback-prize-english-language-learning_template/model/model_xgboost.py
-8
View File
@@ -7,11 +7,6 @@ import xgboost as xgb
from sklearn.multioutput import MultiOutputRegressor
def select(X: pd.DataFrame) -> pd.DataFrame:
# Ignore feature selection logic
return X
def is_sparse_df(df: pd.DataFrame) -> bool:
# 检查 DataFrame 中的每一列是否为稀疏类型
return any(isinstance(dtype, pd.SparseDtype) for dtype in df.dtypes)
@@ -19,8 +14,6 @@ def is_sparse_df(df: pd.DataFrame) -> bool:
def fit(X_train: pd.DataFrame, y_train: pd.DataFrame, X_valid: pd.DataFrame, y_valid: pd.DataFrame):
"""Define and train the model. Merge feature_select"""
X_train = select(X_train)
xgb_estimator = xgb.XGBRegressor(
n_estimators=500, random_state=0, objective="reg:squarederror", tree_method="hist", device="cuda"
)
@@ -38,7 +31,6 @@ def predict(model, X_test):
"""
Keep feature select's consistency.
"""
X_test = select(X_test)
if is_sparse_df(X_test):
X_test = X_test.sparse.to_coo()
y_pred = model.predict(X_test)
rdagent/scenarios/kaggle/experiment/feedback-prize-english-language-learning_template/model/select_randomforest.py
+12
View File
@@ -0,0 +1,12 @@
import pandas as pd
def select(X: pd.DataFrame) -> pd.DataFrame:
"""
Select relevant features. To be used in fit & predict function.
"""
# For now, we assume all features are relevant. This can be expanded to feature selection logic.
if X.columns.nlevels == 1:
return X
X.columns = ["_".join(str(col)).strip() for col in X.columns.values]
return X
rdagent/scenarios/kaggle/experiment/feedback-prize-english-language-learning_template/model/select_xgboost.py
+12
View File
@@ -0,0 +1,12 @@
import pandas as pd
def select(X: pd.DataFrame) -> pd.DataFrame:
"""
Select relevant features. To be used in fit & predict function.
"""
# For now, we assume all features are relevant. This can be expanded to feature selection logic.
if X.columns.nlevels == 1:
return X
X.columns = ["_".join(str(col)).strip() for col in X.columns.values]
return X
rdagent/scenarios/kaggle/experiment/feedback-prize-english-language-learning_template/train.py
+19 -26
View File
@@ -34,53 +34,46 @@ for f in DIRNAME.glob("feature/feat*.py"):
X_valid_f = cls.transform(X_valid)
X_test_f = cls.transform(X_test)
X_train_l.append(X_train_f)
X_valid_l.append(X_valid_f)
X_test_l.append(X_test_f)
if X_train_f.shape[-1] == X_valid_f.shape[-1] and X_train_f.shape[-1] == X_test_f.shape[-1]:
X_train_l.append(X_train_f)
X_valid_l.append(X_valid_f)
X_test_l.append(X_test_f)
X_train = pd.concat(X_train_l, axis=1, keys=[f"feature_{i}" for i in range(len(X_train_l))])
X_valid = pd.concat(X_valid_l, axis=1, keys=[f"feature_{i}" for i in range(len(X_valid_l))])
X_test = pd.concat(X_test_l, axis=1, keys=[f"feature_{i}" for i in range(len(X_test_l))])
print(X_train.shape, X_valid.shape, X_test.shape)
# 3) Train the model
def flatten_columns(df: pd.DataFrame) -> pd.DataFrame:
"""
Flatten the columns of a DataFrame with MultiIndex columns,
for (feature_0, a), (feature_0, b) -> feature_0_a, feature_0_b
"""
if df.columns.nlevels == 1:
return df
df.columns = ["_".join(str(col)).strip() for col in df.columns.values]
return df
X_train = flatten_columns(X_train)
X_valid = flatten_columns(X_valid)
X_test = flatten_columns(X_test)
model_l = [] # list[tuple[model, predict_func]]
model_l = [] # list[tuple[model, predict_func,]]
for f in DIRNAME.glob("model/model*.py"):
select_python_path = f.with_name(f.stem.replace("model", "select") + f.suffix)
select_m = import_module_from_path(select_python_path.stem, select_python_path)
X_train_selected = select_m.select(X_train.copy())
X_valid_selected = select_m.select(X_valid.copy())
m = import_module_from_path(f.stem, f)
model_l.append((m.fit(X_train, y_train, X_valid, y_valid), m.predict))
model_l.append((m.fit(X_train_selected, y_train, X_valid_selected, y_valid), m.predict))
# 4) Evaluate the model on the validation set
y_valid_pred_l = []
metrics_all = []
for model, predict_func in model_l:
y_valid_pred = predict_func(model, X_valid)
y_valid_pred_l.append(y_valid_pred)
X_valid_selected = select_m.select(X_valid.copy())
y_valid_pred = predict_func(model, X_valid_selected)
metrics = MCRMSE(y_valid, y_valid_pred)
print(f"MCRMSE on valid set: {metrics}")
metrics_all.append(metrics)
# 5) Save the validation accuracy
min_index = np.argmin(metrics_all)
pd.Series(data=[metrics_all[min_index]], index=["MCRMSE"]).to_csv("submission_score.csv")
y_test_pred = model_l[min_index][1](model_l[min_index][0], X_test)
# 6) Make predictions on the test set and save them
X_test_selected = select_m.select(X_test.copy())
y_test_pred = model_l[min_index][1](model_l[min_index][0], X_test_selected)
# 7) Submit predictions for the test set
submission_result = pd.read_csv("/kaggle/input/sample_submission.csv")
submission_result["cohesion"] = y_test_pred[:, 0]
submission_result["syntax"] = y_test_pred[:, 1]
rdagent/scenarios/kaggle/experiment/forest-cover-type-prediction_template/model/model_catboost.py
-29
View File
@@ -1,29 +0,0 @@
import pandas as pd
from catboost import CatBoostClassifier
def fit(X_train: pd.DataFrame, y_train: pd.Series, X_valid: pd.DataFrame, y_valid: pd.Series):
# Define CatBoost parameters
cat_params = {
"iterations": 5000,
"learning_rate": 0.03,
"od_wait": 1000,
"depth": 7,
"task_type": "GPU",
"l2_leaf_reg": 3,
"eval_metric": "Accuracy",
"devices": "0",
"verbose": 1000,
}
# Initialize and train the CatBoost model
model = CatBoostClassifier(**cat_params)
model.fit(X_train, y_train, eval_set=(X_valid, y_valid))
return model
def predict(model, X: pd.DataFrame):
# Predict using the trained model
y_pred = model.predict(X)
return y_pred.reshape(-1, 1)
rdagent/scenarios/kaggle/experiment/forest-cover-type-prediction_template/model/model_dnn.py → rdagent/scenarios/kaggle/experiment/forest-cover-type-prediction_template/model/model_nn.py
View File
rdagent/scenarios/kaggle/experiment/forest-cover-type-prediction_template/model/model_randomforest.py
+3 -18
View File
@@ -10,14 +10,6 @@ from sklearn.ensemble import RandomForestClassifier
from sklearn.metrics import accuracy_score
def select(X: pd.DataFrame) -> pd.DataFrame:
"""
Select relevant features. To be used in fit & predict function.
"""
# For now, we assume all features are relevant. This can be expanded to feature selection logic.
return X
def fit(X_train: pd.DataFrame, y_train: pd.Series, X_valid: pd.DataFrame, y_valid: pd.Series):
"""
Define and train the Random Forest model. Merge feature selection into the pipeline.
@@ -25,15 +17,11 @@ def fit(X_train: pd.DataFrame, y_train: pd.Series, X_valid: pd.DataFrame, y_vali
# Initialize the Random Forest model
model = RandomForestClassifier(n_estimators=200, random_state=32, n_jobs=-1)
# Select features (if any feature selection is needed)
X_train_selected = select(X_train)
X_valid_selected = select(X_valid)
# Fit the model
model.fit(X_train_selected, y_train)
model.fit(X_train, y_train)
# Validate the model
y_valid_pred = model.predict(X_valid_selected)
y_valid_pred = model.predict(X_valid)
accuracy = accuracy_score(y_valid, y_valid_pred)
print(f"Validation Accuracy: {accuracy:.4f}")
@@ -44,10 +32,7 @@ def predict(model, X):
"""
Keep feature selection's consistency and make predictions.
"""
# Select features (if any feature selection is needed)
X_selected = select(X)
# Predict using the trained model
y_pred = model.predict(X_selected)
y_pred = model.predict(X)
return y_pred.reshape(-1, 1)
rdagent/scenarios/kaggle/experiment/forest-cover-type-prediction_template/model/model_xgboost.py
-8
View File
@@ -6,15 +6,8 @@ import pandas as pd
import xgboost as xgb
def select(X: pd.DataFrame) -> pd.DataFrame:
# Ignore feature selection logic
return X
def fit(X_train: pd.DataFrame, y_train: pd.DataFrame, X_valid: pd.DataFrame, y_valid: pd.DataFrame):
"""Define and train the model. Merge feature_select"""
X_train = select(X_train)
X_valid = select(X_valid)
dtrain = xgb.DMatrix(X_train, label=y_train)
dvalid = xgb.DMatrix(X_valid, label=y_valid)
@@ -37,7 +30,6 @@ def predict(model, X):
"""
Keep feature select's consistency.
"""
X = select(X)
dtest = xgb.DMatrix(X)
y_pred = model.predict(dtest)
return y_pred.astype(int).reshape(-1, 1)
rdagent/scenarios/kaggle/experiment/forest-cover-type-prediction_template/model/select_nn.py
+12
View File
@@ -0,0 +1,12 @@
import pandas as pd
def select(X: pd.DataFrame) -> pd.DataFrame:
"""
Select relevant features. To be used in fit & predict function.
"""
# For now, we assume all features are relevant. This can be expanded to feature selection logic.
if X.columns.nlevels == 1:
return X
X.columns = ["_".join(str(col)).strip() for col in X.columns.values]
return X
rdagent/scenarios/kaggle/experiment/forest-cover-type-prediction_template/model/select_randomforest.py
+12
View File
@@ -0,0 +1,12 @@
import pandas as pd
def select(X: pd.DataFrame) -> pd.DataFrame:
"""
Select relevant features. To be used in fit & predict function.
"""
# For now, we assume all features are relevant. This can be expanded to feature selection logic.
if X.columns.nlevels == 1:
return X
X.columns = ["_".join(str(col)).strip() for col in X.columns.values]
return X
rdagent/scenarios/kaggle/experiment/forest-cover-type-prediction_template/model/select_xgboost.py
+12
View File
@@ -0,0 +1,12 @@
import pandas as pd
def select(X: pd.DataFrame) -> pd.DataFrame:
"""
Select relevant features. To be used in fit & predict function.
"""
# For now, we assume all features are relevant. This can be expanded to feature selection logic.
if X.columns.nlevels == 1:
return X
X.columns = ["_".join(str(col)).strip() for col in X.columns.values]
return X
rdagent/scenarios/kaggle/experiment/forest-cover-type-prediction_template/train.py
+48 -99
View File
@@ -4,11 +4,8 @@ from pathlib import Path
import numpy as np
import pandas as pd
from scipy import stats
from sklearn.impute import SimpleImputer
from sklearn.metrics import accuracy_score
from sklearn.model_selection import KFold
from sklearn.preprocessing import StandardScaler
from fea_share_preprocess import clean_and_impute_data, preprocess_script
from sklearn.metrics import accuracy_score, matthews_corrcoef
# Set random seed for reproducibility
SEED = 42
@@ -17,6 +14,12 @@ np.random.seed(SEED)
DIRNAME = Path(__file__).absolute().resolve().parent
def compute_metrics_for_classification(y_true, y_pred):
"""Compute MCC for classification."""
mcc = matthews_corrcoef(y_true, y_pred)
return mcc
def import_module_from_path(module_name, module_path):
spec = importlib.util.spec_from_file_location(module_name, module_path)
module = importlib.util.module_from_spec(spec)
@@ -25,117 +28,63 @@ def import_module_from_path(module_name, module_path):
# 1) Preprocess the data
data_df = pd.read_csv("/kaggle/input/train.csv")
data_df = data_df.drop(["Id"], axis=1)
X_train, X_valid, y_train, y_valid, X_test, ids = preprocess_script()
X_train = data_df.drop(["Cover_Type"], axis=1)
y_train = data_df["Cover_Type"] - 1
# 2) Auto feature engineering
X_train_l, X_valid_l = [], []
X_test_l = []
submission_df = pd.read_csv("/kaggle/input/test.csv")
ids = submission_df["Id"]
X_test = submission_df.drop(["Id"], axis=1)
# Set up KFold
kf = KFold(n_splits=5, shuffle=True, random_state=SEED)
# Store results
accuracies = []
y_test_pred_l = []
scaler = StandardScaler()
# 3) Train and evaluate using KFold
fold_number = 1
for train_index, valid_index in kf.split(X_train):
print(f"Starting fold {fold_number}...")
X_train_l, X_valid_l, X_test_l = [], [], [] # Reset feature lists for each fold
X_tr, X_val = X_train.iloc[train_index], X_train.iloc[valid_index]
y_tr, y_val = y_train.iloc[train_index], y_train.iloc[valid_index]
X_te = X_test
# Feature engineering
for f in DIRNAME.glob("feature/feat*.py"):
cls = import_module_from_path(f.stem, f).feature_engineering_cls()
cls.fit(X_tr)
X_train_f = cls.transform(X_tr)
X_valid_f = cls.transform(X_val)
X_test_f = cls.transform(X_te)
for f in DIRNAME.glob("feature/feat*.py"):
cls = import_module_from_path(f.stem, f).feature_engineering_cls()
cls.fit(X_train)
X_train_f = cls.transform(X_train)
X_valid_f = cls.transform(X_valid)
X_test_f = cls.transform(X_test)
if X_train_f.shape[-1] == X_valid_f.shape[-1] and X_train_f.shape[-1] == X_test_f.shape[-1]:
X_train_l.append(X_train_f)
X_valid_l.append(X_valid_f)
X_test_l.append(X_test_f)
X_tr = pd.concat(X_train_l, axis=1, keys=[f"feature_{i}" for i in range(len(X_train_l))])
X_val = pd.concat(X_valid_l, axis=1, keys=[f"feature_{i}" for i in range(len(X_valid_l))])
X_te = pd.concat(X_test_l, axis=1, keys=[f"feature_{i}" for i in range(len(X_test_l))])
X_train = pd.concat(X_train_l, axis=1, keys=[f"feature_{i}" for i in range(len(X_train_l))])
X_valid = pd.concat(X_valid_l, axis=1, keys=[f"feature_{i}" for i in range(len(X_valid_l))])
X_test = pd.concat(X_test_l, axis=1, keys=[f"feature_{i}" for i in range(len(X_test_l))])
print("Shape of X_tr: ", X_tr.shape, " Shape of X_val: ", X_val.shape, " Shape of X_te: ", X_te.shape)
print(X_train.shape, X_valid.shape, X_test.shape)
# Replace inf and -inf with NaN
X_tr.replace([np.inf, -np.inf], np.nan, inplace=True)
X_val.replace([np.inf, -np.inf], np.nan, inplace=True)
X_te.replace([np.inf, -np.inf], np.nan, inplace=True)
# Handle inf and -inf values
X_train, X_valid, X_test = clean_and_impute_data(X_train, X_valid, X_test)
# Impute missing values
imputer = SimpleImputer(strategy="mean")
X_tr = pd.DataFrame(imputer.fit_transform(X_tr), columns=X_tr.columns)
X_val = pd.DataFrame(imputer.transform(X_val), columns=X_val.columns)
X_te = pd.DataFrame(imputer.transform(X_te), columns=X_te.columns)
# Standardize the data
X_tr = pd.DataFrame(scaler.fit_transform(X_tr), columns=X_tr.columns)
X_val = pd.DataFrame(scaler.transform(X_val), columns=X_val.columns)
X_te = pd.DataFrame(scaler.transform(X_te), columns=X_te.columns)
model_l = [] # list[tuple[model, predict_func]]
for f in DIRNAME.glob("model/model*.py"):
select_python_path = f.with_name(f.stem.replace("model", "select") + f.suffix)
select_m = import_module_from_path(select_python_path.stem, select_python_path)
X_train_selected = select_m.select(X_train.copy())
X_valid_selected = select_m.select(X_valid.copy())
# Remove duplicate columns
X_tr = X_tr.loc[:, ~X_tr.columns.duplicated()]
X_val = X_val.loc[:, ~X_val.columns.duplicated()]
X_te = X_te.loc[:, ~X_te.columns.duplicated()]
m = import_module_from_path(f.stem, f)
model_l.append((m.fit(X_train_selected, y_train, X_valid_selected, y_valid), m.predict, select_m))
# Train the model
def flatten_columns(df: pd.DataFrame) -> pd.DataFrame:
"""
Flatten the columns of a DataFrame with MultiIndex columns,
for (feature_0, a), (feature_0, b) -> feature_0_a, feature_0_b
"""
if df.columns.nlevels == 1:
return df
df.columns = ["_".join(col).strip() for col in df.columns.values]
return df
# 4) Evaluate the model on the validation set
metrics_all = []
for model, predict_func, select_m in model_l:
X_valid_selected = select_m.select(X_valid.copy())
y_valid_pred = predict_func(model, X_valid_selected)
accuracy = accuracy_score(y_valid, y_valid_pred)
print(f"final accuracy on valid set: {accuracy}")
metrics_all.append(accuracy)
X_tr = flatten_columns(X_tr)
X_val = flatten_columns(X_val)
X_te = flatten_columns(X_te)
# 5) Save the validation accuracy
min_index = np.argmax(metrics_all)
pd.Series(data=[metrics_all[min_index]], index=["multi-class accuracy"]).to_csv("submission_score.csv")
model_l = [] # list[tuple[model, predict_func]]
for f in DIRNAME.glob("model/model*.py"):
m = import_module_from_path(f.stem, f)
model_l.append((m.fit(X_tr, y_tr, X_val, y_val), m.predict))
# 6) Make predictions on the test set and save them
X_test_selected = model_l[min_index][2].select(X_test.copy())
y_test_pred = model_l[min_index][1](model_l[min_index][0], X_test_selected).flatten() + 1
# Evaluate the model on the validation set
y_valid_pred_l = []
for model, predict_func in model_l:
y_valid_pred = predict_func(model, X_val)
y_valid_pred_l.append(y_valid_pred)
y_test_pred_l.append(predict_func(model, X_te))
# Majority vote ensemble
y_valid_pred_ensemble = stats.mode(y_valid_pred_l, axis=0)[0].flatten()
# Compute metrics
accuracy = accuracy_score(y_val, y_valid_pred_ensemble)
accuracies.append(accuracy)
print(f"Fold {fold_number} accuracy: {accuracy}")
fold_number += 1
# Print average accuracy
accuracy = np.mean(accuracies)
print(f"Average accuracy across folds: {accuracy}")
pd.Series(data=[accuracy], index=["multi-class accuracy"]).to_csv("submission_score.csv")
y_test_pred = stats.mode(y_test_pred_l, axis=0)[0].flatten() + 1
# 7) Submit predictions for the test set
submission_result = pd.DataFrame(y_test_pred, columns=["Cover_Type"])
submission_result.insert(0, "Id", ids)
rdagent/scenarios/kaggle/experiment/forest-cover-type-prediction_template/train_past.py
+101 -86
View File
@@ -1,12 +1,13 @@
import importlib.util
import random
from collections import defaultdict
from pathlib import Path
import numpy as np
import pandas as pd
from fea_share_preprocess import clean_and_impute_data, preprocess_script
from scipy import stats
from sklearn.metrics import accuracy_score, matthews_corrcoef
from sklearn.impute import SimpleImputer
from sklearn.metrics import accuracy_score
from sklearn.preprocessing import StandardScaler
# Set random seed for reproducibility
SEED = 42
@@ -15,19 +16,6 @@ np.random.seed(SEED)
DIRNAME = Path(__file__).absolute().resolve().parent
# support various method for metrics calculation
def compute_metrics_for_classification(y_true, y_pred):
"""Compute accuracy metric for classification."""
accuracy = accuracy_score(y_true, y_pred)
return accuracy
def compute_metrics_for_classification(y_true, y_pred):
"""Compute MCC for classification."""
mcc = matthews_corrcoef(y_true, y_pred)
return mcc
def import_module_from_path(module_name, module_path):
spec = importlib.util.spec_from_file_location(module_name, module_path)
module = importlib.util.module_from_spec(spec)
@@ -36,81 +24,108 @@ def import_module_from_path(module_name, module_path):
# 1) Preprocess the data
X_train, X_valid, y_train, y_valid, X_test, ids = preprocess_script()
# 2) Auto feature engineering
X_train_l, X_valid_l = [], []
X_test_l = []
for f in DIRNAME.glob("feature/feat*.py"):
cls = import_module_from_path(f.stem, f).feature_engineering_cls()
cls.fit(X_train)
X_train_f = cls.transform(X_train)
X_valid_f = cls.transform(X_valid)
X_test_f = cls.transform(X_test)
X_train_l.append(X_train_f)
X_valid_l.append(X_valid_f)
X_test_l.append(X_test_f)
X_train = pd.concat(X_train_l, axis=1, keys=[f"feature_{i}" for i in range(len(X_train_l))])
X_valid = pd.concat(X_valid_l, axis=1, keys=[f"feature_{i}" for i in range(len(X_valid_l))])
X_test = pd.concat(X_test_l, axis=1, keys=[f"feature_{i}" for i in range(len(X_test_l))])
print(X_train.shape, X_valid.shape, X_test.shape)
# Handle inf and -inf values
X_train, X_valid, X_test = clean_and_impute_data(X_train, X_valid, X_test)
print("-1")
data_df = pd.read_csv("/kaggle/input/train.csv")
data_df = data_df.drop(["Id"], axis=1)
print("0")
X_train = data_df.drop(["Cover_Type"], axis=1)
y_train = data_df["Cover_Type"] - 1
print("81")
submission_df = pd.read_csv("/kaggle/input/test.csv")
ids = submission_df["Id"]
X_test = submission_df.drop(["Id"], axis=1)
# 3) Train the model
def flatten_columns(df: pd.DataFrame) -> pd.DataFrame:
"""
Flatten the columns of a DataFrame with MultiIndex columns,
for (feature_0, a), (feature_0, b) -> feature_0_a, feature_0_b
"""
if df.columns.nlevels == 1:
return df
df.columns = ["_".join(col).strip() for col in df.columns.values]
return df
X_train = flatten_columns(X_train)
X_valid = flatten_columns(X_valid)
X_test = flatten_columns(X_test)
model_l = [] # list[tuple[model, predict_func]]
for f in DIRNAME.glob("model/model*.py"):
m = import_module_from_path(f.stem, f)
model_l.append((m.fit(X_train, y_train, X_valid, y_valid), m.predict))
# 4) Evaluate the model on the validation set
y_valid_pred_l = []
for model, predict_func in model_l:
y_valid_pred = predict_func(model, X_valid)
y_valid_pred_l.append(y_valid_pred)
print(y_valid_pred)
print(y_valid_pred.shape)
# 5) Ensemble
# Majority vote ensemble
y_valid_pred_ensemble = stats.mode(y_valid_pred_l, axis=0)[0].flatten()
# Compute metrics
accuracy = accuracy_score(y_valid, y_valid_pred_ensemble)
print(f"final accuracy on valid set: {accuracy}")
# 6) Save the validation metrics
pd.Series(data=[accuracy], index=["multi-class accuracy"]).to_csv("submission_score.csv")
# 7) Make predictions on the test set and save them
# Store results
accuracies = []
y_test_pred_l = []
for model, predict_func in model_l:
y_test_pred_l.append(predict_func(model, X_test))
scaler = StandardScaler()
# For multiclass classification, use the mode of the predictions
y_test_pred = stats.mode(y_test_pred_l, axis=0)[0].flatten() + 1
print("12")
# 3) Train and evaluate using KFold
fold_number = 1
model_count = defaultdict(int)
print("123")
for train_index, valid_index in kf.split(X_train):
print(f"Starting fold {fold_number}...")
X_train_l, X_valid_l, X_test_l = [], [], [] # Reset feature lists for each fold
X_tr, X_val = X_train.iloc[train_index], X_train.iloc[valid_index]
y_tr, y_val = y_train.iloc[train_index], y_train.iloc[valid_index]
X_te = X_test
# Feature engineering
for f in DIRNAME.glob("feature/feat*.py"):
cls = import_module_from_path(f.stem, f).feature_engineering_cls()
cls.fit(X_tr)
X_train_f = cls.transform(X_tr)
X_valid_f = cls.transform(X_val)
X_test_f = cls.transform(X_te)
X_train_l.append(X_train_f)
X_valid_l.append(X_valid_f)
X_test_l.append(X_test_f)
X_tr = pd.concat(X_train_l, axis=1, keys=[f"feature_{i}" for i in range(len(X_train_l))])
X_val = pd.concat(X_valid_l, axis=1, keys=[f"feature_{i}" for i in range(len(X_valid_l))])
X_te = pd.concat(X_test_l, axis=1, keys=[f"feature_{i}" for i in range(len(X_test_l))])
print("Shape of X_tr: ", X_tr.shape, " Shape of X_val: ", X_val.shape, " Shape of X_te: ", X_te.shape)
# Replace inf and -inf with NaN
X_tr.replace([np.inf, -np.inf], np.nan, inplace=True)
X_val.replace([np.inf, -np.inf], np.nan, inplace=True)
X_te.replace([np.inf, -np.inf], np.nan, inplace=True)
# Impute missing values
imputer = SimpleImputer(strategy="mean")
X_tr = pd.DataFrame(imputer.fit_transform(X_tr), columns=X_tr.columns)
X_val = pd.DataFrame(imputer.transform(X_val), columns=X_val.columns)
X_te = pd.DataFrame(imputer.transform(X_te), columns=X_te.columns)
# Standardize the data
X_tr = pd.DataFrame(scaler.fit_transform(X_tr), columns=X_tr.columns)
X_val = pd.DataFrame(scaler.transform(X_val), columns=X_val.columns)
X_te = pd.DataFrame(scaler.transform(X_te), columns=X_te.columns)
# Remove duplicate columns
X_tr = X_tr.loc[:, ~X_tr.columns.duplicated()]
X_val = X_val.loc[:, ~X_val.columns.duplicated()]
X_te = X_te.loc[:, ~X_te.columns.duplicated()]
model_l = [] # list[tuple[model, predict_func]]
for f in DIRNAME.glob("model/model*.py"):
select_python_path = f.with_name(f.stem.replace("model", "select") + f.suffix)
select_m = import_module_from_path(select_python_path.stem, select_python_path)
X_train_selected = select_m.select(X_tr.copy())
X_valid_selected = select_m.select(X_val.copy())
m = import_module_from_path(f.stem, f)
model_l.append((m.fit(X_train_selected, y_tr, X_valid_selected, y_val), m.predict))
# 4) Evaluate the models on the validation set and choose the best one
best_accuracy = -1
best = None
for model, predict_func in model_l:
X_valid_selected = select_m.select(X_val.copy())
y_valid_pred = predict_func(model, X_valid_selected)
accuracy = accuracy_score(y_val, y_valid_pred)
print(f"Accuracy on valid set: {accuracy}")
if accuracy > best_accuracy:
best_accuracy = accuracy
best = (model, predict_func)
model_count[best] += 1
fold_number += 1
# 5) Save the validation accuracy
final_model = max(model_count, key=model_count.get)
pd.Series(data=best_accuracy, index=["multi-class accuracy"]).to_csv("submission_score.csv")
# 6) Make predictions on the test set and save them
X_test_selected = select_m.select(X_te.copy())
y_test_pred = final_model[1](final_model[0], X_test_selected).flatten() + 1
submission_result = pd.DataFrame(y_test_pred, columns=["Cover_Type"])
submission_result.insert(0, "Id", ids)
rdagent/scenarios/kaggle/experiment/kaggle_experiment.py
+43 -8
View File
@@ -14,9 +14,28 @@ from rdagent.components.coder.model_coder.model import (
)
from rdagent.scenarios.kaggle.experiment.workspace import KGFBWorkspace
KG_MODEL_TYPE_XGBOOST = "XGBoost"
KG_MODEL_TYPE_RANDOMFOREST = "RandomForest"
KG_MODEL_TYPE_LIGHTGBM = "LightGBM"
KG_MODEL_TYPE_NN = "NN"
KG_MODEL_MAPPING = {
KG_MODEL_TYPE_XGBOOST: "model/model_xgboost.py",
KG_MODEL_TYPE_RANDOMFOREST: "model/model_randomforest.py",
KG_MODEL_TYPE_LIGHTGBM: "model/model_lightgbm.py",
KG_MODEL_TYPE_NN: "model/model_nn.py",
}
KG_SELECT_MAPPING = {
KG_MODEL_TYPE_XGBOOST: "model/select_xgboost.py",
KG_MODEL_TYPE_RANDOMFOREST: "model/select_randomforest.py",
KG_MODEL_TYPE_LIGHTGBM: "model/select_lightgbm.py",
KG_MODEL_TYPE_NN: "model/select_nn.py",
}
class KGModelExperiment(ModelExperiment[ModelTask, KGFBWorkspace, ModelFBWorkspace]):
def __init__(self, *args, **kwargs) -> None:
def __init__(self, *args, source_feature_size: int = None, **kwargs) -> None:
super().__init__(*args, **kwargs)
self.experiment_workspace = KGFBWorkspace(
template_folder_path=Path(__file__).parent / f"{KAGGLE_IMPLEMENT_SETTING.competition}_template"
@@ -26,13 +45,21 @@ class KGModelExperiment(ModelExperiment[ModelTask, KGFBWorkspace, ModelFBWorkspa
self.experiment_workspace.data_description = deepcopy(
self.based_experiments[-1].experiment_workspace.data_description
)
self.experiment_workspace.model_description = deepcopy(
self.based_experiments[-1].experiment_workspace.model_description
)
else:
self.experiment_workspace.data_description = [
(
FactorTask(
factor_name="Original features",
factor_description="The original features",
factor_formulation="",
).get_task_information(),
source_feature_size,
)
]
class KGFactorExperiment(FeatureExperiment[FactorTask, KGFBWorkspace, FactorFBWorkspace]):
def __init__(self, *args, **kwargs) -> None:
def __init__(self, *args, source_feature_size: int = None, **kwargs) -> None:
super().__init__(*args, **kwargs)
self.experiment_workspace = KGFBWorkspace(
template_folder_path=Path(__file__).parent / f"{KAGGLE_IMPLEMENT_SETTING.competition}_template"
@@ -42,6 +69,14 @@ class KGFactorExperiment(FeatureExperiment[FactorTask, KGFBWorkspace, FactorFBWo
self.experiment_workspace.data_description = deepcopy(
self.based_experiments[-1].experiment_workspace.data_description
)
self.experiment_workspace.model_description = deepcopy(
self.based_experiments[-1].experiment_workspace.model_description
)
else:
self.experiment_workspace.data_description = [
(
FactorTask(
factor_name="Original features",
factor_description="The original features",
factor_formulation="",
).get_task_information(),
source_feature_size,
)
]
rdagent/scenarios/kaggle/experiment/meta_tpl/fea_share_preprocess.py → rdagent/scenarios/kaggle/experiment/meta_tpl_deprecated/fea_share_preprocess.py
View File
rdagent/scenarios/kaggle/experiment/meta_tpl/feature/feature.py → rdagent/scenarios/kaggle/experiment/meta_tpl_deprecated/feature/feature.py
View File
rdagent/scenarios/kaggle/experiment/meta_tpl/model/model_nn.py → rdagent/scenarios/kaggle/experiment/meta_tpl_deprecated/model/model_nn.py
View File
rdagent/scenarios/kaggle/experiment/meta_tpl/model/model_randomforest.py → rdagent/scenarios/kaggle/experiment/meta_tpl_deprecated/model/model_randomforest.py
View File
rdagent/scenarios/kaggle/experiment/meta_tpl/model/model_xgboost.py → rdagent/scenarios/kaggle/experiment/meta_tpl_deprecated/model/model_xgboost.py
View File
rdagent/scenarios/kaggle/experiment/meta_tpl/train.py → rdagent/scenarios/kaggle/experiment/meta_tpl_deprecated/train.py
View File
rdagent/scenarios/kaggle/experiment/optiver-realized-volatility-prediction_template/model/model_randomforest.py
+3 -18
View File
@@ -4,14 +4,6 @@ from sklearn.ensemble import RandomForestRegressor
from sklearn.metrics import mean_squared_error
def select(X: pd.DataFrame) -> pd.DataFrame:
"""
Select relevant features. To be used in fit & predict function.
"""
# For now, we assume all features are relevant. This can be expanded to feature selection logic.
return X
def fit(X_train: pd.DataFrame, y_train: pd.Series, X_valid: pd.DataFrame, y_valid: pd.Series):
"""
Define and train the Random Forest model. Merge feature selection into the pipeline.
@@ -19,15 +11,11 @@ def fit(X_train: pd.DataFrame, y_train: pd.Series, X_valid: pd.DataFrame, y_vali
# Initialize the Random Forest model
model = RandomForestRegressor(n_estimators=100, random_state=32, n_jobs=-1)
# Select features (if any feature selection is needed)
X_train_selected = select(X_train)
X_valid_selected = select(X_valid)
# Fit the model
model.fit(X_train_selected, y_train)
model.fit(X_train, y_train)
# Validate the model
y_valid_pred = model.predict(X_valid_selected)
y_valid_pred = model.predict(X_valid)
mse = mean_squared_error(y_valid, y_valid_pred)
rmse = np.sqrt(mse)
print(f"Validation RMSE: {rmse:.4f}")
@@ -39,10 +27,7 @@ def predict(model, X):
"""
Keep feature selection's consistency and make predictions.
"""
# Select features (if any feature selection is needed)
X_selected = select(X)
# Predict using the trained model
y_pred = model.predict(X_selected)
y_pred = model.predict(X)
return y_pred.reshape(-1, 1)
rdagent/scenarios/kaggle/experiment/optiver-realized-volatility-prediction_template/model/model_xgboost.py
-8
View File
@@ -2,15 +2,8 @@ import pandas as pd
import xgboost as xgb
def select(X: pd.DataFrame) -> pd.DataFrame:
# Ignore feature selection logic
return X
def fit(X_train: pd.DataFrame, y_train: pd.DataFrame, X_valid: pd.DataFrame, y_valid: pd.DataFrame):
"""Define and train the model. Merge feature_select"""
X_train = select(X_train)
X_valid = select(X_valid)
dtrain = xgb.DMatrix(X_train, label=y_train)
dvalid = xgb.DMatrix(X_valid, label=y_valid)
@@ -33,7 +26,6 @@ def predict(model, X):
"""
Keep feature select's consistency.
"""
X = select(X)
dtest = xgb.DMatrix(X)
y_pred = model.predict(dtest)
return y_pred.reshape(-1, 1)
rdagent/scenarios/kaggle/experiment/optiver-realized-volatility-prediction_template/model/select_randomforest.py
+12
View File
@@ -0,0 +1,12 @@
import pandas as pd
def select(X: pd.DataFrame) -> pd.DataFrame:
"""
Select relevant features. To be used in fit & predict function.
"""
# For now, we assume all features are relevant. This can be expanded to feature selection logic.
if X.columns.nlevels == 1:
return X
X.columns = ["_".join(str(col)).strip() for col in X.columns.values]
return X
rdagent/scenarios/kaggle/experiment/optiver-realized-volatility-prediction_template/model/select_xgboost.py
+12
View File
@@ -0,0 +1,12 @@
import pandas as pd
def select(X: pd.DataFrame) -> pd.DataFrame:
"""
Select relevant features. To be used in fit & predict function.
"""
# For now, we assume all features are relevant. This can be expanded to feature selection logic.
if X.columns.nlevels == 1:
return X
X.columns = ["_".join(str(col)).strip() for col in X.columns.values]
return X
rdagent/scenarios/kaggle/experiment/optiver-realized-volatility-prediction_template/train.py
+18 -27
View File
@@ -42,9 +42,10 @@ for f in DIRNAME.glob("feature/feat*.py"):
X_valid_f = cls.transform(X_valid)
X_test_f = cls.transform(X_test)
X_train_l.append(X_train_f)
X_valid_l.append(X_valid_f)
X_test_l.append(X_test_f)
if X_train_f.shape[-1] == X_valid_f.shape[-1] and X_train_f.shape[-1] == X_test_f.shape[-1]:
X_train_l.append(X_train_f)
X_valid_l.append(X_valid_f)
X_test_l.append(X_test_f)
X_train = pd.concat(X_train_l, axis=1, keys=[f"feature_{i}" for i in range(len(X_train_l))])
X_valid = pd.concat(X_valid_l, axis=1, keys=[f"feature_{i}" for i in range(len(X_valid_l))])
@@ -70,42 +71,32 @@ X_test = X_test.loc[:, ~X_test.columns.duplicated()]
# 3) Train the model
def flatten_columns(df: pd.DataFrame) -> pd.DataFrame:
"""
Flatten the columns of a DataFrame with MultiIndex columns,
for (feature_0, a), (feature_0, b) -> feature_0_a, feature_0_b
"""
if df.columns.nlevels == 1:
return df
df.columns = ["_".join(col).strip() for col in df.columns.values]
return df
X_train = flatten_columns(X_train)
X_valid = flatten_columns(X_valid)
X_test = flatten_columns(X_test)
model_l = [] # list[tuple[model, predict_func,]]
for f in DIRNAME.glob("model/model*.py"):
select_python_path = f.with_name(f.stem.replace("model", "select") + f.suffix)
select_m = import_module_from_path(select_python_path.stem, select_python_path)
X_train_selected = select_m.select(X_train.copy())
X_valid_selected = select_m.select(X_valid.copy())
m = import_module_from_path(f.stem, f)
model_l.append((m.fit(X_train, y_train, X_valid, y_valid), m.predict))
model_l.append((m.fit(X_train_selected, y_train, X_valid_selected, y_valid), m.predict, select_m))
# 4) Evaluate the model on the validation set
y_valid_pred_l = []
metrics_all = []
for model, predict_func in model_l:
y_valid_pred_l.append(predict_func(model, X_valid))
metrics = compute_rmspe(y_valid, y_valid_pred_l[-1].ravel())
for model, predict_func, select_m in model_l:
X_valid_selected = select_m.select(X_valid.copy())
y_valid_pred = predict_func(model, X_valid_selected)
metrics = compute_rmspe(y_valid, y_valid_pred.ravel())
print(f"RMSPE on valid set: {metrics}")
metrics_all.append(metrics)
# 5) Save the validation accuracy
min_index = np.argmin(metrics_all)
pd.Series(data=[metrics_all[min_index]], index=["RMSPE"]).to_csv("submission_score.csv")
y_test_pred = model_l[min_index][1](model_l[min_index][0], X_test).ravel()
# 6) Make predictions on the test set and save them
X_test_selected = model_l[min_index][2].select(X_test.copy())
y_test_pred = model_l[min_index][1](model_l[min_index][0], X_test_selected).ravel()
# 5) Submit predictions for the test set
submission_result = pd.DataFrame({"row_id": ids, "target": y_test_pred})
submission_result.to_csv("submission.csv", index=False)
rdagent/scenarios/kaggle/experiment/playground-series-s3e11_template/model/model_randomforest.py
-8
View File
@@ -2,15 +2,8 @@ import pandas as pd
from sklearn.ensemble import RandomForestRegressor
def select(X: pd.DataFrame) -> pd.DataFrame:
# Ignore feature selection logic
return X
def fit(X_train: pd.DataFrame, y_train: pd.DataFrame, X_valid: pd.DataFrame, y_valid: pd.DataFrame):
"""Define and train the Random Forest model. Merge feature_select"""
X_train = select(X_train)
rf_params = {
"n_estimators": 100,
"max_depth": 10,
@@ -30,6 +23,5 @@ def predict(model, X_test):
"""
Keep feature select's consistency.
"""
X_test = select(X_test)
y_pred = model.predict(X_test)
return y_pred.reshape(-1, 1)
rdagent/scenarios/kaggle/experiment/playground-series-s3e11_template/model/model_xgboost.py
-8
View File
@@ -6,15 +6,8 @@ import pandas as pd
import xgboost as xgb
def select(X: pd.DataFrame) -> pd.DataFrame:
# Ignore feature selection logic
return X
def fit(X_train: pd.DataFrame, y_train: pd.DataFrame, X_valid: pd.DataFrame, y_valid: pd.DataFrame):
"""Define and train the model. Merge feature_select"""
X_train = select(X_train)
xgb_params = {
"n_estimators": 280,
"learning_rate": 0.05,
@@ -37,6 +30,5 @@ def predict(model, X_test):
"""
Keep feature select's consistency.
"""
X_test = select(X_test)
y_pred = model.predict(X_test)
return y_pred.reshape(-1, 1)
rdagent/scenarios/kaggle/experiment/playground-series-s3e11_template/model/select_randomforest.py
+12
View File
@@ -0,0 +1,12 @@
import pandas as pd
def select(X: pd.DataFrame) -> pd.DataFrame:
"""
Select relevant features. To be used in fit & predict function.
"""
# For now, we assume all features are relevant. This can be expanded to feature selection logic.
if X.columns.nlevels == 1:
return X
X.columns = ["_".join(str(col)).strip() for col in X.columns.values]
return X
rdagent/scenarios/kaggle/experiment/playground-series-s3e11_template/model/select_xgboost.py
+12
View File
@@ -0,0 +1,12 @@
import pandas as pd
def select(X: pd.DataFrame) -> pd.DataFrame:
"""
Select relevant features. To be used in fit & predict function.
"""
# For now, we assume all features are relevant. This can be expanded to feature selection logic.
if X.columns.nlevels == 1:
return X
X.columns = ["_".join(str(col)).strip() for col in X.columns.values]
return X
rdagent/scenarios/kaggle/experiment/playground-series-s3e11_template/train.py
+18 -24
View File
@@ -30,9 +30,10 @@ for f in DIRNAME.glob("feature/feat*.py"):
X_valid_f = cls.transform(X_valid)
X_test_f = cls.transform(X_test)
X_train_l.append(X_train_f)
X_valid_l.append(X_valid_f)
X_test_l.append(X_test_f)
if X_train_f.shape[-1] == X_valid_f.shape[-1] and X_train_f.shape[-1] == X_test_f.shape[-1]:
X_train_l.append(X_train_f)
X_valid_l.append(X_valid_f)
X_test_l.append(X_test_f)
X_train = pd.concat(X_train_l, axis=1, keys=[f"feature_{i}" for i in range(len(X_train_l))])
X_valid = pd.concat(X_valid_l, axis=1, keys=[f"feature_{i}" for i in range(len(X_valid_l))])
@@ -40,41 +41,34 @@ X_test = pd.concat(X_test_l, axis=1, keys=[f"feature_{i}" for i in range(len(X_t
# 3) Train the model
def flatten_columns(df: pd.DataFrame) -> pd.DataFrame:
"""
Flatten the columns of a DataFrame with MultiIndex columns,
for (feature_0, a), (feature_0, b) -> feature_0_a, feature_0_b
"""
if df.columns.nlevels == 1:
return df
df.columns = ["_".join(col).strip() for col in df.columns.values]
return df
X_train = flatten_columns(X_train)
X_valid = flatten_columns(X_valid)
X_test = flatten_columns(X_test)
model_l = [] # list[tuple[model, predict_func]]
for f in DIRNAME.glob("model/model*.py"):
select_python_path = f.with_name(f.stem.replace("model", "select") + f.suffix)
select_m = import_module_from_path(select_python_path.stem, select_python_path)
X_train_selected = select_m.select(X_train.copy())
X_valid_selected = select_m.select(X_valid.copy())
m = import_module_from_path(f.stem, f)
model_l.append((m.fit(X_train, y_train, X_valid, y_valid), m.predict))
model_l.append((m.fit(X_train_selected, y_train, X_valid_selected, y_valid), m.predict, select_m))
# 4) Evaluate the model on the validation set
y_valid_pred_l = []
metrics_all = []
for model, predict_func in model_l:
y_valid_pred = predict_func(model, X_valid)
y_valid_pred_l.append(y_valid_pred)
for model, predict_func, select_m in model_l:
X_valid_selected = select_m.select(X_valid.copy())
y_valid_pred = predict_func(model, X_valid_selected)
metrics = mean_squared_error(y_valid, y_valid_pred, squared=False)
print(f"RMLSE on valid set: {metrics}")
metrics_all.append(metrics)
# 5) Save the validation accuracy
min_index = np.argmin(metrics_all)
pd.Series(data=[metrics_all[min_index]], index=["RMLSE"]).to_csv("submission_score.csv")
y_test_pred = model_l[min_index][1](model_l[min_index][0], X_test)
# 6) Make predictions on the test set and save them
X_test_selected = model_l[min_index][2].select(X_test.copy())
y_test_pred = model_l[min_index][1](model_l[min_index][0], X_test_selected)
# 7) Submit predictions for the test set
submission_result = pd.DataFrame(np.expm1(y_test_pred), columns=["cost"])
submission_result.insert(0, "id", ids)
rdagent/scenarios/kaggle/experiment/playground-series-s3e26_template/model/model_randomforest.py
+2 -22
View File
@@ -10,14 +10,6 @@ from sklearn.ensemble import RandomForestClassifier
from sklearn.metrics import accuracy_score
def select(X: pd.DataFrame) -> pd.DataFrame:
"""
Select relevant features. To be used in fit & predict function.
"""
# For now, we assume all features are relevant. This can be expanded to feature selection logic.
return X
def fit(X_train: pd.DataFrame, y_train: pd.Series, X_valid: pd.DataFrame, y_valid: pd.Series):
"""
Define and train the Random Forest model. Merge feature selection into the pipeline.
@@ -25,17 +17,8 @@ def fit(X_train: pd.DataFrame, y_train: pd.Series, X_valid: pd.DataFrame, y_vali
# Initialize the Random Forest model
model = RandomForestClassifier(n_estimators=100, random_state=32, n_jobs=-1)
# Select features (if any feature selection is needed)
X_train_selected = select(X_train)
X_valid_selected = select(X_valid)
# Fit the model
model.fit(X_train_selected, y_train)
# Validate the model
y_valid_pred = model.predict(X_valid_selected)
# accuracy = accuracy_score(y_valid, y_valid_pred)
# print(f"Validation Accuracy: {accuracy:.4f}")
model.fit(X_train, y_train)
return model
@@ -44,11 +27,8 @@ def predict(model, X):
"""
Keep feature selection's consistency and make predictions.
"""
# Select features (if any feature selection is needed)
X_selected = select(X)
# Predict using the trained model
y_pred_prob = model.predict_proba(X_selected)
y_pred_prob = model.predict_proba(X)
# Apply threshold to get boolean predictions
return y_pred_prob
rdagent/scenarios/kaggle/experiment/playground-series-s3e26_template/model/model_xgboost.py
-8
View File
@@ -7,15 +7,8 @@ import pandas as pd
import xgboost as xgb
def select(X: pd.DataFrame) -> pd.DataFrame:
# Ignore feature selection logic
return X
def fit(X_train: pd.DataFrame, y_train: pd.DataFrame, X_valid: pd.DataFrame, y_valid: pd.DataFrame):
"""Define and train the model. Merge feature_select"""
X_train = select(X_train)
X_valid = select(X_valid)
dtrain = xgb.DMatrix(X_train, label=y_train)
dvalid = xgb.DMatrix(X_valid, label=y_valid)
num_classes = len(np.unique(y_train))
@@ -40,7 +33,6 @@ def predict(model, X):
"""
Keep feature select's consistency.
"""
X = select(X)
dtest = xgb.DMatrix(X)
y_pred_prob = model.predict(dtest)
return y_pred_prob
rdagent/scenarios/kaggle/experiment/playground-series-s3e26_template/model/select_randomforest.py
+12
View File
@@ -0,0 +1,12 @@
import pandas as pd
def select(X: pd.DataFrame) -> pd.DataFrame:
"""
Select relevant features. To be used in fit & predict function.
"""
# For now, we assume all features are relevant. This can be expanded to feature selection logic.
if X.columns.nlevels == 1:
return X
X.columns = ["_".join(str(col)).strip() for col in X.columns.values]
return X
rdagent/scenarios/kaggle/experiment/playground-series-s3e26_template/model/select_xgboost.py
+12
View File
@@ -0,0 +1,12 @@
import pandas as pd
def select(X: pd.DataFrame) -> pd.DataFrame:
"""
Select relevant features. To be used in fit & predict function.
"""
# For now, we assume all features are relevant. This can be expanded to feature selection logic.
if X.columns.nlevels == 1:
return X
X.columns = ["_".join(str(col)).strip() for col in X.columns.values]
return X
rdagent/scenarios/kaggle/experiment/playground-series-s3e26_template/train.py
+22 -45
View File
@@ -53,6 +53,7 @@ X_train = pd.concat(X_train_l, axis=1)
X_valid = pd.concat(X_valid_l, axis=1)
X_test = pd.concat(X_test_l, axis=1)
print(X_train.shape, X_valid.shape, X_test.shape)
# Handle inf and -inf values
X_train.replace([np.inf, -np.inf], np.nan, inplace=True)
@@ -72,62 +73,38 @@ X_train = X_train.loc[:, ~X_train.columns.duplicated()]
X_valid = X_valid.loc[:, ~X_valid.columns.duplicated()]
X_test = X_test.loc[:, ~X_test.columns.duplicated()]
print(X_train.shape, X_valid.shape, X_test.shape)
# 3) Train the model
def flatten_columns(df: pd.DataFrame) -> pd.DataFrame:
"""
Flatten the columns of a DataFrame with MultiIndex columns,
for (feature_0, a), (feature_0, b) -> feature_0_a, feature_0_b
"""
if df.columns.nlevels == 1:
return df
df.columns = ["_".join(col).strip() for col in df.columns.values]
return df
X_train = flatten_columns(X_train)
X_valid = flatten_columns(X_valid)
X_test = flatten_columns(X_test)
model_l = [] # list[tuple[model, predict_func]]
model_l = [] # list[tuple[model, predict_func,]]
for f in DIRNAME.glob("model/model*.py"):
select_python_path = f.with_name(f.stem.replace("model", "select") + f.suffix)
select_m = import_module_from_path(select_python_path.stem, select_python_path)
X_train_selected = select_m.select(X_train.copy())
X_valid_selected = select_m.select(X_valid.copy())
m = import_module_from_path(f.stem, f)
model_l.append((m.fit(X_train, y_train, X_valid, y_valid), m.predict))
model_l.append((m.fit(X_train_selected, y_train, X_valid_selected, y_valid), m.predict, select_m))
# 4) Evaluate the model on the validation set
y_valid_pred_l = []
for model, predict_func in model_l:
y_valid_pred_l.append(predict_func(model, X_valid))
print(predict_func(model, X_valid))
print(predict_func(model, X_valid).shape)
metrics_all = []
for model, predict_func, select_m in model_l:
X_valid_selected = select_m.select(X_valid.copy())
y_valid_pred = predict_func(model, X_valid_selected)
logloss = compute_metrics_for_classification(y_valid, y_valid_pred)
print(f"log_loss on valid set: {logloss}")
metrics_all.append(logloss)
# 5) Ensemble
from scipy import stats
# 5) Save the validation accuracy
min_index = np.argmin(metrics_all)
pd.Series(data=[metrics_all[min_index]], index=["log_loss"]).to_csv("submission_score.csv")
# average probabilities ensemble
y_valid_pred_proba = np.mean(y_valid_pred_l, axis=0)
# Compute metrics
logloss = compute_metrics_for_classification(y_valid, y_valid_pred_proba)
print(f"final log_loss on valid set: {logloss}")
# 6) Save the validation metrics
pd.Series(data=[logloss], index=["log_loss"]).to_csv("submission_score.csv")
# 7) Make predictions on the test set and save them
y_test_pred_l = []
for model, predict_func in model_l:
y_test_pred_l.append(predict_func(model, X_test))
# For multiclass classification, use the mode of the predictions
y_test_pred_proba = np.mean(y_test_pred_l, axis=0)
# 6) Make predictions on the test set and save them
X_test_selected = model_l[min_index][2].select(X_test.copy())
y_test_pred = model_l[min_index][1](model_l[min_index][0], X_test_selected)
class_labels = ["Status_" + label for label in status_encoder.classes_]
# modified_labels = ["Status_" + label for label in class_labels]
submission_result = pd.DataFrame(y_test_pred_proba, columns=class_labels)
submission_result = pd.DataFrame(y_test_pred, columns=class_labels)
submission_result.insert(0, "id", test_ids)
submission_result.to_csv("submission.csv", index=False)
rdagent/scenarios/kaggle/experiment/playground-series-s4e8_template/model/model_randomforest.py
+3 -18
View File
@@ -10,14 +10,6 @@ from sklearn.ensemble import RandomForestClassifier
from sklearn.metrics import accuracy_score
def select(X: pd.DataFrame) -> pd.DataFrame:
"""
Select relevant features. To be used in fit & predict function.
"""
# For now, we assume all features are relevant. This can be expanded to feature selection logic.
return X
def fit(X_train: pd.DataFrame, y_train: pd.Series, X_valid: pd.DataFrame, y_valid: pd.Series):
"""
Define and train the Random Forest model. Merge feature selection into the pipeline.
@@ -25,15 +17,11 @@ def fit(X_train: pd.DataFrame, y_train: pd.Series, X_valid: pd.DataFrame, y_vali
# Initialize the Random Forest model
model = RandomForestClassifier(n_estimators=100, random_state=32, n_jobs=-1)
# Select features (if any feature selection is needed)
X_train_selected = select(X_train)
X_valid_selected = select(X_valid)
# Fit the model
model.fit(X_train_selected, y_train)
model.fit(X_train, y_train)
# Validate the model
y_valid_pred = model.predict(X_valid_selected)
y_valid_pred = model.predict(X_valid)
accuracy = accuracy_score(y_valid, y_valid_pred)
print(f"Validation Accuracy: {accuracy:.4f}")
@@ -44,11 +32,8 @@ def predict(model, X):
"""
Keep feature selection's consistency and make predictions.
"""
# Select features (if any feature selection is needed)
X_selected = select(X)
# Predict using the trained model
y_pred_prob = model.predict_proba(X_selected)[:, 1]
y_pred_prob = model.predict_proba(X)[:, 1]
# Apply threshold to get boolean predictions
return y_pred_prob.reshape(-1, 1)
rdagent/scenarios/kaggle/experiment/playground-series-s4e8_template/model/model_xgboost.py
-9
View File
@@ -6,19 +6,11 @@ import pandas as pd
import xgboost as xgb
def select(X: pd.DataFrame) -> pd.DataFrame:
# Ignore feature selection logic
return X
def fit(X_train: pd.DataFrame, y_train: pd.DataFrame, X_valid: pd.DataFrame, y_valid: pd.DataFrame):
"""Define and train the model. Merge feature_select"""
X_train = select(X_train)
X_valid = select(X_valid)
dtrain = xgb.DMatrix(X_train, label=y_train)
dvalid = xgb.DMatrix(X_valid, label=y_valid)
# TODO: for quick running....
params = {
"nthread": -1,
"tree_method": "gpu_hist",
@@ -36,7 +28,6 @@ def predict(model, X):
"""
Keep feature select's consistency.
"""
X = select(X)
dtest = xgb.DMatrix(X)
y_pred_prob = model.predict(dtest)
return y_pred_prob.reshape(-1, 1)
rdagent/scenarios/kaggle/experiment/playground-series-s4e8_template/model/select_randomforest.py
+12
View File
@@ -0,0 +1,12 @@
import pandas as pd
def select(X: pd.DataFrame) -> pd.DataFrame:
"""
Select relevant features. To be used in fit & predict function.
"""
# For now, we assume all features are relevant. This can be expanded to feature selection logic.
if X.columns.nlevels == 1:
return X
X.columns = ["_".join(str(col)).strip() for col in X.columns.values]
return X
rdagent/scenarios/kaggle/experiment/playground-series-s4e8_template/model/select_xgboost.py
+12
View File
@@ -0,0 +1,12 @@
import pandas as pd
def select(X: pd.DataFrame) -> pd.DataFrame:
"""
Select relevant features. To be used in fit & predict function.
"""
# For now, we assume all features are relevant. This can be expanded to feature selection logic.
if X.columns.nlevels == 1:
return X
X.columns = ["_".join(str(col)).strip() for col in X.columns.values]
return X
rdagent/scenarios/kaggle/experiment/playground-series-s4e8_template/train.py
+25 -42
View File
@@ -6,7 +6,6 @@ import numpy as np
import pandas as pd
from fea_share_preprocess import preprocess_script
from sklearn.metrics import matthews_corrcoef
from sklearn.preprocessing import LabelEncoder
# Set random seed for reproducibility
SEED = 42
@@ -43,9 +42,10 @@ for f in DIRNAME.glob("feature/feat*.py"):
X_valid_f = cls.transform(X_valid)
X_test_f = cls.transform(X_test)
X_train_l.append(X_train_f)
X_valid_l.append(X_valid_f)
X_test_l.append(X_test_f)
if X_train_f.shape[-1] == X_valid_f.shape[-1] and X_train_f.shape[-1] == X_test_f.shape[-1]:
X_train_l.append(X_train_f)
X_valid_l.append(X_valid_f)
X_test_l.append(X_test_f)
X_train = pd.concat(X_train_l, axis=1, keys=[f"feature_{i}" for i in range(len(X_train_l))])
X_valid = pd.concat(X_valid_l, axis=1, keys=[f"feature_{i}" for i in range(len(X_valid_l))])
@@ -71,55 +71,38 @@ X_train = X_train.loc[:, ~X_train.columns.duplicated()]
X_valid = X_valid.loc[:, ~X_valid.columns.duplicated()]
X_test = X_test.loc[:, ~X_test.columns.duplicated()]
# 3) Train the model
def flatten_columns(df: pd.DataFrame) -> pd.DataFrame:
"""
Flatten the columns of a DataFrame with MultiIndex columns,
for (feature_0, a), (feature_0, b) -> feature_0_a, feature_0_b
"""
if df.columns.nlevels == 1:
return df
df.columns = ["_".join(col).strip() for col in df.columns.values]
return df
X_train = flatten_columns(X_train)
X_valid = flatten_columns(X_valid)
X_test = flatten_columns(X_test)
model_l = [] # list[tuple[model, predict_func,]]
for f in DIRNAME.glob("model/model*.py"):
select_python_path = f.with_name(f.stem.replace("model", "select") + f.suffix)
select_m = import_module_from_path(select_python_path.stem, select_python_path)
X_train_selected = select_m.select(X_train.copy())
X_valid_selected = select_m.select(X_valid.copy())
m = import_module_from_path(f.stem, f)
model_l.append((m.fit(X_train, y_train, X_valid, y_valid), m.predict))
model_l.append((m.fit(X_train_selected, y_train, X_valid_selected, y_valid), m.predict, select_m))
# 4) Evaluate the model on the validation set
y_valid_pred_l = []
for model, predict_func in model_l:
y_valid_pred_l.append(predict_func(model, X_valid))
metrics_all = []
for model, predict_func, select_m in model_l:
X_valid_selected = select_m.select(X_valid.copy())
y_valid_pred = predict_func(model, X_valid_selected)
y_valid_pred = (y_valid_pred > 0.5).astype(int)
metrics = compute_metrics_for_classification(y_valid, y_valid_pred)
print("MCC on validation set: ", metrics)
metrics_all.append(metrics)
# 5) Ensemble
# TODO: ensemble method in a script
# Average the predictions and apply a threshold to determine class labels
y_valid_pred = np.mean(y_valid_pred_l, axis=0)
y_valid_pred = (y_valid_pred > 0.5).astype(int)
# 5) Save the validation accuracy
min_index = np.argmin(metrics_all)
pd.Series(data=[metrics_all[min_index]], index=["MCC"]).to_csv("submission_score.csv")
mcc = compute_metrics_for_classification(y_valid, y_valid_pred)
print("MCC on validation set: ", mcc)
# 6) Save the validation accuracy
pd.Series(data=[mcc], index=["MCC"]).to_csv("submission_score.csv")
# 7) Make predictions on the test set and save them
y_test_pred_l = []
for m, m_pred in model_l:
y_test_pred_l.append(m_pred(m, X_test)) # TODO Make this an ensemble. Currently it uses the last prediction
y_test_pred = np.mean(y_test_pred_l, axis=0)
# 6) Make predictions on the test set and save them
X_test_selected = model_l[min_index][2].select(X_test.copy())
y_test_pred = model_l[min_index][1](model_l[min_index][0], X_test_selected)
y_test_pred = (y_test_pred > 0.5).astype(int)
y_test_pred_labels = np.where(y_test_pred == 1, "p", "e") # 将整数转换回 'e' 或 'p'
# 8) Submit predictions for the test set
# 7) Submit predictions for the test set
submission_result = pd.DataFrame({"id": ids, "class": y_test_pred_labels.ravel()})
submission_result.to_csv("submission.csv", index=False)
rdagent/scenarios/kaggle/experiment/playground-series-s4e9_template/model/model_randomforest.py
+3 -18
View File
@@ -4,14 +4,6 @@ from sklearn.ensemble import RandomForestRegressor
from sklearn.metrics import mean_squared_error
def select(X: pd.DataFrame) -> pd.DataFrame:
"""
Select relevant features. To be used in fit & predict function.
"""
# For now, we assume all features are relevant. This can be expanded to feature selection logic.
return X
def fit(X_train: pd.DataFrame, y_train: pd.Series, X_valid: pd.DataFrame, y_valid: pd.Series):
"""
Define and train the Random Forest model. Merge feature selection into the pipeline.
@@ -19,15 +11,11 @@ def fit(X_train: pd.DataFrame, y_train: pd.Series, X_valid: pd.DataFrame, y_vali
# Initialize the Random Forest model
model = RandomForestRegressor(n_estimators=100, random_state=32, n_jobs=-1)
# Select features (if any feature selection is needed)
X_train_selected = select(X_train)
X_valid_selected = select(X_valid)
# Fit the model
model.fit(X_train_selected, y_train)
model.fit(X_train, y_train)
# Validate the model
y_valid_pred = model.predict(X_valid_selected)
y_valid_pred = model.predict(X_valid)
mse = mean_squared_error(y_valid, y_valid_pred)
rmse = np.sqrt(mse)
print(f"Validation RMSE: {rmse:.4f}")
@@ -39,10 +27,7 @@ def predict(model, X):
"""
Keep feature selection's consistency and make predictions.
"""
# Select features (if any feature selection is needed)
X_selected = select(X)
# Predict using the trained model
y_pred = model.predict(X_selected)
y_pred = model.predict(X)
return y_pred.reshape(-1, 1)
rdagent/scenarios/kaggle/experiment/playground-series-s4e9_template/model/model_xgboost.py
-8
View File
@@ -2,15 +2,8 @@ import pandas as pd
import xgboost as xgb
def select(X: pd.DataFrame) -> pd.DataFrame:
# Ignore feature selection logic
return X
def fit(X_train: pd.DataFrame, y_train: pd.DataFrame, X_valid: pd.DataFrame, y_valid: pd.DataFrame):
"""Define and train the model. Merge feature_select"""
X_train = select(X_train)
X_valid = select(X_valid)
dtrain = xgb.DMatrix(X_train, label=y_train)
dvalid = xgb.DMatrix(X_valid, label=y_valid)
@@ -33,7 +26,6 @@ def predict(model, X):
"""
Keep feature select's consistency.
"""
X = select(X)
dtest = xgb.DMatrix(X)
y_pred = model.predict(dtest)
return y_pred.reshape(-1, 1)
rdagent/scenarios/kaggle/experiment/playground-series-s4e9_template/model/select_randomforest.py
+12
View File
@@ -0,0 +1,12 @@
import pandas as pd
def select(X: pd.DataFrame) -> pd.DataFrame:
"""
Select relevant features. To be used in fit & predict function.
"""
# For now, we assume all features are relevant. This can be expanded to feature selection logic.
if X.columns.nlevels == 1:
return X
X.columns = ["_".join(str(col)).strip() for col in X.columns.values]
return X
rdagent/scenarios/kaggle/experiment/playground-series-s4e9_template/model/select_xgboost.py
+12
View File
@@ -0,0 +1,12 @@
import pandas as pd
def select(X: pd.DataFrame) -> pd.DataFrame:
"""
Select relevant features. To be used in fit & predict function.
"""
# For now, we assume all features are relevant. This can be expanded to feature selection logic.
if X.columns.nlevels == 1:
return X
X.columns = ["_".join(str(col)).strip() for col in X.columns.values]
return X
rdagent/scenarios/kaggle/experiment/playground-series-s4e9_template/train.py
+24 -40
View File
@@ -29,10 +29,8 @@ def import_module_from_path(module_name, module_path):
return module
print("begin preprocess")
# 1) Preprocess the data
X_train, X_valid, y_train, y_valid, X_test, ids = preprocess_script()
print("preprocess done")
# 2) Auto feature engineering
X_train_l, X_valid_l = [], []
@@ -45,9 +43,10 @@ for f in DIRNAME.glob("feature/feat*.py"):
X_valid_f = cls.transform(X_valid)
X_test_f = cls.transform(X_test)
X_train_l.append(X_train_f)
X_valid_l.append(X_valid_f)
X_test_l.append(X_test_f)
if X_train_f.shape[-1] == X_valid_f.shape[-1] and X_train_f.shape[-1] == X_test_f.shape[-1]:
X_train_l.append(X_train_f)
X_valid_l.append(X_valid_f)
X_test_l.append(X_test_f)
X_train = pd.concat(X_train_l, axis=1, keys=[f"feature_{i}" for i in range(len(X_train_l))])
X_valid = pd.concat(X_valid_l, axis=1, keys=[f"feature_{i}" for i in range(len(X_valid_l))])
@@ -75,48 +74,33 @@ X_test = X_test.loc[:, ~X_test.columns.duplicated()]
# 3) Train the model
def flatten_columns(df: pd.DataFrame) -> pd.DataFrame:
"""
Flatten the columns of a DataFrame with MultiIndex columns,
for (feature_0, a), (feature_0, b) -> feature_0_a, feature_0_b
"""
if df.columns.nlevels == 1:
return df
df.columns = ["_".join(col).strip() for col in df.columns.values]
return df
X_train = flatten_columns(X_train)
X_valid = flatten_columns(X_valid)
X_test = flatten_columns(X_test)
model_l = [] # list[tuple[model, predict_func,]]
for f in DIRNAME.glob("model/model*.py"):
select_python_path = f.with_name(f.stem.replace("model", "select") + f.suffix)
select_m = import_module_from_path(select_python_path.stem, select_python_path)
X_train_selected = select_m.select(X_train.copy())
X_valid_selected = select_m.select(X_valid.copy())
m = import_module_from_path(f.stem, f)
model_l.append((m.fit(X_train, y_train, X_valid, y_valid), m.predict))
model_l.append((m.fit(X_train_selected, y_train, X_valid_selected, y_valid), m.predict, select_m))
# 4) Evaluate the model on the validation set
y_valid_pred_l = []
for model, predict_func in model_l:
y_valid_pred_l.append(predict_func(model, X_valid))
print(predict_func(model, X_valid).shape)
metrics_all = []
for model, predict_func, select_m in model_l:
X_valid_selected = select_m.select(X_valid.copy())
y_valid_pred = predict_func(model, X_valid_selected)
rmse = compute_rmse(y_valid, y_valid_pred)
print(f"RMSE on valid set: {rmse}")
metrics_all.append(rmse)
# 5) Ensemble
y_valid_pred = np.mean(y_valid_pred_l, axis=0)
# 5) Save the validation accuracy
min_index = np.argmin(metrics_all)
pd.Series(data=[metrics_all[min_index]], index=["RMSE"]).to_csv("submission_score.csv")
rmse = compute_rmse(y_valid, y_valid_pred)
print("Final RMSE on validation set: ", rmse)
# 6) Make predictions on the test set and save them
X_test_selected = model_l[min_index][2].select(X_test.copy())
y_test_pred = model_l[min_index][1](model_l[min_index][0], X_test_selected).ravel()
# 6) Save the validation RMSE
pd.Series(data=[rmse], index=["RMSE"]).to_csv("submission_score.csv")
# 7) Make predictions on the test set and save them
y_test_pred_l = []
for m, m_pred in model_l:
y_test_pred_l.append(m_pred(m, X_test))
y_test_pred = np.mean(y_test_pred_l, axis=0).ravel()
# 8) Submit predictions for the test set
# 7) Submit predictions for the test set
submission_result = pd.DataFrame({"id": ids, "price": y_test_pred})
submission_result.to_csv("submission.csv", index=False)
rdagent/scenarios/kaggle/experiment/prompts.yaml
+3 -34
View File
@@ -125,22 +125,19 @@ kg_feature_interface: |-
kg_model_interface: |-
Your code should contain several parts:
1. The import part: import the necessary libraries.
2. A select() function that handles feature selection for both training and prediction phases.
The function should take the following arguments:
- X: The features as a pandas DataFrame.
The function should return the selected features as a pandas DataFrame.
3. A function called fit() that trains the model and returns the trained model. If feature selection is applied, it should be done within this function.
2. A function called fit() that trains the model and returns the trained model.
The function should take the following arguments:
- X_train: The training features as a pandas DataFrame.
- y_train: The training labels as a pandas Series.
- X_valid: The validation features as a pandas DataFrame.
- y_valid: The validation labels as a pandas Series.
The function should return the trained model.
4. A function called predict() that makes predictions using the trained model. If feature selection is applied, it should be done within this function.
3. A function called predict() that makes predictions using the trained model.
The function should take the following arguments:
- model: The trained model.
- X: The features as a pandas DataFrame.
The function should return the predicted probabilities or boolean predictions in numpy.ndarray format.
"""
Here are some examples of how your Python code should be structured:
@@ -151,15 +148,9 @@ kg_model_interface: |-
import xgboost
from xgboost import DMatrix
def select(X: pd.DataFrame) -> pd.DataFrame: ... # Implement feature selection logic
def fit(
X_train: pd.DataFrame, y_train: pd.Series, X_valid: pd.DataFrame, y_valid: pd.Series
) -> xgboost.Booster:
X_train = select(X_train)
X_valid = select(X_valid)
dtrain = DMatrix(X_train, label=y_train)
dvalid = DMatrix(X_valid, label=y_valid)
params = ... # Set parameters to XGBoost model
@@ -171,7 +162,6 @@ kg_model_interface: |-
def predict(model: xgboost.Booster, X: pd.DataFrame) -> np.ndarray:
X = select(X)
dtest = DMatrix(X)
y_pred = model.predict(dtest)
@@ -185,15 +175,9 @@ kg_model_interface: |-
from sklearn.ensemble import RandomForestClassifier, RandomForestRegressor
from sklearn.metrics import accuracy_score
def select(X: pd.DataFrame) -> pd.DataFrame: ... # Implement feature selection logic
def fit(
X_train: pd.DataFrame, y_train: pd.Series, X_valid: pd.DataFrame, y_valid: pd.Series
) -> RandomForestClassifier | RandomForestRegressor:
X_train = select(X_train)
X_valid = select(X_valid)
model = RandomForestClassifier(...) # fir classification tasks
model = RandomForestRegressor(...) # for regression tasks
model.fit(X_train, y_train, ...) # Train the model
@@ -202,7 +186,6 @@ kg_model_interface: |-
def predict(model: RandomForestClassifier | RandomForestRegressor, X: pd.DataFrame) -> np.ndarray:
X = select(X)
y_pred = model.predict(X)
return y_pred
@@ -214,15 +197,9 @@ kg_model_interface: |-
import numpy as np
from lightgbm import LGBMClassifier, LGBMRegressor
def select(X: pd.DataFrame) -> pd.DataFrame: ... # Implement feature selection logic
def fit(
X_train: pd.DataFrame, y_train: pd.Series, X_valid: pd.DataFrame, y_valid: pd.Series
) -> LGBMClassifier | LGBMRegressor:
X_train = select(X_train)
X_valid = select(X_valid)
model = LGBMClassifier(...) # for classification tasks, please add parameters here
model = LGBMRegressor(...) # for regression tasks, please add parameters here
@@ -231,7 +208,6 @@ kg_model_interface: |-
def predict(model: LGBMClassifier | LGBMRegressor, X: pd.DataFrame) -> np.ndarray:
X = select(X)
y_pred = model.predict(X)
return y_pred
@@ -254,13 +230,7 @@ kg_model_interface: |-
# Define the forward pass
return x
def select(X: pd.DataFrame) -> pd.DataFrame: ... # Implement feature selection logic
def fit(X_train: pd.DataFrame, y_train: pd.DataFrame, X_valid: pd.DataFrame, y_valid: pd.DataFrame) -> torch.nn.Module:
X_train = select(X_train)
X_valid = select(X_valid)
model = NNModel() # Initialize the model, You can write your own model class
optimizer = torch.optim.Adam(model.parameters(), lr=0.01) # Example optimizer, you can use any optimizer
@@ -293,7 +263,6 @@ kg_model_interface: |-
def predict(model: torch.nn.Module, X: pd.DataFrame) -> np.ndarray:
X = select(X)
X = torch.tensor(X.values).float()
model.eval()
with torch.no_grad():
rdagent/scenarios/kaggle/experiment/scenario.py
-1
View File
@@ -39,7 +39,6 @@ class KGScenario(Scenario):
self.vector_base = None
self._analysis_competition_description()
self.if_action_choosing_based_on_UCB = KAGGLE_IMPLEMENT_SETTING.if_action_choosing_based_on_UCB
self.if_using_feature_selection = KAGGLE_IMPLEMENT_SETTING.if_using_feature_selection
self.if_using_graph_rag = KAGGLE_IMPLEMENT_SETTING.if_using_graph_rag
self.if_using_vector_rag = KAGGLE_IMPLEMENT_SETTING.if_using_vector_rag
rdagent/scenarios/kaggle/experiment/sf-crime_template/model/model_randomforest.py
+2 -23
View File
@@ -7,15 +7,6 @@ baseline model for many classification tasks.
import pandas as pd
from sklearn.ensemble import RandomForestClassifier
from sklearn.metrics import accuracy_score
def select(X: pd.DataFrame) -> pd.DataFrame:
"""
Select relevant features. To be used in fit & predict function.
"""
# For now, we assume all features are relevant. This can be expanded to feature selection logic.
return X
def fit(X_train: pd.DataFrame, y_train: pd.Series, X_valid: pd.DataFrame, y_valid: pd.Series):
@@ -25,17 +16,8 @@ def fit(X_train: pd.DataFrame, y_train: pd.Series, X_valid: pd.DataFrame, y_vali
# Initialize the Random Forest model
model = RandomForestClassifier(n_estimators=10, random_state=32, n_jobs=-1)
# Select features (if any feature selection is needed)
X_train_selected = select(X_train)
X_valid_selected = select(X_valid)
# Fit the model
model.fit(X_train_selected, y_train)
# Validate the model
y_valid_pred = model.predict(X_valid_selected)
# accuracy = accuracy_score(y_valid, y_valid_pred)
# print(f"Validation Accuracy: {accuracy:.4f}")
model.fit(X_train, y_train)
return model
@@ -44,11 +26,8 @@ def predict(model, X):
"""
Keep feature selection's consistency and make predictions.
"""
# Select features (if any feature selection is needed)
X_selected = select(X)
# Predict using the trained model
y_pred_prob = model.predict_proba(X_selected)
y_pred_prob = model.predict_proba(X)
# Apply threshold to get boolean predictions
return y_pred_prob
rdagent/scenarios/kaggle/experiment/sf-crime_template/model/model_xgboost.py
-8
View File
@@ -7,15 +7,8 @@ import pandas as pd
import xgboost as xgb
def select(X: pd.DataFrame) -> pd.DataFrame:
# Ignore feature selection logic
return X
def fit(X_train: pd.DataFrame, y_train: pd.DataFrame, X_valid: pd.DataFrame, y_valid: pd.DataFrame):
"""Define and train the model. Merge feature_select"""
X_train = select(X_train)
X_valid = select(X_valid)
dtrain = xgb.DMatrix(X_train, label=y_train)
dvalid = xgb.DMatrix(X_valid, label=y_valid)
num_classes = len(np.unique(y_train))
@@ -40,7 +33,6 @@ def predict(model, X):
"""
Keep feature select's consistency.
"""
X = select(X)
dtest = xgb.DMatrix(X)
y_pred_prob = model.predict(dtest)
return y_pred_prob
rdagent/scenarios/kaggle/experiment/sf-crime_template/model/select_randomforest.py
+12
View File
@@ -0,0 +1,12 @@
import pandas as pd
def select(X: pd.DataFrame) -> pd.DataFrame:
"""
Select relevant features. To be used in fit & predict function.
"""
# For now, we assume all features are relevant. This can be expanded to feature selection logic.
if X.columns.nlevels == 1:
return X
X.columns = ["_".join(str(col)).strip() for col in X.columns.values]
return X
rdagent/scenarios/kaggle/experiment/sf-crime_template/model/select_xgboost.py
+12
View File
@@ -0,0 +1,12 @@
import pandas as pd
def select(X: pd.DataFrame) -> pd.DataFrame:
"""
Select relevant features. To be used in fit & predict function.
"""
# For now, we assume all features are relevant. This can be expanded to feature selection logic.
if X.columns.nlevels == 1:
return X
X.columns = ["_".join(str(col)).strip() for col in X.columns.values]
return X
rdagent/scenarios/kaggle/experiment/sf-crime_template/train.py
+25 -46
View File
@@ -49,10 +49,11 @@ for f in DIRNAME.glob("feature/feat*.py"):
X_valid_l.append(X_valid_f)
X_test_l.append(X_test_f)
X_train = pd.concat(X_train_l, axis=1)
X_valid = pd.concat(X_valid_l, axis=1)
X_test = pd.concat(X_test_l, axis=1)
X_train = pd.concat(X_train_l, axis=1, keys=[f"feature_{i}" for i in range(len(X_train_l))])
X_valid = pd.concat(X_valid_l, axis=1, keys=[f"feature_{i}" for i in range(len(X_valid_l))])
X_test = pd.concat(X_test_l, axis=1, keys=[f"feature_{i}" for i in range(len(X_test_l))])
print(X_train.shape, X_valid.shape, X_test.shape)
# Handle inf and -inf values
X_train.replace([np.inf, -np.inf], np.nan, inplace=True)
@@ -72,61 +73,39 @@ X_train = X_train.loc[:, ~X_train.columns.duplicated()]
X_valid = X_valid.loc[:, ~X_valid.columns.duplicated()]
X_test = X_test.loc[:, ~X_test.columns.duplicated()]
print(X_train.shape, X_valid.shape, X_test.shape)
# 3) Train the model
def flatten_columns(df: pd.DataFrame) -> pd.DataFrame:
"""
Flatten the columns of a DataFrame with MultiIndex columns,
for (feature_0, a), (feature_0, b) -> feature_0_a, feature_0_b
"""
if df.columns.nlevels == 1:
return df
df.columns = ["_".join(col).strip() for col in df.columns.values]
return df
X_train = flatten_columns(X_train)
X_valid = flatten_columns(X_valid)
X_test = flatten_columns(X_test)
model_l = [] # list[tuple[model, predict_func]]
for f in DIRNAME.glob("model/model*.py"):
select_python_path = f.with_name(f.stem.replace("model", "select") + f.suffix)
select_m = import_module_from_path(select_python_path.stem, select_python_path)
X_train_selected = select_m.select(X_train.copy())
X_valid_selected = select_m.select(X_valid.copy())
m = import_module_from_path(f.stem, f)
model_l.append((m.fit(X_train, y_train, X_valid, y_valid), m.predict))
model_l.append((m.fit(X_train_selected, y_train, X_valid_selected, y_valid), m.predict, select_m))
# 4) Evaluate the model on the validation set
y_valid_pred_l = []
for model, predict_func in model_l:
y_valid_pred_l.append(predict_func(model, X_valid))
# print(predict_func(model, X_valid))
print(predict_func(model, X_valid).shape)
metrics_all = []
for model, predict_func, select_m in model_l:
X_valid_selected = select_m.select(X_valid.copy())
y_valid_pred = predict_func(model, X_valid_selected)
metrics = compute_metrics_for_classification(y_valid, y_valid_pred)
print(f"log_loss on valid set: {metrics}")
metrics_all.append(metrics)
# 5) Ensemble
from scipy import stats
# 5) Save the validation accuracy
min_index = np.argmin(metrics_all)
pd.Series(data=[metrics_all[min_index]], index=["log_loss"]).to_csv("submission_score.csv")
# average probabilities ensemble
y_valid_pred_proba = np.mean(y_valid_pred_l, axis=0)
# 6) Make predictions on the test set and save them
X_test_selected = model_l[min_index][2].select(X_test.copy())
y_test_pred = model_l[min_index][1](model_l[min_index][0], X_test_selected)
# Compute metrics
logloss = compute_metrics_for_classification(y_valid, y_valid_pred_proba)
print(f"final log_loss on valid set: {logloss}")
# 6) Save the validation metrics
pd.Series(data=[logloss], index=["log_loss"]).to_csv("submission_score.csv")
# 7) Make predictions on the test set and save them
y_test_pred_l = []
for model, predict_func in model_l:
y_test_pred_l.append(predict_func(model, X_test))
# For multiclass classification, use the mode of the predictions
y_test_pred_proba = np.mean(y_test_pred_l, axis=0)
# 7) Submit predictions for the test set
class_labels = category_encoder.classes_
submission_result = pd.DataFrame(y_test_pred_proba, columns=class_labels)
submission_result = pd.DataFrame(y_test_pred, columns=class_labels)
submission_result.insert(0, "Id", test_ids)
submission_result.to_csv("submission.csv", index=False)
rdagent/scenarios/kaggle/experiment/spaceship-titanic_template/model/model_nn.py
+2 -2
View File
@@ -49,8 +49,8 @@ def fit(X_train, y_train, X_valid, y_valid):
# Train the model
model.train()
for epoch in range(100):
print(f"Epoch {epoch + 1}/100")
for epoch in range(50):
print(f"Epoch {epoch + 1}/50")
epoch_loss = 0
for X_batch, y_batch in tqdm(train_loader, desc="Training", leave=False):
X_batch, y_batch = X_batch.to(device), y_batch.to(device) # Move data to the device
rdagent/scenarios/kaggle/experiment/spaceship-titanic_template/model/model_randomforest.py
+2 -22
View File
@@ -10,14 +10,6 @@ from sklearn.ensemble import RandomForestClassifier
from sklearn.metrics import accuracy_score
def select(X: pd.DataFrame) -> pd.DataFrame:
"""
Select relevant features. To be used in fit & predict function.
"""
# For now, we assume all features are relevant. This can be expanded to feature selection logic.
return X
def fit(X_train: pd.DataFrame, y_train: pd.Series, X_valid: pd.DataFrame, y_valid: pd.Series):
"""
Define and train the Random Forest model. Merge feature selection into the pipeline.
@@ -25,17 +17,8 @@ def fit(X_train: pd.DataFrame, y_train: pd.Series, X_valid: pd.DataFrame, y_vali
# Initialize the Random Forest model
model = RandomForestClassifier(n_estimators=100, random_state=32, n_jobs=-1)
# Select features (if any feature selection is needed)
X_train_selected = select(X_train)
X_valid_selected = select(X_valid)
# Fit the model
model.fit(X_train_selected, y_train)
# Validate the model
y_valid_pred = model.predict(X_valid_selected)
accuracy = accuracy_score(y_valid, y_valid_pred)
print(f"Validation Accuracy: {accuracy:.4f}")
model.fit(X_train, y_train)
return model
@@ -44,11 +27,8 @@ def predict(model, X):
"""
Keep feature selection's consistency and make predictions.
"""
# Select features (if any feature selection is needed)
X_selected = select(X)
# Predict using the trained model
y_pred_prob = model.predict_proba(X_selected)[:, 1]
y_pred_prob = model.predict_proba(X)[:, 1]
# Apply threshold to get boolean predictions
return y_pred_prob.reshape(-1, 1)
rdagent/scenarios/kaggle/experiment/spaceship-titanic_template/model/model_xgboost.py
-9
View File
@@ -6,19 +6,11 @@ import pandas as pd
import xgboost as xgb
def select(X: pd.DataFrame) -> pd.DataFrame:
# Ignore feature selection logic
return X
def fit(X_train: pd.DataFrame, y_train: pd.DataFrame, X_valid: pd.DataFrame, y_valid: pd.DataFrame):
"""Define and train the model. Merge feature_select"""
X_train = select(X_train)
X_valid = select(X_valid)
dtrain = xgb.DMatrix(X_train, label=y_train)
dvalid = xgb.DMatrix(X_valid, label=y_valid)
# TODO: for quick running....
params = {
"nthread": -1,
"tree_method": "gpu_hist",
@@ -36,7 +28,6 @@ def predict(model, X):
"""
Keep feature select's consistency.
"""
X = select(X)
dtest = xgb.DMatrix(X)
y_pred_prob = model.predict(dtest)
return y_pred_prob.reshape(-1, 1)
rdagent/scenarios/kaggle/experiment/spaceship-titanic_template/model/select_nn.py
+12
View File
@@ -0,0 +1,12 @@
import pandas as pd
def select(X: pd.DataFrame) -> pd.DataFrame:
"""
Select relevant features. To be used in fit & predict function.
"""
# For now, we assume all features are relevant. This can be expanded to feature selection logic.
if X.columns.nlevels == 1:
return X
X.columns = ["_".join(str(col)).strip() for col in X.columns.values]
return X
rdagent/scenarios/kaggle/experiment/spaceship-titanic_template/model/select_randomforest.py
+12
View File
@@ -0,0 +1,12 @@
import pandas as pd
def select(X: pd.DataFrame) -> pd.DataFrame:
"""
Select relevant features. To be used in fit & predict function.
"""
# For now, we assume all features are relevant. This can be expanded to feature selection logic.
if X.columns.nlevels == 1:
return X
X.columns = ["_".join(str(col)).strip() for col in X.columns.values]
return X
rdagent/scenarios/kaggle/experiment/spaceship-titanic_template/model/select_xgboost.py
+12
View File
@@ -0,0 +1,12 @@
import pandas as pd
def select(X: pd.DataFrame) -> pd.DataFrame:
"""
Select relevant features. To be used in fit & predict function.
"""
# For now, we assume all features are relevant. This can be expanded to feature selection logic.
if X.columns.nlevels == 1:
return X
X.columns = ["_".join(str(col)).strip() for col in X.columns.values]
return X
rdagent/scenarios/kaggle/experiment/spaceship-titanic_template/train.py
+25 -41
View File
@@ -5,8 +5,7 @@ from pathlib import Path
import numpy as np
import pandas as pd
from fea_share_preprocess import preprocess_script
from sklearn.metrics import accuracy_score, matthews_corrcoef
from sklearn.preprocessing import LabelEncoder
from sklearn.metrics import accuracy_score
# Set random seed for reproducibility
SEED = 42
@@ -30,7 +29,6 @@ def import_module_from_path(module_name, module_path):
# 1) Preprocess the data
# TODO 如果已经做过数据预处理了，不需要再做了
X_train, X_valid, y_train, y_valid, X_test, passenger_ids = preprocess_script()
# 2) Auto feature engineering
@@ -44,9 +42,10 @@ for f in DIRNAME.glob("feature/feat*.py"):
X_valid_f = cls.transform(X_valid)
X_test_f = cls.transform(X_test)
X_train_l.append(X_train_f)
X_valid_l.append(X_valid_f)
X_test_l.append(X_test_f)
if X_train_f.shape[-1] == X_valid_f.shape[-1] and X_train_f.shape[-1] == X_test_f.shape[-1]:
X_train_l.append(X_train_f)
X_valid_l.append(X_valid_f)
X_test_l.append(X_test_f)
X_train = pd.concat(X_train_l, axis=1, keys=[f"feature_{i}" for i in range(len(X_train_l))])
X_valid = pd.concat(X_valid_l, axis=1, keys=[f"feature_{i}" for i in range(len(X_valid_l))])
@@ -74,49 +73,34 @@ X_test = X_test.loc[:, ~X_test.columns.duplicated()]
# 3) Train the model
def flatten_columns(df: pd.DataFrame) -> pd.DataFrame:
"""
Flatten the columns of a DataFrame with MultiIndex columns,
for (feature_0, a), (feature_0, b) -> feature_0_a, feature_0_b
"""
if df.columns.nlevels == 1:
return df
df.columns = ["_".join(col).strip() for col in df.columns.values]
return df
X_train = flatten_columns(X_train)
X_valid = flatten_columns(X_valid)
X_test = flatten_columns(X_test)
model_l = [] # list[tuple[model, predict_func,]]
for f in DIRNAME.glob("model/model*.py"):
select_python_path = f.with_name(f.stem.replace("model", "select") + f.suffix)
select_m = import_module_from_path(select_python_path.stem, select_python_path)
X_train_selected = select_m.select(X_train.copy())
X_valid_selected = select_m.select(X_valid.copy())
m = import_module_from_path(f.stem, f)
model_l.append((m.fit(X_train, y_train, X_valid, y_valid), m.predict))
model_l.append((m.fit(X_train_selected, y_train, X_valid_selected, y_valid), m.predict, select_m))
# 4) Evaluate the model on the validation set
y_valid_pred_l = []
for model, predict_func in model_l:
y_valid_pred_l.append(predict_func(model, X_valid))
metrics_all = []
for model, predict_func, select_m in model_l:
X_valid_selected = select_m.select(X_valid.copy())
y_valid_pred = predict_func(model, X_valid_selected)
y_valid_pred = (y_valid_pred > 0.5).astype(int)
metrics = compute_metrics_for_classification(y_valid, y_valid_pred)
print(f"Accuracy on valid set: {metrics}")
metrics_all.append(metrics)
# 5) Ensemble
# TODO: ensemble method in a script
# Average the predictions and apply a threshold to determine class labels
y_valid_pred = np.mean(y_valid_pred_l, axis=0)
y_valid_pred = (y_valid_pred > 0.5).astype(int)
mcc = compute_metrics_for_classification(y_valid, y_valid_pred)
print("Final on validation set: ", mcc)
# 5) Save the validation accuracy
min_index = np.argmax(metrics_all)
pd.Series(data=[metrics_all[min_index]], index=["MCC"]).to_csv("submission_score.csv")
# 6) Save the validation accuracy
pd.Series(data=[mcc], index=["MCC"]).to_csv("submission_score.csv")
# 7) Make predictions on the test set and save them
y_test_pred_l = []
for m, m_pred in model_l:
y_test_pred_l.append(m_pred(m, X_test)) # TODO Make this an ensemble. Currently it uses the last prediction
y_test_pred = np.mean(y_test_pred_l, axis=0)
# 6) Make predictions on the test set and save them
X_test_selected = model_l[min_index][2].select(X_test.copy())
y_test_pred = model_l[min_index][1](model_l[min_index][0], X_test_selected)
y_test_pred = (y_test_pred > 0.5).astype(bool)
y_test_pred = y_test_pred.ravel()
rdagent/scenarios/kaggle/experiment/workspace.py
+10 -3
View File
@@ -1,7 +1,7 @@
import subprocess
import zipfile
from pathlib import Path
from typing import Any
from typing import Any, List, Tuple
import pandas as pd
@@ -29,8 +29,15 @@ class KGFBWorkspace(FBWorkspace):
def __init__(self, template_folder_path: Path, *args, **kwargs) -> None:
super().__init__(*args, **kwargs)
self.inject_code_from_folder(template_folder_path)
self.data_description: list[str] = []
self.model_description: dict[str, str] = {}
self.data_description: List[Tuple[str, int]] = []
@property
def model_description(self) -> dict[str, str]:
model_description = {}
for k, v in self.code_dict.items():
if k.startswith("model/"):
model_description[k] = v
return model_description
def generate_preprocess_data(
self,
rdagent/scenarios/kaggle/prompts.yaml
+21 -19
View File
@@ -39,7 +39,7 @@ hypothesis_and_feedback: |-
hypothesis_output_format: |-
The output should follow JSON format. The schema is as follows:
{
"action": "If "hypothesis_specification" provides the action you need to take, please follow "hypothesis_specification" to choose the action. Otherwise, based on previous experimental results, suggest the action you believe is most appropriate at the moment. It should be one of [{% if if_using_feature_selection %}"Feature engineering", "Feature processing", "Model feature selection", "Model tuning"{% else %}"Feature engineering", "Feature processing", "Model tuning"{% endif %}]",
"action": "If "hypothesis_specification" provides the action you need to take, please follow "hypothesis_specification" to choose the action. Otherwise, based on previous experimental results, suggest the action you believe is most appropriate at the moment. It should be one of ["Feature engineering", "Feature processing", "Model feature selection", "Model tuning"]"
"hypothesis": "The new hypothesis generated based on the information provided.",
"reason": "The reason why you generate this hypothesis. It should be comprehensive and logical. It should cover the other keys below and extend them.",
"concise_reason": "Two-line summary. First line focuses on a concise justification for the change. Second line generalizes a knowledge statement.",
@@ -291,26 +291,28 @@ feature_selection_feedback_generation:
3. How might we refine or change the feature selection approach to improve model performance?
4. Are there any domain-specific considerations that should inform our feature selection?
extract_model_task_from_code:
model_feature_selection:
system: |-
You are an expert in analyzing code for machine learning models.
user: |-
Given the following code, summarize the machine learning model including:
- Model architecture
- Hyperparameters
- Formulation and variables
- Model type (one of XGBoost, RandomForest, LightGBM, NN)
You are an assistant for model feature selection in machine learning. Your task is to understand the current feature groups and choose the most relevant features for the model to get the best performance.
Code:
{{ file_content }}
The user is currently working on a Kaggle competition scenario as follows:
{{ scenario }}
Return the information in JSON format with the following structure:
The user is now working on the following model type:
{{ model_type }}
The user will give you several feature groups and their descriptions. Your task is to select the most relevant features for the model to achieve the best performance. You should consider the following:
1. How well do the selected features support the scenario?
2. Are there any features that might be redundant or noisy?
Please answer the chosen group index in JSON format. Example JSON structure for Result Analysis:
{
"name": "",
"description": "",
"architecture": "",
"hyperparameters": {},
"formulation": "",
"variables": {},
"model_type": ""
"Selected Group Index": [1, 3, 5], # List of selected group indices, notice: the index starts from 1
}
user: |-
Current feature groups:
{% for feature in feature_groups %}
Group {{ loop.index }}:
{{ feature }}
{% endfor %}
rdagent/scenarios/kaggle/proposal/proposal.py
+15 -13
View File
@@ -36,7 +36,7 @@ KG_ACTION_MODEL_TUNING = "Model tuning"
KG_ACTION_LIST = [
KG_ACTION_FEATURE_PROCESSING,
KG_ACTION_FEATURE_ENGINEERING,
*([KG_ACTION_MODEL_FEATURE_SELECTION] if KAGGLE_IMPLEMENT_SETTING.if_using_feature_selection else []),
KG_ACTION_MODEL_FEATURE_SELECTION,
KG_ACTION_MODEL_TUNING,
]
@@ -82,13 +82,9 @@ class KGHypothesisGen(ModelHypothesisGen):
def __init__(self, scen: Scenario) -> Tuple[dict, bool]:
super().__init__(scen)
actions = ["Feature engineering", "Feature processing", "Model tuning"]
if KAGGLE_IMPLEMENT_SETTING.if_using_feature_selection:
actions.insert(2, "Model feature selection")
self.action_counts = dict.fromkeys(actions, 0)
self.reward_estimates = {action: 0.0 for action in actions}
if KAGGLE_IMPLEMENT_SETTING.if_using_feature_selection:
self.reward_estimates["Model feature selection"] = 0.2
self.action_counts = dict.fromkeys(KG_ACTION_LIST, 0)
self.reward_estimates = {action: 0.0 for action in KG_ACTION_LIST}
self.reward_estimates["Model feature selection"] = 0.2
self.reward_estimates["Model tuning"] = 1.0
self.confidence_parameter = 1.0
self.initial_performance = 0.0
@@ -239,9 +235,7 @@ class KGHypothesisGen(ModelHypothesisGen):
context_dict = {
"hypothesis_and_feedback": hypothesis_and_feedback,
"RAG": self.generate_RAG_content(trace, hypothesis_and_feedback),
"hypothesis_output_format": Environment(undefined=StrictUndefined)
.from_string(prompt_dict["hypothesis_output_format"])
.render(if_using_feature_selection=KAGGLE_IMPLEMENT_SETTING.if_using_feature_selection),
"hypothesis_output_format": prompt_dict["hypothesis_output_format"],
"hypothesis_specification": (
f"next experiment action is {action}" if self.scen.if_action_choosing_based_on_UCB else None
),
@@ -319,7 +313,11 @@ class KGHypothesis2Experiment(ModelHypothesis2Experiment):
)
exp = KGFactorExperiment(
sub_tasks=tasks, based_experiments=([KGFactorExperiment(sub_tasks=[])] + [t[1] for t in trace.hist if t[2]])
sub_tasks=tasks,
based_experiments=(
[KGFactorExperiment(sub_tasks=[], source_feature_size=trace.scen.input_shape[-1])]
+ [t[1] for t in trace.hist if t[2]]
),
)
return exp
@@ -337,7 +335,11 @@ class KGHypothesis2Experiment(ModelHypothesis2Experiment):
)
)
exp = KGModelExperiment(
sub_tasks=tasks, based_experiments=([KGModelExperiment(sub_tasks=[])] + [t[1] for t in trace.hist if t[2]])
sub_tasks=tasks,
based_experiments=(
[KGModelExperiment(sub_tasks=[], source_feature_size=trace.scen.input_shape[-1])]
+ [t[1] for t in trace.hist if t[2]]
),
)
return exp