From 1d2b8b9867e6181b0fe2a3dbf2f8fd6ad80a009b Mon Sep 17 00:00:00 2001 From: Xu Yang Date: Sat, 28 Sep 2024 00:40:25 +0800 Subject: [PATCH] feat: implement isolated model feature selection loop (#370) * rename meta_tpl * use a isolated coder to deal with model feature selection and refine the structure * fix CI * fix: fix some errors in scenario.py, proposal.py and runner.py and several complex competition scenarios(#365) * fix several bugs in proposal and runner * fix a bug in feedback-prize-english-language-learning * fix some bugs and templates * fix the bug in optiver and nlp problem * delete unnecessary codes * remove unnecessary codes * complete forest and s4e8 * push * feedback & s4e8 & forest * optiver finished * s3e11 & s3e26 * s4e9 finished * sf-crime finished * the last one finished --------- Co-authored-by: WinstonLiyt <104308117+WinstonLiyt@users.noreply.github.com> Co-authored-by: WinstonLiyte <1957922024@qq.com> --- rdagent/app/kaggle/conf.py | 5 +- rdagent/app/kaggle/loop.py | 7 + .../model_coder/CoSTEER/evolving_strategy.py | 11 +- .../model_coder/model_execute_template_v2.txt | 2 +- rdagent/components/proposal/prompts.yaml | 2 +- rdagent/scenarios/kaggle/developer/coder.py | 67 +++++++ .../scenarios/kaggle/developer/feedback.py | 9 +- rdagent/scenarios/kaggle/developer/runner.py | 100 +--------- .../fea_share_preprocess.py | 1 - .../model/model_randomforest.py | 18 +- .../model/model_xgboost.py | 8 - .../model/select_randomforest.py | 12 ++ .../model/select_xgboost.py | 12 ++ .../train.py | 45 ++--- .../model/model_catboost.py | 29 --- .../model/{model_dnn.py => model_nn.py} | 0 .../model/model_randomforest.py | 21 +- .../model/model_xgboost.py | 8 - .../model/select_nn.py | 12 ++ .../model/select_randomforest.py | 12 ++ .../model/select_xgboost.py | 12 ++ .../train.py | 147 +++++--------- .../train_past.py | 187 ++++++++++-------- .../kaggle/experiment/kaggle_experiment.py | 51 ++++- .../fea_share_preprocess.py | 0 .../feature/feature.py | 0 .../model/model_nn.py | 0 .../model/model_randomforest.py | 0 .../model/model_xgboost.py | 0 .../train.py | 0 .../model/model_randomforest.py | 21 +- .../model/model_xgboost.py | 8 - .../model/select_randomforest.py | 12 ++ .../model/select_xgboost.py | 12 ++ .../train.py | 45 ++--- .../model/model_randomforest.py | 8 - .../model/model_xgboost.py | 8 - .../model/select_randomforest.py | 12 ++ .../model/select_xgboost.py | 12 ++ .../playground-series-s3e11_template/train.py | 42 ++-- .../model/model_randomforest.py | 24 +-- .../model/model_xgboost.py | 8 - .../model/select_randomforest.py | 12 ++ .../model/select_xgboost.py | 12 ++ .../playground-series-s3e26_template/train.py | 67 +++---- .../model/model_randomforest.py | 21 +- .../model/model_xgboost.py | 9 - .../model/select_randomforest.py | 12 ++ .../model/select_xgboost.py | 12 ++ .../playground-series-s4e8_template/train.py | 67 +++---- .../model/model_randomforest.py | 21 +- .../model/model_xgboost.py | 8 - .../model/select_randomforest.py | 12 ++ .../model/select_xgboost.py | 12 ++ .../playground-series-s4e9_template/train.py | 64 +++--- .../scenarios/kaggle/experiment/prompts.yaml | 37 +--- .../scenarios/kaggle/experiment/scenario.py | 1 - .../model/model_randomforest.py | 25 +-- .../sf-crime_template/model/model_xgboost.py | 8 - .../model/select_randomforest.py | 12 ++ .../sf-crime_template/model/select_xgboost.py | 12 ++ .../experiment/sf-crime_template/train.py | 71 +++---- .../model/model_nn.py | 4 +- .../model/model_randomforest.py | 24 +-- .../model/model_xgboost.py | 9 - .../model/select_nn.py | 12 ++ .../model/select_randomforest.py | 12 ++ .../model/select_xgboost.py | 12 ++ .../spaceship-titanic_template/train.py | 66 +++---- .../scenarios/kaggle/experiment/workspace.py | 13 +- rdagent/scenarios/kaggle/prompts.yaml | 40 ++-- rdagent/scenarios/kaggle/proposal/proposal.py | 28 +-- 72 files changed, 775 insertions(+), 930 deletions(-) create mode 100644 rdagent/scenarios/kaggle/experiment/feedback-prize-english-language-learning_template/model/select_randomforest.py create mode 100644 rdagent/scenarios/kaggle/experiment/feedback-prize-english-language-learning_template/model/select_xgboost.py delete mode 100644 rdagent/scenarios/kaggle/experiment/forest-cover-type-prediction_template/model/model_catboost.py rename rdagent/scenarios/kaggle/experiment/forest-cover-type-prediction_template/model/{model_dnn.py => model_nn.py} (100%) create mode 100644 rdagent/scenarios/kaggle/experiment/forest-cover-type-prediction_template/model/select_nn.py create mode 100644 rdagent/scenarios/kaggle/experiment/forest-cover-type-prediction_template/model/select_randomforest.py create mode 100644 rdagent/scenarios/kaggle/experiment/forest-cover-type-prediction_template/model/select_xgboost.py rename rdagent/scenarios/kaggle/experiment/{meta_tpl => meta_tpl_deprecated}/fea_share_preprocess.py (100%) rename rdagent/scenarios/kaggle/experiment/{meta_tpl => meta_tpl_deprecated}/feature/feature.py (100%) rename rdagent/scenarios/kaggle/experiment/{meta_tpl => meta_tpl_deprecated}/model/model_nn.py (100%) rename rdagent/scenarios/kaggle/experiment/{meta_tpl => meta_tpl_deprecated}/model/model_randomforest.py (100%) rename rdagent/scenarios/kaggle/experiment/{meta_tpl => meta_tpl_deprecated}/model/model_xgboost.py (100%) rename rdagent/scenarios/kaggle/experiment/{meta_tpl => meta_tpl_deprecated}/train.py (100%) create mode 100644 rdagent/scenarios/kaggle/experiment/optiver-realized-volatility-prediction_template/model/select_randomforest.py create mode 100644 rdagent/scenarios/kaggle/experiment/optiver-realized-volatility-prediction_template/model/select_xgboost.py create mode 100644 rdagent/scenarios/kaggle/experiment/playground-series-s3e11_template/model/select_randomforest.py create mode 100644 rdagent/scenarios/kaggle/experiment/playground-series-s3e11_template/model/select_xgboost.py create mode 100644 rdagent/scenarios/kaggle/experiment/playground-series-s3e26_template/model/select_randomforest.py create mode 100644 rdagent/scenarios/kaggle/experiment/playground-series-s3e26_template/model/select_xgboost.py create mode 100644 rdagent/scenarios/kaggle/experiment/playground-series-s4e8_template/model/select_randomforest.py create mode 100644 rdagent/scenarios/kaggle/experiment/playground-series-s4e8_template/model/select_xgboost.py create mode 100644 rdagent/scenarios/kaggle/experiment/playground-series-s4e9_template/model/select_randomforest.py create mode 100644 rdagent/scenarios/kaggle/experiment/playground-series-s4e9_template/model/select_xgboost.py create mode 100644 rdagent/scenarios/kaggle/experiment/sf-crime_template/model/select_randomforest.py create mode 100644 rdagent/scenarios/kaggle/experiment/sf-crime_template/model/select_xgboost.py create mode 100644 rdagent/scenarios/kaggle/experiment/spaceship-titanic_template/model/select_nn.py create mode 100644 rdagent/scenarios/kaggle/experiment/spaceship-titanic_template/model/select_randomforest.py create mode 100644 rdagent/scenarios/kaggle/experiment/spaceship-titanic_template/model/select_xgboost.py diff --git a/rdagent/app/kaggle/conf.py b/rdagent/app/kaggle/conf.py index da6c856d..b0797643 100644 --- a/rdagent/app/kaggle/conf.py +++ b/rdagent/app/kaggle/conf.py @@ -32,6 +32,9 @@ class KaggleBasePropSetting(BasePropSetting): feature_coder: str = "rdagent.scenarios.kaggle.developer.coder.KGFactorCoSTEER" """Feature Coder class""" + model_feature_selection_coder: str = "rdagent.scenarios.kaggle.developer.coder.KGModelFeatureSelectionCoder" + """Model Feature Selection Coder class""" + model_coder: str = "rdagent.scenarios.kaggle.developer.coder.KGModelCoSTEER" """Model Coder class""" @@ -57,8 +60,6 @@ class KaggleBasePropSetting(BasePropSetting): if_action_choosing_based_on_UCB: bool = False - if_using_feature_selection: bool = False - if_using_graph_rag: bool = False if_using_vector_rag: bool = False diff --git a/rdagent/app/kaggle/loop.py b/rdagent/app/kaggle/loop.py index 901040ea..fad12776 100644 --- a/rdagent/app/kaggle/loop.py +++ b/rdagent/app/kaggle/loop.py @@ -23,6 +23,7 @@ from rdagent.scenarios.kaggle.kaggle_crawler import download_data from rdagent.scenarios.kaggle.proposal.proposal import ( KG_ACTION_FEATURE_ENGINEERING, KG_ACTION_FEATURE_PROCESSING, + KG_ACTION_MODEL_FEATURE_SELECTION, KGTrace, ) @@ -49,6 +50,10 @@ class KaggleRDLoop(RDLoop): self.feature_coder: Developer = import_class(PROP_SETTING.feature_coder)(scen) logger.log_object(self.feature_coder, tag="feature coder") + self.model_feature_selection_coder: Developer = import_class(PROP_SETTING.model_feature_selection_coder)( + scen + ) + logger.log_object(self.model_feature_selection_coder, tag="model feature selection coder") self.model_coder: Developer = import_class(PROP_SETTING.model_coder)(scen) logger.log_object(self.model_coder, tag="model coder") @@ -67,6 +72,8 @@ class KaggleRDLoop(RDLoop): with logger.tag("d"): # develop if prev_out["propose"].action in [KG_ACTION_FEATURE_ENGINEERING, KG_ACTION_FEATURE_PROCESSING]: exp = self.feature_coder.develop(prev_out["exp_gen"]) + elif prev_out["propose"].action == KG_ACTION_MODEL_FEATURE_SELECTION: + exp = self.model_feature_selection_coder.develop(prev_out["exp_gen"]) else: exp = self.model_coder.develop(prev_out["exp_gen"]) logger.log_object(exp.sub_workspace_list, tag="coder result") diff --git a/rdagent/components/coder/model_coder/CoSTEER/evolving_strategy.py b/rdagent/components/coder/model_coder/CoSTEER/evolving_strategy.py index 82476f01..6d3eee5a 100644 --- a/rdagent/components/coder/model_coder/CoSTEER/evolving_strategy.py +++ b/rdagent/components/coder/model_coder/CoSTEER/evolving_strategy.py @@ -21,6 +21,7 @@ from rdagent.core.evolving_framework import EvolvingStrategy from rdagent.core.prompts import Prompts from rdagent.core.utils import multiprocessing_wrapper from rdagent.oai.llm_utils import APIBackend +from rdagent.scenarios.kaggle.experiment.kaggle_experiment import KG_MODEL_MAPPING coder_prompts = Prompts(file_path=Path(__file__).parent.parent / "prompts.yaml") @@ -41,14 +42,8 @@ class ModelCoderEvolvingStrategy(EvolvingStrategy): current_code = "" sota_exp_code_dict = current_exp.based_experiments[-1].experiment_workspace.code_dict if target_task.version == 2: - model_file_mapping = { - "XGBoost": "model/model_xgboost.py", - "RandomForest": "model/model_randomforest.py", - "LightGBM": "model/model_lightgbm.py", - "NN": "model/model_nn.py", - } - if model_type in model_file_mapping: - current_code = sota_exp_code_dict.get(model_file_mapping[model_type], None) + if model_type in KG_MODEL_MAPPING: + current_code = sota_exp_code_dict.get(KG_MODEL_MAPPING[model_type], None) elif "model.py" in sota_exp_code_dict: current_code = sota_exp_code_dict["model.py"] else: diff --git a/rdagent/components/coder/model_coder/model_execute_template_v2.txt b/rdagent/components/coder/model_coder/model_execute_template_v2.txt index 40fbffdb..f16be57a 100644 --- a/rdagent/components/coder/model_coder/model_execute_template_v2.txt +++ b/rdagent/components/coder/model_coder/model_execute_template_v2.txt @@ -4,7 +4,7 @@ import pickle import numpy as np import pandas as pd import torch -from model import fit, predict, select +from model import fit, predict train_X = pd.DataFrame(np.random.randn(8, 30), columns=[f"{i}" for i in range(30)]) train_y = pd.Series(np.random.randint(0, 2, 8)) diff --git a/rdagent/components/proposal/prompts.yaml b/rdagent/components/proposal/prompts.yaml index e5145c9c..24c3a048 100644 --- a/rdagent/components/proposal/prompts.yaml +++ b/rdagent/components/proposal/prompts.yaml @@ -12,7 +12,7 @@ hypothesis_gen: {{ hypothesis_output_format }} user_prompt: |- - {% if hypothesis_and_feedback|length == 0 %} It is the first round of hypothesis generation. The user has no hypothesis on this scenario yet. + {% if hypothesis_and_feedback|length == 0 %}It is the first round of hypothesis generation. The user has no hypothesis on this scenario yet. {% else %}It is not the first round, the user has made several hypothesis on this scenario and did several evaluation on them. The former hypothesis and the corresponding feedbacks are as follows (focus on the last one & the new hypothesis that it provides and reasoning to see if you agree): {{ hypothesis_and_feedback }} diff --git a/rdagent/scenarios/kaggle/developer/coder.py b/rdagent/scenarios/kaggle/developer/coder.py index 07ef97b6..1aa0ead1 100644 --- a/rdagent/scenarios/kaggle/developer/coder.py +++ b/rdagent/scenarios/kaggle/developer/coder.py @@ -1,5 +1,72 @@ +import json +from pathlib import Path + +from jinja2 import Environment, StrictUndefined + from rdagent.components.coder.factor_coder.CoSTEER import FactorCoSTEER from rdagent.components.coder.model_coder.CoSTEER import ModelCoSTEER +from rdagent.core.developer import Developer +from rdagent.core.prompts import Prompts +from rdagent.oai.llm_utils import APIBackend +from rdagent.scenarios.kaggle.experiment.kaggle_experiment import ( + KG_SELECT_MAPPING, + KGModelExperiment, +) KGModelCoSTEER = ModelCoSTEER KGFactorCoSTEER = FactorCoSTEER + +prompt_dict = Prompts(file_path=Path(__file__).parent.parent / "prompts.yaml") + +DEFAULT_SELECTION_CODE = """ +import pandas as pd +def select(X: pd.DataFrame) -> pd.DataFrame: + \""" + Select relevant features. To be used in fit & predict function. + \""" + if X.columns.nlevels == 1: + return X + {% if feature_index_list is not none %} + X = X.loc[:, X.columns.levels[0][{{feature_index_list}}].tolist()] + {% endif %} + X.columns = ["_".join(str(col)).strip() for col in X.columns.values] + return X +""" + + +class KGModelFeatureSelectionCoder(Developer[KGModelExperiment]): + def develop(self, exp: KGModelExperiment) -> KGModelExperiment: + target_model_type = exp.sub_tasks[0].model_type + assert target_model_type in KG_SELECT_MAPPING + if len(exp.experiment_workspace.data_description) == 1: + code = ( + Environment(undefined=StrictUndefined) + .from_string(DEFAULT_SELECTION_CODE) + .render(feature_index_list=None) + ) + else: + system_prompt = ( + Environment(undefined=StrictUndefined) + .from_string(prompt_dict["model_feature_selection"]["system"]) + .render(scenario=self.scen.get_scenario_all_desc(), model_type=exp.sub_tasks[0].model_type) + ) + user_prompt = ( + Environment(undefined=StrictUndefined) + .from_string(prompt_dict["model_feature_selection"]["user"]) + .render(feature_groups=[desc[0] for desc in exp.experiment_workspace.data_description]) + ) + + chosen_index = json.loads( + APIBackend().build_messages_and_create_chat_completion( + user_prompt=user_prompt, system_prompt=system_prompt, json_mode=True + ) + ).get("Selected Group Index", [i + 1 for i in range(len(exp.experiment_workspace.data_description))]) + chosen_index_to_list_index = [i - 1 for i in chosen_index] + + code = ( + Environment(undefined=StrictUndefined) + .from_string(DEFAULT_SELECTION_CODE) + .render(feature_index_list=chosen_index_to_list_index) + ) + exp.experiment_workspace.inject_code(**{KG_SELECT_MAPPING[target_model_type]: code}) + return exp diff --git a/rdagent/scenarios/kaggle/developer/feedback.py b/rdagent/scenarios/kaggle/developer/feedback.py index a36ab742..59a68830 100644 --- a/rdagent/scenarios/kaggle/developer/feedback.py +++ b/rdagent/scenarios/kaggle/developer/feedback.py @@ -117,9 +117,9 @@ class KGHypothesisExperiment2Feedback(HypothesisExperiment2Feedback): "last_hypothesis": trace.hist[-1][0] if trace.hist else None, "last_task_and_code": last_task_and_code, "last_result": trace.hist[-1][1].result if trace.hist else None, - "sota_task_and_code": exp.based_experiments[-1].experiment_workspace.data_description - if exp.based_experiments - else None, + "sota_task_and_code": ( + exp.based_experiments[-1].experiment_workspace.data_description if exp.based_experiments else None + ), "sota_result": exp.based_experiments[-1].result if exp.based_experiments else None, "hypothesis": hypothesis, "exp": exp, @@ -150,6 +150,7 @@ class KGHypothesisExperiment2Feedback(HypothesisExperiment2Feedback): decision = convert2bool(response_json.get("Replace Best Result", "no")) experiment_feedback = { + "current_competition": self.scen.get_competition_full_desc(), "hypothesis_text": hypothesis_text, "current_result": current_result, "model_code": model_code, @@ -163,7 +164,7 @@ class KGHypothesisExperiment2Feedback(HypothesisExperiment2Feedback): self.scen.vector_base.add_experience_to_vector_base(experiment_feedback) self.scen.vector_base.save() elif self.scen.if_using_graph_rag: - trace.knowledge_base.load_from_documents([experiment_feedback], self.scen) + trace.knowledge_base.add_document(experiment_feedback, self.scen) return HypothesisFeedback( observations=observations, diff --git a/rdagent/scenarios/kaggle/developer/runner.py b/rdagent/scenarios/kaggle/developer/runner.py index 95e2a9f4..74740a42 100644 --- a/rdagent/scenarios/kaggle/developer/runner.py +++ b/rdagent/scenarios/kaggle/developer/runner.py @@ -3,17 +3,12 @@ import pickle import shutil from pathlib import Path -from jinja2 import Environment, StrictUndefined - -from rdagent.app.kaggle.conf import KAGGLE_IMPLEMENT_SETTING -from rdagent.components.coder.factor_coder.factor import FactorTask -from rdagent.components.coder.model_coder.model import ModelTask from rdagent.components.runner import CachedRunner from rdagent.components.runner.conf import RUNNER_SETTINGS from rdagent.core.exception import CoderError, FactorEmptyError, ModelEmptyError from rdagent.core.experiment import ASpecificExp from rdagent.core.prompts import Prompts -from rdagent.oai.llm_utils import APIBackend, md5_hash +from rdagent.oai.llm_utils import md5_hash from rdagent.scenarios.kaggle.experiment.kaggle_experiment import ( KGFactorExperiment, KGModelExperiment, @@ -32,48 +27,6 @@ class KGCachedRunner(CachedRunner[ASpecificExp]): codes = "\n".join(codes) return md5_hash(codes) - def extract_model_task_from_code(self, code: str) -> str: - sys_prompt = ( - Environment(undefined=StrictUndefined) - .from_string(prompt_dict["extract_model_task_from_code"]["system"]) - .render() - ) - - user_prompt = ( - Environment(undefined=StrictUndefined) - .from_string(prompt_dict["extract_model_task_from_code"]["user"]) - .render(file_content=code) - ) - - model_task_description = APIBackend().build_messages_and_create_chat_completion( - user_prompt=user_prompt, - system_prompt=sys_prompt, - json_mode=True, - ) - - try: - response_json_analysis = json.loads(model_task_description) - task_desc = f"""name: {response_json_analysis['name']} - description: {response_json_analysis['description']} - """ - task_desc += ( - f"formulation: {response_json_analysis['formulation']}\n" - if response_json_analysis.get("formulation") - else "" - ) - task_desc += f"architecture: {response_json_analysis['architecture']}\n" - task_desc += ( - f"variables: {json.dumps(response_json_analysis['variables'], indent=4)}\n" - if response_json_analysis.get("variables") - else "" - ) - task_desc += f"hyperparameters: {json.dumps(response_json_analysis['hyperparameters'], indent=4)}\n" - task_desc += f"model_type: {response_json_analysis['model_type']}\n" - except json.JSONDecodeError: - task_desc = "Failed to parse LLM's response as JSON" - - return task_desc - def init_develop(self, exp: KGFactorExperiment | KGModelExperiment) -> KGFactorExperiment | KGModelExperiment: """ For the initial development, the experiment serves as a benchmark for feature engineering. @@ -89,39 +42,6 @@ class KGCachedRunner(CachedRunner[ASpecificExp]): result = exp.experiment_workspace.execute(run_env=env_to_use) exp.result = result - sub_task = FactorTask( - factor_name="original features", factor_description="here is the original features", factor_formulation="" - ) - - org_data_path = ( - Path(KAGGLE_IMPLEMENT_SETTING.local_data_path) / KAGGLE_IMPLEMENT_SETTING.competition / "X_valid.pkl" - ) - with open(org_data_path, "rb") as f: - org_data = pickle.load(f) - feature_shape = org_data.shape[-1] - exp.experiment_workspace.data_description.append((sub_task.get_task_information(), feature_shape)) - - model_map = { - "XGBoost": "model_xgboost.py", - "RandomForest": "model_randomforest.py", - "LightGBM": "model_lightgbm.py", - "NN": "model_nn.py", - } - - workspace_path = exp.experiment_workspace.workspace_path / "model" - - for model_name, model_file in model_map.items(): - model_file_path = workspace_path / model_file - - if model_file_path.exists(): - model_description = ( - self.extract_model_task_from_code(model_file_path.read_text()) - + f"""code: {model_file_path.read_text()}""" - ) - else: - model_description = "" - - exp.experiment_workspace.model_description[model_name] = model_description if RUNNER_SETTINGS.cache_result: self.dump_cache_result(exp, result) @@ -135,17 +55,15 @@ class KGModelRunner(KGCachedRunner[KGModelExperiment]): exp.based_experiments[-1] = self.init_develop(exp.based_experiments[-1]) sub_ws = exp.sub_workspace_list[0] - # TODO: There's a possibility of generating a hybrid model (lightgbm + xgboost), which results in having two items in the model_type list. - model_type = sub_ws.target_task.model_type + if sub_ws is not None: + # TODO: There's a possibility of generating a hybrid model (lightgbm + xgboost), which results in having two items in the model_type list. + model_type = sub_ws.target_task.model_type - if sub_ws.code_dict == {}: - raise ModelEmptyError("No model is implemented.") - else: - model_file_name = f"model/model_{model_type.lower()}.py" - exp.experiment_workspace.inject_code(**{model_file_name: sub_ws.code_dict["model.py"]}) - - model_description = sub_ws.target_task.get_task_information() - exp.experiment_workspace.model_description[model_type] = model_description + if sub_ws.code_dict == {}: + raise ModelEmptyError("No model is implemented.") + else: + model_file_name = f"model/model_{model_type.lower()}.py" + exp.experiment_workspace.inject_code(**{model_file_name: sub_ws.code_dict["model.py"]}) if RUNNER_SETTINGS.cache_result: cache_hit, result = self.get_cache_result(exp) diff --git a/rdagent/scenarios/kaggle/experiment/feedback-prize-english-language-learning_template/fea_share_preprocess.py b/rdagent/scenarios/kaggle/experiment/feedback-prize-english-language-learning_template/fea_share_preprocess.py index 49f9d3bf..13476c80 100644 --- a/rdagent/scenarios/kaggle/experiment/feedback-prize-english-language-learning_template/fea_share_preprocess.py +++ b/rdagent/scenarios/kaggle/experiment/feedback-prize-english-language-learning_template/fea_share_preprocess.py @@ -3,7 +3,6 @@ import re import numpy as np # linear algebra import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv) -from sklearn.feature_extraction.text import TfidfVectorizer from sklearn.model_selection import train_test_split diff --git a/rdagent/scenarios/kaggle/experiment/feedback-prize-english-language-learning_template/model/model_randomforest.py b/rdagent/scenarios/kaggle/experiment/feedback-prize-english-language-learning_template/model/model_randomforest.py index 0adc37d6..bcd206a9 100644 --- a/rdagent/scenarios/kaggle/experiment/feedback-prize-english-language-learning_template/model/model_randomforest.py +++ b/rdagent/scenarios/kaggle/experiment/feedback-prize-english-language-learning_template/model/model_randomforest.py @@ -3,14 +3,6 @@ import pandas as pd from sklearn.ensemble import RandomForestRegressor -def select(X: pd.DataFrame) -> pd.DataFrame: - """ - Select relevant features. To be used in fit & predict function. - """ - # For now, we assume all features are relevant. This can be expanded to feature selection logic. - return X - - def fit(X_train: pd.DataFrame, y_train: pd.Series, X_valid: pd.DataFrame, y_valid: pd.Series): """ Define and train the Random Forest model. Merge feature selection into the pipeline. @@ -18,11 +10,8 @@ def fit(X_train: pd.DataFrame, y_train: pd.Series, X_valid: pd.DataFrame, y_vali # Initialize the Random Forest model model = RandomForestRegressor(n_estimators=100, random_state=32, n_jobs=-1) - # Select features (if any feature selection is needed) - X_train_selected = select(X_train) - # Fit the model - model.fit(X_train_selected, y_train) + model.fit(X_train, y_train) return model @@ -31,10 +20,7 @@ def predict(model, X): """ Keep feature selection's consistency and make predictions. """ - # Select features (if any feature selection is needed) - X_selected = select(X) - # Predict using the trained model - y_pred = model.predict(X_selected) + y_pred = model.predict(X) return y_pred diff --git a/rdagent/scenarios/kaggle/experiment/feedback-prize-english-language-learning_template/model/model_xgboost.py b/rdagent/scenarios/kaggle/experiment/feedback-prize-english-language-learning_template/model/model_xgboost.py index 3a433595..6ff4c857 100644 --- a/rdagent/scenarios/kaggle/experiment/feedback-prize-english-language-learning_template/model/model_xgboost.py +++ b/rdagent/scenarios/kaggle/experiment/feedback-prize-english-language-learning_template/model/model_xgboost.py @@ -7,11 +7,6 @@ import xgboost as xgb from sklearn.multioutput import MultiOutputRegressor -def select(X: pd.DataFrame) -> pd.DataFrame: - # Ignore feature selection logic - return X - - def is_sparse_df(df: pd.DataFrame) -> bool: # 检查 DataFrame 中的每一列是否为稀疏类型 return any(isinstance(dtype, pd.SparseDtype) for dtype in df.dtypes) @@ -19,8 +14,6 @@ def is_sparse_df(df: pd.DataFrame) -> bool: def fit(X_train: pd.DataFrame, y_train: pd.DataFrame, X_valid: pd.DataFrame, y_valid: pd.DataFrame): """Define and train the model. Merge feature_select""" - X_train = select(X_train) - xgb_estimator = xgb.XGBRegressor( n_estimators=500, random_state=0, objective="reg:squarederror", tree_method="hist", device="cuda" ) @@ -38,7 +31,6 @@ def predict(model, X_test): """ Keep feature select's consistency. """ - X_test = select(X_test) if is_sparse_df(X_test): X_test = X_test.sparse.to_coo() y_pred = model.predict(X_test) diff --git a/rdagent/scenarios/kaggle/experiment/feedback-prize-english-language-learning_template/model/select_randomforest.py b/rdagent/scenarios/kaggle/experiment/feedback-prize-english-language-learning_template/model/select_randomforest.py new file mode 100644 index 00000000..d2a15dee --- /dev/null +++ b/rdagent/scenarios/kaggle/experiment/feedback-prize-english-language-learning_template/model/select_randomforest.py @@ -0,0 +1,12 @@ +import pandas as pd + + +def select(X: pd.DataFrame) -> pd.DataFrame: + """ + Select relevant features. To be used in fit & predict function. + """ + # For now, we assume all features are relevant. This can be expanded to feature selection logic. + if X.columns.nlevels == 1: + return X + X.columns = ["_".join(str(col)).strip() for col in X.columns.values] + return X diff --git a/rdagent/scenarios/kaggle/experiment/feedback-prize-english-language-learning_template/model/select_xgboost.py b/rdagent/scenarios/kaggle/experiment/feedback-prize-english-language-learning_template/model/select_xgboost.py new file mode 100644 index 00000000..d2a15dee --- /dev/null +++ b/rdagent/scenarios/kaggle/experiment/feedback-prize-english-language-learning_template/model/select_xgboost.py @@ -0,0 +1,12 @@ +import pandas as pd + + +def select(X: pd.DataFrame) -> pd.DataFrame: + """ + Select relevant features. To be used in fit & predict function. + """ + # For now, we assume all features are relevant. This can be expanded to feature selection logic. + if X.columns.nlevels == 1: + return X + X.columns = ["_".join(str(col)).strip() for col in X.columns.values] + return X diff --git a/rdagent/scenarios/kaggle/experiment/feedback-prize-english-language-learning_template/train.py b/rdagent/scenarios/kaggle/experiment/feedback-prize-english-language-learning_template/train.py index 0b336596..ddb4eff5 100644 --- a/rdagent/scenarios/kaggle/experiment/feedback-prize-english-language-learning_template/train.py +++ b/rdagent/scenarios/kaggle/experiment/feedback-prize-english-language-learning_template/train.py @@ -34,53 +34,46 @@ for f in DIRNAME.glob("feature/feat*.py"): X_valid_f = cls.transform(X_valid) X_test_f = cls.transform(X_test) - X_train_l.append(X_train_f) - X_valid_l.append(X_valid_f) - X_test_l.append(X_test_f) + if X_train_f.shape[-1] == X_valid_f.shape[-1] and X_train_f.shape[-1] == X_test_f.shape[-1]: + X_train_l.append(X_train_f) + X_valid_l.append(X_valid_f) + X_test_l.append(X_test_f) X_train = pd.concat(X_train_l, axis=1, keys=[f"feature_{i}" for i in range(len(X_train_l))]) X_valid = pd.concat(X_valid_l, axis=1, keys=[f"feature_{i}" for i in range(len(X_valid_l))]) X_test = pd.concat(X_test_l, axis=1, keys=[f"feature_{i}" for i in range(len(X_test_l))]) +print(X_train.shape, X_valid.shape, X_test.shape) # 3) Train the model -def flatten_columns(df: pd.DataFrame) -> pd.DataFrame: - """ - Flatten the columns of a DataFrame with MultiIndex columns, - for (feature_0, a), (feature_0, b) -> feature_0_a, feature_0_b - """ - if df.columns.nlevels == 1: - return df - df.columns = ["_".join(str(col)).strip() for col in df.columns.values] - return df - - -X_train = flatten_columns(X_train) -X_valid = flatten_columns(X_valid) -X_test = flatten_columns(X_test) - - -model_l = [] # list[tuple[model, predict_func]] +model_l = [] # list[tuple[model, predict_func,]] for f in DIRNAME.glob("model/model*.py"): + select_python_path = f.with_name(f.stem.replace("model", "select") + f.suffix) + select_m = import_module_from_path(select_python_path.stem, select_python_path) + X_train_selected = select_m.select(X_train.copy()) + X_valid_selected = select_m.select(X_valid.copy()) + m = import_module_from_path(f.stem, f) - model_l.append((m.fit(X_train, y_train, X_valid, y_valid), m.predict)) + model_l.append((m.fit(X_train_selected, y_train, X_valid_selected, y_valid), m.predict)) # 4) Evaluate the model on the validation set -y_valid_pred_l = [] metrics_all = [] for model, predict_func in model_l: - y_valid_pred = predict_func(model, X_valid) - y_valid_pred_l.append(y_valid_pred) + X_valid_selected = select_m.select(X_valid.copy()) + y_valid_pred = predict_func(model, X_valid_selected) metrics = MCRMSE(y_valid, y_valid_pred) print(f"MCRMSE on valid set: {metrics}") metrics_all.append(metrics) +# 5) Save the validation accuracy min_index = np.argmin(metrics_all) pd.Series(data=[metrics_all[min_index]], index=["MCRMSE"]).to_csv("submission_score.csv") -y_test_pred = model_l[min_index][1](model_l[min_index][0], X_test) - +# 6) Make predictions on the test set and save them +X_test_selected = select_m.select(X_test.copy()) +y_test_pred = model_l[min_index][1](model_l[min_index][0], X_test_selected) +# 7) Submit predictions for the test set submission_result = pd.read_csv("/kaggle/input/sample_submission.csv") submission_result["cohesion"] = y_test_pred[:, 0] submission_result["syntax"] = y_test_pred[:, 1] diff --git a/rdagent/scenarios/kaggle/experiment/forest-cover-type-prediction_template/model/model_catboost.py b/rdagent/scenarios/kaggle/experiment/forest-cover-type-prediction_template/model/model_catboost.py deleted file mode 100644 index a4420efa..00000000 --- a/rdagent/scenarios/kaggle/experiment/forest-cover-type-prediction_template/model/model_catboost.py +++ /dev/null @@ -1,29 +0,0 @@ -import pandas as pd -from catboost import CatBoostClassifier - - -def fit(X_train: pd.DataFrame, y_train: pd.Series, X_valid: pd.DataFrame, y_valid: pd.Series): - # Define CatBoost parameters - cat_params = { - "iterations": 5000, - "learning_rate": 0.03, - "od_wait": 1000, - "depth": 7, - "task_type": "GPU", - "l2_leaf_reg": 3, - "eval_metric": "Accuracy", - "devices": "0", - "verbose": 1000, - } - - # Initialize and train the CatBoost model - model = CatBoostClassifier(**cat_params) - model.fit(X_train, y_train, eval_set=(X_valid, y_valid)) - - return model - - -def predict(model, X: pd.DataFrame): - # Predict using the trained model - y_pred = model.predict(X) - return y_pred.reshape(-1, 1) diff --git a/rdagent/scenarios/kaggle/experiment/forest-cover-type-prediction_template/model/model_dnn.py b/rdagent/scenarios/kaggle/experiment/forest-cover-type-prediction_template/model/model_nn.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/forest-cover-type-prediction_template/model/model_dnn.py rename to rdagent/scenarios/kaggle/experiment/forest-cover-type-prediction_template/model/model_nn.py diff --git a/rdagent/scenarios/kaggle/experiment/forest-cover-type-prediction_template/model/model_randomforest.py b/rdagent/scenarios/kaggle/experiment/forest-cover-type-prediction_template/model/model_randomforest.py index 18019066..7ed01d02 100644 --- a/rdagent/scenarios/kaggle/experiment/forest-cover-type-prediction_template/model/model_randomforest.py +++ b/rdagent/scenarios/kaggle/experiment/forest-cover-type-prediction_template/model/model_randomforest.py @@ -10,14 +10,6 @@ from sklearn.ensemble import RandomForestClassifier from sklearn.metrics import accuracy_score -def select(X: pd.DataFrame) -> pd.DataFrame: - """ - Select relevant features. To be used in fit & predict function. - """ - # For now, we assume all features are relevant. This can be expanded to feature selection logic. - return X - - def fit(X_train: pd.DataFrame, y_train: pd.Series, X_valid: pd.DataFrame, y_valid: pd.Series): """ Define and train the Random Forest model. Merge feature selection into the pipeline. @@ -25,15 +17,11 @@ def fit(X_train: pd.DataFrame, y_train: pd.Series, X_valid: pd.DataFrame, y_vali # Initialize the Random Forest model model = RandomForestClassifier(n_estimators=200, random_state=32, n_jobs=-1) - # Select features (if any feature selection is needed) - X_train_selected = select(X_train) - X_valid_selected = select(X_valid) - # Fit the model - model.fit(X_train_selected, y_train) + model.fit(X_train, y_train) # Validate the model - y_valid_pred = model.predict(X_valid_selected) + y_valid_pred = model.predict(X_valid) accuracy = accuracy_score(y_valid, y_valid_pred) print(f"Validation Accuracy: {accuracy:.4f}") @@ -44,10 +32,7 @@ def predict(model, X): """ Keep feature selection's consistency and make predictions. """ - # Select features (if any feature selection is needed) - X_selected = select(X) - # Predict using the trained model - y_pred = model.predict(X_selected) + y_pred = model.predict(X) return y_pred.reshape(-1, 1) diff --git a/rdagent/scenarios/kaggle/experiment/forest-cover-type-prediction_template/model/model_xgboost.py b/rdagent/scenarios/kaggle/experiment/forest-cover-type-prediction_template/model/model_xgboost.py index d9720c89..0d665c73 100644 --- a/rdagent/scenarios/kaggle/experiment/forest-cover-type-prediction_template/model/model_xgboost.py +++ b/rdagent/scenarios/kaggle/experiment/forest-cover-type-prediction_template/model/model_xgboost.py @@ -6,15 +6,8 @@ import pandas as pd import xgboost as xgb -def select(X: pd.DataFrame) -> pd.DataFrame: - # Ignore feature selection logic - return X - - def fit(X_train: pd.DataFrame, y_train: pd.DataFrame, X_valid: pd.DataFrame, y_valid: pd.DataFrame): """Define and train the model. Merge feature_select""" - X_train = select(X_train) - X_valid = select(X_valid) dtrain = xgb.DMatrix(X_train, label=y_train) dvalid = xgb.DMatrix(X_valid, label=y_valid) @@ -37,7 +30,6 @@ def predict(model, X): """ Keep feature select's consistency. """ - X = select(X) dtest = xgb.DMatrix(X) y_pred = model.predict(dtest) return y_pred.astype(int).reshape(-1, 1) diff --git a/rdagent/scenarios/kaggle/experiment/forest-cover-type-prediction_template/model/select_nn.py b/rdagent/scenarios/kaggle/experiment/forest-cover-type-prediction_template/model/select_nn.py new file mode 100644 index 00000000..d2a15dee --- /dev/null +++ b/rdagent/scenarios/kaggle/experiment/forest-cover-type-prediction_template/model/select_nn.py @@ -0,0 +1,12 @@ +import pandas as pd + + +def select(X: pd.DataFrame) -> pd.DataFrame: + """ + Select relevant features. To be used in fit & predict function. + """ + # For now, we assume all features are relevant. This can be expanded to feature selection logic. + if X.columns.nlevels == 1: + return X + X.columns = ["_".join(str(col)).strip() for col in X.columns.values] + return X diff --git a/rdagent/scenarios/kaggle/experiment/forest-cover-type-prediction_template/model/select_randomforest.py b/rdagent/scenarios/kaggle/experiment/forest-cover-type-prediction_template/model/select_randomforest.py new file mode 100644 index 00000000..d2a15dee --- /dev/null +++ b/rdagent/scenarios/kaggle/experiment/forest-cover-type-prediction_template/model/select_randomforest.py @@ -0,0 +1,12 @@ +import pandas as pd + + +def select(X: pd.DataFrame) -> pd.DataFrame: + """ + Select relevant features. To be used in fit & predict function. + """ + # For now, we assume all features are relevant. This can be expanded to feature selection logic. + if X.columns.nlevels == 1: + return X + X.columns = ["_".join(str(col)).strip() for col in X.columns.values] + return X diff --git a/rdagent/scenarios/kaggle/experiment/forest-cover-type-prediction_template/model/select_xgboost.py b/rdagent/scenarios/kaggle/experiment/forest-cover-type-prediction_template/model/select_xgboost.py new file mode 100644 index 00000000..d2a15dee --- /dev/null +++ b/rdagent/scenarios/kaggle/experiment/forest-cover-type-prediction_template/model/select_xgboost.py @@ -0,0 +1,12 @@ +import pandas as pd + + +def select(X: pd.DataFrame) -> pd.DataFrame: + """ + Select relevant features. To be used in fit & predict function. + """ + # For now, we assume all features are relevant. This can be expanded to feature selection logic. + if X.columns.nlevels == 1: + return X + X.columns = ["_".join(str(col)).strip() for col in X.columns.values] + return X diff --git a/rdagent/scenarios/kaggle/experiment/forest-cover-type-prediction_template/train.py b/rdagent/scenarios/kaggle/experiment/forest-cover-type-prediction_template/train.py index b6e9841d..1bacd3d0 100644 --- a/rdagent/scenarios/kaggle/experiment/forest-cover-type-prediction_template/train.py +++ b/rdagent/scenarios/kaggle/experiment/forest-cover-type-prediction_template/train.py @@ -4,11 +4,8 @@ from pathlib import Path import numpy as np import pandas as pd -from scipy import stats -from sklearn.impute import SimpleImputer -from sklearn.metrics import accuracy_score -from sklearn.model_selection import KFold -from sklearn.preprocessing import StandardScaler +from fea_share_preprocess import clean_and_impute_data, preprocess_script +from sklearn.metrics import accuracy_score, matthews_corrcoef # Set random seed for reproducibility SEED = 42 @@ -17,6 +14,12 @@ np.random.seed(SEED) DIRNAME = Path(__file__).absolute().resolve().parent +def compute_metrics_for_classification(y_true, y_pred): + """Compute MCC for classification.""" + mcc = matthews_corrcoef(y_true, y_pred) + return mcc + + def import_module_from_path(module_name, module_path): spec = importlib.util.spec_from_file_location(module_name, module_path) module = importlib.util.module_from_spec(spec) @@ -25,117 +28,63 @@ def import_module_from_path(module_name, module_path): # 1) Preprocess the data -data_df = pd.read_csv("/kaggle/input/train.csv") -data_df = data_df.drop(["Id"], axis=1) +X_train, X_valid, y_train, y_valid, X_test, ids = preprocess_script() -X_train = data_df.drop(["Cover_Type"], axis=1) -y_train = data_df["Cover_Type"] - 1 +# 2) Auto feature engineering +X_train_l, X_valid_l = [], [] +X_test_l = [] -submission_df = pd.read_csv("/kaggle/input/test.csv") -ids = submission_df["Id"] -X_test = submission_df.drop(["Id"], axis=1) - -# Set up KFold -kf = KFold(n_splits=5, shuffle=True, random_state=SEED) - -# Store results -accuracies = [] -y_test_pred_l = [] -scaler = StandardScaler() - -# 3) Train and evaluate using KFold -fold_number = 1 -for train_index, valid_index in kf.split(X_train): - print(f"Starting fold {fold_number}...") - - X_train_l, X_valid_l, X_test_l = [], [], [] # Reset feature lists for each fold - X_tr, X_val = X_train.iloc[train_index], X_train.iloc[valid_index] - y_tr, y_val = y_train.iloc[train_index], y_train.iloc[valid_index] - X_te = X_test - - # Feature engineering - for f in DIRNAME.glob("feature/feat*.py"): - cls = import_module_from_path(f.stem, f).feature_engineering_cls() - cls.fit(X_tr) - X_train_f = cls.transform(X_tr) - X_valid_f = cls.transform(X_val) - X_test_f = cls.transform(X_te) +for f in DIRNAME.glob("feature/feat*.py"): + cls = import_module_from_path(f.stem, f).feature_engineering_cls() + cls.fit(X_train) + X_train_f = cls.transform(X_train) + X_valid_f = cls.transform(X_valid) + X_test_f = cls.transform(X_test) + if X_train_f.shape[-1] == X_valid_f.shape[-1] and X_train_f.shape[-1] == X_test_f.shape[-1]: X_train_l.append(X_train_f) X_valid_l.append(X_valid_f) X_test_l.append(X_test_f) - X_tr = pd.concat(X_train_l, axis=1, keys=[f"feature_{i}" for i in range(len(X_train_l))]) - X_val = pd.concat(X_valid_l, axis=1, keys=[f"feature_{i}" for i in range(len(X_valid_l))]) - X_te = pd.concat(X_test_l, axis=1, keys=[f"feature_{i}" for i in range(len(X_test_l))]) +X_train = pd.concat(X_train_l, axis=1, keys=[f"feature_{i}" for i in range(len(X_train_l))]) +X_valid = pd.concat(X_valid_l, axis=1, keys=[f"feature_{i}" for i in range(len(X_valid_l))]) +X_test = pd.concat(X_test_l, axis=1, keys=[f"feature_{i}" for i in range(len(X_test_l))]) - print("Shape of X_tr: ", X_tr.shape, " Shape of X_val: ", X_val.shape, " Shape of X_te: ", X_te.shape) +print(X_train.shape, X_valid.shape, X_test.shape) - # Replace inf and -inf with NaN - X_tr.replace([np.inf, -np.inf], np.nan, inplace=True) - X_val.replace([np.inf, -np.inf], np.nan, inplace=True) - X_te.replace([np.inf, -np.inf], np.nan, inplace=True) +# Handle inf and -inf values +X_train, X_valid, X_test = clean_and_impute_data(X_train, X_valid, X_test) - # Impute missing values - imputer = SimpleImputer(strategy="mean") - X_tr = pd.DataFrame(imputer.fit_transform(X_tr), columns=X_tr.columns) - X_val = pd.DataFrame(imputer.transform(X_val), columns=X_val.columns) - X_te = pd.DataFrame(imputer.transform(X_te), columns=X_te.columns) - # Standardize the data - X_tr = pd.DataFrame(scaler.fit_transform(X_tr), columns=X_tr.columns) - X_val = pd.DataFrame(scaler.transform(X_val), columns=X_val.columns) - X_te = pd.DataFrame(scaler.transform(X_te), columns=X_te.columns) +model_l = [] # list[tuple[model, predict_func]] +for f in DIRNAME.glob("model/model*.py"): + select_python_path = f.with_name(f.stem.replace("model", "select") + f.suffix) + select_m = import_module_from_path(select_python_path.stem, select_python_path) + X_train_selected = select_m.select(X_train.copy()) + X_valid_selected = select_m.select(X_valid.copy()) - # Remove duplicate columns - X_tr = X_tr.loc[:, ~X_tr.columns.duplicated()] - X_val = X_val.loc[:, ~X_val.columns.duplicated()] - X_te = X_te.loc[:, ~X_te.columns.duplicated()] + m = import_module_from_path(f.stem, f) + model_l.append((m.fit(X_train_selected, y_train, X_valid_selected, y_valid), m.predict, select_m)) - # Train the model - def flatten_columns(df: pd.DataFrame) -> pd.DataFrame: - """ - Flatten the columns of a DataFrame with MultiIndex columns, - for (feature_0, a), (feature_0, b) -> feature_0_a, feature_0_b - """ - if df.columns.nlevels == 1: - return df - df.columns = ["_".join(col).strip() for col in df.columns.values] - return df +# 4) Evaluate the model on the validation set +metrics_all = [] +for model, predict_func, select_m in model_l: + X_valid_selected = select_m.select(X_valid.copy()) + y_valid_pred = predict_func(model, X_valid_selected) + accuracy = accuracy_score(y_valid, y_valid_pred) + print(f"final accuracy on valid set: {accuracy}") + metrics_all.append(accuracy) - X_tr = flatten_columns(X_tr) - X_val = flatten_columns(X_val) - X_te = flatten_columns(X_te) +# 5) Save the validation accuracy +min_index = np.argmax(metrics_all) +pd.Series(data=[metrics_all[min_index]], index=["multi-class accuracy"]).to_csv("submission_score.csv") - model_l = [] # list[tuple[model, predict_func]] - for f in DIRNAME.glob("model/model*.py"): - m = import_module_from_path(f.stem, f) - model_l.append((m.fit(X_tr, y_tr, X_val, y_val), m.predict)) +# 6) Make predictions on the test set and save them +X_test_selected = model_l[min_index][2].select(X_test.copy()) +y_test_pred = model_l[min_index][1](model_l[min_index][0], X_test_selected).flatten() + 1 - # Evaluate the model on the validation set - y_valid_pred_l = [] - for model, predict_func in model_l: - y_valid_pred = predict_func(model, X_val) - y_valid_pred_l.append(y_valid_pred) - y_test_pred_l.append(predict_func(model, X_te)) - - # Majority vote ensemble - y_valid_pred_ensemble = stats.mode(y_valid_pred_l, axis=0)[0].flatten() - - # Compute metrics - accuracy = accuracy_score(y_val, y_valid_pred_ensemble) - accuracies.append(accuracy) - print(f"Fold {fold_number} accuracy: {accuracy}") - - fold_number += 1 - -# Print average accuracy -accuracy = np.mean(accuracies) -print(f"Average accuracy across folds: {accuracy}") -pd.Series(data=[accuracy], index=["multi-class accuracy"]).to_csv("submission_score.csv") - -y_test_pred = stats.mode(y_test_pred_l, axis=0)[0].flatten() + 1 +# 7) Submit predictions for the test set submission_result = pd.DataFrame(y_test_pred, columns=["Cover_Type"]) submission_result.insert(0, "Id", ids) diff --git a/rdagent/scenarios/kaggle/experiment/forest-cover-type-prediction_template/train_past.py b/rdagent/scenarios/kaggle/experiment/forest-cover-type-prediction_template/train_past.py index c2161ebf..64648690 100644 --- a/rdagent/scenarios/kaggle/experiment/forest-cover-type-prediction_template/train_past.py +++ b/rdagent/scenarios/kaggle/experiment/forest-cover-type-prediction_template/train_past.py @@ -1,12 +1,13 @@ import importlib.util import random +from collections import defaultdict from pathlib import Path import numpy as np import pandas as pd -from fea_share_preprocess import clean_and_impute_data, preprocess_script -from scipy import stats -from sklearn.metrics import accuracy_score, matthews_corrcoef +from sklearn.impute import SimpleImputer +from sklearn.metrics import accuracy_score +from sklearn.preprocessing import StandardScaler # Set random seed for reproducibility SEED = 42 @@ -15,19 +16,6 @@ np.random.seed(SEED) DIRNAME = Path(__file__).absolute().resolve().parent -# support various method for metrics calculation -def compute_metrics_for_classification(y_true, y_pred): - """Compute accuracy metric for classification.""" - accuracy = accuracy_score(y_true, y_pred) - return accuracy - - -def compute_metrics_for_classification(y_true, y_pred): - """Compute MCC for classification.""" - mcc = matthews_corrcoef(y_true, y_pred) - return mcc - - def import_module_from_path(module_name, module_path): spec = importlib.util.spec_from_file_location(module_name, module_path) module = importlib.util.module_from_spec(spec) @@ -36,81 +24,108 @@ def import_module_from_path(module_name, module_path): # 1) Preprocess the data -X_train, X_valid, y_train, y_valid, X_test, ids = preprocess_script() - -# 2) Auto feature engineering -X_train_l, X_valid_l = [], [] -X_test_l = [] - -for f in DIRNAME.glob("feature/feat*.py"): - cls = import_module_from_path(f.stem, f).feature_engineering_cls() - cls.fit(X_train) - X_train_f = cls.transform(X_train) - X_valid_f = cls.transform(X_valid) - X_test_f = cls.transform(X_test) - - X_train_l.append(X_train_f) - X_valid_l.append(X_valid_f) - X_test_l.append(X_test_f) - -X_train = pd.concat(X_train_l, axis=1, keys=[f"feature_{i}" for i in range(len(X_train_l))]) -X_valid = pd.concat(X_valid_l, axis=1, keys=[f"feature_{i}" for i in range(len(X_valid_l))]) -X_test = pd.concat(X_test_l, axis=1, keys=[f"feature_{i}" for i in range(len(X_test_l))]) - -print(X_train.shape, X_valid.shape, X_test.shape) - -# Handle inf and -inf values -X_train, X_valid, X_test = clean_and_impute_data(X_train, X_valid, X_test) +print("-1") +data_df = pd.read_csv("/kaggle/input/train.csv") +data_df = data_df.drop(["Id"], axis=1) +print("0") +X_train = data_df.drop(["Cover_Type"], axis=1) +y_train = data_df["Cover_Type"] - 1 +print("81") +submission_df = pd.read_csv("/kaggle/input/test.csv") +ids = submission_df["Id"] +X_test = submission_df.drop(["Id"], axis=1) -# 3) Train the model -def flatten_columns(df: pd.DataFrame) -> pd.DataFrame: - """ - Flatten the columns of a DataFrame with MultiIndex columns, - for (feature_0, a), (feature_0, b) -> feature_0_a, feature_0_b - """ - if df.columns.nlevels == 1: - return df - df.columns = ["_".join(col).strip() for col in df.columns.values] - return df - - -X_train = flatten_columns(X_train) -X_valid = flatten_columns(X_valid) -X_test = flatten_columns(X_test) - -model_l = [] # list[tuple[model, predict_func]] -for f in DIRNAME.glob("model/model*.py"): - m = import_module_from_path(f.stem, f) - model_l.append((m.fit(X_train, y_train, X_valid, y_valid), m.predict)) - -# 4) Evaluate the model on the validation set -y_valid_pred_l = [] -for model, predict_func in model_l: - y_valid_pred = predict_func(model, X_valid) - y_valid_pred_l.append(y_valid_pred) - print(y_valid_pred) - print(y_valid_pred.shape) - -# 5) Ensemble -# Majority vote ensemble -y_valid_pred_ensemble = stats.mode(y_valid_pred_l, axis=0)[0].flatten() - -# Compute metrics -accuracy = accuracy_score(y_valid, y_valid_pred_ensemble) -print(f"final accuracy on valid set: {accuracy}") - -# 6) Save the validation metrics -pd.Series(data=[accuracy], index=["multi-class accuracy"]).to_csv("submission_score.csv") - -# 7) Make predictions on the test set and save them +# Store results +accuracies = [] y_test_pred_l = [] -for model, predict_func in model_l: - y_test_pred_l.append(predict_func(model, X_test)) +scaler = StandardScaler() -# For multiclass classification, use the mode of the predictions -y_test_pred = stats.mode(y_test_pred_l, axis=0)[0].flatten() + 1 +print("12") +# 3) Train and evaluate using KFold +fold_number = 1 +model_count = defaultdict(int) +print("123") +for train_index, valid_index in kf.split(X_train): + print(f"Starting fold {fold_number}...") + X_train_l, X_valid_l, X_test_l = [], [], [] # Reset feature lists for each fold + X_tr, X_val = X_train.iloc[train_index], X_train.iloc[valid_index] + y_tr, y_val = y_train.iloc[train_index], y_train.iloc[valid_index] + X_te = X_test + + # Feature engineering + for f in DIRNAME.glob("feature/feat*.py"): + cls = import_module_from_path(f.stem, f).feature_engineering_cls() + cls.fit(X_tr) + X_train_f = cls.transform(X_tr) + X_valid_f = cls.transform(X_val) + X_test_f = cls.transform(X_te) + + X_train_l.append(X_train_f) + X_valid_l.append(X_valid_f) + X_test_l.append(X_test_f) + + X_tr = pd.concat(X_train_l, axis=1, keys=[f"feature_{i}" for i in range(len(X_train_l))]) + X_val = pd.concat(X_valid_l, axis=1, keys=[f"feature_{i}" for i in range(len(X_valid_l))]) + X_te = pd.concat(X_test_l, axis=1, keys=[f"feature_{i}" for i in range(len(X_test_l))]) + + print("Shape of X_tr: ", X_tr.shape, " Shape of X_val: ", X_val.shape, " Shape of X_te: ", X_te.shape) + + # Replace inf and -inf with NaN + X_tr.replace([np.inf, -np.inf], np.nan, inplace=True) + X_val.replace([np.inf, -np.inf], np.nan, inplace=True) + X_te.replace([np.inf, -np.inf], np.nan, inplace=True) + + # Impute missing values + imputer = SimpleImputer(strategy="mean") + X_tr = pd.DataFrame(imputer.fit_transform(X_tr), columns=X_tr.columns) + X_val = pd.DataFrame(imputer.transform(X_val), columns=X_val.columns) + X_te = pd.DataFrame(imputer.transform(X_te), columns=X_te.columns) + + # Standardize the data + X_tr = pd.DataFrame(scaler.fit_transform(X_tr), columns=X_tr.columns) + X_val = pd.DataFrame(scaler.transform(X_val), columns=X_val.columns) + X_te = pd.DataFrame(scaler.transform(X_te), columns=X_te.columns) + + # Remove duplicate columns + X_tr = X_tr.loc[:, ~X_tr.columns.duplicated()] + X_val = X_val.loc[:, ~X_val.columns.duplicated()] + X_te = X_te.loc[:, ~X_te.columns.duplicated()] + + model_l = [] # list[tuple[model, predict_func]] + for f in DIRNAME.glob("model/model*.py"): + select_python_path = f.with_name(f.stem.replace("model", "select") + f.suffix) + select_m = import_module_from_path(select_python_path.stem, select_python_path) + X_train_selected = select_m.select(X_tr.copy()) + X_valid_selected = select_m.select(X_val.copy()) + + m = import_module_from_path(f.stem, f) + model_l.append((m.fit(X_train_selected, y_tr, X_valid_selected, y_val), m.predict)) + + # 4) Evaluate the models on the validation set and choose the best one + best_accuracy = -1 + best = None + for model, predict_func in model_l: + X_valid_selected = select_m.select(X_val.copy()) + y_valid_pred = predict_func(model, X_valid_selected) + accuracy = accuracy_score(y_val, y_valid_pred) + print(f"Accuracy on valid set: {accuracy}") + + if accuracy > best_accuracy: + best_accuracy = accuracy + best = (model, predict_func) + + model_count[best] += 1 + fold_number += 1 + +# 5) Save the validation accuracy +final_model = max(model_count, key=model_count.get) +pd.Series(data=best_accuracy, index=["multi-class accuracy"]).to_csv("submission_score.csv") + +# 6) Make predictions on the test set and save them +X_test_selected = select_m.select(X_te.copy()) +y_test_pred = final_model[1](final_model[0], X_test_selected).flatten() + 1 submission_result = pd.DataFrame(y_test_pred, columns=["Cover_Type"]) submission_result.insert(0, "Id", ids) diff --git a/rdagent/scenarios/kaggle/experiment/kaggle_experiment.py b/rdagent/scenarios/kaggle/experiment/kaggle_experiment.py index 562ea995..ca3e9242 100644 --- a/rdagent/scenarios/kaggle/experiment/kaggle_experiment.py +++ b/rdagent/scenarios/kaggle/experiment/kaggle_experiment.py @@ -14,9 +14,28 @@ from rdagent.components.coder.model_coder.model import ( ) from rdagent.scenarios.kaggle.experiment.workspace import KGFBWorkspace +KG_MODEL_TYPE_XGBOOST = "XGBoost" +KG_MODEL_TYPE_RANDOMFOREST = "RandomForest" +KG_MODEL_TYPE_LIGHTGBM = "LightGBM" +KG_MODEL_TYPE_NN = "NN" + +KG_MODEL_MAPPING = { + KG_MODEL_TYPE_XGBOOST: "model/model_xgboost.py", + KG_MODEL_TYPE_RANDOMFOREST: "model/model_randomforest.py", + KG_MODEL_TYPE_LIGHTGBM: "model/model_lightgbm.py", + KG_MODEL_TYPE_NN: "model/model_nn.py", +} + +KG_SELECT_MAPPING = { + KG_MODEL_TYPE_XGBOOST: "model/select_xgboost.py", + KG_MODEL_TYPE_RANDOMFOREST: "model/select_randomforest.py", + KG_MODEL_TYPE_LIGHTGBM: "model/select_lightgbm.py", + KG_MODEL_TYPE_NN: "model/select_nn.py", +} + class KGModelExperiment(ModelExperiment[ModelTask, KGFBWorkspace, ModelFBWorkspace]): - def __init__(self, *args, **kwargs) -> None: + def __init__(self, *args, source_feature_size: int = None, **kwargs) -> None: super().__init__(*args, **kwargs) self.experiment_workspace = KGFBWorkspace( template_folder_path=Path(__file__).parent / f"{KAGGLE_IMPLEMENT_SETTING.competition}_template" @@ -26,13 +45,21 @@ class KGModelExperiment(ModelExperiment[ModelTask, KGFBWorkspace, ModelFBWorkspa self.experiment_workspace.data_description = deepcopy( self.based_experiments[-1].experiment_workspace.data_description ) - self.experiment_workspace.model_description = deepcopy( - self.based_experiments[-1].experiment_workspace.model_description - ) + else: + self.experiment_workspace.data_description = [ + ( + FactorTask( + factor_name="Original features", + factor_description="The original features", + factor_formulation="", + ).get_task_information(), + source_feature_size, + ) + ] class KGFactorExperiment(FeatureExperiment[FactorTask, KGFBWorkspace, FactorFBWorkspace]): - def __init__(self, *args, **kwargs) -> None: + def __init__(self, *args, source_feature_size: int = None, **kwargs) -> None: super().__init__(*args, **kwargs) self.experiment_workspace = KGFBWorkspace( template_folder_path=Path(__file__).parent / f"{KAGGLE_IMPLEMENT_SETTING.competition}_template" @@ -42,6 +69,14 @@ class KGFactorExperiment(FeatureExperiment[FactorTask, KGFBWorkspace, FactorFBWo self.experiment_workspace.data_description = deepcopy( self.based_experiments[-1].experiment_workspace.data_description ) - self.experiment_workspace.model_description = deepcopy( - self.based_experiments[-1].experiment_workspace.model_description - ) + else: + self.experiment_workspace.data_description = [ + ( + FactorTask( + factor_name="Original features", + factor_description="The original features", + factor_formulation="", + ).get_task_information(), + source_feature_size, + ) + ] diff --git a/rdagent/scenarios/kaggle/experiment/meta_tpl/fea_share_preprocess.py b/rdagent/scenarios/kaggle/experiment/meta_tpl_deprecated/fea_share_preprocess.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/meta_tpl/fea_share_preprocess.py rename to rdagent/scenarios/kaggle/experiment/meta_tpl_deprecated/fea_share_preprocess.py diff --git a/rdagent/scenarios/kaggle/experiment/meta_tpl/feature/feature.py b/rdagent/scenarios/kaggle/experiment/meta_tpl_deprecated/feature/feature.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/meta_tpl/feature/feature.py rename to rdagent/scenarios/kaggle/experiment/meta_tpl_deprecated/feature/feature.py diff --git a/rdagent/scenarios/kaggle/experiment/meta_tpl/model/model_nn.py b/rdagent/scenarios/kaggle/experiment/meta_tpl_deprecated/model/model_nn.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/meta_tpl/model/model_nn.py rename to rdagent/scenarios/kaggle/experiment/meta_tpl_deprecated/model/model_nn.py diff --git a/rdagent/scenarios/kaggle/experiment/meta_tpl/model/model_randomforest.py b/rdagent/scenarios/kaggle/experiment/meta_tpl_deprecated/model/model_randomforest.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/meta_tpl/model/model_randomforest.py rename to rdagent/scenarios/kaggle/experiment/meta_tpl_deprecated/model/model_randomforest.py diff --git a/rdagent/scenarios/kaggle/experiment/meta_tpl/model/model_xgboost.py b/rdagent/scenarios/kaggle/experiment/meta_tpl_deprecated/model/model_xgboost.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/meta_tpl/model/model_xgboost.py rename to rdagent/scenarios/kaggle/experiment/meta_tpl_deprecated/model/model_xgboost.py diff --git a/rdagent/scenarios/kaggle/experiment/meta_tpl/train.py b/rdagent/scenarios/kaggle/experiment/meta_tpl_deprecated/train.py similarity index 100% rename from rdagent/scenarios/kaggle/experiment/meta_tpl/train.py rename to rdagent/scenarios/kaggle/experiment/meta_tpl_deprecated/train.py diff --git a/rdagent/scenarios/kaggle/experiment/optiver-realized-volatility-prediction_template/model/model_randomforest.py b/rdagent/scenarios/kaggle/experiment/optiver-realized-volatility-prediction_template/model/model_randomforest.py index dba55127..bf1b273d 100644 --- a/rdagent/scenarios/kaggle/experiment/optiver-realized-volatility-prediction_template/model/model_randomforest.py +++ b/rdagent/scenarios/kaggle/experiment/optiver-realized-volatility-prediction_template/model/model_randomforest.py @@ -4,14 +4,6 @@ from sklearn.ensemble import RandomForestRegressor from sklearn.metrics import mean_squared_error -def select(X: pd.DataFrame) -> pd.DataFrame: - """ - Select relevant features. To be used in fit & predict function. - """ - # For now, we assume all features are relevant. This can be expanded to feature selection logic. - return X - - def fit(X_train: pd.DataFrame, y_train: pd.Series, X_valid: pd.DataFrame, y_valid: pd.Series): """ Define and train the Random Forest model. Merge feature selection into the pipeline. @@ -19,15 +11,11 @@ def fit(X_train: pd.DataFrame, y_train: pd.Series, X_valid: pd.DataFrame, y_vali # Initialize the Random Forest model model = RandomForestRegressor(n_estimators=100, random_state=32, n_jobs=-1) - # Select features (if any feature selection is needed) - X_train_selected = select(X_train) - X_valid_selected = select(X_valid) - # Fit the model - model.fit(X_train_selected, y_train) + model.fit(X_train, y_train) # Validate the model - y_valid_pred = model.predict(X_valid_selected) + y_valid_pred = model.predict(X_valid) mse = mean_squared_error(y_valid, y_valid_pred) rmse = np.sqrt(mse) print(f"Validation RMSE: {rmse:.4f}") @@ -39,10 +27,7 @@ def predict(model, X): """ Keep feature selection's consistency and make predictions. """ - # Select features (if any feature selection is needed) - X_selected = select(X) - # Predict using the trained model - y_pred = model.predict(X_selected) + y_pred = model.predict(X) return y_pred.reshape(-1, 1) diff --git a/rdagent/scenarios/kaggle/experiment/optiver-realized-volatility-prediction_template/model/model_xgboost.py b/rdagent/scenarios/kaggle/experiment/optiver-realized-volatility-prediction_template/model/model_xgboost.py index 321c2596..e51601f9 100644 --- a/rdagent/scenarios/kaggle/experiment/optiver-realized-volatility-prediction_template/model/model_xgboost.py +++ b/rdagent/scenarios/kaggle/experiment/optiver-realized-volatility-prediction_template/model/model_xgboost.py @@ -2,15 +2,8 @@ import pandas as pd import xgboost as xgb -def select(X: pd.DataFrame) -> pd.DataFrame: - # Ignore feature selection logic - return X - - def fit(X_train: pd.DataFrame, y_train: pd.DataFrame, X_valid: pd.DataFrame, y_valid: pd.DataFrame): """Define and train the model. Merge feature_select""" - X_train = select(X_train) - X_valid = select(X_valid) dtrain = xgb.DMatrix(X_train, label=y_train) dvalid = xgb.DMatrix(X_valid, label=y_valid) @@ -33,7 +26,6 @@ def predict(model, X): """ Keep feature select's consistency. """ - X = select(X) dtest = xgb.DMatrix(X) y_pred = model.predict(dtest) return y_pred.reshape(-1, 1) diff --git a/rdagent/scenarios/kaggle/experiment/optiver-realized-volatility-prediction_template/model/select_randomforest.py b/rdagent/scenarios/kaggle/experiment/optiver-realized-volatility-prediction_template/model/select_randomforest.py new file mode 100644 index 00000000..d2a15dee --- /dev/null +++ b/rdagent/scenarios/kaggle/experiment/optiver-realized-volatility-prediction_template/model/select_randomforest.py @@ -0,0 +1,12 @@ +import pandas as pd + + +def select(X: pd.DataFrame) -> pd.DataFrame: + """ + Select relevant features. To be used in fit & predict function. + """ + # For now, we assume all features are relevant. This can be expanded to feature selection logic. + if X.columns.nlevels == 1: + return X + X.columns = ["_".join(str(col)).strip() for col in X.columns.values] + return X diff --git a/rdagent/scenarios/kaggle/experiment/optiver-realized-volatility-prediction_template/model/select_xgboost.py b/rdagent/scenarios/kaggle/experiment/optiver-realized-volatility-prediction_template/model/select_xgboost.py new file mode 100644 index 00000000..d2a15dee --- /dev/null +++ b/rdagent/scenarios/kaggle/experiment/optiver-realized-volatility-prediction_template/model/select_xgboost.py @@ -0,0 +1,12 @@ +import pandas as pd + + +def select(X: pd.DataFrame) -> pd.DataFrame: + """ + Select relevant features. To be used in fit & predict function. + """ + # For now, we assume all features are relevant. This can be expanded to feature selection logic. + if X.columns.nlevels == 1: + return X + X.columns = ["_".join(str(col)).strip() for col in X.columns.values] + return X diff --git a/rdagent/scenarios/kaggle/experiment/optiver-realized-volatility-prediction_template/train.py b/rdagent/scenarios/kaggle/experiment/optiver-realized-volatility-prediction_template/train.py index c5903aef..39a8b154 100644 --- a/rdagent/scenarios/kaggle/experiment/optiver-realized-volatility-prediction_template/train.py +++ b/rdagent/scenarios/kaggle/experiment/optiver-realized-volatility-prediction_template/train.py @@ -42,9 +42,10 @@ for f in DIRNAME.glob("feature/feat*.py"): X_valid_f = cls.transform(X_valid) X_test_f = cls.transform(X_test) - X_train_l.append(X_train_f) - X_valid_l.append(X_valid_f) - X_test_l.append(X_test_f) + if X_train_f.shape[-1] == X_valid_f.shape[-1] and X_train_f.shape[-1] == X_test_f.shape[-1]: + X_train_l.append(X_train_f) + X_valid_l.append(X_valid_f) + X_test_l.append(X_test_f) X_train = pd.concat(X_train_l, axis=1, keys=[f"feature_{i}" for i in range(len(X_train_l))]) X_valid = pd.concat(X_valid_l, axis=1, keys=[f"feature_{i}" for i in range(len(X_valid_l))]) @@ -70,42 +71,32 @@ X_test = X_test.loc[:, ~X_test.columns.duplicated()] # 3) Train the model -def flatten_columns(df: pd.DataFrame) -> pd.DataFrame: - """ - Flatten the columns of a DataFrame with MultiIndex columns, - for (feature_0, a), (feature_0, b) -> feature_0_a, feature_0_b - """ - if df.columns.nlevels == 1: - return df - df.columns = ["_".join(col).strip() for col in df.columns.values] - return df - - -X_train = flatten_columns(X_train) -X_valid = flatten_columns(X_valid) -X_test = flatten_columns(X_test) - model_l = [] # list[tuple[model, predict_func,]] for f in DIRNAME.glob("model/model*.py"): + select_python_path = f.with_name(f.stem.replace("model", "select") + f.suffix) + select_m = import_module_from_path(select_python_path.stem, select_python_path) + X_train_selected = select_m.select(X_train.copy()) + X_valid_selected = select_m.select(X_valid.copy()) + m = import_module_from_path(f.stem, f) - model_l.append((m.fit(X_train, y_train, X_valid, y_valid), m.predict)) + model_l.append((m.fit(X_train_selected, y_train, X_valid_selected, y_valid), m.predict, select_m)) # 4) Evaluate the model on the validation set -y_valid_pred_l = [] metrics_all = [] - -for model, predict_func in model_l: - y_valid_pred_l.append(predict_func(model, X_valid)) - metrics = compute_rmspe(y_valid, y_valid_pred_l[-1].ravel()) +for model, predict_func, select_m in model_l: + X_valid_selected = select_m.select(X_valid.copy()) + y_valid_pred = predict_func(model, X_valid_selected) + metrics = compute_rmspe(y_valid, y_valid_pred.ravel()) print(f"RMSPE on valid set: {metrics}") metrics_all.append(metrics) +# 5) Save the validation accuracy min_index = np.argmin(metrics_all) - pd.Series(data=[metrics_all[min_index]], index=["RMSPE"]).to_csv("submission_score.csv") -y_test_pred = model_l[min_index][1](model_l[min_index][0], X_test).ravel() +# 6) Make predictions on the test set and save them +X_test_selected = model_l[min_index][2].select(X_test.copy()) +y_test_pred = model_l[min_index][1](model_l[min_index][0], X_test_selected).ravel() -# 5) Submit predictions for the test set submission_result = pd.DataFrame({"row_id": ids, "target": y_test_pred}) submission_result.to_csv("submission.csv", index=False) diff --git a/rdagent/scenarios/kaggle/experiment/playground-series-s3e11_template/model/model_randomforest.py b/rdagent/scenarios/kaggle/experiment/playground-series-s3e11_template/model/model_randomforest.py index 9618a10e..82b6712a 100644 --- a/rdagent/scenarios/kaggle/experiment/playground-series-s3e11_template/model/model_randomforest.py +++ b/rdagent/scenarios/kaggle/experiment/playground-series-s3e11_template/model/model_randomforest.py @@ -2,15 +2,8 @@ import pandas as pd from sklearn.ensemble import RandomForestRegressor -def select(X: pd.DataFrame) -> pd.DataFrame: - # Ignore feature selection logic - return X - - def fit(X_train: pd.DataFrame, y_train: pd.DataFrame, X_valid: pd.DataFrame, y_valid: pd.DataFrame): """Define and train the Random Forest model. Merge feature_select""" - X_train = select(X_train) - rf_params = { "n_estimators": 100, "max_depth": 10, @@ -30,6 +23,5 @@ def predict(model, X_test): """ Keep feature select's consistency. """ - X_test = select(X_test) y_pred = model.predict(X_test) return y_pred.reshape(-1, 1) diff --git a/rdagent/scenarios/kaggle/experiment/playground-series-s3e11_template/model/model_xgboost.py b/rdagent/scenarios/kaggle/experiment/playground-series-s3e11_template/model/model_xgboost.py index 9497f128..16cb7c34 100644 --- a/rdagent/scenarios/kaggle/experiment/playground-series-s3e11_template/model/model_xgboost.py +++ b/rdagent/scenarios/kaggle/experiment/playground-series-s3e11_template/model/model_xgboost.py @@ -6,15 +6,8 @@ import pandas as pd import xgboost as xgb -def select(X: pd.DataFrame) -> pd.DataFrame: - # Ignore feature selection logic - return X - - def fit(X_train: pd.DataFrame, y_train: pd.DataFrame, X_valid: pd.DataFrame, y_valid: pd.DataFrame): """Define and train the model. Merge feature_select""" - X_train = select(X_train) - xgb_params = { "n_estimators": 280, "learning_rate": 0.05, @@ -37,6 +30,5 @@ def predict(model, X_test): """ Keep feature select's consistency. """ - X_test = select(X_test) y_pred = model.predict(X_test) return y_pred.reshape(-1, 1) diff --git a/rdagent/scenarios/kaggle/experiment/playground-series-s3e11_template/model/select_randomforest.py b/rdagent/scenarios/kaggle/experiment/playground-series-s3e11_template/model/select_randomforest.py new file mode 100644 index 00000000..d2a15dee --- /dev/null +++ b/rdagent/scenarios/kaggle/experiment/playground-series-s3e11_template/model/select_randomforest.py @@ -0,0 +1,12 @@ +import pandas as pd + + +def select(X: pd.DataFrame) -> pd.DataFrame: + """ + Select relevant features. To be used in fit & predict function. + """ + # For now, we assume all features are relevant. This can be expanded to feature selection logic. + if X.columns.nlevels == 1: + return X + X.columns = ["_".join(str(col)).strip() for col in X.columns.values] + return X diff --git a/rdagent/scenarios/kaggle/experiment/playground-series-s3e11_template/model/select_xgboost.py b/rdagent/scenarios/kaggle/experiment/playground-series-s3e11_template/model/select_xgboost.py new file mode 100644 index 00000000..d2a15dee --- /dev/null +++ b/rdagent/scenarios/kaggle/experiment/playground-series-s3e11_template/model/select_xgboost.py @@ -0,0 +1,12 @@ +import pandas as pd + + +def select(X: pd.DataFrame) -> pd.DataFrame: + """ + Select relevant features. To be used in fit & predict function. + """ + # For now, we assume all features are relevant. This can be expanded to feature selection logic. + if X.columns.nlevels == 1: + return X + X.columns = ["_".join(str(col)).strip() for col in X.columns.values] + return X diff --git a/rdagent/scenarios/kaggle/experiment/playground-series-s3e11_template/train.py b/rdagent/scenarios/kaggle/experiment/playground-series-s3e11_template/train.py index f82e1f8e..3f2e6281 100644 --- a/rdagent/scenarios/kaggle/experiment/playground-series-s3e11_template/train.py +++ b/rdagent/scenarios/kaggle/experiment/playground-series-s3e11_template/train.py @@ -30,9 +30,10 @@ for f in DIRNAME.glob("feature/feat*.py"): X_valid_f = cls.transform(X_valid) X_test_f = cls.transform(X_test) - X_train_l.append(X_train_f) - X_valid_l.append(X_valid_f) - X_test_l.append(X_test_f) + if X_train_f.shape[-1] == X_valid_f.shape[-1] and X_train_f.shape[-1] == X_test_f.shape[-1]: + X_train_l.append(X_train_f) + X_valid_l.append(X_valid_f) + X_test_l.append(X_test_f) X_train = pd.concat(X_train_l, axis=1, keys=[f"feature_{i}" for i in range(len(X_train_l))]) X_valid = pd.concat(X_valid_l, axis=1, keys=[f"feature_{i}" for i in range(len(X_valid_l))]) @@ -40,41 +41,34 @@ X_test = pd.concat(X_test_l, axis=1, keys=[f"feature_{i}" for i in range(len(X_t # 3) Train the model -def flatten_columns(df: pd.DataFrame) -> pd.DataFrame: - """ - Flatten the columns of a DataFrame with MultiIndex columns, - for (feature_0, a), (feature_0, b) -> feature_0_a, feature_0_b - """ - if df.columns.nlevels == 1: - return df - df.columns = ["_".join(col).strip() for col in df.columns.values] - return df - - -X_train = flatten_columns(X_train) -X_valid = flatten_columns(X_valid) -X_test = flatten_columns(X_test) - model_l = [] # list[tuple[model, predict_func]] for f in DIRNAME.glob("model/model*.py"): + select_python_path = f.with_name(f.stem.replace("model", "select") + f.suffix) + select_m = import_module_from_path(select_python_path.stem, select_python_path) + X_train_selected = select_m.select(X_train.copy()) + X_valid_selected = select_m.select(X_valid.copy()) + m = import_module_from_path(f.stem, f) - model_l.append((m.fit(X_train, y_train, X_valid, y_valid), m.predict)) + model_l.append((m.fit(X_train_selected, y_train, X_valid_selected, y_valid), m.predict, select_m)) # 4) Evaluate the model on the validation set -y_valid_pred_l = [] metrics_all = [] -for model, predict_func in model_l: - y_valid_pred = predict_func(model, X_valid) - y_valid_pred_l.append(y_valid_pred) +for model, predict_func, select_m in model_l: + X_valid_selected = select_m.select(X_valid.copy()) + y_valid_pred = predict_func(model, X_valid_selected) metrics = mean_squared_error(y_valid, y_valid_pred, squared=False) print(f"RMLSE on valid set: {metrics}") metrics_all.append(metrics) +# 5) Save the validation accuracy min_index = np.argmin(metrics_all) pd.Series(data=[metrics_all[min_index]], index=["RMLSE"]).to_csv("submission_score.csv") -y_test_pred = model_l[min_index][1](model_l[min_index][0], X_test) +# 6) Make predictions on the test set and save them +X_test_selected = model_l[min_index][2].select(X_test.copy()) +y_test_pred = model_l[min_index][1](model_l[min_index][0], X_test_selected) +# 7) Submit predictions for the test set submission_result = pd.DataFrame(np.expm1(y_test_pred), columns=["cost"]) submission_result.insert(0, "id", ids) diff --git a/rdagent/scenarios/kaggle/experiment/playground-series-s3e26_template/model/model_randomforest.py b/rdagent/scenarios/kaggle/experiment/playground-series-s3e26_template/model/model_randomforest.py index 31f7a24b..630d0ead 100644 --- a/rdagent/scenarios/kaggle/experiment/playground-series-s3e26_template/model/model_randomforest.py +++ b/rdagent/scenarios/kaggle/experiment/playground-series-s3e26_template/model/model_randomforest.py @@ -10,14 +10,6 @@ from sklearn.ensemble import RandomForestClassifier from sklearn.metrics import accuracy_score -def select(X: pd.DataFrame) -> pd.DataFrame: - """ - Select relevant features. To be used in fit & predict function. - """ - # For now, we assume all features are relevant. This can be expanded to feature selection logic. - return X - - def fit(X_train: pd.DataFrame, y_train: pd.Series, X_valid: pd.DataFrame, y_valid: pd.Series): """ Define and train the Random Forest model. Merge feature selection into the pipeline. @@ -25,17 +17,8 @@ def fit(X_train: pd.DataFrame, y_train: pd.Series, X_valid: pd.DataFrame, y_vali # Initialize the Random Forest model model = RandomForestClassifier(n_estimators=100, random_state=32, n_jobs=-1) - # Select features (if any feature selection is needed) - X_train_selected = select(X_train) - X_valid_selected = select(X_valid) - # Fit the model - model.fit(X_train_selected, y_train) - - # Validate the model - y_valid_pred = model.predict(X_valid_selected) - # accuracy = accuracy_score(y_valid, y_valid_pred) - # print(f"Validation Accuracy: {accuracy:.4f}") + model.fit(X_train, y_train) return model @@ -44,11 +27,8 @@ def predict(model, X): """ Keep feature selection's consistency and make predictions. """ - # Select features (if any feature selection is needed) - X_selected = select(X) - # Predict using the trained model - y_pred_prob = model.predict_proba(X_selected) + y_pred_prob = model.predict_proba(X) # Apply threshold to get boolean predictions return y_pred_prob diff --git a/rdagent/scenarios/kaggle/experiment/playground-series-s3e26_template/model/model_xgboost.py b/rdagent/scenarios/kaggle/experiment/playground-series-s3e26_template/model/model_xgboost.py index b06a0839..4ed913c7 100644 --- a/rdagent/scenarios/kaggle/experiment/playground-series-s3e26_template/model/model_xgboost.py +++ b/rdagent/scenarios/kaggle/experiment/playground-series-s3e26_template/model/model_xgboost.py @@ -7,15 +7,8 @@ import pandas as pd import xgboost as xgb -def select(X: pd.DataFrame) -> pd.DataFrame: - # Ignore feature selection logic - return X - - def fit(X_train: pd.DataFrame, y_train: pd.DataFrame, X_valid: pd.DataFrame, y_valid: pd.DataFrame): """Define and train the model. Merge feature_select""" - X_train = select(X_train) - X_valid = select(X_valid) dtrain = xgb.DMatrix(X_train, label=y_train) dvalid = xgb.DMatrix(X_valid, label=y_valid) num_classes = len(np.unique(y_train)) @@ -40,7 +33,6 @@ def predict(model, X): """ Keep feature select's consistency. """ - X = select(X) dtest = xgb.DMatrix(X) y_pred_prob = model.predict(dtest) return y_pred_prob diff --git a/rdagent/scenarios/kaggle/experiment/playground-series-s3e26_template/model/select_randomforest.py b/rdagent/scenarios/kaggle/experiment/playground-series-s3e26_template/model/select_randomforest.py new file mode 100644 index 00000000..d2a15dee --- /dev/null +++ b/rdagent/scenarios/kaggle/experiment/playground-series-s3e26_template/model/select_randomforest.py @@ -0,0 +1,12 @@ +import pandas as pd + + +def select(X: pd.DataFrame) -> pd.DataFrame: + """ + Select relevant features. To be used in fit & predict function. + """ + # For now, we assume all features are relevant. This can be expanded to feature selection logic. + if X.columns.nlevels == 1: + return X + X.columns = ["_".join(str(col)).strip() for col in X.columns.values] + return X diff --git a/rdagent/scenarios/kaggle/experiment/playground-series-s3e26_template/model/select_xgboost.py b/rdagent/scenarios/kaggle/experiment/playground-series-s3e26_template/model/select_xgboost.py new file mode 100644 index 00000000..d2a15dee --- /dev/null +++ b/rdagent/scenarios/kaggle/experiment/playground-series-s3e26_template/model/select_xgboost.py @@ -0,0 +1,12 @@ +import pandas as pd + + +def select(X: pd.DataFrame) -> pd.DataFrame: + """ + Select relevant features. To be used in fit & predict function. + """ + # For now, we assume all features are relevant. This can be expanded to feature selection logic. + if X.columns.nlevels == 1: + return X + X.columns = ["_".join(str(col)).strip() for col in X.columns.values] + return X diff --git a/rdagent/scenarios/kaggle/experiment/playground-series-s3e26_template/train.py b/rdagent/scenarios/kaggle/experiment/playground-series-s3e26_template/train.py index ca72e886..233ab051 100644 --- a/rdagent/scenarios/kaggle/experiment/playground-series-s3e26_template/train.py +++ b/rdagent/scenarios/kaggle/experiment/playground-series-s3e26_template/train.py @@ -53,6 +53,7 @@ X_train = pd.concat(X_train_l, axis=1) X_valid = pd.concat(X_valid_l, axis=1) X_test = pd.concat(X_test_l, axis=1) +print(X_train.shape, X_valid.shape, X_test.shape) # Handle inf and -inf values X_train.replace([np.inf, -np.inf], np.nan, inplace=True) @@ -72,62 +73,38 @@ X_train = X_train.loc[:, ~X_train.columns.duplicated()] X_valid = X_valid.loc[:, ~X_valid.columns.duplicated()] X_test = X_test.loc[:, ~X_test.columns.duplicated()] -print(X_train.shape, X_valid.shape, X_test.shape) - # 3) Train the model -def flatten_columns(df: pd.DataFrame) -> pd.DataFrame: - """ - Flatten the columns of a DataFrame with MultiIndex columns, - for (feature_0, a), (feature_0, b) -> feature_0_a, feature_0_b - """ - if df.columns.nlevels == 1: - return df - df.columns = ["_".join(col).strip() for col in df.columns.values] - return df - - -X_train = flatten_columns(X_train) -X_valid = flatten_columns(X_valid) -X_test = flatten_columns(X_test) - -model_l = [] # list[tuple[model, predict_func]] +model_l = [] # list[tuple[model, predict_func,]] for f in DIRNAME.glob("model/model*.py"): + select_python_path = f.with_name(f.stem.replace("model", "select") + f.suffix) + select_m = import_module_from_path(select_python_path.stem, select_python_path) + X_train_selected = select_m.select(X_train.copy()) + X_valid_selected = select_m.select(X_valid.copy()) + m = import_module_from_path(f.stem, f) - model_l.append((m.fit(X_train, y_train, X_valid, y_valid), m.predict)) + model_l.append((m.fit(X_train_selected, y_train, X_valid_selected, y_valid), m.predict, select_m)) # 4) Evaluate the model on the validation set -y_valid_pred_l = [] -for model, predict_func in model_l: - y_valid_pred_l.append(predict_func(model, X_valid)) - print(predict_func(model, X_valid)) - print(predict_func(model, X_valid).shape) +metrics_all = [] +for model, predict_func, select_m in model_l: + X_valid_selected = select_m.select(X_valid.copy()) + y_valid_pred = predict_func(model, X_valid_selected) + logloss = compute_metrics_for_classification(y_valid, y_valid_pred) + print(f"log_loss on valid set: {logloss}") + metrics_all.append(logloss) -# 5) Ensemble -from scipy import stats +# 5) Save the validation accuracy +min_index = np.argmin(metrics_all) +pd.Series(data=[metrics_all[min_index]], index=["log_loss"]).to_csv("submission_score.csv") -# average probabilities ensemble -y_valid_pred_proba = np.mean(y_valid_pred_l, axis=0) - -# Compute metrics -logloss = compute_metrics_for_classification(y_valid, y_valid_pred_proba) -print(f"final log_loss on valid set: {logloss}") - -# 6) Save the validation metrics -pd.Series(data=[logloss], index=["log_loss"]).to_csv("submission_score.csv") - -# 7) Make predictions on the test set and save them -y_test_pred_l = [] -for model, predict_func in model_l: - y_test_pred_l.append(predict_func(model, X_test)) - -# For multiclass classification, use the mode of the predictions -y_test_pred_proba = np.mean(y_test_pred_l, axis=0) +# 6) Make predictions on the test set and save them +X_test_selected = model_l[min_index][2].select(X_test.copy()) +y_test_pred = model_l[min_index][1](model_l[min_index][0], X_test_selected) class_labels = ["Status_" + label for label in status_encoder.classes_] -# modified_labels = ["Status_" + label for label in class_labels] -submission_result = pd.DataFrame(y_test_pred_proba, columns=class_labels) +submission_result = pd.DataFrame(y_test_pred, columns=class_labels) submission_result.insert(0, "id", test_ids) submission_result.to_csv("submission.csv", index=False) diff --git a/rdagent/scenarios/kaggle/experiment/playground-series-s4e8_template/model/model_randomforest.py b/rdagent/scenarios/kaggle/experiment/playground-series-s4e8_template/model/model_randomforest.py index 377683b9..33221635 100644 --- a/rdagent/scenarios/kaggle/experiment/playground-series-s4e8_template/model/model_randomforest.py +++ b/rdagent/scenarios/kaggle/experiment/playground-series-s4e8_template/model/model_randomforest.py @@ -10,14 +10,6 @@ from sklearn.ensemble import RandomForestClassifier from sklearn.metrics import accuracy_score -def select(X: pd.DataFrame) -> pd.DataFrame: - """ - Select relevant features. To be used in fit & predict function. - """ - # For now, we assume all features are relevant. This can be expanded to feature selection logic. - return X - - def fit(X_train: pd.DataFrame, y_train: pd.Series, X_valid: pd.DataFrame, y_valid: pd.Series): """ Define and train the Random Forest model. Merge feature selection into the pipeline. @@ -25,15 +17,11 @@ def fit(X_train: pd.DataFrame, y_train: pd.Series, X_valid: pd.DataFrame, y_vali # Initialize the Random Forest model model = RandomForestClassifier(n_estimators=100, random_state=32, n_jobs=-1) - # Select features (if any feature selection is needed) - X_train_selected = select(X_train) - X_valid_selected = select(X_valid) - # Fit the model - model.fit(X_train_selected, y_train) + model.fit(X_train, y_train) # Validate the model - y_valid_pred = model.predict(X_valid_selected) + y_valid_pred = model.predict(X_valid) accuracy = accuracy_score(y_valid, y_valid_pred) print(f"Validation Accuracy: {accuracy:.4f}") @@ -44,11 +32,8 @@ def predict(model, X): """ Keep feature selection's consistency and make predictions. """ - # Select features (if any feature selection is needed) - X_selected = select(X) - # Predict using the trained model - y_pred_prob = model.predict_proba(X_selected)[:, 1] + y_pred_prob = model.predict_proba(X)[:, 1] # Apply threshold to get boolean predictions return y_pred_prob.reshape(-1, 1) diff --git a/rdagent/scenarios/kaggle/experiment/playground-series-s4e8_template/model/model_xgboost.py b/rdagent/scenarios/kaggle/experiment/playground-series-s4e8_template/model/model_xgboost.py index c3f31ec6..b6f6bf5c 100644 --- a/rdagent/scenarios/kaggle/experiment/playground-series-s4e8_template/model/model_xgboost.py +++ b/rdagent/scenarios/kaggle/experiment/playground-series-s4e8_template/model/model_xgboost.py @@ -6,19 +6,11 @@ import pandas as pd import xgboost as xgb -def select(X: pd.DataFrame) -> pd.DataFrame: - # Ignore feature selection logic - return X - - def fit(X_train: pd.DataFrame, y_train: pd.DataFrame, X_valid: pd.DataFrame, y_valid: pd.DataFrame): """Define and train the model. Merge feature_select""" - X_train = select(X_train) - X_valid = select(X_valid) dtrain = xgb.DMatrix(X_train, label=y_train) dvalid = xgb.DMatrix(X_valid, label=y_valid) - # TODO: for quick running.... params = { "nthread": -1, "tree_method": "gpu_hist", @@ -36,7 +28,6 @@ def predict(model, X): """ Keep feature select's consistency. """ - X = select(X) dtest = xgb.DMatrix(X) y_pred_prob = model.predict(dtest) return y_pred_prob.reshape(-1, 1) diff --git a/rdagent/scenarios/kaggle/experiment/playground-series-s4e8_template/model/select_randomforest.py b/rdagent/scenarios/kaggle/experiment/playground-series-s4e8_template/model/select_randomforest.py new file mode 100644 index 00000000..d2a15dee --- /dev/null +++ b/rdagent/scenarios/kaggle/experiment/playground-series-s4e8_template/model/select_randomforest.py @@ -0,0 +1,12 @@ +import pandas as pd + + +def select(X: pd.DataFrame) -> pd.DataFrame: + """ + Select relevant features. To be used in fit & predict function. + """ + # For now, we assume all features are relevant. This can be expanded to feature selection logic. + if X.columns.nlevels == 1: + return X + X.columns = ["_".join(str(col)).strip() for col in X.columns.values] + return X diff --git a/rdagent/scenarios/kaggle/experiment/playground-series-s4e8_template/model/select_xgboost.py b/rdagent/scenarios/kaggle/experiment/playground-series-s4e8_template/model/select_xgboost.py new file mode 100644 index 00000000..d2a15dee --- /dev/null +++ b/rdagent/scenarios/kaggle/experiment/playground-series-s4e8_template/model/select_xgboost.py @@ -0,0 +1,12 @@ +import pandas as pd + + +def select(X: pd.DataFrame) -> pd.DataFrame: + """ + Select relevant features. To be used in fit & predict function. + """ + # For now, we assume all features are relevant. This can be expanded to feature selection logic. + if X.columns.nlevels == 1: + return X + X.columns = ["_".join(str(col)).strip() for col in X.columns.values] + return X diff --git a/rdagent/scenarios/kaggle/experiment/playground-series-s4e8_template/train.py b/rdagent/scenarios/kaggle/experiment/playground-series-s4e8_template/train.py index 752d459c..73c06406 100644 --- a/rdagent/scenarios/kaggle/experiment/playground-series-s4e8_template/train.py +++ b/rdagent/scenarios/kaggle/experiment/playground-series-s4e8_template/train.py @@ -6,7 +6,6 @@ import numpy as np import pandas as pd from fea_share_preprocess import preprocess_script from sklearn.metrics import matthews_corrcoef -from sklearn.preprocessing import LabelEncoder # Set random seed for reproducibility SEED = 42 @@ -43,9 +42,10 @@ for f in DIRNAME.glob("feature/feat*.py"): X_valid_f = cls.transform(X_valid) X_test_f = cls.transform(X_test) - X_train_l.append(X_train_f) - X_valid_l.append(X_valid_f) - X_test_l.append(X_test_f) + if X_train_f.shape[-1] == X_valid_f.shape[-1] and X_train_f.shape[-1] == X_test_f.shape[-1]: + X_train_l.append(X_train_f) + X_valid_l.append(X_valid_f) + X_test_l.append(X_test_f) X_train = pd.concat(X_train_l, axis=1, keys=[f"feature_{i}" for i in range(len(X_train_l))]) X_valid = pd.concat(X_valid_l, axis=1, keys=[f"feature_{i}" for i in range(len(X_valid_l))]) @@ -71,55 +71,38 @@ X_train = X_train.loc[:, ~X_train.columns.duplicated()] X_valid = X_valid.loc[:, ~X_valid.columns.duplicated()] X_test = X_test.loc[:, ~X_test.columns.duplicated()] - # 3) Train the model -def flatten_columns(df: pd.DataFrame) -> pd.DataFrame: - """ - Flatten the columns of a DataFrame with MultiIndex columns, - for (feature_0, a), (feature_0, b) -> feature_0_a, feature_0_b - """ - if df.columns.nlevels == 1: - return df - df.columns = ["_".join(col).strip() for col in df.columns.values] - return df - - -X_train = flatten_columns(X_train) -X_valid = flatten_columns(X_valid) -X_test = flatten_columns(X_test) - model_l = [] # list[tuple[model, predict_func,]] for f in DIRNAME.glob("model/model*.py"): + select_python_path = f.with_name(f.stem.replace("model", "select") + f.suffix) + select_m = import_module_from_path(select_python_path.stem, select_python_path) + X_train_selected = select_m.select(X_train.copy()) + X_valid_selected = select_m.select(X_valid.copy()) + m = import_module_from_path(f.stem, f) - model_l.append((m.fit(X_train, y_train, X_valid, y_valid), m.predict)) + model_l.append((m.fit(X_train_selected, y_train, X_valid_selected, y_valid), m.predict, select_m)) # 4) Evaluate the model on the validation set -y_valid_pred_l = [] -for model, predict_func in model_l: - y_valid_pred_l.append(predict_func(model, X_valid)) +metrics_all = [] +for model, predict_func, select_m in model_l: + X_valid_selected = select_m.select(X_valid.copy()) + y_valid_pred = predict_func(model, X_valid_selected) + y_valid_pred = (y_valid_pred > 0.5).astype(int) + metrics = compute_metrics_for_classification(y_valid, y_valid_pred) + print("MCC on validation set: ", metrics) + metrics_all.append(metrics) -# 5) Ensemble -# TODO: ensemble method in a script -# Average the predictions and apply a threshold to determine class labels -y_valid_pred = np.mean(y_valid_pred_l, axis=0) -y_valid_pred = (y_valid_pred > 0.5).astype(int) +# 5) Save the validation accuracy +min_index = np.argmin(metrics_all) +pd.Series(data=[metrics_all[min_index]], index=["MCC"]).to_csv("submission_score.csv") -mcc = compute_metrics_for_classification(y_valid, y_valid_pred) -print("MCC on validation set: ", mcc) - -# 6) Save the validation accuracy -pd.Series(data=[mcc], index=["MCC"]).to_csv("submission_score.csv") - -# 7) Make predictions on the test set and save them -y_test_pred_l = [] -for m, m_pred in model_l: - y_test_pred_l.append(m_pred(m, X_test)) # TODO Make this an ensemble. Currently it uses the last prediction - -y_test_pred = np.mean(y_test_pred_l, axis=0) +# 6) Make predictions on the test set and save them +X_test_selected = model_l[min_index][2].select(X_test.copy()) +y_test_pred = model_l[min_index][1](model_l[min_index][0], X_test_selected) y_test_pred = (y_test_pred > 0.5).astype(int) y_test_pred_labels = np.where(y_test_pred == 1, "p", "e") # 将整数转换回 'e' 或 'p' -# 8) Submit predictions for the test set +# 7) Submit predictions for the test set submission_result = pd.DataFrame({"id": ids, "class": y_test_pred_labels.ravel()}) submission_result.to_csv("submission.csv", index=False) diff --git a/rdagent/scenarios/kaggle/experiment/playground-series-s4e9_template/model/model_randomforest.py b/rdagent/scenarios/kaggle/experiment/playground-series-s4e9_template/model/model_randomforest.py index dba55127..bf1b273d 100644 --- a/rdagent/scenarios/kaggle/experiment/playground-series-s4e9_template/model/model_randomforest.py +++ b/rdagent/scenarios/kaggle/experiment/playground-series-s4e9_template/model/model_randomforest.py @@ -4,14 +4,6 @@ from sklearn.ensemble import RandomForestRegressor from sklearn.metrics import mean_squared_error -def select(X: pd.DataFrame) -> pd.DataFrame: - """ - Select relevant features. To be used in fit & predict function. - """ - # For now, we assume all features are relevant. This can be expanded to feature selection logic. - return X - - def fit(X_train: pd.DataFrame, y_train: pd.Series, X_valid: pd.DataFrame, y_valid: pd.Series): """ Define and train the Random Forest model. Merge feature selection into the pipeline. @@ -19,15 +11,11 @@ def fit(X_train: pd.DataFrame, y_train: pd.Series, X_valid: pd.DataFrame, y_vali # Initialize the Random Forest model model = RandomForestRegressor(n_estimators=100, random_state=32, n_jobs=-1) - # Select features (if any feature selection is needed) - X_train_selected = select(X_train) - X_valid_selected = select(X_valid) - # Fit the model - model.fit(X_train_selected, y_train) + model.fit(X_train, y_train) # Validate the model - y_valid_pred = model.predict(X_valid_selected) + y_valid_pred = model.predict(X_valid) mse = mean_squared_error(y_valid, y_valid_pred) rmse = np.sqrt(mse) print(f"Validation RMSE: {rmse:.4f}") @@ -39,10 +27,7 @@ def predict(model, X): """ Keep feature selection's consistency and make predictions. """ - # Select features (if any feature selection is needed) - X_selected = select(X) - # Predict using the trained model - y_pred = model.predict(X_selected) + y_pred = model.predict(X) return y_pred.reshape(-1, 1) diff --git a/rdagent/scenarios/kaggle/experiment/playground-series-s4e9_template/model/model_xgboost.py b/rdagent/scenarios/kaggle/experiment/playground-series-s4e9_template/model/model_xgboost.py index 53fa01c3..ca5787b7 100644 --- a/rdagent/scenarios/kaggle/experiment/playground-series-s4e9_template/model/model_xgboost.py +++ b/rdagent/scenarios/kaggle/experiment/playground-series-s4e9_template/model/model_xgboost.py @@ -2,15 +2,8 @@ import pandas as pd import xgboost as xgb -def select(X: pd.DataFrame) -> pd.DataFrame: - # Ignore feature selection logic - return X - - def fit(X_train: pd.DataFrame, y_train: pd.DataFrame, X_valid: pd.DataFrame, y_valid: pd.DataFrame): """Define and train the model. Merge feature_select""" - X_train = select(X_train) - X_valid = select(X_valid) dtrain = xgb.DMatrix(X_train, label=y_train) dvalid = xgb.DMatrix(X_valid, label=y_valid) @@ -33,7 +26,6 @@ def predict(model, X): """ Keep feature select's consistency. """ - X = select(X) dtest = xgb.DMatrix(X) y_pred = model.predict(dtest) return y_pred.reshape(-1, 1) diff --git a/rdagent/scenarios/kaggle/experiment/playground-series-s4e9_template/model/select_randomforest.py b/rdagent/scenarios/kaggle/experiment/playground-series-s4e9_template/model/select_randomforest.py new file mode 100644 index 00000000..d2a15dee --- /dev/null +++ b/rdagent/scenarios/kaggle/experiment/playground-series-s4e9_template/model/select_randomforest.py @@ -0,0 +1,12 @@ +import pandas as pd + + +def select(X: pd.DataFrame) -> pd.DataFrame: + """ + Select relevant features. To be used in fit & predict function. + """ + # For now, we assume all features are relevant. This can be expanded to feature selection logic. + if X.columns.nlevels == 1: + return X + X.columns = ["_".join(str(col)).strip() for col in X.columns.values] + return X diff --git a/rdagent/scenarios/kaggle/experiment/playground-series-s4e9_template/model/select_xgboost.py b/rdagent/scenarios/kaggle/experiment/playground-series-s4e9_template/model/select_xgboost.py new file mode 100644 index 00000000..d2a15dee --- /dev/null +++ b/rdagent/scenarios/kaggle/experiment/playground-series-s4e9_template/model/select_xgboost.py @@ -0,0 +1,12 @@ +import pandas as pd + + +def select(X: pd.DataFrame) -> pd.DataFrame: + """ + Select relevant features. To be used in fit & predict function. + """ + # For now, we assume all features are relevant. This can be expanded to feature selection logic. + if X.columns.nlevels == 1: + return X + X.columns = ["_".join(str(col)).strip() for col in X.columns.values] + return X diff --git a/rdagent/scenarios/kaggle/experiment/playground-series-s4e9_template/train.py b/rdagent/scenarios/kaggle/experiment/playground-series-s4e9_template/train.py index 46c34a85..81222f38 100644 --- a/rdagent/scenarios/kaggle/experiment/playground-series-s4e9_template/train.py +++ b/rdagent/scenarios/kaggle/experiment/playground-series-s4e9_template/train.py @@ -29,10 +29,8 @@ def import_module_from_path(module_name, module_path): return module -print("begin preprocess") # 1) Preprocess the data X_train, X_valid, y_train, y_valid, X_test, ids = preprocess_script() -print("preprocess done") # 2) Auto feature engineering X_train_l, X_valid_l = [], [] @@ -45,9 +43,10 @@ for f in DIRNAME.glob("feature/feat*.py"): X_valid_f = cls.transform(X_valid) X_test_f = cls.transform(X_test) - X_train_l.append(X_train_f) - X_valid_l.append(X_valid_f) - X_test_l.append(X_test_f) + if X_train_f.shape[-1] == X_valid_f.shape[-1] and X_train_f.shape[-1] == X_test_f.shape[-1]: + X_train_l.append(X_train_f) + X_valid_l.append(X_valid_f) + X_test_l.append(X_test_f) X_train = pd.concat(X_train_l, axis=1, keys=[f"feature_{i}" for i in range(len(X_train_l))]) X_valid = pd.concat(X_valid_l, axis=1, keys=[f"feature_{i}" for i in range(len(X_valid_l))]) @@ -75,48 +74,33 @@ X_test = X_test.loc[:, ~X_test.columns.duplicated()] # 3) Train the model -def flatten_columns(df: pd.DataFrame) -> pd.DataFrame: - """ - Flatten the columns of a DataFrame with MultiIndex columns, - for (feature_0, a), (feature_0, b) -> feature_0_a, feature_0_b - """ - if df.columns.nlevels == 1: - return df - df.columns = ["_".join(col).strip() for col in df.columns.values] - return df - - -X_train = flatten_columns(X_train) -X_valid = flatten_columns(X_valid) -X_test = flatten_columns(X_test) - model_l = [] # list[tuple[model, predict_func,]] for f in DIRNAME.glob("model/model*.py"): + select_python_path = f.with_name(f.stem.replace("model", "select") + f.suffix) + select_m = import_module_from_path(select_python_path.stem, select_python_path) + X_train_selected = select_m.select(X_train.copy()) + X_valid_selected = select_m.select(X_valid.copy()) + m = import_module_from_path(f.stem, f) - model_l.append((m.fit(X_train, y_train, X_valid, y_valid), m.predict)) + model_l.append((m.fit(X_train_selected, y_train, X_valid_selected, y_valid), m.predict, select_m)) # 4) Evaluate the model on the validation set -y_valid_pred_l = [] -for model, predict_func in model_l: - y_valid_pred_l.append(predict_func(model, X_valid)) - print(predict_func(model, X_valid).shape) +metrics_all = [] +for model, predict_func, select_m in model_l: + X_valid_selected = select_m.select(X_valid.copy()) + y_valid_pred = predict_func(model, X_valid_selected) + rmse = compute_rmse(y_valid, y_valid_pred) + print(f"RMSE on valid set: {rmse}") + metrics_all.append(rmse) -# 5) Ensemble -y_valid_pred = np.mean(y_valid_pred_l, axis=0) +# 5) Save the validation accuracy +min_index = np.argmin(metrics_all) +pd.Series(data=[metrics_all[min_index]], index=["RMSE"]).to_csv("submission_score.csv") -rmse = compute_rmse(y_valid, y_valid_pred) -print("Final RMSE on validation set: ", rmse) +# 6) Make predictions on the test set and save them +X_test_selected = model_l[min_index][2].select(X_test.copy()) +y_test_pred = model_l[min_index][1](model_l[min_index][0], X_test_selected).ravel() -# 6) Save the validation RMSE -pd.Series(data=[rmse], index=["RMSE"]).to_csv("submission_score.csv") - -# 7) Make predictions on the test set and save them -y_test_pred_l = [] -for m, m_pred in model_l: - y_test_pred_l.append(m_pred(m, X_test)) - -y_test_pred = np.mean(y_test_pred_l, axis=0).ravel() - -# 8) Submit predictions for the test set +# 7) Submit predictions for the test set submission_result = pd.DataFrame({"id": ids, "price": y_test_pred}) submission_result.to_csv("submission.csv", index=False) diff --git a/rdagent/scenarios/kaggle/experiment/prompts.yaml b/rdagent/scenarios/kaggle/experiment/prompts.yaml index 662d3897..5bb00ecc 100644 --- a/rdagent/scenarios/kaggle/experiment/prompts.yaml +++ b/rdagent/scenarios/kaggle/experiment/prompts.yaml @@ -125,22 +125,19 @@ kg_feature_interface: |- kg_model_interface: |- Your code should contain several parts: 1. The import part: import the necessary libraries. - 2. A select() function that handles feature selection for both training and prediction phases. - The function should take the following arguments: - - X: The features as a pandas DataFrame. - The function should return the selected features as a pandas DataFrame. - 3. A function called fit() that trains the model and returns the trained model. If feature selection is applied, it should be done within this function. + 2. A function called fit() that trains the model and returns the trained model. The function should take the following arguments: - X_train: The training features as a pandas DataFrame. - y_train: The training labels as a pandas Series. - X_valid: The validation features as a pandas DataFrame. - y_valid: The validation labels as a pandas Series. The function should return the trained model. - 4. A function called predict() that makes predictions using the trained model. If feature selection is applied, it should be done within this function. + 3. A function called predict() that makes predictions using the trained model. The function should take the following arguments: - model: The trained model. - X: The features as a pandas DataFrame. The function should return the predicted probabilities or boolean predictions in numpy.ndarray format. + """ Here are some examples of how your Python code should be structured: @@ -151,15 +148,9 @@ kg_model_interface: |- import xgboost from xgboost import DMatrix - - def select(X: pd.DataFrame) -> pd.DataFrame: ... # Implement feature selection logic - - def fit( X_train: pd.DataFrame, y_train: pd.Series, X_valid: pd.DataFrame, y_valid: pd.Series ) -> xgboost.Booster: - X_train = select(X_train) - X_valid = select(X_valid) dtrain = DMatrix(X_train, label=y_train) dvalid = DMatrix(X_valid, label=y_valid) params = ... # Set parameters to XGBoost model @@ -171,7 +162,6 @@ kg_model_interface: |- def predict(model: xgboost.Booster, X: pd.DataFrame) -> np.ndarray: - X = select(X) dtest = DMatrix(X) y_pred = model.predict(dtest) @@ -185,15 +175,9 @@ kg_model_interface: |- from sklearn.ensemble import RandomForestClassifier, RandomForestRegressor from sklearn.metrics import accuracy_score - - def select(X: pd.DataFrame) -> pd.DataFrame: ... # Implement feature selection logic - - def fit( X_train: pd.DataFrame, y_train: pd.Series, X_valid: pd.DataFrame, y_valid: pd.Series ) -> RandomForestClassifier | RandomForestRegressor: - X_train = select(X_train) - X_valid = select(X_valid) model = RandomForestClassifier(...) # fir classification tasks model = RandomForestRegressor(...) # for regression tasks model.fit(X_train, y_train, ...) # Train the model @@ -202,7 +186,6 @@ kg_model_interface: |- def predict(model: RandomForestClassifier | RandomForestRegressor, X: pd.DataFrame) -> np.ndarray: - X = select(X) y_pred = model.predict(X) return y_pred @@ -214,15 +197,9 @@ kg_model_interface: |- import numpy as np from lightgbm import LGBMClassifier, LGBMRegressor - - def select(X: pd.DataFrame) -> pd.DataFrame: ... # Implement feature selection logic - - def fit( X_train: pd.DataFrame, y_train: pd.Series, X_valid: pd.DataFrame, y_valid: pd.Series ) -> LGBMClassifier | LGBMRegressor: - X_train = select(X_train) - X_valid = select(X_valid) model = LGBMClassifier(...) # for classification tasks, please add parameters here model = LGBMRegressor(...) # for regression tasks, please add parameters here @@ -231,7 +208,6 @@ kg_model_interface: |- def predict(model: LGBMClassifier | LGBMRegressor, X: pd.DataFrame) -> np.ndarray: - X = select(X) y_pred = model.predict(X) return y_pred @@ -254,13 +230,7 @@ kg_model_interface: |- # Define the forward pass return x - - def select(X: pd.DataFrame) -> pd.DataFrame: ... # Implement feature selection logic - - def fit(X_train: pd.DataFrame, y_train: pd.DataFrame, X_valid: pd.DataFrame, y_valid: pd.DataFrame) -> torch.nn.Module: - X_train = select(X_train) - X_valid = select(X_valid) model = NNModel() # Initialize the model, You can write your own model class optimizer = torch.optim.Adam(model.parameters(), lr=0.01) # Example optimizer, you can use any optimizer @@ -293,7 +263,6 @@ kg_model_interface: |- def predict(model: torch.nn.Module, X: pd.DataFrame) -> np.ndarray: - X = select(X) X = torch.tensor(X.values).float() model.eval() with torch.no_grad(): diff --git a/rdagent/scenarios/kaggle/experiment/scenario.py b/rdagent/scenarios/kaggle/experiment/scenario.py index eddf251f..3cf0b6ac 100644 --- a/rdagent/scenarios/kaggle/experiment/scenario.py +++ b/rdagent/scenarios/kaggle/experiment/scenario.py @@ -39,7 +39,6 @@ class KGScenario(Scenario): self.vector_base = None self._analysis_competition_description() self.if_action_choosing_based_on_UCB = KAGGLE_IMPLEMENT_SETTING.if_action_choosing_based_on_UCB - self.if_using_feature_selection = KAGGLE_IMPLEMENT_SETTING.if_using_feature_selection self.if_using_graph_rag = KAGGLE_IMPLEMENT_SETTING.if_using_graph_rag self.if_using_vector_rag = KAGGLE_IMPLEMENT_SETTING.if_using_vector_rag diff --git a/rdagent/scenarios/kaggle/experiment/sf-crime_template/model/model_randomforest.py b/rdagent/scenarios/kaggle/experiment/sf-crime_template/model/model_randomforest.py index 33ed3eb7..0eefd078 100644 --- a/rdagent/scenarios/kaggle/experiment/sf-crime_template/model/model_randomforest.py +++ b/rdagent/scenarios/kaggle/experiment/sf-crime_template/model/model_randomforest.py @@ -7,15 +7,6 @@ baseline model for many classification tasks. import pandas as pd from sklearn.ensemble import RandomForestClassifier -from sklearn.metrics import accuracy_score - - -def select(X: pd.DataFrame) -> pd.DataFrame: - """ - Select relevant features. To be used in fit & predict function. - """ - # For now, we assume all features are relevant. This can be expanded to feature selection logic. - return X def fit(X_train: pd.DataFrame, y_train: pd.Series, X_valid: pd.DataFrame, y_valid: pd.Series): @@ -25,17 +16,8 @@ def fit(X_train: pd.DataFrame, y_train: pd.Series, X_valid: pd.DataFrame, y_vali # Initialize the Random Forest model model = RandomForestClassifier(n_estimators=10, random_state=32, n_jobs=-1) - # Select features (if any feature selection is needed) - X_train_selected = select(X_train) - X_valid_selected = select(X_valid) - # Fit the model - model.fit(X_train_selected, y_train) - - # Validate the model - y_valid_pred = model.predict(X_valid_selected) - # accuracy = accuracy_score(y_valid, y_valid_pred) - # print(f"Validation Accuracy: {accuracy:.4f}") + model.fit(X_train, y_train) return model @@ -44,11 +26,8 @@ def predict(model, X): """ Keep feature selection's consistency and make predictions. """ - # Select features (if any feature selection is needed) - X_selected = select(X) - # Predict using the trained model - y_pred_prob = model.predict_proba(X_selected) + y_pred_prob = model.predict_proba(X) # Apply threshold to get boolean predictions return y_pred_prob diff --git a/rdagent/scenarios/kaggle/experiment/sf-crime_template/model/model_xgboost.py b/rdagent/scenarios/kaggle/experiment/sf-crime_template/model/model_xgboost.py index 3fac9a30..cd26e961 100644 --- a/rdagent/scenarios/kaggle/experiment/sf-crime_template/model/model_xgboost.py +++ b/rdagent/scenarios/kaggle/experiment/sf-crime_template/model/model_xgboost.py @@ -7,15 +7,8 @@ import pandas as pd import xgboost as xgb -def select(X: pd.DataFrame) -> pd.DataFrame: - # Ignore feature selection logic - return X - - def fit(X_train: pd.DataFrame, y_train: pd.DataFrame, X_valid: pd.DataFrame, y_valid: pd.DataFrame): """Define and train the model. Merge feature_select""" - X_train = select(X_train) - X_valid = select(X_valid) dtrain = xgb.DMatrix(X_train, label=y_train) dvalid = xgb.DMatrix(X_valid, label=y_valid) num_classes = len(np.unique(y_train)) @@ -40,7 +33,6 @@ def predict(model, X): """ Keep feature select's consistency. """ - X = select(X) dtest = xgb.DMatrix(X) y_pred_prob = model.predict(dtest) return y_pred_prob diff --git a/rdagent/scenarios/kaggle/experiment/sf-crime_template/model/select_randomforest.py b/rdagent/scenarios/kaggle/experiment/sf-crime_template/model/select_randomforest.py new file mode 100644 index 00000000..d2a15dee --- /dev/null +++ b/rdagent/scenarios/kaggle/experiment/sf-crime_template/model/select_randomforest.py @@ -0,0 +1,12 @@ +import pandas as pd + + +def select(X: pd.DataFrame) -> pd.DataFrame: + """ + Select relevant features. To be used in fit & predict function. + """ + # For now, we assume all features are relevant. This can be expanded to feature selection logic. + if X.columns.nlevels == 1: + return X + X.columns = ["_".join(str(col)).strip() for col in X.columns.values] + return X diff --git a/rdagent/scenarios/kaggle/experiment/sf-crime_template/model/select_xgboost.py b/rdagent/scenarios/kaggle/experiment/sf-crime_template/model/select_xgboost.py new file mode 100644 index 00000000..d2a15dee --- /dev/null +++ b/rdagent/scenarios/kaggle/experiment/sf-crime_template/model/select_xgboost.py @@ -0,0 +1,12 @@ +import pandas as pd + + +def select(X: pd.DataFrame) -> pd.DataFrame: + """ + Select relevant features. To be used in fit & predict function. + """ + # For now, we assume all features are relevant. This can be expanded to feature selection logic. + if X.columns.nlevels == 1: + return X + X.columns = ["_".join(str(col)).strip() for col in X.columns.values] + return X diff --git a/rdagent/scenarios/kaggle/experiment/sf-crime_template/train.py b/rdagent/scenarios/kaggle/experiment/sf-crime_template/train.py index 516f5a7f..14245c63 100644 --- a/rdagent/scenarios/kaggle/experiment/sf-crime_template/train.py +++ b/rdagent/scenarios/kaggle/experiment/sf-crime_template/train.py @@ -49,10 +49,11 @@ for f in DIRNAME.glob("feature/feat*.py"): X_valid_l.append(X_valid_f) X_test_l.append(X_test_f) -X_train = pd.concat(X_train_l, axis=1) -X_valid = pd.concat(X_valid_l, axis=1) -X_test = pd.concat(X_test_l, axis=1) +X_train = pd.concat(X_train_l, axis=1, keys=[f"feature_{i}" for i in range(len(X_train_l))]) +X_valid = pd.concat(X_valid_l, axis=1, keys=[f"feature_{i}" for i in range(len(X_valid_l))]) +X_test = pd.concat(X_test_l, axis=1, keys=[f"feature_{i}" for i in range(len(X_test_l))]) +print(X_train.shape, X_valid.shape, X_test.shape) # Handle inf and -inf values X_train.replace([np.inf, -np.inf], np.nan, inplace=True) @@ -72,61 +73,39 @@ X_train = X_train.loc[:, ~X_train.columns.duplicated()] X_valid = X_valid.loc[:, ~X_valid.columns.duplicated()] X_test = X_test.loc[:, ~X_test.columns.duplicated()] -print(X_train.shape, X_valid.shape, X_test.shape) - - # 3) Train the model -def flatten_columns(df: pd.DataFrame) -> pd.DataFrame: - """ - Flatten the columns of a DataFrame with MultiIndex columns, - for (feature_0, a), (feature_0, b) -> feature_0_a, feature_0_b - """ - if df.columns.nlevels == 1: - return df - df.columns = ["_".join(col).strip() for col in df.columns.values] - return df - - -X_train = flatten_columns(X_train) -X_valid = flatten_columns(X_valid) -X_test = flatten_columns(X_test) - model_l = [] # list[tuple[model, predict_func]] for f in DIRNAME.glob("model/model*.py"): + select_python_path = f.with_name(f.stem.replace("model", "select") + f.suffix) + select_m = import_module_from_path(select_python_path.stem, select_python_path) + X_train_selected = select_m.select(X_train.copy()) + X_valid_selected = select_m.select(X_valid.copy()) + m = import_module_from_path(f.stem, f) - model_l.append((m.fit(X_train, y_train, X_valid, y_valid), m.predict)) + model_l.append((m.fit(X_train_selected, y_train, X_valid_selected, y_valid), m.predict, select_m)) # 4) Evaluate the model on the validation set -y_valid_pred_l = [] -for model, predict_func in model_l: - y_valid_pred_l.append(predict_func(model, X_valid)) - # print(predict_func(model, X_valid)) - print(predict_func(model, X_valid).shape) +metrics_all = [] +for model, predict_func, select_m in model_l: + X_valid_selected = select_m.select(X_valid.copy()) + y_valid_pred = predict_func(model, X_valid_selected) + metrics = compute_metrics_for_classification(y_valid, y_valid_pred) + print(f"log_loss on valid set: {metrics}") + metrics_all.append(metrics) -# 5) Ensemble -from scipy import stats +# 5) Save the validation accuracy +min_index = np.argmin(metrics_all) +pd.Series(data=[metrics_all[min_index]], index=["log_loss"]).to_csv("submission_score.csv") -# average probabilities ensemble -y_valid_pred_proba = np.mean(y_valid_pred_l, axis=0) +# 6) Make predictions on the test set and save them +X_test_selected = model_l[min_index][2].select(X_test.copy()) +y_test_pred = model_l[min_index][1](model_l[min_index][0], X_test_selected) -# Compute metrics -logloss = compute_metrics_for_classification(y_valid, y_valid_pred_proba) -print(f"final log_loss on valid set: {logloss}") - -# 6) Save the validation metrics -pd.Series(data=[logloss], index=["log_loss"]).to_csv("submission_score.csv") - -# 7) Make predictions on the test set and save them -y_test_pred_l = [] -for model, predict_func in model_l: - y_test_pred_l.append(predict_func(model, X_test)) - -# For multiclass classification, use the mode of the predictions -y_test_pred_proba = np.mean(y_test_pred_l, axis=0) +# 7) Submit predictions for the test set class_labels = category_encoder.classes_ -submission_result = pd.DataFrame(y_test_pred_proba, columns=class_labels) +submission_result = pd.DataFrame(y_test_pred, columns=class_labels) submission_result.insert(0, "Id", test_ids) submission_result.to_csv("submission.csv", index=False) diff --git a/rdagent/scenarios/kaggle/experiment/spaceship-titanic_template/model/model_nn.py b/rdagent/scenarios/kaggle/experiment/spaceship-titanic_template/model/model_nn.py index a80e6654..138881c8 100644 --- a/rdagent/scenarios/kaggle/experiment/spaceship-titanic_template/model/model_nn.py +++ b/rdagent/scenarios/kaggle/experiment/spaceship-titanic_template/model/model_nn.py @@ -49,8 +49,8 @@ def fit(X_train, y_train, X_valid, y_valid): # Train the model model.train() - for epoch in range(100): - print(f"Epoch {epoch + 1}/100") + for epoch in range(50): + print(f"Epoch {epoch + 1}/50") epoch_loss = 0 for X_batch, y_batch in tqdm(train_loader, desc="Training", leave=False): X_batch, y_batch = X_batch.to(device), y_batch.to(device) # Move data to the device diff --git a/rdagent/scenarios/kaggle/experiment/spaceship-titanic_template/model/model_randomforest.py b/rdagent/scenarios/kaggle/experiment/spaceship-titanic_template/model/model_randomforest.py index 377683b9..f0bca61c 100644 --- a/rdagent/scenarios/kaggle/experiment/spaceship-titanic_template/model/model_randomforest.py +++ b/rdagent/scenarios/kaggle/experiment/spaceship-titanic_template/model/model_randomforest.py @@ -10,14 +10,6 @@ from sklearn.ensemble import RandomForestClassifier from sklearn.metrics import accuracy_score -def select(X: pd.DataFrame) -> pd.DataFrame: - """ - Select relevant features. To be used in fit & predict function. - """ - # For now, we assume all features are relevant. This can be expanded to feature selection logic. - return X - - def fit(X_train: pd.DataFrame, y_train: pd.Series, X_valid: pd.DataFrame, y_valid: pd.Series): """ Define and train the Random Forest model. Merge feature selection into the pipeline. @@ -25,17 +17,8 @@ def fit(X_train: pd.DataFrame, y_train: pd.Series, X_valid: pd.DataFrame, y_vali # Initialize the Random Forest model model = RandomForestClassifier(n_estimators=100, random_state=32, n_jobs=-1) - # Select features (if any feature selection is needed) - X_train_selected = select(X_train) - X_valid_selected = select(X_valid) - # Fit the model - model.fit(X_train_selected, y_train) - - # Validate the model - y_valid_pred = model.predict(X_valid_selected) - accuracy = accuracy_score(y_valid, y_valid_pred) - print(f"Validation Accuracy: {accuracy:.4f}") + model.fit(X_train, y_train) return model @@ -44,11 +27,8 @@ def predict(model, X): """ Keep feature selection's consistency and make predictions. """ - # Select features (if any feature selection is needed) - X_selected = select(X) - # Predict using the trained model - y_pred_prob = model.predict_proba(X_selected)[:, 1] + y_pred_prob = model.predict_proba(X)[:, 1] # Apply threshold to get boolean predictions return y_pred_prob.reshape(-1, 1) diff --git a/rdagent/scenarios/kaggle/experiment/spaceship-titanic_template/model/model_xgboost.py b/rdagent/scenarios/kaggle/experiment/spaceship-titanic_template/model/model_xgboost.py index 5dec6433..83d82afb 100644 --- a/rdagent/scenarios/kaggle/experiment/spaceship-titanic_template/model/model_xgboost.py +++ b/rdagent/scenarios/kaggle/experiment/spaceship-titanic_template/model/model_xgboost.py @@ -6,19 +6,11 @@ import pandas as pd import xgboost as xgb -def select(X: pd.DataFrame) -> pd.DataFrame: - # Ignore feature selection logic - return X - - def fit(X_train: pd.DataFrame, y_train: pd.DataFrame, X_valid: pd.DataFrame, y_valid: pd.DataFrame): """Define and train the model. Merge feature_select""" - X_train = select(X_train) - X_valid = select(X_valid) dtrain = xgb.DMatrix(X_train, label=y_train) dvalid = xgb.DMatrix(X_valid, label=y_valid) - # TODO: for quick running.... params = { "nthread": -1, "tree_method": "gpu_hist", @@ -36,7 +28,6 @@ def predict(model, X): """ Keep feature select's consistency. """ - X = select(X) dtest = xgb.DMatrix(X) y_pred_prob = model.predict(dtest) return y_pred_prob.reshape(-1, 1) diff --git a/rdagent/scenarios/kaggle/experiment/spaceship-titanic_template/model/select_nn.py b/rdagent/scenarios/kaggle/experiment/spaceship-titanic_template/model/select_nn.py new file mode 100644 index 00000000..d2a15dee --- /dev/null +++ b/rdagent/scenarios/kaggle/experiment/spaceship-titanic_template/model/select_nn.py @@ -0,0 +1,12 @@ +import pandas as pd + + +def select(X: pd.DataFrame) -> pd.DataFrame: + """ + Select relevant features. To be used in fit & predict function. + """ + # For now, we assume all features are relevant. This can be expanded to feature selection logic. + if X.columns.nlevels == 1: + return X + X.columns = ["_".join(str(col)).strip() for col in X.columns.values] + return X diff --git a/rdagent/scenarios/kaggle/experiment/spaceship-titanic_template/model/select_randomforest.py b/rdagent/scenarios/kaggle/experiment/spaceship-titanic_template/model/select_randomforest.py new file mode 100644 index 00000000..d2a15dee --- /dev/null +++ b/rdagent/scenarios/kaggle/experiment/spaceship-titanic_template/model/select_randomforest.py @@ -0,0 +1,12 @@ +import pandas as pd + + +def select(X: pd.DataFrame) -> pd.DataFrame: + """ + Select relevant features. To be used in fit & predict function. + """ + # For now, we assume all features are relevant. This can be expanded to feature selection logic. + if X.columns.nlevels == 1: + return X + X.columns = ["_".join(str(col)).strip() for col in X.columns.values] + return X diff --git a/rdagent/scenarios/kaggle/experiment/spaceship-titanic_template/model/select_xgboost.py b/rdagent/scenarios/kaggle/experiment/spaceship-titanic_template/model/select_xgboost.py new file mode 100644 index 00000000..d2a15dee --- /dev/null +++ b/rdagent/scenarios/kaggle/experiment/spaceship-titanic_template/model/select_xgboost.py @@ -0,0 +1,12 @@ +import pandas as pd + + +def select(X: pd.DataFrame) -> pd.DataFrame: + """ + Select relevant features. To be used in fit & predict function. + """ + # For now, we assume all features are relevant. This can be expanded to feature selection logic. + if X.columns.nlevels == 1: + return X + X.columns = ["_".join(str(col)).strip() for col in X.columns.values] + return X diff --git a/rdagent/scenarios/kaggle/experiment/spaceship-titanic_template/train.py b/rdagent/scenarios/kaggle/experiment/spaceship-titanic_template/train.py index c042734f..fe8e3ad5 100644 --- a/rdagent/scenarios/kaggle/experiment/spaceship-titanic_template/train.py +++ b/rdagent/scenarios/kaggle/experiment/spaceship-titanic_template/train.py @@ -5,8 +5,7 @@ from pathlib import Path import numpy as np import pandas as pd from fea_share_preprocess import preprocess_script -from sklearn.metrics import accuracy_score, matthews_corrcoef -from sklearn.preprocessing import LabelEncoder +from sklearn.metrics import accuracy_score # Set random seed for reproducibility SEED = 42 @@ -30,7 +29,6 @@ def import_module_from_path(module_name, module_path): # 1) Preprocess the data -# TODO 如果已经做过数据预处理了，不需要再做了 X_train, X_valid, y_train, y_valid, X_test, passenger_ids = preprocess_script() # 2) Auto feature engineering @@ -44,9 +42,10 @@ for f in DIRNAME.glob("feature/feat*.py"): X_valid_f = cls.transform(X_valid) X_test_f = cls.transform(X_test) - X_train_l.append(X_train_f) - X_valid_l.append(X_valid_f) - X_test_l.append(X_test_f) + if X_train_f.shape[-1] == X_valid_f.shape[-1] and X_train_f.shape[-1] == X_test_f.shape[-1]: + X_train_l.append(X_train_f) + X_valid_l.append(X_valid_f) + X_test_l.append(X_test_f) X_train = pd.concat(X_train_l, axis=1, keys=[f"feature_{i}" for i in range(len(X_train_l))]) X_valid = pd.concat(X_valid_l, axis=1, keys=[f"feature_{i}" for i in range(len(X_valid_l))]) @@ -74,49 +73,34 @@ X_test = X_test.loc[:, ~X_test.columns.duplicated()] # 3) Train the model -def flatten_columns(df: pd.DataFrame) -> pd.DataFrame: - """ - Flatten the columns of a DataFrame with MultiIndex columns, - for (feature_0, a), (feature_0, b) -> feature_0_a, feature_0_b - """ - if df.columns.nlevels == 1: - return df - df.columns = ["_".join(col).strip() for col in df.columns.values] - return df - - -X_train = flatten_columns(X_train) -X_valid = flatten_columns(X_valid) -X_test = flatten_columns(X_test) - model_l = [] # list[tuple[model, predict_func,]] for f in DIRNAME.glob("model/model*.py"): + select_python_path = f.with_name(f.stem.replace("model", "select") + f.suffix) + select_m = import_module_from_path(select_python_path.stem, select_python_path) + X_train_selected = select_m.select(X_train.copy()) + X_valid_selected = select_m.select(X_valid.copy()) + m = import_module_from_path(f.stem, f) - model_l.append((m.fit(X_train, y_train, X_valid, y_valid), m.predict)) + model_l.append((m.fit(X_train_selected, y_train, X_valid_selected, y_valid), m.predict, select_m)) # 4) Evaluate the model on the validation set -y_valid_pred_l = [] -for model, predict_func in model_l: - y_valid_pred_l.append(predict_func(model, X_valid)) +metrics_all = [] +for model, predict_func, select_m in model_l: + X_valid_selected = select_m.select(X_valid.copy()) + y_valid_pred = predict_func(model, X_valid_selected) + y_valid_pred = (y_valid_pred > 0.5).astype(int) + metrics = compute_metrics_for_classification(y_valid, y_valid_pred) + print(f"Accuracy on valid set: {metrics}") + metrics_all.append(metrics) -# 5) Ensemble -# TODO: ensemble method in a script -# Average the predictions and apply a threshold to determine class labels -y_valid_pred = np.mean(y_valid_pred_l, axis=0) -y_valid_pred = (y_valid_pred > 0.5).astype(int) -mcc = compute_metrics_for_classification(y_valid, y_valid_pred) -print("Final on validation set: ", mcc) +# 5) Save the validation accuracy +min_index = np.argmax(metrics_all) +pd.Series(data=[metrics_all[min_index]], index=["MCC"]).to_csv("submission_score.csv") -# 6) Save the validation accuracy -pd.Series(data=[mcc], index=["MCC"]).to_csv("submission_score.csv") - -# 7) Make predictions on the test set and save them -y_test_pred_l = [] -for m, m_pred in model_l: - y_test_pred_l.append(m_pred(m, X_test)) # TODO Make this an ensemble. Currently it uses the last prediction - -y_test_pred = np.mean(y_test_pred_l, axis=0) +# 6) Make predictions on the test set and save them +X_test_selected = model_l[min_index][2].select(X_test.copy()) +y_test_pred = model_l[min_index][1](model_l[min_index][0], X_test_selected) y_test_pred = (y_test_pred > 0.5).astype(bool) y_test_pred = y_test_pred.ravel() diff --git a/rdagent/scenarios/kaggle/experiment/workspace.py b/rdagent/scenarios/kaggle/experiment/workspace.py index 7a85f2ab..c8a19172 100644 --- a/rdagent/scenarios/kaggle/experiment/workspace.py +++ b/rdagent/scenarios/kaggle/experiment/workspace.py @@ -1,7 +1,7 @@ import subprocess import zipfile from pathlib import Path -from typing import Any +from typing import Any, List, Tuple import pandas as pd @@ -29,8 +29,15 @@ class KGFBWorkspace(FBWorkspace): def __init__(self, template_folder_path: Path, *args, **kwargs) -> None: super().__init__(*args, **kwargs) self.inject_code_from_folder(template_folder_path) - self.data_description: list[str] = [] - self.model_description: dict[str, str] = {} + self.data_description: List[Tuple[str, int]] = [] + + @property + def model_description(self) -> dict[str, str]: + model_description = {} + for k, v in self.code_dict.items(): + if k.startswith("model/"): + model_description[k] = v + return model_description def generate_preprocess_data( self, diff --git a/rdagent/scenarios/kaggle/prompts.yaml b/rdagent/scenarios/kaggle/prompts.yaml index c6cb9828..a052c2a3 100644 --- a/rdagent/scenarios/kaggle/prompts.yaml +++ b/rdagent/scenarios/kaggle/prompts.yaml @@ -39,7 +39,7 @@ hypothesis_and_feedback: |- hypothesis_output_format: |- The output should follow JSON format. The schema is as follows: { - "action": "If "hypothesis_specification" provides the action you need to take, please follow "hypothesis_specification" to choose the action. Otherwise, based on previous experimental results, suggest the action you believe is most appropriate at the moment. It should be one of [{% if if_using_feature_selection %}"Feature engineering", "Feature processing", "Model feature selection", "Model tuning"{% else %}"Feature engineering", "Feature processing", "Model tuning"{% endif %}]", + "action": "If "hypothesis_specification" provides the action you need to take, please follow "hypothesis_specification" to choose the action. Otherwise, based on previous experimental results, suggest the action you believe is most appropriate at the moment. It should be one of ["Feature engineering", "Feature processing", "Model feature selection", "Model tuning"]" "hypothesis": "The new hypothesis generated based on the information provided.", "reason": "The reason why you generate this hypothesis. It should be comprehensive and logical. It should cover the other keys below and extend them.", "concise_reason": "Two-line summary. First line focuses on a concise justification for the change. Second line generalizes a knowledge statement.", @@ -291,26 +291,28 @@ feature_selection_feedback_generation: 3. How might we refine or change the feature selection approach to improve model performance? 4. Are there any domain-specific considerations that should inform our feature selection? -extract_model_task_from_code: +model_feature_selection: system: |- - You are an expert in analyzing code for machine learning models. - user: |- - Given the following code, summarize the machine learning model including: - - Model architecture - - Hyperparameters - - Formulation and variables - - Model type (one of XGBoost, RandomForest, LightGBM, NN) + You are an assistant for model feature selection in machine learning. Your task is to understand the current feature groups and choose the most relevant features for the model to get the best performance. - Code: - {{ file_content }} + The user is currently working on a Kaggle competition scenario as follows: + {{ scenario }} - Return the information in JSON format with the following structure: + The user is now working on the following model type: + {{ model_type }} + + The user will give you several feature groups and their descriptions. Your task is to select the most relevant features for the model to achieve the best performance. You should consider the following: + 1. How well do the selected features support the scenario? + 2. Are there any features that might be redundant or noisy? + + Please answer the chosen group index in JSON format. Example JSON structure for Result Analysis: { - "name": "", - "description": "", - "architecture": "", - "hyperparameters": {}, - "formulation": "", - "variables": {}, - "model_type": "" + "Selected Group Index": [1, 3, 5], # List of selected group indices, notice: the index starts from 1 } + + user: |- + Current feature groups: + {% for feature in feature_groups %} + Group {{ loop.index }}: + {{ feature }} + {% endfor %} \ No newline at end of file diff --git a/rdagent/scenarios/kaggle/proposal/proposal.py b/rdagent/scenarios/kaggle/proposal/proposal.py index 4b6a1f80..eb670a09 100644 --- a/rdagent/scenarios/kaggle/proposal/proposal.py +++ b/rdagent/scenarios/kaggle/proposal/proposal.py @@ -36,7 +36,7 @@ KG_ACTION_MODEL_TUNING = "Model tuning" KG_ACTION_LIST = [ KG_ACTION_FEATURE_PROCESSING, KG_ACTION_FEATURE_ENGINEERING, - *([KG_ACTION_MODEL_FEATURE_SELECTION] if KAGGLE_IMPLEMENT_SETTING.if_using_feature_selection else []), + KG_ACTION_MODEL_FEATURE_SELECTION, KG_ACTION_MODEL_TUNING, ] @@ -82,13 +82,9 @@ class KGHypothesisGen(ModelHypothesisGen): def __init__(self, scen: Scenario) -> Tuple[dict, bool]: super().__init__(scen) - actions = ["Feature engineering", "Feature processing", "Model tuning"] - if KAGGLE_IMPLEMENT_SETTING.if_using_feature_selection: - actions.insert(2, "Model feature selection") - self.action_counts = dict.fromkeys(actions, 0) - self.reward_estimates = {action: 0.0 for action in actions} - if KAGGLE_IMPLEMENT_SETTING.if_using_feature_selection: - self.reward_estimates["Model feature selection"] = 0.2 + self.action_counts = dict.fromkeys(KG_ACTION_LIST, 0) + self.reward_estimates = {action: 0.0 for action in KG_ACTION_LIST} + self.reward_estimates["Model feature selection"] = 0.2 self.reward_estimates["Model tuning"] = 1.0 self.confidence_parameter = 1.0 self.initial_performance = 0.0 @@ -239,9 +235,7 @@ class KGHypothesisGen(ModelHypothesisGen): context_dict = { "hypothesis_and_feedback": hypothesis_and_feedback, "RAG": self.generate_RAG_content(trace, hypothesis_and_feedback), - "hypothesis_output_format": Environment(undefined=StrictUndefined) - .from_string(prompt_dict["hypothesis_output_format"]) - .render(if_using_feature_selection=KAGGLE_IMPLEMENT_SETTING.if_using_feature_selection), + "hypothesis_output_format": prompt_dict["hypothesis_output_format"], "hypothesis_specification": ( f"next experiment action is {action}" if self.scen.if_action_choosing_based_on_UCB else None ), @@ -319,7 +313,11 @@ class KGHypothesis2Experiment(ModelHypothesis2Experiment): ) exp = KGFactorExperiment( - sub_tasks=tasks, based_experiments=([KGFactorExperiment(sub_tasks=[])] + [t[1] for t in trace.hist if t[2]]) + sub_tasks=tasks, + based_experiments=( + [KGFactorExperiment(sub_tasks=[], source_feature_size=trace.scen.input_shape[-1])] + + [t[1] for t in trace.hist if t[2]] + ), ) return exp @@ -337,7 +335,11 @@ class KGHypothesis2Experiment(ModelHypothesis2Experiment): ) ) exp = KGModelExperiment( - sub_tasks=tasks, based_experiments=([KGModelExperiment(sub_tasks=[])] + [t[1] for t in trace.hist if t[2]]) + sub_tasks=tasks, + based_experiments=( + [KGModelExperiment(sub_tasks=[], source_feature_size=trace.scen.input_shape[-1])] + + [t[1] for t in trace.hist if t[2]] + ), ) return exp