From dbe2cf12bb702f147e9497c3cc9894e23279a053 Mon Sep 17 00:00:00 2001 From: Xu Yang Date: Wed, 11 Sep 2024 15:26:52 +0800 Subject: [PATCH] feat: Kaggle loop update (Feature & Model) (#241) * Init todo * Evaluation & dataset * Generate new data * dataset generation * add the result * Analysis * Factor update * Updates * Reformat analysis.py * CI fix * Revised Preprocessing & Supported Random Forest * Revised to support three models with feature * Further revised prompts * Slight Revision * docs: update contributors (#230) * Revised to support three models with feature * Further revised prompts * Slight Revision * feat: kaggle model and feature (#238) * update first version code * make hypothesis_gen and experiment_builder fit for both feature and model * feat: continue kaggle feature and model coder (#239) * use qlib docker to run qlib models * feature coder ready * model coder ready * fix CI * finish the first round of runner (#240) * Optimized the factor scenario and added the front-end. * fix a small bug * fix a typo * update the kaggle scenario * delete model_template folder * use experiment to run data preprocess script * add source data to scenarios * minor fix * minor bug fix * train.py debug * fixed a bug in train.py and added some TODOs * For Debugging * fix two small bugs in based_exp * fix some bugs * update preprocess * fix a bug in preprocess * fix a bug in train.py * reformat * Follow-up * fix a bug in train.py * fix a bug in workspace * fix a bug in feature duplication * fix a bug in feedback * fix a bug in preprocessed data * fix a bug om feature engineering * fix a ci error * Debugged & Connected * Fixed error on feedback & added other fixes * fix CI errors * fix a CI bug * fix: fix_dotenv_error (#257) * fix_dotenv_error * format with isort * Update rdagent/app/cli.py --------- Co-authored-by: you-n-g * chore(main): release 0.2.1 (#249) Release-As: 0.2.1 * init a scenario for kaggle feature engineering * delete error codes * Delete rdagent/app/kaggle_feature/conf.py --------- Co-authored-by: Young Co-authored-by: Taozhi Wang Co-authored-by: you-n-g Co-authored-by: cyncyw <47289405+taozhiwang@users.noreply.github.com> Co-authored-by: Xisen-Wang Co-authored-by: Haotian Chen <113661982+Hytn@users.noreply.github.com> Co-authored-by: WinstonLiye <1957922024@qq.com> Co-authored-by: WinstonLiyt <104308117+WinstonLiyt@users.noreply.github.com> Co-authored-by: Linlang <30293408+SunsetWolf@users.noreply.github.com> --- rdagent/app/kaggle/conf.py | 24 +- rdagent/app/kaggle/loop.py | 98 ++++++ .../app/{kaggle => kaggle_feature}/model.py | 0 .../coder/factor_coder/CoSTEER/evaluators.py | 28 +- .../components/coder/factor_coder/factor.py | 41 ++- .../factor_execution_template.txt | 13 + .../coder/model_coder/CoSTEER/evaluators.py | 10 +- rdagent/components/coder/model_coder/model.py | 113 +++---- .../model_coder/model_execute_template_v1.txt | 44 +++ .../model_coder/model_execute_template_v2.txt | 20 ++ rdagent/components/proposal/model_proposal.py | 8 +- rdagent/components/proposal/prompts.yaml | 14 +- rdagent/components/runner/conf.py | 2 +- rdagent/core/experiment.py | 18 +- rdagent/log/ui/app.py | 64 ++-- .../scenarios/data_mining/docker/Dockerfile | 2 +- .../data_mining/proposal/model_proposal.py | 10 +- rdagent/scenarios/kaggle/developer/coder.py | 5 + .../scenarios/kaggle/developer/feedback.py | 113 +++++-- .../scenarios/kaggle/developer/model_coder.py | 3 - .../kaggle/developer/model_runner.py | 38 --- rdagent/scenarios/kaggle/developer/runner.py | 121 +++++++ rdagent/scenarios/kaggle/docker/Dockerfile | 5 +- .../kaggle/experiment/kaggle_experiment.py | 21 ++ .../meta_tpl/fea_share_preprocess.py | 77 ++++- .../meta_tpl/{feat.py => feature/feature.py} | 2 +- .../experiment/meta_tpl/model/model_rf.py | 54 +++ .../meta_tpl/{model.py => model/model_xgb.py} | 3 +- .../kaggle/experiment/meta_tpl/train.py | 105 +++--- .../experiment/model_template/README.md | 3 - .../kaggle/experiment/model_template/train.py | 105 ------ .../scenarios/kaggle/experiment/prompts.yaml | 309 +++++++++++++++--- .../{model_experiment.py => scenario.py} | 92 +++--- .../scenarios/kaggle/experiment/workspace.py | 58 +++- rdagent/scenarios/kaggle/kaggle_crawler.py | 9 +- rdagent/scenarios/kaggle/prompts.yaml | 191 +++++++---- .../kaggle/proposal/model_proposal.py | 105 ------ rdagent/scenarios/kaggle/proposal/proposal.py | 181 ++++++++++ rdagent/scenarios/qlib/docker/Dockerfile | 2 +- .../scenarios/qlib/proposal/model_proposal.py | 10 +- rdagent/utils/env.py | 71 +++- requirements.txt | 2 + 42 files changed, 1513 insertions(+), 681 deletions(-) create mode 100644 rdagent/app/kaggle/loop.py rename rdagent/app/{kaggle => kaggle_feature}/model.py (100%) create mode 100644 rdagent/components/coder/factor_coder/factor_execution_template.txt create mode 100644 rdagent/components/coder/model_coder/model_execute_template_v1.txt create mode 100644 rdagent/components/coder/model_coder/model_execute_template_v2.txt create mode 100644 rdagent/scenarios/kaggle/developer/coder.py delete mode 100644 rdagent/scenarios/kaggle/developer/model_coder.py delete mode 100644 rdagent/scenarios/kaggle/developer/model_runner.py create mode 100644 rdagent/scenarios/kaggle/developer/runner.py create mode 100644 rdagent/scenarios/kaggle/experiment/kaggle_experiment.py rename rdagent/scenarios/kaggle/experiment/meta_tpl/{feat.py => feature/feature.py} (94%) create mode 100644 rdagent/scenarios/kaggle/experiment/meta_tpl/model/model_rf.py rename rdagent/scenarios/kaggle/experiment/meta_tpl/{model.py => model/model_xgb.py} (91%) delete mode 100644 rdagent/scenarios/kaggle/experiment/model_template/README.md delete mode 100644 rdagent/scenarios/kaggle/experiment/model_template/train.py rename rdagent/scenarios/kaggle/experiment/{model_experiment.py => scenario.py} (61%) delete mode 100644 rdagent/scenarios/kaggle/proposal/model_proposal.py create mode 100644 rdagent/scenarios/kaggle/proposal/proposal.py diff --git a/rdagent/app/kaggle/conf.py b/rdagent/app/kaggle/conf.py index f8c54ebe..0ffb8f86 100644 --- a/rdagent/app/kaggle/conf.py +++ b/rdagent/app/kaggle/conf.py @@ -13,29 +13,33 @@ class PropSetting(BasePropSetting): """Add 'model_' to the protected namespaces""" # 1) overriding the default - scen: str = "rdagent.scenarios.kaggle.experiment.model_experiment.KGModelScenario" + scen: str = "rdagent.scenarios.kaggle.experiment.scenario.KGScenario" """Scenario class for data mining model""" - hypothesis_gen: str = "rdagent.scenarios.kaggle.proposal.model_proposal.KGModelHypothesisGen" + hypothesis_gen: str = "rdagent.scenarios.kaggle.proposal.proposal.KGHypothesisGen" """Hypothesis generation class""" - hypothesis2experiment: str = "rdagent.scenarios.kaggle.proposal.model_proposal.KGModelHypothesis2Experiment" + hypothesis2experiment: str = "rdagent.scenarios.kaggle.proposal.proposal.KGHypothesis2Experiment" """Hypothesis to experiment class""" - coder: str = "rdagent.scenarios.kaggle.developer.model_coder.KGModelCoSTEER" - """Coder class""" + feature_coder: str = "rdagent.scenarios.kaggle.developer.coder.KGFactorCoSTEER" + """Feature Coder class""" - runner: str = "rdagent.scenarios.kaggle.developer.model_runner.KGModelRunner" - """Runner class""" + model_coder: str = "rdagent.scenarios.kaggle.developer.coder.KGModelCoSTEER" + """Model Coder class""" - summarizer: str = "rdagent.scenarios.kaggle.developer.feedback.KGModelHypothesisExperiment2Feedback" + feature_runner: str = "rdagent.scenarios.kaggle.developer.runner.KGFactorRunner" + """Feature Runner class""" + + model_runner: str = "rdagent.scenarios.kaggle.developer.runner.KGModelRunner" + """Model Runner class""" + + summarizer: str = "rdagent.scenarios.kaggle.developer.feedback.KGHypothesisExperiment2Feedback" """Summarizer class""" evolving_n: int = 10 """Number of evolutions""" - evolving_n: int = 10 - competition: str = "" diff --git a/rdagent/app/kaggle/loop.py b/rdagent/app/kaggle/loop.py new file mode 100644 index 00000000..9658aaa1 --- /dev/null +++ b/rdagent/app/kaggle/loop.py @@ -0,0 +1,98 @@ +from collections import defaultdict +from typing import Any + +import fire + +from rdagent.app.kaggle.conf import PROP_SETTING +from rdagent.components.workflow.conf import BasePropSetting +from rdagent.components.workflow.rd_loop import RDLoop +from rdagent.core.developer import Developer +from rdagent.core.exception import ModelEmptyError +from rdagent.core.proposal import ( + Hypothesis2Experiment, + HypothesisExperiment2Feedback, + HypothesisGen, + Trace, +) +from rdagent.core.scenario import Scenario +from rdagent.core.utils import import_class +from rdagent.log import rdagent_logger as logger +from rdagent.scenarios.kaggle.proposal.proposal import ( + KG_ACTION_FEATURE_ENGINEERING, + KG_ACTION_FEATURE_PROCESSING, +) + + +class ModelRDLoop(RDLoop): + def __init__(self, PROP_SETTING: BasePropSetting): + with logger.tag("init"): + scen: Scenario = import_class(PROP_SETTING.scen)(PROP_SETTING.competition) + logger.log_object(scen, tag="scenario") + + self.hypothesis_gen: HypothesisGen = import_class(PROP_SETTING.hypothesis_gen)(scen) + logger.log_object(self.hypothesis_gen, tag="hypothesis generator") + + self.hypothesis2experiment: Hypothesis2Experiment = import_class(PROP_SETTING.hypothesis2experiment)() + logger.log_object(self.hypothesis2experiment, tag="hypothesis2experiment") + + self.feature_coder: Developer = import_class(PROP_SETTING.feature_coder)(scen) + logger.log_object(self.feature_coder, tag="feature coder") + self.model_coder: Developer = import_class(PROP_SETTING.model_coder)(scen) + logger.log_object(self.model_coder, tag="model coder") + + self.feature_runner: Developer = import_class(PROP_SETTING.feature_runner)(scen) + logger.log_object(self.feature_runner, tag="feature runner") + self.model_runner: Developer = import_class(PROP_SETTING.model_runner)(scen) + logger.log_object(self.model_runner, tag="model runner") + + self.summarizer: HypothesisExperiment2Feedback = import_class(PROP_SETTING.summarizer)(scen) + logger.log_object(self.summarizer, tag="summarizer") + self.trace = Trace(scen=scen) + super(RDLoop, self).__init__() + + def coding(self, prev_out: dict[str, Any]): + with logger.tag("d"): # develop + if prev_out["propose"].action in [KG_ACTION_FEATURE_ENGINEERING, KG_ACTION_FEATURE_PROCESSING]: + exp = self.feature_coder.develop(prev_out["exp_gen"]) + else: + exp = self.model_coder.develop(prev_out["exp_gen"]) + logger.log_object(exp.sub_workspace_list, tag="coder result") + return exp + + def running(self, prev_out: dict[str, Any]): + with logger.tag("ef"): # evaluate and feedback + if prev_out["propose"].action in [KG_ACTION_FEATURE_ENGINEERING, KG_ACTION_FEATURE_PROCESSING]: + exp = self.feature_runner.develop(prev_out["coding"]) + else: + exp = self.model_runner.develop(prev_out["coding"]) + logger.log_object(exp, tag="runner result") + return exp + + skip_loop_error = (ModelEmptyError,) + + +def main(path=None, step_n=None, competition=None): + """ + Auto R&D Evolving loop for models in a kaggle{} scenario. + + You can continue running session by + + .. code-block:: python + + dotenv run -- python rdagent/app/kaggle/loop.py [--competition titanic] $LOG_PATH/__session__/1/0_propose --step_n 1 # `step_n` is a optional paramter + + """ + if competition: + PROP_SETTING.competition = competition + if path is None: + model_loop = ModelRDLoop(PROP_SETTING) + else: + model_loop = ModelRDLoop.load(path) + model_loop.run(step_n=step_n) + + +if __name__ == "__main__": + from dotenv import load_dotenv + + load_dotenv(override=True) + fire.Fire(main) diff --git a/rdagent/app/kaggle/model.py b/rdagent/app/kaggle_feature/model.py similarity index 100% rename from rdagent/app/kaggle/model.py rename to rdagent/app/kaggle_feature/model.py diff --git a/rdagent/components/coder/factor_coder/CoSTEER/evaluators.py b/rdagent/components/coder/factor_coder/CoSTEER/evaluators.py index 8a39886e..ccd08b45 100644 --- a/rdagent/components/coder/factor_coder/CoSTEER/evaluators.py +++ b/rdagent/components/coder/factor_coder/CoSTEER/evaluators.py @@ -161,7 +161,7 @@ class FactorOutputFormatEvaluator(FactorEvaluator): ) buffer = io.StringIO() gen_df.info(buf=buffer) - gen_df_info_str = buffer.getvalue() + gen_df_info_str = f"The use is currently working on a feature related task.\nThe output dataframe info is:\n{buffer.getvalue()}" system_prompt = ( Environment(undefined=StrictUndefined) .from_string( @@ -378,6 +378,7 @@ class FactorValueEvaluator(FactorEvaluator): self, implementation: Workspace, gt_implementation: Workspace, + version: int = 1, # 1 for qlib factors and 2 for kaggle factors **kwargs, ) -> Tuple: conclusions = [] @@ -389,18 +390,21 @@ class FactorValueEvaluator(FactorEvaluator): equal_value_ratio_result = 0 high_correlation_result = False - # Check if both dataframe has only one columns - feedback_str, _ = FactorSingleColumnEvaluator(self.scen).evaluate(implementation, gt_implementation) - conclusions.append(feedback_str) + # Check if both dataframe has only one columns Mute this since factor task might generate more than one columns now + if version == 1: + feedback_str, _ = FactorSingleColumnEvaluator(self.scen).evaluate(implementation, gt_implementation) + conclusions.append(feedback_str) # Check if the index of the dataframe is ("datetime", "instrument") feedback_str, _ = FactorOutputFormatEvaluator(self.scen).evaluate(implementation, gt_implementation) conclusions.append(feedback_str) - - feedback_str, daily_check_result = FactorDatetimeDailyEvaluator(self.scen).evaluate( - implementation, gt_implementation - ) - conclusions.append(feedback_str) + if version == 1: + feedback_str, daily_check_result = FactorDatetimeDailyEvaluator(self.scen).evaluate( + implementation, gt_implementation + ) + conclusions.append(feedback_str) + else: + daily_check_result = None # Check if both dataframe have the same rows count if gt_implementation is not None: @@ -627,7 +631,9 @@ class FactorEvaluatorForCoder(FactorEvaluator): ( factor_feedback.factor_value_feedback, decision_from_value_check, - ) = self.value_evaluator.evaluate(implementation=implementation, gt_implementation=gt_implementation) + ) = self.value_evaluator.evaluate( + implementation=implementation, gt_implementation=gt_implementation, version=target_task.version + ) factor_feedback.final_decision_based_on_gt = gt_implementation is not None @@ -647,7 +653,7 @@ class FactorEvaluatorForCoder(FactorEvaluator): target_task=target_task, implementation=implementation, execution_feedback=factor_feedback.execution_feedback, - value_feedback=factor_feedback.factor_value_feedback, + factor_value_feedback=factor_feedback.factor_value_feedback, gt_implementation=gt_implementation, ) ( diff --git a/rdagent/components/coder/factor_coder/factor.py b/rdagent/components/coder/factor_coder/factor.py index 3fc38e1f..7dde2ebe 100644 --- a/rdagent/components/coder/factor_coder/factor.py +++ b/rdagent/components/coder/factor_coder/factor.py @@ -24,9 +24,11 @@ class FactorTask(Task): factor_name, factor_description, factor_formulation, + *args, variables: dict = {}, resource: str = None, factor_implementation: bool = False, + **kwargs, ) -> None: self.factor_name = factor_name self.factor_description = factor_description @@ -34,6 +36,7 @@ class FactorTask(Task): self.variables = variables self.factor_resources = resource self.factor_implementation = factor_implementation + super().__init__(*args, **kwargs) def get_task_information(self): return f"""factor_name: {self.factor_name} @@ -75,8 +78,8 @@ class FactorFBWorkspace(FBWorkspace): def __init__( self, *args, - executed_factor_value_dataframe=None, - raise_exception=False, + executed_factor_value_dataframe: pd.DataFrame = None, + raise_exception: bool = False, **kwargs, ) -> None: super().__init__(*args, **kwargs) @@ -102,7 +105,10 @@ class FactorFBWorkspace(FBWorkspace): 1. make the directory in workspace path 2. write the code to the file in the workspace path 3. link all the source data to the workspace path folder - 4. execute the code + if call_factor_py is True: + 4. execute the code + else: + 4. generate a script from template to import the factor.py dump get the factor value to result.h5 5. read the factor value from the output file in the workspace path folder returns the execution feedback as a string and the factor value as a pandas dataframe @@ -130,15 +136,21 @@ class FactorFBWorkspace(FBWorkspace): if self.executed_factor_value_dataframe is not None: return self.FB_FROM_CACHE, self.executed_factor_value_dataframe - source_data_path = ( - Path( - FACTOR_IMPLEMENT_SETTINGS.data_folder_debug, + if self.target_task.version == 1: + source_data_path = ( + Path( + FACTOR_IMPLEMENT_SETTINGS.data_folder_debug, + ) + if data_type == "Debug" + else Path( + FACTOR_IMPLEMENT_SETTINGS.data_folder, + ) ) - if data_type == "Debug" - else Path( + elif self.target_task.version == 2: + # TODO you can change the name of the data folder for a better understanding + source_data_path = Path( FACTOR_IMPLEMENT_SETTINGS.data_folder, ) - ) source_data_path.mkdir(exist_ok=True, parents=True) code_path = self.workspace_path / f"factor.py" @@ -147,9 +159,16 @@ class FactorFBWorkspace(FBWorkspace): execution_feedback = self.FB_EXECUTION_SUCCEEDED execution_success = False + + if self.target_task.version == 1: + execution_code_path = code_path + elif self.target_task.version == 2: + execution_code_path = self.workspace_path / f"{uuid.uuid4()}.py" + execution_code_path.write_text((Path(__file__).parent / "factor_execution_template.txt").read_text()) + try: subprocess.check_output( - f"{FACTOR_IMPLEMENT_SETTINGS.python_bin} {code_path}", + f"{FACTOR_IMPLEMENT_SETTINGS.python_bin} {execution_code_path}", shell=True, cwd=self.workspace_path, stderr=subprocess.STDOUT, @@ -161,7 +180,7 @@ class FactorFBWorkspace(FBWorkspace): execution_feedback = ( e.output.decode() - .replace(str(code_path.parent.absolute()), r"/path/to") + .replace(str(execution_code_path.parent.absolute()), r"/path/to") .replace(str(site.getsitepackages()[0]), r"/path/to/site-packages") ) if len(execution_feedback) > 2000: diff --git a/rdagent/components/coder/factor_coder/factor_execution_template.txt b/rdagent/components/coder/factor_coder/factor_execution_template.txt new file mode 100644 index 00000000..af7d4366 --- /dev/null +++ b/rdagent/components/coder/factor_coder/factor_execution_template.txt @@ -0,0 +1,13 @@ +import os + +import numpy as np +import pandas as pd +from factor import feat_eng + +if os.path.exists("valid.pkl"): + valid_df = pd.read_pickle("valid.pkl") +else: + raise FileNotFoundError("No valid data found.") + +new_feat = feat_eng(valid_df) +new_feat.to_hdf("result.h5", key="data", mode="w") diff --git a/rdagent/components/coder/model_coder/CoSTEER/evaluators.py b/rdagent/components/coder/model_coder/CoSTEER/evaluators.py index 01c791f9..96a4ef68 100644 --- a/rdagent/components/coder/model_coder/CoSTEER/evaluators.py +++ b/rdagent/components/coder/model_coder/CoSTEER/evaluators.py @@ -24,7 +24,7 @@ from rdagent.oai.llm_utils import APIBackend evaluate_prompts = Prompts(file_path=Path(__file__).parent.parent / "prompts.yaml") -def shape_evaluator(prediction: torch.Tensor, target_shape: Tuple = None) -> Tuple[str, bool]: +def shape_evaluator(prediction: torch.Tensor | np.ndarray, target_shape: Tuple = None) -> Tuple[str, bool]: if target_shape is None or prediction is None: return ( "No output generated from the model. No shape evaluation conducted.", @@ -279,12 +279,8 @@ class ModelCoderEvaluator(Evaluator): else: gt_tensor = None - if target_task.model_type == "XGBoost": - shape_feedback = "Not applicable for XGBoost models" - shape_decision = True - else: - shape_feedback, shape_decision = shape_evaluator(gen_tensor, (batch_size, 1)) - value_feedback, value_decision = value_evaluator(gt_tensor, gen_tensor) + shape_feedback, shape_decision = shape_evaluator(gen_tensor, (batch_size, 1)) + value_feedback, value_decision = value_evaluator(gen_tensor, gt_tensor) code_feedback, _ = ModelCodeEvaluator(scen=self.scen).evaluate( target_task=target_task, implementation=implementation, diff --git a/rdagent/components/coder/model_coder/model.py b/rdagent/components/coder/model_coder/model.py index d3d114e0..41d5f00b 100644 --- a/rdagent/components/coder/model_coder/model.py +++ b/rdagent/components/coder/model_coder/model.py @@ -1,20 +1,13 @@ -import json import pickle import site import traceback -import uuid from pathlib import Path -from typing import Any, Dict, Optional - -import numpy as np -import torch -import xgboost as xgb +from typing import Dict, Optional from rdagent.components.coder.model_coder.conf import MODEL_IMPL_SETTINGS -from rdagent.core.exception import CodeFormatError from rdagent.core.experiment import Experiment, FBWorkspace, Task from rdagent.oai.llm_utils import md5_hash -from rdagent.utils import get_module_by_module_path +from rdagent.utils.env import KGDockerEnv, QTDockerEnv class ModelTask(Task): @@ -22,11 +15,13 @@ class ModelTask(Task): self, name: str, description: str, - formulation: str, architecture: str, - variables: Dict[str, str], + *args, hyperparameters: Dict[str, str], + formulation: str = None, + variables: Dict[str, str] = None, model_type: Optional[str] = None, + **kwargs, ) -> None: self.name: str = name self.description: str = description @@ -35,16 +30,18 @@ class ModelTask(Task): self.variables: str = variables self.hyperparameters: str = hyperparameters self.model_type: str = model_type # Tabular for tabular model, TimesSeries for time series model, Graph for graph model, XGBoost for XGBoost model + super().__init__(*args, **kwargs) def get_task_information(self): - return f"""name: {self.name} + task_desc = f"""name: {self.name} description: {self.description} -formulation: {self.formulation} -architecture: {self.architecture} -variables: {self.variables} -hyperparameters: {self.hyperparameters} -model_type: {self.model_type} """ + task_desc += f"formulation: {self.formulation}\n" if self.formulation else "" + task_desc += f"architecture: {self.architecture}\n" + task_desc += f"variables: {self.variables}\n" if self.variables else "" + task_desc += f"hyperparameters: {self.hyperparameters}\n" + task_desc += f"model_type: {self.model_type}\n" + return task_desc @staticmethod def from_dict(dict): @@ -66,12 +63,13 @@ class ModelFBWorkspace(FBWorkspace): - the `model.py` that contains a variable named `model_cls` which indicates the implemented model structure - `model_cls` is a instance of `torch.nn.Module`; + We support two ways of interface: + (version 1) for qlib we'll make a script to import the model in the implementation in file `model.py` after setting the cwd into the directory + - from model import model_cls + - initialize the model by initializing it `model_cls(input_dim=INPUT_DIM)` + - And then verify the model. - We'll import the model in the implementation in file `model.py` after setting the cwd into the directory - - from model import model_cls - - initialize the model by initializing it `model_cls(input_dim=INPUT_DIM)` - - And then verify the model. - + (version 2) for kaggle we'll make a script to call the fit and predict function in the implementation in file `model.py` after setting the cwd into the directory """ def execute( @@ -94,51 +92,33 @@ class ModelFBWorkspace(FBWorkspace): Path(MODEL_IMPL_SETTINGS.cache_location).mkdir(exist_ok=True, parents=True) if cache_file_path.exists(): return pickle.load(open(cache_file_path, "rb")) - mod = get_module_by_module_path(str(self.workspace_path / "model.py")) - if self.target_task.model_type != "XGBoost": - model_cls = mod.model_cls + qtde = QTDockerEnv() if self.target_task.version == 1 else KGDockerEnv() + qtde.prepare() - if self.target_task.model_type == "XGBoost": - X_simulated = np.random.rand(100, num_features) # 100 samples, `num_features` features each - y_simulated = np.random.randint(0, 2, 100) # Binary target for example - params = mod.get_params() - num_round = mod.get_num_round() - dtrain = xgb.DMatrix(X_simulated, label=y_simulated) - elif self.target_task.model_type == "Tabular": - input_shape = (batch_size, num_features) - m = model_cls(num_features=input_shape[1]) - data = torch.full(input_shape, input_value) - elif self.target_task.model_type == "TimeSeries": - input_shape = (batch_size, num_features, num_timesteps) - m = model_cls(num_features=input_shape[1], num_timesteps=input_shape[2]) - data = torch.full(input_shape, input_value) - elif self.target_task.model_type == "Graph": - node_feature = torch.randn(batch_size, num_features) - edge_index = torch.randint(0, batch_size, (2, num_edges)) - m = model_cls(num_features=num_features) - data = (node_feature, edge_index) - else: - raise ValueError(f"Unsupported model type: {self.target_task.model_type}") + if self.target_task.version == 1: + dump_code = f""" +MODEL_TYPE = "{self.target_task.model_type}" +BATCH_SIZE = {batch_size} +NUM_FEATURES = {num_features} +NUM_TIMESTEPS = {num_timesteps} +NUM_EDGES = {num_edges} +INPUT_VALUE = {input_value} +PARAM_INIT_VALUE = {param_init_value} +{(Path(__file__).parent / 'model_execute_template_v1.txt').read_text()} +""" + elif self.target_task.version == 2: + dump_code = (Path(__file__).parent / "model_execute_template_v2.txt").read_text() - if self.target_task.model_type == "XGBoost": - bst = xgb.train(params, dtrain, num_round) - y_pred = bst.predict(dtrain) - execution_model_output = y_pred - execution_feedback_str = "Execution successful, model trained and predictions made." - else: - # Initialize all parameters of `m` to `param_init_value` - for _, param in m.named_parameters(): - param.data.fill_(param_init_value) - - # Execute the model - if self.target_task.model_type == "Graph": - out = m(*data) - else: - out = m(data) - - execution_model_output = out.cpu().detach() - execution_feedback_str = f"Execution successful, output tensor shape: {execution_model_output.shape}" + log, results = qtde.dump_python_code_run_and_get_results( + code=dump_code, + dump_file_names=["execution_feedback_str.pkl", "execution_model_output.pkl"], + local_path=str(self.workspace_path), + env={}, + ) + if results is None: + raise RuntimeError(f"Error in running the model code: {log}") + [execution_feedback_str, execution_model_output] = results if MODEL_IMPL_SETTINGS.enable_execution_cache: pickle.dump( @@ -150,10 +130,6 @@ class ModelFBWorkspace(FBWorkspace): execution_feedback_str = f"Execution error: {e}\nTraceback: {traceback.format_exc()}" execution_model_output = None - code_path = self.workspace_path / f"model.py" - execution_feedback_str = execution_feedback_str.replace(str(code_path.parent.absolute()), r"/path/to").replace( - str(site.getsitepackages()[0]), r"/path/to/site-packages" - ) if len(execution_feedback_str) > 2000: execution_feedback_str = ( execution_feedback_str[:1000] + "....hidden long error message...." + execution_feedback_str[-1000:] @@ -161,4 +137,5 @@ class ModelFBWorkspace(FBWorkspace): return execution_feedback_str, execution_model_output +FeatureExperiment = Experiment ModelExperiment = Experiment diff --git a/rdagent/components/coder/model_coder/model_execute_template_v1.txt b/rdagent/components/coder/model_coder/model_execute_template_v1.txt new file mode 100644 index 00000000..bd42538a --- /dev/null +++ b/rdagent/components/coder/model_coder/model_execute_template_v1.txt @@ -0,0 +1,44 @@ +# MODEL_TYPE = "Tabular" +# BATCH_SIZE = 32 +# NUM_FEATURES = 10 +# NUM_TIMESTEPS = 4 +# NUM_EDGES = 20 +# INPUT_VALUE = 1.0 +# PARAM_INIT_VALUE = 1.0 + +import pickle + +import torch +from model import model_cls + +if MODEL_TYPE == "Tabular": + input_shape = (BATCH_SIZE, NUM_FEATURES) + m = model_cls(num_features=input_shape[1]) + data = torch.full(input_shape, INPUT_VALUE) +elif MODEL_TYPE == "TimeSeries": + input_shape = (BATCH_SIZE, NUM_FEATURES, NUM_TIMESTEPS) + m = model_cls(num_features=input_shape[1], num_timesteps=input_shape[2]) + data = torch.full(input_shape, INPUT_VALUE) +elif MODEL_TYPE == "Graph": + node_feature = torch.randn(BATCH_SIZE, NUM_FEATURES) + edge_index = torch.randint(0, BATCH_SIZE, (2, NUM_EDGES)) + m = model_cls(num_features=NUM_FEATURES) + data = (node_feature, edge_index) +else: + raise ValueError(f"Unsupported model type: {MODEL_TYPE}") + +# Initialize all parameters of `m` to `param_init_value` +for _, param in m.named_parameters(): + param.data.fill_(PARAM_INIT_VALUE) + +# Execute the model +if MODEL_TYPE == "Graph": + out = m(*data) +else: + out = m(data) + +execution_model_output = out.cpu().detach() +execution_feedback_str = f"Execution successful, output tensor shape: {execution_model_output.shape}" + +pickle.dump(execution_model_output, open("execution_model_output.pkl", "wb")) +pickle.dump(execution_feedback_str, open("execution_feedback_str.pkl", "wb")) diff --git a/rdagent/components/coder/model_coder/model_execute_template_v2.txt b/rdagent/components/coder/model_coder/model_execute_template_v2.txt new file mode 100644 index 00000000..40fbffdb --- /dev/null +++ b/rdagent/components/coder/model_coder/model_execute_template_v2.txt @@ -0,0 +1,20 @@ +import os +import pickle + +import numpy as np +import pandas as pd +import torch +from model import fit, predict, select + +train_X = pd.DataFrame(np.random.randn(8, 30), columns=[f"{i}" for i in range(30)]) +train_y = pd.Series(np.random.randint(0, 2, 8)) +valid_X = pd.DataFrame(np.random.randn(8, 30), columns=[f"{i}" for i in range(30)]) +valid_y = pd.Series(np.random.randint(0, 2, 8)) + +model = fit(train_X, train_y, valid_X, valid_y) +execution_model_output = predict(model, valid_X) + +execution_feedback_str = f"Execution successful, output numpy ndarray shape: {execution_model_output.shape}" + +pickle.dump(execution_model_output, open("execution_model_output.pkl", "wb")) +pickle.dump(execution_feedback_str, open("execution_feedback_str.pkl", "wb")) diff --git a/rdagent/components/proposal/model_proposal.py b/rdagent/components/proposal/model_proposal.py index a5d28006..409c37d2 100644 --- a/rdagent/components/proposal/model_proposal.py +++ b/rdagent/components/proposal/model_proposal.py @@ -39,7 +39,7 @@ class ModelHypothesisGen(HypothesisGen): Environment(undefined=StrictUndefined) .from_string(ModelHypothesisGen.prompts["hypothesis_gen"]["system_prompt"]) .render( - targets="model", + targets="feature engineering and model building", scenario=self.scen.get_scenario_all_desc(), hypothesis_output_format=context_dict["hypothesis_output_format"], hypothesis_specification=context_dict["hypothesis_specification"], @@ -49,7 +49,7 @@ class ModelHypothesisGen(HypothesisGen): Environment(undefined=StrictUndefined) .from_string(ModelHypothesisGen.prompts["hypothesis_gen"]["user_prompt"]) .render( - targets="model", + targets="feature engineering and model building", hypothesis_and_feedback=context_dict["hypothesis_and_feedback"], RAG=context_dict["RAG"], ) @@ -82,7 +82,7 @@ class ModelHypothesis2Experiment(Hypothesis2Experiment[ModelExperiment]): Environment(undefined=StrictUndefined) .from_string(ModelHypothesis2Experiment.prompts["hypothesis2experiment"]["system_prompt"]) .render( - targets="model", + targets="feature engineering and model building", scenario=trace.scen.get_scenario_all_desc(), experiment_output_format=context["experiment_output_format"], ) @@ -91,7 +91,7 @@ class ModelHypothesis2Experiment(Hypothesis2Experiment[ModelExperiment]): Environment(undefined=StrictUndefined) .from_string(ModelHypothesis2Experiment.prompts["hypothesis2experiment"]["user_prompt"]) .render( - targets="model", + targets="feature engineering and model building", target_hypothesis=context["target_hypothesis"], hypothesis_and_feedback=context["hypothesis_and_feedback"], target_list=context["target_list"], diff --git a/rdagent/components/proposal/prompts.yaml b/rdagent/components/proposal/prompts.yaml index 2a7514cb..e5145c9c 100644 --- a/rdagent/components/proposal/prompts.yaml +++ b/rdagent/components/proposal/prompts.yaml @@ -4,18 +4,22 @@ hypothesis_gen: The {{targets}} are used in a certain scenario, the scenario is as follows: {{ scenario }} The user has made several hypothesis on this scenario and did several evaluation on them. The user will provide this information to you. Check if a new hypothesis has already been proposed. If it is already generated and you agree with it, just use it. If you don't agree, generate a better one. + {% if hypothesis_specification %} To help you generate new hypothesis, the user has prepared some additional information for you. You should use this information to help generate new {{targets}}. + Here are the specifications: {{ hypothesis_specification }} + {% endif %} Please generate the output following the format and specifications below: {{ hypothesis_output_format }} - Here are the specifications: {{ hypothesis_specification }} user_prompt: |- - If it is not the first round, then the user has made several hypothesis on this scenario and did several evaluation on them. + {% if hypothesis_and_feedback|length == 0 %} It is the first round of hypothesis generation. The user has no hypothesis on this scenario yet. + {% else %}It is not the first round, the user has made several hypothesis on this scenario and did several evaluation on them. The former hypothesis and the corresponding feedbacks are as follows (focus on the last one & the new hypothesis that it provides and reasoning to see if you agree): {{ hypothesis_and_feedback }} - To help you generate new {{targets}}, we have prepared the following information for you: - {{ RAG }} - Please generate the new hypothesis based on the information above. Also generate the relevant keys for the reasoning and the distilled knowledge that follows. For those keys, in particular for knowledge, explain in the context of the specific scenario to build up domain knowledge in the specific field rather than genearl knowledge. + {% endif %} + {% if RAG %}To help you generate new {{targets}}, we have prepared the following information for you: + {{ RAG }}{% endif %} + Please generate the new hypothesis based on the information above. Also generate the relevant keys for the reasoning and the distilled knowledge that follows. For those keys, in particular for knowledge, explain in the context of the specific scenario to build up domain knowledge in the specific field rather than general knowledge. hypothesis2experiment: system_prompt: |- diff --git a/rdagent/components/runner/conf.py b/rdagent/components/runner/conf.py index 1791dde7..92b777ca 100644 --- a/rdagent/components/runner/conf.py +++ b/rdagent/components/runner/conf.py @@ -7,7 +7,7 @@ from pydantic_settings import BaseSettings class RunnerSettings(BaseSettings): class Config: - env_prefix = "RUNNER_" # Use MODEL_CODER_ as prefix for environment variables + env_prefix = "RUNNER_" # Use RUNNER_ as prefix for environment variables cache_result: bool = True # whether to cache the result of the docker execution cache_path: str = str(Path.cwd() / "runner_cache/") # the path to store the cache diff --git a/rdagent/core/experiment.py b/rdagent/core/experiment.py index 7b1f00ad..1f9f3f40 100644 --- a/rdagent/core/experiment.py +++ b/rdagent/core/experiment.py @@ -15,6 +15,14 @@ This file contains the all the class about organizing the task in RD-Agent. class Task(ABC): + def __init__(self, version: int = 1) -> None: + """ + The version of the task, default is 1 + Because qlib tasks execution and kaggle tasks execution are different, we need to distinguish them. + TODO: We may align them in the future. + """ + self.version = version + @abstractmethod def get_task_information(self) -> str: """ @@ -113,6 +121,9 @@ class FBWorkspace(Workspace): self.prepare() for k, v in files.items(): self.code_dict[k] = v + target_file_path = self.workspace_path / k + if not target_file_path.parent.exists(): + target_file_path.parent.mkdir(parents=True, exist_ok=True) with Path.open(self.workspace_path / k, "w") as f: f.write(v) @@ -129,9 +140,10 @@ class FBWorkspace(Workspace): """ Load the workspace from the folder """ - for file_path in folder_path.iterdir(): - if file_path.suffix in {".py", ".yaml"}: - self.inject_code(**{file_path.name: file_path.read_text()}) + for file_path in folder_path.rglob("*"): + if file_path.suffix in (".py", ".yaml", ".md"): + relative_path = file_path.relative_to(folder_path) + self.inject_code(**{str(relative_path): file_path.read_text()}) def copy(self) -> FBWorkspace: """ diff --git a/rdagent/log/ui/app.py b/rdagent/log/ui/app.py index 6500405f..0cf5db60 100644 --- a/rdagent/log/ui/app.py +++ b/rdagent/log/ui/app.py @@ -27,6 +27,7 @@ from rdagent.log.storage import FileStorage from rdagent.log.ui.qlib_report_figure import report_figure from rdagent.scenarios.data_mining.experiment.model_experiment import DMModelScenario from rdagent.scenarios.general_model.scenario import GeneralModelScenario +from rdagent.scenarios.kaggle.experiment.scenario import KGScenario from rdagent.scenarios.qlib.experiment.factor_experiment import QlibFactorScenario from rdagent.scenarios.qlib.experiment.factor_from_report_experiment import ( QlibFactorFromReportScenario, @@ -53,13 +54,15 @@ else: main_log_path = None -SELECTED_METRICS = [ +QLIB_SELECTED_METRICS = [ "IC", "1day.excess_return_without_cost.annualized_return", "1day.excess_return_without_cost.information_ratio", "1day.excess_return_without_cost.max_drawdown", ] +SIMILAR_SCENARIOS = (QlibModelScenario, DMModelScenario, QlibFactorScenario, QlibFactorFromReportScenario, KGScenario) + if "log_path" not in state: if main_log_path: state.log_path = next(main_log_path.iterdir()).relative_to(main_log_path) @@ -137,23 +140,25 @@ def get_msgs_until(end_func: Callable[[Message], bool] = lambda _: True): if "model runner result" in tags or "factor runner result" in tags or "runner result" in tags: # factor baseline exp metrics if isinstance(state.scenario, QlibFactorScenario) and state.alpha158_metrics is None: - sms = msg.content.based_experiments[0].result.loc[SELECTED_METRICS] + sms = msg.content.based_experiments[0].result.loc[QLIB_SELECTED_METRICS] sms.name = "alpha158" state.alpha158_metrics = sms # common metrics - if msg.content.result is None: + if msg.content.result is None and isinstance(state.scenario, DMModelScenario): state.metric_series.append(pd.Series([None], index=["AUROC"], name=f"Round {state.lround}")) else: - if len(msg.content.result) < 4: - ps = msg.content.result - ps.index = ["AUROC"] - ps.name = f"Round {state.lround}" - state.metric_series.append(ps) - else: - sms = msg.content.result.loc[SELECTED_METRICS] - sms.name = f"Round {state.lround}" - state.metric_series.append(sms) + sms = msg.content.result + if isinstance(state.scenario, DMModelScenario): + sms.index = ["AUROC"] + elif isinstance( + state.scenario, (QlibModelScenario, QlibFactorFromReportScenario, QlibFactorScenario) + ): + sms = sms.loc[QLIB_SELECTED_METRICS] + elif isinstance(state.scenario, KGScenario): + sms = sms.loc[["MCC"]] + sms.name = f"Round {state.lround}" + state.metric_series.append(sms) elif "hypothesis generation" in tags: state.hypotheses[state.lround] = msg.content elif "ef" in tags and "feedback" in tags: @@ -342,9 +347,7 @@ def metrics_window(df: pd.DataFrame, R: int, C: int, *, height: int = 300, color def summary_window(): - if isinstance( - state.scenario, (QlibModelScenario, DMModelScenario, QlibFactorScenario, QlibFactorFromReportScenario) - ): + if isinstance(state.scenario, SIMILAR_SCENARIOS): st.header("Summary📊", divider="rainbow", anchor="_summary") if state.lround == 0: return @@ -380,7 +383,6 @@ def summary_window(): st.table(df.iloc[0]) elif df.shape[0] > 1: if df.shape[1] == 1: - # suhan's scenario fig = px.line(df, x=df.index, y=df.columns, markers=True) fig.update_layout(xaxis_title="Loop Round", yaxis_title=None) st.plotly_chart(fig) @@ -465,17 +467,9 @@ def tasks_window(tasks: list[FactorTask | ModelTask]): def research_window(): with st.container(border=True): - title = ( - "Research🔍" - if isinstance( - state.scenario, (QlibModelScenario, DMModelScenario, QlibFactorScenario, QlibFactorFromReportScenario) - ) - else "Research🔍 (reader)" - ) + title = "Research🔍" if isinstance(state.scenario, SIMILAR_SCENARIOS) else "Research🔍 (reader)" st.subheader(title, divider="blue", anchor="_research") - if isinstance( - state.scenario, (QlibModelScenario, DMModelScenario, QlibFactorScenario, QlibFactorFromReportScenario) - ): + if isinstance(state.scenario, SIMILAR_SCENARIOS): # pdf image if pim := state.msgs[round]["r.extract_factors_and_implement.load_pdf_screenshot"]: for i in range(min(2, len(pim))): @@ -510,14 +504,12 @@ def research_window(): def feedback_window(): - if isinstance( - state.scenario, (QlibModelScenario, DMModelScenario, QlibFactorScenario, QlibFactorFromReportScenario) - ): + if isinstance(state.scenario, SIMILAR_SCENARIOS): with st.container(border=True): st.subheader("Feedback📝", divider="orange", anchor="_feedback") if state.lround > 0 and isinstance( - state.scenario, (QlibModelScenario, QlibFactorScenario, QlibFactorFromReportScenario) + state.scenario, (QlibModelScenario, QlibFactorScenario, QlibFactorFromReportScenario, KGScenario) ): with st.expander("**Config⚙️**", expanded=True): st.markdown(state.scenario.experiment_setting, unsafe_allow_html=True) @@ -541,13 +533,7 @@ def feedback_window(): @st.fragment def evolving_window(): - title = ( - "Development🛠️" - if isinstance( - state.scenario, (QlibModelScenario, DMModelScenario, QlibFactorScenario, QlibFactorFromReportScenario) - ) - else "Development🛠️ (evolving coder)" - ) + title = "Development🛠️" if isinstance(state.scenario, SIMILAR_SCENARIOS) else "Development🛠️ (evolving coder)" st.subheader(title, divider="green", anchor="_development") # Evolving Status @@ -743,9 +729,7 @@ if state.scenario is not None: summary_window() # R&D Loops Window - if isinstance( - state.scenario, (QlibModelScenario, DMModelScenario, QlibFactorScenario, QlibFactorFromReportScenario) - ): + if isinstance(state.scenario, SIMILAR_SCENARIOS): st.header("R&D Loops♾️", divider="rainbow", anchor="_rdloops") if len(state.msgs) > 1: r_options = list(state.msgs.keys()) diff --git a/rdagent/scenarios/data_mining/docker/Dockerfile b/rdagent/scenarios/data_mining/docker/Dockerfile index ad85b992..fae6e62e 100644 --- a/rdagent/scenarios/data_mining/docker/Dockerfile +++ b/rdagent/scenarios/data_mining/docker/Dockerfile @@ -1,4 +1,4 @@ -FROM pytorch/pytorch:latest +FROM pytorch/pytorch:2.2.1-cuda12.1-cudnn8-runtime # For GPU support, please choose the proper tag from https://hub.docker.com/r/pytorch/pytorch/tags RUN apt-get clean && apt-get update && apt-get install -y \ diff --git a/rdagent/scenarios/data_mining/proposal/model_proposal.py b/rdagent/scenarios/data_mining/proposal/model_proposal.py index fc60e2eb..f1bc0cbc 100644 --- a/rdagent/scenarios/data_mining/proposal/model_proposal.py +++ b/rdagent/scenarios/data_mining/proposal/model_proposal.py @@ -98,7 +98,15 @@ class DMModelHypothesis2Experiment(ModelHypothesis2Experiment): hyperparameters = response_dict[model_name]["hyperparameters"] model_type = response_dict[model_name]["model_type"] tasks.append( - ModelTask(model_name, description, formulation, architecture, variables, hyperparameters, model_type) + ModelTask( + name=model_name, + description=description, + formulation=formulation, + architecture=architecture, + variables=variables, + hyperparameters=hyperparameters, + model_type=model_type, + ) ) exp = DMModelExperiment(tasks) exp.based_experiments = [t[1] for t in trace.hist if t[2]] diff --git a/rdagent/scenarios/kaggle/developer/coder.py b/rdagent/scenarios/kaggle/developer/coder.py new file mode 100644 index 00000000..07ef97b6 --- /dev/null +++ b/rdagent/scenarios/kaggle/developer/coder.py @@ -0,0 +1,5 @@ +from rdagent.components.coder.factor_coder.CoSTEER import FactorCoSTEER +from rdagent.components.coder.model_coder.CoSTEER import ModelCoSTEER + +KGModelCoSTEER = ModelCoSTEER +KGFactorCoSTEER = FactorCoSTEER diff --git a/rdagent/scenarios/kaggle/developer/feedback.py b/rdagent/scenarios/kaggle/developer/feedback.py index 4d373f67..cce49bdc 100644 --- a/rdagent/scenarios/kaggle/developer/feedback.py +++ b/rdagent/scenarios/kaggle/developer/feedback.py @@ -1,6 +1,7 @@ import json from pathlib import Path +import pandas as pd from jinja2 import Environment, StrictUndefined from rdagent.core.experiment import Experiment @@ -19,50 +20,112 @@ feedback_prompts = Prompts(file_path=Path(__file__).parent.parent / "prompts.yam DIRNAME = Path(__file__).absolute().resolve().parent -class KGModelHypothesisExperiment2Feedback(HypothesisExperiment2Feedback): - """Generated feedbacks on the hypothesis from **Executed** Implementations of different tasks & their comparisons with previous performances""" +def process_results(current_result, sota_result): + # Convert the results to dataframes + current_df = pd.DataFrame(current_result) + sota_df = pd.DataFrame(sota_result) + # Combine the dataframes on the Metric index + combined_df = pd.DataFrame({"Current Result": current_df, "SOTA Result": sota_df}) + + # Add a new column to show which result is bigger + combined_df["Bigger Result"] = combined_df.apply( + lambda row: "Equal" + if row["Current Result"] == row["SOTA Result"] + else ("Current Result" if row["Current Result"] > row["SOTA Result"] else "SOTA Result"), + axis=1, + ) + + # Add a note about metric direction + combined_df["Note"] = "Direction of improvement (higher/lower is better) should be judged per metric" + + return combined_df.to_string() + + +class KGHypothesisExperiment2Feedback(HypothesisExperiment2Feedback): def generate_feedback(self, exp: Experiment, hypothesis: Hypothesis, trace: Trace) -> HypothesisFeedback: """ The `ti` should be executed and the results should be included, as well as the comparison between previous results (done by LLM). For example: `mlflow` of Qlib will be included. """ + """ + Generate feedback for the given experiment and hypothesis. + Args: + exp: The experiment to generate feedback for. + hypothesis: The hypothesis to generate feedback for. + trace: The trace of the experiment. + Returns: + Any: The feedback generated for the given experiment and hypothesis. + """ logger.info("Generating feedback...") - # Define the system prompt for hypothesis feedback - system_prompt = feedback_prompts["model_feedback_generation"]["system"] + hypothesis_text = hypothesis.hypothesis + current_result = exp.result + tasks_factors = [] + if exp.sub_tasks: + tasks_factors = [] + for task in exp.sub_tasks: + try: + task_info = task.get_task_information_and_implementation_result() + tasks_factors.append(task_info) + except AttributeError: + print(f"Warning: Task {task} does not have get_task_information_and_implementation_result method") - # Define the user prompt for hypothesis feedback - context = trace.scen - SOTA_hypothesis, SOTA_experiment = trace.get_sota_hypothesis_and_experiment() + # Check if there are any based experiments + if exp.based_experiments: + sota_result = exp.based_experiments[-1].result + # Process the results to filter important metrics + combined_result = process_results(current_result, sota_result) + else: + # If there are no based experiments, we'll only use the current result + combined_result = process_results(current_result, current_result) # Compare with itself + print("Warning: No previous experiments to compare against. Using current result as baseline.") - user_prompt = ( + # Generate the system prompt + sys_prompt = ( Environment(undefined=StrictUndefined) - .from_string(feedback_prompts["model_feedback_generation"]["user"]) + .from_string(feedback_prompts["factor_feedback_generation"]["system"]) + .render(scenario=self.scen.get_scenario_all_desc()) + ) + + # Generate the user prompt based on the action type + if hypothesis.action == "Model Tuning": # TODO Add other prompts here + prompt_key = "model_feedback_generation" + else: + prompt_key = "factor_feedback_generation" + + # Generate the user prompt + usr_prompt = ( + Environment(undefined=StrictUndefined) + .from_string(feedback_prompts[prompt_key]["user"]) .render( - context=context, - last_hypothesis=SOTA_hypothesis, - last_task=SOTA_experiment.sub_tasks[0].get_task_information() if SOTA_hypothesis else None, - last_code=SOTA_experiment.sub_workspace_list[0].code_dict.get("model.py") if SOTA_hypothesis else None, - last_result=SOTA_experiment.result if SOTA_hypothesis else None, - hypothesis=hypothesis, - exp=exp, + hypothesis_text=hypothesis_text, + task_details=tasks_factors, + combined_result=combined_result, ) ) # Call the APIBackend to generate the response for hypothesis feedback - response_hypothesis = APIBackend().build_messages_and_create_chat_completion( - user_prompt=user_prompt, - system_prompt=system_prompt, + response = APIBackend().build_messages_and_create_chat_completion( + user_prompt=usr_prompt, + system_prompt=sys_prompt, json_mode=True, ) # Parse the JSON response to extract the feedback - response_json_hypothesis = json.loads(response_hypothesis) + response_json = json.loads(response) + + # Extract fields from JSON response + observations = response_json.get("Observations", "No observations provided") + hypothesis_evaluation = response_json.get("Feedback for Hypothesis", "No feedback provided") + new_hypothesis = response_json.get("New Hypothesis", "No new hypothesis provided") + reason = response_json.get("Reasoning", "No reasoning provided") + decision = convert2bool(response_json.get("Replace Best Result", "no")) + return HypothesisFeedback( - observations=response_json_hypothesis.get("Observations", "No observations provided"), - hypothesis_evaluation=response_json_hypothesis.get("Feedback for Hypothesis", "No feedback provided"), - new_hypothesis=response_json_hypothesis.get("New Hypothesis", "No new hypothesis provided"), - reason=response_json_hypothesis.get("Reasoning", "No reasoning provided"), - decision=convert2bool(response_json_hypothesis.get("Decision", "false")), + observations=observations, + hypothesis_evaluation=hypothesis_evaluation, + new_hypothesis=new_hypothesis, + reason=reason, + decision=decision, ) diff --git a/rdagent/scenarios/kaggle/developer/model_coder.py b/rdagent/scenarios/kaggle/developer/model_coder.py deleted file mode 100644 index 033a5f3d..00000000 --- a/rdagent/scenarios/kaggle/developer/model_coder.py +++ /dev/null @@ -1,3 +0,0 @@ -from rdagent.components.coder.model_coder.CoSTEER import ModelCoSTEER - -KGModelCoSTEER = ModelCoSTEER diff --git a/rdagent/scenarios/kaggle/developer/model_runner.py b/rdagent/scenarios/kaggle/developer/model_runner.py deleted file mode 100644 index c3225303..00000000 --- a/rdagent/scenarios/kaggle/developer/model_runner.py +++ /dev/null @@ -1,38 +0,0 @@ -import shutil -import uuid -from pathlib import Path - -import pandas as pd - -from rdagent.components.coder.model_coder.model import ModelExperiment, ModelFBWorkspace -from rdagent.components.runner import CachedRunner -from rdagent.components.runner.conf import RUNNER_SETTINGS -from rdagent.core.developer import Developer -from rdagent.core.exception import ModelEmptyError -from rdagent.log import rdagent_logger as logger -from rdagent.scenarios.kaggle.experiment.model_experiment import KGModelExperiment -from rdagent.utils.env import KGDockerEnv - - -class KGModelRunner(CachedRunner[KGModelExperiment]): - def develop(self, exp: KGModelExperiment) -> KGModelExperiment: - if RUNNER_SETTINGS.cache_result: - cache_hit, result = self.get_cache_result(exp) - if cache_hit: - exp.result = result - return exp - - if exp.sub_workspace_list[0].code_dict.get("model.py") is None: - raise ModelEmptyError("model.py is empty") - # to replace & inject code - exp.experiment_workspace.inject_code(**{"model.py": exp.sub_workspace_list[0].code_dict["model.py"]}) - - env_to_use = {"PYTHONPATH": "./"} - - result = exp.experiment_workspace.execute(run_env=env_to_use) - - exp.result = result - if RUNNER_SETTINGS.cache_result: - self.dump_cache_result(exp, result) - - return exp diff --git a/rdagent/scenarios/kaggle/developer/runner.py b/rdagent/scenarios/kaggle/developer/runner.py new file mode 100644 index 00000000..b115d839 --- /dev/null +++ b/rdagent/scenarios/kaggle/developer/runner.py @@ -0,0 +1,121 @@ +import shutil +import uuid +from pathlib import Path + +import pandas as pd + +from rdagent.components.runner import CachedRunner +from rdagent.components.runner.conf import RUNNER_SETTINGS +from rdagent.core.exception import ModelEmptyError +from rdagent.core.experiment import ASpecificExp +from rdagent.oai.llm_utils import md5_hash +from rdagent.scenarios.kaggle.experiment.kaggle_experiment import ( + KGFactorExperiment, + KGModelExperiment, +) + +META_TPL_DIR = Path(__file__).parent.parent / "experiment" / "meta_tpl" + + +class KGCachedRunner(CachedRunner[ASpecificExp]): + def build_from_SOTA(self, exp: ASpecificExp) -> None: + if len(exp.based_experiments) > 0: + exp.experiment_workspace.inject_code(**exp.based_experiments[-1].experiment_workspace.code_dict) + exp.experiment_workspace.data_description = exp.based_experiments[-1].experiment_workspace.data_description + exp.experiment_workspace.model_description = exp.based_experiments[ + -1 + ].experiment_workspace.model_description + + def get_cache_key(self, exp: ASpecificExp) -> str: + codes = [] + for f in sorted((exp.experiment_workspace.workspace_path / "feature").glob("*.py"), key=lambda x: x.name): + codes.append(f.read_text()) + for f in sorted((exp.experiment_workspace.workspace_path / "model").glob("*.py"), key=lambda x: x.name): + codes.append(f.read_text()) + codes = "\n".join(codes) + return md5_hash(codes) + + +class KGModelRunner(KGCachedRunner[KGModelExperiment]): + def develop(self, exp: KGModelExperiment) -> KGModelExperiment: + self.build_from_SOTA(exp) + if exp.sub_workspace_list[0].target_task.model_type == "XGBoost": + exp.experiment_workspace.inject_code(**{"model_xgb.py": exp.sub_workspace_list[0].code_dict["model.py"]}) + elif exp.sub_workspace_list[0].target_task.model_type == "RandomForest": + exp.experiment_workspace.inject_code(**{"model_rf.py": exp.sub_workspace_list[0].code_dict["model.py"]}) + elif exp.sub_workspace_list[0].target_task.model_type == "LightGBM": + exp.experiment_workspace.inject_code(**{"model_lgb.py": exp.sub_workspace_list[0].code_dict["model.py"]}) + elif exp.sub_workspace_list[0].target_task.model_type == "NN": + exp.experiment_workspace.inject_code(**{"model_nn.py": exp.sub_workspace_list[0].code_dict["model.py"]}) + if RUNNER_SETTINGS.cache_result: + cache_hit, result = self.get_cache_result(exp) + if cache_hit: + exp.result = result + return exp + + env_to_use = {"PYTHONPATH": "./"} + + result = exp.experiment_workspace.execute(run_env=env_to_use) + + exp.result = result + if RUNNER_SETTINGS.cache_result: + self.dump_cache_result(exp, result) + + return exp + + +class KGFactorRunner(KGCachedRunner[KGFactorExperiment]): + def init_develop(self, exp: KGFactorExperiment) -> KGFactorExperiment: + """ + For the initial development, the experiment serves as a benchmark for feature engineering. + """ + self.build_from_SOTA(exp) + if RUNNER_SETTINGS.cache_result: + cache_hit, result = self.get_cache_result(exp) + if cache_hit: + exp.result = result + return exp + + env_to_use = {"PYTHONPATH": "./"} + + result = exp.experiment_workspace.execute(run_env=env_to_use) + + exp.result = result + if RUNNER_SETTINGS.cache_result: + self.dump_cache_result(exp, result) + + return exp + + def develop(self, exp: KGFactorExperiment) -> KGFactorExperiment: + if exp.based_experiments and exp.based_experiments[-1].result is None: + exp.based_experiments[-1] = self.init_develop(exp.based_experiments[-1]) + self.build_from_SOTA(exp) + current_feature_file_count = len(list(exp.experiment_workspace.workspace_path.glob("feature/feature*.py"))) + implemented_factor_count = 0 + for sub_ws in exp.sub_workspace_list: + if sub_ws.code_dict == {}: + continue + implemented_factor_count += 1 + target_feature_file_name = f"feature/feature_{current_feature_file_count:05d}.py" + exp.experiment_workspace.inject_code(**{target_feature_file_name: sub_ws.code_dict["factor.py"]}) + feature_shape = sub_ws.execute()[1].shape[-1] + exp.experiment_workspace.data_description.append((sub_ws.target_task.get_task_information(), feature_shape)) + current_feature_file_count += 1 + if implemented_factor_count == 0: + raise ModelEmptyError("No factor is implemented") + + if RUNNER_SETTINGS.cache_result: + cache_hit, result = self.get_cache_result(exp) + if cache_hit: + exp.result = result + return exp + + env_to_use = {"PYTHONPATH": "./"} + + result = exp.experiment_workspace.execute(run_env=env_to_use) + + exp.result = result + if RUNNER_SETTINGS.cache_result: + self.dump_cache_result(exp, result) + + return exp diff --git a/rdagent/scenarios/kaggle/docker/Dockerfile b/rdagent/scenarios/kaggle/docker/Dockerfile index ad85b992..26d3bc3c 100644 --- a/rdagent/scenarios/kaggle/docker/Dockerfile +++ b/rdagent/scenarios/kaggle/docker/Dockerfile @@ -1,4 +1,4 @@ -FROM pytorch/pytorch:latest +FROM pytorch/pytorch:2.2.1-cuda12.1-cudnn8-runtime # For GPU support, please choose the proper tag from https://hub.docker.com/r/pytorch/pytorch/tags RUN apt-get clean && apt-get update && apt-get install -y \ @@ -11,7 +11,7 @@ RUN apt-get clean && apt-get update && apt-get install -y \ WORKDIR /workspace RUN python -m pip install numpy -RUN python -m pip install --upgrade cython +# RUN python -m pip install --upgrade cython # RUN python -m pip install -e . RUN python -m pip install pandas @@ -23,3 +23,4 @@ RUN pip install scikit-learn RUN pip install catboost RUN pip install xgboost RUN pip install sparse +RUN pip install lightgbm diff --git a/rdagent/scenarios/kaggle/experiment/kaggle_experiment.py b/rdagent/scenarios/kaggle/experiment/kaggle_experiment.py new file mode 100644 index 00000000..ee768d95 --- /dev/null +++ b/rdagent/scenarios/kaggle/experiment/kaggle_experiment.py @@ -0,0 +1,21 @@ +from pathlib import Path + +from rdagent.components.coder.factor_coder.factor import FactorFBWorkspace +from rdagent.components.coder.model_coder.model import ( + ModelExperiment, + ModelFBWorkspace, + ModelTask, +) +from rdagent.scenarios.kaggle.experiment.workspace import KGFBWorkspace + + +class KGModelExperiment(ModelExperiment[ModelTask, KGFBWorkspace, ModelFBWorkspace]): + def __init__(self, *args, **kwargs) -> None: + super().__init__(*args, **kwargs) + self.experiment_workspace = KGFBWorkspace(template_folder_path=Path(__file__).parent / "meta_tpl") + + +class KGFactorExperiment(ModelExperiment[ModelTask, KGFBWorkspace, FactorFBWorkspace]): + def __init__(self, *args, **kwargs) -> None: + super().__init__(*args, **kwargs) + self.experiment_workspace = KGFBWorkspace(template_folder_path=Path(__file__).parent / "meta_tpl") diff --git a/rdagent/scenarios/kaggle/experiment/meta_tpl/fea_share_preprocess.py b/rdagent/scenarios/kaggle/experiment/meta_tpl/fea_share_preprocess.py index d1335c56..3bbd7bf2 100644 --- a/rdagent/scenarios/kaggle/experiment/meta_tpl/fea_share_preprocess.py +++ b/rdagent/scenarios/kaggle/experiment/meta_tpl/fea_share_preprocess.py @@ -1,17 +1,39 @@ import pandas as pd from sklearn.compose import ColumnTransformer from sklearn.impute import SimpleImputer +from sklearn.model_selection import train_test_split from sklearn.pipeline import Pipeline -from sklearn.preprocessing import OneHotEncoder +from sklearn.preprocessing import LabelEncoder, OneHotEncoder -def preprocess(X: pd.DataFrame): +def prepreprocess(): """ - We want the X_train & X_test & X_valid to contain the same number of columns & maintain feature consistency. + This method loads the data, drops the unnecessary columns, and splits it into train and validation sets. + """ + # Load and preprocess the data + data_df = pd.read_csv("/kaggle/input/train.csv") + data_df = data_df.head(1200) + data_df = data_df.drop(["id"], axis=1) + + X = data_df.drop(["class"], axis=1) + y = data_df[["class"]] + + label_encoder = LabelEncoder() + y = label_encoder.fit_transform(y) # Convert class labels to numeric + + # Split the data into training and validation sets + X_train, X_valid, y_train, y_valid = train_test_split(X, y, test_size=0.10, random_state=42) + + return X_train, X_valid, y_train, y_valid + + +def preprocess_fit(X_train: pd.DataFrame): + """ + Fits the preprocessor on the training data and returns the fitted preprocessor. """ # Identify numerical and categorical features - numerical_cols = [cname for cname in X.columns if X[cname].dtype in ["int64", "float64"]] - categorical_cols = [cname for cname in X.columns if X[cname].dtype == "object"] + numerical_cols = [cname for cname in X_train.columns if X_train[cname].dtype in ["int64", "float64"]] + categorical_cols = [cname for cname in X_train.columns if X_train[cname].dtype == "object"] # Define preprocessors for numerical and categorical features categorical_transformer = Pipeline( @@ -31,17 +53,50 @@ def preprocess(X: pd.DataFrame): ] ) - # Fit the preprocessor on the data and transform it - preprocessor.fit(X) # TODO depend on its input shape + # Fit the preprocessor on the training data + preprocessor.fit(X_train) + + return preprocessor + + +def preprocess_transform(X: pd.DataFrame, preprocessor): + """ + Transforms the given DataFrame using the fitted preprocessor. + Ensures the processed data has consistent features across train, validation, and test sets. + """ + # Transform the data using the fitted preprocessor X_array = preprocessor.transform(X).toarray() # Get feature names for the columns in the transformed data - feature_names = ( - preprocessor.named_transformers_["cat"]["onehot"].get_feature_names_out(categorical_cols).tolist() - + numerical_cols - ) + categorical_cols = [cname for cname in X.columns if X[cname].dtype == "object"] + feature_names = preprocessor.named_transformers_["cat"]["onehot"].get_feature_names_out( + categorical_cols + ).tolist() + [cname for cname in X.columns if X[cname].dtype in ["int64", "float64"]] # Convert arrays back to DataFrames X_transformed = pd.DataFrame(X_array, columns=feature_names, index=X.index) return X_transformed + + +def preprocess_script(): + """ + This method applies the preprocessing steps to the training, validation, and test datasets. + """ + X_train, X_valid, y_train, y_valid = prepreprocess() + + # Fit the preprocessor on the training data + preprocessor = preprocess_fit(X_train) + + # Preprocess the train, validation, and test data + X_train = preprocess_transform(X_train, preprocessor) + X_valid = preprocess_transform(X_valid, preprocessor) + + # Load and preprocess the test data + submission_df = pd.read_csv("/kaggle/input/test.csv") + submission_df = submission_df.head(500) + passenger_ids = submission_df["id"] + submission_df = submission_df.drop(["id"], axis=1) + X_test = preprocess_transform(submission_df, preprocessor) + + return X_train, X_valid, y_train, y_valid, X_test, passenger_ids diff --git a/rdagent/scenarios/kaggle/experiment/meta_tpl/feat.py b/rdagent/scenarios/kaggle/experiment/meta_tpl/feature/feature.py similarity index 94% rename from rdagent/scenarios/kaggle/experiment/meta_tpl/feat.py rename to rdagent/scenarios/kaggle/experiment/meta_tpl/feature/feature.py index 7dd65426..c25a7ead 100644 --- a/rdagent/scenarios/kaggle/experiment/meta_tpl/feat.py +++ b/rdagent/scenarios/kaggle/experiment/meta_tpl/feature/feature.py @@ -10,4 +10,4 @@ def feat_eng(X: pd.DataFrame): """ return the selected features """ - return X + return None diff --git a/rdagent/scenarios/kaggle/experiment/meta_tpl/model/model_rf.py b/rdagent/scenarios/kaggle/experiment/meta_tpl/model/model_rf.py new file mode 100644 index 00000000..3ec75868 --- /dev/null +++ b/rdagent/scenarios/kaggle/experiment/meta_tpl/model/model_rf.py @@ -0,0 +1,54 @@ +""" +Motivation of the model: +The Random Forest model is chosen for its robustness and ability to handle large datasets with higher dimensionality. +It reduces overfitting by averaging multiple decision trees and typically performs well out of the box, making it a good +baseline model for many classification tasks. +""" + +import pandas as pd +from sklearn.ensemble import RandomForestClassifier +from sklearn.metrics import accuracy_score + + +def select(X: pd.DataFrame) -> pd.DataFrame: + """ + Select relevant features. To be used in fit & predict function. + """ + # For now, we assume all features are relevant. This can be expanded to feature selection logic. + return X + + +def fit(X_train: pd.DataFrame, y_train: pd.Series, X_valid: pd.DataFrame, y_valid: pd.Series): + """ + Define and train the Random Forest model. Merge feature selection into the pipeline. + """ + # Initialize the Random Forest model + model = RandomForestClassifier(n_estimators=100, random_state=32) + + # Select features (if any feature selection is needed) + X_train_selected = select(X_train) + X_valid_selected = select(X_valid) + + # Fit the model + model.fit(X_train_selected, y_train) + + # Validate the model + y_valid_pred = model.predict(X_valid_selected) + accuracy = accuracy_score(y_valid, y_valid_pred) + print(f"Validation Accuracy: {accuracy:.4f}") + + return model + + +def predict(model, X): + """ + Keep feature selection's consistency and make predictions. + """ + # Select features (if any feature selection is needed) + X_selected = select(X) + + # Predict using the trained model + y_pred_prob = model.predict_proba(X_selected)[:, 1] + + # Apply threshold to get boolean predictions + return y_pred_prob diff --git a/rdagent/scenarios/kaggle/experiment/meta_tpl/model.py b/rdagent/scenarios/kaggle/experiment/meta_tpl/model/model_xgb.py similarity index 91% rename from rdagent/scenarios/kaggle/experiment/meta_tpl/model.py rename to rdagent/scenarios/kaggle/experiment/meta_tpl/model/model_xgb.py index e2b69730..839a39e9 100644 --- a/rdagent/scenarios/kaggle/experiment/meta_tpl/model.py +++ b/rdagent/scenarios/kaggle/experiment/meta_tpl/model/model_xgb.py @@ -1,6 +1,7 @@ """ motivation of the model """ + import pandas as pd import xgboost as xgb @@ -33,4 +34,4 @@ def predict(model, X): """ dtest = xgb.DMatrix(X) y_pred_prob = model.predict(dtest) - return y_pred_prob > 0.5 # Apply threshold to get boolean predictions + return y_pred_prob diff --git a/rdagent/scenarios/kaggle/experiment/meta_tpl/train.py b/rdagent/scenarios/kaggle/experiment/meta_tpl/train.py index 97a395f6..99c459b0 100644 --- a/rdagent/scenarios/kaggle/experiment/meta_tpl/train.py +++ b/rdagent/scenarios/kaggle/experiment/meta_tpl/train.py @@ -1,19 +1,12 @@ -import os +import importlib.util import random from pathlib import Path import numpy as np import pandas as pd -import xgboost as xgb -from fea_share_preprocess import preprocess -from sklearn.compose import ColumnTransformer -from sklearn.impute import SimpleImputer +from fea_share_preprocess import preprocess_script from sklearn.metrics import accuracy_score, matthews_corrcoef -from sklearn.model_selection import train_test_split -from sklearn.pipeline import Pipeline -from sklearn.preprocessing import LabelEncoder, OneHotEncoder - -from rdagent.scenarios.kaggle.experiment.meta_tpl.fea_share_preprocess import preprocess +from sklearn.preprocessing import LabelEncoder # Set random seed for reproducibility SEED = 42 @@ -35,66 +28,67 @@ def compute_metrics_for_classification(y_true, y_pred): return mcc -# Load and preprocess the data -data_df = pd.read_csv("/home/v-xisenwang/git_ignore_folder/data/playground-series-s4e8/train.csv") -data_df = data_df.drop(["id"], axis=1) +def import_module_from_path(module_name, module_path): + spec = importlib.util.spec_from_file_location(module_name, module_path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module -X = data_df.drop(["class"], axis=1) -y = data_df[["class"]] - -label_encoder = LabelEncoder() -y = label_encoder.fit_transform(y) # 将类别标签转换为数值 -X_train, X_valid, y_train, y_valid = train_test_split(X, y, test_size=0.10, random_state=SEED) # 1) Preprocess the data -X_train = preprocess(X_train) -X_valid = preprocess(X_valid) - -submission_df = pd.read_csv("/home/v-xisenwang/git_ignore_folder/data/playground-series-s4e8/test.csv") -passenger_ids = submission_df["id"] -submission_df = submission_df.drop(["id"], axis=1) -X_test = preprocess(submission_df) +# TODO 如果已经做过数据预处理了，不需要再做了 +X_train, X_valid, y_train, y_valid, X_test, passenger_ids = preprocess_script() # 2) Auto feature engineering -X_train_l, X_valid_l = [], [] -X_test_l = [] -for f in DIRNAME.glob("feat*.py"): - m = __import__(f.name.strip(".py")) - X_train = m.feat_eng(X_train) - X_valid = m.feat_eng(X_valid) - X_test = m.feat_eng(X_test) +X_train_l, X_valid_l = [X_train], [X_valid] +X_test_l = [X_test] - X_train_l.append(X_train) - X_valid_l.append(X_valid) - X_test_l.append(X_test) +for f in DIRNAME.glob("feature/feat*.py"): + m = import_module_from_path(f.stem, f) + X_train_f = m.feat_eng(X_train) + X_valid_f = m.feat_eng(X_valid) + X_test_f = m.feat_eng(X_test) + + X_train_l.append(X_train_f) + X_valid_l.append(X_valid_f) + X_test_l.append(X_test_f) X_train = pd.concat(X_train_l, axis=1) X_valid = pd.concat(X_valid_l, axis=1) X_test = pd.concat(X_test_l, axis=1) +print(X_train.shape, X_valid.shape, X_test.shape) -def align_features(train_df, valid_df): - # Align the features of validation data to the training data - valid_df = valid_df.reindex(columns=train_df.columns, fill_value=0) - return valid_df +# Handle inf and -inf values +X_train.replace([np.inf, -np.inf], np.nan, inplace=True) +X_valid.replace([np.inf, -np.inf], np.nan, inplace=True) +X_test.replace([np.inf, -np.inf], np.nan, inplace=True) +from sklearn.impute import SimpleImputer -X_valid = align_features(X_train, X_valid) -X_test = align_features(X_train, X_test) +imputer = SimpleImputer(strategy="mean") + +X_train = pd.DataFrame(imputer.fit_transform(X_train), columns=X_train.columns) +X_valid = pd.DataFrame(imputer.transform(X_valid), columns=X_valid.columns) +X_test = pd.DataFrame(imputer.transform(X_test), columns=X_test.columns) + +# Remove duplicate columns +X_train = X_train.loc[:, ~X_train.columns.duplicated()] +X_valid = X_valid.loc[:, ~X_valid.columns.duplicated()] +X_test = X_test.loc[:, ~X_test.columns.duplicated()] # 3) Train the model model_l = [] # list[tuple[model, predict_func,]] -for f in DIRNAME.glob("model*.py"): - # TODO put select() in model.py: fit(X_train, y_train, X_valid, y_valid) - m = __import__(f.name.strip(".py")) +for f in DIRNAME.glob("model/model*.py"): + m = import_module_from_path(f.stem, f) model_l.append((m.fit(X_train, y_train, X_valid, y_valid), m.predict)) -# Evaluate the model on the validation set +# 4) Evaluate the model on the validation set y_valid_pred_l = [] for model, predict_func in model_l: y_valid_pred_l.append(predict_func(model, X_valid)) -# Ensemble +# 5) Ensemble # TODO: ensemble method in a script # Average the predictions and apply a threshold to determine class labels y_valid_pred = np.mean(y_valid_pred_l, axis=0) @@ -103,12 +97,12 @@ y_valid_pred = (y_valid_pred > 0.5).astype(int) mcc = compute_metrics_for_classification(y_valid, y_valid_pred) print("Final on validation set: ", mcc) -# Save the validation accuracy -pd.Series(data=[mcc], index=["MCC"]).to_csv( - "/home/v-xisenwang/RD-Agent/rdagent/scenarios/kaggle/experiment/meta_tpl/submission_score.csv" -) +# 6) Save the validation accuracy +pd.Series(data=[mcc], index=["MCC"]).to_csv("submission_score.csv") -# Make predictions on the test set and save them +# 7) Make predictions on the test set and save them +label_encoder = LabelEncoder() +label_encoder.fit(y_train) y_test_pred_bool_l = [] for m, m_pred in model_l: y_test_pred_bool_l.append( @@ -116,10 +110,11 @@ for m, m_pred in model_l: ) # TODO Make this an ensemble. Currently it uses the last prediction y_test_pred = np.mean(y_test_pred_bool_l, axis=0) -y_test_pred = (y_test_pred > 0.5).astype(int) # TODO Make it a module. Ensemble prediction +y_test_pred = (y_test_pred > 0.5).astype(int) y_test_pred_labels = label_encoder.inverse_transform(y_test_pred) # 将整数转换回 'e' 或 'p' + submission_result = pd.DataFrame({"id": passenger_ids, "class": y_test_pred_labels}) -# submit predictions for the test set -submission_result.to_csv("./submission.csv", index=False) +# 8) Submit predictions for the test set +submission_result.to_csv("submission.csv", index=False) diff --git a/rdagent/scenarios/kaggle/experiment/model_template/README.md b/rdagent/scenarios/kaggle/experiment/model_template/README.md deleted file mode 100644 index fb7c22c9..00000000 --- a/rdagent/scenarios/kaggle/experiment/model_template/README.md +++ /dev/null @@ -1,3 +0,0 @@ -## This folder is a template to be copied from for each model implementation & running process. - -Components: Dummy model.py, versatile conf.yaml, and a result reader. diff --git a/rdagent/scenarios/kaggle/experiment/model_template/train.py b/rdagent/scenarios/kaggle/experiment/model_template/train.py deleted file mode 100644 index e4114500..00000000 --- a/rdagent/scenarios/kaggle/experiment/model_template/train.py +++ /dev/null @@ -1,105 +0,0 @@ -import random - -import numpy as np -import pandas as pd -import xgboost as xgb -from model import get_num_round, get_params -from sklearn.compose import ColumnTransformer -from sklearn.impute import SimpleImputer -from sklearn.metrics import accuracy_score -from sklearn.model_selection import train_test_split -from sklearn.pipeline import Pipeline -from sklearn.preprocessing import OneHotEncoder - -# Set random seed for reproducibility -SEED = 42 -random.seed(SEED) -np.random.seed(SEED) - - -def compute_metrics_for_classification(y_true, y_pred): - """Compute accuracy metric for classification.""" - accuracy = accuracy_score(y_true, y_pred) - return accuracy - - -def train_model(X_train, y_train, X_valid, y_valid): - """Define and train the model.""" - dtrain = xgb.DMatrix(X_train, label=y_train) - dvalid = xgb.DMatrix(X_valid, label=y_valid) - - params = get_params() - num_round = get_num_round() - - evallist = [(dtrain, "train"), (dvalid, "eval")] - bst = xgb.train(params, dtrain, num_round, evallist) - - return bst - - -def predict(model, X): - dtest = xgb.DMatrix(X) - y_pred_prob = model.predict(dtest) - return y_pred_prob > 0.5 # Apply threshold to get boolean predictions - - -if __name__ == "__main__": - # Load and preprocess the data - data_df = pd.read_csv("/root/.data/train.csv") - data_df = data_df.drop(["PassengerId", "Name"], axis=1) - - X = data_df.drop(["Transported"], axis=1) - y = data_df.Transported.to_numpy() - - # Identify numerical and categorical features - numerical_cols = [cname for cname in X.columns if X[cname].dtype in ["int64", "float64"]] - categorical_cols = [cname for cname in X.columns if X[cname].dtype == "object"] - - # Define preprocessors for numerical and categorical features - categorical_transformer = Pipeline( - steps=[ - ("imputer", SimpleImputer(strategy="most_frequent")), - ("onehot", OneHotEncoder(handle_unknown="ignore")), - ] - ) - - numerical_transformer = Pipeline(steps=[("imputer", SimpleImputer(strategy="mean"))]) - - # Combine preprocessing steps - preprocessor = ColumnTransformer( - transformers=[ - ("cat", categorical_transformer, categorical_cols), - ("num", numerical_transformer, numerical_cols), - ] - ) - - # Split the data into training and validation sets - X_train, X_valid, y_train, y_valid = train_test_split(X, y, test_size=0.10, random_state=SEED) - - # Fit the preprocessor on the training data and transform both training and validation data - preprocessor.fit(X_train) - X_train = preprocessor.transform(X_train) - X_valid = preprocessor.transform(X_valid) - - # Train the model - model = train_model(X_train, y_train, X_valid, y_valid) - - # Evaluate the model on the validation set - y_valid_pred = predict(model, X_valid) - accuracy = compute_metrics_for_classification(y_valid, y_valid_pred) - print("Final Accuracy on validation set: ", accuracy) - - # Save the validation accuracy - pd.Series(data=[accuracy], index=["ACC"]).to_csv("./submission_score.csv") - - # Load and preprocess the test set - submission_df = pd.read_csv("/root/.data/test.csv") - passenger_ids = submission_df["PassengerId"] - submission_df = submission_df.drop(["PassengerId", "Name"], axis=1) - X_test = preprocessor.transform(submission_df) - - # Make predictions on the test set and save them - y_test_pred = predict(model, X_test) - submission_result = pd.DataFrame({"PassengerId": passenger_ids, "Transported": y_test_pred}) - # submit predictions for the test set - submission_result.to_csv("./submission.csv", index=False) diff --git a/rdagent/scenarios/kaggle/experiment/prompts.yaml b/rdagent/scenarios/kaggle/experiment/prompts.yaml index 3d220f4a..629ddc95 100644 --- a/rdagent/scenarios/kaggle/experiment/prompts.yaml +++ b/rdagent/scenarios/kaggle/experiment/prompts.yaml @@ -1,74 +1,297 @@ kg_description_template: system: |- You are an assistant that extracts structured information from unstructured text. + The user will provide you a Kaggle competition description, and you need to extract specific details from it. + For the dataset, the competition may not include detailed information about the dataset. The user has read the dataset and provide you the relevant information. Please include it in your response. + Please answer in Json format with the following schema: + { + "Competition Type": "The type of competition, e.g., 'Classification', 'Regression', 'Clustering', 'Prediction", "Time-Series Forecasting", + "Competition Description": "A brief description of the competition", + "Target Description": "A description of the target variable to be predicted", + "Competition Features": "A dict of relevant features used in the competition and their descriptions (if available)", # if you are not sure about the meaning of the feature, please add a (guess) before the description. Importantly, your feature name should be exactly the same as the feature name in the dataset! + } + user: |- - Based on the following competition description, please extract the following details: - 1. Competition Type - 2. Competition Description - 3. Target Description - 4. Competition Features + Competition Description: + {{ competition_descriptions }} + The raw data information: + {{ raw_data_information }} - Competition Description: {{ competition_descriptions }} +kg_background: |- + You are solving a data science tasks and the type of the competition is {{ competition_type }}. + The competition description is:{{competition_description}} + + We provide an overall script in file: train.py. The user will run the train.py script along with several feature and model scripts to train several model to get a good performance on this task. -kg_model_background: |- - You are solving this data science tasks of {{ competition_type }}: - {{competition_description}} + The train.py script is as follows: + ```python + {{ train_script }} + ``` - We provide an overall pipeline in train.py. Now fill in the provided train.py script to train a {{ competition_type }} model to get a good performance on this task. + The final output of our pipeline is from a ensemble of up to four models. Each model is trained on a different subset of the data. + The four model types are: XGBoost, RandomForest, LightGBM and Neural Network (A Pytorch model). - The model is a machine learning or deep learning structure designed to predict {{ target_description }}. - The data is extracted from the competition dataset, focusing on relevant attributes like {{ competition_features }}. - ModelType: The type of the model, "XGBoost" for XGBoost model. + The data is extracted from the competition dataset, focusing on relevant attributes in {{ competition_features }}. + + The user firstly designs and implements a feature book for each model. The feature book is a combination of several features and feature groups. + The feature book is built from: + - Raw features: The raw features are the original features from the dataset. + - generated features: The generated features are the features that are calculated based on the raw features according to some formulations. The calculation should be align with some physical or logical meaning. Don't just simply apply some numeric operations to the raw features. + - feature groups: The feature groups are preprocessed group of features from the raw features like normalization, one hot encoding, etc. + The feature or feature group is defined in the following parts: + - Name: The name of the feature or feature group. + - Description: A description of the feature or feature group. + - Formulation: The formulation of the feature or feature group. + - Variables: The variable list used in the formulation. Notice: The variable should be a specific feature in the dataset. Please make sure the feature name is exactly the same as the feature name in the dataset. + For each model, the user will design and implement the model in a separate script. The model is defined in the following parts: - Name: The name of the model. - Description: A description of the model. - Architecture: The detailed architecture of the model, such as neural network layers or tree structures. - + - ModelType: The type of the model, which should be one of ["XGBoost", "RandomForest", "LightGBM", "NN"]. The model should provide clear and detailed documentation of its architecture and hyperparameters. -kg_model_interface: |- - Your python code should follow the interface to better interact with the user's system. - You code should contain several parts: + The user tries to optimize the performance iteratively by employing one of the feature related or model related action items: + - Feature related: + - "Feature engineering": The user will design several new tasks and implement several new features. The new feature might only affect the model using all the feature book. + - "Feature processing": The user will design a new task to process the feature book like normalization or one hot encoding to improve the model performance. + - Model related: + - "Model feature selection": The user will modify one model to select the most important features from the feature book to improve the model performance. + - "Model tuning": The user will tune the hyperparameters of XGBoost, RandomForest or LightGBM or build or improve the NN model to improve the model performance. + + For each loop, you need to help user decide which action item to choose and provide the corresponding code to implement the action item. + +kg_feature_interface: |- + Your code should contain several parts: 1. The import part: import the necessary libraries. - 2. A class that is a subclass of xgboost. This class should have an __init__ function and a forward function, which inputs a tensor and outputs a tensor. - 3. Set a variable called "model_cls" to the class you defined. + 2. A feat_eng() function that handles feature engineering for each task. + The function should take the following arguments: + - X: The features as a pandas DataFrame. + The function should return the new features as a pandas DataFrame. + The input to `feat_eng` will be a pandas DataFrame, which should be processed to return a new DataFrame containing only the engineered features. + The original columns should be excluded from the returned DataFrame. - The user will save your code into a python file called "model.py". Then the user imports model_cls in file "model.py" after setting the cwd into the directory: + Exception handling will be managed externally, so avoid using try-except blocks in your code. The user will handle any exceptions that arise and provide feedback as needed. + + The feat_eng function can be one of the following: + - Feature engineering: This function calculated one new feature based on the existing raw data. + - Feature processing: This function processes the existing raw data like normalization or one hot encoding and return the processed data in the form of a pandas DataFrame. + + Here is an example of how your Python code should be structured: ```python - from model import get_params, get_num_round - ``` - So your python code should follow the pattern: - ```python - def get_params(): - params = { - ... - } - return params - def get_num_round(): - return xxx + import pandas as pd + + def feat_eng(X: pd.DataFrame): + """ + return the selected features + """ + return X.mean(axis=1).to_frame("mean_feature") # Example feature engineering + return X.fillna(0) # Example feature processing ``` - The model has one types, "XGBoost" for XGBoost model. - The XGBoost Model leverages two critical hyperparameters: "arams" and "num_round". - "params": This hyperparameter encapsulates various settings that dictate the model's behavior and learning process. - "num_round": This hyperparameter specifies the number of training iterations the model will undergo. - User will initialize the XGBoost model with the following code: + To Note: + 1. Ensure that your code meets these requirements and produces a feature-engineered DataFrame that contains only the newly engineered columns, aligning with the user's data and objectives. + 2. Ensure that the index of the output DataFrame matches the index of the original DataFrame. For example: + Incorrect: `normalized_df = pd.DataFrame(normalized_features, columns=X.columns)` + Correct: `normalized_df = pd.DataFrame(normalized_features, columns=X.columns, index=X.index)` + +kg_model_interface: |- + Your code should contain several parts: + 1. The import part: import the necessary libraries. + 2. A select() function that handles feature selection for both training and prediction phases. + The function should take the following arguments: + - X: The features as a pandas DataFrame. + The function should return the selected features as a pandas DataFrame. + 3. A function called fit() that trains the model and returns the trained model. If feature selection is applied, it should be done within this function. + The function should take the following arguments: + - X_train: The training features as a pandas DataFrame. + - y_train: The training labels as a pandas Series. + - X_valid: The validation features as a pandas DataFrame. + - y_valid: The validation labels as a pandas Series. + The function should return the trained model. + 4. A function called predict() that makes predictions using the trained model. If feature selection is applied, it should be done within this function. + The function should take the following arguments: + - model: The trained model. + - X: The features as a pandas DataFrame. + The function should return the predicted probabilities or boolean predictions in numpy.ndarray format. + + Here are some examples of how your Python code should be structured: + + For XGBoost: ```python - params = get_params() - num_round = get_num_round() + import pandas as pd + import numpy as np + import xgboost + from xgboost import DMatrix + + + def select(self, X: pd.DataFrame) -> pd.DataFrame: ... # Implement feature selection logic + + + def fit( + X_train: pd.DataFrame, y_train: pd.Series, X_valid: pd.DataFrame, y_valid: pd.Series + ) -> xgboost.Booster: + X_train = select(X_train) + X_valid = select(X_valid) + dtrain = DMatrix(X_train, label=y_train) + dvalid = DMatrix(X_valid, label=y_valid) + params = ... # Set parameters to XGBoost model + model = xgboost.train(params, dtrain, num_boost_round=100) + y_pred = model.predict(dvalid) + + accuracy = ... # Calculate accuracy + return model + + + def predict(model: xgboost.Booster, X: pd.DataFrame) -> np.ndarray: + X = select(X) + dtest = DMatrix(X) + y_pred = model.predict(dtest) + + return y_pred ``` - No other parameters will be passed to the model so give other parameters a default value or just make them static. - Don't write any try-except block in your python code. The user will catch the exception message and provide the feedback to you. Also, don't write main function in your python code. The user will call the forward method in the model_cls to get the output tensor. + For RandomForest: + ```python + import pandas as pd + import numpy as np + from sklearn.ensemble import RandomForestClassifier, RandomForestRegressor + from sklearn.metrics import accuracy_score - Please notice that your model should only use current features as input. The user will provide the input tensor to the model's forward function. + def select(self, X: pd.DataFrame) -> pd.DataFrame: ... # Implement feature selection logic + + + def fit( + X_train: pd.DataFrame, y_train: pd.Series, X_valid: pd.DataFrame, y_valid: pd.Series + ) -> RandomForestClassifier | RandomForestRegressor: + X_train = select(X_train) + X_valid = select(X_valid) + model = RandomForestClassifier(...) # fir classification tasks + model = RandomForestRegressor(...) # for regression tasks + model.fit(X_train, y_train, ...) # Train the model + + return model + + + def predict(model: RandomForestClassifier | RandomForestRegressor, X: pd.DataFrame) -> np.ndarray: + X = select(X) + y_pred = model.predict(X) + + return y_pred + ``` + + For LightGBM: + ```python + import pandas as pd + import numpy as np + from lightgbm import LGBMClassifier, LGBMRegressor + + + def select(self, X: pd.DataFrame) -> pd.DataFrame: ... # Implement feature selection logic + + + def fit( + X_train: pd.DataFrame, y_train: pd.Series, X_valid: pd.DataFrame, y_valid: pd.Series + ) -> LGBMClassifier | LGBMRegressor: + X_train = select(X_train) + X_valid = select(X_valid) + model = LGBMClassifier(...) # for classification tasks, please add parameters here + model = LGBMRegressor(...) # for regression tasks, please add parameters here + + model.fit(X=X_train, y=y_train, eval_set=[(X_valid, y_valid)]) + return model + + + def predict(model: LGBMClassifier | LGBMRegressor, X: pd.DataFrame) -> np.ndarray: + X = select(X) + y_pred = model.predict(X) + + return y_pred + ``` + + For Neural Network: + ```python + import pandas as pd + import numpy as np + import torch + from torch.utils.data import DataLoader, TensorDataset + + + class NNModel(torch.nn.Module): + def __init__(self): + super(Model, self).__init__() + # Define your model here + + def forward(self, x): + # Define the forward pass + return x + + + def select(self, X: pd.DataFrame) -> pd.DataFrame: ... # Implement feature selection logic + + + def fit(X_train: pd.DataFrame, y_train: pd.DataFrame, X_valid: pd.DataFrame, y_valid: pd.DataFrame) -> torch.nn.Module: + X_train = select(X_train) + X_valid = select(X_valid) + model = NNModel() # Initialize the model, You can write your own model class + + optimizer = torch.optim.Adam(model.parameters(), lr=0.01) # Example optimizer, you can use any optimizer + criterion = torch.nn.CrossEntropyLoss() # Example loss function, you can use any loss function + + train_loader = DataLoader(TensorDataset(X_train, y_train), batch_size=64, shuffle=True) + valid_loader = DataLoader(TensorDataset(X_valid, y_valid), batch_size=64, shuffle=False) + + # Example training loop, you can customize this loop as per your requirement + for epoch in range(10): + model.train() + for X_batch, y_batch in train_loader: + optimizer.zero_grad() + outputs = model(X_batch) + loss = criterion(outputs, y_batch) + loss.backward() + optimizer.step() + + model.eval() + y_pred = [] + with torch.no_grad(): + for X_batch, _ in valid_loader: + outputs = model(X_batch) + y_pred.extend(outputs.squeeze().tolist()) + + y_pred = torch.tensor(y_pred) + accuracy = (y_pred == y_valid).float().mean() + # You can early stop based on the validation, please customize this as per your requirement + return model + + + def predict(model: torch.nn.Module, X: pd.DataFrame) -> np.ndarray: + X = select(X) + X = torch.tensor(X.values).float() + model.eval() + with torch.no_grad(): + y_pred = model(X).squeeze().numpy() + + return y_pred + ``` + +kg_feature_simulator: |- + The data preprocessing method you provide will be used to prepare data by processing it, concatenating the results with other features, and removing unnecessary features before training the model. + The processed data will then be used for model training and prediction. + + User will use your data preprocessing method to do the following steps: + 1. Execute your Python files to process the data. (what you need to do) + 2. Concatenate the processed features with other features and the original data. + 3. Remove any unnecessary features before training the model. + 4. Train a model such as LightGBM, CatBoost, LSTM, or a simple PyTorch model using the processed data. + 5. Evaluate the performance of your preprocessing method and provide feedback. kg_model_output_format: |- - Your output should be an array with the appropriate number of predictions, each prediction being a single value. The output should be a 2D array with dimensions corresponding to the number of predictions and 1 column (e.g., (8, 1) if there are 8 predictions). + For feature related tasks, the output should be a pandas DataFrame with the new features. The columns should be the new features, and the rows should correspond to the number of samples in the input DataFrame. + For model related tasks, the output should be an np.ndarray with the appropriate number of predictions, each prediction being a single value. The output should be a 2D array with dimensions corresponding to the number of predictions and 1 column (e.g., (8, 1) if there are 8 predictions). kg_model_simulator: |- - The models will be trained on the competition dataset and evaluated on their ability to predict whether passengers were transported using metrics like accuracy and AUC-ROC. + The models will be trained on the competition dataset and evaluated on their ability to predict the target. Metrics like accuracy and AUC-ROC is used to evaluate the model performance. Model performance will be iteratively improved based on feedback from evaluation results. \ No newline at end of file diff --git a/rdagent/scenarios/kaggle/experiment/model_experiment.py b/rdagent/scenarios/kaggle/experiment/scenario.py similarity index 61% rename from rdagent/scenarios/kaggle/experiment/model_experiment.py rename to rdagent/scenarios/kaggle/experiment/scenario.py index 8cab9d21..6a9fbfb7 100644 --- a/rdagent/scenarios/kaggle/experiment/model_experiment.py +++ b/rdagent/scenarios/kaggle/experiment/scenario.py @@ -4,40 +4,35 @@ from pathlib import Path import pandas as pd from jinja2 import Environment, StrictUndefined -from rdagent.components.coder.model_coder.model import ( - ModelExperiment, - ModelFBWorkspace, - ModelTask, -) +from rdagent.components.coder.factor_coder.config import FACTOR_IMPLEMENT_SETTINGS from rdagent.core.prompts import Prompts from rdagent.core.scenario import Scenario from rdagent.oai.llm_utils import APIBackend -from rdagent.scenarios.kaggle.experiment.workspace import KGFBWorkspace +from rdagent.scenarios.kaggle.experiment.kaggle_experiment import KGFactorExperiment from rdagent.scenarios.kaggle.kaggle_crawler import crawl_descriptions prompt_dict = Prompts(file_path=Path(__file__).parent / "prompts.yaml") -class KGModelExperiment(ModelExperiment[ModelTask, KGFBWorkspace, ModelFBWorkspace]): - def __init__(self, *args, **kwargs) -> None: - super().__init__(*args, **kwargs) - self.experiment_workspace = KGFBWorkspace(template_folder_path=Path(__file__).parent / "model_template") - - -class KGModelScenario(Scenario): +class KGScenario(Scenario): def __init__(self, competition: str) -> None: super().__init__() self.competition = competition self.competition_descriptions = crawl_descriptions(competition) + self._source_data = self.source_data + self._output_format = self.output_format + self._interface = self.interface + self._simulator = self.simulator + self.competition_type = None self.competition_description = None self.target_description = None self.competition_features = None self._analysis_competition_description() - def _analysis_competition_description(self): - # TODO: use gpt to analyze the competition description + self._background = self.background + def _analysis_competition_description(self): sys_prompt = ( Environment(undefined=StrictUndefined) .from_string(prompt_dict["kg_description_template"]["system"]) @@ -49,6 +44,7 @@ class KGModelScenario(Scenario): .from_string(prompt_dict["kg_description_template"]["user"]) .render( competition_descriptions=self.competition_descriptions, + raw_data_information=self._source_data, ) ) @@ -63,15 +59,19 @@ class KGModelScenario(Scenario): self.competition_description = response_json_analysis.get("Competition Description", "No description provided") self.target_description = response_json_analysis.get("Target Description", "No target provided") self.competition_features = response_json_analysis.get("Competition Features", "No features provided") + self.competition_features = self.source_data @property def background(self) -> str: - background_template = prompt_dict["kg_model_background"] + background_template = prompt_dict["kg_background"] + + train_script = (Path(__file__).parent / "meta_tpl" / "train.py").read_text() background_prompt = ( Environment(undefined=StrictUndefined) .from_string(background_template) .render( + train_script=train_script, competition_type=self.competition_type, competition_description=self.competition_description, target_description=self.target_description, @@ -82,30 +82,26 @@ class KGModelScenario(Scenario): @property def source_data(self) -> str: - kaggle_conf = KGDockerConf() - data_path = Path(f"{kaggle_conf.share_data_path}/{self.competition}") + # TODO later we should improve this part + data_folder = Path(FACTOR_IMPLEMENT_SETTINGS.data_folder) - csv_files = list(data_path.glob("*.csv")) + if (data_folder / "valid.pkl").exists(): + X_valid = pd.read_pickle(data_folder / "valid.pkl") + return X_valid.head() - if not csv_files: - return "No CSV files found in the specified path." + preprocess_experiment = KGFactorExperiment([]) + ( + X_train, + X_valid, + y_train, + y_valid, + X_test, + passenger_ids, + ) = preprocess_experiment.experiment_workspace.generate_preprocess_data() - dataset = pd.concat([pd.read_csv(file) for file in csv_files], ignore_index=True) - - simple_eda = dataset.info(buf=None) # Capture the info output - data_shape = dataset.shape - data_head = dataset.head() - - eda = ( - f"Basic Info about the data:\n{simple_eda}\n" - f"Shape of the dataset: {data_shape}\n" - f"Sample Data:\n{data_head}\n" - ) - - data_description = self.competition_descriptions.get("Data Description", "No description provided") - eda += f"\nData Description:\n{data_description}" - - return eda + data_folder.mkdir(exist_ok=True, parents=True) + X_valid.to_pickle(data_folder / "valid.pkl") + return X_valid.head() @property def output_format(self) -> str: @@ -113,11 +109,19 @@ class KGModelScenario(Scenario): @property def interface(self) -> str: - return prompt_dict["kg_model_interface"] + return f"""The feature code should follow the interface: +{prompt_dict['kg_feature_interface']} +The model code should follow the interface: +{prompt_dict['kg_model_interface']} +""" @property def simulator(self) -> str: - return prompt_dict["kg_model_simulator"] + return f"""The feature code should follow the simulator: +{prompt_dict['kg_feature_simulator']} +The model code should follow the simulator: +{prompt_dict['kg_model_simulator']} +""" @property def rich_style_description(self) -> str: @@ -126,11 +130,13 @@ kaggle scen """ def get_scenario_all_desc(self) -> str: return f"""Background of the scenario: -{self.background} +{self._background} +The source dataset you can use to generate the features: +{self._source_data} The interface you should follow to write the runnable code: -{self.interface} +{self._interface} The output of your code should be in the format: -{self.output_format} +{self._output_format} The simulator user can use to test your model: -{self.simulator} +{self._simulator} """ diff --git a/rdagent/scenarios/kaggle/experiment/workspace.py b/rdagent/scenarios/kaggle/experiment/workspace.py index d2e657a1..3531652c 100644 --- a/rdagent/scenarios/kaggle/experiment/workspace.py +++ b/rdagent/scenarios/kaggle/experiment/workspace.py @@ -7,25 +7,73 @@ import pandas as pd from rdagent.app.kaggle.conf import PROP_SETTING from rdagent.core.experiment import FBWorkspace from rdagent.log import rdagent_logger as logger -from rdagent.utils.env import DockerEnv, KGDockerEnv +from rdagent.utils.env import KGDockerEnv + +KG_FEATURE_PREPROCESS_SCRIPT = """import pickle + +from fea_share_preprocess import preprocess_script + +X_train, X_valid, y_train, y_valid, X_test, passenger_ids = preprocess_script() + +pickle.dump(X_train, open("X_train.pkl", "wb")) +pickle.dump(X_valid, open("X_valid.pkl", "wb")) +pickle.dump(y_train, open("y_train.pkl", "wb")) +pickle.dump(y_valid, open("y_valid.pkl", "wb")) +pickle.dump(X_test, open("X_test.pkl", "wb")) +pickle.dump(passenger_ids, open("passenger_ids.pkl", "wb")) +""" class KGFBWorkspace(FBWorkspace): def __init__(self, template_folder_path: Path, *args, **kwargs) -> None: super().__init__(*args, **kwargs) self.inject_code_from_folder(template_folder_path) + self.data_description: list[str] = [] + self.model_description: str = "" + + def generate_preprocess_data( + self, + ) -> tuple[pd.DataFrame, pd.DataFrame, pd.Series, pd.Series, pd.DataFrame, pd.Series]: + kgde = KGDockerEnv(PROP_SETTING.competition) + kgde.prepare() + + execute_log, results = kgde.dump_python_code_run_and_get_results( + code=KG_FEATURE_PREPROCESS_SCRIPT, + local_path=str(self.workspace_path), + dump_file_names=[ + "X_train.pkl", + "X_valid.pkl", + "y_train.pkl", + "y_valid.pkl", + "X_test.pkl", + "passenger_ids.pkl", + ], + ) + if results is None: + logger.error("Feature preprocess failed.") + raise Exception("Feature preprocess failed.") + else: + X_train, X_valid, y_train, y_valid, X_test, passenger_ids = results + return X_train, X_valid, y_train, y_valid, X_test, passenger_ids def execute(self, run_env: dict = {}, *args, **kwargs) -> str: - qtde = KGDockerEnv(PROP_SETTING.competition) - qtde.prepare() + logger.info(f"Running the experiment in {self.workspace_path}") + kgde = KGDockerEnv(PROP_SETTING.competition) + kgde.prepare() - execute_log = qtde.run( + execute_log = kgde.run( local_path=str(self.workspace_path), entry=f"python train.py", env=run_env, ) - csv_path = self.workspace_path / "submission.csv" + csv_path = self.workspace_path / "submission_score.csv" + + print("WORKSPACE PATH IS HERE --------------------------------------------------------------------------------") + print(self.workspace_path) + print("CSV PATH IS HERE --------------------------------------------------------------------------------------") + print(csv_path) + print("CSV PATH IS HERE --------------------------------------------------------------------------------------") if not csv_path.exists(): logger.error(f"File {csv_path} does not exist.") diff --git a/rdagent/scenarios/kaggle/kaggle_crawler.py b/rdagent/scenarios/kaggle/kaggle_crawler.py index cddf77e4..828f65de 100644 --- a/rdagent/scenarios/kaggle/kaggle_crawler.py +++ b/rdagent/scenarios/kaggle/kaggle_crawler.py @@ -6,7 +6,8 @@ from selenium import webdriver from selenium.webdriver.chrome.service import Service from selenium.webdriver.common.by import By -LOCAL_PATH = "/data/userdata/share/kaggle_competition_descriptions" +from rdagent.log import rdagent_logger as logger +from rdagent.utils.env import KGDockerConf options = webdriver.ChromeOptions() options.add_argument("--no-sandbox") @@ -17,8 +18,8 @@ service = Service("/usr/local/bin/chromedriver") def crawl_descriptions(competition: str, wait: float = 3.0, force: bool = False) -> dict[str, str]: - if (fp := Path(f"{LOCAL_PATH}/{competition}.json")).exists() and not force: - print(f"Found {competition}.json, loading from local file.") + if (fp := Path(f"{KGDockerConf().local_data_path}/{competition}.json")).exists() and not force: + logger.info(f"Found {competition}.json, loading from local file.") with fp.open("r") as f: return json.load(f) @@ -61,7 +62,7 @@ def crawl_descriptions(competition: str, wait: float = 3.0, force: bool = False) descriptions["Data Description"] = data_element.get_attribute("innerHTML") driver.quit() - with open(f"{LOCAL_PATH}/{competition}.json", "w") as f: + with open(f"{KGDockerConf().dockerfile_folder_path}/{competition}.json", "w") as f: json.dump(descriptions, f) return descriptions diff --git a/rdagent/scenarios/kaggle/prompts.yaml b/rdagent/scenarios/kaggle/prompts.yaml index 9154a025..b755bce1 100644 --- a/rdagent/scenarios/kaggle/prompts.yaml +++ b/rdagent/scenarios/kaggle/prompts.yaml @@ -12,66 +12,76 @@ hypothesis_and_feedback: |- hypothesis_output_format: |- The output should follow JSON format. The schema is as follows: { - "hypothesis": "The new hypothesis generated based on the information provided.", - "reason": "The reason why you generate this hypothesis. It should be comprehensive and logical. It should cover the other keys below and extend them.", - "concise_reason": "Two-line summary. First line focuses on a concise justification for the change. Second line generalizes a knowledge statement.", - "concise_observation": "One line summary. It focuses on the observation of the given scenario, data characteristics, or previous experiences (failures & succeses).", - "concise_justification": "One line summary. Justify the hypothesis based on theoretical principles or initial assumptions.", - "concise_knowledge": "One line summary. Transferable knowledge based on theoretical principles. Use conditional grammar. eg. "If...., ..; When..., .; and etc" Make sure that you state things clearly without ambiguity. Eg. avoid saying "previous hypothesis", because one wouldn't know what that is." + "action": "The action that the user wants to take based on the information provided. should be one of ["Feature engineering", "Feature processing", "Model feature selection", "Model tuning"]", Only "Model tuning" For Now + "hypothesis": "The new hypothesis generated based on the information provided.", + "reason": "The reason why you generate this hypothesis. It should be comprehensive and logical. It should cover the other keys below and extend them.", + "concise_reason": "Two-line summary. First line focuses on a concise justification for the change. Second line generalizes a knowledge statement.", + "concise_observation": "One line summary. It focuses on the observation of the given scenario, data characteristics, or previous experiences (failures & succeses).", + "concise_justification": "One line summary. Justify the hypothesis based on theoretical principles or initial assumptions.", + "concise_knowledge": "One line summary. Transferable knowledge based on theoretical principles. Use conditional grammar. eg. "If...., ..; When..., .; and etc" Make sure that you state things clearly without ambiguity. Eg. avoid saying "previous hypothesis", because one wouldn't know what that is." } -model_hypothesis_specification: |- - Additional Specifications: - - Hypotheses should grow and evolve based on the previous hypothesis. If there is no previous hypothesis, start with something simple. Gradually Build Up Upon previous hypothesis & feedbacks. In each round, hypothesis is different. Pay attention to your previous hypothesis. +factor_hypothesis_specification: |- + 1. **Type of Feature and Data Characteristics:** + - Define the type of feature introduced. + - Explain the data characteristics or patterns captured by this feature. + - Omit unnecessary or redundant details. + 2. **Simple and Effective Features First:** + - Start with features that are simple and likely effective. + - Concisely explain why these features are expected to work. + - Avoid complex or combined features initially. + 3. **Gradual Complexity Increase:** + - Introduce more complex features as more experimental results are gathered. + - Discuss potential advantages and complexities. + - Combine features only after simpler ones are tested and validated. + 4. **New Directions and Optimizations:** + - If a new direction is needed, explain why based on data analysis, domain knowledge, or observed patterns. + - Suggest only one new direction at a time for clarity. + - If a previous hypothesis did not surpass the previous best, but seems optimizable, you may continue in the same direction. + - Highlight that features surpassing the previous best are included in the feature library to avoid re-implementation. + 5. **1-3 Features per Generation:** + - Ensure each generation produces 1-3 features. + - Balance simplicity and complexity to build a robust feature library. - Ensure that the hypothesis focuses on the architecture of a PyTorch model. Each hypothesis should address specific architectural choices such as the type of layers, activation functions, regularization techniques, and the overall structure of the model. Avoid hypotheses related to input features or optimization processes. - - Remember: if there is no hypothesis, start with something simple like MLP. - - Usually, a larger model works better than a smaller one. - - Logic for generating a new hypothesis: If the previous hypothesis works, try to inherit from it and grow deeper. If the previous hypotheis doesn't work, try to make changes in the current level. - - Sample hypothesis evolution loop: (This is the entire loop, see what stage you are at. We want hypothesis to continue growing.) Levels include **Model Type**, **Layer Configuration**, **Activation Functions**, **Regularization Techniques** - - 1st Round Hypothesis: The model should be a CNN. - - 2nd Round Hypothesis (If first round worked: CNN is the model type level, which means that we should extend to the next level, like layer configuration): The model should be a CNN. The CNN should have 5 convolutional layers. (Reasoning: As CNN worked, we now specify the layers specification to grow the hypothesis deeper.) - - 3rd Round Hypothesis (If second round didn't work): The model should be a CNN. The CNN should have 3 convolutional layers. (Reasoning: As 5-layer structure didn't work in the 2nd round hypothesis, try something else within the layer configuration level.) - - 4th Round Hypothesis (If third round worked): The model should be a CNN. The CNN should have 3 convolutional layers. Use Leaky ReLU activation for all layers. (As last round worked, now proceed to the next level: activation functions) - - 5th Round Hypothesis (If fourth round worked): The model should be a CNN. The CNN should have 3 convolutional layers. Use Leaky ReLU activation for all layers. Use dropout regularization with a rate of 0.5. (Similar Reasoning & Continuing to Grow to the dropout setup) - - 6th Round Hypothesis (If fourth round didn't work): The model should be a CNN. The CNN should have 5 convolutional layers. Use Leaky ReLU activation for all layers. Use dropout regularization with a rate of 0.3. (Reasoning: As regularisation rate of 0.5 didn't work, we only change a new regularisation and keep the other elements that worked. This means making changes in the current level.) +feature_experiment_output_format: |- + According to the hypothesis, please help user design one or more feature engineering tasks. + The output should follow JSON format. The schema is as follows: + { + "factor or group name 1": { + "description": "description of factor or group name 1", + "formulation": "latex formulation of factor or group name 1", + "variables": { + "variable or function name 1": "description of variable or function 1", + "variable or function name 2": "description of variable or function 2" + } + }, + "factor or group name 2": { + "description": "description of factor or group name 2", + "formulation": "latex formulation of factor or group name 2", + "variables": { + "variable or function name 1": "description of variable or function 1", + "variable or function name 2": "description of variable or function 2" + } + } + # Don't add ellipsis (...) or any filler text that might cause JSON parsing errors here! + } model_experiment_output_format: |- - So far please only design one model to test the hypothesis! + According to the hypothesis, please help user design one model task. + Since we only build one model from four model types: ["XGBoost", "RandomForest", "LightGBM", "NN"]. The output should follow JSON format. The schema is as follows: { - "model_name 1 (The name of the model)": { - "description": "A detailed description of the model", - "formulation": "A LaTeX formula representing the model's formulation", - "architecture": "A detailed description of the model's architecture, e.g., neural network layers or tree structures", - "variables": { - "\\hat{y}_u": "The predicted output for node u", - "variable_name_2": "Description of variable 2", - "variable_name_3": "Description of variable 3" - }, - "hyperparameters": { - "hyperparameter_name_1": "value of hyperparameter 1", - "hyperparameter_name_2": "value of hyperparameter 2", - "hyperparameter_name_3": "value of hyperparameter 3" - }, - "model_type": "XGBoost" # Should be "XGBoost" - }, - "model_name 2 (The name of the model)": { - ... - } + "model_name": "model_name", + "description": "A detailed description of the model", + "architecture": "A detailed description of the model's architecture, e.g., neural network layers or tree structures", + "hyperparameters": { + "hyperparameter_name_1": "value of hyperparameter 1", + "hyperparameter_name_2": "value of hyperparameter 2", + "hyperparameter_name_3": "value of hyperparameter 3" + }, + "model_type": "model type" } - Usually a larger model works better than a smaller one. Hence, the parameters should be larger. + Usually, a larger model works better than a smaller one. Hence, the parameters should be larger. model_feedback_generation: system: |- @@ -91,19 +101,19 @@ model_feedback_generation: Logic for generating a new hypothesis: If the previous hypothesis works, try to inherit from it and grow deeper. If the previous hypotheis doesn't work, try to make changes in the current level. - Sample hypothesis evolution loop: (This is the entire loop, see what stage you are at. We want hypothesis to continue growing.) Levels include **Model Type**, **Layer Configuration**, **Activation Functions**, **Regularization Techniques** + Sample hypothesis evolution loop: (This is the entire loop, see what stage you are at. We want hypotheses to continue growing.) Levels include **Model Type**, **Layer Configuration**, **Activation Functions**, **Regularization Techniques**, **Feature Selection Methods** - 1st Round Hypothesis: The model should be a CNN. + 1st Round Hypothesis: The model should be a CNN with no feature selection. - 2nd Round Hypothesis (If first round worked: CNN is the model type level, which means that we should extend to the next level, like layer configuration): The model should be a CNN. The CNN should have 5 convolutional layers. (Reasoning: As CNN worked, we now specify the layers specification to grow the hypothesis deeper.) + 2nd Round Hypothesis (If first round worked: CNN is the model type level, which means that we should extend to the next level, like layer configuration): The model should be a CNN. The CNN should have 5 convolutional layers, using all available features. (Reasoning: As CNN worked, we now specify the layers specification and feature selection to grow the hypothesis deeper.) - 3rd Round Hypothesis (If second round didn't work): The model should be a CNN. The CNN should have 3 convolutional layers. (Reasoning: As 5-layer structure didn't work in the 2nd round hypothesis, try something else within the layer configuration level.) + 3rd Round Hypothesis (If the second round didn't work): The model should be a CNN. The CNN should have 3 convolutional layers. Use L1 regularization for feature selection. (Reasoning: As the 5-layer structure didn't work in the 2nd round hypothesis, reduce the number of layers and implement feature selection.) - 4th Round Hypothesis (If third round worked): The model should be a CNN. The CNN should have 3 convolutional layers. Use Leaky ReLU activation for all layers. (As last round worked, now proceed to the next level: activation functions) - - 5th Round Hypothesis (If fourth round worked): The model should be a CNN. The CNN should have 3 convolutional layers. Use Leaky ReLU activation for all layers. Use dropout regularization with a rate of 0.5. (Similar Reasoning & Continuing to Grow to the dropout setup) + 4th Round Hypothesis (If the third round worked): The model should be a CNN. The CNN should have 3 convolutional layers. Use Leaky ReLU activation for all layers. Retain only features selected by L1 regularization. (As the last round worked, now proceed to the next level: activation functions) + + 5th Round Hypothesis (If the fourth round worked): The model should be a CNN. The CNN should have 3 convolutional layers. Use Leaky ReLU activation for all layers. Use dropout regularization with a rate of 0.5. Retain only features selected by L1 regularization. (Similar Reasoning & Continuing to Grow to the dropout setup) - 6th Round Hypothesis (If fourth round didn't work): The model should be a CNN. The CNN should have 5 convolutional layers. Use Leaky ReLU activation for all layers. Use dropout regularization with a rate of 0.3. (Reasoning: As regularisation rate of 0.5 didn't work, we only change a new regularisation and keep the other elements that worked. This means making changes in the current level.) + 6th Round Hypothesis (If the fourth round didn't work): The model should be a CNN. The CNN should have 5 convolutional layers. Use Leaky ReLU activation for all layers. Use dropout regularization with a rate of 0.3. Retain features selected by PCA. (Reasoning: As the regularization rate of 0.5 didn't work, change the regularization and use PCA for feature selection while keeping other elements that worked. This means making changes at the current level.) user: |- We are in an experiment of finding hypothesis and validating or rejecting them so that in the end we have a powerful model generated. @@ -128,3 +138,64 @@ model_feedback_generation: Compare and observe. Which result has a better return and lower risk? If the performance increases, the hypothesis should be considered positive (working). Hence, with the hypotheses, relevant reasoning, and results in mind (comparison), provide detailed and constructive feedback and suggest a new hypothesis. + +factor_feedback_generation: + system: |- + You are a professional data feature engineering assistant in data-driven R&D. + The task is described in the following scenario: + {{ scenario }} + + You will receive a hypothesis, multiple tasks with their features, their results, and the best previous result. + Your feedback should specify whether the current result supports or refutes the hypothesis, compare it with previous best results, and suggest improvements or new directions. + + Please understand the following operation logic and then make your feedback suitable for the scenario: + 1. Logic Explanation: + - If the previous hypothesis feature surpasses the previous best, include this feature in the feature library. + - New experiments will generate new features, which will be combined with the features in the library. + - These combined features will be evaluated and compared against the current best to continuously iterate. + 2. Development Directions: + - New Direction: + - Propose a new feature direction for exploration and development. + - Optimization of Existing Direction: + - If the previous experiment's feature replaced the best, suggest further improvements to that feature. + - Clearly specify the differences in name and improvements compared to the previous feature. + - Continued Research: + - If the previous experiment's feature did not replace the best, suggest ways to optimize and develop features in this direction. + 3. Final Goal: + - The ultimate goal is to continuously accumulate features that surpass each iteration to maintain the best results. + + Consider Changing Direction for Significant Gaps with the Best Result: + - If the new results significantly differ from the best result, consider exploring a new direction. + - Avoid re-implementing previous features as those that surpassed the best are already included in the feature library and will be used in each run. + Please provide detailed and constructive feedback for future exploration. + Respond in JSON format. Example JSON structure for Result Analysis: + { + "Observations": "Your overall observations here", + "Feedback for Hypothesis": "Observations related to the hypothesis", + "New Hypothesis": "Your new hypothesis here", + "Reasoning": "Reasoning for the new hypothesis", + "Replace Best Result": "yes or no" + } + user: |- + Target hypothesis: + {{ hypothesis_text }} + Tasks and Features: + {% for task in task_details %} + - {{ task.factor_name }}: {{ task.factor_description }} + - Feature Formulation: {{ task.factor_formulation }} + - Variables: {{ task.variables }} + - Feature Implementation: {{ task.factor_implementation }} + {% if task.factor_implementation == "False" %} + **Note: This feature was not implemented in the current experiment. Only the hypothesis for implemented features can be verified.** + {% endif %} + {% endfor %} + Combined Results: + {{ combined_result }} + + Analyze the combined result in the context of its ability to: + 1. Support or refute the hypothesis. + 2. Show improvement or deterioration compared to the best result. + Consider Changing Direction for Significant Gaps with the Best Result: + - If the new results significantly differ from the best, consider exploring a new direction. + - Avoid re-implementing previous features as those that surpassed the best are already included in the feature library and will be used in each run. + Note: Only features with 'Feature Implementation' as True are implemented and tested in this experiment. If 'Feature Implementation' is False, the hypothesis for that feature cannot be verified in this run. \ No newline at end of file diff --git a/rdagent/scenarios/kaggle/proposal/model_proposal.py b/rdagent/scenarios/kaggle/proposal/model_proposal.py deleted file mode 100644 index 6a181db2..00000000 --- a/rdagent/scenarios/kaggle/proposal/model_proposal.py +++ /dev/null @@ -1,105 +0,0 @@ -import json -from pathlib import Path -from typing import List, Tuple - -from jinja2 import Environment, StrictUndefined - -from rdagent.components.coder.model_coder.model import ModelExperiment, ModelTask -from rdagent.components.proposal.model_proposal import ( - ModelHypothesis, - ModelHypothesis2Experiment, - ModelHypothesisGen, -) -from rdagent.core.prompts import Prompts -from rdagent.core.proposal import Hypothesis, Scenario, Trace -from rdagent.scenarios.kaggle.experiment.model_experiment import KGModelExperiment - -prompt_dict = Prompts(file_path=Path(__file__).parent.parent / "prompts.yaml") - -KGModelHypothesis = ModelHypothesis - - -class KGModelHypothesisGen(ModelHypothesisGen): - """ - # NOTE: we can share this class across different data mining scenarios - # It may better to move the class into components folder like `rdagent/components/proposal/model_proposal.py` - # Here is the use case: - - .. code-block:: python - - class XXXDMModelHypothesisGen(DMModelHypothesisGen): - prompts: Prompts = a_specifc_prompt_dict - """ - - def __init__(self, scen: Scenario) -> Tuple[dict, bool]: - super().__init__(scen) - - def prepare_context(self, trace: Trace) -> Tuple[dict, bool]: - hypothesis_feedback = ( - Environment(undefined=StrictUndefined) - .from_string(prompt_dict["hypothesis_and_feedback"]) - .render(trace=trace) - ) - context_dict = { - "hypothesis_and_feedback": hypothesis_feedback, - "RAG": "", - "hypothesis_output_format": prompt_dict["hypothesis_output_format"], - "hypothesis_specification": "...", - } - return context_dict, True - - def convert_response(self, response: str) -> ModelHypothesis: - response_dict = json.loads(response) - hypothesis = KGModelHypothesis( - hypothesis=response_dict["hypothesis"], - reason=response_dict["reason"], - concise_reason=response_dict["concise_reason"], - concise_observation=response_dict["concise_observation"], - concise_justification=response_dict["concise_justification"], - concise_knowledge=response_dict["concise_knowledge"], - ) - return hypothesis - - -class KGModelHypothesis2Experiment(ModelHypothesis2Experiment): - def prepare_context(self, hypothesis: Hypothesis, trace: Trace) -> Tuple[dict, bool]: - scenario = trace.scen.get_scenario_all_desc() - experiment_output_format = prompt_dict["model_experiment_output_format"] - - hypothesis_and_feedback = ( - Environment(undefined=StrictUndefined) - .from_string(prompt_dict["hypothesis_and_feedback"]) - .render(trace=trace) - ) - - experiment_list: List[ModelExperiment] = [t[1] for t in trace.hist] - - model_list = [] - for experiment in experiment_list: - model_list.extend(experiment.sub_tasks) - - return { - "target_hypothesis": str(hypothesis), - "scenario": scenario, - "hypothesis_and_feedback": hypothesis_and_feedback, - "experiment_output_format": experiment_output_format, - "target_list": model_list, - "RAG": ..., - }, True - - def convert_response(self, response: str, trace: Trace) -> ModelExperiment: - response_dict = json.loads(response) - tasks = [] - for model_name in response_dict: - description = response_dict[model_name]["description"] - formulation = response_dict[model_name]["formulation"] - architecture = response_dict[model_name]["architecture"] - variables = response_dict[model_name]["variables"] - hyperparameters = response_dict[model_name]["hyperparameters"] - model_type = response_dict[model_name]["model_type"] - tasks.append( - ModelTask(model_name, description, formulation, architecture, variables, hyperparameters, model_type) - ) - exp = KGModelExperiment(tasks) - exp.based_experiments = [t[1] for t in trace.hist if t[2]] - return exp diff --git a/rdagent/scenarios/kaggle/proposal/proposal.py b/rdagent/scenarios/kaggle/proposal/proposal.py new file mode 100644 index 00000000..0b136b36 --- /dev/null +++ b/rdagent/scenarios/kaggle/proposal/proposal.py @@ -0,0 +1,181 @@ +import json +from pathlib import Path +from typing import List, Tuple + +from jinja2 import Environment, StrictUndefined + +from rdagent.components.coder.factor_coder.factor import FactorTask +from rdagent.components.coder.model_coder.model import ModelExperiment, ModelTask +from rdagent.components.proposal.model_proposal import ( + ModelHypothesis, + ModelHypothesis2Experiment, + ModelHypothesisGen, +) +from rdagent.core.prompts import Prompts +from rdagent.core.proposal import Hypothesis, Scenario, Trace +from rdagent.scenarios.kaggle.experiment.kaggle_experiment import ( + KGFactorExperiment, + KGModelExperiment, +) + +prompt_dict = Prompts(file_path=Path(__file__).parent.parent / "prompts.yaml") + + +KG_ACTION_FEATURE_ENGINEERING = "Feature engineering" +KG_ACTION_FEATURE_PROCESSING = "Feature processing" +KG_ACTION_MODEL_FEATURE_SELECTION = "Model feature selection" +KG_ACTION_MODEL_TUNING = "Model tuning" +KG_ACTION_LIST = [ + KG_ACTION_FEATURE_ENGINEERING, + KG_ACTION_FEATURE_PROCESSING, + KG_ACTION_MODEL_FEATURE_SELECTION, + KG_ACTION_MODEL_TUNING, +] + + +class KGHypothesis(Hypothesis): + def __init__( + self, + hypothesis: str, + reason: str, + concise_reason: str, + concise_observation: str, + concise_justification: str, + concise_knowledge: str, + action: str, + ) -> None: + super().__init__( + hypothesis, reason, concise_reason, concise_observation, concise_justification, concise_knowledge + ) + self.action = action + + def __str__(self) -> str: + return f"""Chosen Action: {self.action} +Hypothesis: {self.hypothesis} +Reason: {self.reason} +Concise Reason & Knowledge: {self.concise_reason} +Concise Observation: {self.concise_observation} +Concise Justification: {self.concise_justification} +Concise Knowledge: {self.concise_knowledge} +""" + + +class KGHypothesisGen(ModelHypothesisGen): + """ + # NOTE: we can share this class across different data mining scenarios + # It may better to move the class into components folder like `rdagent/components/proposal/model_proposal.py` + # Here is the use case: + + .. code-block:: python + + class XXXDMModelHypothesisGen(DMModelHypothesisGen): + prompts: Prompts = a_specifc_prompt_dict + """ + + def __init__(self, scen: Scenario) -> Tuple[dict, bool]: + super().__init__(scen) + + def prepare_context(self, trace: Trace) -> Tuple[dict, bool]: + hypothesis_feedback = ( + Environment(undefined=StrictUndefined) + .from_string(prompt_dict["hypothesis_and_feedback"]) + .render(trace=trace) + ) + context_dict = { + "hypothesis_and_feedback": hypothesis_feedback, + "RAG": None, + "hypothesis_output_format": prompt_dict["hypothesis_output_format"], + "hypothesis_specification": None, + } + return context_dict, True + + def convert_response(self, response: str) -> ModelHypothesis: + response_dict = json.loads(response) + hypothesis = KGHypothesis( + hypothesis=response_dict["hypothesis"], + reason=response_dict["reason"], + concise_reason=response_dict["concise_reason"], + concise_observation=response_dict["concise_observation"], + concise_justification=response_dict["concise_justification"], + concise_knowledge=response_dict["concise_knowledge"], + action=response_dict["action"], + ) + return hypothesis + + +class KGHypothesis2Experiment(ModelHypothesis2Experiment): + def prepare_context(self, hypothesis: Hypothesis, trace: Trace) -> Tuple[dict, bool]: + scenario = trace.scen.get_scenario_all_desc() + assert isinstance(hypothesis, KGHypothesis) + experiment_output_format = ( + prompt_dict["feature_experiment_output_format"] + if hypothesis.action in [KG_ACTION_FEATURE_ENGINEERING, KG_ACTION_FEATURE_PROCESSING] + else prompt_dict["model_experiment_output_format"] + ) + self.current_action = hypothesis.action + + hypothesis_and_feedback = ( + Environment(undefined=StrictUndefined) + .from_string(prompt_dict["hypothesis_and_feedback"]) + .render(trace=trace) + ) + + experiment_list: List[ModelExperiment] = [t[1] for t in trace.hist] + + model_list = [] + for experiment in experiment_list: + model_list.extend(experiment.sub_tasks) + + return { + "target_hypothesis": str(hypothesis), + "scenario": scenario, + "hypothesis_and_feedback": hypothesis_and_feedback, + "experiment_output_format": experiment_output_format, + "target_list": model_list, + "RAG": ..., + }, True + + def convert_feature_experiment(self, response: str, trace: Trace) -> KGFactorExperiment: + response_dict = json.loads(response) + tasks = [] + + for factor_name in response_dict: + description = response_dict[factor_name]["description"] + formulation = response_dict[factor_name]["formulation"] + variables = response_dict[factor_name]["variables"] + tasks.append( + FactorTask( + factor_name=factor_name, + factor_description=description, + factor_formulation=formulation, + variables=variables, + version=2, + ) + ) + + exp = KGFactorExperiment(tasks) + exp.based_experiments = [KGFactorExperiment(sub_tasks=[])] + [t[1] for t in trace.hist if t[2]] + return exp + + def convert_model_experiment(self, response: str, trace: Trace) -> KGModelExperiment: + response_dict = json.loads(response) + tasks = [] + tasks.append( + ModelTask( + name=response_dict["model_name"], + description=response_dict["description"], + architecture=response_dict["architecture"], + hyperparameters=response_dict["hyperparameters"], + model_type=response_dict["model_type"], + version=2, + ) + ) + exp = KGModelExperiment(tasks) + exp.based_experiments = [t[1] for t in trace.hist if t[2]] + return exp + + def convert_response(self, response: str, trace: Trace) -> ModelExperiment: + if self.current_action in [KG_ACTION_FEATURE_ENGINEERING, KG_ACTION_FEATURE_PROCESSING]: + return self.convert_feature_experiment(response, trace) + elif self.current_action in [KG_ACTION_MODEL_FEATURE_SELECTION, KG_ACTION_MODEL_TUNING]: + return self.convert_model_experiment(response, trace) diff --git a/rdagent/scenarios/qlib/docker/Dockerfile b/rdagent/scenarios/qlib/docker/Dockerfile index 55e1406b..8ceabce5 100644 --- a/rdagent/scenarios/qlib/docker/Dockerfile +++ b/rdagent/scenarios/qlib/docker/Dockerfile @@ -1,4 +1,4 @@ -FROM pytorch/pytorch:latest +FROM pytorch/pytorch:2.2.1-cuda12.1-cudnn8-runtime # For GPU support, please choose the proper tag from https://hub.docker.com/r/pytorch/pytorch/tags diff --git a/rdagent/scenarios/qlib/proposal/model_proposal.py b/rdagent/scenarios/qlib/proposal/model_proposal.py index 9bc53bf7..274c3273 100644 --- a/rdagent/scenarios/qlib/proposal/model_proposal.py +++ b/rdagent/scenarios/qlib/proposal/model_proposal.py @@ -87,7 +87,15 @@ class QlibModelHypothesis2Experiment(ModelHypothesis2Experiment): hyperparameters = response_dict[model_name]["hyperparameters"] model_type = response_dict[model_name]["model_type"] tasks.append( - ModelTask(model_name, description, formulation, architecture, variables, hyperparameters, model_type) + ModelTask( + name=model_name, + description=description, + formulation=formulation, + architecture=architecture, + variables=variables, + hyperparameters=hyperparameters, + model_type=model_type, + ) ) exp = QlibModelExperiment(tasks) exp.based_experiments = [t[1] for t in trace.hist if t[2]] diff --git a/rdagent/utils/env.py b/rdagent/utils/env.py index 196620f3..36fbe044 100644 --- a/rdagent/utils/env.py +++ b/rdagent/utils/env.py @@ -4,12 +4,15 @@ The motiviation of the utils is for environment management Tries to create uniform environment for the agent to run; - All the code and data is expected included in one folder """ + # TODO: move the scenario specific docker env into other folders. import json import os +import pickle import subprocess import sys +import uuid import zipfile from abc import abstractmethod from pathlib import Path @@ -146,6 +149,7 @@ class QlibDockerConf(DockerConf): class DMDockerConf(DockerConf): + # Data Mining Docker class Config: env_prefix = "DM_DOCKER_" @@ -177,7 +181,7 @@ class KGDockerConf(DockerConf): Path("git_ignore_folder/data").resolve(): "/root/.data/" } - share_data_path: str = "/data/userdata/share/kaggle" + local_data_path: str = "/data/userdata/share/kaggle" # physionet.org/files/mimic-eicu-fiddle-feature/1.0.0/FIDDLE_mimic3 @@ -248,9 +252,9 @@ class DockerEnv(Env[DockerConf]): if not self.conf.enable_gpu: return {} gpu_kwargs = { - "device_requests": [docker.types.DeviceRequest(count=-1, capabilities=[["gpu"]])] - if self.conf.enable_gpu - else None, + "device_requests": ( + [docker.types.DeviceRequest(count=-1, capabilities=[["gpu"]])] if self.conf.enable_gpu else None + ), } try: client.containers.run(self.conf.image, "nvidia-smi", **gpu_kwargs) @@ -259,7 +263,13 @@ class DockerEnv(Env[DockerConf]): return {} return gpu_kwargs - def run(self, entry: str | None = None, local_path: str | None = None, env: dict | None = None): + def run( + self, + entry: str | None = None, + local_path: str | None = None, + env: dict | None = None, + running_extra_volume: dict | None = None, + ) -> str: if env is None: env = {} client = docker.from_env() @@ -273,6 +283,9 @@ class DockerEnv(Env[DockerConf]): if self.conf.extra_volumes is not None: for lp, rp in self.conf.extra_volumes.items(): volumns[lp] = {"bind": rp, "mode": "rw"} + if running_extra_volume is not None: + for lp, rp in running_extra_volume.items(): + volumns[lp] = {"bind": rp, "mode": "rw"} log_output = "" @@ -305,6 +318,27 @@ class DockerEnv(Env[DockerConf]): except docker.errors.APIError as e: raise RuntimeError(f"Error while running the container: {e}") + def dump_python_code_run_and_get_results( + self, code: str, dump_file_names: list[str], local_path: str | None = None, env: dict | None = None + ): + """ + Dump the code into the local path and run the code. + """ + random_file_name = f"{uuid.uuid4()}.py" + with open(os.path.join(local_path, random_file_name), "w") as f: + f.write(code) + entry = f"python {random_file_name}" + log_output = self.run(entry, local_path, env) + results = [] + os.remove(os.path.join(local_path, random_file_name)) + for name in dump_file_names: + if os.path.exists(os.path.join(local_path, f"{name}")): + results.append(pickle.load(open(os.path.join(local_path, f"{name}"), "rb"))) + os.remove(os.path.join(local_path, f"{name}")) + else: + return log_output, None + return log_output, results + class QTDockerEnv(DockerEnv): """Qlib Torch Docker""" @@ -349,9 +383,9 @@ class DMDockerEnv(DockerEnv): class KGDockerEnv(DockerEnv): - """Qlib Torch Docker""" + """Kaggle Competition Docker""" - def __init__(self, competition: str, conf: DockerConf = KGDockerConf()): + def __init__(self, competition: str = None, conf: DockerConf = KGDockerConf()): super().__init__(conf) self.competition = competition @@ -361,10 +395,21 @@ class KGDockerEnv(DockerEnv): """ super().prepare() - # download data - data_path = f"{self.conf.share_data_path}/{self.competition}" - subprocess.run(["kaggle", "competitions", "download", "-c", self.competition, "-p", data_path]) + # download data, if competition is not provided, the user is targeting a general docker environment in kaggle + data_path = f"{self.conf.local_data_path}/{self.competition}" + if self.competition is not None and not Path(data_path).exists(): + subprocess.run(["kaggle", "competitions", "download", "-c", self.competition, "-p", data_path]) - # unzip data - with zipfile.ZipFile(f"{data_path}/{self.competition}.zip", "r") as zip_ref: - zip_ref.extractall(data_path) + # unzip data + with zipfile.ZipFile(f"{data_path}/{self.competition}.zip", "r") as zip_ref: + zip_ref.extractall(data_path) + + def run(self, entry: str | None = None, local_path: str | None = None, env: dict | None = None): + super().run( + entry=entry, + local_path=local_path, + env=env, + running_extra_volume=( + {self.conf.local_data_path + "/" + self.competition: "/kaggle/input"} if self.competition else None + ), + ) diff --git a/requirements.txt b/requirements.txt index 59013632..f4805d58 100644 --- a/requirements.txt +++ b/requirements.txt @@ -63,8 +63,10 @@ st-theme selenium kaggle +# tool seaborn setuptools-scm # This is a temporary package installed to pass the test_import test xgboost +lightgbm