From 82bef3a375fc2b758759f7629b506da5f92f5469 Mon Sep 17 00:00:00 2001 From: WinstonLiyt <104308117+WinstonLiyt@users.noreply.github.com> Date: Thu, 12 Sep 2024 00:30:16 +0800 Subject: [PATCH] feat: fix some bugs and add original features' description (#259) * init a scenario for kaggle feature engineering * fix some bugs and add original features' description * refine the process of data downloading * fix a error * revert the code * fix a bug in feedback * fix a ci bug * fix a ci bug --- docs/scens/model_agent_med.rst | 2 +- rdagent/app/data_mining/conf.py | 4 +- rdagent/app/data_mining/model.py | 4 +- rdagent/app/kaggle/conf.py | 4 +- rdagent/app/kaggle/loop.py | 6 +- rdagent/app/kaggle_feature/model.py | 65 ------------------- .../components/coder/factor_coder/factor.py | 6 +- rdagent/components/coder/model_coder/model.py | 1 - .../data_mining/experiment/workspace.py | 4 +- .../scenarios/kaggle/developer/feedback.py | 14 ++-- rdagent/scenarios/kaggle/developer/runner.py | 17 ++++- .../kaggle/experiment/kaggle_experiment.py | 8 ++- .../experiment/meta_tpl/feature/feature.py | 2 +- .../kaggle/experiment/meta_tpl/train.py | 4 +- .../scenarios/kaggle/experiment/scenario.py | 3 +- .../scenarios/kaggle/experiment/workspace.py | 12 ++-- rdagent/scenarios/kaggle/prompts.yaml | 2 +- 17 files changed, 51 insertions(+), 107 deletions(-) delete mode 100644 rdagent/app/kaggle_feature/model.py diff --git a/docs/scens/model_agent_med.rst b/docs/scens/model_agent_med.rst index 5109475d..294830f0 100644 --- a/docs/scens/model_agent_med.rst +++ b/docs/scens/model_agent_med.rst @@ -123,6 +123,6 @@ You can try our demo by running the following command: The following environment variables can be set in the `.env` file to customize the application's behavior: -.. autopydantic_settings:: rdagent.app.data_mining.conf.PropSetting +.. autopydantic_settings:: rdagent.app.data_mining.conf.MedBasePropSetting :settings-show-field-summary: False :exclude-members: Config diff --git a/rdagent/app/data_mining/conf.py b/rdagent/app/data_mining/conf.py index 094666eb..5cd12d75 100644 --- a/rdagent/app/data_mining/conf.py +++ b/rdagent/app/data_mining/conf.py @@ -5,7 +5,7 @@ from pydantic_settings import BaseSettings from rdagent.components.workflow.conf import BasePropSetting -class PropSetting(BasePropSetting): +class MedBasePropSetting(BasePropSetting): class Config: env_prefix = "DM_" """Use `DM_` as prefix for environment variables""" @@ -46,4 +46,4 @@ class PropSetting(BasePropSetting): """Physionet account password""" -PROP_SETTING = PropSetting() +MED_PROP_SETTING = MedBasePropSetting() diff --git a/rdagent/app/data_mining/model.py b/rdagent/app/data_mining/model.py index c6e996bd..8deaaf9c 100644 --- a/rdagent/app/data_mining/model.py +++ b/rdagent/app/data_mining/model.py @@ -1,6 +1,6 @@ import fire -from rdagent.app.data_mining.conf import PROP_SETTING +from rdagent.app.data_mining.conf import MED_PROP_SETTING from rdagent.components.workflow.rd_loop import RDLoop from rdagent.core.exception import ModelEmptyError @@ -21,7 +21,7 @@ def main(path=None, step_n=None): """ if path is None: - model_loop = ModelRDLoop(PROP_SETTING) + model_loop = ModelRDLoop(MED_PROP_SETTING) else: model_loop = ModelRDLoop.load(path) model_loop.run(step_n=step_n) diff --git a/rdagent/app/kaggle/conf.py b/rdagent/app/kaggle/conf.py index 0ffb8f86..62d5fe05 100644 --- a/rdagent/app/kaggle/conf.py +++ b/rdagent/app/kaggle/conf.py @@ -5,7 +5,7 @@ from pydantic_settings import BaseSettings from rdagent.components.workflow.conf import BasePropSetting -class PropSetting(BasePropSetting): +class KaggleBasePropSetting(BasePropSetting): class Config: env_prefix = "KG_" """Use `KG_` as prefix for environment variables""" @@ -43,4 +43,4 @@ class PropSetting(BasePropSetting): competition: str = "" -PROP_SETTING = PropSetting() +KAGGLE_IMPLEMENT_SETTING = KaggleBasePropSetting() diff --git a/rdagent/app/kaggle/loop.py b/rdagent/app/kaggle/loop.py index 9658aaa1..0e823a71 100644 --- a/rdagent/app/kaggle/loop.py +++ b/rdagent/app/kaggle/loop.py @@ -3,7 +3,7 @@ from typing import Any import fire -from rdagent.app.kaggle.conf import PROP_SETTING +from rdagent.app.kaggle.conf import KAGGLE_IMPLEMENT_SETTING from rdagent.components.workflow.conf import BasePropSetting from rdagent.components.workflow.rd_loop import RDLoop from rdagent.core.developer import Developer @@ -83,9 +83,9 @@ def main(path=None, step_n=None, competition=None): """ if competition: - PROP_SETTING.competition = competition + KAGGLE_IMPLEMENT_SETTING.competition = competition if path is None: - model_loop = ModelRDLoop(PROP_SETTING) + model_loop = ModelRDLoop(KAGGLE_IMPLEMENT_SETTING) else: model_loop = ModelRDLoop.load(path) model_loop.run(step_n=step_n) diff --git a/rdagent/app/kaggle_feature/model.py b/rdagent/app/kaggle_feature/model.py deleted file mode 100644 index 43e601b7..00000000 --- a/rdagent/app/kaggle_feature/model.py +++ /dev/null @@ -1,65 +0,0 @@ -from collections import defaultdict - -import fire - -from rdagent.app.kaggle.conf import PROP_SETTING -from rdagent.components.workflow.conf import BasePropSetting -from rdagent.components.workflow.rd_loop import RDLoop -from rdagent.core.exception import ModelEmptyError -from rdagent.core.proposal import ( - Hypothesis2Experiment, - HypothesisExperiment2Feedback, - HypothesisGen, - Trace, -) -from rdagent.core.utils import import_class -from rdagent.log import rdagent_logger as logger - - -class ModelRDLoop(RDLoop): - def __init__(self, PROP_SETTING: BasePropSetting): - with logger.tag("init"): - scen: Scenario = import_class(PROP_SETTING.scen)(PROP_SETTING.competition) - logger.log_object(scen, tag="scenario") - - self.hypothesis_gen: HypothesisGen = import_class(PROP_SETTING.hypothesis_gen)(scen) - logger.log_object(self.hypothesis_gen, tag="hypothesis generator") - - self.hypothesis2experiment: Hypothesis2Experiment = import_class(PROP_SETTING.hypothesis2experiment)() - logger.log_object(self.hypothesis2experiment, tag="hypothesis2experiment") - - self.coder: Developer = import_class(PROP_SETTING.coder)(scen) - logger.log_object(self.coder, tag="coder") - self.runner: Developer = import_class(PROP_SETTING.runner)(scen) - logger.log_object(self.runner, tag="runner") - - self.summarizer: HypothesisExperiment2Feedback = import_class(PROP_SETTING.summarizer)(scen) - logger.log_object(self.summarizer, tag="summarizer") - self.trace = Trace(scen=scen) - super(RDLoop, self).__init__() - - skip_loop_error = (ModelEmptyError,) - - -def main(path=None, step_n=None, competition=None): - """ - Auto R&D Evolving loop for models in a kaggle{} scenario. - - You can continue running session by - - .. code-block:: python - - dotenv run -- python rdagent/app/kaggle/model.py [--competition titanic] $LOG_PATH/__session__/1/0_propose --step_n 1 # `step_n` is a optional paramter - - """ - if competition: - PROP_SETTING.competition = competition - if path is None: - model_loop = ModelRDLoop(PROP_SETTING) - else: - model_loop = ModelRDLoop.load(path) - model_loop.run(step_n=step_n) - - -if __name__ == "__main__": - fire.Fire(main) diff --git a/rdagent/components/coder/factor_coder/factor.py b/rdagent/components/coder/factor_coder/factor.py index 7dde2ebe..f794e4c1 100644 --- a/rdagent/components/coder/factor_coder/factor.py +++ b/rdagent/components/coder/factor_coder/factor.py @@ -9,6 +9,7 @@ from typing import Tuple, Union import pandas as pd from filelock import FileLock +from rdagent.app.kaggle.conf import KAGGLE_IMPLEMENT_SETTING from rdagent.components.coder.factor_coder.config import FACTOR_IMPLEMENT_SETTINGS from rdagent.core.exception import CodeFormatError, CustomRuntimeError, NoOutputError from rdagent.core.experiment import Experiment, FBWorkspace, Task @@ -148,9 +149,7 @@ class FactorFBWorkspace(FBWorkspace): ) elif self.target_task.version == 2: # TODO you can change the name of the data folder for a better understanding - source_data_path = Path( - FACTOR_IMPLEMENT_SETTINGS.data_folder, - ) + source_data_path = Path(FACTOR_IMPLEMENT_SETTINGS.data_folder) / KAGGLE_IMPLEMENT_SETTING.competition source_data_path.mkdir(exist_ok=True, parents=True) code_path = self.workspace_path / f"factor.py" @@ -237,3 +236,4 @@ class FactorFBWorkspace(FBWorkspace): FactorExperiment = Experiment +FeatureExperiment = Experiment diff --git a/rdagent/components/coder/model_coder/model.py b/rdagent/components/coder/model_coder/model.py index 41d5f00b..86f0531c 100644 --- a/rdagent/components/coder/model_coder/model.py +++ b/rdagent/components/coder/model_coder/model.py @@ -137,5 +137,4 @@ PARAM_INIT_VALUE = {param_init_value} return execution_feedback_str, execution_model_output -FeatureExperiment = Experiment ModelExperiment = Experiment diff --git a/rdagent/scenarios/data_mining/experiment/workspace.py b/rdagent/scenarios/data_mining/experiment/workspace.py index 0f9baa6b..4cfa5593 100644 --- a/rdagent/scenarios/data_mining/experiment/workspace.py +++ b/rdagent/scenarios/data_mining/experiment/workspace.py @@ -2,7 +2,7 @@ from pathlib import Path import pandas as pd -from rdagent.app.data_mining.conf import PROP_SETTING +from rdagent.app.data_mining.conf import MED_PROP_SETTING from rdagent.core.experiment import FBWorkspace from rdagent.log import rdagent_logger as logger from rdagent.utils.env import DMDockerEnv @@ -15,7 +15,7 @@ class DMFBWorkspace(FBWorkspace): def execute(self, run_env: dict = {}, *args, **kwargs) -> str: qtde = DMDockerEnv() - qtde.prepare(PROP_SETTING.username, PROP_SETTING.password) + qtde.prepare(MED_PROP_SETTING.username, MED_PROP_SETTING.password) execute_log = qtde.run( local_path=str(self.workspace_path), diff --git a/rdagent/scenarios/kaggle/developer/feedback.py b/rdagent/scenarios/kaggle/developer/feedback.py index cce49bdc..673cce67 100644 --- a/rdagent/scenarios/kaggle/developer/feedback.py +++ b/rdagent/scenarios/kaggle/developer/feedback.py @@ -26,20 +26,20 @@ def process_results(current_result, sota_result): sota_df = pd.DataFrame(sota_result) # Combine the dataframes on the Metric index - combined_df = pd.DataFrame({"Current Result": current_df, "SOTA Result": sota_df}) + combined_df = pd.concat([current_df, sota_df], axis=1) + combined_df.columns = ["current_df", "sota_df"] - # Add a new column to show which result is bigger - combined_df["Bigger Result"] = combined_df.apply( - lambda row: "Equal" - if row["Current Result"] == row["SOTA Result"] - else ("Current Result" if row["Current Result"] > row["SOTA Result"] else "SOTA Result"), + combined_df["the largest"] = combined_df.apply( + lambda row: "sota_df" + if row["sota_df"] > row["current_df"] + else ("Equal" if row["sota_df"] == row["current_df"] else "current_df"), axis=1, ) # Add a note about metric direction combined_df["Note"] = "Direction of improvement (higher/lower is better) should be judged per metric" - return combined_df.to_string() + return combined_df class KGHypothesisExperiment2Feedback(HypothesisExperiment2Feedback): diff --git a/rdagent/scenarios/kaggle/developer/runner.py b/rdagent/scenarios/kaggle/developer/runner.py index b115d839..7a56de0a 100644 --- a/rdagent/scenarios/kaggle/developer/runner.py +++ b/rdagent/scenarios/kaggle/developer/runner.py @@ -1,9 +1,10 @@ +import pickle import shutil -import uuid from pathlib import Path -import pandas as pd - +from rdagent.app.kaggle.conf import KAGGLE_IMPLEMENT_SETTING +from rdagent.components.coder.factor_coder.config import FACTOR_IMPLEMENT_SETTINGS +from rdagent.components.coder.factor_coder.factor import FactorTask from rdagent.components.runner import CachedRunner from rdagent.components.runner.conf import RUNNER_SETTINGS from rdagent.core.exception import ModelEmptyError @@ -81,6 +82,16 @@ class KGFactorRunner(KGCachedRunner[KGFactorExperiment]): result = exp.experiment_workspace.execute(run_env=env_to_use) exp.result = result + sub_task = FactorTask( + factor_name="original features", factor_description="here is the original features", factor_formulation="" + ) + + org_data_path = Path(FACTOR_IMPLEMENT_SETTINGS.data_folder) / KAGGLE_IMPLEMENT_SETTING.competition / "valid.pkl" + with open(org_data_path, "rb") as f: + org_data = pickle.load(f) + feature_shape = org_data.shape[-1] + exp.experiment_workspace.data_description.append((sub_task.get_task_information(), feature_shape)) + if RUNNER_SETTINGS.cache_result: self.dump_cache_result(exp, result) diff --git a/rdagent/scenarios/kaggle/experiment/kaggle_experiment.py b/rdagent/scenarios/kaggle/experiment/kaggle_experiment.py index ee768d95..99602006 100644 --- a/rdagent/scenarios/kaggle/experiment/kaggle_experiment.py +++ b/rdagent/scenarios/kaggle/experiment/kaggle_experiment.py @@ -1,6 +1,10 @@ from pathlib import Path -from rdagent.components.coder.factor_coder.factor import FactorFBWorkspace +from rdagent.components.coder.factor_coder.factor import ( + FactorFBWorkspace, + FactorTask, + FeatureExperiment, +) from rdagent.components.coder.model_coder.model import ( ModelExperiment, ModelFBWorkspace, @@ -15,7 +19,7 @@ class KGModelExperiment(ModelExperiment[ModelTask, KGFBWorkspace, ModelFBWorkspa self.experiment_workspace = KGFBWorkspace(template_folder_path=Path(__file__).parent / "meta_tpl") -class KGFactorExperiment(ModelExperiment[ModelTask, KGFBWorkspace, FactorFBWorkspace]): +class KGFactorExperiment(FeatureExperiment[FactorTask, KGFBWorkspace, FactorFBWorkspace]): def __init__(self, *args, **kwargs) -> None: super().__init__(*args, **kwargs) self.experiment_workspace = KGFBWorkspace(template_folder_path=Path(__file__).parent / "meta_tpl") diff --git a/rdagent/scenarios/kaggle/experiment/meta_tpl/feature/feature.py b/rdagent/scenarios/kaggle/experiment/meta_tpl/feature/feature.py index c25a7ead..7dd65426 100644 --- a/rdagent/scenarios/kaggle/experiment/meta_tpl/feature/feature.py +++ b/rdagent/scenarios/kaggle/experiment/meta_tpl/feature/feature.py @@ -10,4 +10,4 @@ def feat_eng(X: pd.DataFrame): """ return the selected features """ - return None + return X diff --git a/rdagent/scenarios/kaggle/experiment/meta_tpl/train.py b/rdagent/scenarios/kaggle/experiment/meta_tpl/train.py index 99c459b0..445cf269 100644 --- a/rdagent/scenarios/kaggle/experiment/meta_tpl/train.py +++ b/rdagent/scenarios/kaggle/experiment/meta_tpl/train.py @@ -40,8 +40,8 @@ def import_module_from_path(module_name, module_path): X_train, X_valid, y_train, y_valid, X_test, passenger_ids = preprocess_script() # 2) Auto feature engineering -X_train_l, X_valid_l = [X_train], [X_valid] -X_test_l = [X_test] +X_train_l, X_valid_l = [], [] +X_test_l = [] for f in DIRNAME.glob("feature/feat*.py"): m = import_module_from_path(f.stem, f) diff --git a/rdagent/scenarios/kaggle/experiment/scenario.py b/rdagent/scenarios/kaggle/experiment/scenario.py index 6a9fbfb7..c241ebf7 100644 --- a/rdagent/scenarios/kaggle/experiment/scenario.py +++ b/rdagent/scenarios/kaggle/experiment/scenario.py @@ -82,8 +82,7 @@ class KGScenario(Scenario): @property def source_data(self) -> str: - # TODO later we should improve this part - data_folder = Path(FACTOR_IMPLEMENT_SETTINGS.data_folder) + data_folder = Path(FACTOR_IMPLEMENT_SETTINGS.data_folder) / self.competition if (data_folder / "valid.pkl").exists(): X_valid = pd.read_pickle(data_folder / "valid.pkl") diff --git a/rdagent/scenarios/kaggle/experiment/workspace.py b/rdagent/scenarios/kaggle/experiment/workspace.py index 3531652c..42388785 100644 --- a/rdagent/scenarios/kaggle/experiment/workspace.py +++ b/rdagent/scenarios/kaggle/experiment/workspace.py @@ -4,7 +4,7 @@ from pathlib import Path import pandas as pd -from rdagent.app.kaggle.conf import PROP_SETTING +from rdagent.app.kaggle.conf import KAGGLE_IMPLEMENT_SETTING from rdagent.core.experiment import FBWorkspace from rdagent.log import rdagent_logger as logger from rdagent.utils.env import KGDockerEnv @@ -34,7 +34,7 @@ class KGFBWorkspace(FBWorkspace): def generate_preprocess_data( self, ) -> tuple[pd.DataFrame, pd.DataFrame, pd.Series, pd.Series, pd.DataFrame, pd.Series]: - kgde = KGDockerEnv(PROP_SETTING.competition) + kgde = KGDockerEnv(KAGGLE_IMPLEMENT_SETTING.competition) kgde.prepare() execute_log, results = kgde.dump_python_code_run_and_get_results( @@ -58,7 +58,7 @@ class KGFBWorkspace(FBWorkspace): def execute(self, run_env: dict = {}, *args, **kwargs) -> str: logger.info(f"Running the experiment in {self.workspace_path}") - kgde = KGDockerEnv(PROP_SETTING.competition) + kgde = KGDockerEnv(KAGGLE_IMPLEMENT_SETTING.competition) kgde.prepare() execute_log = kgde.run( @@ -69,11 +69,7 @@ class KGFBWorkspace(FBWorkspace): csv_path = self.workspace_path / "submission_score.csv" - print("WORKSPACE PATH IS HERE --------------------------------------------------------------------------------") - print(self.workspace_path) - print("CSV PATH IS HERE --------------------------------------------------------------------------------------") - print(csv_path) - print("CSV PATH IS HERE --------------------------------------------------------------------------------------") + logger.info(self.workspace_path) if not csv_path.exists(): logger.error(f"File {csv_path} does not exist.") diff --git a/rdagent/scenarios/kaggle/prompts.yaml b/rdagent/scenarios/kaggle/prompts.yaml index b755bce1..c2c616b2 100644 --- a/rdagent/scenarios/kaggle/prompts.yaml +++ b/rdagent/scenarios/kaggle/prompts.yaml @@ -12,7 +12,7 @@ hypothesis_and_feedback: |- hypothesis_output_format: |- The output should follow JSON format. The schema is as follows: { - "action": "The action that the user wants to take based on the information provided. should be one of ["Feature engineering", "Feature processing", "Model feature selection", "Model tuning"]", Only "Model tuning" For Now + "action": "The action that the user wants to take based on the information provided. should be one of ["Feature engineering", "Feature processing", "Model feature selection", "Model tuning"]" "hypothesis": "The new hypothesis generated based on the information provided.", "reason": "The reason why you generate this hypothesis. It should be comprehensive and logical. It should cover the other keys below and extend them.", "concise_reason": "Two-line summary. First line focuses on a concise justification for the change. Second line generalizes a knowledge statement.",