feat: fix some bugs and add original features' description (#259)

* init a scenario for kaggle feature engineering

* fix some bugs and add original features' description

* refine the process of data downloading

* fix a error

* revert the code

* fix a bug in feedback

* fix a ci bug

* fix a ci bug

docs/scens/model_agent_med.rst

@@ -123,6 +123,6 @@ You can try our demo by running the following command:
 
 The following environment variables can be set in the `.env` file to customize the application's behavior:
 
-.. autopydantic_settings:: rdagent.app.data_mining.conf.PropSetting
+.. autopydantic_settings:: rdagent.app.data_mining.conf.MedBasePropSetting
     :settings-show-field-summary: False
     :exclude-members: Config

rdagent/app/data_mining/conf.py

@@ -5,7 +5,7 @@ from pydantic_settings import BaseSettings
 from rdagent.components.workflow.conf import BasePropSetting
 
 
-class PropSetting(BasePropSetting):
+class MedBasePropSetting(BasePropSetting):
     class Config:
         env_prefix = "DM_"
         """Use `DM_` as prefix for environment variables"""
@@ -46,4 +46,4 @@ class PropSetting(BasePropSetting):
     """Physionet account password"""
 
 
-PROP_SETTING = PropSetting()
+MED_PROP_SETTING = MedBasePropSetting()

rdagent/app/data_mining/model.py

@@ -1,6 +1,6 @@
 import fire
 
-from rdagent.app.data_mining.conf import PROP_SETTING
+from rdagent.app.data_mining.conf import MED_PROP_SETTING
 from rdagent.components.workflow.rd_loop import RDLoop
 from rdagent.core.exception import ModelEmptyError
 
@@ -21,7 +21,7 @@ def main(path=None, step_n=None):
 
     """
     if path is None:
-        model_loop = ModelRDLoop(PROP_SETTING)
+        model_loop = ModelRDLoop(MED_PROP_SETTING)
     else:
         model_loop = ModelRDLoop.load(path)
     model_loop.run(step_n=step_n)

rdagent/app/kaggle/conf.py

@@ -5,7 +5,7 @@ from pydantic_settings import BaseSettings
 from rdagent.components.workflow.conf import BasePropSetting
 
 
-class PropSetting(BasePropSetting):
+class KaggleBasePropSetting(BasePropSetting):
     class Config:
         env_prefix = "KG_"
         """Use `KG_` as prefix for environment variables"""
@@ -43,4 +43,4 @@ class PropSetting(BasePropSetting):
     competition: str = ""
 
 
-PROP_SETTING = PropSetting()
+KAGGLE_IMPLEMENT_SETTING = KaggleBasePropSetting()

rdagent/app/kaggle/loop.py

@@ -3,7 +3,7 @@ from typing import Any
 
 import fire
 
-from rdagent.app.kaggle.conf import PROP_SETTING
+from rdagent.app.kaggle.conf import KAGGLE_IMPLEMENT_SETTING
 from rdagent.components.workflow.conf import BasePropSetting
 from rdagent.components.workflow.rd_loop import RDLoop
 from rdagent.core.developer import Developer
@@ -83,9 +83,9 @@ def main(path=None, step_n=None, competition=None):
 
     """
     if competition:
-        PROP_SETTING.competition = competition
+        KAGGLE_IMPLEMENT_SETTING.competition = competition
     if path is None:
-        model_loop = ModelRDLoop(PROP_SETTING)
+        model_loop = ModelRDLoop(KAGGLE_IMPLEMENT_SETTING)
     else:
         model_loop = ModelRDLoop.load(path)
     model_loop.run(step_n=step_n)

rdagent/app/kaggle_feature/model.py

@@ -1,65 +0,0 @@
-from collections import defaultdict
-
-import fire
-
-from rdagent.app.kaggle.conf import PROP_SETTING
-from rdagent.components.workflow.conf import BasePropSetting
-from rdagent.components.workflow.rd_loop import RDLoop
-from rdagent.core.exception import ModelEmptyError
-from rdagent.core.proposal import (
-    Hypothesis2Experiment,
-    HypothesisExperiment2Feedback,
-    HypothesisGen,
-    Trace,
-)
-from rdagent.core.utils import import_class
-from rdagent.log import rdagent_logger as logger
-
-
-class ModelRDLoop(RDLoop):
-    def __init__(self, PROP_SETTING: BasePropSetting):
-        with logger.tag("init"):
-            scen: Scenario = import_class(PROP_SETTING.scen)(PROP_SETTING.competition)
-            logger.log_object(scen, tag="scenario")
-
-            self.hypothesis_gen: HypothesisGen = import_class(PROP_SETTING.hypothesis_gen)(scen)
-            logger.log_object(self.hypothesis_gen, tag="hypothesis generator")
-
-            self.hypothesis2experiment: Hypothesis2Experiment = import_class(PROP_SETTING.hypothesis2experiment)()
-            logger.log_object(self.hypothesis2experiment, tag="hypothesis2experiment")
-
-            self.coder: Developer = import_class(PROP_SETTING.coder)(scen)
-            logger.log_object(self.coder, tag="coder")
-            self.runner: Developer = import_class(PROP_SETTING.runner)(scen)
-            logger.log_object(self.runner, tag="runner")
-
-            self.summarizer: HypothesisExperiment2Feedback = import_class(PROP_SETTING.summarizer)(scen)
-            logger.log_object(self.summarizer, tag="summarizer")
-            self.trace = Trace(scen=scen)
-            super(RDLoop, self).__init__()
-
-    skip_loop_error = (ModelEmptyError,)
-
-
-def main(path=None, step_n=None, competition=None):
-    """
-    Auto R&D Evolving loop for models in a kaggle{} scenario.
-
-    You can continue running session by
-
-    .. code-block:: python
-
-        dotenv run -- python rdagent/app/kaggle/model.py [--competition titanic] $LOG_PATH/__session__/1/0_propose  --step_n 1   # `step_n` is a optional paramter
-
-    """
-    if competition:
-        PROP_SETTING.competition = competition
-    if path is None:
-        model_loop = ModelRDLoop(PROP_SETTING)
-    else:
-        model_loop = ModelRDLoop.load(path)
-    model_loop.run(step_n=step_n)
-
-
-if __name__ == "__main__":
-    fire.Fire(main)

rdagent/components/coder/factor_coder/factor.py

@@ -9,6 +9,7 @@ from typing import Tuple, Union
 import pandas as pd
 from filelock import FileLock
 
+from rdagent.app.kaggle.conf import KAGGLE_IMPLEMENT_SETTING
 from rdagent.components.coder.factor_coder.config import FACTOR_IMPLEMENT_SETTINGS
 from rdagent.core.exception import CodeFormatError, CustomRuntimeError, NoOutputError
 from rdagent.core.experiment import Experiment, FBWorkspace, Task
@@ -148,9 +149,7 @@ class FactorFBWorkspace(FBWorkspace):
                 )
             elif self.target_task.version == 2:
                 # TODO you can change the name of the data folder for a better understanding
-                source_data_path = Path(
-                    FACTOR_IMPLEMENT_SETTINGS.data_folder,
-                )
+                source_data_path = Path(FACTOR_IMPLEMENT_SETTINGS.data_folder) / KAGGLE_IMPLEMENT_SETTING.competition
 
             source_data_path.mkdir(exist_ok=True, parents=True)
             code_path = self.workspace_path / f"factor.py"
@@ -237,3 +236,4 @@ class FactorFBWorkspace(FBWorkspace):
 
 
 FactorExperiment = Experiment
+FeatureExperiment = Experiment

rdagent/components/coder/model_coder/model.py

@@ -137,5 +137,4 @@ PARAM_INIT_VALUE = {param_init_value}
         return execution_feedback_str, execution_model_output
 
 
-FeatureExperiment = Experiment
 ModelExperiment = Experiment

rdagent/scenarios/data_mining/experiment/workspace.py

@@ -2,7 +2,7 @@ from pathlib import Path
 
 import pandas as pd
 
-from rdagent.app.data_mining.conf import PROP_SETTING
+from rdagent.app.data_mining.conf import MED_PROP_SETTING
 from rdagent.core.experiment import FBWorkspace
 from rdagent.log import rdagent_logger as logger
 from rdagent.utils.env import DMDockerEnv
@@ -15,7 +15,7 @@ class DMFBWorkspace(FBWorkspace):
 
     def execute(self, run_env: dict = {}, *args, **kwargs) -> str:
         qtde = DMDockerEnv()
-        qtde.prepare(PROP_SETTING.username, PROP_SETTING.password)
+        qtde.prepare(MED_PROP_SETTING.username, MED_PROP_SETTING.password)
 
         execute_log = qtde.run(
             local_path=str(self.workspace_path),

rdagent/scenarios/kaggle/developer/feedback.py

@@ -26,20 +26,20 @@ def process_results(current_result, sota_result):
     sota_df = pd.DataFrame(sota_result)
 
     # Combine the dataframes on the Metric index
-    combined_df = pd.DataFrame({"Current Result": current_df, "SOTA Result": sota_df})
+    combined_df = pd.concat([current_df, sota_df], axis=1)
+    combined_df.columns = ["current_df", "sota_df"]
 
-    # Add a new column to show which result is bigger
-    combined_df["Bigger Result"] = combined_df.apply(
-        lambda row: "Equal"
-        if row["Current Result"] == row["SOTA Result"]
-        else ("Current Result" if row["Current Result"] > row["SOTA Result"] else "SOTA Result"),
+    combined_df["the largest"] = combined_df.apply(
+        lambda row: "sota_df"
+        if row["sota_df"] > row["current_df"]
+        else ("Equal" if row["sota_df"] == row["current_df"] else "current_df"),
         axis=1,
     )
 
     # Add a note about metric direction
     combined_df["Note"] = "Direction of improvement (higher/lower is better) should be judged per metric"
 
-    return combined_df.to_string()
+    return combined_df
 
 
 class KGHypothesisExperiment2Feedback(HypothesisExperiment2Feedback):

rdagent/scenarios/kaggle/developer/runner.py

@@ -1,9 +1,10 @@
+import pickle
 import shutil
-import uuid
 from pathlib import Path
 
-import pandas as pd
-
+from rdagent.app.kaggle.conf import KAGGLE_IMPLEMENT_SETTING
+from rdagent.components.coder.factor_coder.config import FACTOR_IMPLEMENT_SETTINGS
+from rdagent.components.coder.factor_coder.factor import FactorTask
 from rdagent.components.runner import CachedRunner
 from rdagent.components.runner.conf import RUNNER_SETTINGS
 from rdagent.core.exception import ModelEmptyError
@@ -81,6 +82,16 @@ class KGFactorRunner(KGCachedRunner[KGFactorExperiment]):
         result = exp.experiment_workspace.execute(run_env=env_to_use)
 
         exp.result = result
+        sub_task = FactorTask(
+            factor_name="original features", factor_description="here is the original features", factor_formulation=""
+        )
+
+        org_data_path = Path(FACTOR_IMPLEMENT_SETTINGS.data_folder) / KAGGLE_IMPLEMENT_SETTING.competition / "valid.pkl"
+        with open(org_data_path, "rb") as f:
+            org_data = pickle.load(f)
+        feature_shape = org_data.shape[-1]
+        exp.experiment_workspace.data_description.append((sub_task.get_task_information(), feature_shape))
+
         if RUNNER_SETTINGS.cache_result:
             self.dump_cache_result(exp, result)
 

rdagent/scenarios/kaggle/experiment/kaggle_experiment.py

@@ -1,6 +1,10 @@
 from pathlib import Path
 
-from rdagent.components.coder.factor_coder.factor import FactorFBWorkspace
+from rdagent.components.coder.factor_coder.factor import (
+    FactorFBWorkspace,
+    FactorTask,
+    FeatureExperiment,
+)
 from rdagent.components.coder.model_coder.model import (
     ModelExperiment,
     ModelFBWorkspace,
@@ -15,7 +19,7 @@ class KGModelExperiment(ModelExperiment[ModelTask, KGFBWorkspace, ModelFBWorkspa
         self.experiment_workspace = KGFBWorkspace(template_folder_path=Path(__file__).parent / "meta_tpl")
 
 
-class KGFactorExperiment(ModelExperiment[ModelTask, KGFBWorkspace, FactorFBWorkspace]):
+class KGFactorExperiment(FeatureExperiment[FactorTask, KGFBWorkspace, FactorFBWorkspace]):
     def __init__(self, *args, **kwargs) -> None:
         super().__init__(*args, **kwargs)
         self.experiment_workspace = KGFBWorkspace(template_folder_path=Path(__file__).parent / "meta_tpl")

rdagent/scenarios/kaggle/experiment/meta_tpl/feature/feature.py

@@ -10,4 +10,4 @@ def feat_eng(X: pd.DataFrame):
     """
     return the selected features
     """
-    return None
+    return X

rdagent/scenarios/kaggle/experiment/meta_tpl/train.py

@@ -40,8 +40,8 @@ def import_module_from_path(module_name, module_path):
 X_train, X_valid, y_train, y_valid, X_test, passenger_ids = preprocess_script()
 
 # 2) Auto feature engineering
-X_train_l, X_valid_l = [X_train], [X_valid]
-X_test_l = [X_test]
+X_train_l, X_valid_l = [], []
+X_test_l = []
 
 for f in DIRNAME.glob("feature/feat*.py"):
     m = import_module_from_path(f.stem, f)

rdagent/scenarios/kaggle/experiment/scenario.py

@@ -82,8 +82,7 @@ class KGScenario(Scenario):
 
     @property
     def source_data(self) -> str:
-        # TODO later we should improve this part
-        data_folder = Path(FACTOR_IMPLEMENT_SETTINGS.data_folder)
+        data_folder = Path(FACTOR_IMPLEMENT_SETTINGS.data_folder) / self.competition
 
         if (data_folder / "valid.pkl").exists():
             X_valid = pd.read_pickle(data_folder / "valid.pkl")

rdagent/scenarios/kaggle/experiment/workspace.py

@@ -4,7 +4,7 @@ from pathlib import Path
 
 import pandas as pd
 
-from rdagent.app.kaggle.conf import PROP_SETTING
+from rdagent.app.kaggle.conf import KAGGLE_IMPLEMENT_SETTING
 from rdagent.core.experiment import FBWorkspace
 from rdagent.log import rdagent_logger as logger
 from rdagent.utils.env import KGDockerEnv
@@ -34,7 +34,7 @@ class KGFBWorkspace(FBWorkspace):
     def generate_preprocess_data(
         self,
     ) -> tuple[pd.DataFrame, pd.DataFrame, pd.Series, pd.Series, pd.DataFrame, pd.Series]:
-        kgde = KGDockerEnv(PROP_SETTING.competition)
+        kgde = KGDockerEnv(KAGGLE_IMPLEMENT_SETTING.competition)
         kgde.prepare()
 
         execute_log, results = kgde.dump_python_code_run_and_get_results(
@@ -58,7 +58,7 @@ class KGFBWorkspace(FBWorkspace):
 
     def execute(self, run_env: dict = {}, *args, **kwargs) -> str:
         logger.info(f"Running the experiment in {self.workspace_path}")
-        kgde = KGDockerEnv(PROP_SETTING.competition)
+        kgde = KGDockerEnv(KAGGLE_IMPLEMENT_SETTING.competition)
         kgde.prepare()
 
         execute_log = kgde.run(
@@ -69,11 +69,7 @@ class KGFBWorkspace(FBWorkspace):
 
         csv_path = self.workspace_path / "submission_score.csv"
 
-        print("WORKSPACE PATH IS HERE --------------------------------------------------------------------------------")
-        print(self.workspace_path)
-        print("CSV PATH IS HERE --------------------------------------------------------------------------------------")
-        print(csv_path)
-        print("CSV PATH IS HERE --------------------------------------------------------------------------------------")
+        logger.info(self.workspace_path)
 
         if not csv_path.exists():
             logger.error(f"File {csv_path} does not exist.")

rdagent/scenarios/kaggle/prompts.yaml

@@ -12,7 +12,7 @@ hypothesis_and_feedback: |-
 hypothesis_output_format: |-
   The output should follow JSON format. The schema is as follows:
   {
-    "action": "The action that the user wants to take based on the information provided. should be one of ["Feature engineering", "Feature processing", "Model feature selection", "Model tuning"]", Only "Model tuning" For Now
+    "action": "The action that the user wants to take based on the information provided. should be one of ["Feature engineering", "Feature processing", "Model feature selection", "Model tuning"]"
     "hypothesis": "The new hypothesis generated based on the information provided.",
     "reason": "The reason why you generate this hypothesis. It should be comprehensive and logical. It should cover the other keys below and extend them.",
     "concise_reason": "Two-line summary. First line focuses on a concise justification for the change. Second line generalizes a knowledge statement.",