feat: Kaggle loop update (Feature & Model) (#241)

* Init todo

* Evaluation & dataset

* Generate new data

* dataset generation

* add the result

* Analysis

* Factor update

* Updates

* Reformat analysis.py

* CI fix

* Revised Preprocessing & Supported Random Forest

* Revised to support three models with feature

* Further revised prompts

* Slight Revision

* docs: update contributors (#230)

* Revised to support three models with feature

* Further revised prompts

* Slight Revision

* feat: kaggle model and feature (#238)

* update first version code

* make hypothesis_gen and experiment_builder fit for both feature and model

* feat: continue kaggle feature and model coder (#239)

* use qlib docker to run qlib models

* feature coder ready

* model coder ready

* fix CI

* finish the first round of runner (#240)

* Optimized the factor scenario and added the front-end.

* fix a small bug

* fix a typo

* update the kaggle scenario

* delete model_template folder

* use experiment to run data preprocess script

* add source data to scenarios

* minor fix

* minor bug fix

* train.py debug

* fixed a bug in train.py and added some TODOs

* For Debugging

* fix two small bugs in based_exp

* fix some bugs

* update preprocess

* fix a bug in preprocess

* fix a bug in train.py

* reformat

* Follow-up

* fix a bug in train.py

* fix a bug in workspace

* fix a bug in feature duplication

* fix a bug in feedback

* fix a bug in preprocessed data

* fix a bug om feature engineering

* fix a ci error

* Debugged & Connected

* Fixed error on feedback & added other fixes

* fix CI errors

* fix a CI bug

* fix: fix_dotenv_error (#257)

* fix_dotenv_error

* format with isort

* Update rdagent/app/cli.py

---------

Co-authored-by: you-n-g <you-n-g@users.noreply.github.com>

* chore(main): release 0.2.1 (#249)

Release-As: 0.2.1

* init a scenario for kaggle feature engineering

* delete error codes

* Delete rdagent/app/kaggle_feature/conf.py

---------

Co-authored-by: Young <afe.young@gmail.com>
Co-authored-by: Taozhi Wang <taozhi.mark.wang@gmail.com>
Co-authored-by: you-n-g <you-n-g@users.noreply.github.com>
Co-authored-by: cyncyw <47289405+taozhiwang@users.noreply.github.com>
Co-authored-by: Xisen-Wang <xisen_application@163.com>
Co-authored-by: Haotian Chen <113661982+Hytn@users.noreply.github.com>
Co-authored-by: WinstonLiye <1957922024@qq.com>
Co-authored-by: WinstonLiyt <104308117+WinstonLiyt@users.noreply.github.com>
Co-authored-by: Linlang <30293408+SunsetWolf@users.noreply.github.com>

rdagent/app/kaggle/conf.py

@@ -13,29 +13,33 @@ class PropSetting(BasePropSetting):
         """Add 'model_' to the protected namespaces"""
 
     # 1) overriding the default
-    scen: str = "rdagent.scenarios.kaggle.experiment.model_experiment.KGModelScenario"
+    scen: str = "rdagent.scenarios.kaggle.experiment.scenario.KGScenario"
     """Scenario class for data mining model"""
 
-    hypothesis_gen: str = "rdagent.scenarios.kaggle.proposal.model_proposal.KGModelHypothesisGen"
+    hypothesis_gen: str = "rdagent.scenarios.kaggle.proposal.proposal.KGHypothesisGen"
     """Hypothesis generation class"""
 
-    hypothesis2experiment: str = "rdagent.scenarios.kaggle.proposal.model_proposal.KGModelHypothesis2Experiment"
+    hypothesis2experiment: str = "rdagent.scenarios.kaggle.proposal.proposal.KGHypothesis2Experiment"
     """Hypothesis to experiment class"""
 
-    coder: str = "rdagent.scenarios.kaggle.developer.model_coder.KGModelCoSTEER"
-    """Coder class"""
+    feature_coder: str = "rdagent.scenarios.kaggle.developer.coder.KGFactorCoSTEER"
+    """Feature Coder class"""
 
-    runner: str = "rdagent.scenarios.kaggle.developer.model_runner.KGModelRunner"
-    """Runner class"""
+    model_coder: str = "rdagent.scenarios.kaggle.developer.coder.KGModelCoSTEER"
+    """Model Coder class"""
 
-    summarizer: str = "rdagent.scenarios.kaggle.developer.feedback.KGModelHypothesisExperiment2Feedback"
+    feature_runner: str = "rdagent.scenarios.kaggle.developer.runner.KGFactorRunner"
+    """Feature Runner class"""
+
+    model_runner: str = "rdagent.scenarios.kaggle.developer.runner.KGModelRunner"
+    """Model Runner class"""
+
+    summarizer: str = "rdagent.scenarios.kaggle.developer.feedback.KGHypothesisExperiment2Feedback"
     """Summarizer class"""
 
     evolving_n: int = 10
     """Number of evolutions"""
 
-    evolving_n: int = 10
-
     competition: str = ""
 
 

rdagent/app/kaggle/loop.py

@@ -0,0 +1,98 @@
+from collections import defaultdict
+from typing import Any
+
+import fire
+
+from rdagent.app.kaggle.conf import PROP_SETTING
+from rdagent.components.workflow.conf import BasePropSetting
+from rdagent.components.workflow.rd_loop import RDLoop
+from rdagent.core.developer import Developer
+from rdagent.core.exception import ModelEmptyError
+from rdagent.core.proposal import (
+    Hypothesis2Experiment,
+    HypothesisExperiment2Feedback,
+    HypothesisGen,
+    Trace,
+)
+from rdagent.core.scenario import Scenario
+from rdagent.core.utils import import_class
+from rdagent.log import rdagent_logger as logger
+from rdagent.scenarios.kaggle.proposal.proposal import (
+    KG_ACTION_FEATURE_ENGINEERING,
+    KG_ACTION_FEATURE_PROCESSING,
+)
+
+
+class ModelRDLoop(RDLoop):
+    def __init__(self, PROP_SETTING: BasePropSetting):
+        with logger.tag("init"):
+            scen: Scenario = import_class(PROP_SETTING.scen)(PROP_SETTING.competition)
+            logger.log_object(scen, tag="scenario")
+
+            self.hypothesis_gen: HypothesisGen = import_class(PROP_SETTING.hypothesis_gen)(scen)
+            logger.log_object(self.hypothesis_gen, tag="hypothesis generator")
+
+            self.hypothesis2experiment: Hypothesis2Experiment = import_class(PROP_SETTING.hypothesis2experiment)()
+            logger.log_object(self.hypothesis2experiment, tag="hypothesis2experiment")
+
+            self.feature_coder: Developer = import_class(PROP_SETTING.feature_coder)(scen)
+            logger.log_object(self.feature_coder, tag="feature coder")
+            self.model_coder: Developer = import_class(PROP_SETTING.model_coder)(scen)
+            logger.log_object(self.model_coder, tag="model coder")
+
+            self.feature_runner: Developer = import_class(PROP_SETTING.feature_runner)(scen)
+            logger.log_object(self.feature_runner, tag="feature runner")
+            self.model_runner: Developer = import_class(PROP_SETTING.model_runner)(scen)
+            logger.log_object(self.model_runner, tag="model runner")
+
+            self.summarizer: HypothesisExperiment2Feedback = import_class(PROP_SETTING.summarizer)(scen)
+            logger.log_object(self.summarizer, tag="summarizer")
+            self.trace = Trace(scen=scen)
+            super(RDLoop, self).__init__()
+
+    def coding(self, prev_out: dict[str, Any]):
+        with logger.tag("d"):  # develop
+            if prev_out["propose"].action in [KG_ACTION_FEATURE_ENGINEERING, KG_ACTION_FEATURE_PROCESSING]:
+                exp = self.feature_coder.develop(prev_out["exp_gen"])
+            else:
+                exp = self.model_coder.develop(prev_out["exp_gen"])
+            logger.log_object(exp.sub_workspace_list, tag="coder result")
+        return exp
+
+    def running(self, prev_out: dict[str, Any]):
+        with logger.tag("ef"):  # evaluate and feedback
+            if prev_out["propose"].action in [KG_ACTION_FEATURE_ENGINEERING, KG_ACTION_FEATURE_PROCESSING]:
+                exp = self.feature_runner.develop(prev_out["coding"])
+            else:
+                exp = self.model_runner.develop(prev_out["coding"])
+            logger.log_object(exp, tag="runner result")
+        return exp
+
+    skip_loop_error = (ModelEmptyError,)
+
+
+def main(path=None, step_n=None, competition=None):
+    """
+    Auto R&D Evolving loop for models in a kaggle{} scenario.
+
+    You can continue running session by
+
+    .. code-block:: python
+
+        dotenv run -- python rdagent/app/kaggle/loop.py [--competition titanic] $LOG_PATH/__session__/1/0_propose  --step_n 1   # `step_n` is a optional paramter
+
+    """
+    if competition:
+        PROP_SETTING.competition = competition
+    if path is None:
+        model_loop = ModelRDLoop(PROP_SETTING)
+    else:
+        model_loop = ModelRDLoop.load(path)
+    model_loop.run(step_n=step_n)
+
+
+if __name__ == "__main__":
+    from dotenv import load_dotenv
+
+    load_dotenv(override=True)
+    fire.Fire(main)

rdagent/components/coder/factor_coder/CoSTEER/evaluators.py

@@ -161,7 +161,7 @@ class FactorOutputFormatEvaluator(FactorEvaluator):
             )
         buffer = io.StringIO()
         gen_df.info(buf=buffer)
-        gen_df_info_str = buffer.getvalue()
+        gen_df_info_str = f"The use is currently working on a feature related task.\nThe output dataframe info is:\n{buffer.getvalue()}"
         system_prompt = (
             Environment(undefined=StrictUndefined)
             .from_string(
@@ -378,6 +378,7 @@ class FactorValueEvaluator(FactorEvaluator):
         self,
         implementation: Workspace,
         gt_implementation: Workspace,
+        version: int = 1,  # 1 for qlib factors and 2 for kaggle factors
         **kwargs,
     ) -> Tuple:
         conclusions = []
@@ -389,18 +390,21 @@ class FactorValueEvaluator(FactorEvaluator):
         equal_value_ratio_result = 0
         high_correlation_result = False
 
-        # Check if both dataframe has only one columns
-        feedback_str, _ = FactorSingleColumnEvaluator(self.scen).evaluate(implementation, gt_implementation)
-        conclusions.append(feedback_str)
+        # Check if both dataframe has only one columns Mute this since factor task might generate more than one columns now
+        if version == 1:
+            feedback_str, _ = FactorSingleColumnEvaluator(self.scen).evaluate(implementation, gt_implementation)
+            conclusions.append(feedback_str)
 
         # Check if the index of the dataframe is ("datetime", "instrument")
         feedback_str, _ = FactorOutputFormatEvaluator(self.scen).evaluate(implementation, gt_implementation)
         conclusions.append(feedback_str)
-
-        feedback_str, daily_check_result = FactorDatetimeDailyEvaluator(self.scen).evaluate(
-            implementation, gt_implementation
-        )
-        conclusions.append(feedback_str)
+        if version == 1:
+            feedback_str, daily_check_result = FactorDatetimeDailyEvaluator(self.scen).evaluate(
+                implementation, gt_implementation
+            )
+            conclusions.append(feedback_str)
+        else:
+            daily_check_result = None
 
         # Check if both dataframe have the same rows count
         if gt_implementation is not None:
@@ -627,7 +631,9 @@ class FactorEvaluatorForCoder(FactorEvaluator):
                 (
                     factor_feedback.factor_value_feedback,
                     decision_from_value_check,
-                ) = self.value_evaluator.evaluate(implementation=implementation, gt_implementation=gt_implementation)
+                ) = self.value_evaluator.evaluate(
+                    implementation=implementation, gt_implementation=gt_implementation, version=target_task.version
+                )
 
             factor_feedback.final_decision_based_on_gt = gt_implementation is not None
 
@@ -647,7 +653,7 @@ class FactorEvaluatorForCoder(FactorEvaluator):
                     target_task=target_task,
                     implementation=implementation,
                     execution_feedback=factor_feedback.execution_feedback,
-                    value_feedback=factor_feedback.factor_value_feedback,
+                    factor_value_feedback=factor_feedback.factor_value_feedback,
                     gt_implementation=gt_implementation,
                 )
                 (

rdagent/components/coder/factor_coder/factor.py

@@ -24,9 +24,11 @@ class FactorTask(Task):
         factor_name,
         factor_description,
         factor_formulation,
+        *args,
         variables: dict = {},
         resource: str = None,
         factor_implementation: bool = False,
+        **kwargs,
     ) -> None:
         self.factor_name = factor_name
         self.factor_description = factor_description
@@ -34,6 +36,7 @@ class FactorTask(Task):
         self.variables = variables
         self.factor_resources = resource
         self.factor_implementation = factor_implementation
+        super().__init__(*args, **kwargs)
 
     def get_task_information(self):
         return f"""factor_name: {self.factor_name}
@@ -75,8 +78,8 @@ class FactorFBWorkspace(FBWorkspace):
     def __init__(
         self,
         *args,
-        executed_factor_value_dataframe=None,
-        raise_exception=False,
+        executed_factor_value_dataframe: pd.DataFrame = None,
+        raise_exception: bool = False,
         **kwargs,
     ) -> None:
         super().__init__(*args, **kwargs)
@@ -102,7 +105,10 @@ class FactorFBWorkspace(FBWorkspace):
         1. make the directory in workspace path
         2. write the code to the file in the workspace path
         3. link all the source data to the workspace path folder
-        4. execute the code
+        if call_factor_py is True:
+            4. execute the code
+        else:
+            4. generate a script from template to import the factor.py dump get the factor value to result.h5
         5. read the factor value from the output file in the workspace path folder
         returns the execution feedback as a string and the factor value as a pandas dataframe
 
@@ -130,15 +136,21 @@ class FactorFBWorkspace(FBWorkspace):
             if self.executed_factor_value_dataframe is not None:
                 return self.FB_FROM_CACHE, self.executed_factor_value_dataframe
 
-            source_data_path = (
-                Path(
-                    FACTOR_IMPLEMENT_SETTINGS.data_folder_debug,
+            if self.target_task.version == 1:
+                source_data_path = (
+                    Path(
+                        FACTOR_IMPLEMENT_SETTINGS.data_folder_debug,
+                    )
+                    if data_type == "Debug"
+                    else Path(
+                        FACTOR_IMPLEMENT_SETTINGS.data_folder,
+                    )
                 )
-                if data_type == "Debug"
-                else Path(
+            elif self.target_task.version == 2:
+                # TODO you can change the name of the data folder for a better understanding
+                source_data_path = Path(
                     FACTOR_IMPLEMENT_SETTINGS.data_folder,
                 )
-            )
 
             source_data_path.mkdir(exist_ok=True, parents=True)
             code_path = self.workspace_path / f"factor.py"
@@ -147,9 +159,16 @@ class FactorFBWorkspace(FBWorkspace):
 
             execution_feedback = self.FB_EXECUTION_SUCCEEDED
             execution_success = False
+
+            if self.target_task.version == 1:
+                execution_code_path = code_path
+            elif self.target_task.version == 2:
+                execution_code_path = self.workspace_path / f"{uuid.uuid4()}.py"
+                execution_code_path.write_text((Path(__file__).parent / "factor_execution_template.txt").read_text())
+
             try:
                 subprocess.check_output(
-                    f"{FACTOR_IMPLEMENT_SETTINGS.python_bin} {code_path}",
+                    f"{FACTOR_IMPLEMENT_SETTINGS.python_bin} {execution_code_path}",
                     shell=True,
                     cwd=self.workspace_path,
                     stderr=subprocess.STDOUT,
@@ -161,7 +180,7 @@ class FactorFBWorkspace(FBWorkspace):
 
                 execution_feedback = (
                     e.output.decode()
-                    .replace(str(code_path.parent.absolute()), r"/path/to")
+                    .replace(str(execution_code_path.parent.absolute()), r"/path/to")
                     .replace(str(site.getsitepackages()[0]), r"/path/to/site-packages")
                 )
                 if len(execution_feedback) > 2000:

rdagent/components/coder/factor_coder/factor_execution_template.txt

@@ -0,0 +1,13 @@
+import os
+
+import numpy as np
+import pandas as pd
+from factor import feat_eng
+
+if os.path.exists("valid.pkl"):
+    valid_df = pd.read_pickle("valid.pkl")
+else:
+    raise FileNotFoundError("No valid data found.")
+
+new_feat = feat_eng(valid_df)
+new_feat.to_hdf("result.h5", key="data", mode="w")

rdagent/components/coder/model_coder/CoSTEER/evaluators.py

@@ -24,7 +24,7 @@ from rdagent.oai.llm_utils import APIBackend
 evaluate_prompts = Prompts(file_path=Path(__file__).parent.parent / "prompts.yaml")
 
 
-def shape_evaluator(prediction: torch.Tensor, target_shape: Tuple = None) -> Tuple[str, bool]:
+def shape_evaluator(prediction: torch.Tensor | np.ndarray, target_shape: Tuple = None) -> Tuple[str, bool]:
     if target_shape is None or prediction is None:
         return (
             "No output generated from the model. No shape evaluation conducted.",
@@ -279,12 +279,8 @@ class ModelCoderEvaluator(Evaluator):
         else:
             gt_tensor = None
 
-        if target_task.model_type == "XGBoost":
-            shape_feedback = "Not applicable for XGBoost models"
-            shape_decision = True
-        else:
-            shape_feedback, shape_decision = shape_evaluator(gen_tensor, (batch_size, 1))
-        value_feedback, value_decision = value_evaluator(gt_tensor, gen_tensor)
+        shape_feedback, shape_decision = shape_evaluator(gen_tensor, (batch_size, 1))
+        value_feedback, value_decision = value_evaluator(gen_tensor, gt_tensor)
         code_feedback, _ = ModelCodeEvaluator(scen=self.scen).evaluate(
             target_task=target_task,
             implementation=implementation,

rdagent/components/coder/model_coder/model.py

@@ -1,20 +1,13 @@
-import json
 import pickle
 import site
 import traceback
-import uuid
 from pathlib import Path
-from typing import Any, Dict, Optional
-
-import numpy as np
-import torch
-import xgboost as xgb
+from typing import Dict, Optional
 
 from rdagent.components.coder.model_coder.conf import MODEL_IMPL_SETTINGS
-from rdagent.core.exception import CodeFormatError
 from rdagent.core.experiment import Experiment, FBWorkspace, Task
 from rdagent.oai.llm_utils import md5_hash
-from rdagent.utils import get_module_by_module_path
+from rdagent.utils.env import KGDockerEnv, QTDockerEnv
 
 
 class ModelTask(Task):
@@ -22,11 +15,13 @@ class ModelTask(Task):
         self,
         name: str,
         description: str,
-        formulation: str,
         architecture: str,
-        variables: Dict[str, str],
+        *args,
         hyperparameters: Dict[str, str],
+        formulation: str = None,
+        variables: Dict[str, str] = None,
         model_type: Optional[str] = None,
+        **kwargs,
     ) -> None:
         self.name: str = name
         self.description: str = description
@@ -35,16 +30,18 @@ class ModelTask(Task):
         self.variables: str = variables
         self.hyperparameters: str = hyperparameters
         self.model_type: str = model_type  # Tabular for tabular model, TimesSeries for time series model, Graph for graph model, XGBoost for XGBoost model
+        super().__init__(*args, **kwargs)
 
     def get_task_information(self):
-        return f"""name: {self.name}
+        task_desc = f"""name: {self.name}
 description: {self.description}
-formulation: {self.formulation}
-architecture: {self.architecture}
-variables: {self.variables}
-hyperparameters: {self.hyperparameters}
-model_type: {self.model_type}
 """
+        task_desc += f"formulation: {self.formulation}\n" if self.formulation else ""
+        task_desc += f"architecture: {self.architecture}\n"
+        task_desc += f"variables: {self.variables}\n" if self.variables else ""
+        task_desc += f"hyperparameters: {self.hyperparameters}\n"
+        task_desc += f"model_type: {self.model_type}\n"
+        return task_desc
 
     @staticmethod
     def from_dict(dict):
@@ -66,12 +63,13 @@ class ModelFBWorkspace(FBWorkspace):
         - the `model.py` that contains a variable named `model_cls` which indicates the implemented model structure
             - `model_cls` is a instance of `torch.nn.Module`;
 
+    We support two ways of interface:
+        (version 1) for qlib we'll make a script to import the model in the implementation in file `model.py` after setting the cwd into the directory
+            - from model import model_cls
+            - initialize the model by initializing it `model_cls(input_dim=INPUT_DIM)`
+            - And then verify the model.
 
-    We'll import the model in the implementation in file `model.py` after setting the cwd into the directory
-    - from model import model_cls
-    - initialize the model by initializing it `model_cls(input_dim=INPUT_DIM)`
-    - And then verify the model.
-
+        (version 2) for kaggle we'll make a script to call the fit and predict function in the implementation in file `model.py` after setting the cwd into the directory
     """
 
     def execute(
@@ -94,51 +92,33 @@ class ModelFBWorkspace(FBWorkspace):
                 Path(MODEL_IMPL_SETTINGS.cache_location).mkdir(exist_ok=True, parents=True)
                 if cache_file_path.exists():
                     return pickle.load(open(cache_file_path, "rb"))
-            mod = get_module_by_module_path(str(self.workspace_path / "model.py"))
 
-            if self.target_task.model_type != "XGBoost":
-                model_cls = mod.model_cls
+            qtde = QTDockerEnv() if self.target_task.version == 1 else KGDockerEnv()
+            qtde.prepare()
 
-            if self.target_task.model_type == "XGBoost":
-                X_simulated = np.random.rand(100, num_features)  # 100 samples, `num_features` features each
-                y_simulated = np.random.randint(0, 2, 100)  # Binary target for example
-                params = mod.get_params()
-                num_round = mod.get_num_round()
-                dtrain = xgb.DMatrix(X_simulated, label=y_simulated)
-            elif self.target_task.model_type == "Tabular":
-                input_shape = (batch_size, num_features)
-                m = model_cls(num_features=input_shape[1])
-                data = torch.full(input_shape, input_value)
-            elif self.target_task.model_type == "TimeSeries":
-                input_shape = (batch_size, num_features, num_timesteps)
-                m = model_cls(num_features=input_shape[1], num_timesteps=input_shape[2])
-                data = torch.full(input_shape, input_value)
-            elif self.target_task.model_type == "Graph":
-                node_feature = torch.randn(batch_size, num_features)
-                edge_index = torch.randint(0, batch_size, (2, num_edges))
-                m = model_cls(num_features=num_features)
-                data = (node_feature, edge_index)
-            else:
-                raise ValueError(f"Unsupported model type: {self.target_task.model_type}")
+            if self.target_task.version == 1:
+                dump_code = f"""
+MODEL_TYPE = "{self.target_task.model_type}"
+BATCH_SIZE = {batch_size}
+NUM_FEATURES = {num_features}
+NUM_TIMESTEPS = {num_timesteps}
+NUM_EDGES = {num_edges}
+INPUT_VALUE = {input_value}
+PARAM_INIT_VALUE = {param_init_value}
+{(Path(__file__).parent / 'model_execute_template_v1.txt').read_text()}
+"""
+            elif self.target_task.version == 2:
+                dump_code = (Path(__file__).parent / "model_execute_template_v2.txt").read_text()
 
-            if self.target_task.model_type == "XGBoost":
-                bst = xgb.train(params, dtrain, num_round)
-                y_pred = bst.predict(dtrain)
-                execution_model_output = y_pred
-                execution_feedback_str = "Execution successful, model trained and predictions made."
-            else:
-                # Initialize all parameters of `m` to `param_init_value`
-                for _, param in m.named_parameters():
-                    param.data.fill_(param_init_value)
-
-                # Execute the model
-                if self.target_task.model_type == "Graph":
-                    out = m(*data)
-                else:
-                    out = m(data)
-
-                execution_model_output = out.cpu().detach()
-                execution_feedback_str = f"Execution successful, output tensor shape: {execution_model_output.shape}"
+            log, results = qtde.dump_python_code_run_and_get_results(
+                code=dump_code,
+                dump_file_names=["execution_feedback_str.pkl", "execution_model_output.pkl"],
+                local_path=str(self.workspace_path),
+                env={},
+            )
+            if results is None:
+                raise RuntimeError(f"Error in running the model code: {log}")
+            [execution_feedback_str, execution_model_output] = results
 
             if MODEL_IMPL_SETTINGS.enable_execution_cache:
                 pickle.dump(
@@ -150,10 +130,6 @@ class ModelFBWorkspace(FBWorkspace):
             execution_feedback_str = f"Execution error: {e}\nTraceback: {traceback.format_exc()}"
             execution_model_output = None
 
-        code_path = self.workspace_path / f"model.py"
-        execution_feedback_str = execution_feedback_str.replace(str(code_path.parent.absolute()), r"/path/to").replace(
-            str(site.getsitepackages()[0]), r"/path/to/site-packages"
-        )
         if len(execution_feedback_str) > 2000:
             execution_feedback_str = (
                 execution_feedback_str[:1000] + "....hidden long error message...." + execution_feedback_str[-1000:]
@@ -161,4 +137,5 @@ class ModelFBWorkspace(FBWorkspace):
         return execution_feedback_str, execution_model_output
 
 
+FeatureExperiment = Experiment
 ModelExperiment = Experiment

rdagent/components/coder/model_coder/model_execute_template_v1.txt

@@ -0,0 +1,44 @@
+# MODEL_TYPE = "Tabular"
+# BATCH_SIZE = 32
+# NUM_FEATURES = 10
+# NUM_TIMESTEPS = 4
+# NUM_EDGES = 20
+# INPUT_VALUE = 1.0
+# PARAM_INIT_VALUE = 1.0
+
+import pickle
+
+import torch
+from model import model_cls
+
+if MODEL_TYPE == "Tabular":
+    input_shape = (BATCH_SIZE, NUM_FEATURES)
+    m = model_cls(num_features=input_shape[1])
+    data = torch.full(input_shape, INPUT_VALUE)
+elif MODEL_TYPE == "TimeSeries":
+    input_shape = (BATCH_SIZE, NUM_FEATURES, NUM_TIMESTEPS)
+    m = model_cls(num_features=input_shape[1], num_timesteps=input_shape[2])
+    data = torch.full(input_shape, INPUT_VALUE)
+elif MODEL_TYPE == "Graph":
+    node_feature = torch.randn(BATCH_SIZE, NUM_FEATURES)
+    edge_index = torch.randint(0, BATCH_SIZE, (2, NUM_EDGES))
+    m = model_cls(num_features=NUM_FEATURES)
+    data = (node_feature, edge_index)
+else:
+    raise ValueError(f"Unsupported model type: {MODEL_TYPE}")
+
+# Initialize all parameters of `m` to `param_init_value`
+for _, param in m.named_parameters():
+    param.data.fill_(PARAM_INIT_VALUE)
+
+# Execute the model
+if MODEL_TYPE == "Graph":
+    out = m(*data)
+else:
+    out = m(data)
+
+execution_model_output = out.cpu().detach()
+execution_feedback_str = f"Execution successful, output tensor shape: {execution_model_output.shape}"
+
+pickle.dump(execution_model_output, open("execution_model_output.pkl", "wb"))
+pickle.dump(execution_feedback_str, open("execution_feedback_str.pkl", "wb"))

rdagent/components/coder/model_coder/model_execute_template_v2.txt

@@ -0,0 +1,20 @@
+import os
+import pickle
+
+import numpy as np
+import pandas as pd
+import torch
+from model import fit, predict, select
+
+train_X = pd.DataFrame(np.random.randn(8, 30), columns=[f"{i}" for i in range(30)])
+train_y = pd.Series(np.random.randint(0, 2, 8))
+valid_X = pd.DataFrame(np.random.randn(8, 30), columns=[f"{i}" for i in range(30)])
+valid_y = pd.Series(np.random.randint(0, 2, 8))
+
+model = fit(train_X, train_y, valid_X, valid_y)
+execution_model_output = predict(model, valid_X)
+
+execution_feedback_str = f"Execution successful, output numpy ndarray shape: {execution_model_output.shape}"
+
+pickle.dump(execution_model_output, open("execution_model_output.pkl", "wb"))
+pickle.dump(execution_feedback_str, open("execution_feedback_str.pkl", "wb"))

rdagent/components/proposal/model_proposal.py

@@ -39,7 +39,7 @@ class ModelHypothesisGen(HypothesisGen):
             Environment(undefined=StrictUndefined)
             .from_string(ModelHypothesisGen.prompts["hypothesis_gen"]["system_prompt"])
             .render(
-                targets="model",
+                targets="feature engineering and model building",
                 scenario=self.scen.get_scenario_all_desc(),
                 hypothesis_output_format=context_dict["hypothesis_output_format"],
                 hypothesis_specification=context_dict["hypothesis_specification"],
@@ -49,7 +49,7 @@ class ModelHypothesisGen(HypothesisGen):
             Environment(undefined=StrictUndefined)
             .from_string(ModelHypothesisGen.prompts["hypothesis_gen"]["user_prompt"])
             .render(
-                targets="model",
+                targets="feature engineering and model building",
                 hypothesis_and_feedback=context_dict["hypothesis_and_feedback"],
                 RAG=context_dict["RAG"],
             )
@@ -82,7 +82,7 @@ class ModelHypothesis2Experiment(Hypothesis2Experiment[ModelExperiment]):
             Environment(undefined=StrictUndefined)
             .from_string(ModelHypothesis2Experiment.prompts["hypothesis2experiment"]["system_prompt"])
             .render(
-                targets="model",
+                targets="feature engineering and model building",
                 scenario=trace.scen.get_scenario_all_desc(),
                 experiment_output_format=context["experiment_output_format"],
             )
@@ -91,7 +91,7 @@ class ModelHypothesis2Experiment(Hypothesis2Experiment[ModelExperiment]):
             Environment(undefined=StrictUndefined)
             .from_string(ModelHypothesis2Experiment.prompts["hypothesis2experiment"]["user_prompt"])
             .render(
-                targets="model",
+                targets="feature engineering and model building",
                 target_hypothesis=context["target_hypothesis"],
                 hypothesis_and_feedback=context["hypothesis_and_feedback"],
                 target_list=context["target_list"],

rdagent/components/proposal/prompts.yaml

@@ -4,18 +4,22 @@ hypothesis_gen:
     The {{targets}} are used in a certain scenario, the scenario is as follows:
     {{ scenario }}
     The user has made several hypothesis on this scenario and did several evaluation on them. The user will provide this information to you. Check if a new hypothesis has already been proposed. If it is already generated and you agree with it, just use it. If you don't agree, generate a better one.
+    {% if hypothesis_specification %}
     To help you generate new hypothesis, the user has prepared some additional information for you. You should use this information to help generate new {{targets}}.
+    Here are the specifications: {{ hypothesis_specification }}
+    {% endif %}
     Please generate the output following the format and specifications below:
     {{ hypothesis_output_format }}
-    Here are the specifications: {{ hypothesis_specification }}
 
   user_prompt: |-
-    If it is not the first round, then the user has made several hypothesis on this scenario and did several evaluation on them.
+    {% if hypothesis_and_feedback|length == 0 %}    It is the first round of hypothesis generation. The user has no hypothesis on this scenario yet.
+    {% else %}It is not the first round, the user has made several hypothesis on this scenario and did several evaluation on them.
     The former hypothesis and the corresponding feedbacks are as follows (focus on the last one & the new hypothesis that it provides and reasoning to see if you agree):
     {{ hypothesis_and_feedback }}
-    To help you generate new {{targets}}, we have prepared the following information for you:
-    {{ RAG }}
-    Please generate the new hypothesis based on the information above. Also generate the relevant keys for the reasoning and the distilled knowledge that follows. For those keys, in particular for knowledge, explain in the context of the specific scenario to build up domain knowledge in the specific field rather than genearl knowledge.
+    {% endif %}
+    {% if RAG %}To help you generate new {{targets}}, we have prepared the following information for you:
+    {{ RAG }}{% endif %}
+    Please generate the new hypothesis based on the information above. Also generate the relevant keys for the reasoning and the distilled knowledge that follows. For those keys, in particular for knowledge, explain in the context of the specific scenario to build up domain knowledge in the specific field rather than general knowledge.
 
 hypothesis2experiment:
   system_prompt: |-

rdagent/components/runner/conf.py

@@ -7,7 +7,7 @@ from pydantic_settings import BaseSettings
 
 class RunnerSettings(BaseSettings):
     class Config:
-        env_prefix = "RUNNER_"  # Use MODEL_CODER_ as prefix for environment variables
+        env_prefix = "RUNNER_"  # Use RUNNER_ as prefix for environment variables
 
     cache_result: bool = True  # whether to cache the result of the docker execution
     cache_path: str = str(Path.cwd() / "runner_cache/")  # the path to store the cache

rdagent/core/experiment.py

@@ -15,6 +15,14 @@ This file contains the all the class about organizing the task in RD-Agent.
 
 
 class Task(ABC):
+    def __init__(self, version: int = 1) -> None:
+        """
+        The version of the task, default is 1
+        Because qlib tasks execution and kaggle tasks execution are different, we need to distinguish them.
+        TODO: We may align them in the future.
+        """
+        self.version = version
+
     @abstractmethod
     def get_task_information(self) -> str:
         """
@@ -113,6 +121,9 @@ class FBWorkspace(Workspace):
         self.prepare()
         for k, v in files.items():
             self.code_dict[k] = v
+            target_file_path = self.workspace_path / k
+            if not target_file_path.parent.exists():
+                target_file_path.parent.mkdir(parents=True, exist_ok=True)
             with Path.open(self.workspace_path / k, "w") as f:
                 f.write(v)
 
@@ -129,9 +140,10 @@ class FBWorkspace(Workspace):
         """
         Load the workspace from the folder
         """
-        for file_path in folder_path.iterdir():
-            if file_path.suffix in {".py", ".yaml"}:
-                self.inject_code(**{file_path.name: file_path.read_text()})
+        for file_path in folder_path.rglob("*"):
+            if file_path.suffix in (".py", ".yaml", ".md"):
+                relative_path = file_path.relative_to(folder_path)
+                self.inject_code(**{str(relative_path): file_path.read_text()})
 
     def copy(self) -> FBWorkspace:
         """

rdagent/log/ui/app.py

@@ -27,6 +27,7 @@ from rdagent.log.storage import FileStorage
 from rdagent.log.ui.qlib_report_figure import report_figure
 from rdagent.scenarios.data_mining.experiment.model_experiment import DMModelScenario
 from rdagent.scenarios.general_model.scenario import GeneralModelScenario
+from rdagent.scenarios.kaggle.experiment.scenario import KGScenario
 from rdagent.scenarios.qlib.experiment.factor_experiment import QlibFactorScenario
 from rdagent.scenarios.qlib.experiment.factor_from_report_experiment import (
     QlibFactorFromReportScenario,
@@ -53,13 +54,15 @@ else:
     main_log_path = None
 
 
-SELECTED_METRICS = [
+QLIB_SELECTED_METRICS = [
     "IC",
     "1day.excess_return_without_cost.annualized_return",
     "1day.excess_return_without_cost.information_ratio",
     "1day.excess_return_without_cost.max_drawdown",
 ]
 
+SIMILAR_SCENARIOS = (QlibModelScenario, DMModelScenario, QlibFactorScenario, QlibFactorFromReportScenario, KGScenario)
+
 if "log_path" not in state:
     if main_log_path:
         state.log_path = next(main_log_path.iterdir()).relative_to(main_log_path)
@@ -137,23 +140,25 @@ def get_msgs_until(end_func: Callable[[Message], bool] = lambda _: True):
                     if "model runner result" in tags or "factor runner result" in tags or "runner result" in tags:
                         # factor baseline exp metrics
                         if isinstance(state.scenario, QlibFactorScenario) and state.alpha158_metrics is None:
-                            sms = msg.content.based_experiments[0].result.loc[SELECTED_METRICS]
+                            sms = msg.content.based_experiments[0].result.loc[QLIB_SELECTED_METRICS]
                             sms.name = "alpha158"
                             state.alpha158_metrics = sms
 
                         # common metrics
-                        if msg.content.result is None:
+                        if msg.content.result is None and isinstance(state.scenario, DMModelScenario):
                             state.metric_series.append(pd.Series([None], index=["AUROC"], name=f"Round {state.lround}"))
                         else:
-                            if len(msg.content.result) < 4:
-                                ps = msg.content.result
-                                ps.index = ["AUROC"]
-                                ps.name = f"Round {state.lround}"
-                                state.metric_series.append(ps)
-                            else:
-                                sms = msg.content.result.loc[SELECTED_METRICS]
-                                sms.name = f"Round {state.lround}"
-                                state.metric_series.append(sms)
+                            sms = msg.content.result
+                            if isinstance(state.scenario, DMModelScenario):
+                                sms.index = ["AUROC"]
+                            elif isinstance(
+                                state.scenario, (QlibModelScenario, QlibFactorFromReportScenario, QlibFactorScenario)
+                            ):
+                                sms = sms.loc[QLIB_SELECTED_METRICS]
+                            elif isinstance(state.scenario, KGScenario):
+                                sms = sms.loc[["MCC"]]
+                            sms.name = f"Round {state.lround}"
+                            state.metric_series.append(sms)
                     elif "hypothesis generation" in tags:
                         state.hypotheses[state.lround] = msg.content
                     elif "ef" in tags and "feedback" in tags:
@@ -342,9 +347,7 @@ def metrics_window(df: pd.DataFrame, R: int, C: int, *, height: int = 300, color
 
 
 def summary_window():
-    if isinstance(
-        state.scenario, (QlibModelScenario, DMModelScenario, QlibFactorScenario, QlibFactorFromReportScenario)
-    ):
+    if isinstance(state.scenario, SIMILAR_SCENARIOS):
         st.header("Summary📊", divider="rainbow", anchor="_summary")
         if state.lround == 0:
             return
@@ -380,7 +383,6 @@ def summary_window():
                     st.table(df.iloc[0])
                 elif df.shape[0] > 1:
                     if df.shape[1] == 1:
-                        # suhan's scenario
                         fig = px.line(df, x=df.index, y=df.columns, markers=True)
                         fig.update_layout(xaxis_title="Loop Round", yaxis_title=None)
                         st.plotly_chart(fig)
@@ -465,17 +467,9 @@ def tasks_window(tasks: list[FactorTask | ModelTask]):
 
 def research_window():
     with st.container(border=True):
-        title = (
-            "Research🔍"
-            if isinstance(
-                state.scenario, (QlibModelScenario, DMModelScenario, QlibFactorScenario, QlibFactorFromReportScenario)
-            )
-            else "Research🔍 (reader)"
-        )
+        title = "Research🔍" if isinstance(state.scenario, SIMILAR_SCENARIOS) else "Research🔍 (reader)"
         st.subheader(title, divider="blue", anchor="_research")
-        if isinstance(
-            state.scenario, (QlibModelScenario, DMModelScenario, QlibFactorScenario, QlibFactorFromReportScenario)
-        ):
+        if isinstance(state.scenario, SIMILAR_SCENARIOS):
             # pdf image
             if pim := state.msgs[round]["r.extract_factors_and_implement.load_pdf_screenshot"]:
                 for i in range(min(2, len(pim))):
@@ -510,14 +504,12 @@ def research_window():
 
 
 def feedback_window():
-    if isinstance(
-        state.scenario, (QlibModelScenario, DMModelScenario, QlibFactorScenario, QlibFactorFromReportScenario)
-    ):
+    if isinstance(state.scenario, SIMILAR_SCENARIOS):
         with st.container(border=True):
             st.subheader("Feedback📝", divider="orange", anchor="_feedback")
 
             if state.lround > 0 and isinstance(
-                state.scenario, (QlibModelScenario, QlibFactorScenario, QlibFactorFromReportScenario)
+                state.scenario, (QlibModelScenario, QlibFactorScenario, QlibFactorFromReportScenario, KGScenario)
             ):
                 with st.expander("**Config⚙️**", expanded=True):
                     st.markdown(state.scenario.experiment_setting, unsafe_allow_html=True)
@@ -541,13 +533,7 @@ def feedback_window():
 
 @st.fragment
 def evolving_window():
-    title = (
-        "Development🛠️"
-        if isinstance(
-            state.scenario, (QlibModelScenario, DMModelScenario, QlibFactorScenario, QlibFactorFromReportScenario)
-        )
-        else "Development🛠️ (evolving coder)"
-    )
+    title = "Development🛠️" if isinstance(state.scenario, SIMILAR_SCENARIOS) else "Development🛠️ (evolving coder)"
     st.subheader(title, divider="green", anchor="_development")
 
     # Evolving Status
@@ -743,9 +729,7 @@ if state.scenario is not None:
     summary_window()
 
     # R&D Loops Window
-    if isinstance(
-        state.scenario, (QlibModelScenario, DMModelScenario, QlibFactorScenario, QlibFactorFromReportScenario)
-    ):
+    if isinstance(state.scenario, SIMILAR_SCENARIOS):
         st.header("R&D Loops♾️", divider="rainbow", anchor="_rdloops")
         if len(state.msgs) > 1:
             r_options = list(state.msgs.keys())

rdagent/scenarios/data_mining/docker/Dockerfile

@@ -1,4 +1,4 @@
-FROM pytorch/pytorch:latest
+FROM pytorch/pytorch:2.2.1-cuda12.1-cudnn8-runtime
 # For GPU support, please choose the proper tag from https://hub.docker.com/r/pytorch/pytorch/tags
 
 RUN apt-get clean && apt-get update && apt-get install -y \  

rdagent/scenarios/data_mining/proposal/model_proposal.py

@@ -98,7 +98,15 @@ class DMModelHypothesis2Experiment(ModelHypothesis2Experiment):
             hyperparameters = response_dict[model_name]["hyperparameters"]
             model_type = response_dict[model_name]["model_type"]
             tasks.append(
-                ModelTask(model_name, description, formulation, architecture, variables, hyperparameters, model_type)
+                ModelTask(
+                    name=model_name,
+                    description=description,
+                    formulation=formulation,
+                    architecture=architecture,
+                    variables=variables,
+                    hyperparameters=hyperparameters,
+                    model_type=model_type,
+                )
             )
         exp = DMModelExperiment(tasks)
         exp.based_experiments = [t[1] for t in trace.hist if t[2]]

rdagent/scenarios/kaggle/developer/coder.py

@@ -0,0 +1,5 @@
+from rdagent.components.coder.factor_coder.CoSTEER import FactorCoSTEER
+from rdagent.components.coder.model_coder.CoSTEER import ModelCoSTEER
+
+KGModelCoSTEER = ModelCoSTEER
+KGFactorCoSTEER = FactorCoSTEER

rdagent/scenarios/kaggle/developer/feedback.py

@@ -1,6 +1,7 @@
 import json
 from pathlib import Path
 
+import pandas as pd
 from jinja2 import Environment, StrictUndefined
 
 from rdagent.core.experiment import Experiment
@@ -19,50 +20,112 @@ feedback_prompts = Prompts(file_path=Path(__file__).parent.parent / "prompts.yam
 DIRNAME = Path(__file__).absolute().resolve().parent
 
 
-class KGModelHypothesisExperiment2Feedback(HypothesisExperiment2Feedback):
-    """Generated feedbacks on the hypothesis from **Executed** Implementations of different tasks & their comparisons with previous performances"""
+def process_results(current_result, sota_result):
+    # Convert the results to dataframes
+    current_df = pd.DataFrame(current_result)
+    sota_df = pd.DataFrame(sota_result)
 
+    # Combine the dataframes on the Metric index
+    combined_df = pd.DataFrame({"Current Result": current_df, "SOTA Result": sota_df})
+
+    # Add a new column to show which result is bigger
+    combined_df["Bigger Result"] = combined_df.apply(
+        lambda row: "Equal"
+        if row["Current Result"] == row["SOTA Result"]
+        else ("Current Result" if row["Current Result"] > row["SOTA Result"] else "SOTA Result"),
+        axis=1,
+    )
+
+    # Add a note about metric direction
+    combined_df["Note"] = "Direction of improvement (higher/lower is better) should be judged per metric"
+
+    return combined_df.to_string()
+
+
+class KGHypothesisExperiment2Feedback(HypothesisExperiment2Feedback):
     def generate_feedback(self, exp: Experiment, hypothesis: Hypothesis, trace: Trace) -> HypothesisFeedback:
         """
         The `ti` should be executed and the results should be included, as well as the comparison between previous results (done by LLM).
         For example: `mlflow` of Qlib will be included.
         """
 
+        """
+        Generate feedback for the given experiment and hypothesis.
+        Args:
+            exp: The experiment to generate feedback for.
+            hypothesis: The hypothesis to generate feedback for.
+            trace: The trace of the experiment.
+        Returns:
+            Any: The feedback generated for the given experiment and hypothesis.
+        """
         logger.info("Generating feedback...")
-        # Define the system prompt for hypothesis feedback
-        system_prompt = feedback_prompts["model_feedback_generation"]["system"]
+        hypothesis_text = hypothesis.hypothesis
+        current_result = exp.result
+        tasks_factors = []
+        if exp.sub_tasks:
+            tasks_factors = []
+            for task in exp.sub_tasks:
+                try:
+                    task_info = task.get_task_information_and_implementation_result()
+                    tasks_factors.append(task_info)
+                except AttributeError:
+                    print(f"Warning: Task {task} does not have get_task_information_and_implementation_result method")
 
-        # Define the user prompt for hypothesis feedback
-        context = trace.scen
-        SOTA_hypothesis, SOTA_experiment = trace.get_sota_hypothesis_and_experiment()
+        # Check if there are any based experiments
+        if exp.based_experiments:
+            sota_result = exp.based_experiments[-1].result
+            # Process the results to filter important metrics
+            combined_result = process_results(current_result, sota_result)
+        else:
+            # If there are no based experiments, we'll only use the current result
+            combined_result = process_results(current_result, current_result)  # Compare with itself
+            print("Warning: No previous experiments to compare against. Using current result as baseline.")
 
-        user_prompt = (
+        # Generate the system prompt
+        sys_prompt = (
             Environment(undefined=StrictUndefined)
-            .from_string(feedback_prompts["model_feedback_generation"]["user"])
+            .from_string(feedback_prompts["factor_feedback_generation"]["system"])
+            .render(scenario=self.scen.get_scenario_all_desc())
+        )
+
+        # Generate the user prompt based on the action type
+        if hypothesis.action == "Model Tuning":  # TODO Add other prompts here
+            prompt_key = "model_feedback_generation"
+        else:
+            prompt_key = "factor_feedback_generation"
+
+        # Generate the user prompt
+        usr_prompt = (
+            Environment(undefined=StrictUndefined)
+            .from_string(feedback_prompts[prompt_key]["user"])
             .render(
-                context=context,
-                last_hypothesis=SOTA_hypothesis,
-                last_task=SOTA_experiment.sub_tasks[0].get_task_information() if SOTA_hypothesis else None,
-                last_code=SOTA_experiment.sub_workspace_list[0].code_dict.get("model.py") if SOTA_hypothesis else None,
-                last_result=SOTA_experiment.result if SOTA_hypothesis else None,
-                hypothesis=hypothesis,
-                exp=exp,
+                hypothesis_text=hypothesis_text,
+                task_details=tasks_factors,
+                combined_result=combined_result,
             )
         )
 
         # Call the APIBackend to generate the response for hypothesis feedback
-        response_hypothesis = APIBackend().build_messages_and_create_chat_completion(
-            user_prompt=user_prompt,
-            system_prompt=system_prompt,
+        response = APIBackend().build_messages_and_create_chat_completion(
+            user_prompt=usr_prompt,
+            system_prompt=sys_prompt,
             json_mode=True,
         )
 
         # Parse the JSON response to extract the feedback
-        response_json_hypothesis = json.loads(response_hypothesis)
+        response_json = json.loads(response)
+
+        # Extract fields from JSON response
+        observations = response_json.get("Observations", "No observations provided")
+        hypothesis_evaluation = response_json.get("Feedback for Hypothesis", "No feedback provided")
+        new_hypothesis = response_json.get("New Hypothesis", "No new hypothesis provided")
+        reason = response_json.get("Reasoning", "No reasoning provided")
+        decision = convert2bool(response_json.get("Replace Best Result", "no"))
+
         return HypothesisFeedback(
-            observations=response_json_hypothesis.get("Observations", "No observations provided"),
-            hypothesis_evaluation=response_json_hypothesis.get("Feedback for Hypothesis", "No feedback provided"),
-            new_hypothesis=response_json_hypothesis.get("New Hypothesis", "No new hypothesis provided"),
-            reason=response_json_hypothesis.get("Reasoning", "No reasoning provided"),
-            decision=convert2bool(response_json_hypothesis.get("Decision", "false")),
+            observations=observations,
+            hypothesis_evaluation=hypothesis_evaluation,
+            new_hypothesis=new_hypothesis,
+            reason=reason,
+            decision=decision,
         )

rdagent/scenarios/kaggle/developer/model_coder.py

@@ -1,3 +0,0 @@
-from rdagent.components.coder.model_coder.CoSTEER import ModelCoSTEER
-
-KGModelCoSTEER = ModelCoSTEER

rdagent/scenarios/kaggle/developer/model_runner.py

@@ -1,38 +0,0 @@
-import shutil
-import uuid
-from pathlib import Path
-
-import pandas as pd
-
-from rdagent.components.coder.model_coder.model import ModelExperiment, ModelFBWorkspace
-from rdagent.components.runner import CachedRunner
-from rdagent.components.runner.conf import RUNNER_SETTINGS
-from rdagent.core.developer import Developer
-from rdagent.core.exception import ModelEmptyError
-from rdagent.log import rdagent_logger as logger
-from rdagent.scenarios.kaggle.experiment.model_experiment import KGModelExperiment
-from rdagent.utils.env import KGDockerEnv
-
-
-class KGModelRunner(CachedRunner[KGModelExperiment]):
-    def develop(self, exp: KGModelExperiment) -> KGModelExperiment:
-        if RUNNER_SETTINGS.cache_result:
-            cache_hit, result = self.get_cache_result(exp)
-            if cache_hit:
-                exp.result = result
-                return exp
-
-        if exp.sub_workspace_list[0].code_dict.get("model.py") is None:
-            raise ModelEmptyError("model.py is empty")
-        # to replace & inject code
-        exp.experiment_workspace.inject_code(**{"model.py": exp.sub_workspace_list[0].code_dict["model.py"]})
-
-        env_to_use = {"PYTHONPATH": "./"}
-
-        result = exp.experiment_workspace.execute(run_env=env_to_use)
-
-        exp.result = result
-        if RUNNER_SETTINGS.cache_result:
-            self.dump_cache_result(exp, result)
-
-        return exp

rdagent/scenarios/kaggle/developer/runner.py

@@ -0,0 +1,121 @@
+import shutil
+import uuid
+from pathlib import Path
+
+import pandas as pd
+
+from rdagent.components.runner import CachedRunner
+from rdagent.components.runner.conf import RUNNER_SETTINGS
+from rdagent.core.exception import ModelEmptyError
+from rdagent.core.experiment import ASpecificExp
+from rdagent.oai.llm_utils import md5_hash
+from rdagent.scenarios.kaggle.experiment.kaggle_experiment import (
+    KGFactorExperiment,
+    KGModelExperiment,
+)
+
+META_TPL_DIR = Path(__file__).parent.parent / "experiment" / "meta_tpl"
+
+
+class KGCachedRunner(CachedRunner[ASpecificExp]):
+    def build_from_SOTA(self, exp: ASpecificExp) -> None:
+        if len(exp.based_experiments) > 0:
+            exp.experiment_workspace.inject_code(**exp.based_experiments[-1].experiment_workspace.code_dict)
+            exp.experiment_workspace.data_description = exp.based_experiments[-1].experiment_workspace.data_description
+            exp.experiment_workspace.model_description = exp.based_experiments[
+                -1
+            ].experiment_workspace.model_description
+
+    def get_cache_key(self, exp: ASpecificExp) -> str:
+        codes = []
+        for f in sorted((exp.experiment_workspace.workspace_path / "feature").glob("*.py"), key=lambda x: x.name):
+            codes.append(f.read_text())
+        for f in sorted((exp.experiment_workspace.workspace_path / "model").glob("*.py"), key=lambda x: x.name):
+            codes.append(f.read_text())
+        codes = "\n".join(codes)
+        return md5_hash(codes)
+
+
+class KGModelRunner(KGCachedRunner[KGModelExperiment]):
+    def develop(self, exp: KGModelExperiment) -> KGModelExperiment:
+        self.build_from_SOTA(exp)
+        if exp.sub_workspace_list[0].target_task.model_type == "XGBoost":
+            exp.experiment_workspace.inject_code(**{"model_xgb.py": exp.sub_workspace_list[0].code_dict["model.py"]})
+        elif exp.sub_workspace_list[0].target_task.model_type == "RandomForest":
+            exp.experiment_workspace.inject_code(**{"model_rf.py": exp.sub_workspace_list[0].code_dict["model.py"]})
+        elif exp.sub_workspace_list[0].target_task.model_type == "LightGBM":
+            exp.experiment_workspace.inject_code(**{"model_lgb.py": exp.sub_workspace_list[0].code_dict["model.py"]})
+        elif exp.sub_workspace_list[0].target_task.model_type == "NN":
+            exp.experiment_workspace.inject_code(**{"model_nn.py": exp.sub_workspace_list[0].code_dict["model.py"]})
+        if RUNNER_SETTINGS.cache_result:
+            cache_hit, result = self.get_cache_result(exp)
+            if cache_hit:
+                exp.result = result
+                return exp
+
+        env_to_use = {"PYTHONPATH": "./"}
+
+        result = exp.experiment_workspace.execute(run_env=env_to_use)
+
+        exp.result = result
+        if RUNNER_SETTINGS.cache_result:
+            self.dump_cache_result(exp, result)
+
+        return exp
+
+
+class KGFactorRunner(KGCachedRunner[KGFactorExperiment]):
+    def init_develop(self, exp: KGFactorExperiment) -> KGFactorExperiment:
+        """
+        For the initial development, the experiment serves as a benchmark for feature engineering.
+        """
+        self.build_from_SOTA(exp)
+        if RUNNER_SETTINGS.cache_result:
+            cache_hit, result = self.get_cache_result(exp)
+            if cache_hit:
+                exp.result = result
+                return exp
+
+        env_to_use = {"PYTHONPATH": "./"}
+
+        result = exp.experiment_workspace.execute(run_env=env_to_use)
+
+        exp.result = result
+        if RUNNER_SETTINGS.cache_result:
+            self.dump_cache_result(exp, result)
+
+        return exp
+
+    def develop(self, exp: KGFactorExperiment) -> KGFactorExperiment:
+        if exp.based_experiments and exp.based_experiments[-1].result is None:
+            exp.based_experiments[-1] = self.init_develop(exp.based_experiments[-1])
+        self.build_from_SOTA(exp)
+        current_feature_file_count = len(list(exp.experiment_workspace.workspace_path.glob("feature/feature*.py")))
+        implemented_factor_count = 0
+        for sub_ws in exp.sub_workspace_list:
+            if sub_ws.code_dict == {}:
+                continue
+            implemented_factor_count += 1
+            target_feature_file_name = f"feature/feature_{current_feature_file_count:05d}.py"
+            exp.experiment_workspace.inject_code(**{target_feature_file_name: sub_ws.code_dict["factor.py"]})
+            feature_shape = sub_ws.execute()[1].shape[-1]
+            exp.experiment_workspace.data_description.append((sub_ws.target_task.get_task_information(), feature_shape))
+            current_feature_file_count += 1
+        if implemented_factor_count == 0:
+            raise ModelEmptyError("No factor is implemented")
+
+        if RUNNER_SETTINGS.cache_result:
+            cache_hit, result = self.get_cache_result(exp)
+            if cache_hit:
+                exp.result = result
+                return exp
+
+        env_to_use = {"PYTHONPATH": "./"}
+
+        result = exp.experiment_workspace.execute(run_env=env_to_use)
+
+        exp.result = result
+        if RUNNER_SETTINGS.cache_result:
+            self.dump_cache_result(exp, result)
+
+        return exp

rdagent/scenarios/kaggle/docker/Dockerfile

@@ -1,4 +1,4 @@
-FROM pytorch/pytorch:latest
+FROM pytorch/pytorch:2.2.1-cuda12.1-cudnn8-runtime
 # For GPU support, please choose the proper tag from https://hub.docker.com/r/pytorch/pytorch/tags
 
 RUN apt-get clean && apt-get update && apt-get install -y \  
@@ -11,7 +11,7 @@ RUN apt-get clean && apt-get update && apt-get install -y \
 WORKDIR /workspace
 
 RUN python -m pip install numpy
-RUN python -m pip install --upgrade cython
+# RUN python -m pip install --upgrade cython
 # RUN python -m pip install -e .
 
 RUN python -m pip install pandas
@@ -23,3 +23,4 @@ RUN pip install scikit-learn
 RUN pip install catboost
 RUN pip install xgboost
 RUN pip install sparse
+RUN pip install lightgbm 

rdagent/scenarios/kaggle/experiment/kaggle_experiment.py

@@ -0,0 +1,21 @@
+from pathlib import Path
+
+from rdagent.components.coder.factor_coder.factor import FactorFBWorkspace
+from rdagent.components.coder.model_coder.model import (
+    ModelExperiment,
+    ModelFBWorkspace,
+    ModelTask,
+)
+from rdagent.scenarios.kaggle.experiment.workspace import KGFBWorkspace
+
+
+class KGModelExperiment(ModelExperiment[ModelTask, KGFBWorkspace, ModelFBWorkspace]):
+    def __init__(self, *args, **kwargs) -> None:
+        super().__init__(*args, **kwargs)
+        self.experiment_workspace = KGFBWorkspace(template_folder_path=Path(__file__).parent / "meta_tpl")
+
+
+class KGFactorExperiment(ModelExperiment[ModelTask, KGFBWorkspace, FactorFBWorkspace]):
+    def __init__(self, *args, **kwargs) -> None:
+        super().__init__(*args, **kwargs)
+        self.experiment_workspace = KGFBWorkspace(template_folder_path=Path(__file__).parent / "meta_tpl")

rdagent/scenarios/kaggle/experiment/meta_tpl/fea_share_preprocess.py

@@ -1,17 +1,39 @@
 import pandas as pd
 from sklearn.compose import ColumnTransformer
 from sklearn.impute import SimpleImputer
+from sklearn.model_selection import train_test_split
 from sklearn.pipeline import Pipeline
-from sklearn.preprocessing import OneHotEncoder
+from sklearn.preprocessing import LabelEncoder, OneHotEncoder
 
 
-def preprocess(X: pd.DataFrame):
+def prepreprocess():
     """
-    We want the X_train & X_test & X_valid to contain the same number of columns & maintain feature consistency.
+    This method loads the data, drops the unnecessary columns, and splits it into train and validation sets.
+    """
+    # Load and preprocess the data
+    data_df = pd.read_csv("/kaggle/input/train.csv")
+    data_df = data_df.head(1200)
+    data_df = data_df.drop(["id"], axis=1)
+
+    X = data_df.drop(["class"], axis=1)
+    y = data_df[["class"]]
+
+    label_encoder = LabelEncoder()
+    y = label_encoder.fit_transform(y)  # Convert class labels to numeric
+
+    # Split the data into training and validation sets
+    X_train, X_valid, y_train, y_valid = train_test_split(X, y, test_size=0.10, random_state=42)
+
+    return X_train, X_valid, y_train, y_valid
+
+
+def preprocess_fit(X_train: pd.DataFrame):
+    """
+    Fits the preprocessor on the training data and returns the fitted preprocessor.
     """
     # Identify numerical and categorical features
-    numerical_cols = [cname for cname in X.columns if X[cname].dtype in ["int64", "float64"]]
-    categorical_cols = [cname for cname in X.columns if X[cname].dtype == "object"]
+    numerical_cols = [cname for cname in X_train.columns if X_train[cname].dtype in ["int64", "float64"]]
+    categorical_cols = [cname for cname in X_train.columns if X_train[cname].dtype == "object"]
 
     # Define preprocessors for numerical and categorical features
     categorical_transformer = Pipeline(
@@ -31,17 +53,50 @@ def preprocess(X: pd.DataFrame):
         ]
     )
 
-    # Fit the preprocessor on the data and transform it
-    preprocessor.fit(X)  # TODO depend on its input shape
+    # Fit the preprocessor on the training data
+    preprocessor.fit(X_train)
+
+    return preprocessor
+
+
+def preprocess_transform(X: pd.DataFrame, preprocessor):
+    """
+    Transforms the given DataFrame using the fitted preprocessor.
+    Ensures the processed data has consistent features across train, validation, and test sets.
+    """
+    # Transform the data using the fitted preprocessor
     X_array = preprocessor.transform(X).toarray()
 
     # Get feature names for the columns in the transformed data
-    feature_names = (
-        preprocessor.named_transformers_["cat"]["onehot"].get_feature_names_out(categorical_cols).tolist()
-        + numerical_cols
-    )
+    categorical_cols = [cname for cname in X.columns if X[cname].dtype == "object"]
+    feature_names = preprocessor.named_transformers_["cat"]["onehot"].get_feature_names_out(
+        categorical_cols
+    ).tolist() + [cname for cname in X.columns if X[cname].dtype in ["int64", "float64"]]
 
     # Convert arrays back to DataFrames
     X_transformed = pd.DataFrame(X_array, columns=feature_names, index=X.index)
 
     return X_transformed
+
+
+def preprocess_script():
+    """
+    This method applies the preprocessing steps to the training, validation, and test datasets.
+    """
+    X_train, X_valid, y_train, y_valid = prepreprocess()
+
+    # Fit the preprocessor on the training data
+    preprocessor = preprocess_fit(X_train)
+
+    # Preprocess the train, validation, and test data
+    X_train = preprocess_transform(X_train, preprocessor)
+    X_valid = preprocess_transform(X_valid, preprocessor)
+
+    # Load and preprocess the test data
+    submission_df = pd.read_csv("/kaggle/input/test.csv")
+    submission_df = submission_df.head(500)
+    passenger_ids = submission_df["id"]
+    submission_df = submission_df.drop(["id"], axis=1)
+    X_test = preprocess_transform(submission_df, preprocessor)
+
+    return X_train, X_valid, y_train, y_valid, X_test, passenger_ids

rdagent/scenarios/kaggle/experiment/meta_tpl/feature/feature.py

@@ -10,4 +10,4 @@ def feat_eng(X: pd.DataFrame):
     """
     return the selected features
     """
-    return X
+    return None

rdagent/scenarios/kaggle/experiment/meta_tpl/model/model_rf.py

@@ -0,0 +1,54 @@
+"""
+Motivation of the model:
+The Random Forest model is chosen for its robustness and ability to handle large datasets with higher dimensionality.
+It reduces overfitting by averaging multiple decision trees and typically performs well out of the box, making it a good
+baseline model for many classification tasks.
+"""
+
+import pandas as pd
+from sklearn.ensemble import RandomForestClassifier
+from sklearn.metrics import accuracy_score
+
+
+def select(X: pd.DataFrame) -> pd.DataFrame:
+    """
+    Select relevant features. To be used in fit & predict function.
+    """
+    # For now, we assume all features are relevant. This can be expanded to feature selection logic.
+    return X
+
+
+def fit(X_train: pd.DataFrame, y_train: pd.Series, X_valid: pd.DataFrame, y_valid: pd.Series):
+    """
+    Define and train the Random Forest model. Merge feature selection into the pipeline.
+    """
+    # Initialize the Random Forest model
+    model = RandomForestClassifier(n_estimators=100, random_state=32)
+
+    # Select features (if any feature selection is needed)
+    X_train_selected = select(X_train)
+    X_valid_selected = select(X_valid)
+
+    # Fit the model
+    model.fit(X_train_selected, y_train)
+
+    # Validate the model
+    y_valid_pred = model.predict(X_valid_selected)
+    accuracy = accuracy_score(y_valid, y_valid_pred)
+    print(f"Validation Accuracy: {accuracy:.4f}")
+
+    return model
+
+
+def predict(model, X):
+    """
+    Keep feature selection's consistency and make predictions.
+    """
+    # Select features (if any feature selection is needed)
+    X_selected = select(X)
+
+    # Predict using the trained model
+    y_pred_prob = model.predict_proba(X_selected)[:, 1]
+
+    # Apply threshold to get boolean predictions
+    return y_pred_prob

rdagent/scenarios/kaggle/experiment/meta_tpl/model/model_xgb.py

@@ -1,6 +1,7 @@
 """
 motivation  of the model
 """
+
 import pandas as pd
 import xgboost as xgb
 
@@ -33,4 +34,4 @@ def predict(model, X):
     """
     dtest = xgb.DMatrix(X)
     y_pred_prob = model.predict(dtest)
-    return y_pred_prob > 0.5  # Apply threshold to get boolean predictions
+    return y_pred_prob

rdagent/scenarios/kaggle/experiment/meta_tpl/train.py

@@ -1,19 +1,12 @@
-import os
+import importlib.util
 import random
 from pathlib import Path
 
 import numpy as np
 import pandas as pd
-import xgboost as xgb
-from fea_share_preprocess import preprocess
-from sklearn.compose import ColumnTransformer
-from sklearn.impute import SimpleImputer
+from fea_share_preprocess import preprocess_script
 from sklearn.metrics import accuracy_score, matthews_corrcoef
-from sklearn.model_selection import train_test_split
-from sklearn.pipeline import Pipeline
-from sklearn.preprocessing import LabelEncoder, OneHotEncoder
-
-from rdagent.scenarios.kaggle.experiment.meta_tpl.fea_share_preprocess import preprocess
+from sklearn.preprocessing import LabelEncoder
 
 # Set random seed for reproducibility
 SEED = 42
@@ -35,66 +28,67 @@ def compute_metrics_for_classification(y_true, y_pred):
     return mcc
 
 
-# Load and preprocess the data
-data_df = pd.read_csv("/home/v-xisenwang/git_ignore_folder/data/playground-series-s4e8/train.csv")
-data_df = data_df.drop(["id"], axis=1)
+def import_module_from_path(module_name, module_path):
+    spec = importlib.util.spec_from_file_location(module_name, module_path)
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    return module
 
-X = data_df.drop(["class"], axis=1)
-y = data_df[["class"]]
-
-label_encoder = LabelEncoder()
-y = label_encoder.fit_transform(y)  # 将类别标签转换为数值
-X_train, X_valid, y_train, y_valid = train_test_split(X, y, test_size=0.10, random_state=SEED)
 
 # 1) Preprocess the data
-X_train = preprocess(X_train)
-X_valid = preprocess(X_valid)
-
-submission_df = pd.read_csv("/home/v-xisenwang/git_ignore_folder/data/playground-series-s4e8/test.csv")
-passenger_ids = submission_df["id"]
-submission_df = submission_df.drop(["id"], axis=1)
-X_test = preprocess(submission_df)
+# TODO 如果已经做过数据预处理了，不需要再做了
+X_train, X_valid, y_train, y_valid, X_test, passenger_ids = preprocess_script()
 
 # 2) Auto feature engineering
-X_train_l, X_valid_l = [], []
-X_test_l = []
-for f in DIRNAME.glob("feat*.py"):
-    m = __import__(f.name.strip(".py"))
-    X_train = m.feat_eng(X_train)
-    X_valid = m.feat_eng(X_valid)
-    X_test = m.feat_eng(X_test)
+X_train_l, X_valid_l = [X_train], [X_valid]
+X_test_l = [X_test]
 
-    X_train_l.append(X_train)
-    X_valid_l.append(X_valid)
-    X_test_l.append(X_test)
+for f in DIRNAME.glob("feature/feat*.py"):
+    m = import_module_from_path(f.stem, f)
+    X_train_f = m.feat_eng(X_train)
+    X_valid_f = m.feat_eng(X_valid)
+    X_test_f = m.feat_eng(X_test)
+
+    X_train_l.append(X_train_f)
+    X_valid_l.append(X_valid_f)
+    X_test_l.append(X_test_f)
 
 X_train = pd.concat(X_train_l, axis=1)
 X_valid = pd.concat(X_valid_l, axis=1)
 X_test = pd.concat(X_test_l, axis=1)
 
+print(X_train.shape, X_valid.shape, X_test.shape)
 
-def align_features(train_df, valid_df):
-    # Align the features of validation data to the training data
-    valid_df = valid_df.reindex(columns=train_df.columns, fill_value=0)
-    return valid_df
+# Handle inf and -inf values
+X_train.replace([np.inf, -np.inf], np.nan, inplace=True)
+X_valid.replace([np.inf, -np.inf], np.nan, inplace=True)
+X_test.replace([np.inf, -np.inf], np.nan, inplace=True)
 
+from sklearn.impute import SimpleImputer
 
-X_valid = align_features(X_train, X_valid)
-X_test = align_features(X_train, X_test)
+imputer = SimpleImputer(strategy="mean")
+
+X_train = pd.DataFrame(imputer.fit_transform(X_train), columns=X_train.columns)
+X_valid = pd.DataFrame(imputer.transform(X_valid), columns=X_valid.columns)
+X_test = pd.DataFrame(imputer.transform(X_test), columns=X_test.columns)
+
+# Remove duplicate columns
+X_train = X_train.loc[:, ~X_train.columns.duplicated()]
+X_valid = X_valid.loc[:, ~X_valid.columns.duplicated()]
+X_test = X_test.loc[:, ~X_test.columns.duplicated()]
 
 # 3) Train the model
 model_l = []  # list[tuple[model, predict_func,]]
-for f in DIRNAME.glob("model*.py"):
-    # TODO put select() in model.py: fit(X_train, y_train, X_valid, y_valid)
-    m = __import__(f.name.strip(".py"))
+for f in DIRNAME.glob("model/model*.py"):
+    m = import_module_from_path(f.stem, f)
     model_l.append((m.fit(X_train, y_train, X_valid, y_valid), m.predict))
 
-# Evaluate the model on the validation set
+# 4) Evaluate the model on the validation set
 y_valid_pred_l = []
 for model, predict_func in model_l:
     y_valid_pred_l.append(predict_func(model, X_valid))
 
-# Ensemble
+# 5) Ensemble
 # TODO: ensemble method in a script
 # Average the predictions and apply a threshold to determine class labels
 y_valid_pred = np.mean(y_valid_pred_l, axis=0)
@@ -103,12 +97,12 @@ y_valid_pred = (y_valid_pred > 0.5).astype(int)
 mcc = compute_metrics_for_classification(y_valid, y_valid_pred)
 print("Final on validation set: ", mcc)
 
-# Save the validation accuracy
-pd.Series(data=[mcc], index=["MCC"]).to_csv(
-    "/home/v-xisenwang/RD-Agent/rdagent/scenarios/kaggle/experiment/meta_tpl/submission_score.csv"
-)
+# 6) Save the validation accuracy
+pd.Series(data=[mcc], index=["MCC"]).to_csv("submission_score.csv")
 
-# Make predictions on the test set and save them
+# 7) Make predictions on the test set and save them
+label_encoder = LabelEncoder()
+label_encoder.fit(y_train)
 y_test_pred_bool_l = []
 for m, m_pred in model_l:
     y_test_pred_bool_l.append(
@@ -116,10 +110,11 @@ for m, m_pred in model_l:
     )  # TODO Make this an ensemble. Currently it uses the last prediction
 
 y_test_pred = np.mean(y_test_pred_bool_l, axis=0)
-y_test_pred = (y_test_pred > 0.5).astype(int)  # TODO Make it a module. Ensemble prediction
+y_test_pred = (y_test_pred > 0.5).astype(int)
 
 y_test_pred_labels = label_encoder.inverse_transform(y_test_pred)  # 将整数转换回 'e' 或 'p'
+
 submission_result = pd.DataFrame({"id": passenger_ids, "class": y_test_pred_labels})
 
-# submit predictions for the test set
-submission_result.to_csv("./submission.csv", index=False)
+# 8) Submit predictions for the test set
+submission_result.to_csv("submission.csv", index=False)

rdagent/scenarios/kaggle/experiment/model_template/README.md

@@ -1,3 +0,0 @@
-## This folder is a template to be copied from for each model implementation & running process. 
-
-Components: Dummy model.py, versatile conf.yaml, and a result reader. 

rdagent/scenarios/kaggle/experiment/model_template/train.py

@@ -1,105 +0,0 @@
-import random
-
-import numpy as np
-import pandas as pd
-import xgboost as xgb
-from model import get_num_round, get_params
-from sklearn.compose import ColumnTransformer
-from sklearn.impute import SimpleImputer
-from sklearn.metrics import accuracy_score
-from sklearn.model_selection import train_test_split
-from sklearn.pipeline import Pipeline
-from sklearn.preprocessing import OneHotEncoder
-
-# Set random seed for reproducibility
-SEED = 42
-random.seed(SEED)
-np.random.seed(SEED)
-
-
-def compute_metrics_for_classification(y_true, y_pred):
-    """Compute accuracy metric for classification."""
-    accuracy = accuracy_score(y_true, y_pred)
-    return accuracy
-
-
-def train_model(X_train, y_train, X_valid, y_valid):
-    """Define and train the model."""
-    dtrain = xgb.DMatrix(X_train, label=y_train)
-    dvalid = xgb.DMatrix(X_valid, label=y_valid)
-
-    params = get_params()
-    num_round = get_num_round()
-
-    evallist = [(dtrain, "train"), (dvalid, "eval")]
-    bst = xgb.train(params, dtrain, num_round, evallist)
-
-    return bst
-
-
-def predict(model, X):
-    dtest = xgb.DMatrix(X)
-    y_pred_prob = model.predict(dtest)
-    return y_pred_prob > 0.5  # Apply threshold to get boolean predictions
-
-
-if __name__ == "__main__":
-    # Load and preprocess the data
-    data_df = pd.read_csv("/root/.data/train.csv")
-    data_df = data_df.drop(["PassengerId", "Name"], axis=1)
-
-    X = data_df.drop(["Transported"], axis=1)
-    y = data_df.Transported.to_numpy()
-
-    # Identify numerical and categorical features
-    numerical_cols = [cname for cname in X.columns if X[cname].dtype in ["int64", "float64"]]
-    categorical_cols = [cname for cname in X.columns if X[cname].dtype == "object"]
-
-    # Define preprocessors for numerical and categorical features
-    categorical_transformer = Pipeline(
-        steps=[
-            ("imputer", SimpleImputer(strategy="most_frequent")),
-            ("onehot", OneHotEncoder(handle_unknown="ignore")),
-        ]
-    )
-
-    numerical_transformer = Pipeline(steps=[("imputer", SimpleImputer(strategy="mean"))])
-
-    # Combine preprocessing steps
-    preprocessor = ColumnTransformer(
-        transformers=[
-            ("cat", categorical_transformer, categorical_cols),
-            ("num", numerical_transformer, numerical_cols),
-        ]
-    )
-
-    # Split the data into training and validation sets
-    X_train, X_valid, y_train, y_valid = train_test_split(X, y, test_size=0.10, random_state=SEED)
-
-    # Fit the preprocessor on the training data and transform both training and validation data
-    preprocessor.fit(X_train)
-    X_train = preprocessor.transform(X_train)
-    X_valid = preprocessor.transform(X_valid)
-
-    # Train the model
-    model = train_model(X_train, y_train, X_valid, y_valid)
-
-    # Evaluate the model on the validation set
-    y_valid_pred = predict(model, X_valid)
-    accuracy = compute_metrics_for_classification(y_valid, y_valid_pred)
-    print("Final Accuracy on validation set: ", accuracy)
-
-    # Save the validation accuracy
-    pd.Series(data=[accuracy], index=["ACC"]).to_csv("./submission_score.csv")
-
-    # Load and preprocess the test set
-    submission_df = pd.read_csv("/root/.data/test.csv")
-    passenger_ids = submission_df["PassengerId"]
-    submission_df = submission_df.drop(["PassengerId", "Name"], axis=1)
-    X_test = preprocessor.transform(submission_df)
-
-    # Make predictions on the test set and save them
-    y_test_pred = predict(model, X_test)
-    submission_result = pd.DataFrame({"PassengerId": passenger_ids, "Transported": y_test_pred})
-    # submit predictions for the test set
-    submission_result.to_csv("./submission.csv", index=False)

rdagent/scenarios/kaggle/experiment/prompts.yaml

@@ -1,74 +1,297 @@
 kg_description_template:
   system: |-
     You are an assistant that extracts structured information from unstructured text.
+    The user will provide you a Kaggle competition description, and you need to extract specific details from it.
+    For the dataset, the competition may not include detailed information about the dataset. The user has read the dataset and provide you the relevant information. Please include it in your response.
+    Please answer in Json format with the following schema:
+    {
+      "Competition Type": "The type of competition, e.g., 'Classification', 'Regression', 'Clustering', 'Prediction", "Time-Series Forecasting",
+      "Competition Description": "A brief description of the competition",
+      "Target Description": "A description of the target variable to be predicted",
+      "Competition Features": "A dict of relevant features used in the competition and their descriptions (if available)", # if you are not sure about the meaning of the feature, please add a (guess) before the description. Importantly, your feature name should be exactly the same as the feature name in the dataset!
+    }
+
 
   user: |-
-    Based on the following competition description, please extract the following details:
-    1. Competition Type 
-    2. Competition Description
-    3. Target Description
-    4. Competition Features
+    Competition Description: 
+    {{ competition_descriptions }}
+    The raw data information:
+    {{ raw_data_information }}
 
-    Competition Description: {{ competition_descriptions }}
+kg_background: |-
+  You are solving a data science tasks and the type of the competition is {{ competition_type }}.
+  The competition description is:{{competition_description}}
+  
+  We provide an overall script in file: train.py. The user will run the train.py script along with several feature and model scripts to train several model to get a good performance on this task.
 
-kg_model_background: |-
-  You are solving this data science tasks of {{ competition_type }}: 
-  {{competition_description}}
+  The train.py script is as follows:
+  ```python
+  {{ train_script }}
+  ```
   
-  We provide an overall pipeline in train.py. Now fill in the provided train.py script to train a {{ competition_type }} model to get a good performance on this task.
+  The final output of our pipeline is from a ensemble of up to four models. Each model is trained on a different subset of the data.
+  The four model types are: XGBoost, RandomForest, LightGBM and Neural Network (A Pytorch model).
   
-  The model is a machine learning or deep learning structure designed to predict {{ target_description }}. 
-  The data is extracted from the competition dataset, focusing on relevant attributes like {{ competition_features }}.
-  ModelType: The type of the model, "XGBoost" for XGBoost model.
+  The data is extracted from the competition dataset, focusing on relevant attributes in {{ competition_features }}.
+
+  The user firstly designs and implements a feature book for each model. The feature book is a combination of several features and feature groups.
+  The feature book is built from:
+  - Raw features: The raw features are the original features from the dataset.
+  - generated features: The generated features are the features that are calculated based on the raw features according to some formulations. The calculation should be align with some physical or logical meaning. Don't just simply apply some numeric operations to the raw features.
+  - feature groups: The feature groups are preprocessed group of features from the raw features like normalization, one hot encoding, etc.
+  The feature or feature group is defined in the following parts:
+  - Name: The name of the feature or feature group.
+  - Description: A description of the feature or feature group.
+  - Formulation: The formulation of the feature or feature group.
+  - Variables: The variable list used in the formulation. Notice: The variable should be a specific feature in the dataset. Please make sure the feature name is exactly the same as the feature name in the dataset.
   
+  For each model, the user will design and implement the model in a separate script.
   The model is defined in the following parts:
   - Name: The name of the model.
   - Description: A description of the model.
   - Architecture: The detailed architecture of the model, such as neural network layers or tree structures.
-  
+  - ModelType: The type of the model, which should be one of ["XGBoost", "RandomForest", "LightGBM", "NN"].
   The model should provide clear and detailed documentation of its architecture and hyperparameters.
 
-kg_model_interface: |-
-  Your python code should follow the interface to better interact with the user's system.
-  You code should contain several parts:
+  The user tries to optimize the performance iteratively by employing one of the feature related or model related action items:
+  - Feature related:
+    - "Feature engineering": The user will design several new tasks and implement several new features. The new feature might only affect the model using all the feature book.
+    - "Feature processing": The user will design a new task to process the feature book like normalization or one hot encoding to improve the model performance.
+  - Model related:
+    - "Model feature selection": The user will modify one model to select the most important features from the feature book to improve the model performance.
+    - "Model tuning": The user will tune the hyperparameters of XGBoost, RandomForest or LightGBM or build or improve the NN model to improve the model performance.
+
+  For each loop, you need to help user decide which action item to choose and provide the corresponding code to implement the action item.
+
+kg_feature_interface: |-
+  Your code should contain several parts:
   1. The import part: import the necessary libraries.
-  2. A class that is a subclass of xgboost. This class should have an __init__ function and a forward function, which inputs a tensor and outputs a tensor.
-  3. Set a variable called "model_cls" to the class you defined.
+  2. A feat_eng() function that handles feature engineering for each task.
+    The function should take the following arguments:
+      - X: The features as a pandas DataFrame.
+    The function should return the new features as a pandas DataFrame.
+  The input to `feat_eng` will be a pandas DataFrame, which should be processed to return a new DataFrame containing only the engineered features. 
+  The original columns should be excluded from the returned DataFrame.
 
-  The user will save your code into a python file called "model.py". Then the user imports model_cls in file "model.py" after setting the cwd into the directory:
+  Exception handling will be managed externally, so avoid using try-except blocks in your code. The user will handle any exceptions that arise and provide feedback as needed.
+  
+  The feat_eng function can be one of the following:
+  - Feature engineering: This function calculated one new feature based on the existing raw data.
+  - Feature processing: This function processes the existing raw data like normalization or one hot encoding and return the processed data in the form of a pandas DataFrame.
+
+  Here is an example of how your Python code should be structured:
   ```python
-  from model import get_params, get_num_round
-  ```
-  So your python code should follow the pattern:
-  ```python
-  def get_params():
-    params = {
-      ...
-    }
-    return params
-  def get_num_round():
-    return xxx
+  import pandas as pd
+
+  def feat_eng(X: pd.DataFrame):
+      """
+      return the selected features
+      """
+      return X.mean(axis=1).to_frame("mean_feature") # Example feature engineering
+      return X.fillna(0) # Example feature processing
   ```
 
-  The model has one types,  "XGBoost" for XGBoost model.
-  The XGBoost Model leverages two critical hyperparameters: "arams" and "num_round".
-  "params": This hyperparameter encapsulates various settings that dictate the model's behavior and learning process.
-  "num_round": This hyperparameter specifies the number of training iterations the model will undergo.
-  User will initialize the XGBoost model with the following code:
+  To Note:
+  1. Ensure that your code meets these requirements and produces a feature-engineered DataFrame that contains only the newly engineered columns, aligning with the user's data and objectives.
+  2. Ensure that the index of the output DataFrame matches the index of the original DataFrame. For example:
+    Incorrect: `normalized_df = pd.DataFrame(normalized_features, columns=X.columns)`
+    Correct: `normalized_df = pd.DataFrame(normalized_features, columns=X.columns, index=X.index)`
+
+kg_model_interface: |-
+  Your code should contain several parts:
+  1. The import part: import the necessary libraries.
+  2. A select() function that handles feature selection for both training and prediction phases.
+    The function should take the following arguments:
+      - X: The features as a pandas DataFrame.
+    The function should return the selected features as a pandas DataFrame.
+  3. A function called fit() that trains the model and returns the trained model. If feature selection is applied, it should be done within this function.
+    The function should take the following arguments:
+      - X_train: The training features as a pandas DataFrame.
+      - y_train: The training labels as a pandas Series.
+      - X_valid: The validation features as a pandas DataFrame.
+      - y_valid: The validation labels as a pandas Series.
+    The function should return the trained model.
+  4. A function called predict() that makes predictions using the trained model. If feature selection is applied, it should be done within this function.
+    The function should take the following arguments:
+      - model: The trained model.
+      - X: The features as a pandas DataFrame.
+    The function should return the predicted probabilities or boolean predictions in numpy.ndarray format.
+    
+  Here are some examples of how your Python code should be structured:
+
+  For XGBoost:
   ```python
-  params = get_params()
-  num_round = get_num_round()
+  import pandas as pd
+  import numpy as np
+  import xgboost
+  from xgboost import DMatrix
+
+
+  def select(self, X: pd.DataFrame) -> pd.DataFrame: ...  # Implement feature selection logic
+
+
+  def fit(
+      X_train: pd.DataFrame, y_train: pd.Series, X_valid: pd.DataFrame, y_valid: pd.Series
+  ) -> xgboost.Booster:
+      X_train = select(X_train)
+      X_valid = select(X_valid)
+      dtrain = DMatrix(X_train, label=y_train)
+      dvalid = DMatrix(X_valid, label=y_valid)
+      params = ...  # Set parameters to XGBoost model
+      model = xgboost.train(params, dtrain, num_boost_round=100)
+      y_pred = model.predict(dvalid)
+
+      accuracy = ...  # Calculate accuracy
+      return model
+
+
+  def predict(model: xgboost.Booster, X: pd.DataFrame) -> np.ndarray:
+      X = select(X)
+      dtest = DMatrix(X)
+      y_pred = model.predict(dtest)
+
+      return y_pred
   ```
-  No other parameters will be passed to the model so give other parameters a default value or just make them static.
 
-  Don't write any try-except block in your python code. The user will catch the exception message and provide the feedback to you. Also, don't write main function in your python code. The user will call the forward method in the model_cls to get the output tensor.
+  For RandomForest:
+  ```python
+  import pandas as pd
+  import numpy as np
+  from sklearn.ensemble import RandomForestClassifier, RandomForestRegressor
+  from sklearn.metrics import accuracy_score
 
-  Please notice that your model should only use current features as input. The user will provide the input tensor to the model's forward function.
 
+  def select(self, X: pd.DataFrame) -> pd.DataFrame: ...  # Implement feature selection logic
+
+
+  def fit(
+      X_train: pd.DataFrame, y_train: pd.Series, X_valid: pd.DataFrame, y_valid: pd.Series
+  ) -> RandomForestClassifier | RandomForestRegressor:
+      X_train = select(X_train)
+      X_valid = select(X_valid)
+      model = RandomForestClassifier(...)  # fir classification tasks
+      model = RandomForestRegressor(...)  # for regression tasks
+      model.fit(X_train, y_train, ...) # Train the model
+
+      return model
+
+
+  def predict(model: RandomForestClassifier | RandomForestRegressor, X: pd.DataFrame) -> np.ndarray:
+      X = select(X)
+      y_pred = model.predict(X)
+
+      return y_pred
+  ```
+
+  For LightGBM:
+  ```python
+  import pandas as pd
+  import numpy as np
+  from lightgbm import LGBMClassifier, LGBMRegressor
+
+
+  def select(self, X: pd.DataFrame) -> pd.DataFrame: ...  # Implement feature selection logic
+
+
+  def fit(
+      X_train: pd.DataFrame, y_train: pd.Series, X_valid: pd.DataFrame, y_valid: pd.Series
+  ) -> LGBMClassifier | LGBMRegressor:
+      X_train = select(X_train)
+      X_valid = select(X_valid)
+      model = LGBMClassifier(...)  # for classification tasks, please add parameters here
+      model = LGBMRegressor(...)  # for regression tasks, please add parameters here
+
+      model.fit(X=X_train, y=y_train, eval_set=[(X_valid, y_valid)])
+      return model
+
+
+  def predict(model: LGBMClassifier | LGBMRegressor, X: pd.DataFrame) -> np.ndarray:
+      X = select(X)
+      y_pred = model.predict(X)
+
+      return y_pred
+  ```
+
+  For Neural Network:
+  ```python
+  import pandas as pd
+  import numpy as np
+  import torch
+  from torch.utils.data import DataLoader, TensorDataset
+
+
+  class NNModel(torch.nn.Module):
+      def __init__(self):
+          super(Model, self).__init__()
+          # Define your model here
+
+      def forward(self, x):
+          # Define the forward pass
+          return x
+
+
+  def select(self, X: pd.DataFrame) -> pd.DataFrame: ...  # Implement feature selection logic
+
+
+  def fit(X_train: pd.DataFrame, y_train: pd.DataFrame, X_valid: pd.DataFrame, y_valid: pd.DataFrame) -> torch.nn.Module:
+      X_train = select(X_train)
+      X_valid = select(X_valid)
+      model = NNModel()  # Initialize the model, You can write your own model class
+
+      optimizer = torch.optim.Adam(model.parameters(), lr=0.01)  # Example optimizer, you can use any optimizer
+      criterion = torch.nn.CrossEntropyLoss()  # Example loss function, you can use any loss function
+
+      train_loader = DataLoader(TensorDataset(X_train, y_train), batch_size=64, shuffle=True)
+      valid_loader = DataLoader(TensorDataset(X_valid, y_valid), batch_size=64, shuffle=False)
+
+      # Example training loop, you can customize this loop as per your requirement
+      for epoch in range(10):
+          model.train()
+          for X_batch, y_batch in train_loader:
+              optimizer.zero_grad()
+              outputs = model(X_batch)
+              loss = criterion(outputs, y_batch)
+              loss.backward()
+              optimizer.step()
+
+          model.eval()
+          y_pred = []
+          with torch.no_grad():
+              for X_batch, _ in valid_loader:
+                  outputs = model(X_batch)
+                  y_pred.extend(outputs.squeeze().tolist())
+
+          y_pred = torch.tensor(y_pred)
+          accuracy = (y_pred == y_valid).float().mean()
+          # You can early stop based on the validation, please customize this as per your requirement
+      return model
+
+
+  def predict(model: torch.nn.Module, X: pd.DataFrame) -> np.ndarray:
+      X = select(X)
+      X = torch.tensor(X.values).float()
+      model.eval()
+      with torch.no_grad():
+          y_pred = model(X).squeeze().numpy()
+
+      return y_pred
+  ```
+
+kg_feature_simulator: |-
+  The data preprocessing method you provide will be used to prepare data by processing it, concatenating the results with other features, and removing unnecessary features before training the model. 
+  The processed data will then be used for model training and prediction.
+  
+  User will use your data preprocessing method to do the following steps:
+  1. Execute your Python files to process the data. (what you need to do)
+  2. Concatenate the processed features with other features and the original data.
+  3. Remove any unnecessary features before training the model.
+  4. Train a model such as LightGBM, CatBoost, LSTM, or a simple PyTorch model using the processed data.
+  5. Evaluate the performance of your preprocessing method and provide feedback.
 
 kg_model_output_format: |-
-  Your output should be an array with the appropriate number of predictions, each prediction being a single value. The output should be a 2D array with dimensions corresponding to the number of predictions and 1 column (e.g., (8, 1) if there are 8 predictions).
+  For feature related tasks, the output should be a pandas DataFrame with the new features. The columns should be the new features, and the rows should correspond to the number of samples in the input DataFrame.
+  For model related tasks, the output should be an np.ndarray with the appropriate number of predictions, each prediction being a single value. The output should be a 2D array with dimensions corresponding to the number of predictions and 1 column (e.g., (8, 1) if there are 8 predictions).
   
 kg_model_simulator: |-
-  The models will be trained on the competition dataset and evaluated on their ability to predict whether passengers were transported using metrics like accuracy and AUC-ROC. 
+  The models will be trained on the competition dataset and evaluated on their ability to predict the target. Metrics like accuracy and AUC-ROC is used to evaluate the model performance. 
   Model performance will be iteratively improved based on feedback from evaluation results.

rdagent/scenarios/kaggle/experiment/scenario.py

@@ -4,40 +4,35 @@ from pathlib import Path
 import pandas as pd
 from jinja2 import Environment, StrictUndefined
 
-from rdagent.components.coder.model_coder.model import (
-    ModelExperiment,
-    ModelFBWorkspace,
-    ModelTask,
-)
+from rdagent.components.coder.factor_coder.config import FACTOR_IMPLEMENT_SETTINGS
 from rdagent.core.prompts import Prompts
 from rdagent.core.scenario import Scenario
 from rdagent.oai.llm_utils import APIBackend
-from rdagent.scenarios.kaggle.experiment.workspace import KGFBWorkspace
+from rdagent.scenarios.kaggle.experiment.kaggle_experiment import KGFactorExperiment
 from rdagent.scenarios.kaggle.kaggle_crawler import crawl_descriptions
 
 prompt_dict = Prompts(file_path=Path(__file__).parent / "prompts.yaml")
 
 
-class KGModelExperiment(ModelExperiment[ModelTask, KGFBWorkspace, ModelFBWorkspace]):
-    def __init__(self, *args, **kwargs) -> None:
-        super().__init__(*args, **kwargs)
-        self.experiment_workspace = KGFBWorkspace(template_folder_path=Path(__file__).parent / "model_template")
-
-
-class KGModelScenario(Scenario):
+class KGScenario(Scenario):
     def __init__(self, competition: str) -> None:
         super().__init__()
         self.competition = competition
         self.competition_descriptions = crawl_descriptions(competition)
+        self._source_data = self.source_data
+        self._output_format = self.output_format
+        self._interface = self.interface
+        self._simulator = self.simulator
+
         self.competition_type = None
         self.competition_description = None
         self.target_description = None
         self.competition_features = None
         self._analysis_competition_description()
 
-    def _analysis_competition_description(self):
-        # TODO: use gpt to analyze the competition description
+        self._background = self.background
 
+    def _analysis_competition_description(self):
         sys_prompt = (
             Environment(undefined=StrictUndefined)
             .from_string(prompt_dict["kg_description_template"]["system"])
@@ -49,6 +44,7 @@ class KGModelScenario(Scenario):
             .from_string(prompt_dict["kg_description_template"]["user"])
             .render(
                 competition_descriptions=self.competition_descriptions,
+                raw_data_information=self._source_data,
             )
         )
 
@@ -63,15 +59,19 @@ class KGModelScenario(Scenario):
         self.competition_description = response_json_analysis.get("Competition Description", "No description provided")
         self.target_description = response_json_analysis.get("Target Description", "No target provided")
         self.competition_features = response_json_analysis.get("Competition Features", "No features provided")
+        self.competition_features = self.source_data
 
     @property
     def background(self) -> str:
-        background_template = prompt_dict["kg_model_background"]
+        background_template = prompt_dict["kg_background"]
+
+        train_script = (Path(__file__).parent / "meta_tpl" / "train.py").read_text()
 
         background_prompt = (
             Environment(undefined=StrictUndefined)
             .from_string(background_template)
             .render(
+                train_script=train_script,
                 competition_type=self.competition_type,
                 competition_description=self.competition_description,
                 target_description=self.target_description,
@@ -82,30 +82,26 @@ class KGModelScenario(Scenario):
 
     @property
     def source_data(self) -> str:
-        kaggle_conf = KGDockerConf()
-        data_path = Path(f"{kaggle_conf.share_data_path}/{self.competition}")
+        # TODO later we should improve this part
+        data_folder = Path(FACTOR_IMPLEMENT_SETTINGS.data_folder)
 
-        csv_files = list(data_path.glob("*.csv"))
+        if (data_folder / "valid.pkl").exists():
+            X_valid = pd.read_pickle(data_folder / "valid.pkl")
+            return X_valid.head()
 
-        if not csv_files:
-            return "No CSV files found in the specified path."
+        preprocess_experiment = KGFactorExperiment([])
+        (
+            X_train,
+            X_valid,
+            y_train,
+            y_valid,
+            X_test,
+            passenger_ids,
+        ) = preprocess_experiment.experiment_workspace.generate_preprocess_data()
 
-        dataset = pd.concat([pd.read_csv(file) for file in csv_files], ignore_index=True)
-
-        simple_eda = dataset.info(buf=None)  # Capture the info output
-        data_shape = dataset.shape
-        data_head = dataset.head()
-
-        eda = (
-            f"Basic Info about the data:\n{simple_eda}\n"
-            f"Shape of the dataset: {data_shape}\n"
-            f"Sample Data:\n{data_head}\n"
-        )
-
-        data_description = self.competition_descriptions.get("Data Description", "No description provided")
-        eda += f"\nData Description:\n{data_description}"
-
-        return eda
+        data_folder.mkdir(exist_ok=True, parents=True)
+        X_valid.to_pickle(data_folder / "valid.pkl")
+        return X_valid.head()
 
     @property
     def output_format(self) -> str:
@@ -113,11 +109,19 @@ class KGModelScenario(Scenario):
 
     @property
     def interface(self) -> str:
-        return prompt_dict["kg_model_interface"]
+        return f"""The feature code should follow the interface:
+{prompt_dict['kg_feature_interface']}
+The model code should follow the interface:
+{prompt_dict['kg_model_interface']}
+"""
 
     @property
     def simulator(self) -> str:
-        return prompt_dict["kg_model_simulator"]
+        return f"""The feature code should follow the simulator:
+{prompt_dict['kg_feature_simulator']}
+The model code should follow the simulator:
+{prompt_dict['kg_model_simulator']}
+"""
 
     @property
     def rich_style_description(self) -> str:
@@ -126,11 +130,13 @@ kaggle scen """
 
     def get_scenario_all_desc(self) -> str:
         return f"""Background of the scenario:
-{self.background}
+{self._background}
+The source dataset you can use to generate the features:
+{self._source_data}
 The interface you should follow to write the runnable code:
-{self.interface}
+{self._interface}
 The output of your code should be in the format:
-{self.output_format}
+{self._output_format}
 The simulator user can use to test your model:
-{self.simulator}
+{self._simulator}
 """

rdagent/scenarios/kaggle/experiment/workspace.py

@@ -7,25 +7,73 @@ import pandas as pd
 from rdagent.app.kaggle.conf import PROP_SETTING
 from rdagent.core.experiment import FBWorkspace
 from rdagent.log import rdagent_logger as logger
-from rdagent.utils.env import DockerEnv, KGDockerEnv
+from rdagent.utils.env import KGDockerEnv
+
+KG_FEATURE_PREPROCESS_SCRIPT = """import pickle
+
+from fea_share_preprocess import preprocess_script
+
+X_train, X_valid, y_train, y_valid, X_test, passenger_ids = preprocess_script()
+
+pickle.dump(X_train, open("X_train.pkl", "wb"))
+pickle.dump(X_valid, open("X_valid.pkl", "wb"))
+pickle.dump(y_train, open("y_train.pkl", "wb"))
+pickle.dump(y_valid, open("y_valid.pkl", "wb"))
+pickle.dump(X_test, open("X_test.pkl", "wb"))
+pickle.dump(passenger_ids, open("passenger_ids.pkl", "wb"))
+"""
 
 
 class KGFBWorkspace(FBWorkspace):
     def __init__(self, template_folder_path: Path, *args, **kwargs) -> None:
         super().__init__(*args, **kwargs)
         self.inject_code_from_folder(template_folder_path)
+        self.data_description: list[str] = []
+        self.model_description: str = ""
+
+    def generate_preprocess_data(
+        self,
+    ) -> tuple[pd.DataFrame, pd.DataFrame, pd.Series, pd.Series, pd.DataFrame, pd.Series]:
+        kgde = KGDockerEnv(PROP_SETTING.competition)
+        kgde.prepare()
+
+        execute_log, results = kgde.dump_python_code_run_and_get_results(
+            code=KG_FEATURE_PREPROCESS_SCRIPT,
+            local_path=str(self.workspace_path),
+            dump_file_names=[
+                "X_train.pkl",
+                "X_valid.pkl",
+                "y_train.pkl",
+                "y_valid.pkl",
+                "X_test.pkl",
+                "passenger_ids.pkl",
+            ],
+        )
+        if results is None:
+            logger.error("Feature preprocess failed.")
+            raise Exception("Feature preprocess failed.")
+        else:
+            X_train, X_valid, y_train, y_valid, X_test, passenger_ids = results
+            return X_train, X_valid, y_train, y_valid, X_test, passenger_ids
 
     def execute(self, run_env: dict = {}, *args, **kwargs) -> str:
-        qtde = KGDockerEnv(PROP_SETTING.competition)
-        qtde.prepare()
+        logger.info(f"Running the experiment in {self.workspace_path}")
+        kgde = KGDockerEnv(PROP_SETTING.competition)
+        kgde.prepare()
 
-        execute_log = qtde.run(
+        execute_log = kgde.run(
             local_path=str(self.workspace_path),
             entry=f"python train.py",
             env=run_env,
         )
 
-        csv_path = self.workspace_path / "submission.csv"
+        csv_path = self.workspace_path / "submission_score.csv"
+
+        print("WORKSPACE PATH IS HERE --------------------------------------------------------------------------------")
+        print(self.workspace_path)
+        print("CSV PATH IS HERE --------------------------------------------------------------------------------------")
+        print(csv_path)
+        print("CSV PATH IS HERE --------------------------------------------------------------------------------------")
 
         if not csv_path.exists():
             logger.error(f"File {csv_path} does not exist.")

rdagent/scenarios/kaggle/kaggle_crawler.py

@@ -6,7 +6,8 @@ from selenium import webdriver
 from selenium.webdriver.chrome.service import Service
 from selenium.webdriver.common.by import By
 
-LOCAL_PATH = "/data/userdata/share/kaggle_competition_descriptions"
+from rdagent.log import rdagent_logger as logger
+from rdagent.utils.env import KGDockerConf
 
 options = webdriver.ChromeOptions()
 options.add_argument("--no-sandbox")
@@ -17,8 +18,8 @@ service = Service("/usr/local/bin/chromedriver")
 
 
 def crawl_descriptions(competition: str, wait: float = 3.0, force: bool = False) -> dict[str, str]:
-    if (fp := Path(f"{LOCAL_PATH}/{competition}.json")).exists() and not force:
-        print(f"Found {competition}.json, loading from local file.")
+    if (fp := Path(f"{KGDockerConf().local_data_path}/{competition}.json")).exists() and not force:
+        logger.info(f"Found {competition}.json, loading from local file.")
         with fp.open("r") as f:
             return json.load(f)
 
@@ -61,7 +62,7 @@ def crawl_descriptions(competition: str, wait: float = 3.0, force: bool = False)
     descriptions["Data Description"] = data_element.get_attribute("innerHTML")
 
     driver.quit()
-    with open(f"{LOCAL_PATH}/{competition}.json", "w") as f:
+    with open(f"{KGDockerConf().dockerfile_folder_path}/{competition}.json", "w") as f:
         json.dump(descriptions, f)
     return descriptions
 

rdagent/scenarios/kaggle/prompts.yaml

@@ -12,66 +12,76 @@ hypothesis_and_feedback: |-
 hypothesis_output_format: |-
   The output should follow JSON format. The schema is as follows:
   {
-  "hypothesis": "The new hypothesis generated based on the information provided.",
-  "reason": "The reason why you generate this hypothesis. It should be comprehensive and logical. It should cover the other keys below and extend them.",
-  "concise_reason": "Two-line summary. First line focuses on a concise justification for the change. Second line generalizes a knowledge statement.",
-  "concise_observation": "One line summary. It focuses on the observation of the given scenario, data characteristics, or previous experiences (failures & succeses).",
-  "concise_justification": "One line summary. Justify the hypothesis based on theoretical principles or initial assumptions.",
-  "concise_knowledge": "One line summary. Transferable knowledge based on theoretical principles. Use conditional grammar. eg. "If...., ..; When..., .; and etc" Make sure that you state things clearly without ambiguity. Eg. avoid saying "previous hypothesis", because one wouldn't know what that is."
+    "action": "The action that the user wants to take based on the information provided. should be one of ["Feature engineering", "Feature processing", "Model feature selection", "Model tuning"]", Only "Model tuning" For Now
+    "hypothesis": "The new hypothesis generated based on the information provided.",
+    "reason": "The reason why you generate this hypothesis. It should be comprehensive and logical. It should cover the other keys below and extend them.",
+    "concise_reason": "Two-line summary. First line focuses on a concise justification for the change. Second line generalizes a knowledge statement.",
+    "concise_observation": "One line summary. It focuses on the observation of the given scenario, data characteristics, or previous experiences (failures & succeses).",
+    "concise_justification": "One line summary. Justify the hypothesis based on theoretical principles or initial assumptions.",
+    "concise_knowledge": "One line summary. Transferable knowledge based on theoretical principles. Use conditional grammar. eg. "If...., ..; When..., .; and etc" Make sure that you state things clearly without ambiguity. Eg. avoid saying "previous hypothesis", because one wouldn't know what that is."
   }
 
-model_hypothesis_specification: |-
-  Additional Specifications:
-    
-    Hypotheses should grow and evolve based on the previous hypothesis. If there is no previous hypothesis, start with something simple. Gradually Build Up Upon previous hypothesis & feedbacks. In each round, hypothesis is different. Pay attention to your previous hypothesis.
+factor_hypothesis_specification: |-
+  1. **Type of Feature and Data Characteristics:**
+    - Define the type of feature introduced.
+    - Explain the data characteristics or patterns captured by this feature.
+    - Omit unnecessary or redundant details.
+  2. **Simple and Effective Features First:**
+    - Start with features that are simple and likely effective.
+    - Concisely explain why these features are expected to work.
+    - Avoid complex or combined features initially.
+  3. **Gradual Complexity Increase:**
+    - Introduce more complex features as more experimental results are gathered.
+    - Discuss potential advantages and complexities.
+    - Combine features only after simpler ones are tested and validated.
+  4. **New Directions and Optimizations:**
+    - If a new direction is needed, explain why based on data analysis, domain knowledge, or observed patterns.
+    - Suggest only one new direction at a time for clarity.
+    - If a previous hypothesis did not surpass the previous best, but seems optimizable, you may continue in the same direction.
+    - Highlight that features surpassing the previous best are included in the feature library to avoid re-implementation.
+  5. **1-3 Features per Generation:**
+    - Ensure each generation produces 1-3 features.
+    - Balance simplicity and complexity to build a robust feature library.
 
-    Ensure that the hypothesis focuses on the architecture of a PyTorch model. Each hypothesis should address specific architectural choices such as the type of layers, activation functions, regularization techniques, and the overall structure of the model. Avoid hypotheses related to input features or optimization processes.
-
-  Remember: if there is no hypothesis, start with something simple like MLP.
-
-  Usually, a larger model works better than a smaller one. 
-
-  Logic for generating a new hypothesis: If the previous hypothesis works, try to inherit from it and grow deeper. If the previous hypotheis doesn't work, try to make changes in the current level.
-
-  Sample hypothesis evolution loop: (This is the entire loop, see what stage you are at. We want hypothesis to continue growing.) Levels include **Model Type**, **Layer Configuration**, **Activation Functions**, **Regularization Techniques**
-
-    1st Round Hypothesis: The model should be a CNN. 
-
-    2nd Round Hypothesis (If first round worked: CNN is the model type level, which means that we should extend to the next level, like layer configuration): The model should be a CNN. The CNN should have 5 convolutional layers. (Reasoning: As CNN worked, we now specify the layers specification to grow the hypothesis deeper.)
-
-    3rd Round Hypothesis (If second round didn't work): The model should be a CNN. The CNN should have 3 convolutional layers. (Reasoning: As 5-layer structure didn't work in the 2nd round hypothesis, try something else within the layer configuration level.)
-
-    4th Round Hypothesis (If third round worked): The model should be a CNN. The CNN should have 3 convolutional layers. Use Leaky ReLU activation for all layers. (As last round worked, now proceed to the next level: activation functions)
-    
-    5th Round Hypothesis (If fourth round worked): The model should be a CNN. The CNN should have 3 convolutional layers. Use Leaky ReLU activation for all layers. Use dropout regularization with a rate of 0.5. (Similar Reasoning & Continuing to Grow to the dropout setup)
-
-    6th Round Hypothesis (If fourth round didn't work):  The model should be a CNN. The CNN should have 5 convolutional layers. Use Leaky ReLU activation for all layers. Use dropout regularization with a rate of 0.3. (Reasoning: As regularisation rate of 0.5 didn't work, we only change a new regularisation and keep the other elements that worked. This means making changes in the current level.)    
+feature_experiment_output_format: |-
+  According to the hypothesis, please help user design one or more feature engineering tasks.
+  The output should follow JSON format. The schema is as follows:
+  {
+      "factor or group name 1": {
+          "description": "description of factor or group name 1",
+          "formulation": "latex formulation of factor or group name 1",
+          "variables": {
+              "variable or function name 1": "description of variable or function 1",
+              "variable or function name 2": "description of variable or function 2"
+          }
+      },
+      "factor or group name 2": {
+          "description": "description of factor or group name 2",
+          "formulation": "latex formulation of factor or group name 2",
+          "variables": {
+              "variable or function name 1": "description of variable or function 1",
+              "variable or function name 2": "description of variable or function 2"
+          }
+      }
+      # Don't add ellipsis (...) or any filler text that might cause JSON parsing errors here!
+  }
 
 model_experiment_output_format: |-
-  So far please only design one model to test the hypothesis! 
+  According to the hypothesis, please help user design one model task.
+  Since we only build one model from four model types: ["XGBoost", "RandomForest", "LightGBM", "NN"].  
   The output should follow JSON format. The schema is as follows: 
   {
-    "model_name 1 (The name of the model)": {
-        "description": "A detailed description of the model",
-        "formulation": "A LaTeX formula representing the model's formulation",
-        "architecture": "A detailed description of the model's architecture, e.g., neural network layers or tree structures",
-        "variables": {
-            "\\hat{y}_u": "The predicted output for node u",
-            "variable_name_2": "Description of variable 2",
-            "variable_name_3": "Description of variable 3"
-        },
-        "hyperparameters": {
-            "hyperparameter_name_1": "value of hyperparameter 1",
-            "hyperparameter_name_2": "value of hyperparameter 2",
-            "hyperparameter_name_3": "value of hyperparameter 3"
-        },
-        "model_type": "XGBoost" # Should be "XGBoost"
-    },
-    "model_name 2 (The name of the model)": {
-        ...
-    }
+      "model_name": "model_name",
+      "description": "A detailed description of the model",
+      "architecture": "A detailed description of the model's architecture, e.g., neural network layers or tree structures",
+      "hyperparameters": {
+          "hyperparameter_name_1": "value of hyperparameter 1",
+          "hyperparameter_name_2": "value of hyperparameter 2",
+          "hyperparameter_name_3": "value of hyperparameter 3"
+      },
+      "model_type": "model type"
   }
-  Usually a larger model works better than a smaller one. Hence, the parameters should be larger.
+  Usually, a larger model works better than a smaller one. Hence, the parameters should be larger.
 
 model_feedback_generation:
   system: |-
@@ -91,19 +101,19 @@ model_feedback_generation:
     
     Logic for generating a new hypothesis: If the previous hypothesis works, try to inherit from it and grow deeper. If the previous hypotheis doesn't work, try to make changes in the current level.
 
-    Sample hypothesis evolution loop: (This is the entire loop, see what stage you are at. We want hypothesis to continue growing.) Levels include **Model Type**, **Layer Configuration**, **Activation Functions**, **Regularization Techniques**
+    Sample hypothesis evolution loop: (This is the entire loop, see what stage you are at. We want hypotheses to continue growing.) Levels include **Model Type**, **Layer Configuration**, **Activation Functions**, **Regularization Techniques**, **Feature Selection Methods**
 
-      1st Round Hypothesis: The model should be a CNN. 
+    1st Round Hypothesis: The model should be a CNN with no feature selection.
 
-      2nd Round Hypothesis (If first round worked: CNN is the model type level, which means that we should extend to the next level, like layer configuration): The model should be a CNN. The CNN should have 5 convolutional layers. (Reasoning: As CNN worked, we now specify the layers specification to grow the hypothesis deeper.)
+    2nd Round Hypothesis (If first round worked: CNN is the model type level, which means that we should extend to the next level, like layer configuration): The model should be a CNN. The CNN should have 5 convolutional layers, using all available features. (Reasoning: As CNN worked, we now specify the layers specification and feature selection to grow the hypothesis deeper.)
 
-      3rd Round Hypothesis (If second round didn't work): The model should be a CNN. The CNN should have 3 convolutional layers. (Reasoning: As 5-layer structure didn't work in the 2nd round hypothesis, try something else within the layer configuration level.)
+    3rd Round Hypothesis (If the second round didn't work): The model should be a CNN. The CNN should have 3 convolutional layers. Use L1 regularization for feature selection. (Reasoning: As the 5-layer structure didn't work in the 2nd round hypothesis, reduce the number of layers and implement feature selection.)
 
-      4th Round Hypothesis (If third round worked): The model should be a CNN. The CNN should have 3 convolutional layers. Use Leaky ReLU activation for all layers. (As last round worked, now proceed to the next level: activation functions)
-      
-      5th Round Hypothesis (If fourth round worked): The model should be a CNN. The CNN should have 3 convolutional layers. Use Leaky ReLU activation for all layers. Use dropout regularization with a rate of 0.5. (Similar Reasoning & Continuing to Grow to the dropout setup)
+    4th Round Hypothesis (If the third round worked): The model should be a CNN. The CNN should have 3 convolutional layers. Use Leaky ReLU activation for all layers. Retain only features selected by L1 regularization. (As the last round worked, now proceed to the next level: activation functions)
+    
+    5th Round Hypothesis (If the fourth round worked): The model should be a CNN. The CNN should have 3 convolutional layers. Use Leaky ReLU activation for all layers. Use dropout regularization with a rate of 0.5. Retain only features selected by L1 regularization. (Similar Reasoning & Continuing to Grow to the dropout setup)
 
-      6th Round Hypothesis (If fourth round didn't work):  The model should be a CNN. The CNN should have 5 convolutional layers. Use Leaky ReLU activation for all layers. Use dropout regularization with a rate of 0.3. (Reasoning: As regularisation rate of 0.5 didn't work, we only change a new regularisation and keep the other elements that worked. This means making changes in the current level.)    
+    6th Round Hypothesis (If the fourth round didn't work):  The model should be a CNN. The CNN should have 5 convolutional layers. Use Leaky ReLU activation for all layers. Use dropout regularization with a rate of 0.3. Retain features selected by PCA. (Reasoning: As the regularization rate of 0.5 didn't work, change the regularization and use PCA for feature selection while keeping other elements that worked. This means making changes at the current level.)
 
   user: |-
     We are in an experiment of finding hypothesis and validating or rejecting them so that in the end we have a powerful model generated.
@@ -128,3 +138,64 @@ model_feedback_generation:
 
     Compare and observe. Which result has a better return and lower risk? If the performance increases, the hypothesis should be considered positive (working). 
     Hence, with the hypotheses, relevant reasoning, and results in mind (comparison), provide detailed and constructive feedback and suggest a new hypothesis. 
+
+factor_feedback_generation:
+  system: |-
+    You are a professional data feature engineering assistant in data-driven R&D. 
+    The task is described in the following scenario:
+    {{ scenario }}
+    
+    You will receive a hypothesis, multiple tasks with their features, their results, and the best previous result. 
+    Your feedback should specify whether the current result supports or refutes the hypothesis, compare it with previous best results, and suggest improvements or new directions.
+    
+    Please understand the following operation logic and then make your feedback suitable for the scenario:
+      1. Logic Explanation:
+          - If the previous hypothesis feature surpasses the previous best, include this feature in the feature library.
+          - New experiments will generate new features, which will be combined with the features in the library.
+          - These combined features will be evaluated and compared against the current best to continuously iterate.
+      2. Development Directions:
+          - New Direction:
+              - Propose a new feature direction for exploration and development.
+          - Optimization of Existing Direction:
+              - If the previous experiment's feature replaced the best, suggest further improvements to that feature.
+              - Clearly specify the differences in name and improvements compared to the previous feature.
+          - Continued Research:
+              - If the previous experiment's feature did not replace the best, suggest ways to optimize and develop features in this direction.
+      3. Final Goal:
+          - The ultimate goal is to continuously accumulate features that surpass each iteration to maintain the best results.
+    
+    Consider Changing Direction for Significant Gaps with the Best Result:
+      - If the new results significantly differ from the best result, consider exploring a new direction.
+      - Avoid re-implementing previous features as those that surpassed the best are already included in the feature library and will be used in each run.
+    Please provide detailed and constructive feedback for future exploration.
+    Respond in JSON format. Example JSON structure for Result Analysis:
+    {
+      "Observations": "Your overall observations here",
+      "Feedback for Hypothesis": "Observations related to the hypothesis",
+      "New Hypothesis": "Your new hypothesis here",
+      "Reasoning": "Reasoning for the new hypothesis",
+      "Replace Best Result": "yes or no"
+    }
+  user: |-
+    Target hypothesis: 
+    {{ hypothesis_text }}
+    Tasks and Features:
+    {% for task in task_details %}
+      - {{ task.factor_name }}: {{ task.factor_description }}
+        - Feature Formulation: {{ task.factor_formulation }}
+        - Variables: {{ task.variables }}
+        - Feature Implementation: {{ task.factor_implementation }}
+        {% if task.factor_implementation == "False" %}
+        **Note: This feature was not implemented in the current experiment. Only the hypothesis for implemented features can be verified.**
+        {% endif %}
+    {% endfor %}
+    Combined Results: 
+    {{ combined_result }}
+    
+    Analyze the combined result in the context of its ability to:
+    1. Support or refute the hypothesis.
+    2. Show improvement or deterioration compared to the best result.
+    Consider Changing Direction for Significant Gaps with the Best Result:
+      - If the new results significantly differ from the best, consider exploring a new direction.
+      - Avoid re-implementing previous features as those that surpassed the best are already included in the feature library and will be used in each run.
+    Note: Only features with 'Feature Implementation' as True are implemented and tested in this experiment. If 'Feature Implementation' is False, the hypothesis for that feature cannot be verified in this run.

rdagent/scenarios/kaggle/proposal/model_proposal.py

@@ -1,105 +0,0 @@
-import json
-from pathlib import Path
-from typing import List, Tuple
-
-from jinja2 import Environment, StrictUndefined
-
-from rdagent.components.coder.model_coder.model import ModelExperiment, ModelTask
-from rdagent.components.proposal.model_proposal import (
-    ModelHypothesis,
-    ModelHypothesis2Experiment,
-    ModelHypothesisGen,
-)
-from rdagent.core.prompts import Prompts
-from rdagent.core.proposal import Hypothesis, Scenario, Trace
-from rdagent.scenarios.kaggle.experiment.model_experiment import KGModelExperiment
-
-prompt_dict = Prompts(file_path=Path(__file__).parent.parent / "prompts.yaml")
-
-KGModelHypothesis = ModelHypothesis
-
-
-class KGModelHypothesisGen(ModelHypothesisGen):
-    """
-    # NOTE: we can share this class across different data mining scenarios
-    # It may better to move the class into components folder like `rdagent/components/proposal/model_proposal.py`
-    # Here is the use case:
-
-    .. code-block:: python
-
-        class XXXDMModelHypothesisGen(DMModelHypothesisGen):
-            prompts: Prompts = a_specifc_prompt_dict
-    """
-
-    def __init__(self, scen: Scenario) -> Tuple[dict, bool]:
-        super().__init__(scen)
-
-    def prepare_context(self, trace: Trace) -> Tuple[dict, bool]:
-        hypothesis_feedback = (
-            Environment(undefined=StrictUndefined)
-            .from_string(prompt_dict["hypothesis_and_feedback"])
-            .render(trace=trace)
-        )
-        context_dict = {
-            "hypothesis_and_feedback": hypothesis_feedback,
-            "RAG": "",
-            "hypothesis_output_format": prompt_dict["hypothesis_output_format"],
-            "hypothesis_specification": "...",
-        }
-        return context_dict, True
-
-    def convert_response(self, response: str) -> ModelHypothesis:
-        response_dict = json.loads(response)
-        hypothesis = KGModelHypothesis(
-            hypothesis=response_dict["hypothesis"],
-            reason=response_dict["reason"],
-            concise_reason=response_dict["concise_reason"],
-            concise_observation=response_dict["concise_observation"],
-            concise_justification=response_dict["concise_justification"],
-            concise_knowledge=response_dict["concise_knowledge"],
-        )
-        return hypothesis
-
-
-class KGModelHypothesis2Experiment(ModelHypothesis2Experiment):
-    def prepare_context(self, hypothesis: Hypothesis, trace: Trace) -> Tuple[dict, bool]:
-        scenario = trace.scen.get_scenario_all_desc()
-        experiment_output_format = prompt_dict["model_experiment_output_format"]
-
-        hypothesis_and_feedback = (
-            Environment(undefined=StrictUndefined)
-            .from_string(prompt_dict["hypothesis_and_feedback"])
-            .render(trace=trace)
-        )
-
-        experiment_list: List[ModelExperiment] = [t[1] for t in trace.hist]
-
-        model_list = []
-        for experiment in experiment_list:
-            model_list.extend(experiment.sub_tasks)
-
-        return {
-            "target_hypothesis": str(hypothesis),
-            "scenario": scenario,
-            "hypothesis_and_feedback": hypothesis_and_feedback,
-            "experiment_output_format": experiment_output_format,
-            "target_list": model_list,
-            "RAG": ...,
-        }, True
-
-    def convert_response(self, response: str, trace: Trace) -> ModelExperiment:
-        response_dict = json.loads(response)
-        tasks = []
-        for model_name in response_dict:
-            description = response_dict[model_name]["description"]
-            formulation = response_dict[model_name]["formulation"]
-            architecture = response_dict[model_name]["architecture"]
-            variables = response_dict[model_name]["variables"]
-            hyperparameters = response_dict[model_name]["hyperparameters"]
-            model_type = response_dict[model_name]["model_type"]
-            tasks.append(
-                ModelTask(model_name, description, formulation, architecture, variables, hyperparameters, model_type)
-            )
-        exp = KGModelExperiment(tasks)
-        exp.based_experiments = [t[1] for t in trace.hist if t[2]]
-        return exp

rdagent/scenarios/kaggle/proposal/proposal.py

@@ -0,0 +1,181 @@
+import json
+from pathlib import Path
+from typing import List, Tuple
+
+from jinja2 import Environment, StrictUndefined
+
+from rdagent.components.coder.factor_coder.factor import FactorTask
+from rdagent.components.coder.model_coder.model import ModelExperiment, ModelTask
+from rdagent.components.proposal.model_proposal import (
+    ModelHypothesis,
+    ModelHypothesis2Experiment,
+    ModelHypothesisGen,
+)
+from rdagent.core.prompts import Prompts
+from rdagent.core.proposal import Hypothesis, Scenario, Trace
+from rdagent.scenarios.kaggle.experiment.kaggle_experiment import (
+    KGFactorExperiment,
+    KGModelExperiment,
+)
+
+prompt_dict = Prompts(file_path=Path(__file__).parent.parent / "prompts.yaml")
+
+
+KG_ACTION_FEATURE_ENGINEERING = "Feature engineering"
+KG_ACTION_FEATURE_PROCESSING = "Feature processing"
+KG_ACTION_MODEL_FEATURE_SELECTION = "Model feature selection"
+KG_ACTION_MODEL_TUNING = "Model tuning"
+KG_ACTION_LIST = [
+    KG_ACTION_FEATURE_ENGINEERING,
+    KG_ACTION_FEATURE_PROCESSING,
+    KG_ACTION_MODEL_FEATURE_SELECTION,
+    KG_ACTION_MODEL_TUNING,
+]
+
+
+class KGHypothesis(Hypothesis):
+    def __init__(
+        self,
+        hypothesis: str,
+        reason: str,
+        concise_reason: str,
+        concise_observation: str,
+        concise_justification: str,
+        concise_knowledge: str,
+        action: str,
+    ) -> None:
+        super().__init__(
+            hypothesis, reason, concise_reason, concise_observation, concise_justification, concise_knowledge
+        )
+        self.action = action
+
+    def __str__(self) -> str:
+        return f"""Chosen Action: {self.action}
+Hypothesis: {self.hypothesis}
+Reason: {self.reason}
+Concise Reason & Knowledge: {self.concise_reason}
+Concise Observation: {self.concise_observation}
+Concise Justification: {self.concise_justification}
+Concise Knowledge: {self.concise_knowledge}
+"""
+
+
+class KGHypothesisGen(ModelHypothesisGen):
+    """
+    # NOTE: we can share this class across different data mining scenarios
+    # It may better to move the class into components folder like `rdagent/components/proposal/model_proposal.py`
+    # Here is the use case:
+
+    .. code-block:: python
+
+        class XXXDMModelHypothesisGen(DMModelHypothesisGen):
+            prompts: Prompts = a_specifc_prompt_dict
+    """
+
+    def __init__(self, scen: Scenario) -> Tuple[dict, bool]:
+        super().__init__(scen)
+
+    def prepare_context(self, trace: Trace) -> Tuple[dict, bool]:
+        hypothesis_feedback = (
+            Environment(undefined=StrictUndefined)
+            .from_string(prompt_dict["hypothesis_and_feedback"])
+            .render(trace=trace)
+        )
+        context_dict = {
+            "hypothesis_and_feedback": hypothesis_feedback,
+            "RAG": None,
+            "hypothesis_output_format": prompt_dict["hypothesis_output_format"],
+            "hypothesis_specification": None,
+        }
+        return context_dict, True
+
+    def convert_response(self, response: str) -> ModelHypothesis:
+        response_dict = json.loads(response)
+        hypothesis = KGHypothesis(
+            hypothesis=response_dict["hypothesis"],
+            reason=response_dict["reason"],
+            concise_reason=response_dict["concise_reason"],
+            concise_observation=response_dict["concise_observation"],
+            concise_justification=response_dict["concise_justification"],
+            concise_knowledge=response_dict["concise_knowledge"],
+            action=response_dict["action"],
+        )
+        return hypothesis
+
+
+class KGHypothesis2Experiment(ModelHypothesis2Experiment):
+    def prepare_context(self, hypothesis: Hypothesis, trace: Trace) -> Tuple[dict, bool]:
+        scenario = trace.scen.get_scenario_all_desc()
+        assert isinstance(hypothesis, KGHypothesis)
+        experiment_output_format = (
+            prompt_dict["feature_experiment_output_format"]
+            if hypothesis.action in [KG_ACTION_FEATURE_ENGINEERING, KG_ACTION_FEATURE_PROCESSING]
+            else prompt_dict["model_experiment_output_format"]
+        )
+        self.current_action = hypothesis.action
+
+        hypothesis_and_feedback = (
+            Environment(undefined=StrictUndefined)
+            .from_string(prompt_dict["hypothesis_and_feedback"])
+            .render(trace=trace)
+        )
+
+        experiment_list: List[ModelExperiment] = [t[1] for t in trace.hist]
+
+        model_list = []
+        for experiment in experiment_list:
+            model_list.extend(experiment.sub_tasks)
+
+        return {
+            "target_hypothesis": str(hypothesis),
+            "scenario": scenario,
+            "hypothesis_and_feedback": hypothesis_and_feedback,
+            "experiment_output_format": experiment_output_format,
+            "target_list": model_list,
+            "RAG": ...,
+        }, True
+
+    def convert_feature_experiment(self, response: str, trace: Trace) -> KGFactorExperiment:
+        response_dict = json.loads(response)
+        tasks = []
+
+        for factor_name in response_dict:
+            description = response_dict[factor_name]["description"]
+            formulation = response_dict[factor_name]["formulation"]
+            variables = response_dict[factor_name]["variables"]
+            tasks.append(
+                FactorTask(
+                    factor_name=factor_name,
+                    factor_description=description,
+                    factor_formulation=formulation,
+                    variables=variables,
+                    version=2,
+                )
+            )
+
+        exp = KGFactorExperiment(tasks)
+        exp.based_experiments = [KGFactorExperiment(sub_tasks=[])] + [t[1] for t in trace.hist if t[2]]
+        return exp
+
+    def convert_model_experiment(self, response: str, trace: Trace) -> KGModelExperiment:
+        response_dict = json.loads(response)
+        tasks = []
+        tasks.append(
+            ModelTask(
+                name=response_dict["model_name"],
+                description=response_dict["description"],
+                architecture=response_dict["architecture"],
+                hyperparameters=response_dict["hyperparameters"],
+                model_type=response_dict["model_type"],
+                version=2,
+            )
+        )
+        exp = KGModelExperiment(tasks)
+        exp.based_experiments = [t[1] for t in trace.hist if t[2]]
+        return exp
+
+    def convert_response(self, response: str, trace: Trace) -> ModelExperiment:
+        if self.current_action in [KG_ACTION_FEATURE_ENGINEERING, KG_ACTION_FEATURE_PROCESSING]:
+            return self.convert_feature_experiment(response, trace)
+        elif self.current_action in [KG_ACTION_MODEL_FEATURE_SELECTION, KG_ACTION_MODEL_TUNING]:
+            return self.convert_model_experiment(response, trace)

rdagent/scenarios/qlib/docker/Dockerfile

@@ -1,4 +1,4 @@
-FROM pytorch/pytorch:latest
+FROM pytorch/pytorch:2.2.1-cuda12.1-cudnn8-runtime
 
 # For GPU support, please choose the proper tag from https://hub.docker.com/r/pytorch/pytorch/tags
 

rdagent/scenarios/qlib/proposal/model_proposal.py

@@ -87,7 +87,15 @@ class QlibModelHypothesis2Experiment(ModelHypothesis2Experiment):
             hyperparameters = response_dict[model_name]["hyperparameters"]
             model_type = response_dict[model_name]["model_type"]
             tasks.append(
-                ModelTask(model_name, description, formulation, architecture, variables, hyperparameters, model_type)
+                ModelTask(
+                    name=model_name,
+                    description=description,
+                    formulation=formulation,
+                    architecture=architecture,
+                    variables=variables,
+                    hyperparameters=hyperparameters,
+                    model_type=model_type,
+                )
             )
         exp = QlibModelExperiment(tasks)
         exp.based_experiments = [t[1] for t in trace.hist if t[2]]

rdagent/utils/env.py

@@ -4,12 +4,15 @@ The motiviation of the utils is for environment management
 Tries to create uniform environment for the agent to run;
 - All the code and data is expected included in one folder
 """
+
 # TODO: move the scenario specific docker env into other folders.
 
 import json
 import os
+import pickle
 import subprocess
 import sys
+import uuid
 import zipfile
 from abc import abstractmethod
 from pathlib import Path
@@ -146,6 +149,7 @@ class QlibDockerConf(DockerConf):
 
 
 class DMDockerConf(DockerConf):
+    # Data Mining Docker
     class Config:
         env_prefix = "DM_DOCKER_"
 
@@ -177,7 +181,7 @@ class KGDockerConf(DockerConf):
         Path("git_ignore_folder/data").resolve(): "/root/.data/"
     }
 
-    share_data_path: str = "/data/userdata/share/kaggle"
+    local_data_path: str = "/data/userdata/share/kaggle"
 
 
 # physionet.org/files/mimic-eicu-fiddle-feature/1.0.0/FIDDLE_mimic3
@@ -248,9 +252,9 @@ class DockerEnv(Env[DockerConf]):
         if not self.conf.enable_gpu:
             return {}
         gpu_kwargs = {
-            "device_requests": [docker.types.DeviceRequest(count=-1, capabilities=[["gpu"]])]
-            if self.conf.enable_gpu
-            else None,
+            "device_requests": (
+                [docker.types.DeviceRequest(count=-1, capabilities=[["gpu"]])] if self.conf.enable_gpu else None
+            ),
         }
         try:
             client.containers.run(self.conf.image, "nvidia-smi", **gpu_kwargs)
@@ -259,7 +263,13 @@ class DockerEnv(Env[DockerConf]):
             return {}
         return gpu_kwargs
 
-    def run(self, entry: str | None = None, local_path: str | None = None, env: dict | None = None):
+    def run(
+        self,
+        entry: str | None = None,
+        local_path: str | None = None,
+        env: dict | None = None,
+        running_extra_volume: dict | None = None,
+    ) -> str:
         if env is None:
             env = {}
         client = docker.from_env()
@@ -273,6 +283,9 @@ class DockerEnv(Env[DockerConf]):
         if self.conf.extra_volumes is not None:
             for lp, rp in self.conf.extra_volumes.items():
                 volumns[lp] = {"bind": rp, "mode": "rw"}
+        if running_extra_volume is not None:
+            for lp, rp in running_extra_volume.items():
+                volumns[lp] = {"bind": rp, "mode": "rw"}
 
         log_output = ""
 
@@ -305,6 +318,27 @@ class DockerEnv(Env[DockerConf]):
         except docker.errors.APIError as e:
             raise RuntimeError(f"Error while running the container: {e}")
 
+    def dump_python_code_run_and_get_results(
+        self, code: str, dump_file_names: list[str], local_path: str | None = None, env: dict | None = None
+    ):
+        """
+        Dump the code into the local path and run the code.
+        """
+        random_file_name = f"{uuid.uuid4()}.py"
+        with open(os.path.join(local_path, random_file_name), "w") as f:
+            f.write(code)
+        entry = f"python {random_file_name}"
+        log_output = self.run(entry, local_path, env)
+        results = []
+        os.remove(os.path.join(local_path, random_file_name))
+        for name in dump_file_names:
+            if os.path.exists(os.path.join(local_path, f"{name}")):
+                results.append(pickle.load(open(os.path.join(local_path, f"{name}"), "rb")))
+                os.remove(os.path.join(local_path, f"{name}"))
+            else:
+                return log_output, None
+        return log_output, results
+
 
 class QTDockerEnv(DockerEnv):
     """Qlib Torch Docker"""
@@ -349,9 +383,9 @@ class DMDockerEnv(DockerEnv):
 
 
 class KGDockerEnv(DockerEnv):
-    """Qlib Torch Docker"""
+    """Kaggle Competition Docker"""
 
-    def __init__(self, competition: str, conf: DockerConf = KGDockerConf()):
+    def __init__(self, competition: str = None, conf: DockerConf = KGDockerConf()):
         super().__init__(conf)
         self.competition = competition
 
@@ -361,10 +395,21 @@ class KGDockerEnv(DockerEnv):
         """
         super().prepare()
 
-        # download data
-        data_path = f"{self.conf.share_data_path}/{self.competition}"
-        subprocess.run(["kaggle", "competitions", "download", "-c", self.competition, "-p", data_path])
+        # download data, if competition is not provided, the user is targeting a general docker environment in kaggle
+        data_path = f"{self.conf.local_data_path}/{self.competition}"
+        if self.competition is not None and not Path(data_path).exists():
+            subprocess.run(["kaggle", "competitions", "download", "-c", self.competition, "-p", data_path])
 
-        # unzip data
-        with zipfile.ZipFile(f"{data_path}/{self.competition}.zip", "r") as zip_ref:
-            zip_ref.extractall(data_path)
+            # unzip data
+            with zipfile.ZipFile(f"{data_path}/{self.competition}.zip", "r") as zip_ref:
+                zip_ref.extractall(data_path)
+
+    def run(self, entry: str | None = None, local_path: str | None = None, env: dict | None = None):
+        super().run(
+            entry=entry,
+            local_path=local_path,
+            env=env,
+            running_extra_volume=(
+                {self.conf.local_data_path + "/" + self.competition: "/kaggle/input"} if self.competition else None
+            ),
+        )

requirements.txt

@@ -63,8 +63,10 @@ st-theme
 selenium
 kaggle
 
+# tool
 seaborn
 setuptools-scm
 
 # This is a temporary package installed to pass the test_import test
 xgboost
+lightgbm