feat: Kaggle loop update (Feature & Model) (#241)

* Init todo * Evaluation & dataset * Generate new data * dataset generation * add the result * Analysis * Factor update * Updates * Reformat analysis.py * CI fix * Revised Preprocessing & Supported Random Forest * Revised to support three models with feature * Further revised prompts * Slight Revision * docs: update contributors (#230) * Revised to support three models with feature * Further revised prompts * Slight Revision * feat: kaggle model and feature (#238) * update first version code * make hypothesis_gen and experiment_builder fit for both feature and model * feat: continue kaggle feature and model coder (#239) * use qlib docker to run qlib models * feature coder ready * model coder ready * fix CI * finish the first round of runner (#240) * Optimized the factor scenario and added the front-end. * fix a small bug * fix a typo * update the kaggle scenario * delete model_template folder * use experiment to run data preprocess script * add source data to scenarios * minor fix * minor bug fix * train.py debug * fixed a bug in train.py and added some TODOs * For Debugging * fix two small bugs in based_exp * fix some bugs * update preprocess * fix a bug in preprocess * fix a bug in train.py * reformat * Follow-up * fix a bug in train.py * fix a bug in workspace * fix a bug in feature duplication * fix a bug in feedback * fix a bug in preprocessed data * fix a bug om feature engineering * fix a ci error * Debugged & Connected * Fixed error on feedback & added other fixes * fix CI errors * fix a CI bug * fix: fix_dotenv_error (#257) * fix_dotenv_error * format with isort * Update rdagent/app/cli.py --------- Co-authored-by: you-n-g <you-n-g@users.noreply.github.com> * chore(main): release 0.2.1 (#249) Release-As: 0.2.1 * init a scenario for kaggle feature engineering * delete error codes * Delete rdagent/app/kaggle_feature/conf.py --------- Co-authored-by: Young <afe.young@gmail.com> Co-authored-by: Taozhi Wang <taozhi.mark.wang@gmail.com> Co-authored-by: you-n-g <you-n-g@users.noreply.github.com> Co-authored-by: cyncyw <47289405+taozhiwang@users.noreply.github.com> Co-authored-by: Xisen-Wang <xisen_application@163.com> Co-authored-by: Haotian Chen <113661982+Hytn@users.noreply.github.com> Co-authored-by: WinstonLiye <1957922024@qq.com> Co-authored-by: WinstonLiyt <104308117+WinstonLiyt@users.noreply.github.com> Co-authored-by: Linlang <30293408+SunsetWolf@users.noreply.github.com>
2026-08-04 18:57:44 +00:00 · 2024-09-11 15:26:52 +08:00
parent 11fb03ef62
commit dbe2cf12bb
42 changed files with 1513 additions and 681 deletions
@@ -161,7 +161,7 @@ class FactorOutputFormatEvaluator(FactorEvaluator):
            )
        buffer = io.StringIO()
        gen_df.info(buf=buffer)
-        gen_df_info_str = buffer.getvalue()
+        gen_df_info_str = f"The use is currently working on a feature related task.\nThe output dataframe info is:\n{buffer.getvalue()}"
        system_prompt = (
            Environment(undefined=StrictUndefined)
            .from_string(
@@ -378,6 +378,7 @@ class FactorValueEvaluator(FactorEvaluator):
        self,
        implementation: Workspace,
        gt_implementation: Workspace,
+        version: int = 1,  # 1 for qlib factors and 2 for kaggle factors
        **kwargs,
    ) -> Tuple:
        conclusions = []
@@ -389,18 +390,21 @@ class FactorValueEvaluator(FactorEvaluator):
        equal_value_ratio_result = 0
        high_correlation_result = False

-        # Check if both dataframe has only one columns
-        feedback_str, _ = FactorSingleColumnEvaluator(self.scen).evaluate(implementation, gt_implementation)
-        conclusions.append(feedback_str)
+        # Check if both dataframe has only one columns Mute this since factor task might generate more than one columns now
+        if version == 1:
+            feedback_str, _ = FactorSingleColumnEvaluator(self.scen).evaluate(implementation, gt_implementation)
+            conclusions.append(feedback_str)

        # Check if the index of the dataframe is ("datetime", "instrument")
        feedback_str, _ = FactorOutputFormatEvaluator(self.scen).evaluate(implementation, gt_implementation)
        conclusions.append(feedback_str)
-
-        feedback_str, daily_check_result = FactorDatetimeDailyEvaluator(self.scen).evaluate(
-            implementation, gt_implementation
-        )
-        conclusions.append(feedback_str)
+        if version == 1:
+            feedback_str, daily_check_result = FactorDatetimeDailyEvaluator(self.scen).evaluate(
+                implementation, gt_implementation
+            )
+            conclusions.append(feedback_str)
+        else:
+            daily_check_result = None

        # Check if both dataframe have the same rows count
        if gt_implementation is not None:
@@ -627,7 +631,9 @@ class FactorEvaluatorForCoder(FactorEvaluator):
                (
                    factor_feedback.factor_value_feedback,
                    decision_from_value_check,
-                ) = self.value_evaluator.evaluate(implementation=implementation, gt_implementation=gt_implementation)
+                ) = self.value_evaluator.evaluate(
+                    implementation=implementation, gt_implementation=gt_implementation, version=target_task.version
+                )

            factor_feedback.final_decision_based_on_gt = gt_implementation is not None

@@ -647,7 +653,7 @@ class FactorEvaluatorForCoder(FactorEvaluator):
                    target_task=target_task,
                    implementation=implementation,
                    execution_feedback=factor_feedback.execution_feedback,
-                    value_feedback=factor_feedback.factor_value_feedback,
+                    factor_value_feedback=factor_feedback.factor_value_feedback,
                    gt_implementation=gt_implementation,
                )
                (
@@ -24,9 +24,11 @@ class FactorTask(Task):
        factor_name,
        factor_description,
        factor_formulation,
+        *args,
        variables: dict = {},
        resource: str = None,
        factor_implementation: bool = False,
+        **kwargs,
    ) -> None:
        self.factor_name = factor_name
        self.factor_description = factor_description
@@ -34,6 +36,7 @@ class FactorTask(Task):
        self.variables = variables
        self.factor_resources = resource
        self.factor_implementation = factor_implementation
+        super().__init__(*args, **kwargs)

    def get_task_information(self):
        return f"""factor_name: {self.factor_name}
@@ -75,8 +78,8 @@ class FactorFBWorkspace(FBWorkspace):
    def __init__(
        self,
        *args,
-        executed_factor_value_dataframe=None,
-        raise_exception=False,
+        executed_factor_value_dataframe: pd.DataFrame = None,
+        raise_exception: bool = False,
        **kwargs,
    ) -> None:
        super().__init__(*args, **kwargs)
@@ -102,7 +105,10 @@ class FactorFBWorkspace(FBWorkspace):
        1. make the directory in workspace path
        2. write the code to the file in the workspace path
        3. link all the source data to the workspace path folder
-        4. execute the code
+        if call_factor_py is True:
+            4. execute the code
+        else:
+            4. generate a script from template to import the factor.py dump get the factor value to result.h5
        5. read the factor value from the output file in the workspace path folder
        returns the execution feedback as a string and the factor value as a pandas dataframe

@@ -130,15 +136,21 @@ class FactorFBWorkspace(FBWorkspace):
            if self.executed_factor_value_dataframe is not None:
                return self.FB_FROM_CACHE, self.executed_factor_value_dataframe

-            source_data_path = (
-                Path(
-                    FACTOR_IMPLEMENT_SETTINGS.data_folder_debug,
+            if self.target_task.version == 1:
+                source_data_path = (
+                    Path(
+                        FACTOR_IMPLEMENT_SETTINGS.data_folder_debug,
+                    )
+                    if data_type == "Debug"
+                    else Path(
+                        FACTOR_IMPLEMENT_SETTINGS.data_folder,
+                    )
                )
-                if data_type == "Debug"
-                else Path(
+            elif self.target_task.version == 2:
+                # TODO you can change the name of the data folder for a better understanding
+                source_data_path = Path(
                    FACTOR_IMPLEMENT_SETTINGS.data_folder,
                )
-            )

            source_data_path.mkdir(exist_ok=True, parents=True)
            code_path = self.workspace_path / f"factor.py"
@@ -147,9 +159,16 @@ class FactorFBWorkspace(FBWorkspace):

            execution_feedback = self.FB_EXECUTION_SUCCEEDED
            execution_success = False
+
+            if self.target_task.version == 1:
+                execution_code_path = code_path
+            elif self.target_task.version == 2:
+                execution_code_path = self.workspace_path / f"{uuid.uuid4()}.py"
+                execution_code_path.write_text((Path(__file__).parent / "factor_execution_template.txt").read_text())
+
            try:
                subprocess.check_output(
-                    f"{FACTOR_IMPLEMENT_SETTINGS.python_bin} {code_path}",
+                    f"{FACTOR_IMPLEMENT_SETTINGS.python_bin} {execution_code_path}",
                    shell=True,
                    cwd=self.workspace_path,
                    stderr=subprocess.STDOUT,
@@ -161,7 +180,7 @@ class FactorFBWorkspace(FBWorkspace):

                execution_feedback = (
                    e.output.decode()
-                    .replace(str(code_path.parent.absolute()), r"/path/to")
+                    .replace(str(execution_code_path.parent.absolute()), r"/path/to")
                    .replace(str(site.getsitepackages()[0]), r"/path/to/site-packages")
                )
                if len(execution_feedback) > 2000:
@@ -0,0 +1,13 @@
+import os
+
+import numpy as np
+import pandas as pd
+from factor import feat_eng
+
+if os.path.exists("valid.pkl"):
+    valid_df = pd.read_pickle("valid.pkl")
+else:
+    raise FileNotFoundError("No valid data found.")
+
+new_feat = feat_eng(valid_df)
+new_feat.to_hdf("result.h5", key="data", mode="w")
@@ -24,7 +24,7 @@ from rdagent.oai.llm_utils import APIBackend
 evaluate_prompts = Prompts(file_path=Path(__file__).parent.parent / "prompts.yaml")


-def shape_evaluator(prediction: torch.Tensor, target_shape: Tuple = None) -> Tuple[str, bool]:
+def shape_evaluator(prediction: torch.Tensor | np.ndarray, target_shape: Tuple = None) -> Tuple[str, bool]:
    if target_shape is None or prediction is None:
        return (
            "No output generated from the model. No shape evaluation conducted.",
@@ -279,12 +279,8 @@ class ModelCoderEvaluator(Evaluator):
        else:
            gt_tensor = None

-        if target_task.model_type == "XGBoost":
-            shape_feedback = "Not applicable for XGBoost models"
-            shape_decision = True
-        else:
-            shape_feedback, shape_decision = shape_evaluator(gen_tensor, (batch_size, 1))
-        value_feedback, value_decision = value_evaluator(gt_tensor, gen_tensor)
+        shape_feedback, shape_decision = shape_evaluator(gen_tensor, (batch_size, 1))
+        value_feedback, value_decision = value_evaluator(gen_tensor, gt_tensor)
        code_feedback, _ = ModelCodeEvaluator(scen=self.scen).evaluate(
            target_task=target_task,
            implementation=implementation,
@@ -1,20 +1,13 @@
-import json
 import pickle
 import site
 import traceback
-import uuid
 from pathlib import Path
-from typing import Any, Dict, Optional
-
-import numpy as np
-import torch
-import xgboost as xgb
+from typing import Dict, Optional

 from rdagent.components.coder.model_coder.conf import MODEL_IMPL_SETTINGS
-from rdagent.core.exception import CodeFormatError
 from rdagent.core.experiment import Experiment, FBWorkspace, Task
 from rdagent.oai.llm_utils import md5_hash
-from rdagent.utils import get_module_by_module_path
+from rdagent.utils.env import KGDockerEnv, QTDockerEnv


 class ModelTask(Task):
@@ -22,11 +15,13 @@ class ModelTask(Task):
        self,
        name: str,
        description: str,
-        formulation: str,
        architecture: str,
-        variables: Dict[str, str],
+        *args,
        hyperparameters: Dict[str, str],
+        formulation: str = None,
+        variables: Dict[str, str] = None,
        model_type: Optional[str] = None,
+        **kwargs,
    ) -> None:
        self.name: str = name
        self.description: str = description
@@ -35,16 +30,18 @@ class ModelTask(Task):
        self.variables: str = variables
        self.hyperparameters: str = hyperparameters
        self.model_type: str = model_type  # Tabular for tabular model, TimesSeries for time series model, Graph for graph model, XGBoost for XGBoost model
+        super().__init__(*args, **kwargs)

    def get_task_information(self):
-        return f"""name: {self.name}
+        task_desc = f"""name: {self.name}
 description: {self.description}
-formulation: {self.formulation}
-architecture: {self.architecture}
-variables: {self.variables}
-hyperparameters: {self.hyperparameters}
-model_type: {self.model_type}
 """
+        task_desc += f"formulation: {self.formulation}\n" if self.formulation else ""
+        task_desc += f"architecture: {self.architecture}\n"
+        task_desc += f"variables: {self.variables}\n" if self.variables else ""
+        task_desc += f"hyperparameters: {self.hyperparameters}\n"
+        task_desc += f"model_type: {self.model_type}\n"
+        return task_desc

    @staticmethod
    def from_dict(dict):
@@ -66,12 +63,13 @@ class ModelFBWorkspace(FBWorkspace):
        - the `model.py` that contains a variable named `model_cls` which indicates the implemented model structure
            - `model_cls` is a instance of `torch.nn.Module`;

+    We support two ways of interface:
+        (version 1) for qlib we'll make a script to import the model in the implementation in file `model.py` after setting the cwd into the directory
+            - from model import model_cls
+            - initialize the model by initializing it `model_cls(input_dim=INPUT_DIM)`
+            - And then verify the model.

-    We'll import the model in the implementation in file `model.py` after setting the cwd into the directory
-    - from model import model_cls
-    - initialize the model by initializing it `model_cls(input_dim=INPUT_DIM)`
-    - And then verify the model.
-
+        (version 2) for kaggle we'll make a script to call the fit and predict function in the implementation in file `model.py` after setting the cwd into the directory
    """

    def execute(
@@ -94,51 +92,33 @@ class ModelFBWorkspace(FBWorkspace):
                Path(MODEL_IMPL_SETTINGS.cache_location).mkdir(exist_ok=True, parents=True)
                if cache_file_path.exists():
                    return pickle.load(open(cache_file_path, "rb"))
-            mod = get_module_by_module_path(str(self.workspace_path / "model.py"))

-            if self.target_task.model_type != "XGBoost":
-                model_cls = mod.model_cls
+            qtde = QTDockerEnv() if self.target_task.version == 1 else KGDockerEnv()
+            qtde.prepare()

-            if self.target_task.model_type == "XGBoost":
-                X_simulated = np.random.rand(100, num_features)  # 100 samples, `num_features` features each
-                y_simulated = np.random.randint(0, 2, 100)  # Binary target for example
-                params = mod.get_params()
-                num_round = mod.get_num_round()
-                dtrain = xgb.DMatrix(X_simulated, label=y_simulated)
-            elif self.target_task.model_type == "Tabular":
-                input_shape = (batch_size, num_features)
-                m = model_cls(num_features=input_shape[1])
-                data = torch.full(input_shape, input_value)
-            elif self.target_task.model_type == "TimeSeries":
-                input_shape = (batch_size, num_features, num_timesteps)
-                m = model_cls(num_features=input_shape[1], num_timesteps=input_shape[2])
-                data = torch.full(input_shape, input_value)
-            elif self.target_task.model_type == "Graph":
-                node_feature = torch.randn(batch_size, num_features)
-                edge_index = torch.randint(0, batch_size, (2, num_edges))
-                m = model_cls(num_features=num_features)
-                data = (node_feature, edge_index)
-            else:
-                raise ValueError(f"Unsupported model type: {self.target_task.model_type}")
+            if self.target_task.version == 1:
+                dump_code = f"""
+MODEL_TYPE = "{self.target_task.model_type}"
+BATCH_SIZE = {batch_size}
+NUM_FEATURES = {num_features}
+NUM_TIMESTEPS = {num_timesteps}
+NUM_EDGES = {num_edges}
+INPUT_VALUE = {input_value}
+PARAM_INIT_VALUE = {param_init_value}
+{(Path(__file__).parent / 'model_execute_template_v1.txt').read_text()}
+"""
+            elif self.target_task.version == 2:
+                dump_code = (Path(__file__).parent / "model_execute_template_v2.txt").read_text()

-            if self.target_task.model_type == "XGBoost":
-                bst = xgb.train(params, dtrain, num_round)
-                y_pred = bst.predict(dtrain)
-                execution_model_output = y_pred
-                execution_feedback_str = "Execution successful, model trained and predictions made."
-            else:
-                # Initialize all parameters of `m` to `param_init_value`
-                for _, param in m.named_parameters():
-                    param.data.fill_(param_init_value)
-
-                # Execute the model
-                if self.target_task.model_type == "Graph":
-                    out = m(*data)
-                else:
-                    out = m(data)
-
-                execution_model_output = out.cpu().detach()
-                execution_feedback_str = f"Execution successful, output tensor shape: {execution_model_output.shape}"
+            log, results = qtde.dump_python_code_run_and_get_results(
+                code=dump_code,
+                dump_file_names=["execution_feedback_str.pkl", "execution_model_output.pkl"],
+                local_path=str(self.workspace_path),
+                env={},
+            )
+            if results is None:
+                raise RuntimeError(f"Error in running the model code: {log}")
+            [execution_feedback_str, execution_model_output] = results

            if MODEL_IMPL_SETTINGS.enable_execution_cache:
                pickle.dump(
@@ -150,10 +130,6 @@ class ModelFBWorkspace(FBWorkspace):
            execution_feedback_str = f"Execution error: {e}\nTraceback: {traceback.format_exc()}"
            execution_model_output = None

-        code_path = self.workspace_path / f"model.py"
-        execution_feedback_str = execution_feedback_str.replace(str(code_path.parent.absolute()), r"/path/to").replace(
-            str(site.getsitepackages()[0]), r"/path/to/site-packages"
-        )
        if len(execution_feedback_str) > 2000:
            execution_feedback_str = (
                execution_feedback_str[:1000] + "....hidden long error message...." + execution_feedback_str[-1000:]
@@ -161,4 +137,5 @@ class ModelFBWorkspace(FBWorkspace):
        return execution_feedback_str, execution_model_output


+FeatureExperiment = Experiment
 ModelExperiment = Experiment
@@ -0,0 +1,44 @@
+# MODEL_TYPE = "Tabular"
+# BATCH_SIZE = 32
+# NUM_FEATURES = 10
+# NUM_TIMESTEPS = 4
+# NUM_EDGES = 20
+# INPUT_VALUE = 1.0
+# PARAM_INIT_VALUE = 1.0
+
+import pickle
+
+import torch
+from model import model_cls
+
+if MODEL_TYPE == "Tabular":
+    input_shape = (BATCH_SIZE, NUM_FEATURES)
+    m = model_cls(num_features=input_shape[1])
+    data = torch.full(input_shape, INPUT_VALUE)
+elif MODEL_TYPE == "TimeSeries":
+    input_shape = (BATCH_SIZE, NUM_FEATURES, NUM_TIMESTEPS)
+    m = model_cls(num_features=input_shape[1], num_timesteps=input_shape[2])
+    data = torch.full(input_shape, INPUT_VALUE)
+elif MODEL_TYPE == "Graph":
+    node_feature = torch.randn(BATCH_SIZE, NUM_FEATURES)
+    edge_index = torch.randint(0, BATCH_SIZE, (2, NUM_EDGES))
+    m = model_cls(num_features=NUM_FEATURES)
+    data = (node_feature, edge_index)
+else:
+    raise ValueError(f"Unsupported model type: {MODEL_TYPE}")
+
+# Initialize all parameters of `m` to `param_init_value`
+for _, param in m.named_parameters():
+    param.data.fill_(PARAM_INIT_VALUE)
+
+# Execute the model
+if MODEL_TYPE == "Graph":
+    out = m(*data)
+else:
+    out = m(data)
+
+execution_model_output = out.cpu().detach()
+execution_feedback_str = f"Execution successful, output tensor shape: {execution_model_output.shape}"
+
+pickle.dump(execution_model_output, open("execution_model_output.pkl", "wb"))
+pickle.dump(execution_feedback_str, open("execution_feedback_str.pkl", "wb"))
@@ -0,0 +1,20 @@
+import os
+import pickle
+
+import numpy as np
+import pandas as pd
+import torch
+from model import fit, predict, select
+
+train_X = pd.DataFrame(np.random.randn(8, 30), columns=[f"{i}" for i in range(30)])
+train_y = pd.Series(np.random.randint(0, 2, 8))
+valid_X = pd.DataFrame(np.random.randn(8, 30), columns=[f"{i}" for i in range(30)])
+valid_y = pd.Series(np.random.randint(0, 2, 8))
+
+model = fit(train_X, train_y, valid_X, valid_y)
+execution_model_output = predict(model, valid_X)
+
+execution_feedback_str = f"Execution successful, output numpy ndarray shape: {execution_model_output.shape}"
+
+pickle.dump(execution_model_output, open("execution_model_output.pkl", "wb"))
+pickle.dump(execution_feedback_str, open("execution_feedback_str.pkl", "wb"))