Upload the configuration file for running Docker.

.gitignore

@@ -155,6 +155,6 @@ git_ignore_folder/
 *.db
 
 # Docker
-env_factor/
-env_tpl/
+env_factor/mlruns/
+env_tpl
 mlruns/

rdagent/scenarios/qlib/task_generator/env_factor/conf.yaml

@@ -0,0 +1,73 @@
+qlib_init:
+    provider_uri: "~/.qlib/qlib_data/cn_data"
+    region: cn
+
+market: &market csi300
+benchmark: &benchmark SH000300
+
+data_handler_config: &data_handler_config
+    start_time: 2008-01-01
+    end_time: 2020-08-01
+    fit_start_time: 2008-01-01
+    fit_end_time: 2014-12-31
+    instruments: *market
+port_analysis_config: &port_analysis_config
+    strategy:
+        class: TopkDropoutStrategy
+        module_path: qlib.contrib.strategy
+        kwargs:
+            signal: <PRED>
+            topk: 50
+            n_drop: 5
+    backtest:
+        start_time: 2017-01-01
+        end_time: 2020-08-01
+        account: 100000000
+        benchmark: *benchmark
+        exchange_kwargs:
+            limit_threshold: 0.095
+            deal_price: close
+            open_cost: 0.0005
+            close_cost: 0.0015
+            min_cost: 5
+task:
+    model:
+        class: LGBModel
+        module_path: qlib.contrib.model.gbdt
+        kwargs:
+            loss: mse
+            colsample_bytree: 0.8879
+            learning_rate: 0.2
+            subsample: 0.8789
+            lambda_l1: 205.6999
+            lambda_l2: 580.9768
+            max_depth: 8
+            num_leaves: 210
+            num_threads: 20
+    dataset:
+        class: DatasetH
+        module_path: qlib.data.dataset
+        kwargs:
+            handler:
+                class: Alpha158
+                module_path: qlib.contrib.data.handler
+                kwargs: *data_handler_config
+            segments:
+                train: [2008-01-01, 2014-12-31]
+                valid: [2015-01-01, 2016-12-31]
+                test: [2017-01-01, 2020-08-01]
+    record: 
+        - class: SignalRecord
+          module_path: qlib.workflow.record_temp
+          kwargs: 
+            model: <MODEL>
+            dataset: <DATASET>
+        - class: SigAnaRecord
+          module_path: qlib.workflow.record_temp
+          kwargs: 
+            ana_long_short: False
+            ann_scaler: 252
+        - class: PortAnaRecord
+          module_path: qlib.workflow.record_temp
+          kwargs: 
+            config: *port_analysis_config

rdagent/scenarios/qlib/task_generator/env_factor/conf_combined.yaml

@@ -0,0 +1,93 @@
+qlib_init:
+    provider_uri: "~/.qlib/qlib_data/cn_data"
+    region: cn
+
+market: &market csi300
+benchmark: &benchmark SH000300
+
+data_handler_config: &data_handler_config
+    start_time: 2008-01-01
+    end_time: 2022-08-01
+    instruments: *market
+    data_loader:
+        class: NestedDataLoader
+        kwargs:
+            dataloader_l:
+                - class: qlib.contrib.data.loader.Alpha158DL
+                  kwargs:
+                    config:
+                        label: 
+                            - ["Ref($close, -2)/Ref($close, -1) - 1"]
+                            - ["LABEL0"]
+                - class: qlib.data.dataset.loader.StaticDataLoader
+                  kwargs:
+                    # config: "/home/finco/v-yuanteli/RD-Agent/rdagent/scenarios/qlib/task_generator/env_factor/combined_factors_df.pkl"
+                    config: "combined_factors_df.pkl"
+
+    learn_processors:
+        - class: DropnaLabel
+        - class: CSZScoreNorm
+          kwargs:
+              fields_group: label
+
+port_analysis_config: &port_analysis_config
+    strategy:
+        class: TopkDropoutStrategy
+        module_path: qlib.contrib.strategy
+        kwargs:
+            signal: <PRED>
+            topk: 50
+            n_drop: 5
+    backtest:
+        start_time: 2017-01-01
+        end_time: 2020-08-01
+        account: 100000000
+        benchmark: *benchmark
+        exchange_kwargs:
+            limit_threshold: 0.095
+            deal_price: close
+            open_cost: 0.0005
+            close_cost: 0.0015
+            min_cost: 5
+
+task:
+    model:
+        class: LGBModel
+        module_path: qlib.contrib.model.gbdt
+        kwargs:
+            loss: mse
+            colsample_bytree: 0.8879
+            learning_rate: 0.2
+            subsample: 0.8789
+            lambda_l1: 205.6999
+            lambda_l2: 580.9768
+            max_depth: 8
+            num_leaves: 210
+            num_threads: 20
+    dataset:
+        class: DatasetH
+        module_path: qlib.data.dataset
+        kwargs:
+            handler:
+                class: DataHandlerLP
+                module_path: qlib.contrib.data.handler
+                kwargs: *data_handler_config
+            segments:
+                train: [2008-01-01, 2014-12-31]
+                valid: [2015-01-01, 2016-12-31]
+                test: [2017-01-01, 2020-08-01]
+    record: 
+        - class: SignalRecord
+          module_path: qlib.workflow.record_temp
+          kwargs: 
+            model: <MODEL>
+            dataset: <DATASET>
+        - class: SigAnaRecord
+          module_path: qlib.workflow.record_temp
+          kwargs: 
+            ana_long_short: False
+            ann_scaler: 252
+        - class: PortAnaRecord
+          module_path: qlib.workflow.record_temp
+          kwargs: 
+            config: *port_analysis_config

rdagent/scenarios/qlib/task_generator/env_factor/read_exp_res.py

@@ -0,0 +1,52 @@
+from pathlib import Path
+import qlib
+from mlflow.tracking import MlflowClient
+from mlflow.entities import ViewType
+import pandas as pd
+import pickle
+import os
+
+qlib.init()
+
+from qlib.workflow import R
+# here is the documents of the https://qlib.readthedocs.io/en/latest/component/recorder.html
+
+# TODO: list all the recorder and metrics 
+
+# Assuming you have already listed the experiments
+experiments = R.list_experiments()
+
+# Iterate through each experiment to find the latest recorder
+experiment_name = None
+latest_recorder = None
+for experiment in experiments:
+    # print(f"Experiment: {experiment}")
+    recorders = R.list_recorders(experiment_name=experiment)
+    for recorder_id in recorders:
+        if recorder_id is not None:
+            experiment_name = experiment
+            recorder = R.get_recorder(recorder_id=recorder_id, experiment_name=experiment)
+            end_time = recorder.info['end_time']
+            if latest_recorder is None or end_time > latest_recorder.info['end_time']:
+                latest_recorder = recorder
+
+# Check if the latest recorder is found
+if latest_recorder is None:
+    print("No recorders found")
+else:
+    print(f"Latest recorder: {latest_recorder}")
+
+    # Load the specified file from the latest recorder
+    file_path = "portfolio_analysis/port_analysis_1day.pkl"
+    indicator_analysis_df = latest_recorder.load_object(file_path)
+
+    # Optionally convert to DataFrame if not already in DataFrame format
+    if not isinstance(indicator_analysis_df, pd.DataFrame):
+        indicator_analysis_df = pd.DataFrame(indicator_analysis_df)
+    
+    output_path = os.path.join(str(Path(__file__).resolve().parent), "qlib_res.pkl")
+    with open(output_path, "wb") as f:
+        pickle.dump(indicator_analysis_df, f)
+
+    print("here2")
+    print(output_path)