diff --git a/.gitignore b/.gitignore index a1dd635e..176e76d2 100644 --- a/.gitignore +++ b/.gitignore @@ -155,6 +155,6 @@ git_ignore_folder/ *.db # Docker -env_factor/ -env_tpl/ +env_factor/mlruns/ +env_tpl mlruns/ \ No newline at end of file diff --git a/rdagent/scenarios/qlib/task_generator/env_factor/conf.yaml b/rdagent/scenarios/qlib/task_generator/env_factor/conf.yaml new file mode 100644 index 00000000..b8483b02 --- /dev/null +++ b/rdagent/scenarios/qlib/task_generator/env_factor/conf.yaml @@ -0,0 +1,73 @@ +qlib_init: + provider_uri: "~/.qlib/qlib_data/cn_data" + region: cn + +market: &market csi300 +benchmark: &benchmark SH000300 + +data_handler_config: &data_handler_config + start_time: 2008-01-01 + end_time: 2020-08-01 + fit_start_time: 2008-01-01 + fit_end_time: 2014-12-31 + instruments: *market +port_analysis_config: &port_analysis_config + strategy: + class: TopkDropoutStrategy + module_path: qlib.contrib.strategy + kwargs: + signal: + topk: 50 + n_drop: 5 + backtest: + start_time: 2017-01-01 + end_time: 2020-08-01 + account: 100000000 + benchmark: *benchmark + exchange_kwargs: + limit_threshold: 0.095 + deal_price: close + open_cost: 0.0005 + close_cost: 0.0015 + min_cost: 5 +task: + model: + class: LGBModel + module_path: qlib.contrib.model.gbdt + kwargs: + loss: mse + colsample_bytree: 0.8879 + learning_rate: 0.2 + subsample: 0.8789 + lambda_l1: 205.6999 + lambda_l2: 580.9768 + max_depth: 8 + num_leaves: 210 + num_threads: 20 + dataset: + class: DatasetH + module_path: qlib.data.dataset + kwargs: + handler: + class: Alpha158 + module_path: qlib.contrib.data.handler + kwargs: *data_handler_config + segments: + train: [2008-01-01, 2014-12-31] + valid: [2015-01-01, 2016-12-31] + test: [2017-01-01, 2020-08-01] + record: + - class: SignalRecord + module_path: qlib.workflow.record_temp + kwargs: + model: + dataset: + - class: SigAnaRecord + module_path: qlib.workflow.record_temp + kwargs: + ana_long_short: False + ann_scaler: 252 + - class: PortAnaRecord + module_path: qlib.workflow.record_temp + kwargs: + config: *port_analysis_config diff --git a/rdagent/scenarios/qlib/task_generator/env_factor/conf_combined.yaml b/rdagent/scenarios/qlib/task_generator/env_factor/conf_combined.yaml new file mode 100644 index 00000000..b7768bc0 --- /dev/null +++ b/rdagent/scenarios/qlib/task_generator/env_factor/conf_combined.yaml @@ -0,0 +1,93 @@ +qlib_init: + provider_uri: "~/.qlib/qlib_data/cn_data" + region: cn + +market: &market csi300 +benchmark: &benchmark SH000300 + +data_handler_config: &data_handler_config + start_time: 2008-01-01 + end_time: 2022-08-01 + instruments: *market + data_loader: + class: NestedDataLoader + kwargs: + dataloader_l: + - class: qlib.contrib.data.loader.Alpha158DL + kwargs: + config: + label: + - ["Ref($close, -2)/Ref($close, -1) - 1"] + - ["LABEL0"] + - class: qlib.data.dataset.loader.StaticDataLoader + kwargs: + # config: "/home/finco/v-yuanteli/RD-Agent/rdagent/scenarios/qlib/task_generator/env_factor/combined_factors_df.pkl" + config: "combined_factors_df.pkl" + + learn_processors: + - class: DropnaLabel + - class: CSZScoreNorm + kwargs: + fields_group: label + +port_analysis_config: &port_analysis_config + strategy: + class: TopkDropoutStrategy + module_path: qlib.contrib.strategy + kwargs: + signal: + topk: 50 + n_drop: 5 + backtest: + start_time: 2017-01-01 + end_time: 2020-08-01 + account: 100000000 + benchmark: *benchmark + exchange_kwargs: + limit_threshold: 0.095 + deal_price: close + open_cost: 0.0005 + close_cost: 0.0015 + min_cost: 5 + +task: + model: + class: LGBModel + module_path: qlib.contrib.model.gbdt + kwargs: + loss: mse + colsample_bytree: 0.8879 + learning_rate: 0.2 + subsample: 0.8789 + lambda_l1: 205.6999 + lambda_l2: 580.9768 + max_depth: 8 + num_leaves: 210 + num_threads: 20 + dataset: + class: DatasetH + module_path: qlib.data.dataset + kwargs: + handler: + class: DataHandlerLP + module_path: qlib.contrib.data.handler + kwargs: *data_handler_config + segments: + train: [2008-01-01, 2014-12-31] + valid: [2015-01-01, 2016-12-31] + test: [2017-01-01, 2020-08-01] + record: + - class: SignalRecord + module_path: qlib.workflow.record_temp + kwargs: + model: + dataset: + - class: SigAnaRecord + module_path: qlib.workflow.record_temp + kwargs: + ana_long_short: False + ann_scaler: 252 + - class: PortAnaRecord + module_path: qlib.workflow.record_temp + kwargs: + config: *port_analysis_config \ No newline at end of file diff --git a/rdagent/scenarios/qlib/task_generator/env_factor/read_exp_res.py b/rdagent/scenarios/qlib/task_generator/env_factor/read_exp_res.py new file mode 100644 index 00000000..9695008f --- /dev/null +++ b/rdagent/scenarios/qlib/task_generator/env_factor/read_exp_res.py @@ -0,0 +1,52 @@ +from pathlib import Path +import qlib +from mlflow.tracking import MlflowClient +from mlflow.entities import ViewType +import pandas as pd +import pickle +import os + +qlib.init() + +from qlib.workflow import R +# here is the documents of the https://qlib.readthedocs.io/en/latest/component/recorder.html + +# TODO: list all the recorder and metrics + +# Assuming you have already listed the experiments +experiments = R.list_experiments() + +# Iterate through each experiment to find the latest recorder +experiment_name = None +latest_recorder = None +for experiment in experiments: + # print(f"Experiment: {experiment}") + recorders = R.list_recorders(experiment_name=experiment) + for recorder_id in recorders: + if recorder_id is not None: + experiment_name = experiment + recorder = R.get_recorder(recorder_id=recorder_id, experiment_name=experiment) + end_time = recorder.info['end_time'] + if latest_recorder is None or end_time > latest_recorder.info['end_time']: + latest_recorder = recorder + +# Check if the latest recorder is found +if latest_recorder is None: + print("No recorders found") +else: + print(f"Latest recorder: {latest_recorder}") + + # Load the specified file from the latest recorder + file_path = "portfolio_analysis/port_analysis_1day.pkl" + indicator_analysis_df = latest_recorder.load_object(file_path) + + # Optionally convert to DataFrame if not already in DataFrame format + if not isinstance(indicator_analysis_df, pd.DataFrame): + indicator_analysis_df = pd.DataFrame(indicator_analysis_df) + + output_path = os.path.join(str(Path(__file__).resolve().parent), "qlib_res.pkl") + with open(output_path, "wb") as f: + pickle.dump(indicator_analysis_df, f) + + print("here2") + print(output_path) \ No newline at end of file