diff --git a/rdagent/scenarios/qlib/developer/factor_runner.py b/rdagent/scenarios/qlib/developer/factor_runner.py index ade70b1c..466bce24 100644 --- a/rdagent/scenarios/qlib/developer/factor_runner.py +++ b/rdagent/scenarios/qlib/developer/factor_runner.py @@ -105,10 +105,13 @@ class QlibFactorRunner(CachedRunner[QlibFactorExperiment]): combined_factors = combined_factors.loc[:, ~combined_factors.columns.duplicated(keep="last")] new_columns = pd.MultiIndex.from_product([["feature"], combined_factors.columns]) combined_factors.columns = new_columns + # Due to the rdagent and qlib docker image in the numpy version of the difference, + # the `combined_factors_df.pkl` file could not be loaded correctly in qlib dokcer, + # so we changed the file type of `combined_factors_df` from pkl to parquet. + target_path = exp.experiment_workspace.workspace_path / "combined_factors_df.parquet" # Save the combined factors to the workspace - with open(exp.experiment_workspace.workspace_path / "combined_factors_df.pkl", "wb") as f: - pickle.dump(combined_factors, f) + combined_factors.to_parquet(target_path, engine="pyarrow") result = exp.experiment_workspace.execute( qlib_config_name=f"conf.yaml" if len(exp.based_experiments) == 0 else "conf_combined.yaml" diff --git a/rdagent/scenarios/qlib/docker/Dockerfile b/rdagent/scenarios/qlib/docker/Dockerfile index 8ceabce5..755ec2f7 100644 --- a/rdagent/scenarios/qlib/docker/Dockerfile +++ b/rdagent/scenarios/qlib/docker/Dockerfile @@ -9,11 +9,11 @@ RUN apt-get clean && apt-get update && apt-get install -y \ build-essential \ && rm -rf /var/lib/apt/lists/* -RUN git clone https://github.com/microsoft/qlib.git +RUN git clone --depth=100 https://github.com/microsoft/qlib.git WORKDIR /workspace/qlib -RUN git reset c9ed050ef034fe6519c14b59f3d207abcb693282 --hard +RUN git reset 4d621bff99bb403231e26bb1dddae6e77d2af170 --hard RUN python -m pip install --upgrade cython RUN python -m pip install -e . diff --git a/rdagent/scenarios/qlib/experiment/factor_template/conf_combined.yaml b/rdagent/scenarios/qlib/experiment/factor_template/conf_combined.yaml index f32e7ea0..e98305e3 100644 --- a/rdagent/scenarios/qlib/experiment/factor_template/conf_combined.yaml +++ b/rdagent/scenarios/qlib/experiment/factor_template/conf_combined.yaml @@ -21,7 +21,7 @@ data_handler_config: &data_handler_config - ["LABEL0"] - class: qlib.data.dataset.loader.StaticDataLoader kwargs: - config: "combined_factors_df.pkl" + config: "combined_factors_df.parquet" learn_processors: - class: DropnaLabel diff --git a/requirements.txt b/requirements.txt index 8f5f0b27..69720cf1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,6 +10,7 @@ fuzzywuzzy openai litellm azure.identity +pyarrow numpy # we use numpy as default data format. So we have to install numpy pandas # we use pandas as default data format. So we have to install pandas