fix: fix combined_factors_df.pkl not loading in docker (#697)

* Fix combined_factors_df.pkl not loading in docker

* change combined factors file type

* change qlib commit id

rdagent/scenarios/qlib/developer/factor_runner.py

@@ -105,10 +105,13 @@ class QlibFactorRunner(CachedRunner[QlibFactorExperiment]):
             combined_factors = combined_factors.loc[:, ~combined_factors.columns.duplicated(keep="last")]
             new_columns = pd.MultiIndex.from_product([["feature"], combined_factors.columns])
             combined_factors.columns = new_columns
+            # Due to the rdagent and qlib docker image in the numpy version of the difference,
+            # the `combined_factors_df.pkl` file could not be loaded correctly in qlib dokcer,
+            # so we changed the file type of `combined_factors_df` from pkl to parquet.
+            target_path = exp.experiment_workspace.workspace_path / "combined_factors_df.parquet"
 
             # Save the combined factors to the workspace
-            with open(exp.experiment_workspace.workspace_path / "combined_factors_df.pkl", "wb") as f:
-                pickle.dump(combined_factors, f)
+            combined_factors.to_parquet(target_path, engine="pyarrow")
 
         result = exp.experiment_workspace.execute(
             qlib_config_name=f"conf.yaml" if len(exp.based_experiments) == 0 else "conf_combined.yaml"

rdagent/scenarios/qlib/docker/Dockerfile

@@ -9,11 +9,11 @@ RUN apt-get clean && apt-get update && apt-get install -y \
     build-essential \
     && rm -rf /var/lib/apt/lists/* 
 
-RUN git clone https://github.com/microsoft/qlib.git
+RUN git clone --depth=100 https://github.com/microsoft/qlib.git
 
 WORKDIR /workspace/qlib
 
-RUN git reset c9ed050ef034fe6519c14b59f3d207abcb693282 --hard
+RUN git reset 4d621bff99bb403231e26bb1dddae6e77d2af170 --hard
 
 RUN python -m pip install --upgrade cython
 RUN python -m pip install -e .

rdagent/scenarios/qlib/experiment/factor_template/conf_combined.yaml

@@ -21,7 +21,7 @@ data_handler_config: &data_handler_config
                             - ["LABEL0"]
                 - class: qlib.data.dataset.loader.StaticDataLoader
                   kwargs:
-                    config: "combined_factors_df.pkl"
+                    config: "combined_factors_df.parquet"
 
     learn_processors:
         - class: DropnaLabel

requirements.txt

@@ -10,6 +10,7 @@ fuzzywuzzy
 openai
 litellm
 azure.identity
+pyarrow
 
 numpy # we use numpy as default data format. So we have to install numpy
 pandas # we use pandas as default data format. So we have to install pandas