diff --git a/rdagent/components/coder/factor_coder/config.py b/rdagent/components/coder/factor_coder/config.py index 236275a3..1adb0d92 100644 --- a/rdagent/components/coder/factor_coder/config.py +++ b/rdagent/components/coder/factor_coder/config.py @@ -7,13 +7,16 @@ SELECT_METHOD = Literal["random", "scheduler"] class FactorImplementSettings(BaseSettings): - file_based_execution_data_folder: str = str( + factor_data_folder: str = str( (Path().cwd() / "git_ignore_folder" / "factor_implementation_source_data").absolute(), ) - file_based_execution_workspace: str = str( + factor_data_folder_debug: str = str( + (Path().cwd() / "git_ignore_folder" / "factor_implementation_source_data_debug").absolute(), + ) + factor_execution_workspace: str = str( (Path().cwd() / "git_ignore_folder" / "factor_implementation_workspace").absolute(), ) - implementation_execution_cache_location: str = str( + factor_cache_location: str = str( (Path().cwd() / "git_ignore_folder" / "factor_implementation_execution_cache").absolute(), ) enable_execution_cache: bool = True # whether to enable the execution cache diff --git a/rdagent/components/coder/factor_coder/factor.py b/rdagent/components/coder/factor_coder/factor.py index 1cf70dcd..aa4fecac 100644 --- a/rdagent/components/coder/factor_coder/factor.py +++ b/rdagent/components/coder/factor_coder/factor.py @@ -95,11 +95,11 @@ class FileBasedFactorImplementation(FBImplementation): def prepare(self, *args, **kwargs): self.workspace_path = Path( - FACTOR_IMPLEMENT_SETTINGS.file_based_execution_workspace, + FACTOR_IMPLEMENT_SETTINGS.factor_execution_workspace, ) / str(uuid.uuid4()) self.workspace_path.mkdir(exist_ok=True, parents=True) - def execute(self, store_result: bool = False) -> Tuple[str, pd.DataFrame]: + def execute(self, store_result: bool = False, data_type: str = "Debug") -> Tuple[str, pd.DataFrame]: """ execute the implementation and get the factor value by the following steps: 1. make the directory in workspace path @@ -120,14 +120,10 @@ class FileBasedFactorImplementation(FBImplementation): raise ValueError(self.FB_CODE_NOT_SET) with FileLock(self.workspace_path / "execution.lock"): if FACTOR_IMPLEMENT_SETTINGS.enable_execution_cache: - # NOTE: cache the result for the same code - target_file_name = md5_hash(self.code_dict["factor.py"]) - cache_file_path = ( - Path(FACTOR_IMPLEMENT_SETTINGS.implementation_execution_cache_location) / f"{target_file_name}.pkl" - ) - Path(FACTOR_IMPLEMENT_SETTINGS.implementation_execution_cache_location).mkdir( - exist_ok=True, parents=True - ) + # NOTE: cache the result for the same code and same data type + target_file_name = md5_hash(data_type + self.code_dict["factor.py"]) + cache_file_path = Path(FACTOR_IMPLEMENT_SETTINGS.factor_cache_location) / f"{target_file_name}.pkl" + Path(FACTOR_IMPLEMENT_SETTINGS.factor_cache_location).mkdir(exist_ok=True, parents=True) if cache_file_path.exists() and not self.raise_exception: cached_res = pickle.load(open(cache_file_path, "rb")) if store_result and cached_res[1] is not None: @@ -137,8 +133,14 @@ class FileBasedFactorImplementation(FBImplementation): if self.executed_factor_value_dataframe is not None: return self.FB_FROM_CACHE, self.executed_factor_value_dataframe - source_data_path = Path( - FACTOR_IMPLEMENT_SETTINGS.file_based_execution_data_folder, + source_data_path = ( + Path( + FACTOR_IMPLEMENT_SETTINGS.factor_data_folder_debug, + ) + if data_type == "Debug" + else Path( + FACTOR_IMPLEMENT_SETTINGS.factor_data_folder, + ) ) source_data_path.mkdir(exist_ok=True, parents=True) diff --git a/rdagent/components/coder/factor_coder/utils.py b/rdagent/components/coder/factor_coder/utils.py index dd705b29..283f7624 100644 --- a/rdagent/components/coder/factor_coder/utils.py +++ b/rdagent/components/coder/factor_coder/utils.py @@ -22,7 +22,7 @@ def get_data_folder_intro(): It is for preparing prompting message. """ content_l = [] - for p in Path(FACTOR_IMPLEMENT_SETTINGS.file_based_execution_data_folder).iterdir(): + for p in Path(FACTOR_IMPLEMENT_SETTINGS.factor_data_folder).iterdir(): if p.name.endswith(".h5"): df = pd.read_hdf(p) # get df.head() as string with full width diff --git a/rdagent/components/coder/model_coder/conf.py b/rdagent/components/coder/model_coder/conf.py index 74283e1a..70af7d29 100644 --- a/rdagent/components/coder/model_coder/conf.py +++ b/rdagent/components/coder/model_coder/conf.py @@ -8,10 +8,10 @@ class ModelImplSettings(BaseSettings): class Config: env_prefix = "MODEL_IMPL_" # Use MODEL_IMPL_ as prefix for environment variables - file_based_execution_workspace: str = str( + model_execution_workspace: str = str( (Path().cwd() / "git_ignore_folder" / "model_implementation_workspace").absolute(), ) - implementation_execution_cache_location: str = str( + model_cache_location: str = str( (Path().cwd() / "git_ignore_folder" / "model_implementation_execution_cache").absolute(), ) diff --git a/rdagent/components/coder/model_coder/model.py b/rdagent/components/coder/model_coder/model.py index 27d1f13c..7ddf634b 100644 --- a/rdagent/components/coder/model_coder/model.py +++ b/rdagent/components/coder/model_coder/model.py @@ -68,7 +68,7 @@ class ModelImplementation(FBImplementation): Prepare for the workspace; """ unique_id = uuid.uuid4() - self.workspace_path = Path(MODEL_IMPL_SETTINGS.file_based_execution_workspace) / f"M{unique_id}" + self.workspace_path = Path(MODEL_IMPL_SETTINGS.model_execution_workspace) / f"M{unique_id}" # start with `M` so that it can be imported via python self.workspace_path.mkdir(parents=True, exist_ok=True) @@ -84,10 +84,8 @@ class ModelImplementation(FBImplementation): if MODEL_IMPL_SETTINGS.enable_execution_cache: # NOTE: cache the result for the same code target_file_name = md5_hash(self.code_dict["model.py"]) - cache_file_path = ( - Path(MODEL_IMPL_SETTINGS.implementation_execution_cache_location) / f"{target_file_name}.pkl" - ) - Path(MODEL_IMPL_SETTINGS.implementation_execution_cache_location).mkdir(exist_ok=True, parents=True) + cache_file_path = Path(MODEL_IMPL_SETTINGS.model_cache_location) / f"{target_file_name}.pkl" + Path(MODEL_IMPL_SETTINGS.model_cache_location).mkdir(exist_ok=True, parents=True) if cache_file_path.exists(): return pickle.load(open(cache_file_path, "rb")) mod = get_module_by_module_path(str(self.workspace_path / "model.py")) diff --git a/rdagent/scenarios/qlib/task_generator/data.py b/rdagent/scenarios/qlib/task_generator/data.py index 8b4ed8cc..bc44f203 100644 --- a/rdagent/scenarios/qlib/task_generator/data.py +++ b/rdagent/scenarios/qlib/task_generator/data.py @@ -159,7 +159,7 @@ class QlibFactorRunner(TaskGenerator[QlibFactorExperiment]): for exp in exp_or_list: # Iterate over sub-implementations and execute them to get each factor data for implementation in exp.sub_implementations: - message, df = implementation.execute() + message, df = implementation.execute(data_type="All") # Check if factor generation was successful if df is not None: