feat: use unified pickle cacher & move llm config into a isolated config (#424)

* simplify RDAgent conf

* add unified cacher(untested)

* fix small bugs

* fix a bug

* fix a small bug in runner

* use hash_key = None to skip cache

* fix CI

* in factor execution, ignore cache when raise exception

* add file locker to avoid mp calling

* fix CI

* use function __module__ name as folder in cache

rdagent/components/coder/model_coder/CoSTEER/evaluators.py

@@ -19,6 +19,7 @@ from rdagent.core.experiment import Task, Workspace
 from rdagent.core.prompts import Prompts
 from rdagent.core.utils import multiprocessing_wrapper
 from rdagent.log import rdagent_logger as logger
+from rdagent.oai.llm_conf import LLM_SETTINGS
 from rdagent.oai.llm_utils import APIBackend
 
 evaluate_prompts = Prompts(file_path=Path(__file__).parent.parent / "prompts.yaml")
@@ -90,9 +91,11 @@ class ModelCodeEvaluator(Evaluator):
             Environment(undefined=StrictUndefined)
             .from_string(evaluate_prompts["evaluator_code_feedback"]["system"])
             .render(
-                scenario=self.scen.get_scenario_all_desc(target_task)
-                if self.scen is not None
-                else "No scenario description."
+                scenario=(
+                    self.scen.get_scenario_all_desc(target_task)
+                    if self.scen is not None
+                    else "No scenario description."
+                )
             )
         )
 
@@ -116,7 +119,7 @@ class ModelCodeEvaluator(Evaluator):
                     user_prompt=user_prompt,
                     system_prompt=system_prompt,
                 )
-                > RD_AGENT_SETTINGS.chat_token_limit
+                > LLM_SETTINGS.chat_token_limit
             ):
                 execution_feedback_to_render = execution_feedback_to_render[len(execution_feedback_to_render) // 2 :]
             else:
@@ -150,9 +153,11 @@ class ModelFinalEvaluator(Evaluator):
             Environment(undefined=StrictUndefined)
             .from_string(evaluate_prompts["evaluator_final_feedback"]["system"])
             .render(
-                scenario=self.scen.get_scenario_all_desc(target_task)
-                if self.scen is not None
-                else "No scenario description."
+                scenario=(
+                    self.scen.get_scenario_all_desc(target_task)
+                    if self.scen is not None
+                    else "No scenario description."
+                )
             )
         )
 
@@ -176,7 +181,7 @@ class ModelFinalEvaluator(Evaluator):
                     user_prompt=user_prompt,
                     system_prompt=system_prompt,
                 )
-                > RD_AGENT_SETTINGS.chat_token_limit
+                > LLM_SETTINGS.chat_token_limit
             ):
                 execution_feedback_to_render = execution_feedback_to_render[len(execution_feedback_to_render) // 2 :]
             else:

rdagent/components/coder/model_coder/CoSTEER/evolving_strategy.py

@@ -20,6 +20,7 @@ from rdagent.core.conf import RD_AGENT_SETTINGS
 from rdagent.core.evolving_framework import EvolvingStrategy
 from rdagent.core.prompts import Prompts
 from rdagent.core.utils import multiprocessing_wrapper
+from rdagent.oai.llm_conf import LLM_SETTINGS
 from rdagent.oai.llm_utils import APIBackend
 from rdagent.scenarios.kaggle.experiment.kaggle_experiment import KG_MODEL_MAPPING
 
@@ -100,7 +101,7 @@ class ModelCoderEvolvingStrategy(EvolvingStrategy):
                         user_prompt=user_prompt,
                         system_prompt=system_prompt,
                     )
-                    < RD_AGENT_SETTINGS.chat_token_limit
+                    < LLM_SETTINGS.chat_token_limit
                 ):
                     break
                 elif len(queried_former_failed_knowledge_to_render) > 1:

rdagent/components/coder/model_coder/conf.py

@@ -10,10 +10,6 @@ class ModelImplSettings(BaseSettings):
 
     coder_use_cache: bool = False
 
-    cache_location: str = str(
-        (Path().cwd() / "git_ignore_folder" / "model_implementation_execution_cache").absolute(),
-    )
-
     knowledge_base_path: Union[str, None] = None
     new_knowledge_base_path: Union[str, None] = None
 
@@ -23,7 +19,5 @@ class ModelImplSettings(BaseSettings):
     query_similar_success_limit: int = 5
     fail_task_trial_limit: int = 20
 
-    enable_execution_cache: bool = True  # whether to enable the execution cache
-
 
 MODEL_IMPL_SETTINGS = ModelImplSettings()

rdagent/components/coder/model_coder/model.py

@@ -6,6 +6,7 @@ from typing import Dict, Optional
 
 from rdagent.components.coder.model_coder.conf import MODEL_IMPL_SETTINGS
 from rdagent.core.experiment import Experiment, FBWorkspace, Task
+from rdagent.core.utils import cache_with_pickle
 from rdagent.oai.llm_utils import md5_hash
 from rdagent.utils.env import KGDockerEnv, QTDockerEnv
 
@@ -71,6 +72,21 @@ class ModelFBWorkspace(FBWorkspace):
         (version 2) for kaggle we'll make a script to call the fit and predict function in the implementation in file `model.py` after setting the cwd into the directory
     """
 
+    def hash_func(
+        self,
+        batch_size: int = 8,
+        num_features: int = 10,
+        num_timesteps: int = 4,
+        num_edges: int = 20,
+        input_value: float = 1.0,
+        param_init_value: float = 1.0,
+    ) -> str:
+        target_file_name = f"{batch_size}_{num_features}_{num_timesteps}_{input_value}_{param_init_value}"
+        for code_file_name in sorted(list(self.code_dict.keys())):
+            target_file_name = f"{target_file_name}_{self.code_dict[code_file_name]}"
+        return md5_hash(target_file_name)
+
+    @cache_with_pickle(hash_func)
     def execute(
         self,
         batch_size: int = 8,
@@ -82,17 +98,6 @@ class ModelFBWorkspace(FBWorkspace):
     ):
         super().execute()
         try:
-            if MODEL_IMPL_SETTINGS.enable_execution_cache:
-                # NOTE: cache the result for the same code
-                target_file_name = f"{batch_size}_{num_features}_{num_timesteps}_{input_value}_{param_init_value}"
-                for code_file_name in sorted(list(self.code_dict.keys())):
-                    target_file_name = f"{target_file_name}_{self.code_dict[code_file_name]}"
-                target_file_name = md5_hash(target_file_name)
-                cache_file_path = Path(MODEL_IMPL_SETTINGS.cache_location) / f"{target_file_name}.pkl"
-                Path(MODEL_IMPL_SETTINGS.cache_location).mkdir(exist_ok=True, parents=True)
-                if cache_file_path.exists():
-                    return pickle.load(open(cache_file_path, "rb"))
-
             qtde = QTDockerEnv() if self.target_task.version == 1 else KGDockerEnv()
             qtde.prepare()
 
@@ -121,12 +126,6 @@ PARAM_INIT_VALUE = {param_init_value}
                 raise RuntimeError(f"Error in running the model code: {log}")
             [execution_feedback_str, execution_model_output] = results
 
-            if MODEL_IMPL_SETTINGS.enable_execution_cache:
-                pickle.dump(
-                    (execution_feedback_str, execution_model_output),
-                    open(cache_file_path, "wb"),
-                )
-
         except Exception as e:
             execution_feedback_str = f"Execution error: {e}\nTraceback: {traceback.format_exc()}"
             execution_model_output = None