first version of model runner and model feedback (#70)

* Implemented model.py

- Need to run within the RDAgent folder (relevant path)
- Each time copy a template & insert code & run qlib & store result back to experiment

* Create model.py

* Create conf.yaml

This is the sample conf.yaml to be copied each time.

This has gone several times of iteration and is now working for both tabular and Time-Series data.

* Create read_exp.py

This is to read the results within Qlib

* Create ReadMe.md

* Update model.py

* Create test_model.py

A testing file that separates model code generation and running&feedback section.

* move the template folder

* help xisen finish the model runner

* help xisen fix improve model feedback generation

* delete debug file

* rename readme.md

---------

Co-authored-by: Xisen Wang <118058822+Xisen-Wang@users.noreply.github.com>

rdagent/components/coder/model_coder/CoSTEER/evaluators.py

@@ -241,12 +241,12 @@ class ModelCoderEvaluator(Evaluator):
             )
         assert isinstance(target_task, ModelTask)
 
-        batch_size, num_features, num_timesteps = (
-            random.randint(6, 10),
-            random.randint(6, 10),
-            random.randint(6, 10),
-        )
-        input_value, param_init_value = random.random(), random.random()
+        # NOTE: Use fixed input to test the model to avoid randomness
+        batch_size = 8
+        num_features = 30
+        num_timesteps = 40
+        input_value = 0.4
+        param_init_value = 0.6
 
         assert isinstance(implementation, ModelImplementation)
         model_execution_feedback, gen_tensor = implementation.execute(

rdagent/components/coder/model_coder/CoSTEER/evolving_strategy.py

@@ -86,13 +86,12 @@ class ModelCoderEvolvingStrategy(EvolvingStrategy):
                     queried_similar_successful_knowledge_to_render = queried_similar_successful_knowledge_to_render[1:]
 
             code = json.loads(
-                APIBackend(use_chat_cache=True).build_messages_and_create_chat_completion(
+                APIBackend(use_chat_cache=False).build_messages_and_create_chat_completion(
                     user_prompt=user_prompt,
                     system_prompt=system_prompt,
                     json_mode=True,
                 ),
             )["code"]
-            # ast.parse(code)
             model_implementation = ModelImplementation(
                 target_task,
             )

rdagent/components/coder/model_coder/model.py

@@ -83,7 +83,9 @@ class ModelImplementation(FBImplementation):
         try:
             if MODEL_IMPL_SETTINGS.enable_execution_cache:
                 # NOTE: cache the result for the same code
-                target_file_name = md5_hash(self.code_dict["model.py"])
+                target_file_name = md5_hash(
+                    f"{batch_size}_{num_features}_{num_timesteps}_{input_value}_{param_init_value}_{self.code_dict['model.py']}"
+                )
                 cache_file_path = Path(MODEL_IMPL_SETTINGS.model_cache_location) / f"{target_file_name}.pkl"
                 Path(MODEL_IMPL_SETTINGS.model_cache_location).mkdir(exist_ok=True, parents=True)
                 if cache_file_path.exists():

rdagent/components/runner/__init__.py

@@ -0,0 +1,33 @@
+import pickle
+from pathlib import Path
+from typing import Tuple
+
+from rdagent.components.runner.conf import RUNNER_SETTINGS
+from rdagent.core.experiment import ASpecificExp, Experiment
+from rdagent.core.task_generator import TaskGenerator
+from rdagent.oai.llm_utils import md5_hash
+
+
+class CachedRunner(TaskGenerator[ASpecificExp]):
+    def get_cache_key(self, exp: Experiment) -> str:
+        all_tasks = []
+        for based_exp in exp.based_experiments:
+            all_tasks.extend(based_exp.sub_tasks)
+        all_tasks.extend(exp.sub_tasks)
+        task_info_list = [task.get_task_information() for task in all_tasks]
+        task_info_str = "\n".join(task_info_list)
+        return md5_hash(task_info_str)
+
+    def get_cache_result(self, exp: Experiment) -> Tuple[bool, object]:
+        task_info_key = self.get_cache_key(exp)
+        Path(RUNNER_SETTINGS.runner_cache_path).mkdir(parents=True, exist_ok=True)
+        cache_path = Path(RUNNER_SETTINGS.runner_cache_path) / f"{task_info_key}.pkl"
+        if cache_path.exists():
+            return True, pickle.load(open(cache_path, "rb"))
+        else:
+            return False, None
+
+    def dump_cache_result(self, exp: Experiment, result: object):
+        task_info_key = self.get_cache_key(exp)
+        cache_path = Path(RUNNER_SETTINGS.runner_cache_path) / f"{task_info_key}.pkl"
+        pickle.dump(result, open(cache_path, "wb"))

rdagent/components/runner/conf.py

@@ -0,0 +1,19 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+from dotenv import load_dotenv
+from pydantic_settings import BaseSettings
+
+# make sure that env variable is loaded while calling Config()
+load_dotenv(verbose=True, override=True)
+
+from pydantic_settings import BaseSettings
+
+
+class RunnerSettings(BaseSettings):
+    runner_cache_result: bool = True  # whether to cache the result of the docker execution
+    runner_cache_path: str = str(Path.cwd() / "runner_cache/")  # the path to store the cache
+
+
+RUNNER_SETTINGS = RunnerSettings()