Merge pull request #86 from microsoft/recover-branch-xisen-3

Model Optimisation Update 

After optimization, the rd agent loop for model in the qlib scenario should now work.

rdagent/app/qlib_rd_loop/RDAgent.py

@@ -0,0 +1,104 @@
+import pickle
+from rdagent.app.qlib_rd_loop.conf import PROP_SETTING
+from rdagent.core.developer import Developer
+from rdagent.core.exception import ModelEmptyException
+from rdagent.core.proposal import (
+    Hypothesis2Experiment,
+    HypothesisExperiment2Feedback,
+    HypothesisGen,
+    Trace,
+)
+from rdagent.core.scenario import Scenario
+from rdagent.core.utils import import_class
+from rdagent.log import rdagent_logger as logger
+
+# TODO: we can design a workflow that can automatically save session and traceback in the future
+class Model_RD_Agent:
+    def __init__(self):
+        self.scen: Scenario = import_class(PROP_SETTING.model_scen)()
+        self.hypothesis_gen: HypothesisGen = import_class(PROP_SETTING.model_hypothesis_gen)(self.scen)
+        self.hypothesis2experiment: Hypothesis2Experiment = import_class(PROP_SETTING.model_hypothesis2experiment)()
+        self.qlib_model_coder: Developer = import_class(PROP_SETTING.model_coder)(self.scen)
+        self.qlib_model_runner: Developer = import_class(PROP_SETTING.model_runner)(self.scen)
+        self.qlib_model_summarizer: HypothesisExperiment2Feedback = import_class(PROP_SETTING.model_summarizer)(self.scen)
+        self.trace = Trace(scen=self.scen)
+
+    def generate_hypothesis(self):
+        hypothesis = self.hypothesis_gen.gen(self.trace)
+        self.dump_objects(hypothesis=hypothesis, trace=self.trace, filename='step_hypothesis.pkl')
+        return hypothesis
+
+    def convert_hypothesis(self, hypothesis):
+        exp = self.hypothesis2experiment.convert(hypothesis, self.trace)
+        self.dump_objects(exp=exp, hypothesis=hypothesis, trace=self.trace, filename='step_experiment.pkl')
+        return exp
+
+    def generate_code(self, exp):
+        exp = self.qlib_model_coder.develop(exp)
+        self.dump_objects(exp=exp, trace=self.trace, filename='step_code.pkl')
+        return exp
+
+    def run_experiment(self, exp):
+        exp = self.qlib_model_runner.develop(exp)
+        self.dump_objects(exp=exp, trace=self.trace, filename='step_run.pkl')
+        return exp
+
+    def generate_feedback(self, exp, hypothesis):
+        feedback = self.qlib_model_summarizer.generateFeedback(exp, hypothesis, self.trace)
+        self.dump_objects(exp=exp, hypothesis=hypothesis, feedback=feedback, trace=self.trace, filename='step_feedback.pkl')
+        return feedback
+
+    def append_to_trace(self, hypothesis, exp, feedback):
+        self.trace.hist.append((hypothesis, exp, feedback))
+        self.dump_objects(trace=self.trace, filename='step_trace.pkl')
+
+    def dump_objects(self, exp=None, hypothesis=None, feedback=None, trace=None, filename='dumped_objects.pkl'):
+        with open(filename, 'wb') as f:
+            pickle.dump((exp, hypothesis, feedback, trace or self.trace), f)
+
+    def load_objects(self, filename):
+        with open(filename, 'rb') as f:
+            return pickle.load(f)
+
+def process_steps(agent):
+    # Load trace if available
+    try:
+        _, _, _, trace = agent.load_objects('step_trace.pkl')
+        agent.trace = trace
+        print(trace)
+    except FileNotFoundError:
+        pass
+
+    # Step 1: Generate hypothesis
+    try:
+        _, hypothesis, _, _ = agent.load_objects('step_hypothesis.pkl')
+    except FileNotFoundError:
+        hypothesis = agent.generate_hypothesis()
+
+    # Step 2: Convert hypothesis
+    try:
+        exp, _, _, _ = agent.load_objects('step_experiment.pkl')
+    except FileNotFoundError:
+        exp = agent.convert_hypothesis(hypothesis)
+
+    # Step 3: Generate code
+    try:
+        exp, _, _, _ = agent.load_objects('step_code.pkl')
+    except FileNotFoundError:
+        exp = agent.generate_code(exp)
+
+    # Step 4: Run experiment
+    try:
+        exp, _, _, _ = agent.load_objects('step_run.pkl')
+    except FileNotFoundError:
+        exp = agent.run_experiment(exp)
+
+    # Step 5: Generate feedback
+    feedback = agent.generate_feedback(exp, hypothesis)
+
+    # Step 6: Append to trace
+    agent.append_to_trace(hypothesis, exp, feedback)
+
+if __name__ == "__main__":
+    agent = Model_RD_Agent()
+    process_steps(agent)

rdagent/components/proposal/model_proposal.py

@@ -40,6 +40,7 @@ class ModelHypothesisGen(HypothesisGen):
                 targets="model",
                 scenario=self.scen.get_scenario_all_desc(),
                 hypothesis_output_format=context_dict["hypothesis_output_format"],
+                hypothesis_specification = context_dict["hypothesis_specification"],
             )
         )
         user_prompt = (

rdagent/components/proposal/prompts.yaml

@@ -3,10 +3,11 @@ hypothesis_gen:
     The user is trying to generate new hypothesis on the {{targets}} in data-driven research and development.
     The {{targets}} are used in a certain scenario, the scenario is as follows:
     {{ scenario }}
-    The user has made several hypothesis on this scenario and did several evaluation on them. The user will provide this information to you.
+    The user has made several hypothesis on this scenario and did several evaluation on them. The user will provide this information to you. Check if a new hypothesis has already been proposed. If it is already generated and you agree with it, just use it. If you don't agree, generate a better one.
     To help you generate new hypothesis, the user has prepared some additional information for you. You should use this information to help generate new {{targets}}.
-    Please generate the output following the format below:
+    Please generate the output following the format and specifications below:
     {{ hypothesis_output_format }}
+    Here are the specifications: {{ hypothesis_specification }}
   user_prompt: |-
     The user has made several hypothesis on this scenario and did several evaluation on them.
     The former hypothesis and the corresponding feedbacks are as follows:

rdagent/scenarios/qlib/docker/Dockerfile

@@ -1,4 +1,4 @@
-FROM pytorch/pytorch:latest
+FROM pytorch/pytorch: latest
 
 # For GPU support, please choose the proper tag from https://hub.docker.com/r/pytorch/pytorch/tags
 
@@ -20,4 +20,4 @@ RUN python -m pip install --upgrade cython
 RUN python -m pip install -e .
 
 RUN pip install catboost
-RUN pip install xgboost
+RUN pip install xgboost

rdagent/scenarios/qlib/experiment/model_template/conf.yaml

@@ -55,7 +55,7 @@ task:
         class: GeneralPTNN
         module_path: qlib.contrib.model.pytorch_general_nn
         kwargs:
-            n_epochs: 1
+            n_epochs: 10
             lr: 2e-3
             early_stop: 10
             batch_size: 800

rdagent/scenarios/qlib/experiment/prompts.yaml

@@ -101,11 +101,11 @@ qlib_model_output_format: |-
   The user will evaluate the shape of the output tensor so the tensor read from "output.pth" should be 8 numbers.
 
 qlib_model_simulator: |-
-  The models will be sent into Qlib to train and evaluate their performance in predicting future returns based on historical factor values.
+  The models will be sent into Qlib to train and evaluate their performance in predicting future returns. Hypothesis is improved upon checking the feedback on the results. 
   Qlib is an AI-oriented quantitative investment platform that aims to realize the potential, empower research, and create value using AI technologies in quantitative investment, from exploring ideas to implementing productions. Qlib supports diverse machine learning modeling paradigms, including supervised learning, market dynamics modeling, and reinforcement learning (RL).
   User will use Qlib to automatically perform the following tasks:
   1. Generate a baseline factor table.
   2. Train the model defined in your class Net to predict the next several days' returns based on the factor values.
   3. Build a portfolio based on the predicted returns using a specific strategy.
   4. Evaluate the portfolio's performance, including metrics such as return, Sharpe ratio, max drawdown, and others.
-  5. Iterate on model improvements based on performance evaluations and feedback.
+  5. Iterate on growing the hypothesis to enable model improvements based on performance evaluations and feedback.

rdagent/scenarios/qlib/prompts.yaml

@@ -1,7 +1,11 @@
 hypothesis_and_feedback: |-
   {% for hypothesis, experiment, feedback in trace.hist %}
   Hypothesis {{ loop.index }}: {{ hypothesis }}
-  Feedback: {{ feedback }}
+  Observation on the result with the hypothesis: {{ feedback.observations }}
+  Feedback on the original hypothesis:  {{ feedback.hypothesis_evaluation }}
+  New Feedback for Context (For you to agree or improve upon):  {{ feedback.new_hypothesis }}
+  Reasoning for new hypothesis:  {{ feedback.reason }}
+  Did changing to this hypothesis work? (focus on the change):  {{ feedback.decision }}
   {% endfor %}
 
 hypothesis_output_format: |-
@@ -11,6 +15,55 @@ hypothesis_output_format: |-
     "reason": "The reason why you generate this hypothesis."
   }
 
+model_hypothesis_specification: |-
+  Additional Specifications:
+    Hypotheses should grow and evolve based on the previous hypothesis. If there is no previous hypothesis, start with something simple. Gradually Build Up Upon previous hypothesis & feedbacks. 
+    Ensure that the hypothesis focuses on the architecture of a PyTorch model. Each hypothesis should address specific architectural choices such as the type of layers, activation functions, regularization techniques, and the overall structure of the model. Avoid hypotheses related to input features or optimization processes.
+
+  Sample Hypotheses (Only learn from the format as these are not the knowledge):
+    - "The model should use batch normalization to improve training stability."
+    - "The data has spatial dependencies, so we need a 3D CNN."
+    - "The model should include dropout layers to prevent overfitting."
+    - "The data exhibits long-term dependencies, so we need an LSTM/GRU model."
+    - "The model should have three layers. The model should be a ResNet."
+    - "The activation function should be ReLU for all hidden layers."
+    - "The model should include a fully connected layer at the end."
+    - "The model should use a combination of CNN and RNN layers for feature extraction and sequence modeling."
+
+  Remember: if there is no hypothesis, start with something simple like MLP.
+
+  Sample hypothesis evolution loop: (This is the entire loop, see what stage you are at. We want hypothesis to continue growing.) Levels include **Model Type**, **Layer Configuration**, **Activation Functions**, **Regularization Techniques**
+
+  1st Round Hypothesis: The model should be a CNN.
+
+  2nd Round Hypothesis (If first round worked): The model should be a CNN. The CNN should have 5 convolutional layers. (Reasoning: As CNN worked, we now specify the layers specification to grow the hypothesis deeper)
+
+  3rd Round Hypothesis (If second round worked): The model should be a CNN. The CNN should have 5 convolutional layers. Use Leaky ReLU activation for all layers. (Similar Reasoning & Continuing to Grow)
+
+  4th Round Hypothesis (If third round worked): The model should be a CNN. The CNN should have 5 convolutional layers. Use Leaky ReLU activation for all layers. Use dropout regularization with a rate of 0.5.
+
+  5th Round Hypothesis (If fourth round didn't work):  The model should be a CNN. The CNN should have 5 convolutional layers. Use Leaky ReLU activation for all layers. Use dropout regularization with a rate of 0.3. (Reasoning: As regularisation rate of 0.5 didn't work, we only change a new regularisation and keep the other elements that worked)    
+
+  Sample JSON Output:
+  [
+    {
+      "hypothesis": "The CNN should have 3 convolutional layers.",
+      "reason": "Three convolutional layers are often sufficient to capture spatial features in the data without making the model overly complex."
+    },
+    {
+      "hypothesis": "The CNN should have 5 convolutional layers.",
+      "reason": "Five convolutional layers can help in capturing more intricate spatial features, potentially improving the model's performance on complex datasets."
+    },
+    {
+      "hypothesis": "The CNN should use residual connections.",
+      "reason": "Residual connections can help mitigate the vanishing gradient problem, facilitating the training of deeper networks."
+    },
+    {
+      "hypothesis": "The CNN should include a fully connected layer at the end.",
+      "reason": "A fully connected layer at the end can integrate features extracted by the convolutional layers and provide a robust output for classification tasks."
+    }
+  ]
+
 factor_experiment_output_format: |-
   The output should follow JSON format. The schema is as follows:
   {
@@ -35,20 +88,20 @@ factor_experiment_output_format: |-
 
 model_experiment_output_format: |-
   So far please only design one model to test the hypothesis!
-  The output should follow JSON format. The schema is as follows:
+  The output should follow JSON format. The schema is as follows: 
   {
-      "model name 1": {
-          "description": "detailed description of model 1",
-          "formulation": "latex formulation of model 1",
-          "variables": {
-              "variable or function name 1": "description of variable or function 1",
-              "variable or function name 2": "description of variable or function 2"
-          }
-          "model_type": "type of model 1, Tabular or TimesSeries" # Should be one of "Tabular" or "TimeSeries"
-      }
-      # Don't add ellipsis (...) or any filler text that might cause JSON parsing errors here!
+    "model_name (The name of the model)": {
+        "description": "A detailed description of the model",
+        "architecture": "A detailed description of the model's architecture, e.g., neural network layers or tree structures",
+        "hyperparameters": {
+            "hyperparameter_name_1": "value of hyperparameter 1",
+            "hyperparameter_name_2": "value of hyperparameter 2",
+            "hyperparameter_name_3": "value of hyperparameter 3"
+        },
+        "model_type": "Tabular or TimeSeries"  # Should be one of "Tabular" or "TimeSeries"
+    }
   }
-
+  
 factor_feedback_generation:
   system: |-
     You are a professional result analysis assistant on data driven R&D.
@@ -98,14 +151,14 @@ model_feedback_generation:
     We are in an experiment of finding hypothesis and validating or rejecting them so that in the end we have a powerful model generated.
     Here are the context: {{context}}. 
 
-    {% if last_hypothesis %}
+    {% if last_hypothesis %} 
     Last Round Information:
     Hypothesis: {{last_hypothesis.hypothesis}}
     Task: {{last_task}}
     Result: {{last_result}}
     {% else %}
-    This is the first round. No previous information available.
-    {% endif %}
+    This is the first round. No previous information available. As long as the performance is reasonable, treat it as successful. Do not set the threshold too high.  
+    {% endif %} 
     
     Now let's come to this round. You will receive the result and you will evaluate if the performance increases or decreases. 
     Hypothesis: {{hypothesis.hypothesis}}

rdagent/scenarios/qlib/proposal/model_proposal.py

@@ -31,8 +31,9 @@ class QlibModelHypothesisGen(ModelHypothesisGen):
         )
         context_dict = {
             "hypothesis_and_feedback": hypothesis_feedback,
-            "RAG": ...,
+            "RAG": "In Quantitative Finance, market data could be time-series, and GRU model/LSTM model are suitable for them. Do not generate GNN model as for now.",
             "hypothesis_output_format": prompt_dict["hypothesis_output_format"],
+            "hypothesis_specification": prompt_dict["model_hypothesis_specification"]
         }
         return context_dict, True
 
@@ -73,10 +74,10 @@ class QlibModelHypothesis2Experiment(ModelHypothesis2Experiment):
         tasks = []
         for model_name in response_dict:
             description = response_dict[model_name]["description"]
-            formulation = response_dict[model_name]["formulation"]
-            variables = response_dict[model_name]["variables"]
+            architecture = response_dict[model_name]["architecture"]
+            hyperparameters = response_dict[model_name]["hyperparameters"]
             model_type = response_dict[model_name]["model_type"]
-            tasks.append(ModelTask(model_name, description, formulation, variables, model_type))
+            tasks.append(ModelTask(model_name, description, architecture, hyperparameters, model_type))
         exp = QlibModelExperiment(tasks)
         exp.based_experiments = [t[1] for t in trace.hist if t[2]]
         return exp