Merge pull request #106 from microsoft/Continous-Update-Branch

Further Optimised Workflow, Enabled Graph Input, Stored rich-text-description, Logged Back-Testing, Enabled Concise_Hypothesis

rdagent/components/coder/model_coder/model.py

@@ -16,19 +16,23 @@ from rdagent.utils import get_module_by_module_path
 
 class ModelTask(Task):
     def __init__(
-        self, name: str, description: str, formulation: str, variables: Dict[str, str], model_type: Optional[str] = None
+        self, name: str, description: str, formulation: str, architecture: str, variables: Dict[str, str], hyperparameters: Dict[str, str], model_type: Optional[str] = None
     ) -> None:
         self.name: str = name
         self.description: str = description
         self.formulation: str = formulation
+        self.architecture: str = architecture
         self.variables: str = variables
-        self.model_type: str = model_type  # Tabular for tabular model, TimesSeries for time series model
+        self.hyperparameters: str = hyperparameters
+        self.model_type: str = model_type  # Tabular for tabular model, TimesSeries for time series model, Graph for graph model 
 
     def get_task_information(self):
         return f"""name: {self.name}
 description: {self.description}
 formulation: {self.formulation}
+architecture: {self.architecture}
 variables: {self.variables}
+hyperparameters: {self.hyperparameters}
 model_type: {self.model_type}
 """
 
@@ -65,6 +69,8 @@ class ModelFBWorkspace(FBWorkspace):
         batch_size: int = 8,
         num_features: int = 10,
         num_timesteps: int = 4,
+        num_nodes: int = 50,
+        num_edges: int = 100,
         input_value: float = 1.0,
         param_init_value: float = 1.0,
     ):
@@ -85,21 +91,37 @@ class ModelFBWorkspace(FBWorkspace):
             if self.target_task.model_type == "Tabular":
                 input_shape = (batch_size, num_features)
                 m = model_cls(num_features=input_shape[1])
+                data = torch.full(input_shape, input_value)
             elif self.target_task.model_type == "TimeSeries":
                 input_shape = (batch_size, num_features, num_timesteps)
                 m = model_cls(num_features=input_shape[1], num_timesteps=input_shape[2])
-            data = torch.full(input_shape, input_value)
+                data = torch.full(input_shape, input_value)
+            elif self.target_task.model_type == "Graph":
+                node_feature = torch.randn(batch_size, num_nodes, num_features)
+                edge_index = torch.randint(0, num_nodes, (2, num_edges))
+                m = model_cls(num_nodes=num_nodes, num_features=num_features)
+                data = (node_feature, edge_index)
+            else:
+                raise ValueError(f"Unsupported model type: {self.target_task.model_type}")
 
-            # initialize all parameters of `m` to `param_init_value`
+            # Initialize all parameters of `m` to `param_init_value`
             for _, param in m.named_parameters():
                 param.data.fill_(param_init_value)
-            out = m(data)
+            
+            # Execute the model
+            if self.target_task.model_type == "Graph":
+                out = m(*data)
+            else:
+                out = m(data)
+            
             execution_model_output = out.cpu().detach()
             execution_feedback_str = f"Execution successful, output tensor shape: {execution_model_output.shape}"
+            
             if MODEL_IMPL_SETTINGS.enable_execution_cache:
                 pickle.dump((execution_feedback_str, execution_model_output), open(cache_file_path, "wb"))
+            
             return execution_feedback_str, execution_model_output
-
+        
         except Exception as e:
             return f"Execution error: {e}", None
 

rdagent/components/coder/model_coder/prompts.yaml

@@ -1,11 +1,42 @@
 extract_model_formulation_system: |-
-    offer description of the proposed model in this paper, write a latex formula with variable of the model. the format should be like 
-    "Model Name": {
-        "description": "",
-        "formulation": "",
+    offer description of the proposed model in this paper, write a latex formula with variable as well as the architecture of the model. the format should be like 
+    {
+    "model_name (The name of the model)": {
+        "description": "A detailed description of the model",
+        "formulation": "A LaTeX formula representing the model's formulation",
+        "architecture": "A detailed description of the model's architecture, e.g., neural network layers or tree structures",
         "variables": {
-        "\\hat{y}_u": "The predicted output for node u",
-        }"
+            "\\hat{y}_u": "The predicted output for node u",
+            "variable_name_2": "Description of variable 2",
+            "variable_name_3": "Description of variable 3"
+        },
+        "hyperparameters": {
+            "hyperparameter_name_1": "value of hyperparameter 1",
+            "hyperparameter_name_2": "value of hyperparameter 2",
+            "hyperparameter_name_3": "value of hyperparameter 3"
+        },
+        "model_type": "Tabular or TimeSeries or Graph"  # Should be one of "Tabular", "TimeSeries", or "Graph"
+    }
+    }
+    Eg. 
+    {
+    "ABC Model": {
+        "description": "A detailed description of the model",
+        "formulation": "A LaTeX formula representing the model's formulation",
+        "architecture": "A detailed description of the model's architecture, e.g., neural network layers or tree structures",
+        "variables": {
+            "\\hat{y}_u": "The predicted output for node u",
+            "variable_name_2": "Description of variable 2",
+            "variable_name_3": "Description of variable 3"
+        },
+        "hyperparameters": {
+            "hyperparameter_name_1": "value of hyperparameter 1",
+            "hyperparameter_name_2": "value of hyperparameter 2",
+            "hyperparameter_name_3": "value of hyperparameter 3"
+        },
+        "model_type": "Tabular or TimeSeries or Graph"  # Should be one of "Tabular", "TimeSeries", or "Graph"
+    }
+    }
     such format content should be begin with ```json and end with ``` and the content should be in json format.
 
 evolving_strategy_model_coder:

rdagent/components/coder/model_coder/task_loader.py

@@ -105,8 +105,9 @@ class ModelExperimentLoaderFromDict(ModelTaskLoader):
                 name=model_name,
                 description=model_data["description"],
                 formulation=model_data["formulation"],
+                architecture=model_data["architecture"],
                 variables=model_data["variables"],
-                key=model_name,
+                model_type=model_data["model_type"],
             )
             task_l.append(task)
         return QlibModelExperiment(sub_tasks=task_l)

rdagent/components/proposal/prompts.yaml

@@ -10,7 +10,7 @@ hypothesis_gen:
     Here are the specifications: {{ hypothesis_specification }}
   user_prompt: |-
     The user has made several hypothesis on this scenario and did several evaluation on them.
-    The former hypothesis and the corresponding feedbacks are as follows:
+    The former hypothesis and the corresponding feedbacks are as follows (focus on the last one & the new hypothesis that it provides and reasoning to see if you agree):
     {{ hypothesis_and_feedback }}
     To help you generate new {{targets}}, we have prepared the following information for you:
     {{ RAG }}

rdagent/core/proposal.py

@@ -23,9 +23,10 @@ class Hypothesis:
     - Belief
     """
 
-    def __init__(self, hypothesis: str, reason: str) -> None:
+    def __init__(self, hypothesis: str, reason: str, concise_reason = str) -> None:
         self.hypothesis: str = hypothesis
         self.reason: str = reason
+        self.concise_reason: str = concise_reason
 
     def __str__(self) -> str:
         return f"""Hypothesis: {self.hypothesis}

rdagent/core/scenario.py

@@ -27,6 +27,15 @@ class Scenario(ABC):
     def simulator(self) -> str:
         """Simulator description"""
 
+    @property
+    @abstractmethod
+    def rich_style_description(self) -> str:
+        """Rich style description to present"""
+
     @abstractmethod
     def get_scenario_all_desc(self) -> str:
         """Combine all the description together"""
+
+
+
+    

rdagent/scenarios/qlib/experiment/model_experiment.py

@@ -38,6 +38,10 @@ class QlibModelScenario(Scenario):
     @property
     def simulator(self) -> str:
         return prompt_dict["qlib_model_simulator"]
+    
+    @property
+    def rich_style_description(self)->str:
+        return "Below is QlibModel Evolving Automatic R&D Demo."
 
     def get_scenario_all_desc(self) -> str:
         return f"""Background of the scenario:

rdagent/scenarios/qlib/experiment/model_template/conf.yaml

@@ -62,7 +62,7 @@ task:
             metric: loss
             loss: mse
             n_jobs: 20
-            GPU: 3
+            GPU: 2
             # loss: mse
             # lr: 0.002
             # optimizer: adam

rdagent/scenarios/qlib/experiment/model_template/read_exp_res.py

@@ -39,7 +39,11 @@ else:
     # Load the specified file from the latest recorder
     metrics = pd.Series(latest_recorder.list_metrics())
 
+
     output_path = Path(__file__).resolve().parent / "qlib_res.csv"
     metrics.to_csv(output_path)
 
     print(f"Output has been saved to {output_path}")
+
+    ret_data_frame = latest_recorder.load_object("portfolio_analysis/report_normal_1day.pkl")
+    ret_data_frame.to_pickle("ret.pkl")

rdagent/scenarios/qlib/experiment/prompts.yaml

@@ -48,7 +48,7 @@ qlib_factor_simulator: |-
 
 qlib_model_background: |-
   The model is a machine learning or deep learning structure used in quantitative investment to predict the returns and risks of a portfolio or a single asset. Models are employed by investors to generate forecasts based on historical data and identified factors, which are central to many quantitative investment strategies.
-  Each model takes the factors as input and predicts the future returns.
+  Each model takes the factors as input and predicts the future returns. Usually, the bigger the model is, the better the performance would be.
   The model is defined in the following parts:
   1. Name: The name of the model.
   2. Description: The description of the model.

rdagent/scenarios/qlib/experiment/workspace.py

@@ -30,6 +30,9 @@ class QlibFBWorkspace(FBWorkspace):
             env=run_env,
         )
 
+        ret_df = pd.read_pickle(self.workspace_path / "ret.pkl")
+        logger.log_object(ret_df, tag="returns")  # TODO: tag
+
         csv_path = self.workspace_path / "qlib_res.csv"
 
         if not csv_path.exists():

rdagent/scenarios/qlib/prompts.yaml

@@ -12,7 +12,8 @@ hypothesis_output_format: |-
   The output should follow JSON format. The schema is as follows:
   {
     "hypothesis": "The new hypothesis generated based on the information provided.",
-    "reason": "The reason why you generate this hypothesis."
+    "reason": "The reason why you generate this hypothesis.",
+    "concise_reason": One line summary that focuses on the justification for the change that leads to the hypothesis (like a part of a knowledge that we are building),
   }
 
 model_hypothesis_specification: |-
@@ -140,7 +141,13 @@ model_experiment_output_format: |-
   {
     "model_name 1 (The name of the model)": {
         "description": "A detailed description of the model",
+        "formulation": "A LaTeX formula representing the model's formulation",
         "architecture": "A detailed description of the model's architecture, e.g., neural network layers or tree structures",
+        "variables": {
+            "\\hat{y}_u": "The predicted output for node u",
+            "variable_name_2": "Description of variable 2",
+            "variable_name_3": "Description of variable 3"
+        },
         "hyperparameters": {
             "hyperparameter_name_1": "value of hyperparameter 1",
             "hyperparameter_name_2": "value of hyperparameter 2",
@@ -218,6 +225,8 @@ model_feedback_generation:
       "Reasoning": "Provide reasoning for the hypothesis here.",
       "Decision": <true or false>,
     }
+
+    Focus on the changes in hypothesis and justify why do hypothesis evolve like this. Also, increase complexity as the hypothesis evolves  (give more layers, more neurons, and etc)
     
     Logic for generating a new hypothesis: If the previous hypothesis works, try to inherit from it and grow deeper. If the previous hypotheis doesn't work, try to make changes in the current level.
 

rdagent/scenarios/qlib/proposal/model_proposal.py

@@ -39,7 +39,7 @@ class QlibModelHypothesisGen(ModelHypothesisGen):
 
     def convert_response(self, response: str) -> ModelHypothesis:
         response_dict = json.loads(response)
-        hypothesis = QlibModelHypothesis(hypothesis=response_dict["hypothesis"], reason=response_dict["reason"])
+        hypothesis = QlibModelHypothesis(hypothesis=response_dict["hypothesis"], reason=response_dict["reason"], concise_reason=response_dict["concise_reason"])
         return hypothesis
 
 
@@ -74,10 +74,12 @@ class QlibModelHypothesis2Experiment(ModelHypothesis2Experiment):
         tasks = []
         for model_name in response_dict:
             description = response_dict[model_name]["description"]
+            formulation = response_dict[model_name]["formulation"]
             architecture = response_dict[model_name]["architecture"]
+            variables = response_dict[model_name]["variables"]
             hyperparameters = response_dict[model_name]["hyperparameters"]
             model_type = response_dict[model_name]["model_type"]
-            tasks.append(ModelTask(model_name, description, architecture, hyperparameters, model_type))
+            tasks.append(ModelTask(model_name, description, formulation, architecture, variables, hyperparameters, model_type))
         exp = QlibModelExperiment(tasks)
         exp.based_experiments = [t[1] for t in trace.hist if t[2]]
         return exp