From 82a2a5d5b4d2cfc274ca9c9c7c0cc7d2986b6ef0 Mon Sep 17 00:00:00 2001 From: Xu Yang Date: Tue, 30 Jul 2024 19:38:37 +0800 Subject: [PATCH] refine the graph model interface (#136) --- .../model_extraction_and_code/prompts.yaml | 6 ++--- .../coder/factor_coder/prompts.yaml | 4 +++ rdagent/components/coder/model_coder/model.py | 25 +++++++++++++------ .../components/coder/model_coder/prompts.yaml | 2 ++ 4 files changed, 26 insertions(+), 11 deletions(-) diff --git a/rdagent/app/model_extraction_and_code/prompts.yaml b/rdagent/app/model_extraction_and_code/prompts.yaml index ddd01e50..2bbe894e 100644 --- a/rdagent/app/model_extraction_and_code/prompts.yaml +++ b/rdagent/app/model_extraction_and_code/prompts.yaml @@ -32,14 +32,14 @@ general_model_interface: |- The input shape to a tabular model is (batch_size, num_features). The input shape to a time series model is (batch_size, num_features, num_timesteps). The input to a graph model are two tensors. - node_feature: a tensor of shape (batch_size, num_nodes, num_features) + node_feature: a tensor of shape (batch_size, num_features) edge_index: a tensor of shape (2, num_edges) The batch_size is a dynamic value which is determined by the input of the forward function. The output shape of the model should be (batch_size, 1). - The "num_features", "num_timesteps", and "num_nodes" are static and will be provided to the model through the init function. + The "num_features", "num_timesteps" are static and will be provided to the model through the init function. User will initialize the tabular model with the following code: @@ -51,7 +51,7 @@ general_model_interface: |- User will initialize the graph model with the following code: - model = model_cls(num_features=num_features, num_nodes=num_nodes) + model = model_cls(num_features=num_features) No other parameters will be passed to the model, so give other parameters a default value or make them static. diff --git a/rdagent/components/coder/factor_coder/prompts.yaml b/rdagent/components/coder/factor_coder/prompts.yaml index 61d8b69e..66d2c5bf 100644 --- a/rdagent/components/coder/factor_coder/prompts.yaml +++ b/rdagent/components/coder/factor_coder/prompts.yaml @@ -14,6 +14,8 @@ evaluator_code_feedback_v1_system: |- If the ground truth code is not provided, your critic should consider checking whether the user's code is reasonable and correct. Notice that your critics are not for user to debug the code. They are sent to the coding agent to correct the code. So don't give any following items for the user to check like "Please check the code line XXX". + + You suggestion should not include any code, just some clear and short suggestions. Please point out very critical issues in your response, ignore non-important issues to avoid confusion. If no big issue found in the code, you can response "No critics found". You should provide the suggestion to each of your critic to help the user improve the code. Please response the critic in the following format. Here is an example structure for the output: critic 1: The critic message to critic 1 @@ -129,6 +131,8 @@ evolving_strategy_error_summary_v2_system: |- The user has found some tasks that met similar errors, and their final correct solutions. Please refer to these similar errors and their solutions, provide some clear, short and accurate critics that might help you solve the issues in your code. + You suggestion should not include any code, just some clear and short suggestions. Please point out very critical issues in your response, ignore non-important issues to avoid confusion. If no big issue found in the code, you can response "No critics found". + Please response the critic in the following format. Here is an example structure for the output: critic 1: The critic message to critic 1 critic 2: The critic message to critic 2 diff --git a/rdagent/components/coder/model_coder/model.py b/rdagent/components/coder/model_coder/model.py index 7d680185..a1e53ee9 100644 --- a/rdagent/components/coder/model_coder/model.py +++ b/rdagent/components/coder/model_coder/model.py @@ -1,6 +1,7 @@ import json import pickle import site +import traceback import uuid from pathlib import Path from typing import Any, Dict, Optional @@ -78,8 +79,7 @@ class ModelFBWorkspace(FBWorkspace): batch_size: int = 8, num_features: int = 10, num_timesteps: int = 4, - num_nodes: int = 50, - num_edges: int = 100, + num_edges: int = 20, input_value: float = 1.0, param_init_value: float = 1.0, ): @@ -106,9 +106,9 @@ class ModelFBWorkspace(FBWorkspace): m = model_cls(num_features=input_shape[1], num_timesteps=input_shape[2]) data = torch.full(input_shape, input_value) elif self.target_task.model_type == "Graph": - node_feature = torch.randn(batch_size, num_nodes, num_features) - edge_index = torch.randint(0, num_nodes, (2, num_edges)) - m = model_cls(num_nodes=num_nodes, num_features=num_features) + node_feature = torch.randn(batch_size, num_features) + edge_index = torch.randint(0, batch_size, (2, num_edges)) + m = model_cls(num_features=num_features) data = (node_feature, edge_index) else: raise ValueError(f"Unsupported model type: {self.target_task.model_type}") @@ -129,10 +129,19 @@ class ModelFBWorkspace(FBWorkspace): if MODEL_IMPL_SETTINGS.enable_execution_cache: pickle.dump((execution_feedback_str, execution_model_output), open(cache_file_path, "wb")) - return execution_feedback_str, execution_model_output - except Exception as e: - return f"Execution error: {e}", None + execution_feedback_str = f"Execution error: {e}\nTraceback: {traceback.format_exc()}" + execution_model_output = None + + code_path = self.workspace_path / f"model.py" + execution_feedback_str = execution_feedback_str.replace(str(code_path.parent.absolute()), r"/path/to").replace( + str(site.getsitepackages()[0]), r"/path/to/site-packages" + ) + if len(execution_feedback_str) > 2000: + execution_feedback_str = ( + execution_feedback_str[:1000] + "....hidden long error message...." + execution_feedback_str[-1000:] + ) + return execution_feedback_str, execution_model_output ModelExperiment = Experiment diff --git a/rdagent/components/coder/model_coder/prompts.yaml b/rdagent/components/coder/model_coder/prompts.yaml index b5f069a7..b914a120 100644 --- a/rdagent/components/coder/model_coder/prompts.yaml +++ b/rdagent/components/coder/model_coder/prompts.yaml @@ -106,6 +106,8 @@ evaluator_code_feedback: Notice that your critics are not for user to debug the code. They are sent to the coding agent to correct the code. So don't give any following items for the user to check like "Please check the code line XXX". + You suggestion should not include any code, just some clear and short suggestions. Please point out very critical issues in your response, ignore non-important issues to avoid confusion. If no big issue found in the code, you can response "No critics found". + You should provide the suggestion to each of your critic to help the user improve the code. Please response the critic in the following format. Here is an example structure for the output: critic 1: The critic message to critic 1 critic 2: The critic message to critic 2