diff --git a/rdagent/components/coder/model_coder/model.py b/rdagent/components/coder/model_coder/model.py index 67fe2609..eeff3d7c 100644 --- a/rdagent/components/coder/model_coder/model.py +++ b/rdagent/components/coder/model_coder/model.py @@ -16,19 +16,23 @@ from rdagent.utils import get_module_by_module_path class ModelTask(Task): def __init__( - self, name: str, description: str, formulation: str, variables: Dict[str, str], model_type: Optional[str] = None + self, name: str, description: str, formulation: str, architecture: str, variables: Dict[str, str], hyperparameters: Dict[str, str], model_type: Optional[str] = None ) -> None: self.name: str = name self.description: str = description self.formulation: str = formulation + self.architecture: str = architecture self.variables: str = variables - self.model_type: str = model_type # Tabular for tabular model, TimesSeries for time series model + self.hyperparameters: str = hyperparameters + self.model_type: str = model_type # Tabular for tabular model, TimesSeries for time series model, Graph for graph model def get_task_information(self): return f"""name: {self.name} description: {self.description} formulation: {self.formulation} +architecture: {self.architecture} variables: {self.variables} +hyperparameters: {self.hyperparameters} model_type: {self.model_type} """ @@ -65,6 +69,8 @@ class ModelFBWorkspace(FBWorkspace): batch_size: int = 8, num_features: int = 10, num_timesteps: int = 4, + num_nodes: int = 50, + num_edges: int = 100, input_value: float = 1.0, param_init_value: float = 1.0, ): @@ -85,21 +91,37 @@ class ModelFBWorkspace(FBWorkspace): if self.target_task.model_type == "Tabular": input_shape = (batch_size, num_features) m = model_cls(num_features=input_shape[1]) + data = torch.full(input_shape, input_value) elif self.target_task.model_type == "TimeSeries": input_shape = (batch_size, num_features, num_timesteps) m = model_cls(num_features=input_shape[1], num_timesteps=input_shape[2]) - data = torch.full(input_shape, input_value) + data = torch.full(input_shape, input_value) + elif self.target_task.model_type == "Graph": + node_feature = torch.randn(batch_size, num_nodes, num_features) + edge_index = torch.randint(0, num_nodes, (2, num_edges)) + m = model_cls(num_nodes=num_nodes, num_features=num_features) + data = (node_feature, edge_index) + else: + raise ValueError(f"Unsupported model type: {self.target_task.model_type}") - # initialize all parameters of `m` to `param_init_value` + # Initialize all parameters of `m` to `param_init_value` for _, param in m.named_parameters(): param.data.fill_(param_init_value) - out = m(data) + + # Execute the model + if self.target_task.model_type == "Graph": + out = m(*data) + else: + out = m(data) + execution_model_output = out.cpu().detach() execution_feedback_str = f"Execution successful, output tensor shape: {execution_model_output.shape}" + if MODEL_IMPL_SETTINGS.enable_execution_cache: pickle.dump((execution_feedback_str, execution_model_output), open(cache_file_path, "wb")) + return execution_feedback_str, execution_model_output - + except Exception as e: return f"Execution error: {e}", None diff --git a/rdagent/components/coder/model_coder/prompts.yaml b/rdagent/components/coder/model_coder/prompts.yaml index b5679941..b5f069a7 100644 --- a/rdagent/components/coder/model_coder/prompts.yaml +++ b/rdagent/components/coder/model_coder/prompts.yaml @@ -1,11 +1,42 @@ extract_model_formulation_system: |- - offer description of the proposed model in this paper, write a latex formula with variable of the model. the format should be like - "Model Name": { - "description": "", - "formulation": "", + offer description of the proposed model in this paper, write a latex formula with variable as well as the architecture of the model. the format should be like + { + "model_name (The name of the model)": { + "description": "A detailed description of the model", + "formulation": "A LaTeX formula representing the model's formulation", + "architecture": "A detailed description of the model's architecture, e.g., neural network layers or tree structures", "variables": { - "\\hat{y}_u": "The predicted output for node u", - }" + "\\hat{y}_u": "The predicted output for node u", + "variable_name_2": "Description of variable 2", + "variable_name_3": "Description of variable 3" + }, + "hyperparameters": { + "hyperparameter_name_1": "value of hyperparameter 1", + "hyperparameter_name_2": "value of hyperparameter 2", + "hyperparameter_name_3": "value of hyperparameter 3" + }, + "model_type": "Tabular or TimeSeries or Graph" # Should be one of "Tabular", "TimeSeries", or "Graph" + } + } + Eg. + { + "ABC Model": { + "description": "A detailed description of the model", + "formulation": "A LaTeX formula representing the model's formulation", + "architecture": "A detailed description of the model's architecture, e.g., neural network layers or tree structures", + "variables": { + "\\hat{y}_u": "The predicted output for node u", + "variable_name_2": "Description of variable 2", + "variable_name_3": "Description of variable 3" + }, + "hyperparameters": { + "hyperparameter_name_1": "value of hyperparameter 1", + "hyperparameter_name_2": "value of hyperparameter 2", + "hyperparameter_name_3": "value of hyperparameter 3" + }, + "model_type": "Tabular or TimeSeries or Graph" # Should be one of "Tabular", "TimeSeries", or "Graph" + } + } such format content should be begin with ```json and end with ``` and the content should be in json format. evolving_strategy_model_coder: diff --git a/rdagent/components/coder/model_coder/task_loader.py b/rdagent/components/coder/model_coder/task_loader.py index 5929f10e..ad047c1a 100644 --- a/rdagent/components/coder/model_coder/task_loader.py +++ b/rdagent/components/coder/model_coder/task_loader.py @@ -105,8 +105,9 @@ class ModelExperimentLoaderFromDict(ModelTaskLoader): name=model_name, description=model_data["description"], formulation=model_data["formulation"], + architecture=model_data["architecture"], variables=model_data["variables"], - key=model_name, + model_type=model_data["model_type"], ) task_l.append(task) return QlibModelExperiment(sub_tasks=task_l) diff --git a/rdagent/components/proposal/prompts.yaml b/rdagent/components/proposal/prompts.yaml index 1573d7fc..12de3cd0 100644 --- a/rdagent/components/proposal/prompts.yaml +++ b/rdagent/components/proposal/prompts.yaml @@ -10,7 +10,7 @@ hypothesis_gen: Here are the specifications: {{ hypothesis_specification }} user_prompt: |- The user has made several hypothesis on this scenario and did several evaluation on them. - The former hypothesis and the corresponding feedbacks are as follows: + The former hypothesis and the corresponding feedbacks are as follows (focus on the last one & the new hypothesis that it provides and reasoning to see if you agree): {{ hypothesis_and_feedback }} To help you generate new {{targets}}, we have prepared the following information for you: {{ RAG }} diff --git a/rdagent/core/proposal.py b/rdagent/core/proposal.py index 4a86aef1..f2745b58 100644 --- a/rdagent/core/proposal.py +++ b/rdagent/core/proposal.py @@ -23,9 +23,10 @@ class Hypothesis: - Belief """ - def __init__(self, hypothesis: str, reason: str) -> None: + def __init__(self, hypothesis: str, reason: str, concise_reason = str) -> None: self.hypothesis: str = hypothesis self.reason: str = reason + self.concise_reason: str = concise_reason def __str__(self) -> str: return f"""Hypothesis: {self.hypothesis} diff --git a/rdagent/core/scenario.py b/rdagent/core/scenario.py index 31f5f5d3..18ffcd3f 100644 --- a/rdagent/core/scenario.py +++ b/rdagent/core/scenario.py @@ -27,6 +27,15 @@ class Scenario(ABC): def simulator(self) -> str: """Simulator description""" + @property + @abstractmethod + def rich_style_description(self) -> str: + """Rich style description to present""" + @abstractmethod def get_scenario_all_desc(self) -> str: """Combine all the description together""" + + + + diff --git a/rdagent/scenarios/qlib/experiment/model_experiment.py b/rdagent/scenarios/qlib/experiment/model_experiment.py index 7b35b474..3da3c15e 100644 --- a/rdagent/scenarios/qlib/experiment/model_experiment.py +++ b/rdagent/scenarios/qlib/experiment/model_experiment.py @@ -38,6 +38,10 @@ class QlibModelScenario(Scenario): @property def simulator(self) -> str: return prompt_dict["qlib_model_simulator"] + + @property + def rich_style_description(self)->str: + return "Below is QlibModel Evolving Automatic R&D Demo." def get_scenario_all_desc(self) -> str: return f"""Background of the scenario: diff --git a/rdagent/scenarios/qlib/experiment/model_template/conf.yaml b/rdagent/scenarios/qlib/experiment/model_template/conf.yaml index f8169e52..661f4f1f 100644 --- a/rdagent/scenarios/qlib/experiment/model_template/conf.yaml +++ b/rdagent/scenarios/qlib/experiment/model_template/conf.yaml @@ -62,7 +62,7 @@ task: metric: loss loss: mse n_jobs: 20 - GPU: 3 + GPU: 2 # loss: mse # lr: 0.002 # optimizer: adam diff --git a/rdagent/scenarios/qlib/experiment/model_template/read_exp_res.py b/rdagent/scenarios/qlib/experiment/model_template/read_exp_res.py index 19930248..01976cb9 100644 --- a/rdagent/scenarios/qlib/experiment/model_template/read_exp_res.py +++ b/rdagent/scenarios/qlib/experiment/model_template/read_exp_res.py @@ -39,7 +39,11 @@ else: # Load the specified file from the latest recorder metrics = pd.Series(latest_recorder.list_metrics()) + output_path = Path(__file__).resolve().parent / "qlib_res.csv" metrics.to_csv(output_path) print(f"Output has been saved to {output_path}") + + ret_data_frame = latest_recorder.load_object("portfolio_analysis/report_normal_1day.pkl") + ret_data_frame.to_pickle("ret.pkl") diff --git a/rdagent/scenarios/qlib/experiment/prompts.yaml b/rdagent/scenarios/qlib/experiment/prompts.yaml index 4638aae8..237e77ae 100644 --- a/rdagent/scenarios/qlib/experiment/prompts.yaml +++ b/rdagent/scenarios/qlib/experiment/prompts.yaml @@ -48,7 +48,7 @@ qlib_factor_simulator: |- qlib_model_background: |- The model is a machine learning or deep learning structure used in quantitative investment to predict the returns and risks of a portfolio or a single asset. Models are employed by investors to generate forecasts based on historical data and identified factors, which are central to many quantitative investment strategies. - Each model takes the factors as input and predicts the future returns. + Each model takes the factors as input and predicts the future returns. Usually, the bigger the model is, the better the performance would be. The model is defined in the following parts: 1. Name: The name of the model. 2. Description: The description of the model. diff --git a/rdagent/scenarios/qlib/experiment/workspace.py b/rdagent/scenarios/qlib/experiment/workspace.py index 6f549a7e..00871440 100644 --- a/rdagent/scenarios/qlib/experiment/workspace.py +++ b/rdagent/scenarios/qlib/experiment/workspace.py @@ -30,6 +30,9 @@ class QlibFBWorkspace(FBWorkspace): env=run_env, ) + ret_df = pd.read_pickle(self.workspace_path / "ret.pkl") + logger.log_object(ret_df, tag="returns") # TODO: tag + csv_path = self.workspace_path / "qlib_res.csv" if not csv_path.exists(): diff --git a/rdagent/scenarios/qlib/prompts.yaml b/rdagent/scenarios/qlib/prompts.yaml index 9436b069..19814df6 100644 --- a/rdagent/scenarios/qlib/prompts.yaml +++ b/rdagent/scenarios/qlib/prompts.yaml @@ -12,7 +12,8 @@ hypothesis_output_format: |- The output should follow JSON format. The schema is as follows: { "hypothesis": "The new hypothesis generated based on the information provided.", - "reason": "The reason why you generate this hypothesis." + "reason": "The reason why you generate this hypothesis.", + "concise_reason": One line summary that focuses on the justification for the change that leads to the hypothesis (like a part of a knowledge that we are building), } model_hypothesis_specification: |- @@ -140,7 +141,13 @@ model_experiment_output_format: |- { "model_name 1 (The name of the model)": { "description": "A detailed description of the model", + "formulation": "A LaTeX formula representing the model's formulation", "architecture": "A detailed description of the model's architecture, e.g., neural network layers or tree structures", + "variables": { + "\\hat{y}_u": "The predicted output for node u", + "variable_name_2": "Description of variable 2", + "variable_name_3": "Description of variable 3" + }, "hyperparameters": { "hyperparameter_name_1": "value of hyperparameter 1", "hyperparameter_name_2": "value of hyperparameter 2", @@ -218,6 +225,8 @@ model_feedback_generation: "Reasoning": "Provide reasoning for the hypothesis here.", "Decision": , } + + Focus on the changes in hypothesis and justify why do hypothesis evolve like this. Also, increase complexity as the hypothesis evolves (give more layers, more neurons, and etc) Logic for generating a new hypothesis: If the previous hypothesis works, try to inherit from it and grow deeper. If the previous hypotheis doesn't work, try to make changes in the current level. diff --git a/rdagent/scenarios/qlib/proposal/model_proposal.py b/rdagent/scenarios/qlib/proposal/model_proposal.py index 1eac990c..fb2bcefa 100644 --- a/rdagent/scenarios/qlib/proposal/model_proposal.py +++ b/rdagent/scenarios/qlib/proposal/model_proposal.py @@ -39,7 +39,7 @@ class QlibModelHypothesisGen(ModelHypothesisGen): def convert_response(self, response: str) -> ModelHypothesis: response_dict = json.loads(response) - hypothesis = QlibModelHypothesis(hypothesis=response_dict["hypothesis"], reason=response_dict["reason"]) + hypothesis = QlibModelHypothesis(hypothesis=response_dict["hypothesis"], reason=response_dict["reason"], concise_reason=response_dict["concise_reason"]) return hypothesis @@ -74,10 +74,12 @@ class QlibModelHypothesis2Experiment(ModelHypothesis2Experiment): tasks = [] for model_name in response_dict: description = response_dict[model_name]["description"] + formulation = response_dict[model_name]["formulation"] architecture = response_dict[model_name]["architecture"] + variables = response_dict[model_name]["variables"] hyperparameters = response_dict[model_name]["hyperparameters"] model_type = response_dict[model_name]["model_type"] - tasks.append(ModelTask(model_name, description, architecture, hyperparameters, model_type)) + tasks.append(ModelTask(model_name, description, formulation, architecture, variables, hyperparameters, model_type)) exp = QlibModelExperiment(tasks) exp.based_experiments = [t[1] for t in trace.hist if t[2]] return exp