From 95474d01aa241a8839346bdf5c5887978fe5601f Mon Sep 17 00:00:00 2001 From: Xisen-Wang Date: Tue, 23 Jul 2024 12:01:52 +0000 Subject: [PATCH 1/6] Further Revision on Prompts --- rdagent/components/proposal/prompts.yaml | 2 +- rdagent/scenarios/qlib/experiment/model_template/conf.yaml | 2 +- rdagent/scenarios/qlib/experiment/prompts.yaml | 2 +- rdagent/scenarios/qlib/prompts.yaml | 2 ++ 4 files changed, 5 insertions(+), 3 deletions(-) diff --git a/rdagent/components/proposal/prompts.yaml b/rdagent/components/proposal/prompts.yaml index 1573d7fc..12de3cd0 100644 --- a/rdagent/components/proposal/prompts.yaml +++ b/rdagent/components/proposal/prompts.yaml @@ -10,7 +10,7 @@ hypothesis_gen: Here are the specifications: {{ hypothesis_specification }} user_prompt: |- The user has made several hypothesis on this scenario and did several evaluation on them. - The former hypothesis and the corresponding feedbacks are as follows: + The former hypothesis and the corresponding feedbacks are as follows (focus on the last one & the new hypothesis that it provides and reasoning to see if you agree): {{ hypothesis_and_feedback }} To help you generate new {{targets}}, we have prepared the following information for you: {{ RAG }} diff --git a/rdagent/scenarios/qlib/experiment/model_template/conf.yaml b/rdagent/scenarios/qlib/experiment/model_template/conf.yaml index f8169e52..661f4f1f 100644 --- a/rdagent/scenarios/qlib/experiment/model_template/conf.yaml +++ b/rdagent/scenarios/qlib/experiment/model_template/conf.yaml @@ -62,7 +62,7 @@ task: metric: loss loss: mse n_jobs: 20 - GPU: 3 + GPU: 2 # loss: mse # lr: 0.002 # optimizer: adam diff --git a/rdagent/scenarios/qlib/experiment/prompts.yaml b/rdagent/scenarios/qlib/experiment/prompts.yaml index 4638aae8..237e77ae 100644 --- a/rdagent/scenarios/qlib/experiment/prompts.yaml +++ b/rdagent/scenarios/qlib/experiment/prompts.yaml @@ -48,7 +48,7 @@ qlib_factor_simulator: |- qlib_model_background: |- The model is a machine learning or deep learning structure used in quantitative investment to predict the returns and risks of a portfolio or a single asset. Models are employed by investors to generate forecasts based on historical data and identified factors, which are central to many quantitative investment strategies. - Each model takes the factors as input and predicts the future returns. + Each model takes the factors as input and predicts the future returns. Usually, the bigger the model is, the better the performance would be. The model is defined in the following parts: 1. Name: The name of the model. 2. Description: The description of the model. diff --git a/rdagent/scenarios/qlib/prompts.yaml b/rdagent/scenarios/qlib/prompts.yaml index 504aa93d..24e8fc6d 100644 --- a/rdagent/scenarios/qlib/prompts.yaml +++ b/rdagent/scenarios/qlib/prompts.yaml @@ -152,6 +152,8 @@ model_feedback_generation: "Reasoning": "Provide reasoning for the hypothesis here.", "Decision": , } + + Focus on the changes in hypothesis and justify why do hypothesis evolve like this. Also, increase complexity as the hypothesis evolves (give more layers, more neurons, and etc) Logic for generating a new hypothesis: If the previous hypothesis works, try to inherit from it and grow deeper. If the previous hypotheis doesn't work, try to make changes in the current level. From 509b4d7745ddbae3f94dfa18eeb82222fe9787e9 Mon Sep 17 00:00:00 2001 From: Xisen-Wang Date: Wed, 24 Jul 2024 09:14:20 +0000 Subject: [PATCH 2/6] Revised relevant code to enable graph input --- .../model_extraction_and_implementation.py | 2 +- rdagent/components/coder/model_coder/model.py | 32 +++++++++++++++---- .../components/coder/model_coder/prompts.yaml | 17 ++++++++-- .../coder/model_coder/task_loader.py | 3 +- 4 files changed, 44 insertions(+), 10 deletions(-) diff --git a/rdagent/app/model_extraction_and_code/model_extraction_and_implementation.py b/rdagent/app/model_extraction_and_code/model_extraction_and_implementation.py index 1114cf13..ec22b6ae 100644 --- a/rdagent/app/model_extraction_and_code/model_extraction_and_implementation.py +++ b/rdagent/app/model_extraction_and_code/model_extraction_and_implementation.py @@ -8,7 +8,7 @@ from rdagent.scenarios.qlib.developer.model_coder import QlibModelCoSTEER from rdagent.scenarios.qlib.experiment.model_experiment import QlibModelScenario -def extract_models_and_implement(report_file_path: str = "../test_doc") -> None: +def extract_models_and_implement(report_file_path: str = "/home/xiaoyang/PaperImpBench/raw_paper/4096.pdf") -> None: scenario = QlibModelScenario() exp = ModelExperimentLoaderFromPDFfiles().load(report_file_path) exp = QlibModelCoSTEER(scenario).develop(exp) diff --git a/rdagent/components/coder/model_coder/model.py b/rdagent/components/coder/model_coder/model.py index 67fe2609..75a7c66e 100644 --- a/rdagent/components/coder/model_coder/model.py +++ b/rdagent/components/coder/model_coder/model.py @@ -16,18 +16,20 @@ from rdagent.utils import get_module_by_module_path class ModelTask(Task): def __init__( - self, name: str, description: str, formulation: str, variables: Dict[str, str], model_type: Optional[str] = None + self, name: str, description: str, formulation: str, architecture: str, variables: Dict[str, str], model_type: Optional[str] = None ) -> None: self.name: str = name self.description: str = description self.formulation: str = formulation + self.architecture: str = architecture self.variables: str = variables - self.model_type: str = model_type # Tabular for tabular model, TimesSeries for time series model + self.model_type: str = model_type # Tabular for tabular model, TimesSeries for time series model, Graph for graph model def get_task_information(self): return f"""name: {self.name} description: {self.description} formulation: {self.formulation} +architecture: {self.architecture} variables: {self.variables} model_type: {self.model_type} """ @@ -65,6 +67,8 @@ class ModelFBWorkspace(FBWorkspace): batch_size: int = 8, num_features: int = 10, num_timesteps: int = 4, + num_nodes: int = 50, + num_edges: int = 100, input_value: float = 1.0, param_init_value: float = 1.0, ): @@ -85,21 +89,37 @@ class ModelFBWorkspace(FBWorkspace): if self.target_task.model_type == "Tabular": input_shape = (batch_size, num_features) m = model_cls(num_features=input_shape[1]) + data = torch.full(input_shape, input_value) elif self.target_task.model_type == "TimeSeries": input_shape = (batch_size, num_features, num_timesteps) m = model_cls(num_features=input_shape[1], num_timesteps=input_shape[2]) - data = torch.full(input_shape, input_value) + data = torch.full(input_shape, input_value) + elif self.target_task.model_type == "Graph": + node_feature = torch.randn(batch_size, num_nodes, num_features) + edge_index = torch.randint(0, num_nodes, (2, num_edges)) + m = model_cls(num_nodes=num_nodes, num_features=num_features) + data = (node_feature, edge_index) + else: + raise ValueError(f"Unsupported model type: {self.target_task.model_type}") - # initialize all parameters of `m` to `param_init_value` + # Initialize all parameters of `m` to `param_init_value` for _, param in m.named_parameters(): param.data.fill_(param_init_value) - out = m(data) + + # Execute the model + if self.target_task.model_type == "Graph": + out = m(*data) + else: + out = m(data) + execution_model_output = out.cpu().detach() execution_feedback_str = f"Execution successful, output tensor shape: {execution_model_output.shape}" + if MODEL_IMPL_SETTINGS.enable_execution_cache: pickle.dump((execution_feedback_str, execution_model_output), open(cache_file_path, "wb")) + return execution_feedback_str, execution_model_output - + except Exception as e: return f"Execution error: {e}", None diff --git a/rdagent/components/coder/model_coder/prompts.yaml b/rdagent/components/coder/model_coder/prompts.yaml index b5679941..107c4189 100644 --- a/rdagent/components/coder/model_coder/prompts.yaml +++ b/rdagent/components/coder/model_coder/prompts.yaml @@ -1,11 +1,24 @@ extract_model_formulation_system: |- - offer description of the proposed model in this paper, write a latex formula with variable of the model. the format should be like - "Model Name": { + offer description of the proposed model in this paper, write a latex formula with variable as well as the architecture of the model. the format should be like + "Model Name (replace with a reasonable model type)": { "description": "", "formulation": "", + "architecture": "", "variables": { "\\hat{y}_u": "The predicted output for node u", }" + "model_type":"Tabular", "TimeSeries", or "Graph", (depending on which type of the model it is) + } + Eg. + "ABC Model":{ + "description": "", + "formulation": "", + "architecture": "", + "variables": { + "\\hat{y}_u": "The predicted output for node u", + }" + "model_type":"Tabular", + } such format content should be begin with ```json and end with ``` and the content should be in json format. evolving_strategy_model_coder: diff --git a/rdagent/components/coder/model_coder/task_loader.py b/rdagent/components/coder/model_coder/task_loader.py index 5929f10e..ad047c1a 100644 --- a/rdagent/components/coder/model_coder/task_loader.py +++ b/rdagent/components/coder/model_coder/task_loader.py @@ -105,8 +105,9 @@ class ModelExperimentLoaderFromDict(ModelTaskLoader): name=model_name, description=model_data["description"], formulation=model_data["formulation"], + architecture=model_data["architecture"], variables=model_data["variables"], - key=model_name, + model_type=model_data["model_type"], ) task_l.append(task) return QlibModelExperiment(sub_tasks=task_l) From 86d954775a7c1d11e73c503578b3bfd872afc4af Mon Sep 17 00:00:00 2001 From: Xisen-Wang Date: Wed, 24 Jul 2024 10:26:06 +0000 Subject: [PATCH 3/6] Revised to enable logging of backtesting in Qlib & storing of rich-text-description in Trace --- rdagent/core/scenario.py | 9 +++++++++ rdagent/scenarios/qlib/experiment/model_experiment.py | 4 ++++ .../qlib/experiment/model_template/read_exp_res.py | 4 ++++ rdagent/scenarios/qlib/experiment/workspace.py | 3 +++ 4 files changed, 20 insertions(+) diff --git a/rdagent/core/scenario.py b/rdagent/core/scenario.py index 31f5f5d3..18ffcd3f 100644 --- a/rdagent/core/scenario.py +++ b/rdagent/core/scenario.py @@ -27,6 +27,15 @@ class Scenario(ABC): def simulator(self) -> str: """Simulator description""" + @property + @abstractmethod + def rich_style_description(self) -> str: + """Rich style description to present""" + @abstractmethod def get_scenario_all_desc(self) -> str: """Combine all the description together""" + + + + diff --git a/rdagent/scenarios/qlib/experiment/model_experiment.py b/rdagent/scenarios/qlib/experiment/model_experiment.py index 7b35b474..3da3c15e 100644 --- a/rdagent/scenarios/qlib/experiment/model_experiment.py +++ b/rdagent/scenarios/qlib/experiment/model_experiment.py @@ -38,6 +38,10 @@ class QlibModelScenario(Scenario): @property def simulator(self) -> str: return prompt_dict["qlib_model_simulator"] + + @property + def rich_style_description(self)->str: + return "Below is QlibModel Evolving Automatic R&D Demo." def get_scenario_all_desc(self) -> str: return f"""Background of the scenario: diff --git a/rdagent/scenarios/qlib/experiment/model_template/read_exp_res.py b/rdagent/scenarios/qlib/experiment/model_template/read_exp_res.py index 19930248..01976cb9 100644 --- a/rdagent/scenarios/qlib/experiment/model_template/read_exp_res.py +++ b/rdagent/scenarios/qlib/experiment/model_template/read_exp_res.py @@ -39,7 +39,11 @@ else: # Load the specified file from the latest recorder metrics = pd.Series(latest_recorder.list_metrics()) + output_path = Path(__file__).resolve().parent / "qlib_res.csv" metrics.to_csv(output_path) print(f"Output has been saved to {output_path}") + + ret_data_frame = latest_recorder.load_object("portfolio_analysis/report_normal_1day.pkl") + ret_data_frame.to_pickle("ret.pkl") diff --git a/rdagent/scenarios/qlib/experiment/workspace.py b/rdagent/scenarios/qlib/experiment/workspace.py index b9b58038..98f50124 100644 --- a/rdagent/scenarios/qlib/experiment/workspace.py +++ b/rdagent/scenarios/qlib/experiment/workspace.py @@ -36,6 +36,9 @@ class QlibFBWorkspace(FBWorkspace): env=run_env, ) + ret_df = pd.read_pickle(self.workspace_path / "ret.pkl") + logger.log_object(ret_df, tag="returns") # TODO: tag + csv_path = self.workspace_path / "qlib_res.csv" if not csv_path.exists(): From 202ea4a566f963170c8a3a377e323b5784a8fdfd Mon Sep 17 00:00:00 2001 From: Xisen-Wang Date: Wed, 24 Jul 2024 11:33:04 +0000 Subject: [PATCH 4/6] Unified ModelTask structure --- rdagent/components/coder/model_coder/model.py | 4 +- .../components/coder/model_coder/prompts.yaml | 48 +++++++++++++------ rdagent/core/proposal.py | 3 +- rdagent/scenarios/qlib/prompts.yaml | 11 ++++- .../scenarios/qlib/proposal/model_proposal.py | 6 ++- 5 files changed, 51 insertions(+), 21 deletions(-) diff --git a/rdagent/components/coder/model_coder/model.py b/rdagent/components/coder/model_coder/model.py index 75a7c66e..eeff3d7c 100644 --- a/rdagent/components/coder/model_coder/model.py +++ b/rdagent/components/coder/model_coder/model.py @@ -16,13 +16,14 @@ from rdagent.utils import get_module_by_module_path class ModelTask(Task): def __init__( - self, name: str, description: str, formulation: str, architecture: str, variables: Dict[str, str], model_type: Optional[str] = None + self, name: str, description: str, formulation: str, architecture: str, variables: Dict[str, str], hyperparameters: Dict[str, str], model_type: Optional[str] = None ) -> None: self.name: str = name self.description: str = description self.formulation: str = formulation self.architecture: str = architecture self.variables: str = variables + self.hyperparameters: str = hyperparameters self.model_type: str = model_type # Tabular for tabular model, TimesSeries for time series model, Graph for graph model def get_task_information(self): @@ -31,6 +32,7 @@ description: {self.description} formulation: {self.formulation} architecture: {self.architecture} variables: {self.variables} +hyperparameters: {self.hyperparameters} model_type: {self.model_type} """ diff --git a/rdagent/components/coder/model_coder/prompts.yaml b/rdagent/components/coder/model_coder/prompts.yaml index 107c4189..b5f069a7 100644 --- a/rdagent/components/coder/model_coder/prompts.yaml +++ b/rdagent/components/coder/model_coder/prompts.yaml @@ -1,23 +1,41 @@ extract_model_formulation_system: |- offer description of the proposed model in this paper, write a latex formula with variable as well as the architecture of the model. the format should be like - "Model Name (replace with a reasonable model type)": { - "description": "", - "formulation": "", - "architecture": "", + { + "model_name (The name of the model)": { + "description": "A detailed description of the model", + "formulation": "A LaTeX formula representing the model's formulation", + "architecture": "A detailed description of the model's architecture, e.g., neural network layers or tree structures", "variables": { - "\\hat{y}_u": "The predicted output for node u", - }" - "model_type":"Tabular", "TimeSeries", or "Graph", (depending on which type of the model it is) - } + "\\hat{y}_u": "The predicted output for node u", + "variable_name_2": "Description of variable 2", + "variable_name_3": "Description of variable 3" + }, + "hyperparameters": { + "hyperparameter_name_1": "value of hyperparameter 1", + "hyperparameter_name_2": "value of hyperparameter 2", + "hyperparameter_name_3": "value of hyperparameter 3" + }, + "model_type": "Tabular or TimeSeries or Graph" # Should be one of "Tabular", "TimeSeries", or "Graph" + } + } Eg. - "ABC Model":{ - "description": "", - "formulation": "", - "architecture": "", + { + "ABC Model": { + "description": "A detailed description of the model", + "formulation": "A LaTeX formula representing the model's formulation", + "architecture": "A detailed description of the model's architecture, e.g., neural network layers or tree structures", "variables": { - "\\hat{y}_u": "The predicted output for node u", - }" - "model_type":"Tabular", + "\\hat{y}_u": "The predicted output for node u", + "variable_name_2": "Description of variable 2", + "variable_name_3": "Description of variable 3" + }, + "hyperparameters": { + "hyperparameter_name_1": "value of hyperparameter 1", + "hyperparameter_name_2": "value of hyperparameter 2", + "hyperparameter_name_3": "value of hyperparameter 3" + }, + "model_type": "Tabular or TimeSeries or Graph" # Should be one of "Tabular", "TimeSeries", or "Graph" + } } such format content should be begin with ```json and end with ``` and the content should be in json format. diff --git a/rdagent/core/proposal.py b/rdagent/core/proposal.py index 5f32efc0..878903c5 100644 --- a/rdagent/core/proposal.py +++ b/rdagent/core/proposal.py @@ -22,9 +22,10 @@ class Hypothesis: - Belief """ - def __init__(self, hypothesis: str, reason: str) -> None: + def __init__(self, hypothesis: str, reason: str, concise_reason = str) -> None: self.hypothesis: str = hypothesis self.reason: str = reason + self.concise_reason: str = concise_reason def __str__(self) -> str: return f"""Hypothesis: {self.hypothesis} diff --git a/rdagent/scenarios/qlib/prompts.yaml b/rdagent/scenarios/qlib/prompts.yaml index 24e8fc6d..8b88c5e2 100644 --- a/rdagent/scenarios/qlib/prompts.yaml +++ b/rdagent/scenarios/qlib/prompts.yaml @@ -12,7 +12,8 @@ hypothesis_output_format: |- The output should follow JSON format. The schema is as follows: { "hypothesis": "The new hypothesis generated based on the information provided.", - "reason": "The reason why you generate this hypothesis." + "reason": "The reason why you generate this hypothesis.", + "concise_reason": One line summary that focuses on the justification for the change that leads to the hypothesis (like a part of a knowledge that we are building), } model_hypothesis_specification: |- @@ -85,13 +86,19 @@ model_experiment_output_format: |- { "model_name (The name of the model)": { "description": "A detailed description of the model", + "formulation": "A LaTeX formula representing the model's formulation", "architecture": "A detailed description of the model's architecture, e.g., neural network layers or tree structures", + "variables": { + "\\hat{y}_u": "The predicted output for node u", + "variable_name_2": "Description of variable 2", + "variable_name_3": "Description of variable 3" + }, "hyperparameters": { "hyperparameter_name_1": "value of hyperparameter 1", "hyperparameter_name_2": "value of hyperparameter 2", "hyperparameter_name_3": "value of hyperparameter 3" }, - "model_type": "Tabular or TimeSeries" # Should be one of "Tabular" or "TimeSeries" + "model_type": "Tabular or TimeSeries # Should be one of "Tabular" or "TimeSeries" } } Usually a larger model works better than a smaller one. Hence, the parameters should be larger. diff --git a/rdagent/scenarios/qlib/proposal/model_proposal.py b/rdagent/scenarios/qlib/proposal/model_proposal.py index 1eac990c..fb2bcefa 100644 --- a/rdagent/scenarios/qlib/proposal/model_proposal.py +++ b/rdagent/scenarios/qlib/proposal/model_proposal.py @@ -39,7 +39,7 @@ class QlibModelHypothesisGen(ModelHypothesisGen): def convert_response(self, response: str) -> ModelHypothesis: response_dict = json.loads(response) - hypothesis = QlibModelHypothesis(hypothesis=response_dict["hypothesis"], reason=response_dict["reason"]) + hypothesis = QlibModelHypothesis(hypothesis=response_dict["hypothesis"], reason=response_dict["reason"], concise_reason=response_dict["concise_reason"]) return hypothesis @@ -74,10 +74,12 @@ class QlibModelHypothesis2Experiment(ModelHypothesis2Experiment): tasks = [] for model_name in response_dict: description = response_dict[model_name]["description"] + formulation = response_dict[model_name]["formulation"] architecture = response_dict[model_name]["architecture"] + variables = response_dict[model_name]["variables"] hyperparameters = response_dict[model_name]["hyperparameters"] model_type = response_dict[model_name]["model_type"] - tasks.append(ModelTask(model_name, description, architecture, hyperparameters, model_type)) + tasks.append(ModelTask(model_name, description, formulation, architecture, variables, hyperparameters, model_type)) exp = QlibModelExperiment(tasks) exp.based_experiments = [t[1] for t in trace.hist if t[2]] return exp From f5556e3b8e81b43bb2b19df06f92679cc49bd74b Mon Sep 17 00:00:00 2001 From: Xisen Wang <118058822+Xisen-Wang@users.noreply.github.com> Date: Wed, 24 Jul 2024 19:36:15 +0800 Subject: [PATCH 5/6] Update model_extraction_and_implementation.py --- .../model_extraction_and_implementation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rdagent/app/model_extraction_and_code/model_extraction_and_implementation.py b/rdagent/app/model_extraction_and_code/model_extraction_and_implementation.py index ec22b6ae..ef92f09f 100644 --- a/rdagent/app/model_extraction_and_code/model_extraction_and_implementation.py +++ b/rdagent/app/model_extraction_and_code/model_extraction_and_implementation.py @@ -8,7 +8,7 @@ from rdagent.scenarios.qlib.developer.model_coder import QlibModelCoSTEER from rdagent.scenarios.qlib.experiment.model_experiment import QlibModelScenario -def extract_models_and_implement(report_file_path: str = "/home/xiaoyang/PaperImpBench/raw_paper/4096.pdf") -> None: +def extract_models_and_implement(report_file_path: str = ".../test") -> None: scenario = QlibModelScenario() exp = ModelExperimentLoaderFromPDFfiles().load(report_file_path) exp = QlibModelCoSTEER(scenario).develop(exp) From c4046b388f8751ff903ffb4f4561ca60f6c5348b Mon Sep 17 00:00:00 2001 From: Xisen Wang <118058822+Xisen-Wang@users.noreply.github.com> Date: Wed, 24 Jul 2024 19:36:37 +0800 Subject: [PATCH 6/6] Update model_extraction_and_implementation.py --- .../model_extraction_and_implementation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rdagent/app/model_extraction_and_code/model_extraction_and_implementation.py b/rdagent/app/model_extraction_and_code/model_extraction_and_implementation.py index ef92f09f..1114cf13 100644 --- a/rdagent/app/model_extraction_and_code/model_extraction_and_implementation.py +++ b/rdagent/app/model_extraction_and_code/model_extraction_and_implementation.py @@ -8,7 +8,7 @@ from rdagent.scenarios.qlib.developer.model_coder import QlibModelCoSTEER from rdagent.scenarios.qlib.experiment.model_experiment import QlibModelScenario -def extract_models_and_implement(report_file_path: str = ".../test") -> None: +def extract_models_and_implement(report_file_path: str = "../test_doc") -> None: scenario = QlibModelScenario() exp = ModelExperimentLoaderFromPDFfiles().load(report_file_path) exp = QlibModelCoSTEER(scenario).develop(exp)