From 0a4bca5ffa61fa9c6777ccd64af375f21af8058a Mon Sep 17 00:00:00 2001 From: Xisen Wang <118058822+Xisen-Wang@users.noreply.github.com> Date: Thu, 1 Aug 2024 10:53:37 +0800 Subject: [PATCH] Added three new keys on hypothesis reasoning (#138) * Added three new keys on hypothesis reasoning * Updated two scenario rich text --- .../model_extraction_and_code/GeneralModel.py | 44 +++++++------------ rdagent/components/proposal/prompts.yaml | 2 +- rdagent/core/proposal.py | 8 +++- .../qlib/experiment/model_experiment.py | 43 ++++++------------ rdagent/scenarios/qlib/prompts.yaml | 7 ++- .../scenarios/qlib/proposal/model_proposal.py | 3 ++ 6 files changed, 46 insertions(+), 61 deletions(-) diff --git a/rdagent/app/model_extraction_and_code/GeneralModel.py b/rdagent/app/model_extraction_and_code/GeneralModel.py index 7d417326..7a7e39ca 100644 --- a/rdagent/app/model_extraction_and_code/GeneralModel.py +++ b/rdagent/app/model_extraction_and_code/GeneralModel.py @@ -31,34 +31,22 @@ class GeneralModelScenario(Scenario): @property def rich_style_description(self) -> str: return """ -# General Model Scenario - -## Overview - -This demo automates the extraction and iterative development of models from academic papers, ensuring functionality and correctness. - -### Scenario: Auto-Developing Model Code from Academic Papers - -#### Overview - -This scenario automates the development of PyTorch models by reading academic papers or other sources. It supports various data types, including tabular, time-series, and graph data. The primary workflow involves two main components: the Reader and the Coder. - -#### Workflow Components - -1. **Reader** - - Parses and extracts relevant model information from academic papers or sources, including architectures, parameters, and implementation details. - - Uses Large Language Models to convert content into a structured format for the Coder. - -2. **Evolving Coder** - - Translates structured information from the Reader into executable PyTorch code. - - Utilizes an evolving coding mechanism to ensure correct tensor shapes, verified with sample input tensors. - - Iteratively refines the code to align with source material specifications. - -#### Supported Data Types - -- **Tabular Data:** Structured data with rows and columns, such as spreadsheets or databases. -- **Time-Series Data:** Sequential data points indexed in time order, useful for forecasting and temporal pattern recognition. -- **Graph Data:** Data structured as nodes and edges, suitable for network analysis and relational tasks. +# [General Model Research Co-Pilot] (#_scenario) + +## [Overview](#_summary) + +This demo automates the extraction and development of PyTorch models from academic papers. It supports various model types through two main components: Reader and Coder. + +#### [Workflow Components](#_rdloops) + +1. **[Reader](#_research)** + - Extracts model information from papers, including architectures and parameters. + - Converts content into a structured format using Large Language Models. + +2. **[Evolving Coder](#_development)** + - Translates structured information into executable PyTorch code. + - Ensures correct tensor shapes with an evolving coding mechanism. + - Refines the code to match source specifications. """ diff --git a/rdagent/components/proposal/prompts.yaml b/rdagent/components/proposal/prompts.yaml index 4e3b427d..e482e4d5 100644 --- a/rdagent/components/proposal/prompts.yaml +++ b/rdagent/components/proposal/prompts.yaml @@ -14,7 +14,7 @@ hypothesis_gen: {{ hypothesis_and_feedback }} To help you generate new {{targets}}, we have prepared the following information for you: {{ RAG }} - Please generate the new hypothesis based on the information above. + Please generate the new hypothesis based on the information above. Also generate the relevant keys for the reasoning and the distilled knowledge that follows. hypothesis2experiment: system_prompt: |- diff --git a/rdagent/core/proposal.py b/rdagent/core/proposal.py index e01ab71e..b3e4697e 100644 --- a/rdagent/core/proposal.py +++ b/rdagent/core/proposal.py @@ -25,15 +25,21 @@ class Hypothesis: - Belief """ - def __init__(self, hypothesis: str, reason: str, concise_reason: str) -> None: + def __init__(self, hypothesis: str, reason: str, concise_reason: str, concise_observation: str, concise_justification: str, concise_knowledge: str) -> None: self.hypothesis: str = hypothesis self.reason: str = reason self.concise_reason: str = concise_reason + self.concise_observation: str = concise_observation + self.concise_justification: str = concise_justification + self.concise_knowledge: str = concise_knowledge def __str__(self) -> str: return f"""Hypothesis: {self.hypothesis} Reason: {self.reason} Concise Reason & Knowledge: {self.concise_reason} + Concise Observation: {self.concise_observation} + Concise Justification: {self.concise_justification} + Concise Knowledge: {self.concise_knowledge} """ # source: data_ana | model_nan = None diff --git a/rdagent/scenarios/qlib/experiment/model_experiment.py b/rdagent/scenarios/qlib/experiment/model_experiment.py index f1c99c8d..ddb04761 100644 --- a/rdagent/scenarios/qlib/experiment/model_experiment.py +++ b/rdagent/scenarios/qlib/experiment/model_experiment.py @@ -43,39 +43,24 @@ class QlibModelScenario(Scenario): def rich_style_description(self) -> str: return """ ### Qlib Model Evolving Automatic R&D Demo - -#### Overview - -The demo showcases the iterative process of hypothesis generation, knowledge construction, and decision-making. It highlights how models evolve through continuous feedback and refinement. - -#### Key Steps - -1. **Hypothesis** - - Generate and propose initial hypotheses based on data and domain knowledge. - -2. **Knowledge Construction** - - Develop, test, and refine models to gather empirical results. - - Analyze feedback and incorporate new insights. - -3. **Decision Pair** - - Make data-driven decisions based on validated hypotheses and constructed knowledge. - - Iterate the process to evolve and improve models continuously. - -#### Automated R&D - -- **R (Research)** + +#### [Overview](#_summary) + +The demo showcases the iterative process of hypothesis generation, knowledge construction, and decision-making in model construction in quantitative finance. It highlights how models evolve through continuous feedback and refinement. + +#### [Automated R&D](#_rdloops) + +- **[R (Research)](#_research)** - Iteration of ideas and hypotheses. - Continuous learning and knowledge construction. - -- **D (Development)** + +- **[D (Development)](#_development)** - Evolving code generation and model refinement. - Automated implementation and testing of models. - -#### Objective - -To demonstrate the dynamic evolution of models through the Qlib platform, emphasizing how each iteration enhances the accuracy and reliability of the resulting models. - - """ + +#### [Objective](#_summary) + +To demonstrate the dynamic evolution of models through the Qlib platform, emphasizing how each iteration enhances the accuracy and reliability of the resulting models. """ def get_scenario_all_desc(self) -> str: return f"""Background of the scenario: diff --git a/rdagent/scenarios/qlib/prompts.yaml b/rdagent/scenarios/qlib/prompts.yaml index 63a7db8d..d1936480 100644 --- a/rdagent/scenarios/qlib/prompts.yaml +++ b/rdagent/scenarios/qlib/prompts.yaml @@ -13,8 +13,11 @@ hypothesis_output_format: |- The output should follow JSON format. The schema is as follows: { "hypothesis": "The new hypothesis generated based on the information provided.", Note that this should focus on model architecture, not training process or feature engineering or anything else - "reason": "The reason why you generate this hypothesis.", - "concise_reason": Two line summary. First line focuses on the a concise justification for the change. 2nd Line learns from first line and previous experiences (hypothesis & experiments & code & feedbacks) to generalise a knowledge statement (use tend to/because/if/generally/etc. ). + "reason": "The reason why you generate this hypothesis. It should be comprehensive and logical. It should cover the other keys below and extend them.", + "concise_reason": Two line summary. First line focuses on the a concise justification for the change. 2nd Line learns from first line and previous experiences (hypothesis & experiments & code & feedbacks) to generalise a knowledge statement (use tend to/because/if/generally/etc. ). This is the summary of the three keys below. + "concise_observation": One line summary. It focuses on the observation of the given scenario, data characteristics, or previous experiences (failures & succeses). + "concise_justification": One line summary. It focuses on the justification for the change in new hypothesis and the route of exploration supporting the growth of the hypothesis, based on the observation. + "concise_knowledge": One line summary. It focuses on a transferable knowledege that comes with the new hypothesis. Use conditional grammar. eg. "If...., ..; When..., .; and etc" } model_hypothesis_specification: |- diff --git a/rdagent/scenarios/qlib/proposal/model_proposal.py b/rdagent/scenarios/qlib/proposal/model_proposal.py index 796e0a1c..9bc53bf7 100644 --- a/rdagent/scenarios/qlib/proposal/model_proposal.py +++ b/rdagent/scenarios/qlib/proposal/model_proposal.py @@ -43,6 +43,9 @@ class QlibModelHypothesisGen(ModelHypothesisGen): hypothesis=response_dict["hypothesis"], reason=response_dict["reason"], concise_reason=response_dict["concise_reason"], + concise_observation=response_dict["concise_observation"], + concise_justification=response_dict["concise_justification"], + concise_knowledge=response_dict["concise_knowledge"], ) return hypothesis