From f61453fbb1a0be94eb9beca94e82ca5b00405f26 Mon Sep 17 00:00:00 2001 From: Xu Yang Date: Thu, 4 Jul 2024 15:56:14 +0800 Subject: [PATCH] First version of factor idea proposal (#46) * update all code * save code * update first version of factor proposal * change a comment * remove a useless comment --------- Co-authored-by: xuyang1 --- rdagent/app/model_proposal/conf.py | 12 --- rdagent/app/qlib_rd_loop/conf.py | 17 ++++ rdagent/app/qlib_rd_loop/factor.py | 38 +++++++++ .../run.py => qlib_rd_loop/model.py} | 0 .../idea_proposal/factor_proposal.py | 68 +++++++++++---- rdagent/components/idea_proposal/prompts.yaml | 28 +++++-- .../evolving/evolvable_subjects.py | 2 +- .../factor_implementation/factor.py | 2 +- rdagent/core/experiment.py | 2 +- rdagent/core/proposal.py | 59 +++++++++---- .../qlib/experiment/factor_experiment.py | 38 +++++++++ .../scenarios/qlib/experiment/prompts.yaml | 25 ++++++ .../scenarios/qlib/factor_idea_proposal.py | 40 --------- rdagent/scenarios/qlib/factor_proposal.py | 83 +++++++++++++++++++ rdagent/scenarios/qlib/prompts.yaml | 28 ++++++- rdagent/scenarios/qlib/task_generator/data.py | 5 +- .../scenarios/qlib/task_generator/feedback.py | 5 ++ 17 files changed, 354 insertions(+), 98 deletions(-) delete mode 100644 rdagent/app/model_proposal/conf.py create mode 100644 rdagent/app/qlib_rd_loop/conf.py create mode 100644 rdagent/app/qlib_rd_loop/factor.py rename rdagent/app/{model_proposal/run.py => qlib_rd_loop/model.py} (100%) create mode 100644 rdagent/scenarios/qlib/experiment/prompts.yaml delete mode 100644 rdagent/scenarios/qlib/factor_idea_proposal.py create mode 100644 rdagent/scenarios/qlib/factor_proposal.py diff --git a/rdagent/app/model_proposal/conf.py b/rdagent/app/model_proposal/conf.py deleted file mode 100644 index f3455c61..00000000 --- a/rdagent/app/model_proposal/conf.py +++ /dev/null @@ -1,12 +0,0 @@ -from pydantic_settings import BaseSettings - - -class ModelPropSetting(BaseSettings): - """""" - - scen: str # a.b.c:XXXClass - # TODO: inital keywards should be included in the settings - ... - - -MODEL_PROP_SETTING = ModelPropSetting() diff --git a/rdagent/app/qlib_rd_loop/conf.py b/rdagent/app/qlib_rd_loop/conf.py new file mode 100644 index 00000000..76802bbd --- /dev/null +++ b/rdagent/app/qlib_rd_loop/conf.py @@ -0,0 +1,17 @@ +from pydantic_settings import BaseSettings + + +class PropSetting(BaseSettings): + """""" + + scen: str = "rdagent.scenarios.qlib.experiment.factor_experiment.QlibFactorScenario" + hypothesis_gen: str = "rdagent.scenarios.qlib.factor_proposal.QlibFactorHypothesisGen" + hypothesis2experiment: str = "rdagent.scenarios.qlib.factor_proposal.QlibFactorHypothesis2Experiment" + qlib_factor_coder: str = "rdagent.components.task_implementation.factor_implementation.CoSTEER.CoSTEERFG" + qlib_factor_runner: str = "rdagent.scenarios.qlib.task_generator.data.QlibFactorRunner" + qlib_factor_summarizer: str = "rdagent.scenarios.qlib.task_generator.feedback.QlibFactorExperiment2Feedback" + + evolving_n: int = 10 + + +PROP_SETTING = PropSetting() diff --git a/rdagent/app/qlib_rd_loop/factor.py b/rdagent/app/qlib_rd_loop/factor.py new file mode 100644 index 00000000..dce78331 --- /dev/null +++ b/rdagent/app/qlib_rd_loop/factor.py @@ -0,0 +1,38 @@ +""" +TODO: Factor Structure RD-Loop +""" + +# import_from +from rdagent.app.qlib_rd_loop.conf import PROP_SETTING +from rdagent.core.proposal import ( + Experiment2Feedback, + Hypothesis2Experiment, + HypothesisGen, + HypothesisSet, + Trace, +) +from rdagent.core.task_generator import TaskGenerator +from rdagent.core.utils import import_class + +scen = import_class(PROP_SETTING.scen)() + +hypothesis_gen: HypothesisGen = import_class(PROP_SETTING.hypothesis_gen)(scen) + +hypothesis2experiment: Hypothesis2Experiment = import_class(PROP_SETTING.hypothesis2experiment)() + +qlib_factor_coder: TaskGenerator = import_class(PROP_SETTING.qlib_factor_coder)() +qlib_factor_runner: TaskGenerator = import_class(PROP_SETTING.qlib_factor_runner)() + +qlib_factor_summarizer: Experiment2Feedback = import_class(PROP_SETTING.qlib_factor_summarizer)() + + +trace = Trace(scen=scen) +hs = HypothesisSet(trace=trace) +for _ in range(PROP_SETTING.evolving_n): + hypothesis = hypothesis_gen.gen(trace) + exp = hypothesis2experiment.convert(hs) + exp = qlib_factor_coder.generate(exp) + exp = qlib_factor_runner.generate(exp) + feedback = qlib_factor_summarizer.summarize(exp) + + trace.hist.append((hypothesis, exp, feedback)) diff --git a/rdagent/app/model_proposal/run.py b/rdagent/app/qlib_rd_loop/model.py similarity index 100% rename from rdagent/app/model_proposal/run.py rename to rdagent/app/qlib_rd_loop/model.py diff --git a/rdagent/components/idea_proposal/factor_proposal.py b/rdagent/components/idea_proposal/factor_proposal.py index 79f3cd6c..f9342f59 100644 --- a/rdagent/components/idea_proposal/factor_proposal.py +++ b/rdagent/components/idea_proposal/factor_proposal.py @@ -1,13 +1,18 @@ from abc import abstractmethod from pathlib import Path +from typing import Tuple from jinja2 import Environment, StrictUndefined +from rdagent.components.task_implementation.factor_implementation.factor import ( + FactorExperiment, +) from rdagent.core.prompts import Prompts from rdagent.core.proposal import ( Hypothesis, - Hypothesis2Task, + Hypothesis2Experiment, HypothesisGen, + HypothesisSet, Scenario, Trace, ) @@ -22,40 +27,71 @@ FactorHypothesis = Hypothesis class FactorHypothesisGen(HypothesisGen): def __init__(self, scen: Scenario): super().__init__(scen) - self.gen_context_flag = False - self.gen_context_dict = None - self.gen_json_flag = False # The following methods are scenario related so they should be implemented in the subclass @abstractmethod - def prepare_gen_context(self, trace: Trace) -> None: ... + def prepare_context(self, trace: Trace) -> Tuple[dict, bool]: ... @abstractmethod - def gen_response_to_hypothesis_list(self, response: str) -> FactorHypothesis: ... + def convert_response(self, response: str) -> FactorHypothesis: ... def gen(self, trace: Trace) -> FactorHypothesis: - assert self.gen_context_flag, "Please call prepare_gen_context before calling gen." - self.gen_context_flag = False # reset the flag + context_dict, json_flag = self.prepare_context(trace) system_prompt = ( Environment(undefined=StrictUndefined) .from_string(prompt_dict["factor_hypothesis_gen"]["system_prompt"]) - .render(scenario=self.scen.get_scenario_all_desc()) + .render( + scenario=self.scen.get_scenario_all_desc(), + hypothesis_output_format=context_dict["hypothesis_output_format"], + ) ) user_prompt = ( Environment(undefined=StrictUndefined) .from_string(prompt_dict["factor_hypothesis_gen"]["user_prompt"]) - .render(self.gen_context_dict) + .render( + hypothesis_and_feedback=context_dict["hypothesis_and_feedback"], + RAG=context_dict["RAG"], + ) ) - resp = APIBackend().build_messages_and_create_chat_completion( - user_prompt, system_prompt, json_mode=self.gen_json_flag - ) + resp = APIBackend().build_messages_and_create_chat_completion(user_prompt, system_prompt, json_mode=json_flag) - hypothesis = self.gen_response_to_hypothesis_list(resp) + hypothesis = self.convert_response(resp) return hypothesis -class FactorHypothesis2Task(Hypothesis2Task): - def convert(self, bs: FactorHypothesis) -> None: ... +class FactorHypothesis2Experiment(Hypothesis2Experiment[FactorExperiment]): + def __init__(self) -> None: + super().__init__() + + @abstractmethod + def prepare_context(self, hs: HypothesisSet) -> Tuple[dict, bool]: ... + + @abstractmethod + def convert_response(self, response: str) -> FactorExperiment: ... + + def convert(self, hs: HypothesisSet) -> FactorExperiment: + context, json_flag = self.prepare_context(hs) + system_prompt = ( + Environment(undefined=StrictUndefined) + .from_string(prompt_dict["factor_hypothesis2experiment"]["system_prompt"]) + .render( + scenario=hs.trace.scen.get_scenario_all_desc(), + experiment_output_format=context["experiment_output_format"], + ) + ) + user_prompt = ( + Environment(undefined=StrictUndefined) + .from_string(prompt_dict["factor_hypothesis2experiment"]["user_prompt"]) + .render( + hypothesis_and_feedback=context["hypothesis_and_feedback"], + factor_list=context["factor_list"], + RAG=context["RAG"], + ) + ) + + resp = APIBackend().build_messages_and_create_chat_completion(user_prompt, system_prompt, json_mode=json_flag) + + return self.convert_response(resp) diff --git a/rdagent/components/idea_proposal/prompts.yaml b/rdagent/components/idea_proposal/prompts.yaml index ad42a5be..e79fa70c 100644 --- a/rdagent/components/idea_proposal/prompts.yaml +++ b/rdagent/components/idea_proposal/prompts.yaml @@ -3,20 +3,36 @@ factor_hypothesis_gen: The user is trying to generate new hypothesis on the factors in data-driven research and development. The factors are used in a certain scenario, the scenario is as follows: {{ scenario }} - The user has made several hypothesis on this sencario and did several evaluation on them. The user will provide this information to you. + The user has made several hypothesis on this scenario and did several evaluation on them. The user will provide this information to you. To help you generate new hypothesis, the user has prepared some additional information for you. You should use this information to help generate new factors. + Please generate the output following the format below: + {{ hypothesis_output_format }} user_prompt: |- - The user has made several hypothesis on this sencario and did several evaluation on them. + The user has made several hypothesis on this scenario and did several evaluation on them. The former hypothesis and the corresponding feedbacks are as follows: {{ hypothesis_and_feedback }} To help you generate new factors, we have prepared the following information for you: {{ RAG }} - Please generate the new hypothesis based on the information above and generate the output following the format below: - {{ factor_output_format }} + Please generate the new hypothesis based on the information above. -factor_hypothesis_to_tasks: +factor_hypothesis2experiment: system_prompt: |- The user is trying to generate new factors based on the hypothesis generated in the previous step. The factors are used in certain scenario, the scenario is as follows: {{ scenario }} - + The user will use the factors generated to do some experiments. The user will provide this information to you: + 1. The hypothesis generated in the previous steps and their corresponding feedbacks. + 2. Former proposed factors on similar hypothesis. + 3. Some additional information to help you generate new factors. + Please generate the output following the format below: + {{ experiment_output_format }} + + user_prompt: |- + The user has made several hypothesis on this scenario and did several evaluation on them. + The former hypothesis and the corresponding feedbacks are as follows: + {{ hypothesis_and_feedback }} + The former proposed factors on similar hypothesis are as follows: + {{ factor_list }} + To help you generate new factors, we have prepared the following information for you: + {{ RAG }} + Please generate the new factors based on the information above. \ No newline at end of file diff --git a/rdagent/components/task_implementation/factor_implementation/evolving/evolvable_subjects.py b/rdagent/components/task_implementation/factor_implementation/evolving/evolvable_subjects.py index d339c4f6..a79d96a6 100644 --- a/rdagent/components/task_implementation/factor_implementation/evolving/evolvable_subjects.py +++ b/rdagent/components/task_implementation/factor_implementation/evolving/evolvable_subjects.py @@ -7,7 +7,7 @@ from rdagent.core.evolving_framework import EvolvableSubjects from rdagent.core.log import RDAgentLog -class FactorEvolvingItem(FactorExperiment[FactorTask, FileBasedFactorImplementation], EvolvableSubjects): +class FactorEvolvingItem(FactorExperiment, EvolvableSubjects): """ Intermediate item of factor implementation. """ diff --git a/rdagent/components/task_implementation/factor_implementation/factor.py b/rdagent/components/task_implementation/factor_implementation/factor.py index 59056886..91232693 100644 --- a/rdagent/components/task_implementation/factor_implementation/factor.py +++ b/rdagent/components/task_implementation/factor_implementation/factor.py @@ -220,4 +220,4 @@ class FileBasedFactorImplementation(FBImplementation): return FileBasedFactorImplementation(task, code=code, **kwargs) -FactorExperiment = Experiment +class FactorExperiment(Experiment[FactorTask, FileBasedFactorImplementation]): ... diff --git a/rdagent/core/experiment.py b/rdagent/core/experiment.py index 7e8c95ac..b209e3c9 100644 --- a/rdagent/core/experiment.py +++ b/rdagent/core/experiment.py @@ -119,7 +119,7 @@ class Experiment(ABC, Generic[ASpecificTask, ASpecificImp]): The experiment is a sequence of tasks and the implementations of the tasks after generated by the TaskGenerator. """ - def __init__(self, sub_tasks: Sequence[Task]) -> None: + def __init__(self, sub_tasks: Sequence[ASpecificTask]) -> None: self.sub_tasks = sub_tasks self.sub_implementations: Sequence[ASpecificImp] = [None for _ in self.sub_tasks] diff --git a/rdagent/core/proposal.py b/rdagent/core/proposal.py index d70b4a26..703292a0 100644 --- a/rdagent/core/proposal.py +++ b/rdagent/core/proposal.py @@ -2,7 +2,8 @@ """ -from typing import Dict, List, Tuple +from abc import ABC, abstractmethod +from typing import Dict, Generic, List, Tuple, TypeVar from rdagent.core.evolving_framework import Feedback from rdagent.core.experiment import Experiment, Implementation, Loader, Task @@ -18,8 +19,9 @@ class Hypothesis: - Belief """ - hypothesis: str = None - reason: str = None + def __init__(self, hypothesis: str, reason: str) -> None: + self.hypothesis: str = hypothesis + self.reason: str = reason # source: data_ana | model_nan = None @@ -27,16 +29,29 @@ class Hypothesis: # Origin(path of repo/data/feedback) => view/summarization => generated Hypothesis -class Scenario: - def get_repo_path(self): - """codebase""" +class Scenario(ABC): - def get_data(self): - """ "data info""" + @property + @abstractmethod + def background(self): + """Background information""" - def get_env(self): - """env description""" + @property + @abstractmethod + def source_data(self): + """Source data description""" + @property + @abstractmethod + def interface(self): + """Interface description about how to run the code""" + + @property + @abstractmethod + def simulator(self): + """Simulator description""" + + @abstractmethod def get_scenario_all_desc(self) -> str: """Combine all the description together""" @@ -44,9 +59,13 @@ class Scenario: class HypothesisFeedback(Feedback): ... -class Trace: - scen: Scenario - hist: list[Tuple[Hypothesis, Experiment, HypothesisFeedback]] +ASpecificScen = TypeVar("ASpecificScen", bound=Scenario) + + +class Trace(Generic[ASpecificScen]): + def __init__(self, scen: ASpecificScen) -> None: + self.scen: ASpecificScen = scen + self.hist: list[Tuple[Hypothesis, Experiment, HypothesisFeedback]] = [] class HypothesisGen: @@ -74,16 +93,21 @@ class HypothesisSet: true_hypothesis or false_hypothesis """ - hypothesis_list: list[Hypothesis] - trace: Trace + def __init__(self, trace: Trace, hypothesis_list: list[Hypothesis] = []) -> None: + self.hypothesis_list: list[Hypothesis] = hypothesis_list + self.trace: Trace = trace -class Hypothesis2Experiment(Loader[Experiment]): +ASpecificExp = TypeVar("ASpecificExp", bound=Experiment) + + +class Hypothesis2Experiment(ABC, Generic[ASpecificExp]): """ [Abstract description => concrete description] => Code implement """ - def convert(self, bs: HypothesisSet) -> Experiment: + @abstractmethod + def convert(self, hs: HypothesisSet) -> ASpecificExp: """Connect the idea proposal to implementation""" ... @@ -99,3 +123,4 @@ class Experiment2Feedback: The `ti` should be executed and the results should be included. For example: `mlflow` of Qlib will be included. """ + return HypothesisFeedback() diff --git a/rdagent/scenarios/qlib/experiment/factor_experiment.py b/rdagent/scenarios/qlib/experiment/factor_experiment.py index 69960f68..e0401b68 100644 --- a/rdagent/scenarios/qlib/experiment/factor_experiment.py +++ b/rdagent/scenarios/qlib/experiment/factor_experiment.py @@ -1,5 +1,43 @@ +from pathlib import Path + from rdagent.components.task_implementation.factor_implementation.factor import ( FactorExperiment, ) +from rdagent.components.task_implementation.factor_implementation.utils import ( + get_data_folder_intro, +) +from rdagent.core.prompts import Prompts +from rdagent.core.proposal import Scenario + +prompt_dict = Prompts(file_path=Path(__file__).parent / "prompts.yaml") QlibFactorExperiment = FactorExperiment + + +class QlibFactorScenario(Scenario): + @property + def background(self) -> str: + return prompt_dict["qlib_factor_background"] + + @property + def source_data(self) -> str: + return get_data_folder_intro() + + @property + def interface(self) -> str: + return prompt_dict["qlib_factor_interface"] + + @property + def simulator(self) -> str: + return prompt_dict["qlib_factor_simulator"] + + def get_scenario_all_desc(self) -> str: + return f"""Background of the scenario: +{self.background} +The source data you can use: +{self.source_data} +The interface you should follow to write the runnable code: +{self.interface} +The simulator user can use to test your factor: +{self.simulator} +""" diff --git a/rdagent/scenarios/qlib/experiment/prompts.yaml b/rdagent/scenarios/qlib/experiment/prompts.yaml new file mode 100644 index 00000000..6219e011 --- /dev/null +++ b/rdagent/scenarios/qlib/experiment/prompts.yaml @@ -0,0 +1,25 @@ +qlib_factor_background: |- + The factor is a characteristic or variable used in quant investment that can help explain the returns and risks of a portfolio or a single asset. Factors are used by investors to identify and exploit sources of excess returns, and they are central to many quantitative investment strategies. + Each number in the factor represents a physics value to an instrument on a day. + User will train a model to predict the next several days return based on the factor values of the previous days. + The factor is defined in the following parts: + 1. Name: The name of the factor. + 2. Description: The description of the factor. + 3. Formulation: The formulation of the factor. + 4. Variables: The variables or functions used in the formulation of the factor. + The factor might not provide all the parts of the information above since some might not be applicable. + Please specifically give all the hyperparameter in the factors like the window size, look back period, and so on. One factor should statically defines one output with a static source data. For example, last 10 days momentum and last 20 days momentum should be two different factors. + +qlib_factor_interface: |- + Your code should follow the interface to better interact with the user's system. + Your code should contain the following part: the import part, the function part, and the main part. You should write a main function name: "calculate_{function_name}" and call this function in "if __name__ == __main__" part. Don't write any try-except block in your code. The user will catch the exception message and provide the feedback to you. + User will write your code into a python file and execute the file directly with "python {your_file_name}.py". You should calculate the factor values and save the result into a HDF5(H5) file named "result.h5" in the same directory as your python file. The result file is a HDF5(H5) file containing a pandas dataframe. The index of the dataframe is the "datetime" and "instrument", and the single column name is the factor name,and the value is the factor value. The result file should be saved in the same directory as your python file. + +qlib_factor_simulator: |- + The factors will be sent into Qlib to train a model to predict the next several days return based on the factor values of the previous days. + Qlib is an AI-oriented quantitative investment platform that aims to realize the potential, empower research, and create value using AI technologies in quantitative investment, from exploring ideas to implementing productions. Qlib supports diverse machine learning modeling paradigms. including supervised learning, market dynamics modeling, and RL. + User will use Qlib to automatically do the following things: + 1. generate a new factor table based on the factor values. + 2. train a model like LightGBM, CatBoost, LSTM or simple PyTorch model to predict the next several days return based on the factor values. + 3. build a portfolio based on the predicted return based on a strategy. + 4. evaluate the portfolio's performance including the return, sharpe ratio, max drawdown, and so on. \ No newline at end of file diff --git a/rdagent/scenarios/qlib/factor_idea_proposal.py b/rdagent/scenarios/qlib/factor_idea_proposal.py deleted file mode 100644 index a7d41a16..00000000 --- a/rdagent/scenarios/qlib/factor_idea_proposal.py +++ /dev/null @@ -1,40 +0,0 @@ -import json -from pathlib import Path - -from jinja2 import Environment, StrictUndefined - -from rdagent.components.idea_proposal.factor_proposal import ( - FactorHypothesis, - FactorHypothesisGen, -) -from rdagent.core.prompts import Prompts -from rdagent.core.proposal import Scenario, Trace - -prompt_dict = Prompts(file_path=Path(__file__).parent / "prompts.yaml") - -QlibFactorHypothesis = FactorHypothesis - - -class QlibFactorHypothesisGen(FactorHypothesisGen): - def __init__(self, scen: Scenario): - super().__init__(scen) - self.gen_json_flag = True - - def prepare_gen_context(self, trace: Trace) -> None: - self.gen_context_flag = True - - hypothesis_feedback = ( - Environment(undefined=StrictUndefined) - .from_string(prompt_dict["hypothesis_and_feedback"]) - .render(trace=trace) - ) - self.gen_context_dict = { - "hypothesis_and_feedback": hypothesis_feedback, - "RAG": ..., - "factor_output_format": prompt_dict["output_format"], - } - - def gen_response_to_hypothesis_list(self, response: str) -> FactorHypothesis: - response_dict = json.loads(response) - hypothesis = QlibFactorHypothesis(hypothesis=response_dict["hypothesis"], reason=response_dict["reason"]) - return hypothesis diff --git a/rdagent/scenarios/qlib/factor_proposal.py b/rdagent/scenarios/qlib/factor_proposal.py new file mode 100644 index 00000000..ab2d494c --- /dev/null +++ b/rdagent/scenarios/qlib/factor_proposal.py @@ -0,0 +1,83 @@ +import json +from pathlib import Path +from typing import List, Tuple + +from jinja2 import Environment, StrictUndefined + +from rdagent.components.idea_proposal.factor_proposal import ( + FactorHypothesis, + FactorHypothesis2Experiment, + FactorHypothesisGen, +) +from rdagent.components.task_implementation.factor_implementation.factor import ( + FactorExperiment, + FactorTask, +) +from rdagent.components.task_implementation.factor_implementation.utils import ( + get_data_folder_intro, +) +from rdagent.core.prompts import Prompts +from rdagent.core.proposal import HypothesisSet, Scenario, Trace + +prompt_dict = Prompts(file_path=Path(__file__).parent / "prompts.yaml") + +QlibFactorHypothesis = FactorHypothesis + + +class QlibFactorHypothesisGen(FactorHypothesisGen): + def __init__(self, scen: Scenario) -> Tuple[dict, bool]: + super().__init__(scen) + + def prepare_context(self, trace: Trace) -> None: + hypothesis_feedback = ( + Environment(undefined=StrictUndefined) + .from_string(prompt_dict["hypothesis_and_feedback"]) + .render(trace=trace) + ) + context_dict = { + "hypothesis_and_feedback": hypothesis_feedback, + "RAG": ..., + "hypothesis_output_format": prompt_dict["hypothesis_output_format"], + } + return context_dict, True + + def convert_response(self, response: str) -> FactorHypothesis: + response_dict = json.loads(response) + hypothesis = QlibFactorHypothesis(hypothesis=response_dict["hypothesis"], reason=response_dict["reason"]) + return hypothesis + + +class QlibFactorHypothesis2Experiment(FactorHypothesis2Experiment): + def prepare_context(self, hs: HypothesisSet) -> Tuple[dict | bool]: + scenario = hs.trace.scen.get_scenario_all_desc() + experiment_output_format = prompt_dict["experiment_output_format"] + + hypothesis_and_feedback = ( + Environment(undefined=StrictUndefined) + .from_string(prompt_dict["hypothesis_and_feedback"]) + .render(trace=hs.trace) + ) + + experiment_list: List[FactorExperiment] = [t[1] for t in hs.trace.hist] + + factor_list = [] + for experiment in experiment_list: + factor_list.extend(experiment.sub_tasks) + + return { + "scenario": scenario, + "hypothesis_and_feedback": hypothesis_and_feedback, + "experiment_output_format": experiment_output_format, + "factor_list": factor_list, + "RAG": ..., + }, True + + def convert_response(self, response: str) -> FactorExperiment: + response_dict = json.loads(response) + tasks = [] + for factor_name in response_dict: + description = response_dict[factor_name]["description"] + formulation = response_dict[factor_name]["formulation"] + variables = response_dict[factor_name]["variables"] + tasks.append(FactorTask(factor_name, description, formulation, variables)) + return FactorExperiment(tasks) diff --git a/rdagent/scenarios/qlib/prompts.yaml b/rdagent/scenarios/qlib/prompts.yaml index 336d1602..6f48af23 100644 --- a/rdagent/scenarios/qlib/prompts.yaml +++ b/rdagent/scenarios/qlib/prompts.yaml @@ -1,12 +1,34 @@ hypothesis_and_feedback: |- - {% for hypothesis, feedback in trace %} + {% for hypothesis, experiment, feedback in trace.hist %} Hypothesis {{ loop.index }}: {{ hypothesis }} Feedback: {{ feedback }} {% endfor %} -output_format: |- +hypothesis_output_format: |- The output should follow JSON format. The schema is as follows: { "hypothesis": "The new hypothesis generated based on the information provided.", "reason": "The reason why you generate this hypothesis." - } \ No newline at end of file + } + +experiment_output_format: |- + The output should follow JSON format. The schema is as follows: + { + "factor name 1": { + "description": "description of factor 1", + "formulation": "latex formulation of factor 1", + "variables": { + "variable or function name 1": "description of variable or function 1", + "variable or function name 2": "description of variable or function 2" + } + }, + "factor name 2": { + "description": "description of factor 1", + "formulation": "latex formulation of factor 2", + "variables": { + "variable or function name 1": "description of variable or function 1", + "variable or function name 2": "description of variable or function 2" + } + } + # Don't add ellipsis (...) or any filler text that might cause JSON parsing errors here! + } diff --git a/rdagent/scenarios/qlib/task_generator/data.py b/rdagent/scenarios/qlib/task_generator/data.py index a052286e..abc63a85 100644 --- a/rdagent/scenarios/qlib/task_generator/data.py +++ b/rdagent/scenarios/qlib/task_generator/data.py @@ -2,7 +2,7 @@ from rdagent.core.task_generator import TaskGenerator from rdagent.scenarios.qlib.experiment.factor_experiment import QlibFactorExperiment -class QlibDataImplementation(TaskGenerator[QlibFactorExperiment]): +class QlibFactorRunner(TaskGenerator[QlibFactorExperiment]): """ Docker run Everything in a folder @@ -13,3 +13,6 @@ class QlibDataImplementation(TaskGenerator[QlibFactorExperiment]): - TODO: implement a qlib handler """ + + def generate(self, exp: QlibFactorExperiment) -> QlibFactorExperiment: + return exp # TODO IMPLEMENT THIS diff --git a/rdagent/scenarios/qlib/task_generator/feedback.py b/rdagent/scenarios/qlib/task_generator/feedback.py index f2e8ce91..ea4a2ce8 100644 --- a/rdagent/scenarios/qlib/task_generator/feedback.py +++ b/rdagent/scenarios/qlib/task_generator/feedback.py @@ -1,2 +1,7 @@ # TODO: # Implement to feedback. + +from rdagent.core.proposal import Experiment2Feedback + + +class QlibFactorExperiment2Feedback(Experiment2Feedback): ...