diff --git a/rdagent/app/CI/run.py b/rdagent/app/CI/run.py index d3671cc2..82d2a1df 100644 --- a/rdagent/app/CI/run.py +++ b/rdagent/app/CI/run.py @@ -23,9 +23,9 @@ from rich.table import Table from rich.text import Text from tree_sitter import Language, Node, Parser +from rdagent.core.evaluation import Evaluator +from rdagent.core.evolving_agent import EvoAgent from rdagent.core.evolving_framework import ( - Evaluator, - EvoAgent, EvolvableSubjects, EvolvingStrategy, EvoStep, diff --git a/rdagent/app/factor_extraction_and_implementation/factor_extract_and_implement.py b/rdagent/app/factor_extraction_and_implementation/factor_extract_and_implement.py index f0fd5435..830e8216 100644 --- a/rdagent/app/factor_extraction_and_implementation/factor_extract_and_implement.py +++ b/rdagent/app/factor_extraction_and_implementation/factor_extract_and_implement.py @@ -1,18 +1,18 @@ # %% from dotenv import load_dotenv +from rdagent.scenarios.qlib.factor_experiment_loader.pdf_loader import ( + FactorImplementationExperimentLoaderFromPDFfiles, +) from rdagent.scenarios.qlib.factor_task_implementation import ( COSTEERFG_QUANT_FACTOR_IMPLEMENTATION, ) -from rdagent.scenarios.qlib.factor_task_loader.pdf_loader import ( - FactorImplementationTaskLoaderFromPDFfiles, -) assert load_dotenv() def extract_factors_and_implement(report_file_path: str) -> None: - factor_tasks = FactorImplementationTaskLoaderFromPDFfiles().load(report_file_path) + factor_tasks = FactorImplementationExperimentLoaderFromPDFfiles().load(report_file_path) implementation_result = COSTEERFG_QUANT_FACTOR_IMPLEMENTATION().generate(factor_tasks) # Qlib to run the implementation return implementation_result diff --git a/rdagent/app/model_implementation/eval.py b/rdagent/app/model_implementation/eval.py index 1460e618..e28aed5b 100644 --- a/rdagent/app/model_implementation/eval.py +++ b/rdagent/app/model_implementation/eval.py @@ -2,13 +2,19 @@ from pathlib import Path DIRNAME = Path(__file__).absolute().resolve().parent -from rdagent.components.task_implementation.model_implementation.benchmark.eval import ModelImpValEval -from rdagent.components.task_implementation.model_implementation.one_shot import ModelTaskGen -from rdagent.components.task_implementation.model_implementation.task import ModelImpLoader, ModelTaskLoderJson - +from rdagent.components.task_implementation.model_implementation.benchmark.eval import ( + ModelImpValEval, +) +from rdagent.components.task_implementation.model_implementation.one_shot import ( + ModelTaskGen, +) +from rdagent.components.task_implementation.model_implementation.task import ( + ModelImpLoader, + ModelTaskLoaderJson, +) bench_folder = DIRNAME.parent.parent / "components" / "task_implementation" / "model_implementation" / "benchmark" -mtl = ModelTaskLoderJson(str(bench_folder / "model_dict.json")) +mtl = ModelTaskLoaderJson(str(bench_folder / "model_dict.json")) task_l = mtl.load() diff --git a/rdagent/app/model_proposal/run.py b/rdagent/app/model_proposal/run.py index fad65bf0..26b8d306 100644 --- a/rdagent/app/model_proposal/run.py +++ b/rdagent/app/model_proposal/run.py @@ -5,32 +5,37 @@ TODO: move the following code to a new class: Model_RD_Agent # import_from from rdagent.app.model_proposal.conf import MODEL_PROP_SETTING -from rdagent.core.implementation import TaskGenerator -from rdagent.core.proposal import Belief2Task, BeliefSet, Imp2Feedback, Trace +from rdagent.core.proposal import ( + Experiment2Feedback, + Hypothesis2Experiment, + HypothesisSet, + Trace, +) +from rdagent.core.task_generator import TaskGenerator # load_from_cls_uri scen = load_from_cls_uri(MODEL_PROP_SETTING.scen)() -belief_gen = load_from_cls_uri(MODEL_PROP_SETTING.belief_gen)(scen) +hypothesis_gen = load_from_cls_uri(MODEL_PROP_SETTING.hypothesis_gen)(scen) -belief2task: Belief2Task = load_from_cls_uri(MODEL_PROP_SETTING.belief2task)() +hypothesis2task: Hypothesis2Experiment = load_from_cls_uri(MODEL_PROP_SETTING.hypothesis2task)() task_gen: TaskGenerator = load_from_cls_uri(MODEL_PROP_SETTING.task_gen)(scen) # for implementation -imp2feedback: Imp2Feedback = load_from_cls_uri(MODEL_PROP_SETTING.imp2feedback)(scen) # for implementation +imp2feedback: Experiment2Feedback = load_from_cls_uri(MODEL_PROP_SETTING.imp2feedback)(scen) # for implementation iter_n = MODEL_PROP_SETTING.iter_n trace = Trace() -belief_set = BeliefSet() +hypothesis_set = HypothesisSet() for _ in range(iter_n): - belief = belief_gen.gen(trace) - task = belief2task.convert(belief) + hypothesis = hypothesis_gen.gen(trace) + task = hypothesis2task.convert(hypothesis) imp = task_gen.gen(task) imp.execute() feedback = imp2feedback.summarize(imp) - trace.hist.append((belief, feedback)) + trace.hist.append((hypothesis, feedback)) diff --git a/rdagent/components/benchmark/eval_method.py b/rdagent/components/benchmark/eval_method.py index 8bc81012..4161605a 100644 --- a/rdagent/components/benchmark/eval_method.py +++ b/rdagent/components/benchmark/eval_method.py @@ -4,6 +4,9 @@ from typing import List, Tuple, Union from tqdm import tqdm +from rdagent.components.task_implementation.factor_implementation.config import ( + FACTOR_IMPLEMENT_SETTINGS, +) from rdagent.components.task_implementation.factor_implementation.evolving.evaluators import ( FactorImplementationCorrelationEvaluator, FactorImplementationEvaluator, @@ -14,18 +17,25 @@ from rdagent.components.task_implementation.factor_implementation.evolving.evalu FactorImplementationSingleColumnEvaluator, FactorImplementationValuesEvaluator, ) -from rdagent.components.task_implementation.factor_implementation.evolving.factor import ( +from rdagent.components.task_implementation.factor_implementation.factor import ( FileBasedFactorImplementation, ) -from rdagent.components.task_implementation.factor_implementation.share_modules.factor_implementation_config import ( - FACTOR_IMPLEMENT_SETTINGS, -) from rdagent.core.exception import ImplementRunException -from rdagent.core.implementation import TaskGenerator -from rdagent.core.task import TaskImplementation, TestCase +from rdagent.core.experiment import Implementation, Task +from rdagent.core.task_generator import TaskGenerator from rdagent.core.utils import multiprocessing_wrapper +class TestCase: + def __init__( + self, + target_task: list[Task] = [], + ground_truth: list[Implementation] = [], + ): + self.ground_truth = ground_truth + self.target_task = target_task + + class BaseEval: """ The benchmark benchmark evaluation. @@ -56,7 +66,7 @@ class BaseEval: self, path: Union[Path, str], **kwargs, - ) -> List[TaskImplementation]: + ) -> List[Implementation]: path = Path(path) fi_l = [] for tc in self.test_cases: @@ -69,8 +79,8 @@ class BaseEval: def eval_case( self, - case_gt: TaskImplementation, - case_gen: TaskImplementation, + case_gt: Implementation, + case_gen: Implementation, ) -> List[Union[Tuple[FactorImplementationEvaluator, object], Exception]]: """Parameters ---------- @@ -137,11 +147,11 @@ class FactorImplementEval(BaseEval): print("Manually interrupted the evaluation. Saving existing results") break - if len(gen_factor_l.corresponding_implementations) != len(self.test_cases.ground_truth): + if len(gen_factor_l.sub_implementations) != len(self.test_cases.ground_truth): raise ValueError( "The number of cases to eval should be equal to the number of test cases.", ) - gen_factor_l_all_rounds.extend(gen_factor_l.corresponding_implementations) + gen_factor_l_all_rounds.extend(gen_factor_l.sub_implementations) test_cases_all_rounds.extend(self.test_cases.ground_truth) eval_res_list = multiprocessing_wrapper( diff --git a/rdagent/components/idea_proposal/factor_proposal.py b/rdagent/components/idea_proposal/factor_proposal.py index e69de29b..79f3cd6c 100644 --- a/rdagent/components/idea_proposal/factor_proposal.py +++ b/rdagent/components/idea_proposal/factor_proposal.py @@ -0,0 +1,61 @@ +from abc import abstractmethod +from pathlib import Path + +from jinja2 import Environment, StrictUndefined + +from rdagent.core.prompts import Prompts +from rdagent.core.proposal import ( + Hypothesis, + Hypothesis2Task, + HypothesisGen, + Scenario, + Trace, +) +from rdagent.oai.llm_utils import APIBackend + +prompt_dict = Prompts(file_path=Path(__file__).parent / "prompts.yaml") + + +FactorHypothesis = Hypothesis + + +class FactorHypothesisGen(HypothesisGen): + def __init__(self, scen: Scenario): + super().__init__(scen) + self.gen_context_flag = False + self.gen_context_dict = None + self.gen_json_flag = False + + # The following methods are scenario related so they should be implemented in the subclass + @abstractmethod + def prepare_gen_context(self, trace: Trace) -> None: ... + + @abstractmethod + def gen_response_to_hypothesis_list(self, response: str) -> FactorHypothesis: ... + + def gen(self, trace: Trace) -> FactorHypothesis: + assert self.gen_context_flag, "Please call prepare_gen_context before calling gen." + self.gen_context_flag = False # reset the flag + + system_prompt = ( + Environment(undefined=StrictUndefined) + .from_string(prompt_dict["factor_hypothesis_gen"]["system_prompt"]) + .render(scenario=self.scen.get_scenario_all_desc()) + ) + user_prompt = ( + Environment(undefined=StrictUndefined) + .from_string(prompt_dict["factor_hypothesis_gen"]["user_prompt"]) + .render(self.gen_context_dict) + ) + + resp = APIBackend().build_messages_and_create_chat_completion( + user_prompt, system_prompt, json_mode=self.gen_json_flag + ) + + hypothesis = self.gen_response_to_hypothesis_list(resp) + + return hypothesis + + +class FactorHypothesis2Task(Hypothesis2Task): + def convert(self, bs: FactorHypothesis) -> None: ... diff --git a/rdagent/components/idea_proposal/prompts.yaml b/rdagent/components/idea_proposal/prompts.yaml new file mode 100644 index 00000000..ad42a5be --- /dev/null +++ b/rdagent/components/idea_proposal/prompts.yaml @@ -0,0 +1,22 @@ +factor_hypothesis_gen: + system_prompt: |- + The user is trying to generate new hypothesis on the factors in data-driven research and development. + The factors are used in a certain scenario, the scenario is as follows: + {{ scenario }} + The user has made several hypothesis on this sencario and did several evaluation on them. The user will provide this information to you. + To help you generate new hypothesis, the user has prepared some additional information for you. You should use this information to help generate new factors. + user_prompt: |- + The user has made several hypothesis on this sencario and did several evaluation on them. + The former hypothesis and the corresponding feedbacks are as follows: + {{ hypothesis_and_feedback }} + To help you generate new factors, we have prepared the following information for you: + {{ RAG }} + Please generate the new hypothesis based on the information above and generate the output following the format below: + {{ factor_output_format }} + +factor_hypothesis_to_tasks: + system_prompt: |- + The user is trying to generate new factors based on the hypothesis generated in the previous step. + The factors are used in certain scenario, the scenario is as follows: + {{ scenario }} + diff --git a/rdagent/components/loader/experiment_loader.py b/rdagent/components/loader/experiment_loader.py new file mode 100644 index 00000000..ee929303 --- /dev/null +++ b/rdagent/components/loader/experiment_loader.py @@ -0,0 +1,12 @@ +from rdagent.components.task_implementation.factor_implementation.factor import ( + FactorExperiment, +) +from rdagent.core.experiment import Loader + + +class FactorExperimentLoader(Loader[FactorExperiment]): + pass + + +class ModelExperimentLoader(Loader[FactorExperiment]): + pass diff --git a/rdagent/components/loader/task_loader.py b/rdagent/components/loader/task_loader.py new file mode 100644 index 00000000..862b557c --- /dev/null +++ b/rdagent/components/loader/task_loader.py @@ -0,0 +1,12 @@ +from rdagent.components.task_implementation.factor_implementation.factor import ( + FactorTask, +) +from rdagent.core.experiment import Loader + + +class FactorTaskLoader(Loader[FactorTask]): + pass + + +class ModelTaskLoader(Loader[FactorTask]): + pass diff --git a/rdagent/components/task_implementation/factor_implementation/CoSTEER.py b/rdagent/components/task_implementation/factor_implementation/CoSTEER.py index 554193ba..6b90ad04 100644 --- a/rdagent/components/task_implementation/factor_implementation/CoSTEER.py +++ b/rdagent/components/task_implementation/factor_implementation/CoSTEER.py @@ -2,31 +2,33 @@ import pickle from pathlib import Path from typing import List +from rdagent.components.task_implementation.factor_implementation.config import ( + FACTOR_IMPLEMENT_SETTINGS, +) from rdagent.components.task_implementation.factor_implementation.evolving.evaluators import ( FactorImplementationEvaluatorV1, FactorImplementationsMultiEvaluator, ) +from rdagent.components.task_implementation.factor_implementation.evolving.evolvable_subjects import ( + FactorEvolvingItem, +) from rdagent.components.task_implementation.factor_implementation.evolving.evolving_strategy import ( FactorEvolvingStrategyWithGraph, ) -from rdagent.components.task_implementation.factor_implementation.evolving.factor import ( - FactorEvovlingItem, - FactorImplementTask, -) from rdagent.components.task_implementation.factor_implementation.evolving.knowledge_management import ( FactorImplementationGraphKnowledgeBase, FactorImplementationGraphRAGStrategy, FactorImplementationKnowledgeBaseV1, ) -from rdagent.components.task_implementation.factor_implementation.share_modules.factor_implementation_config import ( - FACTOR_IMPLEMENT_SETTINGS, +from rdagent.components.task_implementation.factor_implementation.factor import ( + FactorExperiment, ) from rdagent.core.evolving_agent import RAGEvoAgent -from rdagent.core.implementation import TaskGenerator -from rdagent.core.task import TaskImplementation +from rdagent.core.experiment import Experiment +from rdagent.core.task_generator import TaskGenerator -class CoSTEERFG(TaskGenerator): +class CoSTEERFG(TaskGenerator[FactorExperiment]): def __init__( self, with_knowledge: bool = True, @@ -74,7 +76,7 @@ class CoSTEERFG(TaskGenerator): ) return factor_knowledge_base - def generate(self, tasks: List[FactorImplementTask]) -> List[TaskImplementation]: + def generate(self, exp: FactorExperiment) -> FactorExperiment: # init knowledge base factor_knowledge_base = self.load_or_init_knowledge_base( former_knowledge_base_path=self.knowledge_base_path, @@ -83,8 +85,8 @@ class CoSTEERFG(TaskGenerator): # init rag method self.rag = FactorImplementationGraphRAGStrategy(factor_knowledge_base) - # init indermediate items - factor_implementations = FactorEvovlingItem(target_factor_tasks=tasks) + # init intermediate items + factor_implementations = FactorEvolvingItem(sub_tasks=exp.sub_tasks) self.evolve_agent = RAGEvoAgent(max_loop=self.max_loop, evolving_strategy=self.evolving_strategy, rag=self.rag) @@ -100,5 +102,5 @@ class CoSTEERFG(TaskGenerator): if self.new_knowledge_base_path is not None: pickle.dump(factor_knowledge_base, open(self.new_knowledge_base_path, "wb")) self.knowledge_base = factor_knowledge_base - self.latest_factor_implementations = tasks + self.latest_factor_implementations = exp.sub_tasks return factor_implementations diff --git a/rdagent/components/task_implementation/factor_implementation/share_modules/factor_implementation_config.py b/rdagent/components/task_implementation/factor_implementation/config.py similarity index 100% rename from rdagent/components/task_implementation/factor_implementation/share_modules/factor_implementation_config.py rename to rdagent/components/task_implementation/factor_implementation/config.py diff --git a/rdagent/components/task_implementation/factor_implementation/evolving/evaluators.py b/rdagent/components/task_implementation/factor_implementation/evolving/evaluators.py index 7303de20..19289031 100644 --- a/rdagent/components/task_implementation/factor_implementation/evolving/evaluators.py +++ b/rdagent/components/task_implementation/factor_implementation/evolving/evaluators.py @@ -5,21 +5,23 @@ from pathlib import Path from typing import List, Tuple import pandas as pd -from jinja2 import Template +from jinja2 import Environment, StrictUndefined -from rdagent.components.task_implementation.factor_implementation.evolving.evolving_strategy import ( - FactorEvovlingItem, - FactorImplementTask, -) -from rdagent.components.task_implementation.factor_implementation.share_modules.factor_implementation_config import ( +from rdagent.components.task_implementation.factor_implementation.config import ( FACTOR_IMPLEMENT_SETTINGS, ) +from rdagent.components.task_implementation.factor_implementation.evolving.evolvable_subjects import ( + FactorEvolvingItem, +) +from rdagent.components.task_implementation.factor_implementation.evolving.evolving_strategy import ( + FactorTask, +) from rdagent.core.conf import RD_AGENT_SETTINGS from rdagent.core.evaluation import Evaluator from rdagent.core.evolving_framework import Feedback, QueriedKnowledge +from rdagent.core.experiment import Implementation from rdagent.core.log import RDAgentLog from rdagent.core.prompts import Prompts -from rdagent.core.task import TaskImplementation from rdagent.core.utils import multiprocessing_wrapper from rdagent.oai.llm_utils import APIBackend @@ -33,8 +35,8 @@ class FactorImplementationEvaluator(Evaluator): @abstractmethod def evaluate( self, - gt: TaskImplementation, - gen: TaskImplementation, + gt: Implementation, + gen: Implementation, ) -> Tuple[str, object]: """You can get the dataframe by @@ -52,7 +54,7 @@ class FactorImplementationEvaluator(Evaluator): """ raise NotImplementedError("Please implement the `evaluator` method") - def _get_df(self, gt: TaskImplementation, gen: TaskImplementation): + def _get_df(self, gt: Implementation, gen: Implementation): _, gt_df = gt.execute() _, gen_df = gen.execute() if isinstance(gen_df, pd.Series): @@ -65,11 +67,11 @@ class FactorImplementationEvaluator(Evaluator): class FactorImplementationCodeEvaluator(Evaluator): def evaluate( self, - target_task: FactorImplementTask, - implementation: TaskImplementation, + target_task: FactorTask, + implementation: Implementation, execution_feedback: str, factor_value_feedback: str = "", - gt_implementation: TaskImplementation = None, + gt_implementation: Implementation = None, **kwargs, ): factor_information = target_task.get_factor_information() @@ -78,14 +80,18 @@ class FactorImplementationCodeEvaluator(Evaluator): system_prompt = evaluate_prompts["evaluator_code_feedback_v1_system"] execution_feedback_to_render = execution_feedback - user_prompt = Template( - evaluate_prompts["evaluator_code_feedback_v1_user"], - ).render( - factor_information=factor_information, - code=code, - execution_feedback=execution_feedback_to_render, - factor_value_feedback=factor_value_feedback, - gt_code=gt_implementation.code if gt_implementation else None, + user_prompt = ( + Environment(undefined=StrictUndefined) + .from_string( + evaluate_prompts["evaluator_code_feedback_v1_user"], + ) + .render( + factor_information=factor_information, + code=code, + execution_feedback=execution_feedback_to_render, + factor_value_feedback=factor_value_feedback, + gt_code=gt_implementation.code if gt_implementation else None, + ) ) while ( APIBackend().build_messages_and_calculate_token( @@ -96,14 +102,18 @@ class FactorImplementationCodeEvaluator(Evaluator): > RD_AGENT_SETTINGS.chat_token_limit ): execution_feedback_to_render = execution_feedback_to_render[len(execution_feedback_to_render) // 2 :] - user_prompt = Template( - evaluate_prompts["evaluator_code_feedback_v1_user"], - ).render( - factor_information=factor_information, - code=code, - execution_feedback=execution_feedback_to_render, - factor_value_feedback=factor_value_feedback, - gt_code=gt_implementation.code if gt_implementation else None, + user_prompt = ( + Environment(undefined=StrictUndefined) + .from_string( + evaluate_prompts["evaluator_code_feedback_v1_user"], + ) + .render( + factor_information=factor_information, + code=code, + execution_feedback=execution_feedback_to_render, + factor_value_feedback=factor_value_feedback, + gt_code=gt_implementation.code if gt_implementation else None, + ) ) critic_response = APIBackend().build_messages_and_create_chat_completion( user_prompt=user_prompt, @@ -117,8 +127,8 @@ class FactorImplementationCodeEvaluator(Evaluator): class FactorImplementationSingleColumnEvaluator(FactorImplementationEvaluator): def evaluate( self, - gt: TaskImplementation, - gen: TaskImplementation, + gt: Implementation, + gen: Implementation, ) -> Tuple[str, object]: gt_df, gen_df = self._get_df(gt, gen) @@ -143,8 +153,8 @@ class FactorImplementationSingleColumnEvaluator(FactorImplementationEvaluator): class FactorImplementationIndexFormatEvaluator(FactorImplementationEvaluator): def evaluate( self, - gt: TaskImplementation, - gen: TaskImplementation, + gt: Implementation, + gen: Implementation, ) -> Tuple[str, object]: gt_df, gen_df = self._get_df(gt, gen) idx_name_right = gen_df.index.names == ("datetime", "instrument") @@ -166,8 +176,8 @@ class FactorImplementationIndexFormatEvaluator(FactorImplementationEvaluator): class FactorImplementationRowCountEvaluator(FactorImplementationEvaluator): def evaluate( self, - gt: TaskImplementation, - gen: TaskImplementation, + gt: Implementation, + gen: Implementation, ) -> Tuple[str, object]: gt_df, gen_df = self._get_df(gt, gen) @@ -186,8 +196,8 @@ class FactorImplementationRowCountEvaluator(FactorImplementationEvaluator): class FactorImplementationIndexEvaluator(FactorImplementationEvaluator): def evaluate( self, - gt: TaskImplementation, - gen: TaskImplementation, + gt: Implementation, + gen: Implementation, ) -> Tuple[str, object]: gt_df, gen_df = self._get_df(gt, gen) @@ -206,8 +216,8 @@ class FactorImplementationIndexEvaluator(FactorImplementationEvaluator): class FactorImplementationMissingValuesEvaluator(FactorImplementationEvaluator): def evaluate( self, - gt: TaskImplementation, - gen: TaskImplementation, + gt: Implementation, + gen: Implementation, ) -> Tuple[str, object]: gt_df, gen_df = self._get_df(gt, gen) @@ -226,8 +236,8 @@ class FactorImplementationMissingValuesEvaluator(FactorImplementationEvaluator): class FactorImplementationValuesEvaluator(FactorImplementationEvaluator): def evaluate( self, - gt: TaskImplementation, - gen: TaskImplementation, + gt: Implementation, + gen: Implementation, ) -> Tuple[str, object]: gt_df, gen_df = self._get_df(gt, gen) @@ -259,8 +269,8 @@ class FactorImplementationCorrelationEvaluator(FactorImplementationEvaluator): def evaluate( self, - gt: TaskImplementation, - gen: TaskImplementation, + gt: Implementation, + gen: Implementation, ) -> Tuple[str, object]: gt_df, gen_df = self._get_df(gt, gen) @@ -293,7 +303,7 @@ class FactorImplementationCorrelationEvaluator(FactorImplementationEvaluator): class FactorImplementationValEvaluator(FactorImplementationEvaluator): - def evaluate(self, gt: TaskImplementation, gen: TaskImplementation): + def evaluate(self, gt: Implementation, gen: Implementation): _, gt_df = gt.execute() _, gen_df = gen.execute() # FIXME: refactor the two classes @@ -473,7 +483,7 @@ def shorten_prompt(tpl: str, render_kwargs: dict, shorten_key: str, max_trail: i class FactorImplementationFinalDecisionEvaluator(Evaluator): def evaluate( self, - target_task: FactorImplementTask, + target_task: FactorTask, execution_feedback: str, value_feedback: str, code_feedback: str, @@ -483,17 +493,21 @@ class FactorImplementationFinalDecisionEvaluator(Evaluator): "evaluator_final_decision_v1_system" ] execution_feedback_to_render = execution_feedback - user_prompt = Template( - evaluate_prompts["evaluator_final_decision_v1_user"], - ).render( - factor_information=target_task.get_factor_information(), - execution_feedback=execution_feedback_to_render, - code_feedback=code_feedback, - factor_value_feedback=( - value_feedback - if value_feedback is not None - else "No Ground Truth Value provided, so no evaluation on value is performed." - ), + user_prompt = ( + Environment(undefined=StrictUndefined) + .from_string( + evaluate_prompts["evaluator_final_decision_v1_user"], + ) + .render( + factor_information=target_task.get_factor_information(), + execution_feedback=execution_feedback_to_render, + code_feedback=code_feedback, + factor_value_feedback=( + value_feedback + if value_feedback is not None + else "No Ground Truth Value provided, so no evaluation on value is performed." + ), + ) ) while ( APIBackend().build_messages_and_calculate_token( @@ -504,17 +518,21 @@ class FactorImplementationFinalDecisionEvaluator(Evaluator): > RD_AGENT_SETTINGS.chat_token_limit ): execution_feedback_to_render = execution_feedback_to_render[len(execution_feedback_to_render) // 2 :] - user_prompt = Template( - evaluate_prompts["evaluator_final_decision_v1_user"], - ).render( - factor_information=target_task.get_factor_information(), - execution_feedback=execution_feedback_to_render, - code_feedback=code_feedback, - factor_value_feedback=( - value_feedback - if value_feedback is not None - else "No Ground Truth Value provided, so no evaluation on value is performed." - ), + user_prompt = ( + Environment(undefined=StrictUndefined) + .from_string( + evaluate_prompts["evaluator_final_decision_v1_user"], + ) + .render( + factor_information=target_task.get_factor_information(), + execution_feedback=execution_feedback_to_render, + code_feedback=code_feedback, + factor_value_feedback=( + value_feedback + if value_feedback is not None + else "No Ground Truth Value provided, so no evaluation on value is performed." + ), + ) ) final_evaluation_dict = json.loads( @@ -584,9 +602,9 @@ class FactorImplementationEvaluatorV1(FactorImplementationEvaluator): def evaluate( self, - target_task: FactorImplementTask, - implementation: TaskImplementation, - gt_implementation: TaskImplementation = None, + target_task: FactorTask, + implementation: Implementation, + gt_implementation: Implementation = None, queried_knowledge: QueriedKnowledge = None, **kwargs, ) -> FactorImplementationSingleFeedback: @@ -681,23 +699,23 @@ class FactorImplementationsMultiEvaluator(Evaluator): def evaluate( self, - evo: FactorEvovlingItem, + evo: FactorEvolvingItem, queried_knowledge: QueriedKnowledge = None, **kwargs, ) -> FactorImplementationsMultiFeedback: multi_implementation_feedback = FactorImplementationsMultiFeedback() - # for index in range(len(evo.target_factor_tasks)): - # corresponding_implementation = evo.corresponding_implementations[index] + # for index in range(len(evo.sub_tasks)): + # corresponding_implementation = evo.sub_implementations[index] # corresponding_gt_implementation = ( - # evo.corresponding_gt_implementations[index] - # if evo.corresponding_gt_implementations is not None + # evo.sub_gt_implementations[index] + # if evo.sub_gt_implementations is not None # else None # ) # multi_implementation_feedback.append( # self.single_factor_implementation_evaluator.evaluate( - # target_task=evo.target_factor_tasks[index], + # target_task=evo.sub_tasks[index], # implementation=corresponding_implementation, # gt_implementation=corresponding_gt_implementation, # queried_knowledge=queried_knowledge, @@ -705,18 +723,16 @@ class FactorImplementationsMultiEvaluator(Evaluator): # ) calls = [] - for index in range(len(evo.target_factor_tasks)): - corresponding_implementation = evo.corresponding_implementations[index] + for index in range(len(evo.sub_tasks)): + corresponding_implementation = evo.sub_implementations[index] corresponding_gt_implementation = ( - evo.corresponding_gt_implementations[index] - if evo.corresponding_gt_implementations is not None - else None + evo.sub_gt_implementations[index] if evo.sub_gt_implementations is not None else None ) calls.append( ( self.single_factor_implementation_evaluator.evaluate, ( - evo.target_factor_tasks[index], + evo.sub_tasks[index], corresponding_implementation, corresponding_gt_implementation, queried_knowledge, diff --git a/rdagent/components/task_implementation/factor_implementation/evolving/evolvable_subjects.py b/rdagent/components/task_implementation/factor_implementation/evolving/evolvable_subjects.py new file mode 100644 index 00000000..d339c4f6 --- /dev/null +++ b/rdagent/components/task_implementation/factor_implementation/evolving/evolvable_subjects.py @@ -0,0 +1,30 @@ +from rdagent.components.task_implementation.factor_implementation.factor import ( + FactorExperiment, + FactorTask, + FileBasedFactorImplementation, +) +from rdagent.core.evolving_framework import EvolvableSubjects +from rdagent.core.log import RDAgentLog + + +class FactorEvolvingItem(FactorExperiment[FactorTask, FileBasedFactorImplementation], EvolvableSubjects): + """ + Intermediate item of factor implementation. + """ + + def __init__( + self, + sub_tasks: list[FactorTask], + sub_gt_implementations: list[FileBasedFactorImplementation] = None, + ): + FactorExperiment.__init__(self, sub_tasks=sub_tasks) + self.corresponding_selection: list = None + if sub_gt_implementations is not None and len( + sub_gt_implementations, + ) != len(self.sub_tasks): + self.sub_gt_implementations = None + RDAgentLog().warning( + "The length of sub_gt_implementations is not equal to the length of sub_tasks, set sub_gt_implementations to None", + ) + else: + self.sub_gt_implementations = sub_gt_implementations diff --git a/rdagent/components/task_implementation/factor_implementation/evolving/evolving_strategy.py b/rdagent/components/task_implementation/factor_implementation/evolving/evolving_strategy.py index 36a8ae9a..7189bae2 100644 --- a/rdagent/components/task_implementation/factor_implementation/evolving/evolving_strategy.py +++ b/rdagent/components/task_implementation/factor_implementation/evolving/evolving_strategy.py @@ -6,27 +6,29 @@ from copy import deepcopy from pathlib import Path from typing import TYPE_CHECKING -from jinja2 import Template +from jinja2 import Environment, StrictUndefined -from rdagent.components.task_implementation.factor_implementation.evolving.factor import ( - FactorEvovlingItem, - FactorImplementTask, - FileBasedFactorImplementation, +from rdagent.components.task_implementation.factor_implementation.config import ( + FACTOR_IMPLEMENT_SETTINGS, +) +from rdagent.components.task_implementation.factor_implementation.evolving.evolvable_subjects import ( + FactorEvolvingItem, ) from rdagent.components.task_implementation.factor_implementation.evolving.scheduler import ( LLMSelect, RandomSelect, ) -from rdagent.components.task_implementation.factor_implementation.share_modules.factor_implementation_config import ( - FACTOR_IMPLEMENT_SETTINGS, +from rdagent.components.task_implementation.factor_implementation.factor import ( + FactorTask, + FileBasedFactorImplementation, ) -from rdagent.components.task_implementation.factor_implementation.share_modules.factor_implementation_utils import ( +from rdagent.components.task_implementation.factor_implementation.utils import ( get_data_folder_intro, ) from rdagent.core.conf import RD_AGENT_SETTINGS from rdagent.core.evolving_framework import EvolvingStrategy, QueriedKnowledge +from rdagent.core.experiment import Implementation from rdagent.core.prompts import Prompts -from rdagent.core.task import TaskImplementation from rdagent.core.utils import multiprocessing_wrapper from rdagent.oai.llm_utils import APIBackend @@ -43,27 +45,27 @@ class MultiProcessEvolvingStrategy(EvolvingStrategy): @abstractmethod def implement_one_factor( self, - target_task: FactorImplementTask, + target_task: FactorTask, queried_knowledge: QueriedKnowledge = None, - ) -> TaskImplementation: + ) -> Implementation: raise NotImplementedError def evolve( self, *, - evo: FactorEvovlingItem, + evo: FactorEvolvingItem, queried_knowledge: FactorImplementationQueriedKnowledge | None = None, **kwargs, - ) -> FactorEvovlingItem: + ) -> FactorEvolvingItem: self.num_loop += 1 new_evo = deepcopy(evo) # 1.找出需要evolve的factor to_be_finished_task_index = [] - for index, target_factor_task in enumerate(new_evo.target_factor_tasks): + for index, target_factor_task in enumerate(new_evo.sub_tasks): target_factor_task_desc = target_factor_task.get_factor_information() if target_factor_task_desc in queried_knowledge.success_task_to_knowledge_dict: - new_evo.corresponding_implementations[index] = queried_knowledge.success_task_to_knowledge_dict[ + new_evo.sub_implementations[index] = queried_knowledge.success_task_to_knowledge_dict[ target_factor_task_desc ].implementation elif ( @@ -95,18 +97,18 @@ class MultiProcessEvolvingStrategy(EvolvingStrategy): result = multiprocessing_wrapper( [ - (self.implement_one_factor, (new_evo.target_factor_tasks[target_index], queried_knowledge)) + (self.implement_one_factor, (new_evo.sub_tasks[target_index], queried_knowledge)) for target_index in to_be_finished_task_index ], n=FACTOR_IMPLEMENT_SETTINGS.evo_multi_proc_n, ) for index, target_index in enumerate(to_be_finished_task_index): - new_evo.corresponding_implementations[target_index] = result[index] + new_evo.sub_implementations[target_index] = result[index] # for target_index in to_be_finished_task_index: - # new_evo.corresponding_implementations[target_index] = self.implement_one_factor( - # new_evo.target_factor_tasks[target_index], queried_knowledge + # new_evo.sub_implementations[target_index] = self.implement_one_factor( + # new_evo.sub_tasks[target_index], queried_knowledge # ) new_evo.corresponding_selection = to_be_finished_task_index @@ -117,9 +119,9 @@ class MultiProcessEvolvingStrategy(EvolvingStrategy): class FactorEvolvingStrategy(MultiProcessEvolvingStrategy): def implement_one_factor( self, - target_task: FactorImplementTask, + target_task: FactorTask, queried_knowledge: FactorImplementationQueriedKnowledgeV1 = None, - ) -> TaskImplementation: + ) -> Implementation: factor_information_str = target_task.get_factor_information() if queried_knowledge is not None and factor_information_str in queried_knowledge.success_task_to_knowledge_dict: @@ -140,11 +142,15 @@ class FactorEvolvingStrategy(MultiProcessEvolvingStrategy): queried_former_failed_knowledge_to_render = queried_former_failed_knowledge - system_prompt = Template( - implement_prompts["evolving_strategy_factor_implementation_v1_system"], - ).render( - data_info=get_data_folder_intro(), - queried_former_failed_knowledge=queried_former_failed_knowledge_to_render, + system_prompt = ( + Environment(undefined=StrictUndefined) + .from_string( + implement_prompts["evolving_strategy_factor_implementation_v1_system"], + ) + .render( + data_info=get_data_folder_intro(), + queried_former_failed_knowledge=queried_former_failed_knowledge_to_render, + ) ) session = APIBackend(use_chat_cache=False).build_chat_session( session_system_prompt=system_prompt, @@ -153,7 +159,8 @@ class FactorEvolvingStrategy(MultiProcessEvolvingStrategy): queried_similar_successful_knowledge_to_render = queried_similar_successful_knowledge while True: user_prompt = ( - Template( + Environment(undefined=StrictUndefined) + .from_string( implement_prompts["evolving_strategy_factor_implementation_v1_user"], ) .render( @@ -196,9 +203,9 @@ class FactorEvolvingStrategyWithGraph(MultiProcessEvolvingStrategy): def implement_one_factor( self, - target_task: FactorImplementTask, + target_task: FactorTask, queried_knowledge, - ) -> TaskImplementation: + ) -> Implementation: error_summary = FACTOR_IMPLEMENT_SETTINGS.v2_error_summary # 1. 提取因子的背景信息 target_factor_task_information = target_task.get_factor_information() @@ -231,11 +238,15 @@ class FactorEvolvingStrategyWithGraph(MultiProcessEvolvingStrategy): queried_former_failed_knowledge_to_render = queried_former_failed_knowledge - system_prompt = Template( - implement_prompts["evolving_strategy_factor_implementation_v1_system"], - ).render( - data_info=get_data_folder_intro(), - queried_former_failed_knowledge=queried_former_failed_knowledge_to_render, + system_prompt = ( + Environment(undefined=StrictUndefined) + .from_string( + implement_prompts["evolving_strategy_factor_implementation_v1_system"], + ) + .render( + data_info=get_data_folder_intro(), + queried_former_failed_knowledge=queried_former_failed_knowledge_to_render, + ) ) session = APIBackend(use_chat_cache=False).build_chat_session( @@ -254,7 +265,8 @@ class FactorEvolvingStrategyWithGraph(MultiProcessEvolvingStrategy): and len(queried_former_failed_knowledge_to_render) != 0 ): error_summary_system_prompt = ( - Template(implement_prompts["evolving_strategy_error_summary_v2_system"]) + Environment(undefined=StrictUndefined) + .from_string(implement_prompts["evolving_strategy_error_summary_v2_system"]) .render( factor_information_str=target_factor_task_information, code_and_feedback=queried_former_failed_knowledge_to_render[ @@ -268,7 +280,8 @@ class FactorEvolvingStrategyWithGraph(MultiProcessEvolvingStrategy): ) while True: error_summary_user_prompt = ( - Template(implement_prompts["evolving_strategy_error_summary_v2_user"]) + Environment(undefined=StrictUndefined) + .from_string(implement_prompts["evolving_strategy_error_summary_v2_user"]) .render( queried_similar_component_knowledge=queried_similar_component_knowledge_to_render, ) @@ -287,7 +300,8 @@ class FactorEvolvingStrategyWithGraph(MultiProcessEvolvingStrategy): ) # 构建user_prompt。开始写代码 user_prompt = ( - Template( + Environment(undefined=StrictUndefined) + .from_string( implement_prompts["evolving_strategy_factor_implementation_v2_user"], ) .render( diff --git a/rdagent/components/task_implementation/factor_implementation/evolving/knowledge_management.py b/rdagent/components/task_implementation/factor_implementation/evolving/knowledge_management.py index 5d48d698..400be6ef 100644 --- a/rdagent/components/task_implementation/factor_implementation/evolving/knowledge_management.py +++ b/rdagent/components/task_implementation/factor_implementation/evolving/knowledge_management.py @@ -8,20 +8,20 @@ from itertools import combinations from pathlib import Path from typing import Union -from jinja2 import Template +from jinja2 import Environment, StrictUndefined from rdagent.components.knowledge_management.graph import ( UndirectedGraph, UndirectedNode, ) +from rdagent.components.task_implementation.factor_implementation.config import ( + FACTOR_IMPLEMENT_SETTINGS, +) from rdagent.components.task_implementation.factor_implementation.evolving.evaluators import ( FactorImplementationSingleFeedback, ) from rdagent.components.task_implementation.factor_implementation.evolving.evolving_strategy import ( - FactorImplementTask, -) -from rdagent.components.task_implementation.factor_implementation.share_modules.factor_implementation_config import ( - FACTOR_IMPLEMENT_SETTINGS, + FactorTask, ) from rdagent.core.evolving_framework import ( EvolvableSubjects, @@ -31,9 +31,9 @@ from rdagent.core.evolving_framework import ( QueriedKnowledge, RAGStrategy, ) +from rdagent.core.experiment import Implementation from rdagent.core.log import RDAgentLog from rdagent.core.prompts import Prompts -from rdagent.core.task import TaskImplementation from rdagent.oai.llm_utils import ( APIBackend, calculate_embedding_distance_between_str_list, @@ -43,8 +43,8 @@ from rdagent.oai.llm_utils import ( class FactorImplementationKnowledge(Knowledge): def __init__( self, - target_task: FactorImplementTask, - implementation: TaskImplementation, + target_task: FactorTask, + implementation: Implementation, feedback: FactorImplementationSingleFeedback, ) -> None: """ @@ -116,10 +116,10 @@ class FactorImplementationRAGStrategyV1(RAGStrategy): evo_step = evolving_trace[trace_index] implementations = evo_step.evolvable_subjects feedback = evo_step.feedback - for task_index in range(len(implementations.target_factor_tasks)): - target_task = implementations.target_factor_tasks[task_index] + for task_index in range(len(implementations.sub_tasks)): + target_task = implementations.sub_tasks[task_index] target_task_information = target_task.get_factor_information() - implementation = implementations.corresponding_implementations[task_index] + implementation = implementations.sub_implementations[task_index] single_feedback = feedback[task_index] if single_feedback is None: continue @@ -150,12 +150,12 @@ class FactorImplementationRAGStrategyV1(RAGStrategy): fail_task_trial_limit = FACTOR_IMPLEMENT_SETTINGS.fail_task_trial_limit queried_knowledge = FactorImplementationQueriedKnowledgeV1() - for target_factor_task in evo.target_factor_tasks: + for target_factor_task in evo.sub_tasks: target_factor_task_information = target_factor_task.get_factor_information() if target_factor_task_information in self.knowledgebase.success_task_info_set: - queried_knowledge.success_task_to_knowledge_dict[ - target_factor_task_information - ] = self.knowledgebase.implementation_trace[target_factor_task_information][-1] + queried_knowledge.success_task_to_knowledge_dict[target_factor_task_information] = ( + self.knowledgebase.implementation_trace[target_factor_task_information][-1] + ) elif ( len( self.knowledgebase.implementation_trace.setdefault( @@ -167,14 +167,12 @@ class FactorImplementationRAGStrategyV1(RAGStrategy): ): queried_knowledge.failed_task_info_set.add(target_factor_task_information) else: - queried_knowledge.working_task_to_former_failed_knowledge_dict[ - target_factor_task_information - ] = self.knowledgebase.implementation_trace.setdefault( - target_factor_task_information, - [], - )[ - -v1_query_former_trace_limit: - ] + queried_knowledge.working_task_to_former_failed_knowledge_dict[target_factor_task_information] = ( + self.knowledgebase.implementation_trace.setdefault( + target_factor_task_information, + [], + )[-v1_query_former_trace_limit:] + ) knowledge_base_success_task_list = list( self.knowledgebase.success_task_info_set, @@ -195,9 +193,9 @@ class FactorImplementationRAGStrategyV1(RAGStrategy): )[-1] for index in similar_indexes ] - queried_knowledge.working_task_to_similar_successful_knowledge_dict[ - target_factor_task_information - ] = similar_successful_knowledge + queried_knowledge.working_task_to_similar_successful_knowledge_dict[target_factor_task_information] = ( + similar_successful_knowledge + ) return queried_knowledge @@ -236,11 +234,11 @@ class FactorImplementationGraphRAGStrategy(RAGStrategy): evo_step = evolving_trace[trace_index] implementations = evo_step.evolvable_subjects feedback = evo_step.feedback - for task_index in range(len(implementations.target_factor_tasks)): + for task_index in range(len(implementations.sub_tasks)): single_feedback = feedback[task_index] - target_task = implementations.target_factor_tasks[task_index] + target_task = implementations.sub_tasks[task_index] target_task_information = target_task.get_factor_information() - implementation = implementations.corresponding_implementations[task_index] + implementation = implementations.sub_implementations[task_index] single_feedback = feedback[task_index] if single_feedback is None: continue @@ -323,8 +321,12 @@ class FactorImplementationGraphRAGStrategy(RAGStrategy): all_component_content = "" for _, component_node in enumerate(all_component_nodes): all_component_content += f"{component_node.content}, \n" - analyze_component_system_prompt = Template(self.prompt["analyze_component_prompt_v1_system"]).render( - all_component_content=all_component_content, + analyze_component_system_prompt = ( + Environment(undefined=StrictUndefined) + .from_string(self.prompt["analyze_component_prompt_v1_system"]) + .render( + all_component_content=all_component_content, + ) ) analyze_component_user_prompt = target_factor_task_information @@ -396,7 +398,7 @@ class FactorImplementationGraphRAGStrategy(RAGStrategy): """ fail_task_trial_limit = FACTOR_IMPLEMENT_SETTINGS.fail_task_trial_limit - for target_factor_task in evo.target_factor_tasks: + for target_factor_task in evo.sub_tasks: target_factor_task_information = target_factor_task.get_factor_information() if ( target_factor_task_information not in self.knowledgebase.success_task_to_knowledge_dict @@ -427,9 +429,9 @@ class FactorImplementationGraphRAGStrategy(RAGStrategy): else: current_index += 1 - factor_implementation_queried_graph_knowledge.former_traces[ - target_factor_task_information - ] = former_trace_knowledge[-v2_query_former_trace_limit:] + factor_implementation_queried_graph_knowledge.former_traces[target_factor_task_information] = ( + former_trace_knowledge[-v2_query_former_trace_limit:] + ) else: factor_implementation_queried_graph_knowledge.former_traces[target_factor_task_information] = [] @@ -443,7 +445,7 @@ class FactorImplementationGraphRAGStrategy(RAGStrategy): knowledge_sampler: float = 1.0, ) -> QueriedKnowledge | None: # queried_component_knowledge = FactorImplementationQueriedGraphComponentKnowledge() - for target_factor_task in evo.target_factor_tasks: + for target_factor_task in evo.sub_tasks: target_factor_task_information = target_factor_task.get_factor_information() if ( target_factor_task_information in self.knowledgebase.success_task_to_knowledge_dict @@ -583,7 +585,7 @@ class FactorImplementationGraphRAGStrategy(RAGStrategy): knowledge_sampler: float = 1.0, ) -> QueriedKnowledge | None: # queried_error_knowledge = FactorImplementationQueriedGraphErrorKnowledge() - for task_index, target_factor_task in enumerate(evo.target_factor_tasks): + for task_index, target_factor_task in enumerate(evo.sub_tasks): target_factor_task_information = target_factor_task.get_factor_information() factor_implementation_queried_graph_knowledge.error_with_success_task[target_factor_task_information] = {} if ( diff --git a/rdagent/components/task_implementation/factor_implementation/evolving/scheduler.py b/rdagent/components/task_implementation/factor_implementation/evolving/scheduler.py index d27707dc..ded011ee 100644 --- a/rdagent/components/task_implementation/factor_implementation/evolving/scheduler.py +++ b/rdagent/components/task_implementation/factor_implementation/evolving/scheduler.py @@ -1,12 +1,12 @@ import json from pathlib import Path -from jinja2 import Template +from jinja2 import Environment, StrictUndefined -from rdagent.components.task_implementation.factor_implementation.evolving.factor import ( - FactorEvovlingItem, +from rdagent.components.task_implementation.factor_implementation.evolving.evolvable_subjects import ( + FactorEvolvingItem, ) -from rdagent.components.task_implementation.factor_implementation.share_modules.factor_implementation_utils import ( +from rdagent.components.task_implementation.factor_implementation.utils import ( get_data_folder_intro, ) from rdagent.core.conf import RD_AGENT_SETTINGS @@ -29,18 +29,22 @@ def RandomSelect(to_be_finished_task_index, implementation_factors_per_round): return to_be_finished_task_index -def LLMSelect(to_be_finished_task_index, implementation_factors_per_round, evo: FactorEvovlingItem, former_trace): +def LLMSelect(to_be_finished_task_index, implementation_factors_per_round, evo: FactorEvolvingItem, former_trace): tasks = [] for i in to_be_finished_task_index: # find corresponding former trace for each task - target_factor_task_information = evo.target_factor_tasks[i].get_factor_information() + target_factor_task_information = evo.sub_tasks[i].get_factor_information() if target_factor_task_information in former_trace: - tasks.append((i, evo.target_factor_tasks[i], former_trace[target_factor_task_information])) + tasks.append((i, evo.sub_tasks[i], former_trace[target_factor_task_information])) - system_prompt = Template( - scheduler_prompts["select_implementable_factor_system"], - ).render( - data_info=get_data_folder_intro(), + system_prompt = ( + Environment(undefined=StrictUndefined) + .from_string( + scheduler_prompts["select_implementable_factor_system"], + ) + .render( + data_info=get_data_folder_intro(), + ) ) session = APIBackend(use_chat_cache=False).build_chat_session( @@ -48,11 +52,15 @@ def LLMSelect(to_be_finished_task_index, implementation_factors_per_round, evo: ) while True: - user_prompt = Template( - scheduler_prompts["select_implementable_factor_user"], - ).render( - factor_num=implementation_factors_per_round, - target_factor_tasks=tasks, + user_prompt = ( + Environment(undefined=StrictUndefined) + .from_string( + scheduler_prompts["select_implementable_factor_user"], + ) + .render( + factor_num=implementation_factors_per_round, + sub_tasks=tasks, + ) ) if ( session.build_chat_completion_message_and_calculate_token( diff --git a/rdagent/components/task_implementation/factor_implementation/evolving/factor.py b/rdagent/components/task_implementation/factor_implementation/factor.py similarity index 84% rename from rdagent/components/task_implementation/factor_implementation/evolving/factor.py rename to rdagent/components/task_implementation/factor_implementation/factor.py index 0f3cdbd5..59056886 100644 --- a/rdagent/components/task_implementation/factor_implementation/evolving/factor.py +++ b/rdagent/components/task_implementation/factor_implementation/factor.py @@ -9,27 +9,21 @@ from typing import Tuple, Union import pandas as pd from filelock import FileLock -from rdagent.components.task_implementation.factor_implementation.share_modules.factor_implementation_config import ( +from rdagent.components.task_implementation.factor_implementation.config import ( FACTOR_IMPLEMENT_SETTINGS, ) -from rdagent.core.evolving_framework import EvolvableSubjects from rdagent.core.exception import ( CodeFormatException, NoOutputException, RuntimeErrorException, ) +from rdagent.core.experiment import Experiment, FBImplementation, Task from rdagent.core.log import RDAgentLog -from rdagent.core.task import ( - BaseTask, - FBTaskImplementation, - TaskImplementation, - TestCase, -) from rdagent.oai.llm_utils import md5_hash -class FactorImplementTask(BaseTask): - # TODO: generalized the attributes into the BaseTask +class FactorTask(Task): + # TODO: generalized the attributes into the Task # - factor_* -> * def __init__( self, @@ -53,38 +47,13 @@ variables: {str(self.variables)}""" @staticmethod def from_dict(dict): - return FactorImplementTask(**dict) + return FactorTask(**dict) def __repr__(self) -> str: return f"<{self.__class__.__name__}[{self.factor_name}]>" -class FactorEvovlingItem(EvolvableSubjects): - """ - Intermediate item of factor implementation. - """ - - def __init__( - self, - target_factor_tasks: list[FactorImplementTask], - corresponding_gt_implementations: list[TaskImplementation] = None, - ): - super().__init__() - self.target_factor_tasks = target_factor_tasks - self.corresponding_implementations: list[TaskImplementation] = [None for _ in target_factor_tasks] - self.corresponding_selection: list = None - if corresponding_gt_implementations is not None and len( - corresponding_gt_implementations, - ) != len(target_factor_tasks): - self.corresponding_gt_implementations = None - RDAgentLog().warning( - "The length of corresponding_gt_implementations is not equal to the length of target_factor_tasks, set corresponding_gt_implementations to None", - ) - else: - self.corresponding_gt_implementations = corresponding_gt_implementations - - -class FileBasedFactorImplementation(FBTaskImplementation): +class FileBasedFactorImplementation(FBImplementation): """ This class is used to implement a factor by writing the code to a file. Input data and output factor value are also written to files. @@ -100,7 +69,7 @@ class FileBasedFactorImplementation(FBTaskImplementation): def __init__( self, - target_task: FactorImplementTask, + target_task: FactorTask, code, executed_factor_value_dataframe=None, raise_exception=False, @@ -243,9 +212,12 @@ class FileBasedFactorImplementation(FBTaskImplementation): return self.__str__() @staticmethod - def from_folder(task: FactorImplementTask, path: Union[str, Path], **kwargs): + def from_folder(task: FactorTask, path: Union[str, Path], **kwargs): path = Path(path) factor_path = (path / task.factor_name).with_suffix(".py") with factor_path.open("r") as f: code = f.read() return FileBasedFactorImplementation(task, code=code, **kwargs) + + +FactorExperiment = Experiment diff --git a/rdagent/components/task_implementation/factor_implementation/prompts.yaml b/rdagent/components/task_implementation/factor_implementation/prompts.yaml index d843a107..b6551c46 100644 --- a/rdagent/components/task_implementation/factor_implementation/prompts.yaml +++ b/rdagent/components/task_implementation/factor_implementation/prompts.yaml @@ -45,7 +45,7 @@ evolving_strategy_factor_implementation_v1_system: |- 2. The user might provide you the failed former code and the corresponding feedback to the code. The feedback contains to the execution, the code and the factor value. You should analyze the feedback and try to correct the latest code. 3. The user might provide you the suggestion to the latest fail code and some similar fail to correct pairs. Each pair contains the fail code with similar error and the corresponding corrected version code. You should learn from these suggestion to write the correct code. - Your must write your code based on your former lastest attempt below which consists of your former code and code feedback, you should read the former attempt carefully and must not modify the right part of your former code. + Your must write your code based on your former latest attempt below which consists of your former code and code feedback, you should read the former attempt carefully and must not modify the right part of your former code. {% if queried_former_failed_knowledge|length != 0 %} --------------Your former latest attempt:--------------- {% for former_failed_knowledge in queried_former_failed_knowledge %} @@ -199,7 +199,7 @@ select_implementable_factor_system: |- select_implementable_factor_user: |- Number of factor you should pick: {{ factor_num }} - {% for factor_info in target_factor_tasks %} + {% for factor_info in sub_tasks %} =============Factor index:{{factor_info[0]}}:============= =====Factor name:===== {{ factor_info[1].factor_name }} diff --git a/rdagent/components/task_implementation/factor_implementation/share_modules/factor_implementation_utils.py b/rdagent/components/task_implementation/factor_implementation/utils.py similarity index 81% rename from rdagent/components/task_implementation/factor_implementation/share_modules/factor_implementation_utils.py rename to rdagent/components/task_implementation/factor_implementation/utils.py index b6620ecc..9c7aa42c 100644 --- a/rdagent/components/task_implementation/factor_implementation/share_modules/factor_implementation_utils.py +++ b/rdagent/components/task_implementation/factor_implementation/utils.py @@ -3,12 +3,9 @@ from pathlib import Path import pandas as pd # render it with jinja -from jinja2 import Template +from jinja2 import Environment, StrictUndefined -from rdagent.components.task_implementation.factor_implementation.evolving.factor import ( - FactorImplementTask, -) -from rdagent.components.task_implementation.factor_implementation.share_modules.factor_implementation_config import ( +from rdagent.components.task_implementation.factor_implementation.config import ( FACTOR_IMPLEMENT_SETTINGS, ) @@ -19,11 +16,11 @@ TPL = """ ```` """ # Create a Jinja template from the string -JJ_TPL = Template(TPL) +JJ_TPL = Environment(undefined=StrictUndefined).from_string(TPL) def get_data_folder_intro(): - """Direclty get the info of the data folder. + """Directly get the info of the data folder. It is for preparing prompting message. """ content_l = [] @@ -53,4 +50,4 @@ def get_data_folder_intro(): raise NotImplementedError( f"file type {p.name} is not supported. Please implement its description function.", ) - return "\n ----------------- file spliter -------------\n".join(content_l) + return "\n ----------------- file splitter -------------\n".join(content_l) diff --git a/rdagent/components/task_implementation/model_implementation/one_shot/__init__.py b/rdagent/components/task_implementation/model_implementation/one_shot/__init__.py index 87f3d9e4..afad43db 100644 --- a/rdagent/components/task_implementation/model_implementation/one_shot/__init__.py +++ b/rdagent/components/task_implementation/model_implementation/one_shot/__init__.py @@ -2,13 +2,13 @@ import re from pathlib import Path from typing import Sequence -from jinja2 import Template +from jinja2 import Environment, StrictUndefined from rdagent.components.task_implementation.model_implementation.task import ( ModelImplTask, ModelTaskImpl, ) -from rdagent.core.implementation import TaskGenerator +from rdagent.core.task_generator import TaskGenerator from rdagent.core.prompts import Prompts from rdagent.oai.llm_utils import APIBackend @@ -23,8 +23,8 @@ class ModelTaskGen(TaskGenerator): mti.prepare() pr = Prompts(file_path=DIRNAME / "prompt.yaml") - user_prompt_tpl = Template(pr["code_implement_user"]) - sys_prompt_tpl = Template(pr["code_implement_sys"]) + user_prompt_tpl = Environment(undefined=StrictUndefined).from_string(pr["code_implement_user"]) + sys_prompt_tpl = Environment(undefined=StrictUndefined).from_string(pr["code_implement_sys"]) user_prompt = user_prompt_tpl.render( name=t.name, diff --git a/rdagent/components/task_implementation/model_implementation/task.py b/rdagent/components/task_implementation/model_implementation/task.py index ef6a0d51..85cef1f5 100644 --- a/rdagent/components/task_implementation/model_implementation/task.py +++ b/rdagent/components/task_implementation/model_implementation/task.py @@ -1,26 +1,21 @@ +import json import uuid from pathlib import Path from typing import Dict, Optional, Sequence import torch +from rdagent.components.loader.task_loader import ModelTaskLoader from rdagent.components.task_implementation.model_implementation.conf import ( MODEL_IMPL_SETTINGS, ) from rdagent.core.exception import CodeFormatException -from rdagent.core.task import ( - BaseTask, - FBTaskImplementation, - ImpLoader, - TaskImplementation, - TaskLoader, -) +from rdagent.core.experiment import FBImplementation, ImpLoader, Task from rdagent.utils import get_module_by_module_path -import json -class ModelImplTask(BaseTask): - # TODO: it should change when the BaseTask changes. +class ModelImplTask(Task): + # TODO: it should change when the Task changes. name: str description: str formulation: str @@ -43,6 +38,7 @@ class ModelImplTask(BaseTask): self.formulation = formulation self.variables = variables self.key = key + def get_information(self): return f"""name: {self.name} description: {self.description} @@ -50,16 +46,16 @@ formulation: {self.formulation} variables: {self.variables} key: {self.key} """ - + @staticmethod def from_dict(dict): return ModelImplTask(**dict) - + def __repr__(self) -> str: return f"<{self.__class__.__name__} {self.name}>" -class ModelTaskLoderJson(TaskLoader): +class ModelTaskLoaderJson(ModelTaskLoader): # def __init__(self, json_uri: str, select_model: Optional[str] = None) -> None: # super().__init__() # self.json_uri = json_uri @@ -82,7 +78,7 @@ class ModelTaskLoderJson(TaskLoader): # variables=model_data["variables"], # key=model_name # ) - + # return [model_impl_task] def __init__(self, json_uri: str) -> None: @@ -95,7 +91,7 @@ class ModelTaskLoderJson(TaskLoader): # FIXME: the model in the json file is not right due to extraction error # We should fix them case by case in the future - # + # # formula_info = { # "name": "Anti-Symmetric Deep Graph Network (A-DGN)", # "description": "A framework for stable and non-dissipative DGN design. It ensures long-range information preservation between nodes and prevents gradient vanishing or explosion during training.", @@ -119,12 +115,13 @@ class ModelTaskLoderJson(TaskLoader): description=model_data["description"], formulation=model_data["formulation"], variables=model_data["variables"], - key=model_data["key"] + key=model_data["key"], ) model_impl_task_list.append(model_impl_task) return model_impl_task_list - -class ModelImplementationTaskLoaderFromDict(TaskLoader): + + +class ModelImplementationTaskLoaderFromDict(ModelTaskLoader): def load(self, model_dict: dict) -> list: """Load data from a dict.""" task_l = [] @@ -134,13 +131,13 @@ class ModelImplementationTaskLoaderFromDict(TaskLoader): description=model_data["description"], formulation=model_data["formulation"], variables=model_data["variables"], - key=model_name + key=model_name, ) task_l.append(task) return task_l -class ModelTaskImpl(FBTaskImplementation): +class ModelTaskImpl(FBImplementation): """ It is a Pytorch model implementation task; All the things are placed in a folder. @@ -156,11 +153,11 @@ class ModelTaskImpl(FBTaskImplementation): We'll import the model in the implementation in file `model.py` after setting the cwd into the directory - from model import model_cls - initialize the model by initializing it `model_cls(input_dim=INPUT_DIM)` - - And then verify the modle. + - And then verify the model. """ - def __init__(self, target_task: BaseTask) -> None: + def __init__(self, target_task: Task) -> None: super().__init__(target_task) self.path = None @@ -202,7 +199,7 @@ The the implemented code will be placed in a file like /model.py We'll import the model in the implementation in file `model.py` after setting the cwd into the directory - from model import model_cls (So you must have a variable named `model_cls` in the file) - - So your implelemented code could follow the following pattern + - So your implemented code could follow the following pattern ```Python class XXXLayer(torch.nn.Module): ... diff --git a/rdagent/components/task_implementation/model_implementation/task_extraction.py b/rdagent/components/task_implementation/model_implementation/task_extraction.py index bffcd1e6..c39afadf 100644 --- a/rdagent/components/task_implementation/model_implementation/task_extraction.py +++ b/rdagent/components/task_implementation/model_implementation/task_extraction.py @@ -1,26 +1,23 @@ from __future__ import annotations import json -import multiprocessing as mp import re from pathlib import Path -from typing import Mapping -import numpy as np -import pandas as pd -import tiktoken -from jinja2 import Template -from rdagent.core.conf import RD_AGENT_SETTINGS +from rdagent.components.document_reader.document_reader import ( + load_and_process_pdfs_by_langchain, +) +from rdagent.components.loader.task_loader import ModelTaskLoader +from rdagent.components.task_implementation.model_implementation.task import ( + ModelImplementationTaskLoaderFromDict, +) from rdagent.core.log import RDAgentLog from rdagent.core.prompts import Prompts -from rdagent.core.task import TaskLoader -from rdagent.components.document_reader.document_reader import load_and_process_pdfs_by_langchain -from rdagent.components.task_implementation.model_implementation.task import ModelImplementationTaskLoaderFromDict -from rdagent.oai.llm_utils import APIBackend, create_embedding_with_multiprocessing - +from rdagent.oai.llm_utils import APIBackend document_process_prompts = Prompts(file_path=Path(__file__).parent / "prompts.yaml") + def extract_model_from_doc(doc_content: str) -> dict: """ Extract model information from document content. @@ -67,11 +64,11 @@ def extract_model_from_doc(doc_content: str) -> dict: else: break - RDAgentLog().info(f"已经完成{len(model_dict)}个模型的提取") return model_dict + def merge_file_to_model_dict_to_model_dict( file_to_model_dict: dict[str, dict], ) -> dict: @@ -80,7 +77,7 @@ def merge_file_to_model_dict_to_model_dict( for model_name in file_to_model_dict[file_name]: model_dict.setdefault(model_name, []) model_dict[model_name].append(file_to_model_dict[file_name][model_name]) - + model_dict_simple_deduplication = {} for model_name in model_dict: if len(model_dict[model_name]) > 1: @@ -100,18 +97,24 @@ def extract_model_from_docs(docs_dict): return model_dict -class ModelImplementationTaskLoaderFromPDFfiles(TaskLoader): +class ModelImplementationTaskLoaderFromPDFfiles(ModelTaskLoader): def load(self, file_or_folder_path: Path) -> dict: - docs_dict = load_and_process_pdfs_by_langchain(Path(file_or_folder_path)) # dict{file_path:content} - model_dict = extract_model_from_docs(docs_dict) # dict{file_name: dict{model_name: dict{description, formulation, variables}}} - model_dict = merge_file_to_model_dict_to_model_dict(model_dict) # dict {model_name: dict{description, formulation, variables}} + docs_dict = load_and_process_pdfs_by_langchain(Path(file_or_folder_path)) # dict{file_path:content} + model_dict = extract_model_from_docs( + docs_dict + ) # dict{file_name: dict{model_name: dict{description, formulation, variables}}} + model_dict = merge_file_to_model_dict_to_model_dict( + model_dict + ) # dict {model_name: dict{description, formulation, variables}} return ModelImplementationTaskLoaderFromDict().load(model_dict) + def main(path="../test_doc"): doc_dict = load_and_process_pdfs_by_langchain(Path(path)) - print(doc_dict.keys()) # if you run code like "python -u", the print content will be truncated + print(doc_dict.keys()) # if you run code like "python -u", the print content will be truncated import fire + if __name__ == "__main__": fire.Fire(main) diff --git a/rdagent/components/task_loader/__init__.py b/rdagent/components/task_loader/__init__.py deleted file mode 100644 index 6646680b..00000000 --- a/rdagent/components/task_loader/__init__.py +++ /dev/null @@ -1,9 +0,0 @@ -from rdagent.core.task import TaskLoader - - -class FactorTaskLoader(TaskLoader): - pass - - -class ModelTaskLoader(TaskLoader): - pass diff --git a/rdagent/core/evaluation.py b/rdagent/core/evaluation.py index 2b20e8ab..6b90165f 100644 --- a/rdagent/core/evaluation.py +++ b/rdagent/core/evaluation.py @@ -1,15 +1,15 @@ from abc import ABC, abstractmethod -from rdagent.core.task import BaseTask, TaskImplementation +from rdagent.core.experiment import Task, Implementation class Evaluator(ABC): @abstractmethod def evaluate( self, - target_task: BaseTask, - implementation: TaskImplementation, - gt_implementation: TaskImplementation, + target_task: Task, + implementation: Implementation, + gt_implementation: Implementation, **kwargs, ): raise NotImplementedError diff --git a/rdagent/core/evolving_framework.py b/rdagent/core/evolving_framework.py index 7b586a4b..645e2fad 100644 --- a/rdagent/core/evolving_framework.py +++ b/rdagent/core/evolving_framework.py @@ -33,8 +33,7 @@ class EvolvableSubjects: return copy.deepcopy(self) -class QlibEvolvableSubjects(EvolvableSubjects): - ... +class QlibEvolvableSubjects(EvolvableSubjects): ... @dataclass @@ -42,7 +41,7 @@ class EvoStep: """At a specific step, based on - previous trace - - newly RAG kownledge `QueriedKnowledge` + - newly RAG knowledge `QueriedKnowledge` the EvolvableSubjects is evolved to a new one `EvolvableSubjects`. @@ -73,7 +72,7 @@ class EvolvingStrategy(ABC): class RAGStrategy(ABC): - """Retrival Augmentation Generation Strategy""" + """Retrieval Augmentation Generation Strategy""" def __init__(self, knowledgebase: KnowledgeBase) -> None: self.knowledgebase = knowledgebase diff --git a/rdagent/core/task.py b/rdagent/core/experiment.py similarity index 67% rename from rdagent/core/task.py rename to rdagent/core/experiment.py index 95e8f4a7..7e8c95ac 100644 --- a/rdagent/core/task.py +++ b/rdagent/core/experiment.py @@ -1,25 +1,23 @@ from abc import ABC, abstractmethod from pathlib import Path -from typing import Generic, Optional, Sequence, Tuple, TypeVar - -import pandas as pd +from typing import Generic, Optional, Sequence, TypeVar """ -This file contains the all the data class for rdagent task. +This file contains the all the class about organizing the task in RD-Agent. """ -class BaseTask(ABC): +class Task: # TODO: 把name放在这里作为主键 # Please refer to rdagent/model_implementation/task.py for the implementation # I think the task version applies to the base class. pass -ASpecificTask = TypeVar("ASpecificTask", bound=BaseTask) +ASpecificTask = TypeVar("ASpecificTask", bound=Task) -class TaskImplementation(ABC, Generic[ASpecificTask]): +class Implementation(ABC, Generic[ASpecificTask]): def __init__(self, target_task: ASpecificTask) -> None: self.target_task = target_task @@ -28,7 +26,7 @@ class TaskImplementation(ABC, Generic[ASpecificTask]): """ The execution of the implementation can be dynamic. - So we may passin the data and config dynamically. + So we may pass in the data and config dynamically. """ raise NotImplementedError("execute method is not implemented.") @@ -44,16 +42,16 @@ class TaskImplementation(ABC, Generic[ASpecificTask]): # Some evaluators will input the results and output -ASpecificTaskImp = TypeVar("ASpecificTaskImp", bound=TaskImplementation) +ASpecificImp = TypeVar("ASpecificImp", bound=Implementation) -class ImpLoader(ABC, Generic[ASpecificTask, ASpecificTaskImp]): +class ImpLoader(ABC, Generic[ASpecificTask, ASpecificImp]): @abstractmethod - def load(self, task: ASpecificTask) -> ASpecificTaskImp: + def load(self, task: ASpecificTask) -> ASpecificImp: raise NotImplementedError("load method is not implemented.") -class FBTaskImplementation(TaskImplementation): +class FBImplementation(Implementation): """ File-based task implementation @@ -63,12 +61,12 @@ class FBTaskImplementation(TaskImplementation): - Output - After execution, it will generate the final output as file. - A typical way to run the pipeline of FBTaskImplementation will be + A typical way to run the pipeline of FBImplementation will be (We didn't add it as a method due to that we may pass arguments into `prepare` or `execute` based on our requirements.) .. code-block:: python - def run_pipline(self, **files: str): + def run_pipeline(self, **files: str): self.prepare() self.inject_code(**files) self.execute() @@ -76,9 +74,9 @@ class FBTaskImplementation(TaskImplementation): """ # TODO: - # FileBasedFactorImplementation should inherient from it. + # FileBasedFactorImplementation should inherit from it. # Why not directly reuse FileBasedFactorImplementation. - # Because it has too much concerete dependencies. + # Because it has too much concrete dependencies. # e.g. dataframe, factors path: Optional[Path] @@ -116,17 +114,20 @@ class FBTaskImplementation(TaskImplementation): return list(self.path.iterdir()) -class TestCase: - def __init__( - self, - target_task: list[BaseTask] = [], - ground_truth: list[TaskImplementation] = [], - ): - self.ground_truth = ground_truth - self.target_task = target_task +class Experiment(ABC, Generic[ASpecificTask, ASpecificImp]): + """ + The experiment is a sequence of tasks and the implementations of the tasks after generated by the TaskGenerator. + """ + + def __init__(self, sub_tasks: Sequence[Task]) -> None: + self.sub_tasks = sub_tasks + self.sub_implementations: Sequence[ASpecificImp] = [None for _ in self.sub_tasks] -class TaskLoader: +TaskOrExperiment = TypeVar("TaskOrExperiment", Task, Experiment) + + +class Loader(ABC, Generic[TaskOrExperiment]): @abstractmethod - def load(self, *args, **kwargs) -> Sequence[BaseTask]: + def load(self, *args, **kwargs) -> TaskOrExperiment: raise NotImplementedError("load method is not implemented.") diff --git a/rdagent/core/proposal.py b/rdagent/core/proposal.py new file mode 100644 index 00000000..d70b4a26 --- /dev/null +++ b/rdagent/core/proposal.py @@ -0,0 +1,101 @@ +""" + +""" + +from typing import Dict, List, Tuple + +from rdagent.core.evolving_framework import Feedback +from rdagent.core.experiment import Experiment, Implementation, Loader, Task + +# class data_ana: XXX + + +class Hypothesis: + """ + TODO: We may have better name for it. + + Name Candidates: + - Belief + """ + + hypothesis: str = None + reason: str = None + + # source: data_ana | model_nan = None + + +# Origin(path of repo/data/feedback) => view/summarization => generated Hypothesis + + +class Scenario: + def get_repo_path(self): + """codebase""" + + def get_data(self): + """ "data info""" + + def get_env(self): + """env description""" + + def get_scenario_all_desc(self) -> str: + """Combine all the description together""" + + +class HypothesisFeedback(Feedback): ... + + +class Trace: + scen: Scenario + hist: list[Tuple[Hypothesis, Experiment, HypothesisFeedback]] + + +class HypothesisGen: + def __init__(self, scen: Scenario): + self.scen = scen + + def gen(self, trace: Trace) -> Hypothesis: + # def gen(self, scenario_desc: str, ) -> Hypothesis: + """ + Motivation of the variable `scenario_desc`: + - Mocking a data-scientist is observing the scenario. + + scenario_desc may conclude: + - data observation: + - Original or derivative + - Task information: + """ + + +class HypothesisSet: + """ + # drop, append + + hypothesis_imp: list[float] | None # importance of each hypothesis + true_hypothesis or false_hypothesis + """ + + hypothesis_list: list[Hypothesis] + trace: Trace + + +class Hypothesis2Experiment(Loader[Experiment]): + """ + [Abstract description => concrete description] => Code implement + """ + + def convert(self, bs: HypothesisSet) -> Experiment: + """Connect the idea proposal to implementation""" + ... + + +# Boolean, Reason, Confidence, etc. + + +class Experiment2Feedback: + """ "Generated(summarize) feedback from **Executed** Implementation""" + + def summarize(self, ti: Experiment) -> HypothesisFeedback: + """ + The `ti` should be executed and the results should be included. + For example: `mlflow` of Qlib will be included. + """ diff --git a/rdagent/core/proposal/__init__.py b/rdagent/core/proposal/__init__.py deleted file mode 100644 index 43125809..00000000 --- a/rdagent/core/proposal/__init__.py +++ /dev/null @@ -1,95 +0,0 @@ -""" - -""" - -from typing import Tuple - -from rdagent.core.task import BaseTask, TaskLoader - -# class data_ana: XXX - - -class Belief: - """ - TODO: We may have better name for it. - - Name Candidates: - - Hypothesis - """ - - # source: data_ana | model_nan = None - - -# Origin(path of repo/data/feedback) => view/summarization => generated Belief - - -class Scenario: - def get_repo_path(self): - """codebase""" - - def get_data(self): - """ "data info""" - - def get_env(self): - """env description""" - - -class Trace: - scen: Scenario - hist: list[Tuple[Belief, Feedback]] - - -class BeliefGen: - def __init__(self, scen: Scenario): - self.scen = scen - - def gen(self, trace: Trace) -> Belief: - # def gen(self, scenario_desc: str, ) -> Belief: - """ - Motivation of the variable `scenario_desc`: - - Mocking a data-scientist is observing the scenario. - - scenario_desc may conclude: - - data observation: - - Original or derivative - - Task information: - """ - - -class BeliefSet: - """ - # drop, append - - belief_imp: list[float] | None # importance of each belief - failed_belief or success belief - """ - - belief_l: list[Belief] - feedbacks: Dict[Tuple[Belief, Scenario], BeliefFeedback] - - -class Belief2Task(TaskLoader): - """ - [Abstract description => conceret description] => Code implement - """ - - def convert(self, bs: BeliefSet) -> BaseTask: - """Connect the idea proposal to implementation""" - ... - - -class BeliefFeedback: - ... - - -# Boolean, Reason, Confidence, etc. - - -class Imp2Feedback: - """ "Generated(summarize) feedback from **Executed** Implemenation""" - - def summarize(self, ti: TaskImplementation) -> BeliefFeedback: - """ - The `ti` should be exectued and the results should be included. - For example: `mlflow` of Qlib will be included. - """ diff --git a/rdagent/core/implementation.py b/rdagent/core/task_generator.py similarity index 57% rename from rdagent/core/implementation.py rename to rdagent/core/task_generator.py index be08dc42..7570801a 100644 --- a/rdagent/core/implementation.py +++ b/rdagent/core/task_generator.py @@ -1,14 +1,16 @@ from abc import ABC, abstractmethod -from typing import List, Sequence +from typing import Generic, List, Sequence, TypeVar -from rdagent.core.task import BaseTask, TaskImplementation +from rdagent.core.experiment import Experiment + +ASpecificExp = TypeVar("ASpecificExp", bound=Experiment) -class TaskGenerator(ABC): +class TaskGenerator(ABC, Generic[ASpecificExp]): @abstractmethod - def generate(self, task_l: Sequence[BaseTask]) -> Sequence[TaskImplementation]: + def generate(self, exp: ASpecificExp) -> ASpecificExp: """ - Task Generator should take in a sequence of tasks. + Task Generator should take in an experiment. Because the schedule of different tasks is crucial for the final performance due to it affects the learning process. @@ -19,7 +21,7 @@ class TaskGenerator(ABC): def collect_feedback(self, feedback_obj_l: List[object]): """ When online evaluation. - The preivous feedbacks will be collected to support advanced factor generator + The previous feedbacks will be collected to support advanced factor generator Parameters ---------- diff --git a/rdagent/scenarios/qlib/experiment/factor_experiment.py b/rdagent/scenarios/qlib/experiment/factor_experiment.py new file mode 100644 index 00000000..69960f68 --- /dev/null +++ b/rdagent/scenarios/qlib/experiment/factor_experiment.py @@ -0,0 +1,5 @@ +from rdagent.components.task_implementation.factor_implementation.factor import ( + FactorExperiment, +) + +QlibFactorExperiment = FactorExperiment diff --git a/rdagent/scenarios/qlib/experiment/model_experiment.py b/rdagent/scenarios/qlib/experiment/model_experiment.py new file mode 100644 index 00000000..774eec6d --- /dev/null +++ b/rdagent/scenarios/qlib/experiment/model_experiment.py @@ -0,0 +1 @@ +# TODO define QlibModelExperiment here which should be subclass of Experiment diff --git a/rdagent/scenarios/qlib/factor_task_loader/json_loader.py b/rdagent/scenarios/qlib/factor_experiment_loader/json_loader.py similarity index 60% rename from rdagent/scenarios/qlib/factor_task_loader/json_loader.py rename to rdagent/scenarios/qlib/factor_experiment_loader/json_loader.py index 292922be..aca7944f 100644 --- a/rdagent/scenarios/qlib/factor_task_loader/json_loader.py +++ b/rdagent/scenarios/qlib/factor_experiment_loader/json_loader.py @@ -1,49 +1,54 @@ import json from pathlib import Path -from rdagent.components.task_implementation.factor_implementation.evolving.factor import ( - FactorImplementTask, +from rdagent.components.benchmark.eval_method import TestCase +from rdagent.components.loader.experiment_loader import FactorExperimentLoader +from rdagent.components.task_implementation.factor_implementation.factor import ( + FactorExperiment, + FactorTask, FileBasedFactorImplementation, ) -from rdagent.components.task_loader import FactorTaskLoader -from rdagent.core.task import TaskLoader, TestCase +from rdagent.core.experiment import Loader -class FactorImplementationTaskLoaderFromDict(FactorTaskLoader): +class FactorImplementationExperimentLoaderFromDict(FactorExperimentLoader): def load(self, factor_dict: dict) -> list: """Load data from a dict.""" task_l = [] for factor_name, factor_data in factor_dict.items(): - task = FactorImplementTask( + task = FactorTask( factor_name=factor_name, factor_description=factor_data["description"], factor_formulation=factor_data["formulation"], variables=factor_data["variables"], ) task_l.append(task) - return task_l + exp = FactorExperiment(sub_tasks=task_l) + return exp -class FactorImplementationTaskLoaderFromJsonFile(FactorTaskLoader): +class FactorImplementationExperimentLoaderFromJsonFile(FactorExperimentLoader): def load(self, json_file_path: Path) -> list: with open(json_file_path, "r") as file: factor_dict = json.load(file) - return FactorImplementationTaskLoaderFromDict().load(factor_dict) + return FactorImplementationExperimentLoaderFromDict().load(factor_dict) -class FactorImplementationTaskLoaderFromJsonString(FactorTaskLoader): +class FactorImplementationExperimentLoaderFromJsonString(FactorExperimentLoader): def load(self, json_string: str) -> list: factor_dict = json.loads(json_string) - return FactorImplementationTaskLoaderFromDict().load(factor_dict) + return FactorImplementationExperimentLoaderFromDict().load(factor_dict) -class FactorTestCaseLoaderFromJsonFile(TaskLoader): +# TODO loader only supports generic of task or experiment, testcase might cause CI error here +# class FactorTestCaseLoaderFromJsonFile(Loader[TestCase]): +class FactorTestCaseLoaderFromJsonFile: def load(self, json_file_path: Path) -> list: with open(json_file_path, "r") as file: factor_dict = json.load(file) TestData = TestCase() for factor_name, factor_data in factor_dict.items(): - task = FactorImplementTask( + task = FactorTask( factor_name=factor_name, factor_description=factor_data["description"], factor_formulation=factor_data["formulation"], diff --git a/rdagent/scenarios/qlib/factor_task_loader/pdf_loader.py b/rdagent/scenarios/qlib/factor_experiment_loader/pdf_loader.py similarity index 96% rename from rdagent/scenarios/qlib/factor_task_loader/pdf_loader.py rename to rdagent/scenarios/qlib/factor_experiment_loader/pdf_loader.py index 6ee4ab75..1161d010 100644 --- a/rdagent/scenarios/qlib/factor_task_loader/pdf_loader.py +++ b/rdagent/scenarios/qlib/factor_experiment_loader/pdf_loader.py @@ -8,8 +8,7 @@ from typing import Mapping import numpy as np import pandas as pd -import tiktoken -from jinja2 import Template +from jinja2 import Environment, StrictUndefined from sklearn.cluster import KMeans from sklearn.metrics.pairwise import cosine_similarity from sklearn.preprocessing import normalize @@ -17,13 +16,13 @@ from sklearn.preprocessing import normalize from rdagent.components.document_reader.document_reader import ( load_and_process_pdfs_by_langchain, ) -from rdagent.components.task_loader import FactorTaskLoader +from rdagent.components.loader.experiment_loader import FactorExperimentLoader from rdagent.core.conf import RD_AGENT_SETTINGS from rdagent.core.log import RDAgentLog from rdagent.core.prompts import Prompts from rdagent.oai.llm_utils import APIBackend, create_embedding_with_multiprocessing -from rdagent.scenarios.qlib.factor_task_loader.json_loader import ( - FactorImplementationTaskLoaderFromDict, +from rdagent.scenarios.qlib.factor_experiment_loader.json_loader import ( + FactorImplementationExperimentLoaderFromDict, ) document_process_prompts = Prompts(file_path=Path(__file__).parent / "prompts.yaml") @@ -160,9 +159,13 @@ def __extract_factors_formulation_from_content( ) system_prompt = document_process_prompts["extract_factor_formulation_system"] - current_user_prompt = Template( - document_process_prompts["extract_factor_formulation_user"], - ).render(report_content=content, factor_dict=factor_dict_df.to_string()) + current_user_prompt = ( + Environment(undefined=StrictUndefined) + .from_string( + document_process_prompts["extract_factor_formulation_user"], + ) + .render(report_content=content, factor_dict=factor_dict_df.to_string()) + ) session = APIBackend().build_chat_session(session_system_prompt=system_prompt) factor_to_formulation = {} @@ -576,7 +579,7 @@ def deduplicate_factors_by_llm( # noqa: C901, PLR0912 return llm_deduplicated_factor_dict, final_duplication_names_list -class FactorImplementationTaskLoaderFromPDFfiles(FactorTaskLoader): +class FactorImplementationExperimentLoaderFromPDFfiles(FactorExperimentLoader): def load(self, file_or_folder_path: Path) -> dict: docs_dict = load_and_process_pdfs_by_langchain(Path(file_or_folder_path)) @@ -586,4 +589,4 @@ class FactorImplementationTaskLoaderFromPDFfiles(FactorTaskLoader): factor_viability = check_factor_viability(factor_dict) factor_dict, duplication_names_list = deduplicate_factors_by_llm(factor_dict, factor_viability) - return FactorImplementationTaskLoaderFromDict().load(factor_dict) + return FactorImplementationExperimentLoaderFromDict().load(factor_dict) diff --git a/rdagent/scenarios/qlib/factor_task_loader/prompts.yaml b/rdagent/scenarios/qlib/factor_experiment_loader/prompts.yaml similarity index 100% rename from rdagent/scenarios/qlib/factor_task_loader/prompts.yaml rename to rdagent/scenarios/qlib/factor_experiment_loader/prompts.yaml diff --git a/rdagent/scenarios/qlib/factor_idea_proposal.py b/rdagent/scenarios/qlib/factor_idea_proposal.py new file mode 100644 index 00000000..a7d41a16 --- /dev/null +++ b/rdagent/scenarios/qlib/factor_idea_proposal.py @@ -0,0 +1,40 @@ +import json +from pathlib import Path + +from jinja2 import Environment, StrictUndefined + +from rdagent.components.idea_proposal.factor_proposal import ( + FactorHypothesis, + FactorHypothesisGen, +) +from rdagent.core.prompts import Prompts +from rdagent.core.proposal import Scenario, Trace + +prompt_dict = Prompts(file_path=Path(__file__).parent / "prompts.yaml") + +QlibFactorHypothesis = FactorHypothesis + + +class QlibFactorHypothesisGen(FactorHypothesisGen): + def __init__(self, scen: Scenario): + super().__init__(scen) + self.gen_json_flag = True + + def prepare_gen_context(self, trace: Trace) -> None: + self.gen_context_flag = True + + hypothesis_feedback = ( + Environment(undefined=StrictUndefined) + .from_string(prompt_dict["hypothesis_and_feedback"]) + .render(trace=trace) + ) + self.gen_context_dict = { + "hypothesis_and_feedback": hypothesis_feedback, + "RAG": ..., + "factor_output_format": prompt_dict["output_format"], + } + + def gen_response_to_hypothesis_list(self, response: str) -> FactorHypothesis: + response_dict = json.loads(response) + hypothesis = QlibFactorHypothesis(hypothesis=response_dict["hypothesis"], reason=response_dict["reason"]) + return hypothesis diff --git a/rdagent/scenarios/qlib/prompts.yaml b/rdagent/scenarios/qlib/prompts.yaml new file mode 100644 index 00000000..336d1602 --- /dev/null +++ b/rdagent/scenarios/qlib/prompts.yaml @@ -0,0 +1,12 @@ +hypothesis_and_feedback: |- + {% for hypothesis, feedback in trace %} + Hypothesis {{ loop.index }}: {{ hypothesis }} + Feedback: {{ feedback }} + {% endfor %} + +output_format: |- + The output should follow JSON format. The schema is as follows: + { + "hypothesis": "The new hypothesis generated based on the information provided.", + "reason": "The reason why you generate this hypothesis." + } \ No newline at end of file diff --git a/rdagent/scenarios/qlib/task_implementation/data.py b/rdagent/scenarios/qlib/task_generator/data.py similarity index 52% rename from rdagent/scenarios/qlib/task_implementation/data.py rename to rdagent/scenarios/qlib/task_generator/data.py index 96752733..a052286e 100644 --- a/rdagent/scenarios/qlib/task_implementation/data.py +++ b/rdagent/scenarios/qlib/task_generator/data.py @@ -1,7 +1,8 @@ -from rdagent.core.task import FBTaskImplementation +from rdagent.core.task_generator import TaskGenerator +from rdagent.scenarios.qlib.experiment.factor_experiment import QlibFactorExperiment -class QlibDataTaskImplementation(FBTaskImplementation): +class QlibDataImplementation(TaskGenerator[QlibFactorExperiment]): """ Docker run Everything in a folder diff --git a/rdagent/scenarios/qlib/task_implementation/feedback.py b/rdagent/scenarios/qlib/task_generator/feedback.py similarity index 100% rename from rdagent/scenarios/qlib/task_implementation/feedback.py rename to rdagent/scenarios/qlib/task_generator/feedback.py diff --git a/rdagent/scenarios/qlib/task_implementation/model.py b/rdagent/scenarios/qlib/task_generator/model.py similarity index 71% rename from rdagent/scenarios/qlib/task_implementation/model.py rename to rdagent/scenarios/qlib/task_generator/model.py index db0bb4b0..8c7d2a0c 100644 --- a/rdagent/scenarios/qlib/task_implementation/model.py +++ b/rdagent/scenarios/qlib/task_generator/model.py @@ -1,7 +1,7 @@ -from rdagent.core.task import FBTaskImplementation +from rdagent.core.task_generator import TaskGenerator -class QlibModelTaskImplementation(FBTaskImplementation): +class QlibModelImplementation(TaskGenerator[QlibModelExperiment]): """ Docker run Everything in a folder