diff --git a/rdagent/app/CI/run.py b/rdagent/app/CI/run.py index 469cdf00..aa2ae6fa 100644 --- a/rdagent/app/CI/run.py +++ b/rdagent/app/CI/run.py @@ -13,6 +13,16 @@ from pathlib import Path from typing import Any, Literal import tree_sitter_python +from rich import print +from rich.panel import Panel +from rich.progress import Progress, SpinnerColumn, TimeElapsedColumn +from rich.prompt import Prompt +from rich.rule import Rule +from rich.syntax import Syntax +from rich.table import Table +from rich.text import Text +from tree_sitter import Language, Node, Parser + from rdagent.core.evaluation import Evaluator from rdagent.core.evolving_agent import EvoAgent from rdagent.core.evolving_framework import ( @@ -24,15 +34,6 @@ from rdagent.core.evolving_framework import ( ) from rdagent.core.prompts import Prompts from rdagent.oai.llm_utils import APIBackend -from rich import print -from rich.panel import Panel -from rich.progress import Progress, SpinnerColumn, TimeElapsedColumn -from rich.prompt import Prompt -from rich.rule import Rule -from rich.syntax import Syntax -from rich.table import Table -from rich.text import Text -from tree_sitter import Language, Node, Parser py_parser = Parser(Language(tree_sitter_python.language())) CI_prompts = Prompts(file_path=Path(__file__).parent / "prompts.yaml") @@ -355,7 +356,6 @@ class RuffEvaluator(Evaluator): class MypyEvaluator(Evaluator): - def __init__(self, command: str | None = None) -> None: if command is None: self.command = "mypy . --pretty --no-error-summary --show-column-numbers" @@ -411,12 +411,10 @@ class MypyEvaluator(Evaluator): class MultiEvaluator(Evaluator): - def __init__(self, *evaluators: Evaluator) -> None: self.evaluators = evaluators def evaluate(self, evo: Repo, **kwargs: dict) -> CIFeedback: - all_errors = defaultdict(list) for evaluator in self.evaluators: feedback: CIFeedback = evaluator.evaluate(evo, **kwargs) @@ -438,7 +436,6 @@ class CIEvoStr(EvolvingStrategy): knowledge_l: list[Knowledge] | None = None, **kwargs: dict, ) -> Repo: - @dataclass class CodeFixGroup: start_line: int @@ -633,12 +630,16 @@ class CIEvoStr(EvolvingStrategy): for i in diff: if i.startswith("+"): table.add_row( - "", Text(str(diff_new_lineno), style="green bold"), Text(i, style="green"), + "", + Text(str(diff_new_lineno), style="green bold"), + Text(i, style="green"), ) diff_new_lineno += 1 elif i.startswith("-"): table.add_row( - Text(str(diff_original_lineno), style="red bold"), "", Text(i, style="red"), + Text(str(diff_original_lineno), style="red bold"), + "", + Text(i, style="red"), ) diff_original_lineno += 1 elif i.startswith("?"): @@ -691,13 +692,13 @@ class CIEvoStr(EvolvingStrategy): return evo + class CIEvoAgent(EvoAgent): def __init__(self, evolving_strategy: CIEvoStr) -> None: super().__init__(max_loop=1, evolving_strategy=evolving_strategy) self.evolving_trace = [] def multistep_evolve(self, evo: Repo, eva: Evaluator, **kwargs: Any) -> Repo: - evo = self.evolving_strategy.evolve( evo=evo, evolving_trace=self.evolving_trace, @@ -707,6 +708,7 @@ class CIEvoAgent(EvoAgent): return evo + DIR = None while DIR is None or not DIR.exists(): DIR = Prompt.ask("Please input the [cyan]project directory[/cyan]") diff --git a/rdagent/app/factor_extraction_and_implementation/factor_extract_and_implement.py b/rdagent/app/factor_extraction_and_implementation/factor_extract_and_implement.py index 830e8216..e92a1f00 100644 --- a/rdagent/app/factor_extraction_and_implementation/factor_extract_and_implement.py +++ b/rdagent/app/factor_extraction_and_implementation/factor_extract_and_implement.py @@ -2,7 +2,7 @@ from dotenv import load_dotenv from rdagent.scenarios.qlib.factor_experiment_loader.pdf_loader import ( - FactorImplementationExperimentLoaderFromPDFfiles, + FactorExperimentLoaderFromPDFfiles, ) from rdagent.scenarios.qlib.factor_task_implementation import ( COSTEERFG_QUANT_FACTOR_IMPLEMENTATION, @@ -12,7 +12,7 @@ assert load_dotenv() def extract_factors_and_implement(report_file_path: str) -> None: - factor_tasks = FactorImplementationExperimentLoaderFromPDFfiles().load(report_file_path) + factor_tasks = FactorExperimentLoaderFromPDFfiles().load(report_file_path) implementation_result = COSTEERFG_QUANT_FACTOR_IMPLEMENTATION().generate(factor_tasks) # Qlib to run the implementation return implementation_result diff --git a/rdagent/app/model_extraction_and_implementation/model_extraction_and_implementation.py b/rdagent/app/model_extraction_and_implementation/model_extraction_and_implementation.py index 830bc7bc..0e69087b 100644 --- a/rdagent/app/model_extraction_and_implementation/model_extraction_and_implementation.py +++ b/rdagent/app/model_extraction_and_implementation/model_extraction_and_implementation.py @@ -1,16 +1,14 @@ # %% from dotenv import load_dotenv -from rdagent.components.task_implementation.model_implementation.one_shot import ( - ModelCodeWriter, -) -from rdagent.components.task_implementation.model_implementation.task_loader import ( - ModelImplementationExperimentLoaderFromPDFfiles, +from rdagent.components.coder.model_coder.one_shot import ModelCodeWriter +from rdagent.components.coder.model_coder.task_loader import ( + ModelExperimentLoaderFromPDFfiles, ) def extract_models_and_implement(report_file_path: str = "../test_doc") -> None: - factor_tasks = ModelImplementationExperimentLoaderFromPDFfiles().load(report_file_path) + factor_tasks = ModelExperimentLoaderFromPDFfiles().load(report_file_path) implementation_result = ModelCodeWriter().generate(factor_tasks) return implementation_result diff --git a/rdagent/app/model_implementation/eval.py b/rdagent/app/model_implementation/eval.py index 67efee67..20fe0268 100644 --- a/rdagent/app/model_implementation/eval.py +++ b/rdagent/app/model_implementation/eval.py @@ -2,16 +2,12 @@ from pathlib import Path DIRNAME = Path(__file__).absolute().resolve().parent -from rdagent.components.task_implementation.model_implementation.benchmark.eval import ( - ModelImpValEval, -) -from rdagent.components.task_implementation.model_implementation.model import ( +from rdagent.components.coder.model_coder.benchmark.eval import ModelImpValEval +from rdagent.components.coder.model_coder.model import ( ModelImpLoader, ModelTaskLoaderJson, ) -from rdagent.components.task_implementation.model_implementation.one_shot import ( - ModelCodeWriter, -) +from rdagent.components.coder.model_coder.one_shot import ModelCodeWriter bench_folder = DIRNAME.parent.parent / "components" / "task_implementation" / "model_implementation" / "benchmark" mtl = ModelTaskLoaderJson(str(bench_folder / "model_dict.json")) diff --git a/rdagent/app/qlib_rd_loop/conf.py b/rdagent/app/qlib_rd_loop/conf.py index 76802bbd..6db7c1c3 100644 --- a/rdagent/app/qlib_rd_loop/conf.py +++ b/rdagent/app/qlib_rd_loop/conf.py @@ -7,7 +7,7 @@ class PropSetting(BaseSettings): scen: str = "rdagent.scenarios.qlib.experiment.factor_experiment.QlibFactorScenario" hypothesis_gen: str = "rdagent.scenarios.qlib.factor_proposal.QlibFactorHypothesisGen" hypothesis2experiment: str = "rdagent.scenarios.qlib.factor_proposal.QlibFactorHypothesis2Experiment" - qlib_factor_coder: str = "rdagent.components.task_implementation.factor_implementation.CoSTEER.CoSTEERFG" + qlib_factor_coder: str = "rdagent.scenarios.qlib.factor_task_implementation.QlibFactorCoSTEER" qlib_factor_runner: str = "rdagent.scenarios.qlib.task_generator.data.QlibFactorRunner" qlib_factor_summarizer: str = "rdagent.scenarios.qlib.task_generator.feedback.QlibFactorExperiment2Feedback" diff --git a/rdagent/app/qlib_rd_loop/factor.py b/rdagent/app/qlib_rd_loop/factor.py index dce78331..b6bb0599 100644 --- a/rdagent/app/qlib_rd_loop/factor.py +++ b/rdagent/app/qlib_rd_loop/factor.py @@ -2,6 +2,10 @@ TODO: Factor Structure RD-Loop """ +from dotenv import load_dotenv + +load_dotenv(override=True) + # import_from from rdagent.app.qlib_rd_loop.conf import PROP_SETTING from rdagent.core.proposal import ( @@ -20,8 +24,8 @@ hypothesis_gen: HypothesisGen = import_class(PROP_SETTING.hypothesis_gen)(scen) hypothesis2experiment: Hypothesis2Experiment = import_class(PROP_SETTING.hypothesis2experiment)() -qlib_factor_coder: TaskGenerator = import_class(PROP_SETTING.qlib_factor_coder)() -qlib_factor_runner: TaskGenerator = import_class(PROP_SETTING.qlib_factor_runner)() +qlib_factor_coder: TaskGenerator = import_class(PROP_SETTING.qlib_factor_coder)(scen) +qlib_factor_runner: TaskGenerator = import_class(PROP_SETTING.qlib_factor_runner)(scen) qlib_factor_summarizer: Experiment2Feedback = import_class(PROP_SETTING.qlib_factor_summarizer)() diff --git a/rdagent/app/qlib_rd_loop/model.py b/rdagent/app/qlib_rd_loop/model.py index 26b8d306..c92c1d48 100644 --- a/rdagent/app/qlib_rd_loop/model.py +++ b/rdagent/app/qlib_rd_loop/model.py @@ -5,6 +5,7 @@ TODO: move the following code to a new class: Model_RD_Agent # import_from from rdagent.app.model_proposal.conf import MODEL_PROP_SETTING + from rdagent.core.proposal import ( Experiment2Feedback, Hypothesis2Experiment, diff --git a/rdagent/components/benchmark/eval_method.py b/rdagent/components/benchmark/eval_method.py index 4161605a..e8ea3ca6 100644 --- a/rdagent/components/benchmark/eval_method.py +++ b/rdagent/components/benchmark/eval_method.py @@ -4,22 +4,18 @@ from typing import List, Tuple, Union from tqdm import tqdm -from rdagent.components.task_implementation.factor_implementation.config import ( - FACTOR_IMPLEMENT_SETTINGS, -) -from rdagent.components.task_implementation.factor_implementation.evolving.evaluators import ( - FactorImplementationCorrelationEvaluator, - FactorImplementationEvaluator, - FactorImplementationIndexEvaluator, - FactorImplementationIndexFormatEvaluator, - FactorImplementationMissingValuesEvaluator, - FactorImplementationRowCountEvaluator, - FactorImplementationSingleColumnEvaluator, - FactorImplementationValuesEvaluator, -) -from rdagent.components.task_implementation.factor_implementation.factor import ( - FileBasedFactorImplementation, +from rdagent.components.coder.factor_coder.config import FACTOR_IMPLEMENT_SETTINGS +from rdagent.components.coder.factor_coder.CoSTEER.evaluators import ( + FactorCorrelationEvaluator, + FactorEqualValueCountEvaluator, + FactorEvaluator, + FactorIndexEvaluator, + FactorMissingValuesEvaluator, + FactorOutputFormatEvaluator, + FactorRowCountEvaluator, + FactorSingleColumnEvaluator, ) +from rdagent.components.coder.factor_coder.factor import FileBasedFactorImplementation from rdagent.core.exception import ImplementRunException from rdagent.core.experiment import Implementation, Task from rdagent.core.task_generator import TaskGenerator @@ -43,7 +39,7 @@ class BaseEval: def __init__( self, - evaluator_l: List[FactorImplementationEvaluator], + evaluator_l: List[FactorEvaluator], test_cases: List[TestCase], generate_method: TaskGenerator, catch_eval_except: bool = True, @@ -52,7 +48,7 @@ class BaseEval: ---------- test_cases : List[TestCase] cases to be evaluated, ground truth are included in the test cases. - evaluator_l : List[FactorImplementationEvaluator] + evaluator_l : List[FactorEvaluator] A list of evaluators to evaluate the generated code. catch_eval_except : bool If we want to debug the evaluators, we recommend to set the this parameter to True. @@ -81,7 +77,7 @@ class BaseEval: self, case_gt: Implementation, case_gen: Implementation, - ) -> List[Union[Tuple[FactorImplementationEvaluator, object], Exception]]: + ) -> List[Union[Tuple[FactorEvaluator, object], Exception]]: """Parameters ---------- case_gt : FactorImplementation @@ -91,14 +87,14 @@ class BaseEval: Returns ------- - List[Union[Tuple[FactorImplementationEvaluator, object],Exception]] + List[Union[Tuple[FactorEvaluator, object],Exception]] for each item If the evaluation run successfully, return the evaluate results. Otherwise, return the exception. """ eval_res = [] for ev in self.evaluator_l: try: - eval_res.append((ev, ev.evaluate(case_gt, case_gen))) + eval_res.append((ev, ev.evaluate(implementation=case_gen, gt_implementation=case_gt))) # if the corr ev is successfully evaluated and achieve the best performance, then break except ImplementRunException as e: return e @@ -116,18 +112,18 @@ class FactorImplementEval(BaseEval): self, test_cases: TestCase, method: TaskGenerator, - test_round: int = 10, *args, + test_round: int = 10, **kwargs, ): online_evaluator_l = [ - FactorImplementationSingleColumnEvaluator(), - FactorImplementationIndexFormatEvaluator(), - FactorImplementationRowCountEvaluator(), - FactorImplementationIndexEvaluator(), - FactorImplementationMissingValuesEvaluator(), - FactorImplementationValuesEvaluator(), - FactorImplementationCorrelationEvaluator(hard_check=False), + FactorSingleColumnEvaluator(), + FactorOutputFormatEvaluator(), + FactorRowCountEvaluator(), + FactorIndexEvaluator(), + FactorMissingValuesEvaluator(), + FactorEqualValueCountEvaluator(), + FactorCorrelationEvaluator(hard_check=False), ] super().__init__(online_evaluator_l, test_cases, method, *args, **kwargs) self.test_round = test_round diff --git a/rdagent/components/task_implementation/factor_implementation/CoSTEER.py b/rdagent/components/coder/factor_coder/CoSTEER/__init__.py similarity index 63% rename from rdagent/components/task_implementation/factor_implementation/CoSTEER.py rename to rdagent/components/coder/factor_coder/CoSTEER/__init__.py index 6b90ad04..f48b1e47 100644 --- a/rdagent/components/task_implementation/factor_implementation/CoSTEER.py +++ b/rdagent/components/coder/factor_coder/CoSTEER/__init__.py @@ -1,40 +1,38 @@ import pickle from pathlib import Path -from typing import List -from rdagent.components.task_implementation.factor_implementation.config import ( - FACTOR_IMPLEMENT_SETTINGS, +from rdagent.components.coder.factor_coder.config import FACTOR_IMPLEMENT_SETTINGS +from rdagent.components.coder.factor_coder.CoSTEER.evaluators import ( + FactorEvaluatorForCoder, + FactorMultiEvaluator, ) -from rdagent.components.task_implementation.factor_implementation.evolving.evaluators import ( - FactorImplementationEvaluatorV1, - FactorImplementationsMultiEvaluator, -) -from rdagent.components.task_implementation.factor_implementation.evolving.evolvable_subjects import ( +from rdagent.components.coder.factor_coder.CoSTEER.evolvable_subjects import ( FactorEvolvingItem, ) -from rdagent.components.task_implementation.factor_implementation.evolving.evolving_strategy import ( +from rdagent.components.coder.factor_coder.CoSTEER.evolving_strategy import ( FactorEvolvingStrategyWithGraph, ) -from rdagent.components.task_implementation.factor_implementation.evolving.knowledge_management import ( - FactorImplementationGraphKnowledgeBase, - FactorImplementationGraphRAGStrategy, - FactorImplementationKnowledgeBaseV1, -) -from rdagent.components.task_implementation.factor_implementation.factor import ( - FactorExperiment, +from rdagent.components.coder.factor_coder.CoSTEER.knowledge_management import ( + FactorGraphKnowledgeBase, + FactorGraphRAGStrategy, + FactorKnowledgeBaseV1, ) +from rdagent.components.coder.factor_coder.factor import FactorExperiment from rdagent.core.evolving_agent import RAGEvoAgent -from rdagent.core.experiment import Experiment +from rdagent.core.scenario import Scenario from rdagent.core.task_generator import TaskGenerator -class CoSTEERFG(TaskGenerator[FactorExperiment]): +class FactorCoSTEER(TaskGenerator[FactorExperiment]): def __init__( self, + *args, with_knowledge: bool = True, with_feedback: bool = True, knowledge_self_gen: bool = True, + **kwargs, ) -> None: + super().__init__(*args, **kwargs) self.max_loop = FACTOR_IMPLEMENT_SETTINGS.max_loop self.knowledge_base_path = ( Path(FACTOR_IMPLEMENT_SETTINGS.knowledge_base_path) @@ -49,30 +47,28 @@ class CoSTEERFG(TaskGenerator[FactorExperiment]): self.with_knowledge = with_knowledge self.with_feedback = with_feedback self.knowledge_self_gen = knowledge_self_gen - self.evolving_strategy = FactorEvolvingStrategyWithGraph() + self.evolving_strategy = FactorEvolvingStrategyWithGraph(scen=self.scen) # declare the factor evaluator - self.factor_evaluator = FactorImplementationsMultiEvaluator(FactorImplementationEvaluatorV1()) + self.factor_evaluator = FactorMultiEvaluator(FactorEvaluatorForCoder(scen=self.scen), scen=self.scen) self.evolving_version = 2 def load_or_init_knowledge_base(self, former_knowledge_base_path: Path = None, component_init_list: list = []): if former_knowledge_base_path is not None and former_knowledge_base_path.exists(): factor_knowledge_base = pickle.load(open(former_knowledge_base_path, "rb")) - if self.evolving_version == 1 and not isinstance( - factor_knowledge_base, FactorImplementationKnowledgeBaseV1 - ): + if self.evolving_version == 1 and not isinstance(factor_knowledge_base, FactorKnowledgeBaseV1): raise ValueError("The former knowledge base is not compatible with the current version") elif self.evolving_version == 2 and not isinstance( factor_knowledge_base, - FactorImplementationGraphKnowledgeBase, + FactorGraphKnowledgeBase, ): raise ValueError("The former knowledge base is not compatible with the current version") else: factor_knowledge_base = ( - FactorImplementationGraphKnowledgeBase( + FactorGraphKnowledgeBase( init_component_list=component_init_list, ) if self.evolving_version == 2 - else FactorImplementationKnowledgeBaseV1() + else FactorKnowledgeBaseV1() ) return factor_knowledge_base @@ -83,15 +79,15 @@ class CoSTEERFG(TaskGenerator[FactorExperiment]): component_init_list=[], ) # init rag method - self.rag = FactorImplementationGraphRAGStrategy(factor_knowledge_base) + self.rag = FactorGraphRAGStrategy(factor_knowledge_base) # init intermediate items - factor_implementations = FactorEvolvingItem(sub_tasks=exp.sub_tasks) + factor_experiment = FactorEvolvingItem(sub_tasks=exp.sub_tasks) self.evolve_agent = RAGEvoAgent(max_loop=self.max_loop, evolving_strategy=self.evolving_strategy, rag=self.rag) - factor_implementations = self.evolve_agent.multistep_evolve( - factor_implementations, + factor_experiment = self.evolve_agent.multistep_evolve( + factor_experiment, self.factor_evaluator, with_knowledge=self.with_knowledge, with_feedback=self.with_feedback, @@ -103,4 +99,4 @@ class CoSTEERFG(TaskGenerator[FactorExperiment]): pickle.dump(factor_knowledge_base, open(self.new_knowledge_base_path, "wb")) self.knowledge_base = factor_knowledge_base self.latest_factor_implementations = exp.sub_tasks - return factor_implementations + return factor_experiment diff --git a/rdagent/components/coder/factor_coder/CoSTEER/evaluators.py b/rdagent/components/coder/factor_coder/CoSTEER/evaluators.py new file mode 100644 index 00000000..7cb861dd --- /dev/null +++ b/rdagent/components/coder/factor_coder/CoSTEER/evaluators.py @@ -0,0 +1,610 @@ +import io +import json +import re +from abc import abstractmethod +from pathlib import Path +from typing import List, Tuple + +import pandas as pd +from jinja2 import Environment, StrictUndefined + +from rdagent.components.coder.factor_coder.config import FACTOR_IMPLEMENT_SETTINGS +from rdagent.components.coder.factor_coder.CoSTEER.evolvable_subjects import ( + FactorEvolvingItem, +) +from rdagent.components.coder.factor_coder.factor import FactorTask +from rdagent.core.conf import RD_AGENT_SETTINGS +from rdagent.core.evaluation import Evaluator +from rdagent.core.evolving_framework import Feedback, QueriedKnowledge +from rdagent.core.experiment import Implementation, Task +from rdagent.core.log import RDAgentLog +from rdagent.core.prompts import Prompts +from rdagent.core.utils import multiprocessing_wrapper +from rdagent.oai.llm_utils import APIBackend + +evaluate_prompts = Prompts(file_path=Path(__file__).parent.parent / "prompts.yaml") + + +class FactorEvaluator(Evaluator): + # TODO: + # I think we should have unified interface for all evaluates, for examples. + # So we should adjust the interface of other factors + @abstractmethod + def evaluate( + self, + target_task: Task, + implementation: Implementation, + gt_implementation: Implementation, + **kwargs, + ) -> Tuple[str, object]: + """You can get the dataframe by + + .. code-block:: python + + _, gen_df = implementation.execute() + _, gt_df = gt_implementation.execute() + + Returns + ------- + Tuple[str, object] + - str: the text-based description of the evaluation result + - object: a comparable metric (bool, integer, float ...) None for evaluator with only text-based result + + """ + raise NotImplementedError("Please implement the `evaluator` method") + + def _get_df(self, gt_implementation: Implementation, implementation: Implementation): + if gt_implementation is not None: + _, gt_df = gt_implementation.execute() + if isinstance(gt_df, pd.Series): + gt_df = gt_df.to_frame("gt_factor") + if isinstance(gt_df, pd.DataFrame): + gt_df = gt_df.sort_index() + else: + gt_df = None + + _, gen_df = implementation.execute() + if isinstance(gen_df, pd.Series): + gen_df = gen_df.to_frame("source_factor") + if isinstance(gen_df, pd.DataFrame): + gen_df = gen_df.sort_index() + return gt_df, gen_df + + def __str__(self) -> str: + return self.__class__.__name__ + + +class FactorCodeEvaluator(FactorEvaluator): + def evaluate( + self, + target_task: FactorTask, + implementation: Implementation, + execution_feedback: str, + factor_value_feedback: str = "", + gt_implementation: Implementation = None, + **kwargs, + ): + factor_information = target_task.get_factor_information() + code = implementation.code + + system_prompt = ( + Environment(undefined=StrictUndefined) + .from_string(evaluate_prompts["evaluator_code_feedback_v1_system"]) + .render(scenario=self.scen.get_scenario_all_desc() if self.scen is not None else "No scenario description.") + ) + + execution_feedback_to_render = execution_feedback + for _ in range(10): # 10 times to split the content is enough + user_prompt = ( + Environment(undefined=StrictUndefined) + .from_string( + evaluate_prompts["evaluator_code_feedback_v1_user"], + ) + .render( + factor_information=factor_information, + code=code, + execution_feedback=execution_feedback_to_render, + factor_value_feedback=factor_value_feedback, + gt_code=gt_implementation.code if gt_implementation else None, + ) + ) + if ( + APIBackend().build_messages_and_calculate_token( + user_prompt=user_prompt, + system_prompt=system_prompt, + ) + > RD_AGENT_SETTINGS.chat_token_limit + ): + execution_feedback_to_render = execution_feedback_to_render[len(execution_feedback_to_render) // 2 :] + else: + break + critic_response = APIBackend().build_messages_and_create_chat_completion( + user_prompt=user_prompt, + system_prompt=system_prompt, + json_mode=False, + ) + + return critic_response, None + + +class FactorSingleColumnEvaluator(FactorEvaluator): + def evaluate( + self, + implementation: Implementation, + gt_implementation: Implementation, + ) -> Tuple[str, object]: + _, gen_df = self._get_df(gt_implementation, implementation) + + if len(gen_df.columns) == 1: + return "The source dataframe has only one column which is correct.", True + else: + return ( + "The source dataframe has more than one column. Please check the implementation. We only evaluate the first column.", + False, + ) + + +class FactorOutputFormatEvaluator(FactorEvaluator): + def evaluate( + self, + implementation: Implementation, + gt_implementation: Implementation, + ) -> Tuple[str, object]: + gt_df, gen_df = self._get_df(gt_implementation, implementation) + if gen_df is None: + return ( + "The source dataframe is None. Skip the evaluation of the output format.", + False, + ) + buffer = io.StringIO() + gen_df.info(buf=buffer) + gen_df_info_str = buffer.getvalue() + system_prompt = ( + Environment(undefined=StrictUndefined) + .from_string( + evaluate_prompts["evaluator_output_format_system"], + ) + .render(scenario=self.scen.get_scenario_all_desc() if self.scen is not None else "No scenario description.") + ) + resp = APIBackend().build_messages_and_create_chat_completion( + user_prompt=gen_df_info_str, system_prompt=system_prompt, json_mode=True + ) + resp_dict = json.loads(resp) + if isinstance(resp_dict["output_format_decision"], str) and resp_dict["output_format_decision"].lower() in ( + "true", + "false", + ): + resp_dict["output_format_decision"] = bool(resp_dict["output_format_decision"]) + return ( + resp_dict["output_format_feedback"], + resp_dict["output_format_decision"], + ) + + +class FactorRowCountEvaluator(FactorEvaluator): + def evaluate( + self, + implementation: Implementation, + gt_implementation: Implementation, + ) -> Tuple[str, object]: + gt_df, gen_df = self._get_df(gt_implementation, implementation) + + if gen_df.shape[0] == gt_df.shape[0]: + return "Both dataframes have the same rows count.", True + else: + return ( + f"The source dataframe and the ground truth dataframe have different rows count. The source dataframe has {gen_df.shape[0]} rows, while the ground truth dataframe has {gt_df.shape[0]} rows. Please check the implementation.", + False, + ) + + +class FactorIndexEvaluator(FactorEvaluator): + def evaluate( + self, + implementation: Implementation, + gt_implementation: Implementation, + ) -> Tuple[str, object]: + gt_df, gen_df = self._get_df(gt_implementation, implementation) + + if gen_df.index.equals(gt_df.index): + return "Both dataframes have the same index.", True + else: + return ( + "The source dataframe and the ground truth dataframe have different index. Please check the implementation.", + False, + ) + + +class FactorMissingValuesEvaluator(FactorEvaluator): + def evaluate( + self, + implementation: Implementation, + gt_implementation: Implementation, + ) -> Tuple[str, object]: + gt_df, gen_df = self._get_df(gt_implementation, implementation) + + if gen_df.isna().sum().sum() == gt_df.isna().sum().sum(): + return "Both dataframes have the same missing values.", True + else: + return ( + f"The dataframes do not have the same missing values. The source dataframe has {gen_df.isna().sum().sum()} missing values, while the ground truth dataframe has {gt_df.isna().sum().sum()} missing values. Please check the implementation.", + False, + ) + + +class FactorEqualValueCountEvaluator(FactorEvaluator): + def evaluate( + self, + implementation: Implementation, + gt_implementation: Implementation, + ) -> Tuple[str, object]: + gt_df, gen_df = self._get_df(gt_implementation, implementation) + + try: + close_values = gen_df.sub(gt_df).abs().lt(1e-6) + result_int = close_values.astype(int) + pos_num = result_int.sum().sum() + acc_rate = pos_num / close_values.size + except: + close_values = gen_df + if close_values.all().iloc[0]: + return ( + "All values in the dataframes are equal within the tolerance of 1e-6.", + acc_rate, + ) + else: + return ( + "Some values differ by more than the tolerance of 1e-6. Check for rounding errors or differences in the calculation methods.", + acc_rate, + ) + + +class FactorCorrelationEvaluator(FactorEvaluator): + def __init__(self, hard_check: bool, *args, **kwargs) -> None: + super().__init__(*args, **kwargs) + self.hard_check = hard_check + + def evaluate( + self, + implementation: Implementation, + gt_implementation: Implementation, + ) -> Tuple[str, object]: + gt_df, gen_df = self._get_df(gt_implementation, implementation) + + concat_df = pd.concat([gen_df, gt_df], axis=1) + concat_df.columns = ["source", "gt"] + ic = concat_df.groupby("datetime").apply(lambda df: df["source"].corr(df["gt"])).dropna().mean() + ric = ( + concat_df.groupby("datetime") + .apply(lambda df: df["source"].corr(df["gt"], method="spearman")) + .dropna() + .mean() + ) + + if self.hard_check: + if ic > 0.99 and ric > 0.99: + return ( + f"The dataframes are highly correlated. The ic is {ic:.6f} and the rankic is {ric:.6f}.", + True, + ) + else: + return ( + f"The dataframes are not sufficiently high correlated. The ic is {ic:.6f} and the rankic is {ric:.6f}. Investigate the factors that might be causing the discrepancies and ensure that the logic of the factor calculation is consistent.", + False, + ) + else: + return f"The ic is ({ic:.6f}) and the rankic is ({ric:.6f}).", ic + + +class FactorValueEvaluator(FactorEvaluator): + + def evaluate( + self, + implementation: Implementation, + gt_implementation: Implementation, + **kwargs, + ) -> Tuple: + conclusions = [] + + # Check if both dataframe has only one columns + feedback_str, _ = FactorSingleColumnEvaluator(self.scen).evaluate(implementation, gt_implementation) + conclusions.append(feedback_str) + + # Check if the index of the dataframe is ("datetime", "instrument") + feedback_str, _ = FactorOutputFormatEvaluator(self.scen).evaluate(implementation, gt_implementation) + conclusions.append(feedback_str) + + # Check if both dataframe have the same rows count + if gt_implementation is not None: + feedback_str, _ = FactorRowCountEvaluator(self.scen).evaluate(implementation, gt_implementation) + conclusions.append(feedback_str) + + feedback_str, same_index_result = FactorIndexEvaluator(self.scen).evaluate( + implementation, gt_implementation + ) + conclusions.append(feedback_str) + + feedback_str, _ = FactorMissingValuesEvaluator(self.scen).evaluate(implementation, gt_implementation) + conclusions.append(feedback_str) + + feedback_str, equal_value_ratio_result = FactorEqualValueCountEvaluator(self.scen).evaluate( + implementation, gt_implementation + ) + conclusions.append(feedback_str) + + if same_index_result: + feedback_str, high_correlation_result = FactorCorrelationEvaluator( + hard_check=True, scen=self.scen + ).evaluate(implementation, gt_implementation) + else: + high_correlation_result = False + feedback_str = "The source dataframe and the ground truth dataframe have different index. Give up comparing the values and correlation because it's useless" + conclusions.append(feedback_str) + + # Combine all conclusions into a single string + conclusion_str = "\n".join(conclusions) + + same_value_or_high_correlation = ( + ((equal_value_ratio_result > 0.99) or high_correlation_result) if gt_implementation is not None else False + ) + return conclusion_str, same_value_or_high_correlation + + +class FactorFinalDecisionEvaluator(Evaluator): + def evaluate( + self, + target_task: FactorTask, + execution_feedback: str, + value_feedback: str, + code_feedback: str, + **kwargs, + ) -> Tuple: + system_prompt = ( + Environment(undefined=StrictUndefined) + .from_string(evaluate_prompts["evaluator_final_decision_v1_system"]) + .render(scenario=self.scen.get_scenario_all_desc() if self.scen is not None else "No scenario description.") + ) + execution_feedback_to_render = execution_feedback + + for _ in range(10): # 10 times to split the content is enough + user_prompt = ( + Environment(undefined=StrictUndefined) + .from_string( + evaluate_prompts["evaluator_final_decision_v1_user"], + ) + .render( + factor_information=target_task.get_factor_information(), + execution_feedback=execution_feedback_to_render, + code_feedback=code_feedback, + factor_value_feedback=( + value_feedback + if value_feedback is not None + else "No Ground Truth Value provided, so no evaluation on value is performed." + ), + ) + ) + if ( + APIBackend().build_messages_and_calculate_token( + user_prompt=user_prompt, + system_prompt=system_prompt, + ) + > RD_AGENT_SETTINGS.chat_token_limit + ): + execution_feedback_to_render = execution_feedback_to_render[len(execution_feedback_to_render) // 2 :] + else: + break + + final_evaluation_dict = json.loads( + APIBackend().build_messages_and_create_chat_completion( + user_prompt=user_prompt, + system_prompt=system_prompt, + json_mode=True, + ), + ) + if isinstance(final_evaluation_dict["final_decision"], str) and final_evaluation_dict[ + "final_decision" + ].lower() in ("true", "false"): + final_evaluation_dict["final_decision"] = bool(final_evaluation_dict["final_decision"]) + return ( + final_evaluation_dict["final_decision"], + final_evaluation_dict["final_feedback"], + ) + + +class FactorSingleFeedback: + """This class is a feedback to single implementation which is generated from an evaluator.""" + + def __init__( + self, + execution_feedback: str = None, + value_generated_flag: bool = False, + code_feedback: str = None, + factor_value_feedback: str = None, + final_decision: bool = None, + final_feedback: str = None, + final_decision_based_on_gt: bool = None, + ) -> None: + self.execution_feedback = execution_feedback + self.value_generated_flag = value_generated_flag + self.code_feedback = code_feedback + self.factor_value_feedback = factor_value_feedback + self.final_decision = final_decision + self.final_feedback = final_feedback + self.final_decision_based_on_gt = final_decision_based_on_gt + + def __str__(self) -> str: + return f"""------------------Factor Execution Feedback------------------ +{self.execution_feedback} +------------------Factor Code Feedback------------------ +{self.code_feedback} +------------------Factor Value Feedback------------------ +{self.factor_value_feedback} +------------------Factor Final Feedback------------------ +{self.final_feedback} +------------------Factor Final Decision------------------ +This implementation is {'SUCCESS' if self.final_decision else 'FAIL'}. +""" + + +class FactorMultiFeedback( + Feedback, + List[FactorSingleFeedback], +): + """Feedback contains a list, each element is the corresponding feedback for each factor implementation.""" + + +class FactorEvaluatorForCoder(FactorEvaluator): + """This class is the v1 version of evaluator for a single factor implementation. + It calls several evaluators in share modules to evaluate the factor implementation. + """ + + def __init__(self, *args, **kwargs) -> None: + super().__init__(*args, **kwargs) + self.value_evaluator = FactorValueEvaluator(self.scen) + self.code_evaluator = FactorCodeEvaluator(self.scen) + self.final_decision_evaluator = FactorFinalDecisionEvaluator(self.scen) + + def evaluate( + self, + target_task: FactorTask, + implementation: Implementation, + gt_implementation: Implementation = None, + queried_knowledge: QueriedKnowledge = None, + **kwargs, + ) -> FactorSingleFeedback: + if implementation is None: + return None + + target_task_information = target_task.get_factor_information() + if ( + queried_knowledge is not None + and target_task_information in queried_knowledge.success_task_to_knowledge_dict + ): + return queried_knowledge.success_task_to_knowledge_dict[target_task_information].feedback + elif queried_knowledge is not None and target_task_information in queried_knowledge.failed_task_info_set: + return FactorSingleFeedback( + execution_feedback="This task has failed too many times, skip implementation.", + value_generated_flag=False, + code_feedback="This task has failed too many times, skip code evaluation.", + factor_value_feedback="This task has failed too many times, skip value evaluation.", + final_decision=False, + final_feedback="This task has failed too many times, skip final decision evaluation.", + final_decision_based_on_gt=False, + ) + else: + factor_feedback = FactorSingleFeedback() + + # 1. Get factor execution feedback to generated implementation and remove the long list of numbers in execution feedback + ( + execution_feedback, + gen_df, + ) = implementation.execute() + + execution_feedback = re.sub(r"(?<=\D)(,\s+-?\d+\.\d+){50,}(?=\D)", ", ", execution_feedback) + factor_feedback.execution_feedback = "\n".join( + [line for line in execution_feedback.split("\n") if "warning" not in line.lower()] + ) + + # 2. Get factor value feedback + if gen_df is None: + factor_feedback.factor_value_feedback = "No factor value generated, skip value evaluation." + factor_feedback.value_generated_flag = False + same_value_or_high_correlation = None + else: + factor_feedback.value_generated_flag = True + ( + factor_feedback.factor_value_feedback, + same_value_or_high_correlation, + ) = self.value_evaluator.evaluate(implementation=implementation, gt_implementation=gt_implementation) + + factor_feedback.final_decision_based_on_gt = gt_implementation is not None + + if same_value_or_high_correlation is not None and same_value_or_high_correlation is True: + # To avoid confusion, when same_value_or_high_correlation is True, we do not need code feedback + factor_feedback.code_feedback = "Final decision is True and there are no code critics." + factor_feedback.final_decision = same_value_or_high_correlation + factor_feedback.final_feedback = "Value evaluation passed, skip final decision evaluation." + else: + factor_feedback.code_feedback, _ = self.code_evaluator.evaluate( + target_task=target_task, + implementation=implementation, + execution_feedback=factor_feedback.execution_feedback, + value_feedback=factor_feedback.factor_value_feedback, + gt_implementation=gt_implementation, + ) + ( + factor_feedback.final_decision, + factor_feedback.final_feedback, + ) = self.final_decision_evaluator.evaluate( + target_task=target_task, + execution_feedback=factor_feedback.execution_feedback, + value_feedback=factor_feedback.factor_value_feedback, + code_feedback=factor_feedback.code_feedback, + ) + return factor_feedback + + +class FactorMultiEvaluator(Evaluator): + def __init__(self, single_evaluator, *args, **kwargs) -> None: + super().__init__(*args, **kwargs) + self.single_factor_implementation_evaluator = single_evaluator + + def evaluate( + self, + evo: FactorEvolvingItem, + queried_knowledge: QueriedKnowledge = None, + **kwargs, + ) -> FactorMultiFeedback: + multi_implementation_feedback = FactorMultiFeedback() + + # for index in range(len(evo.sub_tasks)): + # corresponding_implementation = evo.sub_implementations[index] + # corresponding_gt_implementation = ( + # evo.sub_gt_implementations[index] if evo.sub_gt_implementations is not None else None + # ) + + # multi_implementation_feedback.append( + # self.single_factor_implementation_evaluator.evaluate( + # target_task=evo.sub_tasks[index], + # implementation=corresponding_implementation, + # gt_implementation=corresponding_gt_implementation, + # queried_knowledge=queried_knowledge, + # ) + # ) + + calls = [] + for index in range(len(evo.sub_tasks)): + corresponding_implementation = evo.sub_implementations[index] + corresponding_gt_implementation = ( + evo.sub_gt_implementations[index] if evo.sub_gt_implementations is not None else None + ) + calls.append( + ( + self.single_factor_implementation_evaluator.evaluate, + ( + evo.sub_tasks[index], + corresponding_implementation, + corresponding_gt_implementation, + queried_knowledge, + ), + ), + ) + multi_implementation_feedback = multiprocessing_wrapper(calls, n=FACTOR_IMPLEMENT_SETTINGS.evo_multi_proc_n) + + final_decision = [ + None if single_feedback is None else single_feedback.final_decision + for single_feedback in multi_implementation_feedback + ] + RDAgentLog().info(f"Final decisions: {final_decision} True count: {final_decision.count(True)}") + + return multi_implementation_feedback + + +# TODO: +def shorten_prompt(tpl: str, render_kwargs: dict, shorten_key: str, max_trail: int = 10) -> str: + """When the prompt is too long. We have to shorten it. + But we should not truncate the prompt directly, so we should find the key we want to shorten and then shorten it. + """ + # TODO: this should replace most of code in + # - FactorFinalDecisionEvaluator.evaluate + # - FactorCodeEvaluator.evaluate diff --git a/rdagent/components/task_implementation/factor_implementation/evolving/evolvable_subjects.py b/rdagent/components/coder/factor_coder/CoSTEER/evolvable_subjects.py similarity index 92% rename from rdagent/components/task_implementation/factor_implementation/evolving/evolvable_subjects.py rename to rdagent/components/coder/factor_coder/CoSTEER/evolvable_subjects.py index a79d96a6..1bf47073 100644 --- a/rdagent/components/task_implementation/factor_implementation/evolving/evolvable_subjects.py +++ b/rdagent/components/coder/factor_coder/CoSTEER/evolvable_subjects.py @@ -1,4 +1,4 @@ -from rdagent.components.task_implementation.factor_implementation.factor import ( +from rdagent.components.coder.factor_coder.factor import ( FactorExperiment, FactorTask, FileBasedFactorImplementation, diff --git a/rdagent/components/task_implementation/factor_implementation/evolving/evolving_strategy.py b/rdagent/components/coder/factor_coder/CoSTEER/evolving_strategy.py similarity index 93% rename from rdagent/components/task_implementation/factor_implementation/evolving/evolving_strategy.py rename to rdagent/components/coder/factor_coder/CoSTEER/evolving_strategy.py index 7189bae2..4d3fa2f7 100644 --- a/rdagent/components/task_implementation/factor_implementation/evolving/evolving_strategy.py +++ b/rdagent/components/coder/factor_coder/CoSTEER/evolving_strategy.py @@ -8,23 +8,19 @@ from typing import TYPE_CHECKING from jinja2 import Environment, StrictUndefined -from rdagent.components.task_implementation.factor_implementation.config import ( - FACTOR_IMPLEMENT_SETTINGS, -) -from rdagent.components.task_implementation.factor_implementation.evolving.evolvable_subjects import ( +from rdagent.components.coder.factor_coder.config import FACTOR_IMPLEMENT_SETTINGS +from rdagent.components.coder.factor_coder.CoSTEER.evolvable_subjects import ( FactorEvolvingItem, ) -from rdagent.components.task_implementation.factor_implementation.evolving.scheduler import ( +from rdagent.components.coder.factor_coder.CoSTEER.scheduler import ( LLMSelect, RandomSelect, ) -from rdagent.components.task_implementation.factor_implementation.factor import ( +from rdagent.components.coder.factor_coder.factor import ( FactorTask, FileBasedFactorImplementation, ) -from rdagent.components.task_implementation.factor_implementation.utils import ( - get_data_folder_intro, -) +from rdagent.components.coder.factor_coder.utils import get_data_folder_intro from rdagent.core.conf import RD_AGENT_SETTINGS from rdagent.core.evolving_framework import EvolvingStrategy, QueriedKnowledge from rdagent.core.experiment import Implementation @@ -33,9 +29,9 @@ from rdagent.core.utils import multiprocessing_wrapper from rdagent.oai.llm_utils import APIBackend if TYPE_CHECKING: - from rdagent.components.task_implementation.factor_implementation.evolving.knowledge_management import ( - FactorImplementationQueriedKnowledge, - FactorImplementationQueriedKnowledgeV1, + from rdagent.components.coder.factor_coder.CoSTEER.knowledge_management import ( + FactorQueriedKnowledge, + FactorQueriedKnowledgeV1, ) implement_prompts = Prompts(file_path=Path(__file__).parent.parent / "prompts.yaml") @@ -54,7 +50,7 @@ class MultiProcessEvolvingStrategy(EvolvingStrategy): self, *, evo: FactorEvolvingItem, - queried_knowledge: FactorImplementationQueriedKnowledge | None = None, + queried_knowledge: FactorQueriedKnowledge | None = None, **kwargs, ) -> FactorEvolvingItem: self.num_loop += 1 @@ -93,6 +89,7 @@ class MultiProcessEvolvingStrategy(EvolvingStrategy): implementation_factors_per_round, new_evo, queried_knowledge.former_traces, + self.scen, ) result = multiprocessing_wrapper( @@ -120,7 +117,7 @@ class FactorEvolvingStrategy(MultiProcessEvolvingStrategy): def implement_one_factor( self, target_task: FactorTask, - queried_knowledge: FactorImplementationQueriedKnowledgeV1 = None, + queried_knowledge: FactorQueriedKnowledgeV1 = None, ) -> Implementation: factor_information_str = target_task.get_factor_information() @@ -197,7 +194,8 @@ class FactorEvolvingStrategy(MultiProcessEvolvingStrategy): class FactorEvolvingStrategyWithGraph(MultiProcessEvolvingStrategy): - def __init__(self) -> None: + def __init__(self, *args, **kwargs) -> None: + super().__init__(*args, **kwargs) self.num_loop = 0 self.haveSelected = False @@ -244,7 +242,7 @@ class FactorEvolvingStrategyWithGraph(MultiProcessEvolvingStrategy): implement_prompts["evolving_strategy_factor_implementation_v1_system"], ) .render( - data_info=get_data_folder_intro(), + scenario=self.scen.get_scenario_all_desc(), queried_former_failed_knowledge=queried_former_failed_knowledge_to_render, ) ) diff --git a/rdagent/components/task_implementation/factor_implementation/evolving/knowledge_management.py b/rdagent/components/coder/factor_coder/CoSTEER/knowledge_management.py similarity index 95% rename from rdagent/components/task_implementation/factor_implementation/evolving/knowledge_management.py rename to rdagent/components/coder/factor_coder/CoSTEER/knowledge_management.py index 400be6ef..c238d176 100644 --- a/rdagent/components/task_implementation/factor_implementation/evolving/knowledge_management.py +++ b/rdagent/components/coder/factor_coder/CoSTEER/knowledge_management.py @@ -10,19 +10,15 @@ from typing import Union from jinja2 import Environment, StrictUndefined +from rdagent.components.coder.factor_coder.config import FACTOR_IMPLEMENT_SETTINGS +from rdagent.components.coder.factor_coder.CoSTEER.evaluators import ( + FactorSingleFeedback, +) +from rdagent.components.coder.factor_coder.factor import FactorTask from rdagent.components.knowledge_management.graph import ( UndirectedGraph, UndirectedNode, ) -from rdagent.components.task_implementation.factor_implementation.config import ( - FACTOR_IMPLEMENT_SETTINGS, -) -from rdagent.components.task_implementation.factor_implementation.evolving.evaluators import ( - FactorImplementationSingleFeedback, -) -from rdagent.components.task_implementation.factor_implementation.evolving.evolving_strategy import ( - FactorTask, -) from rdagent.core.evolving_framework import ( EvolvableSubjects, EvoStep, @@ -40,12 +36,12 @@ from rdagent.oai.llm_utils import ( ) -class FactorImplementationKnowledge(Knowledge): +class FactorKnowledge(Knowledge): def __init__( self, target_task: FactorTask, implementation: Implementation, - feedback: FactorImplementationSingleFeedback, + feedback: FactorSingleFeedback, ) -> None: """ Initialize a FactorKnowledge object. The FactorKnowledge object is used to store a factor implementation without the ground truth code and value. @@ -68,15 +64,15 @@ class FactorImplementationKnowledge(Knowledge): """ -class FactorImplementationQueriedKnowledge(QueriedKnowledge): +class FactorQueriedKnowledge(QueriedKnowledge): def __init__(self, success_task_to_knowledge_dict: dict = {}, failed_task_info_set: set = set()) -> None: self.success_task_to_knowledge_dict = success_task_to_knowledge_dict self.failed_task_info_set = failed_task_info_set -class FactorImplementationKnowledgeBaseV1(KnowledgeBase): +class FactorKnowledgeBaseV1(KnowledgeBase): def __init__(self) -> None: - self.implementation_trace: dict[str, FactorImplementationKnowledge] = dict() + self.implementation_trace: dict[str, FactorKnowledge] = dict() self.success_task_info_set: set[str] = set() self.task_to_embedding = dict() @@ -88,15 +84,15 @@ class FactorImplementationKnowledgeBaseV1(KnowledgeBase): raise NotImplementedError -class FactorImplementationQueriedKnowledgeV1(FactorImplementationQueriedKnowledge): +class FactorQueriedKnowledgeV1(FactorQueriedKnowledge): def __init__(self) -> None: self.working_task_to_former_failed_knowledge_dict = dict() self.working_task_to_similar_successful_knowledge_dict = dict() super().__init__() -class FactorImplementationRAGStrategyV1(RAGStrategy): - def __init__(self, knowledgebase: FactorImplementationKnowledgeBaseV1) -> None: +class FactorRAGStrategyV1(RAGStrategy): + def __init__(self, knowledgebase: FactorKnowledgeBaseV1) -> None: super().__init__(knowledgebase) self.current_generated_trace_count = 0 @@ -123,7 +119,7 @@ class FactorImplementationRAGStrategyV1(RAGStrategy): single_feedback = feedback[task_index] if single_feedback is None: continue - single_knowledge = FactorImplementationKnowledge( + single_knowledge = FactorKnowledge( target_task=target_task, implementation=implementation, feedback=single_feedback, @@ -149,7 +145,7 @@ class FactorImplementationRAGStrategyV1(RAGStrategy): v1_query_similar_success_limit = FACTOR_IMPLEMENT_SETTINGS.v1_query_similar_success_limit fail_task_trial_limit = FACTOR_IMPLEMENT_SETTINGS.fail_task_trial_limit - queried_knowledge = FactorImplementationQueriedKnowledgeV1() + queried_knowledge = FactorQueriedKnowledgeV1() for target_factor_task in evo.sub_tasks: target_factor_task_information = target_factor_task.get_factor_information() if target_factor_task_information in self.knowledgebase.success_task_info_set: @@ -199,7 +195,7 @@ class FactorImplementationRAGStrategyV1(RAGStrategy): return queried_knowledge -class FactorImplementationQueriedGraphKnowledge(FactorImplementationQueriedKnowledge): +class FactorQueriedGraphKnowledge(FactorQueriedKnowledge): # Aggregation of knowledge def __init__( self, @@ -214,8 +210,8 @@ class FactorImplementationQueriedGraphKnowledge(FactorImplementationQueriedKnowl super().__init__(**kwargs) -class FactorImplementationGraphRAGStrategy(RAGStrategy): - def __init__(self, knowledgebase: FactorImplementationGraphKnowledgeBase) -> None: +class FactorGraphRAGStrategy(RAGStrategy): + def __init__(self, knowledgebase: FactorGraphKnowledgeBase) -> None: super().__init__(knowledgebase) self.current_generated_trace_count = 0 self.prompt = Prompts(file_path=Path(__file__).parent.parent / "prompts.yaml") @@ -242,7 +238,7 @@ class FactorImplementationGraphRAGStrategy(RAGStrategy): single_feedback = feedback[task_index] if single_feedback is None: continue - single_knowledge = FactorImplementationKnowledge( + single_knowledge = FactorKnowledge( target_task=target_task, implementation=implementation, feedback=single_feedback, @@ -288,7 +284,7 @@ class FactorImplementationGraphRAGStrategy(RAGStrategy): def query(self, evo: EvolvableSubjects, evolving_trace: list[EvoStep]) -> QueriedKnowledge | None: conf_knowledge_sampler = FACTOR_IMPLEMENT_SETTINGS.v2_knowledge_sampler - factor_implementation_queried_graph_knowledge = FactorImplementationQueriedGraphKnowledge( + factor_implementation_queried_graph_knowledge = FactorQueriedGraphKnowledge( success_task_to_knowledge_dict=self.knowledgebase.success_task_to_knowledge_dict, ) @@ -390,7 +386,7 @@ class FactorImplementationGraphRAGStrategy(RAGStrategy): def former_trace_query( self, evo: EvolvableSubjects, - factor_implementation_queried_graph_knowledge: FactorImplementationQueriedGraphKnowledge, + factor_implementation_queried_graph_knowledge: FactorQueriedGraphKnowledge, v2_query_former_trace_limit: int = 5, ) -> Union[QueriedKnowledge, set]: """ @@ -440,11 +436,11 @@ class FactorImplementationGraphRAGStrategy(RAGStrategy): def component_query( self, evo: EvolvableSubjects, - factor_implementation_queried_graph_knowledge: FactorImplementationQueriedGraphKnowledge, + factor_implementation_queried_graph_knowledge: FactorQueriedGraphKnowledge, v2_query_component_limit: int = 5, knowledge_sampler: float = 1.0, ) -> QueriedKnowledge | None: - # queried_component_knowledge = FactorImplementationQueriedGraphComponentKnowledge() + # queried_component_knowledge = FactorQueriedGraphComponentKnowledge() for target_factor_task in evo.sub_tasks: target_factor_task_information = target_factor_task.get_factor_information() if ( @@ -580,11 +576,11 @@ class FactorImplementationGraphRAGStrategy(RAGStrategy): def error_query( self, evo: EvolvableSubjects, - factor_implementation_queried_graph_knowledge: FactorImplementationQueriedGraphKnowledge, + factor_implementation_queried_graph_knowledge: FactorQueriedGraphKnowledge, v2_query_error_limit: int = 5, knowledge_sampler: float = 1.0, ) -> QueriedKnowledge | None: - # queried_error_knowledge = FactorImplementationQueriedGraphErrorKnowledge() + # queried_error_knowledge = FactorQueriedGraphErrorKnowledge() for task_index, target_factor_task in enumerate(evo.sub_tasks): target_factor_task_information = target_factor_task.get_factor_information() factor_implementation_queried_graph_knowledge.error_with_success_task[target_factor_task_information] = {} @@ -712,7 +708,7 @@ class FactorImplementationGraphRAGStrategy(RAGStrategy): return factor_implementation_queried_graph_knowledge -class FactorImplementationGraphKnowledgeBase(KnowledgeBase): +class FactorGraphKnowledgeBase(KnowledgeBase): def __init__(self, init_component_list=None) -> None: """ Load knowledge, offer brief information of knowledge and common handle interfaces @@ -735,7 +731,7 @@ class FactorImplementationGraphKnowledgeBase(KnowledgeBase): # Add already success task self.success_task_to_knowledge_dict = {} - # key:node_id(for task trace and success implement), value:knowledge instance(aka 'FactorImplementationKnowledge') + # key:node_id(for task trace and success implement), value:knowledge instance(aka 'FactorKnowledge') self.node_to_implementation_knowledge_dict = {} # store the task description to component nodes diff --git a/rdagent/components/task_implementation/factor_implementation/evolving/scheduler.py b/rdagent/components/coder/factor_coder/CoSTEER/scheduler.py similarity index 75% rename from rdagent/components/task_implementation/factor_implementation/evolving/scheduler.py rename to rdagent/components/coder/factor_coder/CoSTEER/scheduler.py index ded011ee..90296b84 100644 --- a/rdagent/components/task_implementation/factor_implementation/evolving/scheduler.py +++ b/rdagent/components/coder/factor_coder/CoSTEER/scheduler.py @@ -1,17 +1,17 @@ import json from pathlib import Path +from typing import Dict from jinja2 import Environment, StrictUndefined -from rdagent.components.task_implementation.factor_implementation.evolving.evolvable_subjects import ( +from rdagent.components.coder.factor_coder.CoSTEER.evolvable_subjects import ( FactorEvolvingItem, ) -from rdagent.components.task_implementation.factor_implementation.utils import ( - get_data_folder_intro, -) +from rdagent.components.coder.factor_coder.utils import get_data_folder_intro from rdagent.core.conf import RD_AGENT_SETTINGS from rdagent.core.log import RDAgentLog from rdagent.core.prompts import Prompts +from rdagent.core.scenario import Scenario from rdagent.oai.llm_utils import APIBackend scheduler_prompts = Prompts(file_path=Path(__file__).parent.parent / "prompts.yaml") @@ -29,7 +29,13 @@ def RandomSelect(to_be_finished_task_index, implementation_factors_per_round): return to_be_finished_task_index -def LLMSelect(to_be_finished_task_index, implementation_factors_per_round, evo: FactorEvolvingItem, former_trace): +def LLMSelect( + to_be_finished_task_index, + implementation_factors_per_round, + evo: FactorEvolvingItem, + former_trace: Dict, + scen: Scenario, +): tasks = [] for i in to_be_finished_task_index: # find corresponding former trace for each task @@ -43,14 +49,10 @@ def LLMSelect(to_be_finished_task_index, implementation_factors_per_round, evo: scheduler_prompts["select_implementable_factor_system"], ) .render( - data_info=get_data_folder_intro(), + scenario=scen.get_scenario_all_desc(), ) ) - session = APIBackend(use_chat_cache=False).build_chat_session( - session_system_prompt=system_prompt, - ) - while True: user_prompt = ( Environment(undefined=StrictUndefined) @@ -63,15 +65,17 @@ def LLMSelect(to_be_finished_task_index, implementation_factors_per_round, evo: ) ) if ( - session.build_chat_completion_message_and_calculate_token( - user_prompt, + APIBackend().build_messages_and_calculate_token( + user_prompt=user_prompt, + system_prompt=system_prompt, ) < RD_AGENT_SETTINGS.chat_token_limit ): break - response = session.build_chat_completion( + response = APIBackend().build_messages_and_create_chat_completion( user_prompt=user_prompt, + system_prompt=system_prompt, json_mode=True, ) try: diff --git a/rdagent/components/task_implementation/factor_implementation/config.py b/rdagent/components/coder/factor_coder/config.py similarity index 100% rename from rdagent/components/task_implementation/factor_implementation/config.py rename to rdagent/components/coder/factor_coder/config.py diff --git a/rdagent/components/task_implementation/factor_implementation/factor.py b/rdagent/components/coder/factor_coder/factor.py similarity index 98% rename from rdagent/components/task_implementation/factor_implementation/factor.py rename to rdagent/components/coder/factor_coder/factor.py index 91232693..72d01995 100644 --- a/rdagent/components/task_implementation/factor_implementation/factor.py +++ b/rdagent/components/coder/factor_coder/factor.py @@ -9,9 +9,7 @@ from typing import Tuple, Union import pandas as pd from filelock import FileLock -from rdagent.components.task_implementation.factor_implementation.config import ( - FACTOR_IMPLEMENT_SETTINGS, -) +from rdagent.components.coder.factor_coder.config import FACTOR_IMPLEMENT_SETTINGS from rdagent.core.exception import ( CodeFormatException, NoOutputException, diff --git a/rdagent/components/task_implementation/factor_implementation/prompts.yaml b/rdagent/components/coder/factor_coder/prompts.yaml similarity index 73% rename from rdagent/components/task_implementation/factor_implementation/prompts.yaml rename to rdagent/components/coder/factor_coder/prompts.yaml index b6551c46..486b7ef8 100644 --- a/rdagent/components/task_implementation/factor_implementation/prompts.yaml +++ b/rdagent/components/coder/factor_coder/prompts.yaml @@ -1,8 +1,10 @@ + evaluator_code_feedback_v1_system: |- - Your job is to give critic to user's code. User's code is expected to implement some factors in quant investment. The code contains reading data from a HDF5(H5) file, calculate the factor to each instrument on each datetime, and save the result pandas dataframe to a HDF5(H5) file. - - User will firstly provide you the information of the factor, which includes the name of the factor, description of the factor, the formulation of the factor and the description of the formulation. You can check whether user's code is align with the factor. + User is trying to implement some factors in the following scenario: + {{ scenario }} + User will provide you the information of the factor. + Your job is to check whether user's code is align with the factor and the scenario. The user will provide the source python code and the execution error message if execution failed. The user might provide you the ground truth code for you to provide the critic. You should not leak the ground truth code to the user in any form but you can use it to provide the critic. @@ -14,6 +16,7 @@ evaluator_code_feedback_v1_system: |- You should provide the suggestion to each of your critic to help the user improve the code. Please response the critic in the following format. Here is an example structure for the output: critic 1: The critic message to critic 1 critic 2: The critic message to critic 2 + evaluator_code_feedback_v1_user: |- --------------Factor information:--------------- {{ factor_information }} @@ -29,23 +32,19 @@ evaluator_code_feedback_v1_user: |- --------------Ground truth Python code:--------------- {{ gt_code }} {% endif %} + evolving_strategy_factor_implementation_v1_system: |- - The user is trying to implement some factors in quant investment, and you are the one to help write the python code. - - {{ data_info }} - - The user will provide you a formulation of the factor, which contains some function calls and some operators. You need to implement the function calls and operators in python. Your code is expected to align the formulation in any form which means The user needs to get the exact factor values with your code as expected. - - Your code should contain the following part: the import part, the function part, and the main part. You should write a main function name: "calculate_{function_name}" and call this function in "if __name__ == __main__" part. Don't write any try-except block in your code. The user will catch the exception message and provide the feedback to you. - - User will write your code into a python file and execute the file directly with "python {your_file_name}.py". You should calculate the factor values and save the result into a HDF5(H5) file named "result.h5" in the same directory as your python file. The result file is a HDF5(H5) file containing a pandas dataframe. The index of the dataframe is the "datetime" and "instrument", and the single column name is the factor name,and the value is the factor value. The result file should be saved in the same directory as your python file. + User is trying to implement some factors in the following scenario: + {{ scenario }} + Your code is expected to align the scenario in any form which means The user needs to get the exact factor values with your code as expected. To help you write the correct code, the user might provide multiple information that helps you write the correct code: 1. The user might provide you the correct code to similar factors. Your should learn from these code to write the correct code. 2. The user might provide you the failed former code and the corresponding feedback to the code. The feedback contains to the execution, the code and the factor value. You should analyze the feedback and try to correct the latest code. 3. The user might provide you the suggestion to the latest fail code and some similar fail to correct pairs. Each pair contains the fail code with similar error and the corresponding corrected version code. You should learn from these suggestion to write the correct code. - + Your must write your code based on your former latest attempt below which consists of your former code and code feedback, you should read the former attempt carefully and must not modify the right part of your former code. + {% if queried_former_failed_knowledge|length != 0 %} --------------Your former latest attempt:--------------- {% for former_failed_knowledge in queried_former_failed_knowledge %} @@ -56,17 +55,6 @@ evolving_strategy_factor_implementation_v1_system: |- {% endfor %} {% endif %} - A typical format of `result.h5` may be like following: - datetime instrument - 2020-01-02 SZ000001 -0.001796 - SZ000166 0.005780 - SZ000686 0.004228 - SZ000712 0.001298 - SZ000728 0.005330 - ... - 2021-12-31 SZ000750 0.000000 - SZ000776 0.002459 - Please response the code in the following json format. Here is an example structure for the JSON output: { "code": "The Python code as a string." @@ -96,31 +84,6 @@ evolving_strategy_factor_implementation_v1_user: |- {% endfor %} {% endif %} -evaluator_final_decision_v1_system: |- - User is trying to implement some factor in quant investment and has finished a version of implementation. User has finished evaluation and got some feedback from the evaluator. - The evaluator run the code and get the factor value dataframe and provide several feedback regarding user's code and code output. You should analyze the feedback and considering the factor description to give a final decision about the evaluation result. The final decision concludes whether the factor is implemented correctly and if not, detail feedback containing reason and suggestion if the final decision is False. - - The implementation final decision is considered in the following logic: - 1. If the value and the ground truth value are exactly the same under a small tolerance, the implementation is considered correct. - 2. If the value and the ground truth value have a high correlation on ic or rank ic, the implementation is considered correct. - 3. If no ground truth value is not provided, the implementation is considered correct if the code execution is successful and the code feedback is reasonable. - - Please response the critic in the json format. Here is an example structure for the JSON output, please strictly follow the format: - { - "final_decision": True, - "final_feedback": "The final feedback message", - } - -evaluator_final_decision_v1_user: |- - --------------Factor information:--------------- - {{ factor_information }} - --------------Execution feedback:--------------- - {{ execution_feedback }} - --------------Code feedback:--------------- - {{ code_feedback }} - --------------Factor value feedback:--------------- - {{ factor_value_feedback }} - evolving_strategy_factor_implementation_v2_user: |- --------------Target factor information:--------------- {{ factor_information_str }} @@ -154,7 +117,6 @@ evolving_strategy_factor_implementation_v2_user: |- {% endfor %} {% endif %} - evolving_strategy_error_summary_v2_system: |- You are doing the following task: {{factor_information_str}} @@ -183,14 +145,10 @@ evolving_strategy_error_summary_v2_user: |- select_implementable_factor_system: |- - User is trying to implement some factors in quant investment using Python code, You are an assistant who helps the user select the easiest-to-implement factors and some factors may be difficult to implement due to a lack of information or excessive complexity.. - The user will provide the number of factor you should pick and information about the factors, including their descriptions, formulas, and variable explanations. - - At the same time, user will provide with your former attempt to implement the factor and the feedback to the implementation.You need to carefully review your previous attempts. Some factors have been repeatedly tried without success. You should consider discarding these factors. - - Here is the source data that user will use to implement the factors: - {{ data_info }} - + User is trying to implement some factors in the following scenario: + {{ scenario }} + Your job is to help the user select the easiest-to-implement factors. Some factors may be difficult to implement due to a lack of information or excessive complexity. The user will provide the number of factors you should pick and information about the factors, including their descriptions, formulas, and variable explanations. + User will provide you the former attempt to implement the factor and the feedback to the implementation. You need to carefully review your previous attempts. Some factors have been repeatedly tried without success. You should consider discarding these factors. Please analyze the difficulties of the each factors and provide the reason and response the indices of selected implementable factor in the json format. Here is an example structure for the JSON output: { "Analysis": "Analyze the difficulties of the each factors and provide the reason why the factor can be implemented or not." @@ -226,4 +184,42 @@ analyze_component_prompt_v1_system: |- Please response the critic in the json format. Here is an example structure for the JSON output, please strictly follow the format: { "component_no_list": the list containing indices of components. - } \ No newline at end of file + } + +evaluator_output_format_system: |- + User is trying to implement some factors in the following scenario: + {{ scenario }} + User will provide you the format of the output. Please help to check whether the output is align with the format. + Please respond in the JSON format. Here is an example structure for the JSON output: + { + "output_format_decision": True, + "output_format_feedback": "The output format is correct." + } + + +evaluator_final_decision_v1_system: |- + User is trying to implement some factors in the following scenario: + {{ scenario }} + User has finished evaluation and got some feedback from the evaluator. + The evaluator run the code and get the factor value dataframe and provide several feedback regarding user's code and code output. You should analyze the feedback and considering the scenario and factor description to give a final decision about the evaluation result. The final decision concludes whether the factor is implemented correctly and if not, detail feedback containing reason and suggestion if the final decision is False. + + The implementation final decision is considered in the following logic: + 1. If the value and the ground truth value are exactly the same under a small tolerance, the implementation is considered correct. + 2. If the value and the ground truth value have a high correlation on ic or rank ic, the implementation is considered correct. + 3. If no ground truth value is not provided, the implementation is considered correct if the code execution is successful and the code feedback is align with the scenario and factor description. + + Please response the critic in the json format. Here is an example structure for the JSON output, please strictly follow the format: + { + "final_decision": True, + "final_feedback": "The final feedback message", + } + +evaluator_final_decision_v1_user: |- + --------------Factor information:--------------- + {{ factor_information }} + --------------Execution feedback:--------------- + {{ execution_feedback }} + --------------Code feedback:--------------- + {{ code_feedback }} + --------------Factor value feedback:--------------- + {{ factor_value_feedback }} diff --git a/rdagent/components/task_implementation/factor_implementation/utils.py b/rdagent/components/coder/factor_coder/utils.py similarity index 93% rename from rdagent/components/task_implementation/factor_implementation/utils.py rename to rdagent/components/coder/factor_coder/utils.py index 9c7aa42c..dd705b29 100644 --- a/rdagent/components/task_implementation/factor_implementation/utils.py +++ b/rdagent/components/coder/factor_coder/utils.py @@ -5,9 +5,7 @@ import pandas as pd # render it with jinja from jinja2 import Environment, StrictUndefined -from rdagent.components.task_implementation.factor_implementation.config import ( - FACTOR_IMPLEMENT_SETTINGS, -) +from rdagent.components.coder.factor_coder.config import FACTOR_IMPLEMENT_SETTINGS TPL = """ {{file_name}} diff --git a/rdagent/components/task_implementation/model_implementation/benchmark/eval.py b/rdagent/components/coder/model_coder/benchmark/eval.py similarity index 96% rename from rdagent/components/task_implementation/model_implementation/benchmark/eval.py rename to rdagent/components/coder/model_coder/benchmark/eval.py index 01519be9..a87c7475 100644 --- a/rdagent/components/task_implementation/model_implementation/benchmark/eval.py +++ b/rdagent/components/coder/model_coder/benchmark/eval.py @@ -1,9 +1,7 @@ # TODO: inherent from the benchmark base class import torch -from rdagent.components.task_implementation.model_implementation.model import ( - ModelImplementation, -) +from rdagent.components.coder.model_coder.model import ModelImplementation def get_data_conf(init_val): diff --git a/rdagent/components/task_implementation/model_implementation/benchmark/gt_code/A-DGN.py b/rdagent/components/coder/model_coder/benchmark/gt_code/A-DGN.py similarity index 100% rename from rdagent/components/task_implementation/model_implementation/benchmark/gt_code/A-DGN.py rename to rdagent/components/coder/model_coder/benchmark/gt_code/A-DGN.py diff --git a/rdagent/components/task_implementation/model_implementation/benchmark/gt_code/dirgnn.py b/rdagent/components/coder/model_coder/benchmark/gt_code/dirgnn.py similarity index 100% rename from rdagent/components/task_implementation/model_implementation/benchmark/gt_code/dirgnn.py rename to rdagent/components/coder/model_coder/benchmark/gt_code/dirgnn.py diff --git a/rdagent/components/task_implementation/model_implementation/benchmark/gt_code/gpsconv.py b/rdagent/components/coder/model_coder/benchmark/gt_code/gpsconv.py similarity index 100% rename from rdagent/components/task_implementation/model_implementation/benchmark/gt_code/gpsconv.py rename to rdagent/components/coder/model_coder/benchmark/gt_code/gpsconv.py diff --git a/rdagent/components/task_implementation/model_implementation/benchmark/gt_code/linkx.py b/rdagent/components/coder/model_coder/benchmark/gt_code/linkx.py similarity index 99% rename from rdagent/components/task_implementation/model_implementation/benchmark/gt_code/linkx.py rename to rdagent/components/coder/model_coder/benchmark/gt_code/linkx.py index fa9a53c3..fc71e86d 100644 --- a/rdagent/components/task_implementation/model_implementation/benchmark/gt_code/linkx.py +++ b/rdagent/components/coder/model_coder/benchmark/gt_code/linkx.py @@ -166,6 +166,7 @@ class LINKX(torch.nn.Module): f"out_channels={self.out_channels})" ) + model_cls = LINKX if __name__ == "__main__": diff --git a/rdagent/components/task_implementation/model_implementation/benchmark/gt_code/pmlp.py b/rdagent/components/coder/model_coder/benchmark/gt_code/pmlp.py similarity index 95% rename from rdagent/components/task_implementation/model_implementation/benchmark/gt_code/pmlp.py rename to rdagent/components/coder/model_coder/benchmark/gt_code/pmlp.py index de8face9..3dadd76d 100644 --- a/rdagent/components/task_implementation/model_implementation/benchmark/gt_code/pmlp.py +++ b/rdagent/components/coder/model_coder/benchmark/gt_code/pmlp.py @@ -106,7 +106,12 @@ if __name__ == "__main__": edge_index = torch.load("edge_index.pt") # Model instantiation and forward pass - model = PMLP(in_channels=node_features.size(-1), hidden_channels=node_features.size(-1), out_channels=node_features.size(-1), num_layers=1) + model = PMLP( + in_channels=node_features.size(-1), + hidden_channels=node_features.size(-1), + out_channels=node_features.size(-1), + num_layers=1, + ) output = model(node_features, edge_index) # Save output to a file diff --git a/rdagent/components/task_implementation/model_implementation/benchmark/gt_code/visnet.py b/rdagent/components/coder/model_coder/benchmark/gt_code/visnet.py similarity index 100% rename from rdagent/components/task_implementation/model_implementation/benchmark/gt_code/visnet.py rename to rdagent/components/coder/model_coder/benchmark/gt_code/visnet.py diff --git a/rdagent/components/task_implementation/model_implementation/conf.py b/rdagent/components/coder/model_coder/conf.py similarity index 100% rename from rdagent/components/task_implementation/model_implementation/conf.py rename to rdagent/components/coder/model_coder/conf.py diff --git a/rdagent/components/task_implementation/model_implementation/evaluator.py b/rdagent/components/coder/model_coder/evaluator.py similarity index 100% rename from rdagent/components/task_implementation/model_implementation/evaluator.py rename to rdagent/components/coder/model_coder/evaluator.py diff --git a/rdagent/components/task_implementation/model_implementation/gt_code.py b/rdagent/components/coder/model_coder/gt_code.py similarity index 100% rename from rdagent/components/task_implementation/model_implementation/gt_code.py rename to rdagent/components/coder/model_coder/gt_code.py diff --git a/rdagent/components/task_implementation/model_implementation/main.py b/rdagent/components/coder/model_coder/main.py similarity index 100% rename from rdagent/components/task_implementation/model_implementation/main.py rename to rdagent/components/coder/model_coder/main.py diff --git a/rdagent/components/task_implementation/model_implementation/model.py b/rdagent/components/coder/model_coder/model.py similarity index 98% rename from rdagent/components/task_implementation/model_implementation/model.py rename to rdagent/components/coder/model_coder/model.py index d7ed5e8b..9898a3de 100644 --- a/rdagent/components/task_implementation/model_implementation/model.py +++ b/rdagent/components/coder/model_coder/model.py @@ -5,10 +5,8 @@ from typing import Dict, Optional, Sequence import torch +from rdagent.components.coder.model_coder.conf import MODEL_IMPL_SETTINGS from rdagent.components.loader.task_loader import ModelTaskLoader -from rdagent.components.task_implementation.model_implementation.conf import ( - MODEL_IMPL_SETTINGS, -) from rdagent.core.exception import CodeFormatException from rdagent.core.experiment import Experiment, FBImplementation, ImpLoader, Task from rdagent.utils import get_module_by_module_path @@ -128,7 +126,8 @@ We'll import the model in the implementation in file `model.py` after setting th """ -class ModelExperiment(Experiment[ModelTask, ModelImplementation]): ... +class ModelExperiment(Experiment[ModelTask, ModelImplementation]): + ... class ModelTaskLoaderJson(ModelTaskLoader): diff --git a/rdagent/components/task_implementation/model_implementation/one_shot/__init__.py b/rdagent/components/coder/model_coder/one_shot/__init__.py similarity index 92% rename from rdagent/components/task_implementation/model_implementation/one_shot/__init__.py rename to rdagent/components/coder/model_coder/one_shot/__init__.py index 4851bb21..c54fff2d 100644 --- a/rdagent/components/task_implementation/model_implementation/one_shot/__init__.py +++ b/rdagent/components/coder/model_coder/one_shot/__init__.py @@ -1,13 +1,11 @@ import re from pathlib import Path -from typing import Sequence from jinja2 import Environment, StrictUndefined -from rdagent.components.task_implementation.model_implementation.model import ( +from rdagent.components.coder.model_coder.model import ( ModelExperiment, ModelImplementation, - ModelTask, ) from rdagent.core.prompts import Prompts from rdagent.core.task_generator import TaskGenerator diff --git a/rdagent/components/task_implementation/model_implementation/one_shot/prompt.yaml b/rdagent/components/coder/model_coder/one_shot/prompt.yaml similarity index 100% rename from rdagent/components/task_implementation/model_implementation/one_shot/prompt.yaml rename to rdagent/components/coder/model_coder/one_shot/prompt.yaml diff --git a/rdagent/components/task_implementation/model_implementation/prompts.yaml b/rdagent/components/coder/model_coder/prompts.yaml similarity index 100% rename from rdagent/components/task_implementation/model_implementation/prompts.yaml rename to rdagent/components/coder/model_coder/prompts.yaml diff --git a/rdagent/components/task_implementation/model_implementation/task_loader.py b/rdagent/components/coder/model_coder/task_loader.py similarity index 92% rename from rdagent/components/task_implementation/model_implementation/task_loader.py rename to rdagent/components/coder/model_coder/task_loader.py index f11a1f33..d2018f2d 100644 --- a/rdagent/components/task_implementation/model_implementation/task_loader.py +++ b/rdagent/components/coder/model_coder/task_loader.py @@ -4,15 +4,11 @@ import json import re from pathlib import Path +from rdagent.components.coder.model_coder.model import ModelExperiment, ModelTask from rdagent.components.document_reader.document_reader import ( load_and_process_pdfs_by_langchain, ) from rdagent.components.loader.task_loader import ModelTaskLoader -from rdagent.components.task_implementation.model_implementation.model import ( - ModelExperiment, - ModelImplementationTaskLoaderFromDict, - ModelTask, -) from rdagent.core.log import RDAgentLog from rdagent.core.prompts import Prompts from rdagent.oai.llm_utils import APIBackend @@ -99,7 +95,7 @@ def extract_model_from_docs(docs_dict): return model_dict -class ModelImplementationExperimentLoaderFromDict(ModelTaskLoader): +class ModelExperimentLoaderFromDict(ModelTaskLoader): def load(self, model_dict: dict) -> list: """Load data from a dict.""" task_l = [] @@ -115,7 +111,7 @@ class ModelImplementationExperimentLoaderFromDict(ModelTaskLoader): return ModelExperiment(sub_tasks=task_l) -class ModelImplementationExperimentLoaderFromPDFfiles(ModelTaskLoader): +class ModelExperimentLoaderFromPDFfiles(ModelTaskLoader): def load(self, file_or_folder_path: Path) -> dict: docs_dict = load_and_process_pdfs_by_langchain(Path(file_or_folder_path)) # dict{file_path:content} model_dict = extract_model_from_docs( @@ -124,7 +120,7 @@ class ModelImplementationExperimentLoaderFromPDFfiles(ModelTaskLoader): model_dict = merge_file_to_model_dict_to_model_dict( model_dict ) # dict {model_name: dict{description, formulation, variables}} - return ModelImplementationExperimentLoaderFromDict().load(model_dict) + return ModelExperimentLoaderFromDict().load(model_dict) def main(path="../test_doc"): diff --git a/rdagent/components/loader/experiment_loader.py b/rdagent/components/loader/experiment_loader.py index ee929303..00366096 100644 --- a/rdagent/components/loader/experiment_loader.py +++ b/rdagent/components/loader/experiment_loader.py @@ -1,6 +1,4 @@ -from rdagent.components.task_implementation.factor_implementation.factor import ( - FactorExperiment, -) +from rdagent.components.coder.factor_coder.factor import FactorExperiment from rdagent.core.experiment import Loader diff --git a/rdagent/components/loader/task_loader.py b/rdagent/components/loader/task_loader.py index c651a66c..647cc669 100644 --- a/rdagent/components/loader/task_loader.py +++ b/rdagent/components/loader/task_loader.py @@ -1,7 +1,5 @@ -from rdagent.components.task_implementation.factor_implementation.factor import ( - FactorTask, -) -from rdagent.components.task_implementation.model_implementation.model import ModelTask +from rdagent.components.coder.factor_coder.factor import FactorTask +from rdagent.components.coder.model_coder.model import ModelTask from rdagent.core.experiment import Loader diff --git a/rdagent/components/idea_proposal/factor_proposal.py b/rdagent/components/proposal/factor_proposal.py similarity index 95% rename from rdagent/components/idea_proposal/factor_proposal.py rename to rdagent/components/proposal/factor_proposal.py index f9342f59..b01c511f 100644 --- a/rdagent/components/idea_proposal/factor_proposal.py +++ b/rdagent/components/proposal/factor_proposal.py @@ -4,9 +4,7 @@ from typing import Tuple from jinja2 import Environment, StrictUndefined -from rdagent.components.task_implementation.factor_implementation.factor import ( - FactorExperiment, -) +from rdagent.components.coder.factor_coder.factor import FactorExperiment from rdagent.core.prompts import Prompts from rdagent.core.proposal import ( Hypothesis, @@ -30,10 +28,12 @@ class FactorHypothesisGen(HypothesisGen): # The following methods are scenario related so they should be implemented in the subclass @abstractmethod - def prepare_context(self, trace: Trace) -> Tuple[dict, bool]: ... + def prepare_context(self, trace: Trace) -> Tuple[dict, bool]: + ... @abstractmethod - def convert_response(self, response: str) -> FactorHypothesis: ... + def convert_response(self, response: str) -> FactorHypothesis: + ... def gen(self, trace: Trace) -> FactorHypothesis: context_dict, json_flag = self.prepare_context(trace) @@ -67,10 +67,12 @@ class FactorHypothesis2Experiment(Hypothesis2Experiment[FactorExperiment]): super().__init__() @abstractmethod - def prepare_context(self, hs: HypothesisSet) -> Tuple[dict, bool]: ... + def prepare_context(self, hs: HypothesisSet) -> Tuple[dict, bool]: + ... @abstractmethod - def convert_response(self, response: str) -> FactorExperiment: ... + def convert_response(self, response: str) -> FactorExperiment: + ... def convert(self, hs: HypothesisSet) -> FactorExperiment: context, json_flag = self.prepare_context(hs) diff --git a/rdagent/components/idea_proposal/model_proposal.py b/rdagent/components/proposal/model_proposal.py similarity index 100% rename from rdagent/components/idea_proposal/model_proposal.py rename to rdagent/components/proposal/model_proposal.py diff --git a/rdagent/components/idea_proposal/prompts.yaml b/rdagent/components/proposal/prompts.yaml similarity index 100% rename from rdagent/components/idea_proposal/prompts.yaml rename to rdagent/components/proposal/prompts.yaml diff --git a/rdagent/components/task_implementation/factor_implementation/evolving/evaluators.py b/rdagent/components/task_implementation/factor_implementation/evolving/evaluators.py deleted file mode 100644 index 19289031..00000000 --- a/rdagent/components/task_implementation/factor_implementation/evolving/evaluators.py +++ /dev/null @@ -1,750 +0,0 @@ -import json -import re -from abc import abstractmethod -from pathlib import Path -from typing import List, Tuple - -import pandas as pd -from jinja2 import Environment, StrictUndefined - -from rdagent.components.task_implementation.factor_implementation.config import ( - FACTOR_IMPLEMENT_SETTINGS, -) -from rdagent.components.task_implementation.factor_implementation.evolving.evolvable_subjects import ( - FactorEvolvingItem, -) -from rdagent.components.task_implementation.factor_implementation.evolving.evolving_strategy import ( - FactorTask, -) -from rdagent.core.conf import RD_AGENT_SETTINGS -from rdagent.core.evaluation import Evaluator -from rdagent.core.evolving_framework import Feedback, QueriedKnowledge -from rdagent.core.experiment import Implementation -from rdagent.core.log import RDAgentLog -from rdagent.core.prompts import Prompts -from rdagent.core.utils import multiprocessing_wrapper -from rdagent.oai.llm_utils import APIBackend - -evaluate_prompts = Prompts(file_path=Path(__file__).parent.parent / "prompts.yaml") - - -class FactorImplementationEvaluator(Evaluator): - # TODO: - # I think we should have unified interface for all evaluates, for examples. - # So we should adjust the interface of other factors - @abstractmethod - def evaluate( - self, - gt: Implementation, - gen: Implementation, - ) -> Tuple[str, object]: - """You can get the dataframe by - - .. code-block:: python - - _, gt_df = gt.execute() - _, gen_df = gen.execute() - - Returns - ------- - Tuple[str, object] - - str: the text-based description of the evaluation result - - object: a comparable metric (bool, integer, float ...) - - """ - raise NotImplementedError("Please implement the `evaluator` method") - - def _get_df(self, gt: Implementation, gen: Implementation): - _, gt_df = gt.execute() - _, gen_df = gen.execute() - if isinstance(gen_df, pd.Series): - gen_df = gen_df.to_frame("source_factor") - if isinstance(gt_df, pd.Series): - gt_df = gt_df.to_frame("gt_factor") - return gt_df, gen_df - - -class FactorImplementationCodeEvaluator(Evaluator): - def evaluate( - self, - target_task: FactorTask, - implementation: Implementation, - execution_feedback: str, - factor_value_feedback: str = "", - gt_implementation: Implementation = None, - **kwargs, - ): - factor_information = target_task.get_factor_information() - code = implementation.code - - system_prompt = evaluate_prompts["evaluator_code_feedback_v1_system"] - - execution_feedback_to_render = execution_feedback - user_prompt = ( - Environment(undefined=StrictUndefined) - .from_string( - evaluate_prompts["evaluator_code_feedback_v1_user"], - ) - .render( - factor_information=factor_information, - code=code, - execution_feedback=execution_feedback_to_render, - factor_value_feedback=factor_value_feedback, - gt_code=gt_implementation.code if gt_implementation else None, - ) - ) - while ( - APIBackend().build_messages_and_calculate_token( - user_prompt=user_prompt, - system_prompt=system_prompt, - former_messages=[], - ) - > RD_AGENT_SETTINGS.chat_token_limit - ): - execution_feedback_to_render = execution_feedback_to_render[len(execution_feedback_to_render) // 2 :] - user_prompt = ( - Environment(undefined=StrictUndefined) - .from_string( - evaluate_prompts["evaluator_code_feedback_v1_user"], - ) - .render( - factor_information=factor_information, - code=code, - execution_feedback=execution_feedback_to_render, - factor_value_feedback=factor_value_feedback, - gt_code=gt_implementation.code if gt_implementation else None, - ) - ) - critic_response = APIBackend().build_messages_and_create_chat_completion( - user_prompt=user_prompt, - system_prompt=system_prompt, - json_mode=False, - ) - - return critic_response - - -class FactorImplementationSingleColumnEvaluator(FactorImplementationEvaluator): - def evaluate( - self, - gt: Implementation, - gen: Implementation, - ) -> Tuple[str, object]: - gt_df, gen_df = self._get_df(gt, gen) - - if len(gen_df.columns) == 1 and len(gt_df.columns) == 1: - return "Both dataframes have only one column.", True - elif len(gen_df.columns) != 1: - gen_df = gen_df.iloc(axis=1)[ - [ - 0, - ] - ] - return ( - "The source dataframe has more than one column. Please check the implementation. We only evaluate the first column.", - False, - ) - return "", False - - def __str__(self) -> str: - return self.__class__.__name__ - - -class FactorImplementationIndexFormatEvaluator(FactorImplementationEvaluator): - def evaluate( - self, - gt: Implementation, - gen: Implementation, - ) -> Tuple[str, object]: - gt_df, gen_df = self._get_df(gt, gen) - idx_name_right = gen_df.index.names == ("datetime", "instrument") - if idx_name_right: - return ( - 'The index of the dataframe is ("datetime", "instrument") and align with the predefined format.', - True, - ) - else: - return ( - 'The index of the dataframe is not ("datetime", "instrument"). Please check the implementation.', - False, - ) - - def __str__(self) -> str: - return self.__class__.__name__ - - -class FactorImplementationRowCountEvaluator(FactorImplementationEvaluator): - def evaluate( - self, - gt: Implementation, - gen: Implementation, - ) -> Tuple[str, object]: - gt_df, gen_df = self._get_df(gt, gen) - - if gen_df.shape[0] == gt_df.shape[0]: - return "Both dataframes have the same rows count.", True - else: - return ( - f"The source dataframe and the ground truth dataframe have different rows count. The source dataframe has {gen_df.shape[0]} rows, while the ground truth dataframe has {gt_df.shape[0]} rows. Please check the implementation.", - False, - ) - - def __str__(self) -> str: - return self.__class__.__name__ - - -class FactorImplementationIndexEvaluator(FactorImplementationEvaluator): - def evaluate( - self, - gt: Implementation, - gen: Implementation, - ) -> Tuple[str, object]: - gt_df, gen_df = self._get_df(gt, gen) - - if gen_df.index.equals(gt_df.index): - return "Both dataframes have the same index.", True - else: - return ( - "The source dataframe and the ground truth dataframe have different index. Please check the implementation.", - False, - ) - - def __str__(self) -> str: - return self.__class__.__name__ - - -class FactorImplementationMissingValuesEvaluator(FactorImplementationEvaluator): - def evaluate( - self, - gt: Implementation, - gen: Implementation, - ) -> Tuple[str, object]: - gt_df, gen_df = self._get_df(gt, gen) - - if gen_df.isna().sum().sum() == gt_df.isna().sum().sum(): - return "Both dataframes have the same missing values.", True - else: - return ( - f"The dataframes do not have the same missing values. The source dataframe has {gen_df.isna().sum().sum()} missing values, while the ground truth dataframe has {gt_df.isna().sum().sum()} missing values. Please check the implementation.", - False, - ) - - def __str__(self) -> str: - return self.__class__.__name__ - - -class FactorImplementationValuesEvaluator(FactorImplementationEvaluator): - def evaluate( - self, - gt: Implementation, - gen: Implementation, - ) -> Tuple[str, object]: - gt_df, gen_df = self._get_df(gt, gen) - - try: - close_values = gen_df.sub(gt_df).abs().lt(1e-6) - result_int = close_values.astype(int) - pos_num = result_int.sum().sum() - acc_rate = pos_num / close_values.size - except: - close_values = gen_df - if close_values.all().iloc[0]: - return ( - "All values in the dataframes are equal within the tolerance of 1e-6.", - acc_rate, - ) - else: - return ( - "Some values differ by more than the tolerance of 1e-6. Check for rounding errors or differences in the calculation methods.", - acc_rate, - ) - - def __str__(self) -> str: - return self.__class__.__name__ - - -class FactorImplementationCorrelationEvaluator(FactorImplementationEvaluator): - def __init__(self, hard_check: bool) -> None: - self.hard_check = hard_check - - def evaluate( - self, - gt: Implementation, - gen: Implementation, - ) -> Tuple[str, object]: - gt_df, gen_df = self._get_df(gt, gen) - - concat_df = pd.concat([gen_df, gt_df], axis=1) - concat_df.columns = ["source", "gt"] - ic = concat_df.groupby("datetime").apply(lambda df: df["source"].corr(df["gt"])).dropna().mean() - ric = ( - concat_df.groupby("datetime") - .apply(lambda df: df["source"].corr(df["gt"], method="spearman")) - .dropna() - .mean() - ) - - if self.hard_check: - if ic > 0.99 and ric > 0.99: - return ( - f"The dataframes are highly correlated. The ic is {ic:.6f} and the rankic is {ric:.6f}.", - True, - ) - else: - return ( - f"The dataframes are not sufficiently high correlated. The ic is {ic:.6f} and the rankic is {ric:.6f}. Investigate the factors that might be causing the discrepancies and ensure that the logic of the factor calculation is consistent.", - False, - ) - else: - return f"The ic is ({ic:.6f}) and the rankic is ({ric:.6f}).", ic - - def __str__(self) -> str: - return self.__class__.__name__ - - -class FactorImplementationValEvaluator(FactorImplementationEvaluator): - def evaluate(self, gt: Implementation, gen: Implementation): - _, gt_df = gt.execute() - _, gen_df = gen.execute() - # FIXME: refactor the two classes - fiv = FactorImplementationValueEvaluator() - return fiv.evaluate(source_df=gen_df, gt_df=gt_df) - - def __str__(self) -> str: - return self.__class__.__name__ - - -class FactorImplementationValueEvaluator(Evaluator): - # TODO: let's discuss the about the interface of the evaluator - def evaluate( - self, - source_df: pd.DataFrame, - gt_df: pd.DataFrame, - **kwargs, - ) -> Tuple: - conclusions = [] - - if isinstance(source_df, pd.Series): - source_df = source_df.to_frame("source_factor") - conclusions.append( - "The source dataframe is a series, better convert it to a dataframe.", - ) - if gt_df is not None and isinstance(gt_df, pd.Series): - gt_df = gt_df.to_frame("gt_factor") - conclusions.append( - "The ground truth dataframe is a series, convert it to a dataframe.", - ) - - # Check if both dataframe has only one columns - if len(source_df.columns) == 1: - conclusions.append("The source dataframe has only one column which is correct.") - else: - conclusions.append( - "The source dataframe has more than one column. Please check the implementation. We only evaluate the first column.", - ) - source_df = source_df.iloc(axis=1)[ - [ - 0, - ] - ] - - if list(source_df.index.names) != ["datetime", "instrument"]: - conclusions.append( - rf"The index of the dataframe is not (\"datetime\", \"instrument\"), instead is {source_df.index.names}. Please check the implementation.", - ) - else: - conclusions.append( - 'The index of the dataframe is ("datetime", "instrument") and align with the predefined format.', - ) - - # Check if both dataframe have the same rows count - if gt_df is not None: - if source_df.shape[0] == gt_df.shape[0]: - conclusions.append("Both dataframes have the same rows count.") - same_row_count_result = True - else: - conclusions.append( - f"The source dataframe and the ground truth dataframe have different rows count. The source dataframe has {source_df.shape[0]} rows, while the ground truth dataframe has {gt_df.shape[0]} rows. Please check the implementation.", - ) - same_row_count_result = False - - # Check whether both dataframe has the same index - if source_df.index.equals(gt_df.index): - conclusions.append("Both dataframes have the same index.") - same_index_result = True - else: - conclusions.append( - "The source dataframe and the ground truth dataframe have different index. Please check the implementation.", - ) - same_index_result = False - - # Check for the same missing values (NaN) - if source_df.isna().sum().sum() == gt_df.isna().sum().sum(): - conclusions.append("Both dataframes have the same missing values.") - same_missing_values_result = True - else: - conclusions.append( - f"The dataframes do not have the same missing values. The source dataframe has {source_df.isna().sum().sum()} missing values, while the ground truth dataframe has {gt_df.isna().sum().sum()} missing values. Please check the implementation.", - ) - same_missing_values_result = False - - # Check if the values are the same within a small tolerance - if not same_index_result: - conclusions.append( - "The source dataframe and the ground truth dataframe have different index. Give up comparing the values and correlation because it's useless", - ) - same_values_result = False - high_correlation_result = False - else: - close_values = source_df.sub(gt_df).abs().lt(1e-6) - if close_values.all().iloc[0]: - conclusions.append( - "All values in the dataframes are equal within the tolerance of 1e-6.", - ) - same_values_result = True - else: - conclusions.append( - "Some values differ by more than the tolerance of 1e-6. Check for rounding errors or differences in the calculation methods.", - ) - same_values_result = False - - # Check the ic and rankic between the two dataframes - concat_df = pd.concat([source_df, gt_df], axis=1) - concat_df.columns = ["source", "gt"] - try: - ic = concat_df.groupby("datetime").apply(lambda df: df["source"].corr(df["gt"])).dropna().mean() - ric = ( - concat_df.groupby("datetime") - .apply(lambda df: df["source"].corr(df["gt"], method="spearman")) - .dropna() - .mean() - ) - - if ic > 0.99 and ric > 0.99: - conclusions.append( - f"The dataframes are highly correlated. The ic is {ic:.6f} and the rankic is {ric:.6f}.", - ) - high_correlation_result = True - else: - conclusions.append( - f"The dataframes are not sufficiently high correlated. The ic is {ic:.6f} and the rankic is {ric:.6f}. Investigate the factors that might be causing the discrepancies and ensure that the logic of the factor calculation is consistent.", - ) - high_correlation_result = False - - # Check for shifted alignments only in the "datetime" index - max_shift_days = 2 - for shift in range(-max_shift_days, max_shift_days + 1): - if shift == 0: - continue # Skip the case where there is no shift - - shifted_source_df = source_df.groupby(level="instrument").shift(shift) - concat_df = pd.concat([shifted_source_df, gt_df], axis=1) - concat_df.columns = ["source", "gt"] - shifted_ric = ( - concat_df.groupby("datetime") - .apply(lambda df: df["source"].corr(df["gt"], method="spearman")) - .dropna() - .mean() - ) - if shifted_ric > 0.99: - conclusions.append( - f"The dataframes are highly correlated with a shift of {max_shift_days} days in the 'date' index. Shifted rankic: {shifted_ric:.6f}.", - ) - break - else: - conclusions.append( - f"No sufficient correlation found when shifting up to {max_shift_days} days in the 'date' index. Investigate the factors that might be causing discrepancies.", - ) - - except Exception as e: - RDAgentLog().warning(f"Error occurred when calculating the correlation: {str(e)}") - conclusions.append( - f"Some error occurred when calculating the correlation. Investigate the factors that might be causing the discrepancies and ensure that the logic of the factor calculation is consistent. Error: {e}", - ) - high_correlation_result = False - - # Combine all conclusions into a single string - conclusion_str = "\n".join(conclusions) - - final_result = (same_values_result or high_correlation_result) if gt_df is not None else False - return conclusion_str, final_result - - -# TODO: -def shorten_prompt(tpl: str, render_kwargs: dict, shorten_key: str, max_trail: int = 10) -> str: - """When the prompt is too long. We have to shorten it. - But we should not truncate the prompt directly, so we should find the key we want to shorten and then shorten it. - """ - # TODO: this should replace most of code in - # - FactorImplementationFinalDecisionEvaluator.evaluate - # - FactorImplementationCodeEvaluator.evaluate - - -class FactorImplementationFinalDecisionEvaluator(Evaluator): - def evaluate( - self, - target_task: FactorTask, - execution_feedback: str, - value_feedback: str, - code_feedback: str, - **kwargs, - ) -> Tuple: - system_prompt = Prompts(file_path=Path(__file__).parent.parent / "prompts.yaml")[ - "evaluator_final_decision_v1_system" - ] - execution_feedback_to_render = execution_feedback - user_prompt = ( - Environment(undefined=StrictUndefined) - .from_string( - evaluate_prompts["evaluator_final_decision_v1_user"], - ) - .render( - factor_information=target_task.get_factor_information(), - execution_feedback=execution_feedback_to_render, - code_feedback=code_feedback, - factor_value_feedback=( - value_feedback - if value_feedback is not None - else "No Ground Truth Value provided, so no evaluation on value is performed." - ), - ) - ) - while ( - APIBackend().build_messages_and_calculate_token( - user_prompt=user_prompt, - system_prompt=system_prompt, - former_messages=[], - ) - > RD_AGENT_SETTINGS.chat_token_limit - ): - execution_feedback_to_render = execution_feedback_to_render[len(execution_feedback_to_render) // 2 :] - user_prompt = ( - Environment(undefined=StrictUndefined) - .from_string( - evaluate_prompts["evaluator_final_decision_v1_user"], - ) - .render( - factor_information=target_task.get_factor_information(), - execution_feedback=execution_feedback_to_render, - code_feedback=code_feedback, - factor_value_feedback=( - value_feedback - if value_feedback is not None - else "No Ground Truth Value provided, so no evaluation on value is performed." - ), - ) - ) - - final_evaluation_dict = json.loads( - APIBackend().build_messages_and_create_chat_completion( - user_prompt=user_prompt, - system_prompt=system_prompt, - json_mode=True, - ), - ) - return ( - final_evaluation_dict["final_decision"], - final_evaluation_dict["final_feedback"], - ) - - -class FactorImplementationSingleFeedback: - """This class is a feedback to single implementation which is generated from an evaluator.""" - - def __init__( - self, - execution_feedback: str = None, - value_generated_flag: bool = False, - code_feedback: str = None, - factor_value_feedback: str = None, - final_decision: bool = None, - final_feedback: str = None, - final_decision_based_on_gt: bool = None, - ) -> None: - self.execution_feedback = execution_feedback - self.value_generated_flag = value_generated_flag - self.code_feedback = code_feedback - self.factor_value_feedback = factor_value_feedback - self.final_decision = final_decision - self.final_feedback = final_feedback - self.final_decision_based_on_gt = final_decision_based_on_gt - - def __str__(self) -> str: - return f"""------------------Factor Execution Feedback------------------ -{self.execution_feedback} -------------------Factor Code Feedback------------------ -{self.code_feedback} -------------------Factor Value Feedback------------------ -{self.factor_value_feedback} -------------------Factor Final Feedback------------------ -{self.final_feedback} -------------------Factor Final Decision------------------ -This implementation is {'SUCCESS' if self.final_decision else 'FAIL'}. -""" - - -class FactorImplementationsMultiFeedback( - Feedback, - List[FactorImplementationSingleFeedback], -): - """Feedback contains a list, each element is the corresponding feedback for each factor implementation.""" - - -class FactorImplementationEvaluatorV1(FactorImplementationEvaluator): - """This class is the v1 version of evaluator for a single factor implementation. - It calls several evaluators in share modules to evaluate the factor implementation. - """ - - def __init__(self) -> None: - self.code_evaluator = FactorImplementationCodeEvaluator() - self.value_evaluator = FactorImplementationValueEvaluator() - self.final_decision_evaluator = FactorImplementationFinalDecisionEvaluator() - - def evaluate( - self, - target_task: FactorTask, - implementation: Implementation, - gt_implementation: Implementation = None, - queried_knowledge: QueriedKnowledge = None, - **kwargs, - ) -> FactorImplementationSingleFeedback: - if implementation is None: - return None - - target_task_information = target_task.get_factor_information() - if ( - queried_knowledge is not None - and target_task_information in queried_knowledge.success_task_to_knowledge_dict - ): - return queried_knowledge.success_task_to_knowledge_dict[target_task_information].feedback - elif queried_knowledge is not None and target_task_information in queried_knowledge.failed_task_info_set: - return FactorImplementationSingleFeedback( - execution_feedback="This task has failed too many times, skip implementation.", - value_generated_flag=False, - code_feedback="This task has failed too many times, skip code evaluation.", - factor_value_feedback="This task has failed too many times, skip value evaluation.", - final_decision=False, - final_feedback="This task has failed too many times, skip final decision evaluation.", - final_decision_based_on_gt=False, - ) - else: - factor_feedback = FactorImplementationSingleFeedback() - ( - factor_feedback.execution_feedback, - source_df, - ) = implementation.execute() - - # Remove the long list of numbers in the feedback - pattern = r"(?<=\D)(,\s+-?\d+\.\d+){50,}(?=\D)" - factor_feedback.execution_feedback = re.sub(pattern, ", ", factor_feedback.execution_feedback) - execution_feedback_lines = [ - line for line in factor_feedback.execution_feedback.split("\n") if "warning" not in line.lower() - ] - factor_feedback.execution_feedback = "\n".join(execution_feedback_lines) - - if source_df is None: - factor_feedback.factor_value_feedback = "No factor value generated, skip value evaluation." - factor_feedback.value_generated_flag = False - value_decision = None - else: - factor_feedback.value_generated_flag = True - if gt_implementation is not None: - _, gt_df = gt_implementation.execute(store_result=True) - else: - gt_df = None - try: - source_df = source_df.sort_index() - if gt_df is not None: - gt_df = gt_df.sort_index() - ( - factor_feedback.factor_value_feedback, - value_decision, - ) = self.value_evaluator.evaluate(source_df=source_df, gt_df=gt_df) - except Exception as e: - RDAgentLog().warning("Value evaluation failed with exception: %s", e) - factor_feedback.factor_value_feedback = "Value evaluation failed." - value_decision = False - - factor_feedback.final_decision_based_on_gt = gt_implementation is not None - - if value_decision is not None and value_decision is True: - # To avoid confusion, when value_decision is True, we do not need code feedback - factor_feedback.code_feedback = "Final decision is True and there are no code critics." - factor_feedback.final_decision = value_decision - factor_feedback.final_feedback = "Value evaluation passed, skip final decision evaluation." - else: - factor_feedback.code_feedback = self.code_evaluator.evaluate( - target_task=target_task, - implementation=implementation, - execution_feedback=factor_feedback.execution_feedback, - value_feedback=factor_feedback.factor_value_feedback, - gt_implementation=gt_implementation, - ) - ( - factor_feedback.final_decision, - factor_feedback.final_feedback, - ) = self.final_decision_evaluator.evaluate( - target_task=target_task, - execution_feedback=factor_feedback.execution_feedback, - value_feedback=factor_feedback.factor_value_feedback, - code_feedback=factor_feedback.code_feedback, - ) - return factor_feedback - - -class FactorImplementationsMultiEvaluator(Evaluator): - def __init__(self, single_evaluator=FactorImplementationEvaluatorV1()) -> None: - super().__init__() - self.single_factor_implementation_evaluator = single_evaluator - - def evaluate( - self, - evo: FactorEvolvingItem, - queried_knowledge: QueriedKnowledge = None, - **kwargs, - ) -> FactorImplementationsMultiFeedback: - multi_implementation_feedback = FactorImplementationsMultiFeedback() - - # for index in range(len(evo.sub_tasks)): - # corresponding_implementation = evo.sub_implementations[index] - # corresponding_gt_implementation = ( - # evo.sub_gt_implementations[index] - # if evo.sub_gt_implementations is not None - # else None - # ) - - # multi_implementation_feedback.append( - # self.single_factor_implementation_evaluator.evaluate( - # target_task=evo.sub_tasks[index], - # implementation=corresponding_implementation, - # gt_implementation=corresponding_gt_implementation, - # queried_knowledge=queried_knowledge, - # ) - # ) - - calls = [] - for index in range(len(evo.sub_tasks)): - corresponding_implementation = evo.sub_implementations[index] - corresponding_gt_implementation = ( - evo.sub_gt_implementations[index] if evo.sub_gt_implementations is not None else None - ) - calls.append( - ( - self.single_factor_implementation_evaluator.evaluate, - ( - evo.sub_tasks[index], - corresponding_implementation, - corresponding_gt_implementation, - queried_knowledge, - ), - ), - ) - multi_implementation_feedback = multiprocessing_wrapper(calls, n=FACTOR_IMPLEMENT_SETTINGS.evo_multi_proc_n) - - final_decision = [ - None if single_feedback is None else single_feedback.final_decision - for single_feedback in multi_implementation_feedback - ] - RDAgentLog().info(f"Final decisions: {final_decision} True count: {final_decision.count(True)}") - - return multi_implementation_feedback diff --git a/rdagent/core/evaluation.py b/rdagent/core/evaluation.py index 6b90165f..f446fa67 100644 --- a/rdagent/core/evaluation.py +++ b/rdagent/core/evaluation.py @@ -1,9 +1,20 @@ from abc import ABC, abstractmethod -from rdagent.core.experiment import Task, Implementation +from rdagent.core.experiment import Implementation, Task +from rdagent.core.scenario import Scenario + + +class Feedback: + pass class Evaluator(ABC): + def __init__( + self, + scen: Scenario, + ) -> None: + self.scen = scen + @abstractmethod def evaluate( self, diff --git a/rdagent/core/evolving_framework.py b/rdagent/core/evolving_framework.py index 645e2fad..fd99a043 100644 --- a/rdagent/core/evolving_framework.py +++ b/rdagent/core/evolving_framework.py @@ -5,9 +5,8 @@ from abc import ABC, abstractmethod from dataclasses import dataclass from typing import Any - -class Feedback: - pass +from rdagent.core.evaluation import Feedback +from rdagent.core.scenario import Scenario class Knowledge: @@ -54,6 +53,9 @@ class EvoStep: class EvolvingStrategy(ABC): + def __init__(self, scen: Scenario) -> None: + self.scen = scen + @abstractmethod def evolve( self, diff --git a/rdagent/core/exception.py b/rdagent/core/exception.py index 5260bc50..ae1c57b4 100644 --- a/rdagent/core/exception.py +++ b/rdagent/core/exception.py @@ -1,7 +1,7 @@ class ImplementRunException(Exception): """ Exceptions raised when Implementing and running code. - - start: FactorImplementationTask => FactorGenerator + - start: FactorTask => FactorGenerator - end: Get dataframe after execution The more detailed evaluation in dataframe values are managed by the evaluator. diff --git a/rdagent/core/proposal.py b/rdagent/core/proposal.py index 703292a0..144f34a7 100644 --- a/rdagent/core/proposal.py +++ b/rdagent/core/proposal.py @@ -5,8 +5,9 @@ from abc import ABC, abstractmethod from typing import Dict, Generic, List, Tuple, TypeVar -from rdagent.core.evolving_framework import Feedback -from rdagent.core.experiment import Experiment, Implementation, Loader, Task +from rdagent.core.evaluation import Feedback +from rdagent.core.experiment import Experiment +from rdagent.core.scenario import Scenario # class data_ana: XXX @@ -29,33 +30,6 @@ class Hypothesis: # Origin(path of repo/data/feedback) => view/summarization => generated Hypothesis -class Scenario(ABC): - - @property - @abstractmethod - def background(self): - """Background information""" - - @property - @abstractmethod - def source_data(self): - """Source data description""" - - @property - @abstractmethod - def interface(self): - """Interface description about how to run the code""" - - @property - @abstractmethod - def simulator(self): - """Simulator description""" - - @abstractmethod - def get_scenario_all_desc(self) -> str: - """Combine all the description together""" - - class HypothesisFeedback(Feedback): ... diff --git a/rdagent/core/scenario.py b/rdagent/core/scenario.py new file mode 100644 index 00000000..1e90e936 --- /dev/null +++ b/rdagent/core/scenario.py @@ -0,0 +1,32 @@ +from abc import ABC, abstractmethod + + +class Scenario(ABC): + @property + @abstractmethod + def background(self): + """Background information""" + + @property + @abstractmethod + def source_data(self): + """Source data description""" + + @property + @abstractmethod + def interface(self): + """Interface description about how to run the code""" + + @property + @abstractmethod + def output_format(self): + """Output format description""" + + @property + @abstractmethod + def simulator(self): + """Simulator description""" + + @abstractmethod + def get_scenario_all_desc(self) -> str: + """Combine all the description together""" diff --git a/rdagent/core/task_generator.py b/rdagent/core/task_generator.py index 7570801a..17a69d39 100644 --- a/rdagent/core/task_generator.py +++ b/rdagent/core/task_generator.py @@ -2,11 +2,15 @@ from abc import ABC, abstractmethod from typing import Generic, List, Sequence, TypeVar from rdagent.core.experiment import Experiment +from rdagent.core.scenario import Scenario ASpecificExp = TypeVar("ASpecificExp", bound=Experiment) class TaskGenerator(ABC, Generic[ASpecificExp]): + def __init__(self, scen: Scenario) -> None: + self.scen: Scenario = scen + @abstractmethod def generate(self, exp: ASpecificExp) -> ASpecificExp: """ diff --git a/rdagent/scenarios/qlib/experiment/factor_experiment.py b/rdagent/scenarios/qlib/experiment/factor_experiment.py index e0401b68..85e7bd6e 100644 --- a/rdagent/scenarios/qlib/experiment/factor_experiment.py +++ b/rdagent/scenarios/qlib/experiment/factor_experiment.py @@ -1,13 +1,9 @@ from pathlib import Path -from rdagent.components.task_implementation.factor_implementation.factor import ( - FactorExperiment, -) -from rdagent.components.task_implementation.factor_implementation.utils import ( - get_data_folder_intro, -) +from rdagent.components.coder.factor_coder.factor import FactorExperiment +from rdagent.components.coder.factor_coder.utils import get_data_folder_intro from rdagent.core.prompts import Prompts -from rdagent.core.proposal import Scenario +from rdagent.core.scenario import Scenario prompt_dict = Prompts(file_path=Path(__file__).parent / "prompts.yaml") @@ -23,6 +19,10 @@ class QlibFactorScenario(Scenario): def source_data(self) -> str: return get_data_folder_intro() + @property + def output_format(self) -> str: + return prompt_dict["qlib_factor_output_format"] + @property def interface(self) -> str: return prompt_dict["qlib_factor_interface"] @@ -38,6 +38,8 @@ The source data you can use: {self.source_data} The interface you should follow to write the runnable code: {self.interface} +The output of your code should be in the format: +{self.output_format} The simulator user can use to test your factor: {self.simulator} """ diff --git a/rdagent/scenarios/qlib/experiment/prompts.yaml b/rdagent/scenarios/qlib/experiment/prompts.yaml index 6219e011..b66723db 100644 --- a/rdagent/scenarios/qlib/experiment/prompts.yaml +++ b/rdagent/scenarios/qlib/experiment/prompts.yaml @@ -11,9 +11,31 @@ qlib_factor_background: |- Please specifically give all the hyperparameter in the factors like the window size, look back period, and so on. One factor should statically defines one output with a static source data. For example, last 10 days momentum and last 20 days momentum should be two different factors. qlib_factor_interface: |- - Your code should follow the interface to better interact with the user's system. - Your code should contain the following part: the import part, the function part, and the main part. You should write a main function name: "calculate_{function_name}" and call this function in "if __name__ == __main__" part. Don't write any try-except block in your code. The user will catch the exception message and provide the feedback to you. - User will write your code into a python file and execute the file directly with "python {your_file_name}.py". You should calculate the factor values and save the result into a HDF5(H5) file named "result.h5" in the same directory as your python file. The result file is a HDF5(H5) file containing a pandas dataframe. The index of the dataframe is the "datetime" and "instrument", and the single column name is the factor name,and the value is the factor value. The result file should be saved in the same directory as your python file. + Your python code should follow the interface to better interact with the user's system. + Your python code should contain the following part: the import part, the function part, and the main part. You should write a main function name: "calculate_{function_name}" and call this function in "if __name__ == __main__" part. Don't write any try-except block in your python code. The user will catch the exception message and provide the feedback to you. + User will write your python code into a python file and execute the file directly with "python {your_file_name}.py". You should calculate the factor values and save the result into a HDF5(H5) file named "result.h5" in the same directory as your python file. The result file is a HDF5(H5) file containing a pandas dataframe. The index of the dataframe is the "datetime" and "instrument", and the single column name is the factor name,and the value is the factor value. The result file should be saved in the same directory as your python file. + +qlib_factor_output_format: |- + Your output should be a pandas dataframe similar to the following example information: + + MultiIndex: 40914 entries, (Timestamp('2020-01-02 00:00:00'), 'SH600000') to (Timestamp('2021-12-31 00:00:00'), 'SZ300059') + Data columns (total 1 columns): + # Column Non-Null Count Dtype + --- ------ -------------- ----- + 0 your factor name 40914 non-null float64 + dtypes: float64(1) + memory usage: + None + One possible format of `result.h5` may be like following: + datetime instrument + 2020-01-02 SZ000001 -0.001796 + SZ000166 0.005780 + SZ000686 0.004228 + SZ000712 0.001298 + SZ000728 0.005330 + ... + 2021-12-31 SZ000750 0.000000 + SZ000776 0.002459 qlib_factor_simulator: |- The factors will be sent into Qlib to train a model to predict the next several days return based on the factor values of the previous days. diff --git a/rdagent/scenarios/qlib/factor_experiment_loader/json_loader.py b/rdagent/scenarios/qlib/factor_experiment_loader/json_loader.py index aca7944f..609c7910 100644 --- a/rdagent/scenarios/qlib/factor_experiment_loader/json_loader.py +++ b/rdagent/scenarios/qlib/factor_experiment_loader/json_loader.py @@ -2,16 +2,16 @@ import json from pathlib import Path from rdagent.components.benchmark.eval_method import TestCase -from rdagent.components.loader.experiment_loader import FactorExperimentLoader -from rdagent.components.task_implementation.factor_implementation.factor import ( +from rdagent.components.coder.factor_coder.factor import ( FactorExperiment, FactorTask, FileBasedFactorImplementation, ) +from rdagent.components.loader.experiment_loader import FactorExperimentLoader from rdagent.core.experiment import Loader -class FactorImplementationExperimentLoaderFromDict(FactorExperimentLoader): +class FactorExperimentLoaderFromDict(FactorExperimentLoader): def load(self, factor_dict: dict) -> list: """Load data from a dict.""" task_l = [] @@ -27,17 +27,17 @@ class FactorImplementationExperimentLoaderFromDict(FactorExperimentLoader): return exp -class FactorImplementationExperimentLoaderFromJsonFile(FactorExperimentLoader): +class FactorExperimentLoaderFromJsonFile(FactorExperimentLoader): def load(self, json_file_path: Path) -> list: with open(json_file_path, "r") as file: factor_dict = json.load(file) - return FactorImplementationExperimentLoaderFromDict().load(factor_dict) + return FactorExperimentLoaderFromDict().load(factor_dict) -class FactorImplementationExperimentLoaderFromJsonString(FactorExperimentLoader): +class FactorExperimentLoaderFromJsonString(FactorExperimentLoader): def load(self, json_string: str) -> list: factor_dict = json.loads(json_string) - return FactorImplementationExperimentLoaderFromDict().load(factor_dict) + return FactorExperimentLoaderFromDict().load(factor_dict) # TODO loader only supports generic of task or experiment, testcase might cause CI error here diff --git a/rdagent/scenarios/qlib/factor_experiment_loader/pdf_loader.py b/rdagent/scenarios/qlib/factor_experiment_loader/pdf_loader.py index 1161d010..446b7026 100644 --- a/rdagent/scenarios/qlib/factor_experiment_loader/pdf_loader.py +++ b/rdagent/scenarios/qlib/factor_experiment_loader/pdf_loader.py @@ -22,7 +22,7 @@ from rdagent.core.log import RDAgentLog from rdagent.core.prompts import Prompts from rdagent.oai.llm_utils import APIBackend, create_embedding_with_multiprocessing from rdagent.scenarios.qlib.factor_experiment_loader.json_loader import ( - FactorImplementationExperimentLoaderFromDict, + FactorExperimentLoaderFromDict, ) document_process_prompts = Prompts(file_path=Path(__file__).parent / "prompts.yaml") @@ -579,7 +579,7 @@ def deduplicate_factors_by_llm( # noqa: C901, PLR0912 return llm_deduplicated_factor_dict, final_duplication_names_list -class FactorImplementationExperimentLoaderFromPDFfiles(FactorExperimentLoader): +class FactorExperimentLoaderFromPDFfiles(FactorExperimentLoader): def load(self, file_or_folder_path: Path) -> dict: docs_dict = load_and_process_pdfs_by_langchain(Path(file_or_folder_path)) @@ -589,4 +589,4 @@ class FactorImplementationExperimentLoaderFromPDFfiles(FactorExperimentLoader): factor_viability = check_factor_viability(factor_dict) factor_dict, duplication_names_list = deduplicate_factors_by_llm(factor_dict, factor_viability) - return FactorImplementationExperimentLoaderFromDict().load(factor_dict) + return FactorExperimentLoaderFromDict().load(factor_dict) diff --git a/rdagent/scenarios/qlib/factor_proposal.py b/rdagent/scenarios/qlib/factor_proposal.py index ab2d494c..52f65de4 100644 --- a/rdagent/scenarios/qlib/factor_proposal.py +++ b/rdagent/scenarios/qlib/factor_proposal.py @@ -4,18 +4,13 @@ from typing import List, Tuple from jinja2 import Environment, StrictUndefined -from rdagent.components.idea_proposal.factor_proposal import ( +from rdagent.components.coder.factor_coder.factor import FactorExperiment, FactorTask +from rdagent.components.coder.factor_coder.utils import get_data_folder_intro +from rdagent.components.proposal.factor_proposal import ( FactorHypothesis, FactorHypothesis2Experiment, FactorHypothesisGen, ) -from rdagent.components.task_implementation.factor_implementation.factor import ( - FactorExperiment, - FactorTask, -) -from rdagent.components.task_implementation.factor_implementation.utils import ( - get_data_folder_intro, -) from rdagent.core.prompts import Prompts from rdagent.core.proposal import HypothesisSet, Scenario, Trace diff --git a/rdagent/scenarios/qlib/factor_task_implementation/__init__.py b/rdagent/scenarios/qlib/factor_task_implementation/__init__.py index b0ac1c47..0facb840 100644 --- a/rdagent/scenarios/qlib/factor_task_implementation/__init__.py +++ b/rdagent/scenarios/qlib/factor_task_implementation/__init__.py @@ -1,5 +1,4 @@ -from rdagent.components.task_implementation.factor_implementation.CoSTEER import ( - CoSTEERFG, -) +from rdagent.components.coder.factor_coder.CoSTEER import FactorCoSTEER -COSTEERFG_QUANT_FACTOR_IMPLEMENTATION = CoSTEERFG # TODO: This is a placeholder. We need to split the scenario part of the task implementation into this folder +QlibFactorCoSTEER = FactorCoSTEER +# TODO: This is a placeholder. We need to split the scenario part of the task implementation into this folder diff --git a/rdagent/scenarios/qlib/model_task_implementation/__init__.py b/rdagent/scenarios/qlib/model_task_implementation/__init__.py index fcc823d4..fad4c284 100644 --- a/rdagent/scenarios/qlib/model_task_implementation/__init__.py +++ b/rdagent/scenarios/qlib/model_task_implementation/__init__.py @@ -1,6 +1,5 @@ -from rdagent.components.task_implementation.model_implementation.one_shot import ( - ModelCodeWriter, -) +from rdagent.components.coder.model_coder.one_shot import ModelCodeWriter -class QlibModelCodeWriter(ModelCodeWriter): ... +class QlibModelCodeWriter(ModelCodeWriter): + ... diff --git a/rdagent/scenarios/qlib/task_generator/feedback.py b/rdagent/scenarios/qlib/task_generator/feedback.py index ea4a2ce8..8fc42768 100644 --- a/rdagent/scenarios/qlib/task_generator/feedback.py +++ b/rdagent/scenarios/qlib/task_generator/feedback.py @@ -4,4 +4,5 @@ from rdagent.core.proposal import Experiment2Feedback -class QlibFactorExperiment2Feedback(Experiment2Feedback): ... +class QlibFactorExperiment2Feedback(Experiment2Feedback): + ... diff --git a/rdagent/scenarios/qlib/task_generator/model.py b/rdagent/scenarios/qlib/task_generator/model.py index 8c7d2a0c..7fc0fabc 100644 --- a/rdagent/scenarios/qlib/task_generator/model.py +++ b/rdagent/scenarios/qlib/task_generator/model.py @@ -1,7 +1,8 @@ +from rdagent.components.coder.model_coder.model import ModelImplementation from rdagent.core.task_generator import TaskGenerator -class QlibModelImplementation(TaskGenerator[QlibModelExperiment]): +class QlibModelImplementation(TaskGenerator[ModelImplementation]): """ Docker run Everything in a folder diff --git a/rdagent/utils/env.py b/rdagent/utils/env.py index 1590e59d..e334db4d 100644 --- a/rdagent/utils/env.py +++ b/rdagent/utils/env.py @@ -6,11 +6,12 @@ Tries to create uniform environment for the agent to run; """ import os -import docker from abc import abstractmethod -from pydantic import BaseModel -from typing import Generic, TypeVar from pathlib import Path +from typing import Generic, TypeVar + +import docker +from pydantic import BaseModel ASpecificBaseModel = TypeVar("ASpecificBaseModel", bound=BaseModel) @@ -21,6 +22,7 @@ class Env(Generic[ASpecificBaseModel]): - It provides base typing and checking featurs. - loading and dumping the information will be easier: for example, we can use package like `pydantic-yaml` """ + conf: ASpecificBaseModel # different env have different conf. def __init__(self, conf: ASpecificBaseModel): @@ -33,10 +35,7 @@ class Env(Generic[ASpecificBaseModel]): """ @abstractmethod - def run(self, - entry: str | None, - local_path: str | None = None, - env: dict | None = None) -> str: + def run(self, entry: str | None, local_path: str | None = None, env: dict | None = None) -> str: """ Run the folder under the environment. @@ -70,6 +69,7 @@ class LocalEnv(Env[LocalConf]): """ Sometimes local environment may be more convinient for testing """ + conf: LocalConf @@ -87,10 +87,12 @@ class DockerConf(BaseModel): # So we just want to download it once. -QLIB_TORCH_IMAGE = DockerConf(image="linlanglv/qlib_image_nightly_pytorch:nightly", - mount_path="/workspace", - default_entry="qrun conf.yaml", - extra_volumes={Path("~/.qlib/").expanduser().resolve(): "/root/.qlib/"}) +QLIB_TORCH_IMAGE = DockerConf( + image="linlanglv/qlib_image_nightly_pytorch:nightly", + mount_path="/workspace", + default_entry="qrun conf.yaml", + extra_volumes={Path("~/.qlib/").expanduser().resolve(): "/root/.qlib/"}, +) class DockerEnv(Env[DockerConf]): @@ -109,7 +111,6 @@ class DockerEnv(Env[DockerConf]): raise RuntimeError(f"Error while pulling the image: {e}") def run(self, entry: str | None = None, local_path: str | None = None, env: dict | None = None): - if env is None: env = {} client = docker.from_env() @@ -119,10 +120,10 @@ class DockerEnv(Env[DockerConf]): volumns = {} if local_path is not None: local_path = os.path.abspath(local_path) - volumns[local_path] = {'bind': self.conf.mount_path, 'mode': 'rw'} + volumns[local_path] = {"bind": self.conf.mount_path, "mode": "rw"} if self.conf.extra_volumes is not None: for lp, rp in self.conf.extra_volumes.items(): - volumns[lp] = {'bind': rp, 'mode': 'rw'} + volumns[lp] = {"bind": rp, "mode": "rw"} log_output = "" try: diff --git a/test/utils/test_env.py b/test/utils/test_env.py index 2750eb00..ba5e44cd 100644 --- a/test/utils/test_env.py +++ b/test/utils/test_env.py @@ -1,24 +1,24 @@ import os -import sys import subprocess +import sys import unittest from pathlib import Path + sys.path.append(str(Path(__file__).resolve().parent.parent)) -from rdagent.utils.env import QTDockerEnv import shutil +from rdagent.utils.env import QTDockerEnv DIRNAME = Path(__file__).absolute().resolve().parent class EnvUtils(unittest.TestCase): - def setUp(self): pass def tearDown(self): # NOTE: For a docker file, the output are generated with root permission. - # mlrun_p = DIRNAME / "env_tpl" / "mlruns" + # mlrun_p = DIRNAME / "env_tpl" / "mlruns" # if mlrun_p.exists(): # shutil.rmtree(mlrun_p) ... @@ -34,7 +34,7 @@ class EnvUtils(unittest.TestCase): # the stdout are returned as result result = qtde.run(local_path=str(DIRNAME / "env_tpl"), entry="qrun conf.yaml") - mlrun_p = DIRNAME / "env_tpl" / "mlruns" + mlrun_p = DIRNAME / "env_tpl" / "mlruns" self.assertTrue(mlrun_p.exists(), f"Expected output file {mlrun_p} not found")