diff --git a/.gitignore b/.gitignore index 176e76d2..d2ab763a 100644 --- a/.gitignore +++ b/.gitignore @@ -155,6 +155,10 @@ git_ignore_folder/ *.db # Docker -env_factor/mlruns/ +factor_template/mlruns/ env_tpl -mlruns/ \ No newline at end of file +mlruns/ + +# possible output from coder or runner +*.pth +*qlib_res.csv \ No newline at end of file diff --git a/docs/conf.py b/docs/conf.py index d66ca234..244a5626 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -8,9 +8,9 @@ import importlib.metadata -project = 'RDAgent' -copyright = '2024, Microsoft' -author = 'Microsoft' +project = "RDAgent" +copyright = "2024, Microsoft" +author = "Microsoft" # -- General configuration --------------------------------------------------- # https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration @@ -20,13 +20,13 @@ extensions = [] autodoc_member_order = "bysource" # The suffix of source filenames. -source_suffix = '.rst' +source_suffix = ".rst" # The encoding of source files. -source_encoding = 'utf-8' +source_encoding = "utf-8" # The main toctree document. -master_doc = 'index' +master_doc = "index" # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the @@ -42,21 +42,21 @@ language = "en" # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. -exclude_patterns = ['build'] +exclude_patterns = ["build"] # -- Options for HTML output ------------------------------------------------- # https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output # The name of the Pygments (syntax highlighting) style to use. -pygments_style = 'sphinx' +pygments_style = "sphinx" try: import furo - html_theme = 'furo' + html_theme = "furo" html_theme_options = { "navigation_with_keys": True, } except ImportError: - html_theme = 'default' + html_theme = "default" -html_static_path = ['_static'] +html_static_path = ["_static"] diff --git a/rdagent/app/factor_extraction_and_code/factor_extract_and_implement.py b/rdagent/app/factor_extraction_and_code/factor_extract_and_implement.py new file mode 100644 index 00000000..5ff22944 --- /dev/null +++ b/rdagent/app/factor_extraction_and_code/factor_extract_and_implement.py @@ -0,0 +1,28 @@ +# %% +from dotenv import load_dotenv + +from rdagent.log import rdagent_logger as logger +from rdagent.scenarios.qlib.developer.factor_coder import QlibFactorCoSTEER +from rdagent.scenarios.qlib.experiment.factor_experiment import QlibFactorScenario +from rdagent.scenarios.qlib.factor_experiment_loader.pdf_loader import ( + FactorExperimentLoaderFromPDFfiles, +) + +assert load_dotenv() + + +def extract_factors_and_implement(report_file_path: str) -> None: + scenario = QlibFactorScenario() + + with logger.tag("extract_factors_and_implement"): + with logger.tag("load_factor_tasks"): + exp = FactorExperimentLoaderFromPDFfiles().load(report_file_path) + with logger.tag("implement_factors"): + exp = QlibFactorCoSTEER(scenario).develop(exp) + + # Qlib to run the implementation in rd loop + return exp + + +if __name__ == "__main__": + extract_factors_and_implement("/home/xuyang1/workspace/report.pdf") diff --git a/rdagent/app/factor_extraction_and_implementation/factor_extract_and_implement.py b/rdagent/app/factor_extraction_and_implementation/factor_extract_and_implement.py deleted file mode 100644 index de5048d9..00000000 --- a/rdagent/app/factor_extraction_and_implementation/factor_extract_and_implement.py +++ /dev/null @@ -1,24 +0,0 @@ -# %% -from dotenv import load_dotenv - -from rdagent.scenarios.qlib.factor_experiment_loader.pdf_loader import FactorExperimentLoaderFromPDFfiles -from rdagent.scenarios.qlib.factor_task_implementation import QlibFactorCoSTEER -from rdagent.log import rdagent_logger as logger - -assert load_dotenv() - - -def extract_factors_and_implement(report_file_path: str) -> None: - with logger.tag('extract_factors_and_implement'): - with logger.tag('load_factor_tasks'): - factor_tasks = FactorExperimentLoaderFromPDFfiles().load(report_file_path) - with logger.tag('implement_factors'): - implementation_result = QlibFactorCoSTEER().generate(factor_tasks) - # Qlib to run the implementation - return implementation_result - - -if __name__ == "__main__": - extract_factors_and_implement("/home/xuyang1/workspace/report.pdf") - -# %% diff --git a/rdagent/app/model_extraction_and_code/model_extraction_and_implementation.py b/rdagent/app/model_extraction_and_code/model_extraction_and_implementation.py new file mode 100644 index 00000000..1114cf13 --- /dev/null +++ b/rdagent/app/model_extraction_and_code/model_extraction_and_implementation.py @@ -0,0 +1,21 @@ +# %% +from dotenv import load_dotenv + +from rdagent.components.coder.model_coder.task_loader import ( + ModelExperimentLoaderFromPDFfiles, +) +from rdagent.scenarios.qlib.developer.model_coder import QlibModelCoSTEER +from rdagent.scenarios.qlib.experiment.model_experiment import QlibModelScenario + + +def extract_models_and_implement(report_file_path: str = "../test_doc") -> None: + scenario = QlibModelScenario() + exp = ModelExperimentLoaderFromPDFfiles().load(report_file_path) + exp = QlibModelCoSTEER(scenario).develop(exp) + return exp + + +import fire + +if __name__ == "__main__": + fire.Fire(extract_models_and_implement) diff --git a/rdagent/app/model_extraction_and_implementation/model_extraction_and_implementation.py b/rdagent/app/model_extraction_and_implementation/model_extraction_and_implementation.py deleted file mode 100644 index 0e69087b..00000000 --- a/rdagent/app/model_extraction_and_implementation/model_extraction_and_implementation.py +++ /dev/null @@ -1,19 +0,0 @@ -# %% -from dotenv import load_dotenv - -from rdagent.components.coder.model_coder.one_shot import ModelCodeWriter -from rdagent.components.coder.model_coder.task_loader import ( - ModelExperimentLoaderFromPDFfiles, -) - - -def extract_models_and_implement(report_file_path: str = "../test_doc") -> None: - factor_tasks = ModelExperimentLoaderFromPDFfiles().load(report_file_path) - implementation_result = ModelCodeWriter().generate(factor_tasks) - return implementation_result - - -import fire - -if __name__ == "__main__": - fire.Fire(extract_models_and_implement) diff --git a/rdagent/app/model_implementation/eval.py b/rdagent/app/model_implementation/eval.py index 8d69a047..6c3ae06e 100644 --- a/rdagent/app/model_implementation/eval.py +++ b/rdagent/app/model_implementation/eval.py @@ -1,7 +1,7 @@ from pathlib import Path from rdagent.components.coder.model_coder.CoSTEER import ModelCoSTEER -from rdagent.components.loader.task_loader import ModelImpLoader, ModelTaskLoaderJson +from rdagent.components.loader.task_loader import ModelTaskLoaderJson, ModelWsLoader from rdagent.scenarios.qlib.experiment.model_experiment import ( QlibModelExperiment, QlibModelScenario, @@ -23,17 +23,17 @@ model_experiment = QlibModelExperiment(sub_tasks=task_l) # mtg = ModelCodeWriter(scen=QlibModelScenario()) mtg = ModelCoSTEER(scen=QlibModelScenario()) -model_experiment = mtg.generate(model_experiment) +model_experiment = mtg.develop(model_experiment) # TODO: Align it with the benchmark framework after @wenjun's refine the evaluation part. # Currently, we just handcraft a workflow for fast evaluation. -mil = ModelImpLoader(bench_folder / "gt_code") +mil = ModelWsLoader(bench_folder / "gt_code") mie = ModelImpValEval() # Evaluation: eval_l = [] -for impl in model_experiment.sub_implementations: +for impl in model_experiment.sub_workspace_list: print(impl.target_task) gt_impl = mil.load(impl.target_task) eval_l.append(mie.evaluate(gt_impl, impl)) diff --git a/rdagent/app/qlib_rd_loop/conf.py b/rdagent/app/qlib_rd_loop/conf.py index 5e8b8b14..df9d1dd5 100644 --- a/rdagent/app/qlib_rd_loop/conf.py +++ b/rdagent/app/qlib_rd_loop/conf.py @@ -1,30 +1,34 @@ -from pydantic_settings import BaseSettings from pathlib import Path +from pydantic_settings import BaseSettings + class PropSetting(BaseSettings): - """""" + class Config: + env_prefix = "QLIB_" # Use MODEL_CODER_ as prefix for environment variables + protected_namespaces = () # Add 'model_' to the protected namespaces - qlib_factor_scen: str = "rdagent.scenarios.qlib.experiment.factor_experiment.QlibFactorScenario" - qlib_factor_hypothesis_gen: str = "rdagent.scenarios.qlib.factor_proposal.QlibFactorHypothesisGen" - qlib_factor_hypothesis2experiment: str = "rdagent.scenarios.qlib.factor_proposal.QlibFactorHypothesis2Experiment" - qlib_factor_coder: str = "rdagent.scenarios.qlib.factor_task_implementation.QlibFactorCoSTEER" - qlib_factor_runner: str = "rdagent.scenarios.qlib.task_generator.data.QlibFactorRunner" - qlib_factor_summarizer: str = ( - "rdagent.scenarios.qlib.task_generator.feedback.QlibFactorHypothesisExperiment2Feedback" + factor_scen: str = "rdagent.scenarios.qlib.experiment.factor_experiment.QlibFactorScenario" + factor_hypothesis_gen: str = "rdagent.scenarios.qlib.proposal.factor_proposal.QlibFactorHypothesisGen" + factor_hypothesis2experiment: str = ( + "rdagent.scenarios.qlib.proposal.factor_proposal.QlibFactorHypothesis2Experiment" ) + factor_coder: str = "rdagent.scenarios.qlib.developer.factor_coder.QlibFactorCoSTEER" + factor_runner: str = "rdagent.scenarios.qlib.developer.factor_runner.QlibFactorRunner" + factor_summarizer: str = "rdagent.scenarios.qlib.developer.feedback.QlibFactorHypothesisExperiment2Feedback" # TODO: model part is not finished yet - qlib_model_scen: str = "rdagent.scenarios.qlib.experiment.model_experiment.QlibModelScenario" - qlib_model_hypothesis_gen: str = "rdagent.scenarios.qlib.model_proposal.QlibModelHypothesisGen" - qlib_model_hypothesis2experiment: str = "rdagent.scenarios.qlib.model_proposal.QlibModelHypothesis2Experiment" - qlib_model_coder: str = "rdagent.scenarios.qlib.model_task_implementation.QlibModelCoSTEER" - qlib_model_runner: str = "rdagent.scenarios.qlib.task_generator.model.QlibModelRunner" - qlib_model_summarizer: str = "rdagent.scenarios.qlib.task_generator.feedback.QlibModelHypothesisExperiment2Feedback" + model_scen: str = "rdagent.scenarios.qlib.experiment.model_experiment.QlibModelScenario" + model_hypothesis_gen: str = "rdagent.scenarios.qlib.proposal.model_proposal.QlibModelHypothesisGen" + model_hypothesis2experiment: str = "rdagent.scenarios.qlib.proposal.model_proposal.QlibModelHypothesis2Experiment" + model_coder: str = "rdagent.scenarios.qlib.developer.model_coder.QlibModelCoSTEER" + model_runner: str = "rdagent.scenarios.qlib.developer.model_runner.QlibModelRunner" + model_summarizer: str = "rdagent.scenarios.qlib.developer.feedback.QlibModelHypothesisExperiment2Feedback" evolving_n: int = 10 - + py_bin: str = "/usr/bin/python" local_qlib_folder: Path = Path("/home/rdagent/qlib") - + + PROP_SETTING = PropSetting() diff --git a/rdagent/app/qlib_rd_loop/factor.py b/rdagent/app/qlib_rd_loop/factor.py index 32bbbb65..4330702e 100644 --- a/rdagent/app/qlib_rd_loop/factor.py +++ b/rdagent/app/qlib_rd_loop/factor.py @@ -4,39 +4,45 @@ TODO: Factor Structure RD-Loop from dotenv import load_dotenv +from rdagent.core.exception import FactorEmptyException from rdagent.core.scenario import Scenario +from rdagent.log import rdagent_logger as logger load_dotenv(override=True) from rdagent.app.qlib_rd_loop.conf import PROP_SETTING +from rdagent.core.developer import Developer from rdagent.core.proposal import ( Hypothesis2Experiment, HypothesisExperiment2Feedback, HypothesisGen, Trace, ) -from rdagent.core.task_generator import TaskGenerator from rdagent.core.utils import import_class -scen: Scenario = import_class(PROP_SETTING.qlib_factor_scen)() +scen: Scenario = import_class(PROP_SETTING.factor_scen)() -hypothesis_gen: HypothesisGen = import_class(PROP_SETTING.qlib_factor_hypothesis_gen)(scen) +hypothesis_gen: HypothesisGen = import_class(PROP_SETTING.factor_hypothesis_gen)(scen) -hypothesis2experiment: Hypothesis2Experiment = import_class(PROP_SETTING.qlib_factor_hypothesis2experiment)() +hypothesis2experiment: Hypothesis2Experiment = import_class(PROP_SETTING.factor_hypothesis2experiment)() -qlib_factor_coder: TaskGenerator = import_class(PROP_SETTING.qlib_factor_coder)(scen) +qlib_factor_coder: Developer = import_class(PROP_SETTING.factor_coder)(scen) -qlib_factor_runner: TaskGenerator = import_class(PROP_SETTING.qlib_factor_runner)(scen) +qlib_factor_runner: Developer = import_class(PROP_SETTING.factor_runner)(scen) -qlib_factor_summarizer: HypothesisExperiment2Feedback = import_class(PROP_SETTING.qlib_factor_summarizer)(scen) +qlib_factor_summarizer: HypothesisExperiment2Feedback = import_class(PROP_SETTING.factor_summarizer)(scen) trace = Trace(scen=scen) for _ in range(PROP_SETTING.evolving_n): - hypothesis = hypothesis_gen.gen(trace) - exp = hypothesis2experiment.convert(hypothesis, trace) - exp = qlib_factor_coder.generate(exp) - exp = qlib_factor_runner.generate(exp) - feedback = qlib_factor_summarizer.generateFeedback(exp, hypothesis, trace) + try: + hypothesis = hypothesis_gen.gen(trace) + exp = hypothesis2experiment.convert(hypothesis, trace) + exp = qlib_factor_coder.develop(exp) + exp = qlib_factor_runner.develop(exp) + feedback = qlib_factor_summarizer.generateFeedback(exp, hypothesis, trace) - trace.hist.append((hypothesis, exp, feedback)) \ No newline at end of file + trace.hist.append((hypothesis, exp, feedback)) + except FactorEmptyException as e: + logger.warning(e) + continue diff --git a/rdagent/app/qlib_rd_loop/model.py b/rdagent/app/qlib_rd_loop/model.py index e238d6b9..57af0258 100644 --- a/rdagent/app/qlib_rd_loop/model.py +++ b/rdagent/app/qlib_rd_loop/model.py @@ -6,6 +6,8 @@ TODO: move the following code to a new class: Model_RD_Agent # import_from from rdagent.app.qlib_rd_loop.conf import PROP_SETTING +from rdagent.core.developer import Developer +from rdagent.core.exception import ModelEmptyException from rdagent.core.proposal import ( Hypothesis2Experiment, HypothesisExperiment2Feedback, @@ -13,26 +15,30 @@ from rdagent.core.proposal import ( Trace, ) from rdagent.core.scenario import Scenario -from rdagent.core.task_generator import TaskGenerator from rdagent.core.utils import import_class +from rdagent.log import rdagent_logger as logger -scen: Scenario = import_class(PROP_SETTING.qlib_model_scen)() +scen: Scenario = import_class(PROP_SETTING.model_scen)() -hypothesis_gen: HypothesisGen = import_class(PROP_SETTING.qlib_model_hypothesis_gen)(scen) +hypothesis_gen: HypothesisGen = import_class(PROP_SETTING.model_hypothesis_gen)(scen) -hypothesis2experiment: Hypothesis2Experiment = import_class(PROP_SETTING.qlib_model_hypothesis2experiment)() +hypothesis2experiment: Hypothesis2Experiment = import_class(PROP_SETTING.model_hypothesis2experiment)() -qlib_model_coder: TaskGenerator = import_class(PROP_SETTING.qlib_model_coder)(scen) -qlib_model_runner: TaskGenerator = import_class(PROP_SETTING.qlib_model_runner)(scen) +qlib_model_coder: Developer = import_class(PROP_SETTING.model_coder)(scen) +qlib_model_runner: Developer = import_class(PROP_SETTING.model_runner)(scen) -qlib_model_summarizer: HypothesisExperiment2Feedback = import_class(PROP_SETTING.qlib_model_summarizer)(scen) +qlib_model_summarizer: HypothesisExperiment2Feedback = import_class(PROP_SETTING.model_summarizer)(scen) trace = Trace(scen=scen) for _ in range(PROP_SETTING.evolving_n): - hypothesis = hypothesis_gen.gen(trace) - exp = hypothesis2experiment.convert(hypothesis, trace) - exp = qlib_model_coder.generate(exp) - exp = qlib_model_runner.generate(exp) - feedback = qlib_model_summarizer.generateFeedback(exp, hypothesis, trace) + try: + hypothesis = hypothesis_gen.gen(trace) + exp = hypothesis2experiment.convert(hypothesis, trace) + exp = qlib_model_coder.develop(exp) + exp = qlib_model_runner.develop(exp) + feedback = qlib_model_summarizer.generateFeedback(exp, hypothesis, trace) - trace.hist.append((hypothesis, exp, feedback)) + trace.hist.append((hypothesis, exp, feedback)) + except ModelEmptyException as e: + logger.warning(e) + continue diff --git a/rdagent/app/quant_factor_benchmark/eval.py b/rdagent/app/quant_factor_benchmark/eval.py index 0580e8cb..3088be30 100644 --- a/rdagent/app/quant_factor_benchmark/eval.py +++ b/rdagent/app/quant_factor_benchmark/eval.py @@ -1,10 +1,11 @@ -from rdagent.components.benchmark.conf import BenchmarkSettings -from rdagent.components.benchmark.eval_method import FactorImplementEval -from rdagent.core.utils import import_class from rdagent.scenarios.qlib.factor_task_loader.json_loader import ( FactorTestCaseLoaderFromJsonFile, ) +from rdagent.components.benchmark.conf import BenchmarkSettings +from rdagent.components.benchmark.eval_method import FactorImplementEval +from rdagent.core.utils import import_class + # 1.read the settings bs = BenchmarkSettings() diff --git a/rdagent/components/benchmark/eval_method.py b/rdagent/components/benchmark/eval_method.py index e8ea3ca6..054b1c97 100644 --- a/rdagent/components/benchmark/eval_method.py +++ b/rdagent/components/benchmark/eval_method.py @@ -15,10 +15,11 @@ from rdagent.components.coder.factor_coder.CoSTEER.evaluators import ( FactorRowCountEvaluator, FactorSingleColumnEvaluator, ) -from rdagent.components.coder.factor_coder.factor import FileBasedFactorImplementation -from rdagent.core.exception import ImplementRunException -from rdagent.core.experiment import Implementation, Task -from rdagent.core.task_generator import TaskGenerator +from rdagent.components.coder.factor_coder.factor import FactorFBWorkspace +from rdagent.core.conf import RD_AGENT_SETTINGS +from rdagent.core.developer import Developer +from rdagent.core.exception import CoderException +from rdagent.core.experiment import Task, Workspace from rdagent.core.utils import multiprocessing_wrapper @@ -26,7 +27,7 @@ class TestCase: def __init__( self, target_task: list[Task] = [], - ground_truth: list[Implementation] = [], + ground_truth: list[Workspace] = [], ): self.ground_truth = ground_truth self.target_task = target_task @@ -41,7 +42,7 @@ class BaseEval: self, evaluator_l: List[FactorEvaluator], test_cases: List[TestCase], - generate_method: TaskGenerator, + generate_method: Developer, catch_eval_except: bool = True, ): """Parameters @@ -62,12 +63,12 @@ class BaseEval: self, path: Union[Path, str], **kwargs, - ) -> List[Implementation]: + ) -> List[Workspace]: path = Path(path) fi_l = [] for tc in self.test_cases: try: - fi = FileBasedFactorImplementation.from_folder(tc.task, path, **kwargs) + fi = FactorFBWorkspace.from_folder(tc.task, path, **kwargs) fi_l.append(fi) except FileNotFoundError: print("Fail to load test case for factor: ", tc.task.factor_name) @@ -75,8 +76,8 @@ class BaseEval: def eval_case( self, - case_gt: Implementation, - case_gen: Implementation, + case_gt: Workspace, + case_gen: Workspace, ) -> List[Union[Tuple[FactorEvaluator, object], Exception]]: """Parameters ---------- @@ -96,7 +97,7 @@ class BaseEval: try: eval_res.append((ev, ev.evaluate(implementation=case_gen, gt_implementation=case_gt))) # if the corr ev is successfully evaluated and achieve the best performance, then break - except ImplementRunException as e: + except CoderException as e: return e except Exception as e: # exception when evaluation @@ -111,7 +112,7 @@ class FactorImplementEval(BaseEval): def __init__( self, test_cases: TestCase, - method: TaskGenerator, + method: Developer, *args, test_round: int = 10, **kwargs, @@ -137,17 +138,17 @@ class FactorImplementEval(BaseEval): print(f"Eval {_}-th times...") print("========================================================\n") try: - gen_factor_l = self.generate_method.generate(self.test_cases.target_task) + gen_factor_l = self.generate_method.develop(self.test_cases.target_task) except KeyboardInterrupt: # TODO: Why still need to save result after KeyboardInterrupt? print("Manually interrupted the evaluation. Saving existing results") break - if len(gen_factor_l.sub_implementations) != len(self.test_cases.ground_truth): + if len(gen_factor_l.sub_workspace_list) != len(self.test_cases.ground_truth): raise ValueError( "The number of cases to eval should be equal to the number of test cases.", ) - gen_factor_l_all_rounds.extend(gen_factor_l.sub_implementations) + gen_factor_l_all_rounds.extend(gen_factor_l.sub_workspace_list) test_cases_all_rounds.extend(self.test_cases.ground_truth) eval_res_list = multiprocessing_wrapper( @@ -155,7 +156,7 @@ class FactorImplementEval(BaseEval): (self.eval_case, (gt_case, gen_factor)) for gt_case, gen_factor in zip(test_cases_all_rounds, gen_factor_l_all_rounds) ], - n=FACTOR_IMPLEMENT_SETTINGS.evo_multi_proc_n, + n=RD_AGENT_SETTINGS.multi_proc_n, ) for gt_case, eval_res, gen_factor in tqdm(zip(test_cases_all_rounds, eval_res_list, gen_factor_l_all_rounds)): diff --git a/rdagent/components/coder/factor_coder/CoSTEER/__init__.py b/rdagent/components/coder/factor_coder/CoSTEER/__init__.py index 9fa2f84e..55870c7e 100644 --- a/rdagent/components/coder/factor_coder/CoSTEER/__init__.py +++ b/rdagent/components/coder/factor_coder/CoSTEER/__init__.py @@ -9,6 +9,9 @@ from rdagent.components.coder.factor_coder.CoSTEER.evaluators import ( from rdagent.components.coder.factor_coder.CoSTEER.evolvable_subjects import ( FactorEvolvingItem, ) +from rdagent.components.coder.factor_coder.CoSTEER.evolving_agent import ( + FactorRAGEvoAgent, +) from rdagent.components.coder.factor_coder.CoSTEER.evolving_strategy import ( FactorEvolvingStrategyWithGraph, ) @@ -18,18 +21,19 @@ from rdagent.components.coder.factor_coder.CoSTEER.knowledge_management import ( FactorKnowledgeBaseV1, ) from rdagent.components.coder.factor_coder.factor import FactorExperiment +from rdagent.core.developer import Developer from rdagent.core.evolving_agent import RAGEvoAgent from rdagent.core.scenario import Scenario -from rdagent.core.task_generator import TaskGenerator -class FactorCoSTEER(TaskGenerator[FactorExperiment]): +class FactorCoSTEER(Developer[FactorExperiment]): def __init__( self, *args, with_knowledge: bool = True, with_feedback: bool = True, knowledge_self_gen: bool = True, + filter_final_evo: bool = True, **kwargs, ) -> None: super().__init__(*args, **kwargs) @@ -47,6 +51,7 @@ class FactorCoSTEER(TaskGenerator[FactorExperiment]): self.with_knowledge = with_knowledge self.with_feedback = with_feedback self.knowledge_self_gen = knowledge_self_gen + self.filter_final_evo = filter_final_evo self.evolving_strategy = FactorEvolvingStrategyWithGraph(scen=self.scen) # declare the factor evaluator self.factor_evaluator = FactorMultiEvaluator(FactorEvaluatorForCoder(scen=self.scen), scen=self.scen) @@ -72,7 +77,7 @@ class FactorCoSTEER(TaskGenerator[FactorExperiment]): ) return factor_knowledge_base - def generate(self, exp: FactorExperiment) -> FactorExperiment: + def develop(self, exp: FactorExperiment) -> FactorExperiment: # init knowledge base factor_knowledge_base = self.load_or_init_knowledge_base( former_knowledge_base_path=self.knowledge_base_path, @@ -84,7 +89,9 @@ class FactorCoSTEER(TaskGenerator[FactorExperiment]): # init intermediate items factor_experiment = FactorEvolvingItem(sub_tasks=exp.sub_tasks) - self.evolve_agent = RAGEvoAgent(max_loop=self.max_loop, evolving_strategy=self.evolving_strategy, rag=self.rag) + self.evolve_agent = FactorRAGEvoAgent( + max_loop=self.max_loop, evolving_strategy=self.evolving_strategy, rag=self.rag + ) factor_experiment = self.evolve_agent.multistep_evolve( factor_experiment, @@ -92,11 +99,11 @@ class FactorCoSTEER(TaskGenerator[FactorExperiment]): with_knowledge=self.with_knowledge, with_feedback=self.with_feedback, knowledge_self_gen=self.knowledge_self_gen, + filter_final_evo=self.filter_final_evo, ) # save new knowledge base if self.new_knowledge_base_path is not None: pickle.dump(factor_knowledge_base, open(self.new_knowledge_base_path, "wb")) - self.knowledge_base = factor_knowledge_base - factor_experiment.based_experiments = exp.based_experiments - return factor_experiment + exp.sub_workspace_list = factor_experiment.sub_workspace_list + return exp diff --git a/rdagent/components/coder/factor_coder/CoSTEER/evaluators.py b/rdagent/components/coder/factor_coder/CoSTEER/evaluators.py index 36e87a19..081a244d 100644 --- a/rdagent/components/coder/factor_coder/CoSTEER/evaluators.py +++ b/rdagent/components/coder/factor_coder/CoSTEER/evaluators.py @@ -8,7 +8,6 @@ from typing import List, Tuple import pandas as pd from jinja2 import Environment, StrictUndefined -from rdagent.components.coder.factor_coder.config import FACTOR_IMPLEMENT_SETTINGS from rdagent.components.coder.factor_coder.CoSTEER.evolvable_subjects import ( FactorEvolvingItem, ) @@ -16,10 +15,10 @@ from rdagent.components.coder.factor_coder.factor import FactorTask from rdagent.core.conf import RD_AGENT_SETTINGS from rdagent.core.evaluation import Evaluator from rdagent.core.evolving_framework import Feedback, QueriedKnowledge -from rdagent.core.experiment import Implementation, Task -from rdagent.log import rdagent_logger as logger +from rdagent.core.experiment import Task, Workspace from rdagent.core.prompts import Prompts from rdagent.core.utils import multiprocessing_wrapper +from rdagent.log import rdagent_logger as logger from rdagent.oai.llm_utils import APIBackend evaluate_prompts = Prompts(file_path=Path(__file__).parent.parent / "prompts.yaml") @@ -33,8 +32,8 @@ class FactorEvaluator(Evaluator): def evaluate( self, target_task: Task, - implementation: Implementation, - gt_implementation: Implementation, + implementation: Workspace, + gt_implementation: Workspace, **kwargs, ) -> Tuple[str, object]: """You can get the dataframe by @@ -53,7 +52,7 @@ class FactorEvaluator(Evaluator): """ raise NotImplementedError("Please implement the `evaluator` method") - def _get_df(self, gt_implementation: Implementation, implementation: Implementation): + def _get_df(self, gt_implementation: Workspace, implementation: Workspace): if gt_implementation is not None: _, gt_df = gt_implementation.execute() if isinstance(gt_df, pd.Series): @@ -78,10 +77,10 @@ class FactorCodeEvaluator(FactorEvaluator): def evaluate( self, target_task: FactorTask, - implementation: Implementation, + implementation: Workspace, execution_feedback: str, factor_value_feedback: str = "", - gt_implementation: Implementation = None, + gt_implementation: Workspace = None, **kwargs, ): factor_information = target_task.get_task_information() @@ -130,8 +129,8 @@ class FactorCodeEvaluator(FactorEvaluator): class FactorSingleColumnEvaluator(FactorEvaluator): def evaluate( self, - implementation: Implementation, - gt_implementation: Implementation, + implementation: Workspace, + gt_implementation: Workspace, ) -> Tuple[str, object]: _, gen_df = self._get_df(gt_implementation, implementation) @@ -147,8 +146,8 @@ class FactorSingleColumnEvaluator(FactorEvaluator): class FactorOutputFormatEvaluator(FactorEvaluator): def evaluate( self, - implementation: Implementation, - gt_implementation: Implementation, + implementation: Workspace, + gt_implementation: Workspace, ) -> Tuple[str, object]: gt_df, gen_df = self._get_df(gt_implementation, implementation) if gen_df is None: @@ -184,8 +183,8 @@ class FactorOutputFormatEvaluator(FactorEvaluator): class FactorDatetimeDailyEvaluator(FactorEvaluator): def evaluate( self, - implementation: Implementation, - gt_implementation: Implementation, + implementation: Workspace, + gt_implementation: Workspace, ) -> Tuple[str | object]: _, gen_df = self._get_df(gt_implementation, implementation) if gen_df is None: @@ -214,8 +213,8 @@ class FactorDatetimeDailyEvaluator(FactorEvaluator): class FactorRowCountEvaluator(FactorEvaluator): def evaluate( self, - implementation: Implementation, - gt_implementation: Implementation, + implementation: Workspace, + gt_implementation: Workspace, ) -> Tuple[str, object]: gt_df, gen_df = self._get_df(gt_implementation, implementation) @@ -231,8 +230,8 @@ class FactorRowCountEvaluator(FactorEvaluator): class FactorIndexEvaluator(FactorEvaluator): def evaluate( self, - implementation: Implementation, - gt_implementation: Implementation, + implementation: Workspace, + gt_implementation: Workspace, ) -> Tuple[str, object]: gt_df, gen_df = self._get_df(gt_implementation, implementation) @@ -248,8 +247,8 @@ class FactorIndexEvaluator(FactorEvaluator): class FactorMissingValuesEvaluator(FactorEvaluator): def evaluate( self, - implementation: Implementation, - gt_implementation: Implementation, + implementation: Workspace, + gt_implementation: Workspace, ) -> Tuple[str, object]: gt_df, gen_df = self._get_df(gt_implementation, implementation) @@ -265,8 +264,8 @@ class FactorMissingValuesEvaluator(FactorEvaluator): class FactorEqualValueCountEvaluator(FactorEvaluator): def evaluate( self, - implementation: Implementation, - gt_implementation: Implementation, + implementation: Workspace, + gt_implementation: Workspace, ) -> Tuple[str, object]: gt_df, gen_df = self._get_df(gt_implementation, implementation) @@ -296,8 +295,8 @@ class FactorCorrelationEvaluator(FactorEvaluator): def evaluate( self, - implementation: Implementation, - gt_implementation: Implementation, + implementation: Workspace, + gt_implementation: Workspace, ) -> Tuple[str, object]: gt_df, gen_df = self._get_df(gt_implementation, implementation) @@ -327,11 +326,10 @@ class FactorCorrelationEvaluator(FactorEvaluator): class FactorValueEvaluator(FactorEvaluator): - def evaluate( self, - implementation: Implementation, - gt_implementation: Implementation, + implementation: Workspace, + gt_implementation: Workspace, **kwargs, ) -> Tuple: conclusions = [] @@ -508,8 +506,8 @@ class FactorEvaluatorForCoder(FactorEvaluator): def evaluate( self, target_task: FactorTask, - implementation: Implementation, - gt_implementation: Implementation = None, + implementation: Workspace, + gt_implementation: Workspace = None, queried_knowledge: QueriedKnowledge = None, **kwargs, ) -> FactorSingleFeedback: @@ -603,41 +601,21 @@ class FactorMultiEvaluator(Evaluator): queried_knowledge: QueriedKnowledge = None, **kwargs, ) -> FactorMultiFeedback: - multi_implementation_feedback = FactorMultiFeedback() - - # for index in range(len(evo.sub_tasks)): - # corresponding_implementation = evo.sub_implementations[index] - # corresponding_gt_implementation = ( - # evo.sub_gt_implementations[index] if evo.sub_gt_implementations is not None else None - # ) - - # multi_implementation_feedback.append( - # self.single_factor_implementation_evaluator.evaluate( - # target_task=evo.sub_tasks[index], - # implementation=corresponding_implementation, - # gt_implementation=corresponding_gt_implementation, - # queried_knowledge=queried_knowledge, - # ) - # ) - - calls = [] - for index in range(len(evo.sub_tasks)): - corresponding_implementation = evo.sub_implementations[index] - corresponding_gt_implementation = ( - evo.sub_gt_implementations[index] if evo.sub_gt_implementations is not None else None - ) - calls.append( + multi_implementation_feedback = multiprocessing_wrapper( + [ ( self.single_factor_implementation_evaluator.evaluate, ( evo.sub_tasks[index], - corresponding_implementation, - corresponding_gt_implementation, + evo.sub_workspace_list[index], + evo.sub_gt_implementations[index] if evo.sub_gt_implementations is not None else None, queried_knowledge, ), - ), - ) - multi_implementation_feedback = multiprocessing_wrapper(calls, n=FACTOR_IMPLEMENT_SETTINGS.evo_multi_proc_n) + ) + for index in range(len(evo.sub_tasks)) + ], + n=RD_AGENT_SETTINGS.multi_proc_n, + ) final_decision = [ None if single_feedback is None else single_feedback.final_decision diff --git a/rdagent/components/coder/factor_coder/CoSTEER/evolvable_subjects.py b/rdagent/components/coder/factor_coder/CoSTEER/evolvable_subjects.py index 9af63e5f..0a0e4b6d 100644 --- a/rdagent/components/coder/factor_coder/CoSTEER/evolvable_subjects.py +++ b/rdagent/components/coder/factor_coder/CoSTEER/evolvable_subjects.py @@ -1,7 +1,7 @@ from rdagent.components.coder.factor_coder.factor import ( FactorExperiment, + FactorFBWorkspace, FactorTask, - FileBasedFactorImplementation, ) from rdagent.core.evolving_framework import EvolvableSubjects from rdagent.log import rdagent_logger as logger @@ -15,7 +15,7 @@ class FactorEvolvingItem(FactorExperiment, EvolvableSubjects): def __init__( self, sub_tasks: list[FactorTask], - sub_gt_implementations: list[FileBasedFactorImplementation] = None, + sub_gt_implementations: list[FactorFBWorkspace] = None, ): FactorExperiment.__init__(self, sub_tasks=sub_tasks) self.corresponding_selection: list = None diff --git a/rdagent/components/coder/factor_coder/CoSTEER/evolving_agent.py b/rdagent/components/coder/factor_coder/CoSTEER/evolving_agent.py new file mode 100644 index 00000000..feee0ef7 --- /dev/null +++ b/rdagent/components/coder/factor_coder/CoSTEER/evolving_agent.py @@ -0,0 +1,19 @@ +from rdagent.components.coder.factor_coder.CoSTEER.evaluators import FactorMultiFeedback +from rdagent.components.coder.factor_coder.CoSTEER.evolvable_subjects import ( + FactorEvolvingItem, +) +from rdagent.core.evaluation import Feedback +from rdagent.core.evolving_agent import RAGEvoAgent +from rdagent.core.evolving_framework import EvolvableSubjects + + +class FactorRAGEvoAgent(RAGEvoAgent): + def filter_evolvable_subjects_by_feedback(self, evo: EvolvableSubjects, feedback: Feedback) -> EvolvableSubjects: + assert isinstance(evo, FactorEvolvingItem) + assert isinstance(feedback, list) + assert len(evo.sub_workspace_list) == len(feedback) + + for index in range(len(evo.sub_workspace_list)): + if not feedback[index].final_decision: + evo.sub_workspace_list[index].clear() + return evo diff --git a/rdagent/components/coder/factor_coder/CoSTEER/evolving_strategy.py b/rdagent/components/coder/factor_coder/CoSTEER/evolving_strategy.py index aa3a9eb4..97c225ff 100644 --- a/rdagent/components/coder/factor_coder/CoSTEER/evolving_strategy.py +++ b/rdagent/components/coder/factor_coder/CoSTEER/evolving_strategy.py @@ -16,14 +16,11 @@ from rdagent.components.coder.factor_coder.CoSTEER.scheduler import ( LLMSelect, RandomSelect, ) -from rdagent.components.coder.factor_coder.factor import ( - FactorTask, - FileBasedFactorImplementation, -) +from rdagent.components.coder.factor_coder.factor import FactorFBWorkspace, FactorTask from rdagent.components.coder.factor_coder.utils import get_data_folder_intro from rdagent.core.conf import RD_AGENT_SETTINGS from rdagent.core.evolving_framework import EvolvingStrategy, QueriedKnowledge -from rdagent.core.experiment import Implementation +from rdagent.core.experiment import Workspace from rdagent.core.prompts import Prompts from rdagent.core.utils import multiprocessing_wrapper from rdagent.oai.llm_utils import APIBackend @@ -43,7 +40,7 @@ class MultiProcessEvolvingStrategy(EvolvingStrategy): self, target_task: FactorTask, queried_knowledge: QueriedKnowledge = None, - ) -> Implementation: + ) -> Workspace: raise NotImplementedError def evolve( @@ -53,15 +50,12 @@ class MultiProcessEvolvingStrategy(EvolvingStrategy): queried_knowledge: FactorQueriedKnowledge | None = None, **kwargs, ) -> FactorEvolvingItem: - self.num_loop += 1 - new_evo = deepcopy(evo) - # 1.找出需要evolve的factor to_be_finished_task_index = [] - for index, target_factor_task in enumerate(new_evo.sub_tasks): + for index, target_factor_task in enumerate(evo.sub_tasks): target_factor_task_desc = target_factor_task.get_task_information() if target_factor_task_desc in queried_knowledge.success_task_to_knowledge_dict: - new_evo.sub_implementations[index] = queried_knowledge.success_task_to_knowledge_dict[ + evo.sub_workspace_list[index] = queried_knowledge.success_task_to_knowledge_dict[ target_factor_task_desc ].implementation elif ( @@ -87,30 +81,27 @@ class MultiProcessEvolvingStrategy(EvolvingStrategy): to_be_finished_task_index = LLMSelect( to_be_finished_task_index, implementation_factors_per_round, - new_evo, + evo, queried_knowledge.former_traces, self.scen, ) result = multiprocessing_wrapper( [ - (self.implement_one_factor, (new_evo.sub_tasks[target_index], queried_knowledge)) + (self.implement_one_factor, (evo.sub_tasks[target_index], queried_knowledge)) for target_index in to_be_finished_task_index ], - n=FACTOR_IMPLEMENT_SETTINGS.evo_multi_proc_n, + n=RD_AGENT_SETTINGS.multi_proc_n, ) for index, target_index in enumerate(to_be_finished_task_index): - new_evo.sub_implementations[target_index] = result[index] + if evo.sub_workspace_list[target_index] is None: + evo.sub_workspace_list[target_index] = FactorFBWorkspace(target_task=evo.sub_tasks[target_index]) + evo.sub_workspace_list[target_index].inject_code(**{"factor.py": result[index]}) - # for target_index in to_be_finished_task_index: - # new_evo.sub_implementations[target_index] = self.implement_one_factor( - # new_evo.sub_tasks[target_index], queried_knowledge - # ) + evo.corresponding_selection = to_be_finished_task_index - new_evo.corresponding_selection = to_be_finished_task_index - - return new_evo + return evo class FactorEvolvingStrategy(MultiProcessEvolvingStrategy): @@ -118,7 +109,7 @@ class FactorEvolvingStrategy(MultiProcessEvolvingStrategy): self, target_task: FactorTask, queried_knowledge: FactorQueriedKnowledgeV1 = None, - ) -> Implementation: + ) -> str: factor_information_str = target_task.get_task_information() if queried_knowledge is not None and factor_information_str in queried_knowledge.success_task_to_knowledge_dict: @@ -149,7 +140,7 @@ class FactorEvolvingStrategy(MultiProcessEvolvingStrategy): queried_former_failed_knowledge=queried_former_failed_knowledge_to_render, ) ) - session = APIBackend(use_chat_cache=False).build_chat_session( + session = APIBackend(use_chat_cache=FACTOR_IMPLEMENT_SETTINGS.coder_use_cache).build_chat_session( session_system_prompt=system_prompt, ) @@ -185,14 +176,8 @@ class FactorEvolvingStrategy(MultiProcessEvolvingStrategy): json_mode=True, ), )["code"] - # ast.parse(code) - factor_implementation = FileBasedFactorImplementation( - target_task, - ) - factor_implementation.prepare() - factor_implementation.inject_code(**{"factor.py": code}) - return factor_implementation + return code class FactorEvolvingStrategyWithGraph(MultiProcessEvolvingStrategy): @@ -205,7 +190,7 @@ class FactorEvolvingStrategyWithGraph(MultiProcessEvolvingStrategy): self, target_task: FactorTask, queried_knowledge, - ) -> Implementation: + ) -> str: error_summary = FACTOR_IMPLEMENT_SETTINGS.v2_error_summary # 1. 提取因子的背景信息 target_factor_task_information = target_task.get_task_information() @@ -249,7 +234,7 @@ class FactorEvolvingStrategyWithGraph(MultiProcessEvolvingStrategy): ) ) - session = APIBackend(use_chat_cache=False).build_chat_session( + session = APIBackend(use_chat_cache=FACTOR_IMPLEMENT_SETTINGS.coder_use_cache).build_chat_session( session_system_prompt=system_prompt, ) @@ -276,7 +261,9 @@ class FactorEvolvingStrategyWithGraph(MultiProcessEvolvingStrategy): ) .strip("\n") ) - session_summary = APIBackend(use_chat_cache=False).build_chat_session( + session_summary = APIBackend( + use_chat_cache=FACTOR_IMPLEMENT_SETTINGS.coder_use_cache + ).build_chat_session( session_system_prompt=error_summary_system_prompt, ) for _ in range(10): # max attempt to reduce the length of error_summary_user_prompt @@ -335,7 +322,4 @@ class FactorEvolvingStrategyWithGraph(MultiProcessEvolvingStrategy): json_mode=True, ) code = json.loads(response)["code"] - factor_implementation = FileBasedFactorImplementation(target_task) - factor_implementation.prepare() - factor_implementation.inject_code(**{"factor.py": code}) - return factor_implementation + return code diff --git a/rdagent/components/coder/factor_coder/CoSTEER/knowledge_management.py b/rdagent/components/coder/factor_coder/CoSTEER/knowledge_management.py index 8cbf4095..701ae0df 100644 --- a/rdagent/components/coder/factor_coder/CoSTEER/knowledge_management.py +++ b/rdagent/components/coder/factor_coder/CoSTEER/knowledge_management.py @@ -27,9 +27,9 @@ from rdagent.core.evolving_framework import ( QueriedKnowledge, RAGStrategy, ) -from rdagent.core.experiment import Implementation -from rdagent.log import rdagent_logger as logger +from rdagent.core.experiment import Workspace from rdagent.core.prompts import Prompts +from rdagent.log import rdagent_logger as logger from rdagent.oai.llm_utils import ( APIBackend, calculate_embedding_distance_between_str_list, @@ -40,7 +40,7 @@ class FactorKnowledge(Knowledge): def __init__( self, target_task: FactorTask, - implementation: Implementation, + implementation: Workspace, feedback: FactorSingleFeedback, ) -> None: """ @@ -53,7 +53,7 @@ class FactorKnowledge(Knowledge): None """ self.target_task = target_task - self.implementation = implementation + self.implementation = implementation.copy() self.feedback = feedback def get_implementation_and_feedback_str(self) -> str: @@ -115,7 +115,7 @@ class FactorRAGStrategyV1(RAGStrategy): for task_index in range(len(implementations.sub_tasks)): target_task = implementations.sub_tasks[task_index] target_task_information = target_task.get_task_information() - implementation = implementations.sub_implementations[task_index] + implementation = implementations.sub_workspace_list[task_index] single_feedback = feedback[task_index] if single_feedback is None: continue @@ -149,9 +149,9 @@ class FactorRAGStrategyV1(RAGStrategy): for target_factor_task in evo.sub_tasks: target_factor_task_information = target_factor_task.get_task_information() if target_factor_task_information in self.knowledgebase.success_task_info_set: - queried_knowledge.success_task_to_knowledge_dict[target_factor_task_information] = ( - self.knowledgebase.implementation_trace[target_factor_task_information][-1] - ) + queried_knowledge.success_task_to_knowledge_dict[ + target_factor_task_information + ] = self.knowledgebase.implementation_trace[target_factor_task_information][-1] elif ( len( self.knowledgebase.implementation_trace.setdefault( @@ -163,12 +163,14 @@ class FactorRAGStrategyV1(RAGStrategy): ): queried_knowledge.failed_task_info_set.add(target_factor_task_information) else: - queried_knowledge.working_task_to_former_failed_knowledge_dict[target_factor_task_information] = ( - self.knowledgebase.implementation_trace.setdefault( - target_factor_task_information, - [], - )[-v1_query_former_trace_limit:] - ) + queried_knowledge.working_task_to_former_failed_knowledge_dict[ + target_factor_task_information + ] = self.knowledgebase.implementation_trace.setdefault( + target_factor_task_information, + [], + )[ + -v1_query_former_trace_limit: + ] knowledge_base_success_task_list = list( self.knowledgebase.success_task_info_set, @@ -189,9 +191,9 @@ class FactorRAGStrategyV1(RAGStrategy): )[-1] for index in similar_indexes ] - queried_knowledge.working_task_to_similar_successful_knowledge_dict[target_factor_task_information] = ( - similar_successful_knowledge - ) + queried_knowledge.working_task_to_similar_successful_knowledge_dict[ + target_factor_task_information + ] = similar_successful_knowledge return queried_knowledge @@ -234,7 +236,7 @@ class FactorGraphRAGStrategy(RAGStrategy): single_feedback = feedback[task_index] target_task = implementations.sub_tasks[task_index] target_task_information = target_task.get_task_information() - implementation = implementations.sub_implementations[task_index] + implementation = implementations.sub_workspace_list[task_index] single_feedback = feedback[task_index] if single_feedback is None: continue @@ -425,9 +427,9 @@ class FactorGraphRAGStrategy(RAGStrategy): else: current_index += 1 - factor_implementation_queried_graph_knowledge.former_traces[target_factor_task_information] = ( - former_trace_knowledge[-v2_query_former_trace_limit:] - ) + factor_implementation_queried_graph_knowledge.former_traces[ + target_factor_task_information + ] = former_trace_knowledge[-v2_query_former_trace_limit:] else: factor_implementation_queried_graph_knowledge.former_traces[target_factor_task_information] = [] diff --git a/rdagent/components/coder/factor_coder/CoSTEER/scheduler.py b/rdagent/components/coder/factor_coder/CoSTEER/scheduler.py index 42b8a7c8..ed930666 100644 --- a/rdagent/components/coder/factor_coder/CoSTEER/scheduler.py +++ b/rdagent/components/coder/factor_coder/CoSTEER/scheduler.py @@ -9,9 +9,9 @@ from rdagent.components.coder.factor_coder.CoSTEER.evolvable_subjects import ( ) from rdagent.components.coder.factor_coder.utils import get_data_folder_intro from rdagent.core.conf import RD_AGENT_SETTINGS -from rdagent.log import rdagent_logger as logger from rdagent.core.prompts import Prompts from rdagent.core.scenario import Scenario +from rdagent.log import rdagent_logger as logger from rdagent.oai.llm_utils import APIBackend scheduler_prompts = Prompts(file_path=Path(__file__).parent.parent / "prompts.yaml") diff --git a/rdagent/components/coder/factor_coder/config.py b/rdagent/components/coder/factor_coder/config.py index 1adb0d92..cc3c301c 100644 --- a/rdagent/components/coder/factor_coder/config.py +++ b/rdagent/components/coder/factor_coder/config.py @@ -7,16 +7,17 @@ SELECT_METHOD = Literal["random", "scheduler"] class FactorImplementSettings(BaseSettings): - factor_data_folder: str = str( + class Config: + env_prefix = "FACTOR_CODER_" # Use FACTOR_CODER_ as prefix for environment variables + + coder_use_cache: bool = False + data_folder: str = str( (Path().cwd() / "git_ignore_folder" / "factor_implementation_source_data").absolute(), ) - factor_data_folder_debug: str = str( + data_folder_debug: str = str( (Path().cwd() / "git_ignore_folder" / "factor_implementation_source_data_debug").absolute(), ) - factor_execution_workspace: str = str( - (Path().cwd() / "git_ignore_folder" / "factor_implementation_workspace").absolute(), - ) - factor_cache_location: str = str( + cache_location: str = str( (Path().cwd() / "git_ignore_folder" / "factor_implementation_execution_cache").absolute(), ) enable_execution_cache: bool = True # whether to enable the execution cache @@ -35,8 +36,6 @@ class FactorImplementSettings(BaseSettings): v2_error_summary: bool = False v2_knowledge_sampler: float = 1.0 - evo_multi_proc_n: int = 16 # how many processes to use for evolving (including eval & generation) - file_based_execution_timeout: int = 120 # seconds for each factor implementation execution select_method: SELECT_METHOD = "random" @@ -47,5 +46,7 @@ class FactorImplementSettings(BaseSettings): knowledge_base_path: Union[str, None] = None new_knowledge_base_path: Union[str, None] = None + python_bin: str = "python" + FACTOR_IMPLEMENT_SETTINGS = FactorImplementSettings() diff --git a/rdagent/components/coder/factor_coder/factor.py b/rdagent/components/coder/factor_coder/factor.py index 82c193e7..bdf84036 100644 --- a/rdagent/components/coder/factor_coder/factor.py +++ b/rdagent/components/coder/factor_coder/factor.py @@ -15,7 +15,7 @@ from rdagent.core.exception import ( NoOutputException, RuntimeErrorException, ) -from rdagent.core.experiment import Experiment, FBImplementation, Task +from rdagent.core.experiment import Experiment, FBWorkspace, Task from rdagent.log import rdagent_logger as logger from rdagent.oai.llm_utils import md5_hash @@ -51,7 +51,7 @@ variables: {str(self.variables)}""" return f"<{self.__class__.__name__}[{self.factor_name}]>" -class FileBasedFactorImplementation(FBImplementation): +class FactorFBWorkspace(FBWorkspace): """ This class is used to implement a factor by writing the code to a file. Input data and output factor value are also written to files. @@ -90,15 +90,6 @@ class FileBasedFactorImplementation(FBImplementation): check=False, ) - def execute_desc(self): - raise NotImplementedError - - def prepare(self, *args, **kwargs): - self.workspace_path = Path( - FACTOR_IMPLEMENT_SETTINGS.factor_execution_workspace, - ) / str(uuid.uuid4()) - self.workspace_path.mkdir(exist_ok=True, parents=True) - def execute(self, store_result: bool = False, data_type: str = "Debug") -> Tuple[str, pd.DataFrame]: """ execute the implementation and get the factor value by the following steps: @@ -112,6 +103,7 @@ class FileBasedFactorImplementation(FBImplementation): parameters: store_result: if True, store the factor value in the instance variable, this feature is to be used in the gt implementation to avoid multiple execution on the same gt implementation """ + super().execute() if self.code_dict is None or "factor.py" not in self.code_dict: if self.raise_exception: raise CodeFormatException(self.FB_CODE_NOT_SET) @@ -122,8 +114,8 @@ class FileBasedFactorImplementation(FBImplementation): if FACTOR_IMPLEMENT_SETTINGS.enable_execution_cache: # NOTE: cache the result for the same code and same data type target_file_name = md5_hash(data_type + self.code_dict["factor.py"]) - cache_file_path = Path(FACTOR_IMPLEMENT_SETTINGS.factor_cache_location) / f"{target_file_name}.pkl" - Path(FACTOR_IMPLEMENT_SETTINGS.factor_cache_location).mkdir(exist_ok=True, parents=True) + cache_file_path = Path(FACTOR_IMPLEMENT_SETTINGS.cache_location) / f"{target_file_name}.pkl" + Path(FACTOR_IMPLEMENT_SETTINGS.cache_location).mkdir(exist_ok=True, parents=True) if cache_file_path.exists() and not self.raise_exception: cached_res = pickle.load(open(cache_file_path, "rb")) if store_result and cached_res[1] is not None: @@ -135,11 +127,11 @@ class FileBasedFactorImplementation(FBImplementation): source_data_path = ( Path( - FACTOR_IMPLEMENT_SETTINGS.factor_data_folder_debug, + FACTOR_IMPLEMENT_SETTINGS.data_folder_debug, ) if data_type == "Debug" else Path( - FACTOR_IMPLEMENT_SETTINGS.factor_data_folder, + FACTOR_IMPLEMENT_SETTINGS.data_folder, ) ) @@ -151,7 +143,7 @@ class FileBasedFactorImplementation(FBImplementation): execution_feedback = self.FB_EXECUTION_SUCCEEDED try: subprocess.check_output( - f"python {code_path}", + f"{FACTOR_IMPLEMENT_SETTINGS.python_bin} {code_path}", shell=True, cwd=self.workspace_path, stderr=subprocess.STDOUT, @@ -215,7 +207,7 @@ class FileBasedFactorImplementation(FBImplementation): for file_path in path.iterdir(): if file_path.suffix == ".py": code_dict[file_path.name] = file_path.read_text() - return FileBasedFactorImplementation(target_task=task, code_dict=code_dict, **kwargs) + return FactorFBWorkspace(target_task=task, code_dict=code_dict, **kwargs) -class FactorExperiment(Experiment[FactorTask, FileBasedFactorImplementation]): ... +FactorExperiment = Experiment diff --git a/rdagent/components/coder/factor_coder/utils.py b/rdagent/components/coder/factor_coder/utils.py index 283f7624..cfc71457 100644 --- a/rdagent/components/coder/factor_coder/utils.py +++ b/rdagent/components/coder/factor_coder/utils.py @@ -22,7 +22,7 @@ def get_data_folder_intro(): It is for preparing prompting message. """ content_l = [] - for p in Path(FACTOR_IMPLEMENT_SETTINGS.factor_data_folder).iterdir(): + for p in Path(FACTOR_IMPLEMENT_SETTINGS.data_folder).iterdir(): if p.name.endswith(".h5"): df = pd.read_hdf(p) # get df.head() as string with full width diff --git a/rdagent/components/coder/model_coder/CoSTEER/__init__.py b/rdagent/components/coder/model_coder/CoSTEER/__init__.py index a5818987..6c44e96d 100644 --- a/rdagent/components/coder/model_coder/CoSTEER/__init__.py +++ b/rdagent/components/coder/model_coder/CoSTEER/__init__.py @@ -8,6 +8,7 @@ from rdagent.components.coder.model_coder.CoSTEER.evaluators import ( from rdagent.components.coder.model_coder.CoSTEER.evolvable_subjects import ( ModelEvolvingItem, ) +from rdagent.components.coder.model_coder.CoSTEER.evolving_agent import ModelRAGEvoAgent from rdagent.components.coder.model_coder.CoSTEER.evolving_strategy import ( ModelCoderEvolvingStrategy, ) @@ -16,17 +17,18 @@ from rdagent.components.coder.model_coder.CoSTEER.knowledge_management import ( ModelRAGStrategy, ) from rdagent.components.coder.model_coder.model import ModelExperiment +from rdagent.core.developer import Developer from rdagent.core.evolving_agent import RAGEvoAgent -from rdagent.core.task_generator import TaskGenerator -class ModelCoSTEER(TaskGenerator[ModelExperiment]): +class ModelCoSTEER(Developer[ModelExperiment]): def __init__( self, *args, with_knowledge: bool = True, with_feedback: bool = True, knowledge_self_gen: bool = True, + filter_final_evo: bool = True, **kwargs, ) -> None: super().__init__(*args, **kwargs) @@ -44,6 +46,7 @@ class ModelCoSTEER(TaskGenerator[ModelExperiment]): self.with_knowledge = with_knowledge self.with_feedback = with_feedback self.knowledge_self_gen = knowledge_self_gen + self.filter_final_evo = filter_final_evo self.evolving_strategy = ModelCoderEvolvingStrategy(scen=self.scen) self.model_evaluator = ModelCoderMultiEvaluator(scen=self.scen) @@ -57,7 +60,7 @@ class ModelCoSTEER(TaskGenerator[ModelExperiment]): return model_knowledge_base - def generate(self, exp: ModelExperiment) -> ModelExperiment: + def develop(self, exp: ModelExperiment) -> ModelExperiment: # init knowledge base model_knowledge_base = self.load_or_init_knowledge_base( former_knowledge_base_path=self.knowledge_base_path, @@ -69,7 +72,9 @@ class ModelCoSTEER(TaskGenerator[ModelExperiment]): # init intermediate items model_experiment = ModelEvolvingItem(sub_tasks=exp.sub_tasks) - self.evolve_agent = RAGEvoAgent(max_loop=self.max_loop, evolving_strategy=self.evolving_strategy, rag=self.rag) + self.evolve_agent = ModelRAGEvoAgent( + max_loop=self.max_loop, evolving_strategy=self.evolving_strategy, rag=self.rag + ) model_experiment = self.evolve_agent.multistep_evolve( model_experiment, @@ -77,11 +82,11 @@ class ModelCoSTEER(TaskGenerator[ModelExperiment]): with_knowledge=self.with_knowledge, with_feedback=self.with_feedback, knowledge_self_gen=self.knowledge_self_gen, + filter_final_evo=self.filter_final_evo, ) # save new knowledge base if self.new_knowledge_base_path is not None: pickle.dump(model_knowledge_base, open(self.new_knowledge_base_path, "wb")) - self.knowledge_base = model_knowledge_base - model_experiment.based_experiments = exp.based_experiments - return model_experiment + exp.sub_workspace_list = model_experiment.sub_workspace_list + return exp diff --git a/rdagent/components/coder/model_coder/CoSTEER/evaluators.py b/rdagent/components/coder/model_coder/CoSTEER/evaluators.py index 9ae642d4..38a8a5d4 100644 --- a/rdagent/components/coder/model_coder/CoSTEER/evaluators.py +++ b/rdagent/components/coder/model_coder/CoSTEER/evaluators.py @@ -11,14 +11,14 @@ from rdagent.components.coder.model_coder.conf import MODEL_IMPL_SETTINGS from rdagent.components.coder.model_coder.CoSTEER.evolvable_subjects import ( ModelEvolvingItem, ) -from rdagent.components.coder.model_coder.model import ModelImplementation, ModelTask +from rdagent.components.coder.model_coder.model import ModelFBWorkspace, ModelTask from rdagent.core.conf import RD_AGENT_SETTINGS from rdagent.core.evaluation import Evaluator from rdagent.core.evolving_framework import QueriedKnowledge -from rdagent.core.experiment import Implementation, Task -from rdagent.log import rdagent_logger as logger +from rdagent.core.experiment import Task, Workspace from rdagent.core.prompts import Prompts from rdagent.core.utils import multiprocessing_wrapper +from rdagent.log import rdagent_logger as logger from rdagent.oai.llm_utils import APIBackend evaluate_prompts = Prompts(file_path=Path(__file__).parent.parent / "prompts.yaml") @@ -62,15 +62,15 @@ class ModelCodeEvaluator(Evaluator): def evaluate( self, target_task: Task, - implementation: Implementation, - gt_implementation: Implementation, + implementation: Workspace, + gt_implementation: Workspace, model_execution_feedback: str = "", model_value_feedback: str = "", ): assert isinstance(target_task, ModelTask) - assert isinstance(implementation, ModelImplementation) + assert isinstance(implementation, ModelFBWorkspace) if gt_implementation is not None: - assert isinstance(gt_implementation, ModelImplementation) + assert isinstance(gt_implementation, ModelFBWorkspace) model_task_information = target_task.get_task_information() code = implementation.code @@ -120,16 +120,16 @@ class ModelFinalEvaluator(Evaluator): def evaluate( self, target_task: Task, - implementation: Implementation, - gt_implementation: Implementation, + implementation: Workspace, + gt_implementation: Workspace, model_execution_feedback: str, model_value_feedback: str, model_code_feedback: str, ): assert isinstance(target_task, ModelTask) - assert isinstance(implementation, ModelImplementation) + assert isinstance(implementation, ModelFBWorkspace) if gt_implementation is not None: - assert isinstance(gt_implementation, ModelImplementation) + assert isinstance(gt_implementation, ModelFBWorkspace) system_prompt = ( Environment(undefined=StrictUndefined) @@ -219,8 +219,8 @@ class ModelCoderEvaluator(Evaluator): def evaluate( self, target_task: Task, - implementation: Implementation, - gt_implementation: Implementation, + implementation: Workspace, + gt_implementation: Workspace, queried_knowledge: QueriedKnowledge = None, **kwargs, ) -> ModelCoderFeedback: @@ -248,7 +248,7 @@ class ModelCoderEvaluator(Evaluator): input_value = 0.4 param_init_value = 0.6 - assert isinstance(implementation, ModelImplementation) + assert isinstance(implementation, ModelFBWorkspace) model_execution_feedback, gen_tensor = implementation.execute( batch_size=batch_size, num_features=num_features, @@ -257,7 +257,7 @@ class ModelCoderEvaluator(Evaluator): param_init_value=param_init_value, ) if gt_implementation is not None: - assert isinstance(gt_implementation, ModelImplementation) + assert isinstance(gt_implementation, ModelFBWorkspace) _, gt_tensor = gt_implementation.execute( batch_size=batch_size, num_features=num_features, @@ -303,26 +303,21 @@ class ModelCoderMultiEvaluator(Evaluator): queried_knowledge: QueriedKnowledge = None, **kwargs, ) -> List[ModelCoderFeedback]: - multi_implementation_feedback = [] - - calls = [] - for index in range(len(evo.sub_tasks)): - corresponding_implementation = evo.sub_implementations[index] - corresponding_gt_implementation = ( - evo.sub_gt_implementations[index] if evo.sub_gt_implementations is not None else None - ) - calls.append( + multi_implementation_feedback = multiprocessing_wrapper( + [ ( ModelCoderEvaluator(scen=self.scen).evaluate, ( evo.sub_tasks[index], - corresponding_implementation, - corresponding_gt_implementation, + evo.sub_workspace_list[index], + evo.sub_gt_implementations[index] if evo.sub_gt_implementations is not None else None, queried_knowledge, ), - ), - ) - multi_implementation_feedback = multiprocessing_wrapper(calls, n=MODEL_IMPL_SETTINGS.evo_multi_proc_n) + ) + for index in range(len(evo.sub_tasks)) + ], + n=RD_AGENT_SETTINGS.multi_proc_n, + ) final_decision = [ None if single_feedback is None else single_feedback.final_decision diff --git a/rdagent/components/coder/model_coder/CoSTEER/evolvable_subjects.py b/rdagent/components/coder/model_coder/CoSTEER/evolvable_subjects.py index 8ed10dd5..3729f751 100644 --- a/rdagent/components/coder/model_coder/CoSTEER/evolvable_subjects.py +++ b/rdagent/components/coder/model_coder/CoSTEER/evolvable_subjects.py @@ -1,6 +1,6 @@ from rdagent.components.coder.model_coder.model import ( ModelExperiment, - ModelImplementation, + ModelFBWorkspace, ModelTask, ) from rdagent.core.evolving_framework import EvolvableSubjects @@ -15,7 +15,7 @@ class ModelEvolvingItem(ModelExperiment, EvolvableSubjects): def __init__( self, sub_tasks: list[ModelTask], - sub_gt_implementations: list[ModelImplementation] = None, + sub_gt_implementations: list[ModelFBWorkspace] = None, ): ModelExperiment.__init__(self, sub_tasks=sub_tasks) if sub_gt_implementations is not None and len( diff --git a/rdagent/components/coder/model_coder/CoSTEER/evolving_agent.py b/rdagent/components/coder/model_coder/CoSTEER/evolving_agent.py new file mode 100644 index 00000000..34efa7cd --- /dev/null +++ b/rdagent/components/coder/model_coder/CoSTEER/evolving_agent.py @@ -0,0 +1,19 @@ +from rdagent.components.coder.model_coder.CoSTEER.evaluators import ModelCoderFeedback +from rdagent.components.coder.model_coder.CoSTEER.evolvable_subjects import ( + ModelEvolvingItem, +) +from rdagent.core.evaluation import Feedback +from rdagent.core.evolving_agent import RAGEvoAgent +from rdagent.core.evolving_framework import EvolvableSubjects + + +class ModelRAGEvoAgent(RAGEvoAgent): + def filter_evolvable_subjects_by_feedback(self, evo: EvolvableSubjects, feedback: Feedback) -> EvolvableSubjects: + assert isinstance(evo, ModelEvolvingItem) + assert isinstance(feedback, list) + assert len(evo.sub_workspace_list) == len(feedback) + + for index in range(len(evo.sub_workspace_list)): + if not feedback[index].final_decision: + evo.sub_workspace_list[index].clear() + return evo diff --git a/rdagent/components/coder/model_coder/CoSTEER/evolving_strategy.py b/rdagent/components/coder/model_coder/CoSTEER/evolving_strategy.py index 07a6162c..f824c2ba 100644 --- a/rdagent/components/coder/model_coder/CoSTEER/evolving_strategy.py +++ b/rdagent/components/coder/model_coder/CoSTEER/evolving_strategy.py @@ -11,7 +11,7 @@ from rdagent.components.coder.model_coder.CoSTEER.evolvable_subjects import ( from rdagent.components.coder.model_coder.CoSTEER.knowledge_management import ( ModelQueriedKnowledge, ) -from rdagent.components.coder.model_coder.model import ModelImplementation, ModelTask +from rdagent.components.coder.model_coder.model import ModelFBWorkspace, ModelTask from rdagent.core.conf import RD_AGENT_SETTINGS from rdagent.core.evolving_framework import EvolvingStrategy from rdagent.core.prompts import Prompts @@ -26,7 +26,7 @@ class ModelCoderEvolvingStrategy(EvolvingStrategy): self, target_task: ModelTask, queried_knowledge: ModelQueriedKnowledge = None, - ) -> ModelImplementation: + ) -> str: model_information_str = target_task.get_task_information() if queried_knowledge is not None and model_information_str in queried_knowledge.success_task_to_knowledge_dict: @@ -86,19 +86,15 @@ class ModelCoderEvolvingStrategy(EvolvingStrategy): queried_similar_successful_knowledge_to_render = queried_similar_successful_knowledge_to_render[1:] code = json.loads( - APIBackend(use_chat_cache=False).build_messages_and_create_chat_completion( + APIBackend( + use_chat_cache=MODEL_IMPL_SETTINGS.coder_use_cache + ).build_messages_and_create_chat_completion( user_prompt=user_prompt, system_prompt=system_prompt, json_mode=True, ), )["code"] - model_implementation = ModelImplementation( - target_task, - ) - model_implementation.prepare() - model_implementation.inject_code(**{"model.py": code}) - - return model_implementation + return code def evolve( self, @@ -107,14 +103,12 @@ class ModelCoderEvolvingStrategy(EvolvingStrategy): queried_knowledge: ModelQueriedKnowledge | None = None, **kwargs, ) -> ModelEvolvingItem: - new_evo = deepcopy(evo) - # 1.找出需要evolve的model to_be_finished_task_index = [] - for index, target_model_task in enumerate(new_evo.sub_tasks): + for index, target_model_task in enumerate(evo.sub_tasks): target_model_task_desc = target_model_task.get_task_information() if target_model_task_desc in queried_knowledge.success_task_to_knowledge_dict: - new_evo.sub_implementations[index] = queried_knowledge.success_task_to_knowledge_dict[ + evo.sub_workspace_list[index] = queried_knowledge.success_task_to_knowledge_dict[ target_model_task_desc ].implementation elif ( @@ -125,20 +119,17 @@ class ModelCoderEvolvingStrategy(EvolvingStrategy): result = multiprocessing_wrapper( [ - (self.implement_one_model, (new_evo.sub_tasks[target_index], queried_knowledge)) + (self.implement_one_model, (evo.sub_tasks[target_index], queried_knowledge)) for target_index in to_be_finished_task_index ], - n=MODEL_IMPL_SETTINGS.evo_multi_proc_n, + n=RD_AGENT_SETTINGS.multi_proc_n, ) for index, target_index in enumerate(to_be_finished_task_index): - new_evo.sub_implementations[target_index] = result[index] + if evo.sub_workspace_list[target_index] is None: + evo.sub_workspace_list[target_index] = ModelFBWorkspace(target_task=evo.sub_tasks[target_index]) + evo.sub_workspace_list[target_index].inject_code(**{"model.py": result[index]}) - # for target_index in to_be_finished_task_index: - # new_evo.sub_implementations[target_index] = self.implement_one_model( - # new_evo.sub_tasks[target_index], queried_knowledge - # ) + evo.corresponding_selection = to_be_finished_task_index - new_evo.corresponding_selection = to_be_finished_task_index - - return new_evo + return evo diff --git a/rdagent/components/coder/model_coder/CoSTEER/knowledge_management.py b/rdagent/components/coder/model_coder/CoSTEER/knowledge_management.py index a706bf65..bd88eb4c 100644 --- a/rdagent/components/coder/model_coder/CoSTEER/knowledge_management.py +++ b/rdagent/components/coder/model_coder/CoSTEER/knowledge_management.py @@ -9,7 +9,7 @@ from rdagent.core.evolving_framework import ( QueriedKnowledge, RAGStrategy, ) -from rdagent.core.experiment import Implementation +from rdagent.core.experiment import Workspace from rdagent.oai.llm_utils import calculate_embedding_distance_between_str_list @@ -17,7 +17,7 @@ class ModelKnowledge(Knowledge): def __init__( self, target_task: ModelTask, - implementation: Implementation, + implementation: Workspace, feedback: ModelCoderFeedback, ) -> None: """ @@ -30,7 +30,7 @@ class ModelKnowledge(Knowledge): None """ self.target_task = target_task - self.implementation = implementation + self.implementation = implementation.copy() self.feedback = feedback def get_implementation_and_feedback_str(self) -> str: @@ -87,7 +87,7 @@ class ModelRAGStrategy(RAGStrategy): for task_index in range(len(implementations.sub_tasks)): target_task = implementations.sub_tasks[task_index] target_task_information = target_task.get_task_information() - implementation = implementations.sub_implementations[task_index] + implementation = implementations.sub_workspace_list[task_index] single_feedback = feedback[task_index] if single_feedback is None: continue @@ -121,9 +121,9 @@ class ModelRAGStrategy(RAGStrategy): for target_model_task in evo.sub_tasks: target_model_task_information = target_model_task.get_task_information() if target_model_task_information in self.knowledgebase.success_task_info_set: - queried_knowledge.success_task_to_knowledge_dict[target_model_task_information] = ( - self.knowledgebase.implementation_trace[target_model_task_information][-1] - ) + queried_knowledge.success_task_to_knowledge_dict[ + target_model_task_information + ] = self.knowledgebase.implementation_trace[target_model_task_information][-1] elif ( len( self.knowledgebase.implementation_trace.setdefault( @@ -135,12 +135,14 @@ class ModelRAGStrategy(RAGStrategy): ): queried_knowledge.failed_task_info_set.add(target_model_task_information) else: - queried_knowledge.working_task_to_former_failed_knowledge_dict[target_model_task_information] = ( - self.knowledgebase.implementation_trace.setdefault( - target_model_task_information, - [], - )[-query_former_trace_limit:] - ) + queried_knowledge.working_task_to_former_failed_knowledge_dict[ + target_model_task_information + ] = self.knowledgebase.implementation_trace.setdefault( + target_model_task_information, + [], + )[ + -query_former_trace_limit: + ] knowledge_base_success_task_list = list( self.knowledgebase.success_task_info_set, @@ -161,7 +163,7 @@ class ModelRAGStrategy(RAGStrategy): )[-1] for index in similar_indexes ] - queried_knowledge.working_task_to_similar_successful_knowledge_dict[target_model_task_information] = ( - similar_successful_knowledge - ) + queried_knowledge.working_task_to_similar_successful_knowledge_dict[ + target_model_task_information + ] = similar_successful_knowledge return queried_knowledge diff --git a/rdagent/components/coder/model_coder/benchmark/eval.py b/rdagent/components/coder/model_coder/benchmark/eval.py index 77f924e3..b179262a 100644 --- a/rdagent/components/coder/model_coder/benchmark/eval.py +++ b/rdagent/components/coder/model_coder/benchmark/eval.py @@ -1,7 +1,7 @@ # TODO: inherent from the benchmark base class import torch -from rdagent.components.coder.model_coder.model import ModelImplementation +from rdagent.components.coder.model_coder.model import ModelFBWorkspace def get_data_conf(init_val): @@ -32,7 +32,7 @@ class ModelImpValEval: For each hidden output, we can calculate a correlation. The average correlation will be the metrics. """ - def evaluate(self, gt: ModelImplementation, gen: ModelImplementation): + def evaluate(self, gt: ModelFBWorkspace, gen: ModelFBWorkspace): round_n = 10 eval_pairs: list[tuple] = [] diff --git a/rdagent/components/coder/model_coder/conf.py b/rdagent/components/coder/model_coder/conf.py index 70af7d29..ea45041b 100644 --- a/rdagent/components/coder/model_coder/conf.py +++ b/rdagent/components/coder/model_coder/conf.py @@ -6,12 +6,11 @@ from pydantic_settings import BaseSettings class ModelImplSettings(BaseSettings): class Config: - env_prefix = "MODEL_IMPL_" # Use MODEL_IMPL_ as prefix for environment variables + env_prefix = "MODEL_CODER_" # Use MODEL_CODER_ as prefix for environment variables - model_execution_workspace: str = str( - (Path().cwd() / "git_ignore_folder" / "model_implementation_workspace").absolute(), - ) - model_cache_location: str = str( + coder_use_cache: bool = False + + cache_location: str = str( (Path().cwd() / "git_ignore_folder" / "model_implementation_execution_cache").absolute(), ) @@ -24,8 +23,6 @@ class ModelImplSettings(BaseSettings): query_similar_success_limit: int = 5 fail_task_trial_limit: int = 20 - evo_multi_proc_n: int = 1 - enable_execution_cache: bool = True # whether to enable the execution cache diff --git a/rdagent/components/coder/model_coder/main.py b/rdagent/components/coder/model_coder/main.py index 88466ba0..54a0c102 100644 --- a/rdagent/components/coder/model_coder/main.py +++ b/rdagent/components/coder/model_coder/main.py @@ -10,6 +10,10 @@ import os import torch from dotenv import load_dotenv +from rdagent.components.coder.model_coder.CoSTEER.evaluators import ( + shape_evaluator, + value_evaluator, +) from rdagent.oai.llm_utils import APIBackend assert load_dotenv() @@ -68,8 +72,6 @@ for test_mode in ["zeros", "ones", "randn"]: os.system("rm node_features.pt") # load the output and print the shape - from evaluator import shape_evaluator, value_evaluator - try: llm_output = torch.load("llm_output.pt") except: @@ -80,7 +82,7 @@ for test_mode in ["zeros", "ones", "randn"]: average_value_eval.append(value_evaluator(llm_output, gt_output)[1]) print("Shape evaluation: ", average_shape_eval[-1]) - print("Value evaluation:super().generate(task_l) ", average_value_eval[-1]) + print("Value evaluation:super().develop(task_l) ", average_value_eval[-1]) os.system("rm llm_output.pt") os.system("rm gt_output.pt") diff --git a/rdagent/components/coder/model_coder/model.py b/rdagent/components/coder/model_coder/model.py index e80272d2..4fa636f7 100644 --- a/rdagent/components/coder/model_coder/model.py +++ b/rdagent/components/coder/model_coder/model.py @@ -9,7 +9,7 @@ import torch from rdagent.components.coder.model_coder.conf import MODEL_IMPL_SETTINGS from rdagent.core.exception import CodeFormatException -from rdagent.core.experiment import Experiment, FBImplementation, Task +from rdagent.core.experiment import Experiment, FBWorkspace, Task from rdagent.oai.llm_utils import md5_hash from rdagent.utils import get_module_by_module_path @@ -40,7 +40,7 @@ model_type: {self.model_type} return f"<{self.__class__.__name__} {self.name}>" -class ModelImplementation(FBImplementation): +class ModelFBWorkspace(FBWorkspace): """ It is a Pytorch model implementation task; All the things are placed in a folder. @@ -60,18 +60,6 @@ class ModelImplementation(FBImplementation): """ - def __init__(self, target_task: Task) -> None: - super().__init__(target_task) - - def prepare(self) -> None: - """ - Prepare for the workspace; - """ - unique_id = uuid.uuid4() - self.workspace_path = Path(MODEL_IMPL_SETTINGS.model_execution_workspace) / f"M{unique_id}" - # start with `M` so that it can be imported via python - self.workspace_path.mkdir(parents=True, exist_ok=True) - def execute( self, batch_size: int = 8, @@ -80,14 +68,15 @@ class ModelImplementation(FBImplementation): input_value: float = 1.0, param_init_value: float = 1.0, ): + super().execute() try: if MODEL_IMPL_SETTINGS.enable_execution_cache: # NOTE: cache the result for the same code target_file_name = md5_hash( f"{batch_size}_{num_features}_{num_timesteps}_{input_value}_{param_init_value}_{self.code_dict['model.py']}" ) - cache_file_path = Path(MODEL_IMPL_SETTINGS.model_cache_location) / f"{target_file_name}.pkl" - Path(MODEL_IMPL_SETTINGS.model_cache_location).mkdir(exist_ok=True, parents=True) + cache_file_path = Path(MODEL_IMPL_SETTINGS.cache_location) / f"{target_file_name}.pkl" + Path(MODEL_IMPL_SETTINGS.cache_location).mkdir(exist_ok=True, parents=True) if cache_file_path.exists(): return pickle.load(open(cache_file_path, "rb")) mod = get_module_by_module_path(str(self.workspace_path / "model.py")) @@ -115,4 +104,4 @@ class ModelImplementation(FBImplementation): return f"Execution error: {e}", None -class ModelExperiment(Experiment[ModelTask, ModelImplementation]): ... +ModelExperiment = Experiment diff --git a/rdagent/components/coder/model_coder/one_shot/__init__.py b/rdagent/components/coder/model_coder/one_shot/__init__.py index 2b3c7dce..7f7fa83e 100644 --- a/rdagent/components/coder/model_coder/one_shot/__init__.py +++ b/rdagent/components/coder/model_coder/one_shot/__init__.py @@ -3,22 +3,19 @@ from pathlib import Path from jinja2 import Environment, StrictUndefined -from rdagent.components.coder.model_coder.model import ( - ModelExperiment, - ModelImplementation, -) +from rdagent.components.coder.model_coder.model import ModelExperiment, ModelFBWorkspace +from rdagent.core.developer import Developer from rdagent.core.prompts import Prompts -from rdagent.core.task_generator import TaskGenerator from rdagent.oai.llm_utils import APIBackend DIRNAME = Path(__file__).absolute().resolve().parent -class ModelCodeWriter(TaskGenerator[ModelExperiment]): - def generate(self, exp: ModelExperiment) -> ModelExperiment: +class ModelCodeWriter(Developer[ModelExperiment]): + def develop(self, exp: ModelExperiment) -> ModelExperiment: mti_l = [] for t in exp.sub_tasks: - mti = ModelImplementation(t) + mti = ModelFBWorkspace(t) mti.prepare() pr = Prompts(file_path=DIRNAME / "prompt.yaml") @@ -40,5 +37,5 @@ class ModelCodeWriter(TaskGenerator[ModelExperiment]): code = match.group(1) mti.inject_code(**{"model.py": code}) mti_l.append(mti) - exp.sub_implementations = mti_l + exp.sub_workspace_list = mti_l return exp diff --git a/rdagent/components/coder/model_coder/task_loader.py b/rdagent/components/coder/model_coder/task_loader.py index 95cda9fb..5929f10e 100644 --- a/rdagent/components/coder/model_coder/task_loader.py +++ b/rdagent/components/coder/model_coder/task_loader.py @@ -9,12 +9,14 @@ from rdagent.components.document_reader.document_reader import ( load_and_process_pdfs_by_langchain, ) from rdagent.components.loader.task_loader import ModelTaskLoader -from rdagent.log import rdagent_logger as logger from rdagent.core.prompts import Prompts +from rdagent.log import rdagent_logger as logger from rdagent.oai.llm_utils import APIBackend +from rdagent.scenarios.qlib.experiment.model_experiment import QlibModelExperiment document_process_prompts = Prompts(file_path=Path(__file__).parent / "prompts.yaml") + def extract_model_from_doc(doc_content: str) -> dict: """ Extract model information from document content. @@ -107,7 +109,7 @@ class ModelExperimentLoaderFromDict(ModelTaskLoader): key=model_name, ) task_l.append(task) - return ModelExperiment(sub_tasks=task_l) + return QlibModelExperiment(sub_tasks=task_l) class ModelExperimentLoaderFromPDFfiles(ModelTaskLoader): diff --git a/rdagent/components/knowledge_management/vector_base.py b/rdagent/components/knowledge_management/vector_base.py index 17ca514b..c0d72e5e 100644 --- a/rdagent/components/knowledge_management/vector_base.py +++ b/rdagent/components/knowledge_management/vector_base.py @@ -8,6 +8,7 @@ from scipy.spatial.distance import cosine from rdagent.log import rdagent_logger as logger from rdagent.oai.llm_utils import APIBackend + class KnowledgeMetaData: def __init__(self, content: str = "", label: str = None, embedding=None, identity=None): self.label = label diff --git a/rdagent/components/loader/task_loader.py b/rdagent/components/loader/task_loader.py index a67262c0..a3b34450 100644 --- a/rdagent/components/loader/task_loader.py +++ b/rdagent/components/loader/task_loader.py @@ -3,8 +3,8 @@ from pathlib import Path from typing import Sequence from rdagent.components.coder.factor_coder.factor import FactorTask -from rdagent.components.coder.model_coder.model import ModelImplementation, ModelTask -from rdagent.core.experiment import ImpLoader, Loader +from rdagent.components.coder.model_coder.model import ModelFBWorkspace, ModelTask +from rdagent.core.experiment import Loader, WsLoader class FactorTaskLoader(Loader[FactorTask]): @@ -80,16 +80,15 @@ class ModelTaskLoaderJson(ModelTaskLoader): return model_impl_task_list -class ModelImpLoader(ImpLoader[ModelTask, ModelImplementation]): +class ModelWsLoader(WsLoader[ModelTask, ModelFBWorkspace]): def __init__(self, path: Path) -> None: self.path = Path(path) - def load(self, task: ModelTask) -> ModelImplementation: + def load(self, task: ModelTask) -> ModelFBWorkspace: assert task.name is not None - mti = ModelImplementation(task) + mti = ModelFBWorkspace(task) mti.prepare() with open(self.path / f"{task.name}.py", "r") as f: code = f.read() mti.inject_code(**{"model.py": code}) return mti - return mti diff --git a/rdagent/components/proposal/factor_proposal.py b/rdagent/components/proposal/factor_proposal.py index cc1dbc7c..8ec66efc 100644 --- a/rdagent/components/proposal/factor_proposal.py +++ b/rdagent/components/proposal/factor_proposal.py @@ -27,10 +27,12 @@ class FactorHypothesisGen(HypothesisGen): # The following methods are scenario related so they should be implemented in the subclass @abstractmethod - def prepare_context(self, trace: Trace) -> Tuple[dict, bool]: ... + def prepare_context(self, trace: Trace) -> Tuple[dict, bool]: + ... @abstractmethod - def convert_response(self, response: str) -> FactorHypothesis: ... + def convert_response(self, response: str) -> FactorHypothesis: + ... def gen(self, trace: Trace) -> FactorHypothesis: context_dict, json_flag = self.prepare_context(trace) @@ -62,12 +64,13 @@ class FactorHypothesisGen(HypothesisGen): class FactorHypothesis2Experiment(Hypothesis2Experiment[FactorExperiment]): + @abstractmethod + def prepare_context(self, hypothesis: Hypothesis, trace: Trace) -> Tuple[dict, bool]: + ... @abstractmethod - def prepare_context(self, hypothesis: Hypothesis, trace: Trace) -> Tuple[dict, bool]: ... - - @abstractmethod - def convert_response(self, response: str, trace: Trace) -> FactorExperiment: ... + def convert_response(self, response: str, trace: Trace) -> FactorExperiment: + ... def convert(self, hypothesis: Hypothesis, trace: Trace) -> FactorExperiment: context, json_flag = self.prepare_context(hypothesis, trace) diff --git a/rdagent/components/proposal/model_proposal.py b/rdagent/components/proposal/model_proposal.py index 31f1711b..c5f2d8b7 100644 --- a/rdagent/components/proposal/model_proposal.py +++ b/rdagent/components/proposal/model_proposal.py @@ -21,13 +21,14 @@ ModelHypothesis = Hypothesis class ModelHypothesisGen(HypothesisGen): - # The following methods are scenario related so they should be implemented in the subclass @abstractmethod - def prepare_context(self, trace: Trace) -> Tuple[dict, bool]: ... + def prepare_context(self, trace: Trace) -> Tuple[dict, bool]: + ... @abstractmethod - def convert_response(self, response: str) -> ModelHypothesis: ... + def convert_response(self, response: str) -> ModelHypothesis: + ... def gen(self, trace: Trace) -> ModelHypothesis: context_dict, json_flag = self.prepare_context(trace) @@ -63,10 +64,12 @@ class ModelHypothesis2Experiment(Hypothesis2Experiment[ModelExperiment]): super().__init__() @abstractmethod - def prepare_context(self, hypothesis: Hypothesis, trace: Trace) -> Tuple[dict, bool]: ... + def prepare_context(self, hypothesis: Hypothesis, trace: Trace) -> Tuple[dict, bool]: + ... @abstractmethod - def convert_response(self, response: str, trace: Trace) -> ModelExperiment: ... + def convert_response(self, response: str, trace: Trace) -> ModelExperiment: + ... def convert(self, hypothesis: Hypothesis, trace: Trace) -> ModelExperiment: context, json_flag = self.prepare_context(hypothesis, trace) diff --git a/rdagent/components/runner/__init__.py b/rdagent/components/runner/__init__.py index d331f18b..997bcd6a 100644 --- a/rdagent/components/runner/__init__.py +++ b/rdagent/components/runner/__init__.py @@ -3,12 +3,12 @@ from pathlib import Path from typing import Tuple from rdagent.components.runner.conf import RUNNER_SETTINGS +from rdagent.core.developer import Developer from rdagent.core.experiment import ASpecificExp, Experiment -from rdagent.core.task_generator import TaskGenerator from rdagent.oai.llm_utils import md5_hash -class CachedRunner(TaskGenerator[ASpecificExp]): +class CachedRunner(Developer[ASpecificExp]): def get_cache_key(self, exp: Experiment) -> str: all_tasks = [] for based_exp in exp.based_experiments: @@ -20,8 +20,8 @@ class CachedRunner(TaskGenerator[ASpecificExp]): def get_cache_result(self, exp: Experiment) -> Tuple[bool, object]: task_info_key = self.get_cache_key(exp) - Path(RUNNER_SETTINGS.runner_cache_path).mkdir(parents=True, exist_ok=True) - cache_path = Path(RUNNER_SETTINGS.runner_cache_path) / f"{task_info_key}.pkl" + Path(RUNNER_SETTINGS.cache_path).mkdir(parents=True, exist_ok=True) + cache_path = Path(RUNNER_SETTINGS.cache_path) / f"{task_info_key}.pkl" if cache_path.exists(): return True, pickle.load(open(cache_path, "rb")) else: @@ -29,5 +29,5 @@ class CachedRunner(TaskGenerator[ASpecificExp]): def dump_cache_result(self, exp: Experiment, result: object): task_info_key = self.get_cache_key(exp) - cache_path = Path(RUNNER_SETTINGS.runner_cache_path) / f"{task_info_key}.pkl" + cache_path = Path(RUNNER_SETTINGS.cache_path) / f"{task_info_key}.pkl" pickle.dump(result, open(cache_path, "wb")) diff --git a/rdagent/components/runner/conf.py b/rdagent/components/runner/conf.py index b7575d57..e7670ace 100644 --- a/rdagent/components/runner/conf.py +++ b/rdagent/components/runner/conf.py @@ -12,8 +12,11 @@ from pydantic_settings import BaseSettings class RunnerSettings(BaseSettings): - runner_cache_result: bool = True # whether to cache the result of the docker execution - runner_cache_path: str = str(Path.cwd() / "runner_cache/") # the path to store the cache + class Config: + env_prefix = "RUNNER_" # Use MODEL_CODER_ as prefix for environment variables + + cache_result: bool = True # whether to cache the result of the docker execution + cache_path: str = str(Path.cwd() / "runner_cache/") # the path to store the cache RUNNER_SETTINGS = RunnerSettings() diff --git a/rdagent/core/conf.py b/rdagent/core/conf.py index f709d575..edf4aace 100644 --- a/rdagent/core/conf.py +++ b/rdagent/core/conf.py @@ -18,7 +18,7 @@ class RDAgentSettings(BaseSettings): # TODO: (xiao) I think LLMSetting may be a better name. # TODO: (xiao) I think most of the config should be in oai.config # Log configs - # TODO: (xiao) think it can be a seperate config. + # TODO: (xiao) think it can be a separate config. log_trace_path: str | None = None log_llm_chat_content: bool = True @@ -100,5 +100,11 @@ class RDAgentSettings(BaseSettings): max_input_duplicate_factor_group: int = 600 max_output_duplicate_factor_group: int = 20 + # workspace conf + workspace_path: Path = Path.cwd() / "git_ignore_folder" / "RD-Agent_workspace" + + # multi processing conf + multi_proc_n: int = 1 + RD_AGENT_SETTINGS = RDAgentSettings() diff --git a/rdagent/core/task_generator.py b/rdagent/core/developer.py similarity index 58% rename from rdagent/core/task_generator.py rename to rdagent/core/developer.py index 27c91ac4..15227fb2 100644 --- a/rdagent/core/task_generator.py +++ b/rdagent/core/developer.py @@ -5,12 +5,12 @@ from rdagent.core.experiment import ASpecificExp from rdagent.core.scenario import Scenario -class TaskGenerator(ABC, Generic[ASpecificExp]): +class Developer(ABC, Generic[ASpecificExp]): def __init__(self, scen: Scenario) -> None: self.scen: Scenario = scen @abstractmethod - def generate(self, exp: ASpecificExp) -> ASpecificExp: + def develop(self, exp: ASpecificExp) -> ASpecificExp: """ Task Generator should take in an experiment. @@ -19,14 +19,3 @@ class TaskGenerator(ABC, Generic[ASpecificExp]): """ raise NotImplementedError("generate method is not implemented.") - - def collect_feedback(self, feedback_obj_l: List[object]): - """ - When online evaluation. - The previous feedbacks will be collected to support advanced factor generator - - Parameters - ---------- - feedback_obj_l : List[object] - - """ diff --git a/rdagent/core/evaluation.py b/rdagent/core/evaluation.py index f446fa67..01b71c9a 100644 --- a/rdagent/core/evaluation.py +++ b/rdagent/core/evaluation.py @@ -1,6 +1,6 @@ from abc import ABC, abstractmethod -from rdagent.core.experiment import Implementation, Task +from rdagent.core.experiment import Task, Workspace from rdagent.core.scenario import Scenario @@ -19,8 +19,8 @@ class Evaluator(ABC): def evaluate( self, target_task: Task, - implementation: Implementation, - gt_implementation: Implementation, + implementation: Workspace, + gt_implementation: Workspace, **kwargs, ): raise NotImplementedError diff --git a/rdagent/core/evolving_agent.py b/rdagent/core/evolving_agent.py index 42c1e071..f030df64 100644 --- a/rdagent/core/evolving_agent.py +++ b/rdagent/core/evolving_agent.py @@ -1,5 +1,5 @@ from abc import ABC, abstractmethod -from typing import Any +from typing import Any, List from tqdm import tqdm @@ -14,14 +14,18 @@ class EvoAgent(ABC): @abstractmethod def multistep_evolve(self, evo: EvolvableSubjects, eva: Evaluator | Feedback, **kwargs: Any) -> EvolvableSubjects: - pass + ... + + @abstractmethod + def filter_evolvable_subjects_by_feedback(self, evo: EvolvableSubjects, feedback: Feedback) -> EvolvableSubjects: + ... class RAGEvoAgent(EvoAgent): def __init__(self, max_loop, evolving_strategy, rag) -> None: super().__init__(max_loop, evolving_strategy) self.rag = rag - self.evolving_trace = [] + self.evolving_trace: List[EvoStep] = [] def multistep_evolve( self, @@ -31,6 +35,7 @@ class RAGEvoAgent(EvoAgent): with_knowledge: bool = False, with_feedback: bool = True, knowledge_self_gen: bool = False, + filter_final_evo: bool = False, ) -> EvolvableSubjects: for _ in tqdm(range(self.max_loop), "Implementing"): # 1. knowledge self-evolving @@ -60,5 +65,6 @@ class RAGEvoAgent(EvoAgent): # 6. update trace self.evolving_trace.append(es) - + if with_feedback and filter_final_evo: + evo = self.filter_evolvable_subjects_by_feedback(evo, self.evolving_trace[-1].feedback) return evo diff --git a/rdagent/core/evolving_framework.py b/rdagent/core/evolving_framework.py index fd99a043..a84cf699 100644 --- a/rdagent/core/evolving_framework.py +++ b/rdagent/core/evolving_framework.py @@ -32,7 +32,8 @@ class EvolvableSubjects: return copy.deepcopy(self) -class QlibEvolvableSubjects(EvolvableSubjects): ... +class QlibEvolvableSubjects(EvolvableSubjects): + ... @dataclass diff --git a/rdagent/core/exception.py b/rdagent/core/exception.py index ae1c57b4..9086784c 100644 --- a/rdagent/core/exception.py +++ b/rdagent/core/exception.py @@ -1,4 +1,4 @@ -class ImplementRunException(Exception): +class CoderException(Exception): """ Exceptions raised when Implementing and running code. - start: FactorTask => FactorGenerator @@ -8,19 +8,37 @@ class ImplementRunException(Exception): """ -class CodeFormatException(ImplementRunException): +class CodeFormatException(CoderException): """ The generated code is not found due format error. """ -class RuntimeErrorException(ImplementRunException): +class RuntimeErrorException(CoderException): """ The generated code fail to execute the script. """ -class NoOutputException(ImplementRunException): +class NoOutputException(CoderException): """ The code fail to generate output file. """ + + +class RunnerException(Exception): + """ + Exceptions raised when running the code output. + """ + + +class FactorEmptyException(Exception): + """ + Exceptions raised when no factor is generated correctly + """ + + +class ModelEmptyException(Exception): + """ + Exceptions raised when no model is generated correctly + """ diff --git a/rdagent/core/experiment.py b/rdagent/core/experiment.py index c0c661ff..52136011 100644 --- a/rdagent/core/experiment.py +++ b/rdagent/core/experiment.py @@ -1,17 +1,20 @@ +from __future__ import annotations + +import shutil +import uuid from abc import ABC, abstractmethod +from copy import deepcopy from pathlib import Path from typing import Any, Dict, Generic, Optional, Sequence, TypeVar +from rdagent.core.conf import RD_AGENT_SETTINGS + """ This file contains the all the class about organizing the task in RD-Agent. """ class Task(ABC): - # TODO: 把name放在这里作为主键 - # Please refer to rdagent/model_implementation/task.py for the implementation - # I think the task version applies to the base class. - @abstractmethod def get_task_information(self): """ @@ -23,39 +26,44 @@ class Task(ABC): ASpecificTask = TypeVar("ASpecificTask", bound=Task) -class Implementation(ABC, Generic[ASpecificTask]): - # TODO: workspace; - # - code or data(optional) - # - Execute logic - # - `env is not included`. It is a underlying infra - def __init__(self, target_task: ASpecificTask) -> None: - self.target_task = target_task +class Workspace(ABC, Generic[ASpecificTask]): + """ + A workspace is a place to store the task implementation. It evolves as the developer implements the task. + To get a snapshot of the workspace, make sure call `copy` to get a copy of the workspace. + """ + + def __init__(self, target_task: ASpecificTask = None) -> None: + self.target_task: ASpecificTask = target_task @abstractmethod def execute(self, *args, **kwargs) -> object: raise NotImplementedError("execute method is not implemented.") - -ASpecificImp = TypeVar("ASpecificImp", bound=Implementation) - - -class ImpLoader(ABC, Generic[ASpecificTask, ASpecificImp]): @abstractmethod - def load(self, task: ASpecificTask) -> ASpecificImp: + def copy(self) -> Workspace: + raise NotImplementedError("copy method is not implemented.") + + +ASpecificWS = TypeVar("ASpecificWS", bound=Workspace) + + +class WsLoader(ABC, Generic[ASpecificTask, ASpecificWS]): + @abstractmethod + def load(self, task: ASpecificTask) -> ASpecificWS: raise NotImplementedError("load method is not implemented.") -class FBImplementation(Implementation): +class FBWorkspace(Workspace): """ - File-based task implementation + File-based task workspace The implemented task will be a folder which contains related elements. - Data - - Code Implementation + - Code Workspace - Output - After execution, it will generate the final output as file. - A typical way to run the pipeline of FBImplementation will be + A typical way to run the pipeline of FBWorkspace will be (We didn't add it as a method due to that we may pass arguments into `prepare` or `execute` based on our requirements.) .. code-block:: python @@ -67,15 +75,12 @@ class FBImplementation(Implementation): """ - # TODO: - # FileBasedFactorImplementation should inherit from it. - # Why not directly reuse FileBasedFactorImplementation. - # Because it has too much concrete dependencies. - # e.g. dataframe, factors - def __init__(self, *args, code_dict: Dict[str, str] = None, **kwargs) -> None: + def __init__(self, *args, **kwargs) -> None: super().__init__(*args, **kwargs) - self.code_dict = code_dict # The code to be injected into the folder, store them in the variable - self.workspace_path: Optional[Path] = None + self.code_dict = ( + {} + ) # The code injected into the folder, store them in the variable to reproduce the former result + self.workspace_path: Path = RD_AGENT_SETTINGS.workspace_path / uuid.uuid4().hex @property def code(self) -> str: @@ -84,28 +89,26 @@ class FBImplementation(Implementation): code_string += f"File: {file_name}\n{code}\n" return code_string - @abstractmethod def prepare(self, *args, **kwargs): """ - Prepare all the files except the injected code + Prepare the workspace except the injected code - Data - Documentation - - TODO: env? Env is implicitly defined by the document? - typical usage of `*args, **kwargs`: Different methods shares the same data. The data are passed by the arguments. """ - # TODO: model and factor prepare; + self.workspace_path.mkdir(parents=True, exist_ok=True) def inject_code(self, **files: str): """ Inject the code into the folder. { - "model.py": "" + : } """ - self.code_dict = files + self.prepare() for k, v in files.items(): + self.code_dict[k] = v with open(self.workspace_path / k, "w") as f: f.write(v) @@ -114,22 +117,55 @@ class FBImplementation(Implementation): Get the environment description. To be general, we only return a list of filenames. - How to summarize the environment is the responsibility of the TaskGenerator. + How to summarize the environment is the responsibility of the Developer. """ return list(self.workspace_path.iterdir()) + def inject_code_from_folder(self, folder_path: Path): + """ + Load the workspace from the folder + """ + for file_path in folder_path.iterdir(): + if file_path.suffix == ".py" or file_path.suffix == ".yaml": + self.inject_code(**{file_path.name: file_path.read_text()}) -class Experiment(ABC, Generic[ASpecificTask, ASpecificImp]): + def copy(self) -> FBWorkspace: + """ + copy the workspace from the original one + """ + return deepcopy(self) + + def clear(self) -> None: + """ + Clear the workspace + """ + shutil.rmtree(self.workspace_path) + self.code_dict = {} + + @abstractmethod + def execute(self, *args, **kwargs) -> object: + """ + Before each execution, make sure to prepare and inject code + """ + self.prepare() + self.inject_code(**self.code_dict) + + +ASpecificWSForExperiment = TypeVar("ASpecificWSForExperiment", bound=Workspace) +ASpecificWSForSubTasks = TypeVar("ASpecificWSForSubTasks", bound=Workspace) + + +class Experiment(ABC, Generic[ASpecificTask, ASpecificWSForExperiment, ASpecificWSForSubTasks]): """ - The experiment is a sequence of tasks and the implementations of the tasks after generated by the TaskGenerator. + The experiment is a sequence of tasks and the implementations of the tasks after generated by the Developer. """ def __init__(self, sub_tasks: Sequence[ASpecificTask]) -> None: self.sub_tasks = sub_tasks - self.sub_implementations: Sequence[ASpecificImp] = [None for _ in self.sub_tasks] - self.based_experiments: Sequence[Experiment] = [] + self.sub_workspace_list: Sequence[ASpecificWSForSubTasks] = [None for _ in self.sub_tasks] + self.based_experiments: Sequence[ASpecificWSForExperiment] = [] self.result: object = None # The result of the experiment, can be different types in different scenarios. - self.exp_ws: ASpecificImp = None + self.experiment_workspace: ASpecificWSForExperiment = None ASpecificExp = TypeVar("ASpecificExp", bound=Experiment) diff --git a/rdagent/core/proposal.py b/rdagent/core/proposal.py index 7dde0e20..b703dca9 100644 --- a/rdagent/core/proposal.py +++ b/rdagent/core/proposal.py @@ -45,6 +45,13 @@ class HypothesisFeedback(Feedback): def __bool__(self): return self.decision + def __str__(self) -> str: + return f"""Observations: {self.observations} +Hypothesis Evaluation: {self.hypothesis_evaluation} +New Hypothesis: {self.new_hypothesis} +Decision: {self.decision} +Reason: {self.reason}""" + ASpecificScen = TypeVar("ASpecificScen", bound=Scenario) @@ -64,10 +71,11 @@ class Trace(Generic[ASpecificScen]): return None, None -class HypothesisGen: +class HypothesisGen(ABC): def __init__(self, scen: Scenario): self.scen = scen + @abstractmethod def gen(self, trace: Trace) -> Hypothesis: # def gen(self, scenario_desc: str, ) -> Hypothesis: """ @@ -107,5 +115,3 @@ class HypothesisExperiment2Feedback: For example: `mlflow` of Qlib will be included. """ raise NotImplementedError("generateFeedback method is not implemented.") - - # def generateResultComparison() diff --git a/rdagent/log/__init__.py b/rdagent/log/__init__.py index ff6f158d..a0d37eff 100644 --- a/rdagent/log/__init__.py +++ b/rdagent/log/__init__.py @@ -1,4 +1,4 @@ -from .logger import RDAgentLog -from .utils import LogColors +from rdagent.log.logger import RDAgentLog +from rdagent.log.utils import LogColors -rdagent_logger: RDAgentLog = RDAgentLog() \ No newline at end of file +rdagent_logger: RDAgentLog = RDAgentLog() diff --git a/rdagent/log/base.py b/rdagent/log/base.py index d99c6a81..8cec23cc 100644 --- a/rdagent/log/base.py +++ b/rdagent/log/base.py @@ -47,16 +47,17 @@ class View: Display the content in the storage """ + # TODO: pleas fix me @abstractmethod - def display(s: Storage, watch: bool=False): + def display(s: Storage, watch: bool = False): """ Parameters ---------- s : Storage - + watch : bool should we watch the new content and display them """ - ... \ No newline at end of file + ... diff --git a/rdagent/log/logger.py b/rdagent/log/logger.py index 2b9b3ead..8a17768d 100644 --- a/rdagent/log/logger.py +++ b/rdagent/log/logger.py @@ -1,16 +1,17 @@ -import sys import os - -from loguru import logger -from rdagent.core.utils import SingletonBaseClass -from rdagent.core.conf import RD_AGENT_SETTINGS -from pathlib import Path -from psutil import Process +import sys +from contextlib import contextmanager from datetime import datetime, timezone from functools import partial -from contextlib import contextmanager from multiprocessing import Pipe from multiprocessing.connection import Connection +from pathlib import Path + +from loguru import logger +from psutil import Process + +from rdagent.core.conf import RD_AGENT_SETTINGS +from rdagent.core.utils import SingletonBaseClass from .storage import FileStorage from .utils import LogColors, get_caller_info @@ -22,7 +23,7 @@ class RDAgentLog(SingletonBaseClass): Here is an example tag .. code-block:: - + a - b - c @@ -38,6 +39,7 @@ class RDAgentLog(SingletonBaseClass): - 1233-365 ... """ + # TODO: Simplify it to introduce less concepts ( We may merge RDAgentLog, Storage &) # Solution: Storage => PipeLog, View => PipeLogView, RDAgentLog is an instance of PipeLogger # PipeLogger.info(...) , PipeLogger.get_resp() to get feedback from frontend. @@ -48,16 +50,15 @@ class RDAgentLog(SingletonBaseClass): _tag: str = "" def __init__(self, log_trace_path: str | None = RD_AGENT_SETTINGS.log_trace_path) -> None: - if log_trace_path is None: timestamp = datetime.now(timezone.utc).strftime("%Y-%m-%d_%H-%M-%S-%f") log_trace_path: Path = Path.cwd() / "log" / timestamp - + self.log_trace_path = Path(log_trace_path) self.log_trace_path.mkdir(parents=True, exist_ok=True) - + self.storage = FileStorage(log_trace_path) - + self.main_pid = os.getpid() @contextmanager @@ -70,13 +71,13 @@ class RDAgentLog(SingletonBaseClass): # TODO: It may result in error in mutithreading or co-routine self._tag = self._tag + tag yield - self._tag = self._tag[:-len(tag)] + self._tag = self._tag[: -len(tag)] def get_pids(self) -> str: - ''' + """ Returns a string of pids from the current process to the main process. Split by '-'. - ''' + """ pid = os.getpid() process = Process(pid) pid_chain = f"{pid}" @@ -87,7 +88,7 @@ class RDAgentLog(SingletonBaseClass): process = parent_process return pid_chain - def file_format(self, record, raw: bool=False): + def file_format(self, record, raw: bool = False): record["message"] = LogColors.remove_ansi_codes(record["message"]) if raw: return "{message}" @@ -95,24 +96,25 @@ class RDAgentLog(SingletonBaseClass): def log_object(self, obj: object, *, tag: str = "") -> None: caller_info = get_caller_info() - tag = f"{self._tag}.{tag}.{self.get_pids()}".strip('.') + tag = f"{self._tag}.{tag}.{self.get_pids()}".strip(".") logp = self.storage.log(obj, name=tag, save_type="pkl") - file_handler_id = logger.add(self.log_trace_path / tag.replace('.','/') / "common_logs.log", format=self.file_format) + file_handler_id = logger.add( + self.log_trace_path / tag.replace(".", "/") / "common_logs.log", format=self.file_format + ) logger.patch(lambda r: r.update(caller_info)).info(f"Logging object in {logp.absolute()}") logger.remove(file_handler_id) - - def info(self, msg: str, *, tag: str = "", raw: bool=False) -> None: + def info(self, msg: str, *, tag: str = "", raw: bool = False) -> None: # TODO: too much duplicated. due to we have no logger with stream context; caller_info = get_caller_info() if raw: logger.remove() logger.add(sys.stderr, format=lambda r: "{message}") - tag = f"{self._tag}.{tag}.{self.get_pids()}".strip('.') - log_file_path = self.log_trace_path / tag.replace('.','/') / "common_logs.log" + tag = f"{self._tag}.{tag}.{self.get_pids()}".strip(".") + log_file_path = self.log_trace_path / tag.replace(".", "/") / "common_logs.log" if raw: file_handler_id = logger.add(log_file_path, format=partial(self.file_format, raw=True)) else: @@ -131,15 +133,19 @@ class RDAgentLog(SingletonBaseClass): # getattr(logger.patch(lambda r: r.update(caller_info)), level)(msg) caller_info = get_caller_info() - tag = f"{self._tag}.{tag}.{self.get_pids()}".strip('.') - file_handler_id = logger.add(self.log_trace_path / tag.replace('.','/') / "common_logs.log", format=self.file_format) + tag = f"{self._tag}.{tag}.{self.get_pids()}".strip(".") + file_handler_id = logger.add( + self.log_trace_path / tag.replace(".", "/") / "common_logs.log", format=self.file_format + ) logger.patch(lambda r: r.update(caller_info)).warning(msg) logger.remove(file_handler_id) def error(self, msg: str, *, tag: str = "") -> None: caller_info = get_caller_info() - - tag = f"{self._tag}.{tag}.{self.get_pids()}".strip('.') - file_handler_id = logger.add(self.log_trace_path / tag.replace('.','/') / "common_logs.log", format=self.file_format) + + tag = f"{self._tag}.{tag}.{self.get_pids()}".strip(".") + file_handler_id = logger.add( + self.log_trace_path / tag.replace(".", "/") / "common_logs.log", format=self.file_format + ) logger.patch(lambda r: r.update(caller_info)).error(msg) logger.remove(file_handler_id) diff --git a/rdagent/log/storage.py b/rdagent/log/storage.py index 50418a24..92470eab 100644 --- a/rdagent/log/storage.py +++ b/rdagent/log/storage.py @@ -1,11 +1,12 @@ import json import pickle +from datetime import datetime, timezone +from pathlib import Path from typing import Literal -from pathlib import Path -from datetime import datetime, timezone from .base import Storage + class FileStorage(Storage): """ The info are logginged to the file systems @@ -17,18 +18,19 @@ class FileStorage(Storage): self.path = Path(path) self.path.mkdir(parents=True, exist_ok=True) - def log(self, - obj: object, - name: str = "", - save_type: Literal["json", "text", "pkl"] = "text", - timestamp: datetime | None = None, - ) -> Path: + def log( + self, + obj: object, + name: str = "", + save_type: Literal["json", "text", "pkl"] = "text", + timestamp: datetime | None = None, + ) -> Path: # TODO: We can remove the timestamp after we implement PipeLog if timestamp is None: timestamp = datetime.now(timezone.utc) else: timestamp = timestamp.astimezone(timezone.utc) - + cur_p = self.path / name.replace(".", "/") cur_p.mkdir(parents=True, exist_ok=True) diff --git a/rdagent/log/ui/web.py b/rdagent/log/ui/web.py index ad5aada4..6fae6658 100644 --- a/rdagent/log/ui/web.py +++ b/rdagent/log/ui/web.py @@ -1,20 +1,18 @@ -from rdagent.log.base import View, Storage from pathlib import Path +from rdagent.log.base import Storage, View + + class ProcessView(View): def __init__(self, trace_path: Path): - - # Save logs to your desired data structure # ... pass - def display(s: Storage, watch: bool = False): pass - class WebView(View): r""" @@ -55,6 +53,7 @@ class WebView(View): 2. Map path like `a.b.c` to frontend components 3. Display logic """ + def __init__(self, trace_path: Path): pass # Save logs to your desired data structure @@ -74,7 +73,7 @@ class STLWindow: ... def consume_msg(self, msg): - ... # update it's view + ... # update it's view class STLUI: diff --git a/rdagent/log/utils.py b/rdagent/log/utils.py index 1459f888..0d237bc6 100644 --- a/rdagent/log/utils.py +++ b/rdagent/log/utils.py @@ -23,7 +23,7 @@ class LogColors: END = "\033[0m" @classmethod - def get_all_colors(cls: type['LogColors']) -> list: + def get_all_colors(cls: type["LogColors"]) -> list: names = dir(cls) names = [name for name in names if not name.startswith("__") and not callable(getattr(cls, name))] return [getattr(cls, name) for name in names] @@ -52,8 +52,8 @@ class LogColors: """ It is for removing ansi ctrl characters in the string(e.g. colored text) """ - ansi_escape = re.compile(r'\x1B\[[0-?]*[ -/]*[@-~]') - return ansi_escape.sub('', s) + ansi_escape = re.compile(r"\x1B\[[0-?]*[ -/]*[@-~]") + return ansi_escape.sub("", s) def get_caller_info(): @@ -63,8 +63,8 @@ def get_caller_info(): caller_info = stack[2] frame = caller_info[0] info = { - 'line': caller_info.lineno, - 'name': frame.f_globals['__name__'], # Get the module name from the frame's globals - 'function': frame.f_code.co_name, # Get the caller's function name + "line": caller_info.lineno, + "name": frame.f_globals["__name__"], # Get the module name from the frame's globals + "function": frame.f_code.co_name, # Get the caller's function name } return info diff --git a/rdagent/oai/llm_utils.py b/rdagent/oai/llm_utils.py index 35fb0fb5..75e41928 100644 --- a/rdagent/oai/llm_utils.py +++ b/rdagent/oai/llm_utils.py @@ -19,11 +19,13 @@ import numpy as np import tiktoken from rdagent.core.conf import RD_AGENT_SETTINGS -from rdagent.log import LogColors, rdagent_logger as logger from rdagent.core.utils import SingletonBaseClass +from rdagent.log import LogColors +from rdagent.log import rdagent_logger as logger DEFAULT_QLIB_DOT_PATH = Path("./") + def md5_hash(input_string: str) -> str: hash_md5 = hashlib.md5(usedforsecurity=False) input_bytes = input_string.encode("utf-8") @@ -203,7 +205,7 @@ class ChatSession: user prompt should always be provided """ messages = self.build_chat_completion_message(user_prompt) - + with logger.tag(f"session_{self.conversation_id}"): response = self.api_backend._try_create_chat_completion_or_embedding( # noqa: SLF001 messages=messages, @@ -651,7 +653,7 @@ class APIBackend: # TODO: with logger.config(stream=self.chat_stream): and add a `stream_start` flag to add timestamp for first message. if self.cfg.log_llm_chat_content: logger.info(f"{LogColors.CYAN}Response:{LogColors.END}", tag="llm_messages") - + for chunk in response: content = ( chunk.choices[0].delta.content @@ -663,10 +665,10 @@ class APIBackend: resp += content if len(chunk.choices) > 0 and chunk.choices[0].finish_reason is not None: finish_reason = chunk.choices[0].finish_reason - + if self.cfg.log_llm_chat_content: logger.info("\n", raw=True, tag="llm_messages") - + else: resp = response.choices[0].message.content finish_reason = response.choices[0].finish_reason diff --git a/rdagent/scenarios/qlib/factor_task_implementation/__init__.py b/rdagent/scenarios/qlib/developer/factor_coder.py similarity index 100% rename from rdagent/scenarios/qlib/factor_task_implementation/__init__.py rename to rdagent/scenarios/qlib/developer/factor_coder.py diff --git a/rdagent/scenarios/qlib/task_generator/data.py b/rdagent/scenarios/qlib/developer/factor_runner.py similarity index 69% rename from rdagent/scenarios/qlib/task_generator/data.py rename to rdagent/scenarios/qlib/developer/factor_runner.py index 4995bcf0..9e9b880a 100644 --- a/rdagent/scenarios/qlib/task_generator/data.py +++ b/rdagent/scenarios/qlib/developer/factor_runner.py @@ -1,17 +1,14 @@ import pickle -import shutil from pathlib import Path -from typing import List, Tuple +from typing import List import pandas as pd from rdagent.components.runner import CachedRunner from rdagent.components.runner.conf import RUNNER_SETTINGS +from rdagent.core.exception import FactorEmptyException from rdagent.log import rdagent_logger as logger -from rdagent.core.task_generator import TaskGenerator -from rdagent.oai.llm_utils import md5_hash from rdagent.scenarios.qlib.experiment.factor_experiment import QlibFactorExperiment -from rdagent.utils.env import QTDockerEnv DIRNAME = Path(__file__).absolute().resolve().parent DIRNAME_local = Path.cwd() @@ -40,15 +37,15 @@ class QlibFactorRunner(CachedRunner[QlibFactorExperiment]): - results in `mlflow` """ - def generate(self, exp: QlibFactorExperiment) -> QlibFactorExperiment: + def develop(self, exp: QlibFactorExperiment) -> QlibFactorExperiment: """ Generate the experiment by processing and combining factor data, then passing the combined data to Docker for backtest results. """ if exp.based_experiments and exp.based_experiments[-1].result is None: - exp.based_experiments[-1] = self.generate(exp.based_experiments[-1]) + exp.based_experiments[-1] = self.develop(exp.based_experiments[-1]) - if RUNNER_SETTINGS.runner_cache_result: + if RUNNER_SETTINGS.cache_result: cache_hit, result = self.get_cache_result(exp) if cache_hit: exp.result = result @@ -56,12 +53,15 @@ class QlibFactorRunner(CachedRunner[QlibFactorExperiment]): if exp.based_experiments: SOTA_factor = None - if exp.based_experiments.__len__() != 1: + if len(exp.based_experiments) > 1: SOTA_factor = self.process_factor_data(exp.based_experiments) # Process the new factors data new_factors = self.process_factor_data(exp) + if new_factors.empty: + raise FactorEmptyException("No valid factor data found to merge.") + # Combine the SOTA factor and new factors if SOTA factor exists if SOTA_factor is not None and not SOTA_factor.empty: combined_factors = pd.concat([SOTA_factor, new_factors], axis=1).dropna() @@ -73,36 +73,16 @@ class QlibFactorRunner(CachedRunner[QlibFactorExperiment]): new_columns = pd.MultiIndex.from_product([["feature"], combined_factors.columns]) combined_factors.columns = new_columns - # Save the combined factors to a pickle file - combined_factors_path = DIRNAME / "env_factor/combined_factors_df.pkl" - with open(combined_factors_path, "wb") as f: + # Save the combined factors to the workspace + with open(exp.experiment_workspace.workspace_path / "combined_factors_df.pkl", "wb") as f: pickle.dump(combined_factors, f) - # Docker run - # Call Docker, pass the combined factors to Docker, and generate backtest results - qtde = QTDockerEnv() - qtde.prepare() - - # Run the Docker command - execute_log = qtde.run(local_path=str(DIRNAME / "env_factor"), entry="rm -r mlruns") - # Run the Qlib backtest - execute_log = qtde.run( - local_path=str(DIRNAME / "env_factor"), - entry=f"qrun conf.yaml" if len(exp.based_experiments) == 0 else "qrun conf_combined.yaml", + result = exp.experiment_workspace.execute( + qlib_config_name=f"conf.yaml" if len(exp.based_experiments) == 0 else "conf_combined.yaml" ) - execute_log = qtde.run(local_path=str(DIRNAME / "env_factor"), entry="python read_exp_res.py") - - csv_path = DIRNAME / "env_factor/qlib_res.csv" - - if not csv_path.exists(): - logger.error(f"File {csv_path} does not exist.") - return None - - result = pd.read_csv(csv_path, index_col=0).iloc[:, 0] - exp.result = result - if RUNNER_SETTINGS.runner_cache_result: + if RUNNER_SETTINGS.cache_result: self.dump_cache_result(exp, result) return exp @@ -124,11 +104,11 @@ class QlibFactorRunner(CachedRunner[QlibFactorExperiment]): # Collect all exp's dataframes for exp in exp_or_list: # Iterate over sub-implementations and execute them to get each factor data - for implementation in exp.sub_implementations: + for implementation in exp.sub_workspace_list: message, df = implementation.execute(data_type="All") # Check if factor generation was successful - if df is not None: + if df is not None and "datetime" in df.index.names: time_diff = df.index.get_level_values("datetime").to_series().diff().dropna().unique() if pd.Timedelta(minutes=1) not in time_diff: factor_dfs.append(df) diff --git a/rdagent/scenarios/qlib/task_generator/feedback.py b/rdagent/scenarios/qlib/developer/feedback.py similarity index 90% rename from rdagent/scenarios/qlib/task_generator/feedback.py rename to rdagent/scenarios/qlib/developer/feedback.py index 339ab7f0..33b7c57b 100644 --- a/rdagent/scenarios/qlib/task_generator/feedback.py +++ b/rdagent/scenarios/qlib/developer/feedback.py @@ -7,7 +7,6 @@ from pathlib import Path from jinja2 import Environment, StrictUndefined from rdagent.core.experiment import Experiment -from rdagent.log import rdagent_logger as logger from rdagent.core.prompts import Prompts from rdagent.core.proposal import ( Hypothesis, @@ -15,6 +14,7 @@ from rdagent.core.proposal import ( HypothesisFeedback, Trace, ) +from rdagent.log import rdagent_logger as logger from rdagent.oai.llm_utils import APIBackend feedback_prompts = Prompts(file_path=Path(__file__).parent.parent / "prompts.yaml") @@ -76,8 +76,7 @@ class QlibFactorHypothesisExperiment2Feedback(HypothesisExperiment2Feedback): reason = response_json.get("Reasoning", "No reasoning provided") decision = response_json.get("Replace Best Result", "no").lower() == "yes" - # Create HypothesisFeedback object - hypothesis_feedback = HypothesisFeedback( + return HypothesisFeedback( observations=observations, hypothesis_evaluation=hypothesis_evaluation, new_hypothesis=new_hypothesis, @@ -85,17 +84,6 @@ class QlibFactorHypothesisExperiment2Feedback(HypothesisExperiment2Feedback): decision=decision, ) - logger.info( - "Generated Hypothesis Feedback:\n" - f"Observations: {observations}\n" - f"Feedback for Hypothesis: {hypothesis_evaluation}\n" - f"New Hypothesis: {new_hypothesis}\n" - f"Reason: {reason}\n" - f"Replace Best Result: {'Yes' if decision else 'No'}" - ) - - return hypothesis_feedback - class QlibModelHypothesisExperiment2Feedback(HypothesisExperiment2Feedback): """Generated feedbacks on the hypothesis from **Executed** Implementations of different tasks & their comparisons with previous performances""" @@ -106,6 +94,7 @@ class QlibModelHypothesisExperiment2Feedback(HypothesisExperiment2Feedback): For example: `mlflow` of Qlib will be included. """ + logger.info("Generating feedback...") # Define the system prompt for hypothesis feedback system_prompt = feedback_prompts["model_feedback_generation"]["system"] @@ -140,5 +129,5 @@ class QlibModelHypothesisExperiment2Feedback(HypothesisExperiment2Feedback): hypothesis_evaluation=response_json_hypothesis.get("Feedback for Hypothesis", "No feedback provided"), new_hypothesis=response_json_hypothesis.get("New Hypothesis", "No new hypothesis provided"), reason=response_json_hypothesis.get("Reasoning", "No reasoning provided"), - decision=response_json_hypothesis.get("Decision", "false").lower() == "true", + decision=str(response_json_hypothesis.get("Decision", "false")).lower() == "true", ) diff --git a/rdagent/scenarios/qlib/model_task_implementation/__init__.py b/rdagent/scenarios/qlib/developer/model_coder.py similarity index 100% rename from rdagent/scenarios/qlib/model_task_implementation/__init__.py rename to rdagent/scenarios/qlib/developer/model_coder.py diff --git a/rdagent/scenarios/qlib/developer/model_runner.py b/rdagent/scenarios/qlib/developer/model_runner.py new file mode 100644 index 00000000..1fee7b6e --- /dev/null +++ b/rdagent/scenarios/qlib/developer/model_runner.py @@ -0,0 +1,55 @@ +import shutil +import uuid +from pathlib import Path + +import pandas as pd + +from rdagent.components.coder.model_coder.model import ModelExperiment, ModelFBWorkspace +from rdagent.components.runner import CachedRunner +from rdagent.components.runner.conf import RUNNER_SETTINGS +from rdagent.core.developer import Developer +from rdagent.core.exception import ModelEmptyException +from rdagent.log import rdagent_logger as logger +from rdagent.scenarios.qlib.experiment.model_experiment import QlibModelExperiment +from rdagent.utils.env import QTDockerEnv + + +class QlibModelRunner(CachedRunner[QlibModelExperiment]): + """ + Docker run + Everything in a folder + - config.yaml + - Pytorch `model.py` + - results in `mlflow` + + https://github.com/microsoft/qlib/blob/main/qlib/contrib/model/pytorch_nn.py + - pt_model_uri: hard-code `model.py:Net` in the config + - let LLM modify model.py + """ + + def develop(self, exp: QlibModelExperiment) -> QlibModelExperiment: + if RUNNER_SETTINGS.cache_result: + cache_hit, result = self.get_cache_result(exp) + if cache_hit: + exp.result = result + return exp + + if exp.sub_workspace_list[0].code_dict.get("model.py") is None: + raise ModelEmptyException("model.py is empty") + # to replace & inject code + exp.experiment_workspace.inject_code(**{"model.py": exp.sub_workspace_list[0].code_dict["model.py"]}) + + env_to_use = {"PYTHONPATH": "./"} + + if exp.sub_tasks[0].model_type == "TimeSeries": + env_to_use.update({"dataset_cls": "TSDatasetH", "step_len": 20, "num_timesteps": 20}) + elif exp.sub_tasks[0].model_type == "Tabular": + env_to_use.update({"dataset_cls": "DatasetH"}) + + result = exp.experiment_workspace.execute(qlib_config_name="conf.yaml", run_env=env_to_use) + + exp.result = result + if RUNNER_SETTINGS.cache_result: + self.dump_cache_result(exp, result) + + return exp diff --git a/rdagent/scenarios/qlib/experiment/factor_experiment.py b/rdagent/scenarios/qlib/experiment/factor_experiment.py index 85e7bd6e..b4821eb4 100644 --- a/rdagent/scenarios/qlib/experiment/factor_experiment.py +++ b/rdagent/scenarios/qlib/experiment/factor_experiment.py @@ -1,13 +1,22 @@ from pathlib import Path -from rdagent.components.coder.factor_coder.factor import FactorExperiment +from rdagent.components.coder.factor_coder.factor import ( + FactorExperiment, + FactorFBWorkspace, + FactorTask, +) from rdagent.components.coder.factor_coder.utils import get_data_folder_intro from rdagent.core.prompts import Prompts from rdagent.core.scenario import Scenario +from rdagent.scenarios.qlib.experiment.workspace import QlibFBWorkspace prompt_dict = Prompts(file_path=Path(__file__).parent / "prompts.yaml") -QlibFactorExperiment = FactorExperiment + +class QlibFactorExperiment(FactorExperiment[FactorTask, QlibFBWorkspace, FactorFBWorkspace]): + def __init__(self, *args, **kwargs) -> None: + super().__init__(*args, **kwargs) + self.experiment_workspace = QlibFBWorkspace(template_folder_path=Path(__file__).parent / "factor_template") class QlibFactorScenario(Scenario): diff --git a/rdagent/scenarios/qlib/task_generator/env_factor/conf.yaml b/rdagent/scenarios/qlib/experiment/factor_template/conf.yaml similarity index 100% rename from rdagent/scenarios/qlib/task_generator/env_factor/conf.yaml rename to rdagent/scenarios/qlib/experiment/factor_template/conf.yaml diff --git a/rdagent/scenarios/qlib/task_generator/env_factor/conf_combined.yaml b/rdagent/scenarios/qlib/experiment/factor_template/conf_combined.yaml similarity index 97% rename from rdagent/scenarios/qlib/task_generator/env_factor/conf_combined.yaml rename to rdagent/scenarios/qlib/experiment/factor_template/conf_combined.yaml index b7768bc0..c2e081f4 100644 --- a/rdagent/scenarios/qlib/task_generator/env_factor/conf_combined.yaml +++ b/rdagent/scenarios/qlib/experiment/factor_template/conf_combined.yaml @@ -21,7 +21,7 @@ data_handler_config: &data_handler_config - ["LABEL0"] - class: qlib.data.dataset.loader.StaticDataLoader kwargs: - # config: "/home/finco/v-yuanteli/RD-Agent/rdagent/scenarios/qlib/task_generator/env_factor/combined_factors_df.pkl" + # config: "/home/finco/v-yuanteli/RD-Agent/rdagent/scenarios/qlib/task_generator/factor_template/combined_factors_df.pkl" config: "combined_factors_df.pkl" learn_processors: diff --git a/rdagent/scenarios/qlib/task_generator/env_factor/read_exp_res.py b/rdagent/scenarios/qlib/experiment/factor_template/read_exp_res.py similarity index 100% rename from rdagent/scenarios/qlib/task_generator/env_factor/read_exp_res.py rename to rdagent/scenarios/qlib/experiment/factor_template/read_exp_res.py diff --git a/rdagent/scenarios/qlib/experiment/model_experiment.py b/rdagent/scenarios/qlib/experiment/model_experiment.py index 8feb185d..7b35b474 100644 --- a/rdagent/scenarios/qlib/experiment/model_experiment.py +++ b/rdagent/scenarios/qlib/experiment/model_experiment.py @@ -1,12 +1,21 @@ from pathlib import Path -from rdagent.components.coder.model_coder.model import ModelExperiment +from rdagent.components.coder.model_coder.model import ( + ModelExperiment, + ModelFBWorkspace, + ModelTask, +) from rdagent.core.prompts import Prompts from rdagent.core.scenario import Scenario +from rdagent.scenarios.qlib.experiment.workspace import QlibFBWorkspace prompt_dict = Prompts(file_path=Path(__file__).parent / "prompts.yaml") -QlibModelExperiment = ModelExperiment + +class QlibModelExperiment(ModelExperiment[ModelTask, QlibFBWorkspace, ModelFBWorkspace]): + def __init__(self, *args, **kwargs) -> None: + super().__init__(*args, **kwargs) + self.experiment_workspace = QlibFBWorkspace(template_folder_path=Path(__file__).parent / "model_template") class QlibModelScenario(Scenario): diff --git a/rdagent/scenarios/qlib/task_generator/model_template/README.md b/rdagent/scenarios/qlib/experiment/model_template/README.md similarity index 100% rename from rdagent/scenarios/qlib/task_generator/model_template/README.md rename to rdagent/scenarios/qlib/experiment/model_template/README.md diff --git a/rdagent/scenarios/qlib/task_generator/model_template/conf.yaml b/rdagent/scenarios/qlib/experiment/model_template/conf.yaml similarity index 100% rename from rdagent/scenarios/qlib/task_generator/model_template/conf.yaml rename to rdagent/scenarios/qlib/experiment/model_template/conf.yaml diff --git a/rdagent/scenarios/qlib/task_generator/model_template/read_exp_res.py b/rdagent/scenarios/qlib/experiment/model_template/read_exp_res.py similarity index 100% rename from rdagent/scenarios/qlib/task_generator/model_template/read_exp_res.py rename to rdagent/scenarios/qlib/experiment/model_template/read_exp_res.py diff --git a/rdagent/scenarios/qlib/experiment/workspace.py b/rdagent/scenarios/qlib/experiment/workspace.py new file mode 100644 index 00000000..3100eb62 --- /dev/null +++ b/rdagent/scenarios/qlib/experiment/workspace.py @@ -0,0 +1,44 @@ +from pathlib import Path + +import pandas as pd + +from rdagent.core.experiment import FBWorkspace +from rdagent.log import rdagent_logger as logger +from rdagent.utils.env import QTDockerEnv + + +class QlibFBWorkspace(FBWorkspace): + def __init__(self, template_folder_path: Path, *args, **kwargs) -> None: + super().__init__(*args, **kwargs) + self.inject_code_from_folder(template_folder_path) + + def execute(self, qlib_config_name: str = "conf.yaml", run_env: dict = {}, *args, **kwargs) -> str: + qtde = QTDockerEnv() + qtde.prepare() + + # Run the Docker command + execute_log = qtde.run( + local_path=str(self.workspace_path), + entry="rm -r mlruns", + env=run_env, + ) + # Run the Qlib backtest + execute_log = qtde.run( + local_path=str(self.workspace_path), + entry=f"qrun {qlib_config_name}", + env=run_env, + ) + + execute_log = qtde.run( + local_path=str(self.workspace_path), + entry="python read_exp_res.py", + env=run_env, + ) + + csv_path = self.workspace_path / "qlib_res.csv" + + if not csv_path.exists(): + logger.error(f"File {csv_path} does not exist.") + return None + + return pd.read_csv(csv_path, index_col=0).iloc[:, 0] diff --git a/rdagent/scenarios/qlib/factor_experiment_loader/json_loader.py b/rdagent/scenarios/qlib/factor_experiment_loader/json_loader.py index 609c7910..99395c36 100644 --- a/rdagent/scenarios/qlib/factor_experiment_loader/json_loader.py +++ b/rdagent/scenarios/qlib/factor_experiment_loader/json_loader.py @@ -4,11 +4,12 @@ from pathlib import Path from rdagent.components.benchmark.eval_method import TestCase from rdagent.components.coder.factor_coder.factor import ( FactorExperiment, + FactorFBWorkspace, FactorTask, - FileBasedFactorImplementation, ) from rdagent.components.loader.experiment_loader import FactorExperimentLoader from rdagent.core.experiment import Loader +from rdagent.scenarios.qlib.experiment.factor_experiment import QlibFactorExperiment class FactorExperimentLoaderFromDict(FactorExperimentLoader): @@ -23,7 +24,7 @@ class FactorExperimentLoaderFromDict(FactorExperimentLoader): variables=factor_data["variables"], ) task_l.append(task) - exp = FactorExperiment(sub_tasks=task_l) + exp = QlibFactorExperiment(sub_tasks=task_l) return exp @@ -54,7 +55,7 @@ class FactorTestCaseLoaderFromJsonFile: factor_formulation=factor_data["formulation"], variables=factor_data["variables"], ) - gt = FileBasedFactorImplementation(task, code=factor_data["gt_code"]) + gt = FactorFBWorkspace(task, code=factor_data["gt_code"]) gt.execute() TestData.target_task.append(task) TestData.ground_truth.append(gt) diff --git a/rdagent/scenarios/qlib/factor_experiment_loader/pdf_loader.py b/rdagent/scenarios/qlib/factor_experiment_loader/pdf_loader.py index 01029ff0..f5b9f94e 100644 --- a/rdagent/scenarios/qlib/factor_experiment_loader/pdf_loader.py +++ b/rdagent/scenarios/qlib/factor_experiment_loader/pdf_loader.py @@ -18,8 +18,8 @@ from rdagent.components.document_reader.document_reader import ( ) from rdagent.components.loader.experiment_loader import FactorExperimentLoader from rdagent.core.conf import RD_AGENT_SETTINGS -from rdagent.log import rdagent_logger as logger from rdagent.core.prompts import Prompts +from rdagent.log import rdagent_logger as logger from rdagent.oai.llm_utils import APIBackend, create_embedding_with_multiprocessing from rdagent.scenarios.qlib.factor_experiment_loader.json_loader import ( FactorExperimentLoaderFromDict, @@ -27,6 +27,7 @@ from rdagent.scenarios.qlib.factor_experiment_loader.json_loader import ( document_process_prompts = Prompts(file_path=Path(__file__).parent / "prompts.yaml") + def classify_report_from_dict( report_dict: Mapping[str, str], vote_time: int = 1, diff --git a/rdagent/scenarios/qlib/factor_proposal.py b/rdagent/scenarios/qlib/proposal/factor_proposal.py similarity index 91% rename from rdagent/scenarios/qlib/factor_proposal.py rename to rdagent/scenarios/qlib/proposal/factor_proposal.py index b66d752d..a24dbb26 100644 --- a/rdagent/scenarios/qlib/factor_proposal.py +++ b/rdagent/scenarios/qlib/proposal/factor_proposal.py @@ -12,8 +12,9 @@ from rdagent.components.proposal.factor_proposal import ( ) from rdagent.core.prompts import Prompts from rdagent.core.proposal import Hypothesis, Scenario, Trace +from rdagent.scenarios.qlib.experiment.factor_experiment import QlibFactorExperiment -prompt_dict = Prompts(file_path=Path(__file__).parent / "prompts.yaml") +prompt_dict = Prompts(file_path=Path(__file__).parent.parent / "prompts.yaml") QlibFactorHypothesis = FactorHypothesis @@ -75,8 +76,8 @@ class QlibFactorHypothesis2Experiment(FactorHypothesis2Experiment): formulation = response_dict[factor_name]["formulation"] variables = response_dict[factor_name]["variables"] tasks.append(FactorTask(factor_name, description, formulation, variables)) - exp = FactorExperiment(tasks) + exp = QlibFactorExperiment(tasks) exp.based_experiments = [t[1] for t in trace.hist if t[2]] if len(exp.based_experiments) == 0: - exp.based_experiments.append(FactorExperiment(sub_tasks=[])) + exp.based_experiments.append(QlibFactorExperiment(sub_tasks=[])) return exp diff --git a/rdagent/scenarios/qlib/model_proposal.py b/rdagent/scenarios/qlib/proposal/model_proposal.py similarity index 93% rename from rdagent/scenarios/qlib/model_proposal.py rename to rdagent/scenarios/qlib/proposal/model_proposal.py index d5165dee..57f466c7 100644 --- a/rdagent/scenarios/qlib/model_proposal.py +++ b/rdagent/scenarios/qlib/proposal/model_proposal.py @@ -12,8 +12,9 @@ from rdagent.components.proposal.model_proposal import ( ) from rdagent.core.prompts import Prompts from rdagent.core.proposal import Hypothesis, Scenario, Trace +from rdagent.scenarios.qlib.experiment.model_experiment import QlibModelExperiment -prompt_dict = Prompts(file_path=Path(__file__).parent / "prompts.yaml") +prompt_dict = Prompts(file_path=Path(__file__).parent.parent / "prompts.yaml") QlibModelHypothesis = ModelHypothesis @@ -76,6 +77,6 @@ class QlibModelHypothesis2Experiment(ModelHypothesis2Experiment): variables = response_dict[model_name]["variables"] model_type = response_dict[model_name]["model_type"] tasks.append(ModelTask(model_name, description, formulation, variables, model_type)) - exp = ModelExperiment(tasks) + exp = QlibModelExperiment(tasks) exp.based_experiments = [t[1] for t in trace.hist if t[2]] return exp diff --git a/rdagent/scenarios/qlib/task_generator/model.py b/rdagent/scenarios/qlib/task_generator/model.py deleted file mode 100644 index 88c7ac33..00000000 --- a/rdagent/scenarios/qlib/task_generator/model.py +++ /dev/null @@ -1,84 +0,0 @@ -import shutil -import uuid -from pathlib import Path - -import pandas as pd - -from rdagent.components.coder.model_coder.model import ( - ModelExperiment, - ModelImplementation, -) -from rdagent.components.runner import CachedRunner -from rdagent.components.runner.conf import RUNNER_SETTINGS -from rdagent.log import rdagent_logger as logger -from rdagent.core.task_generator import TaskGenerator -from rdagent.utils.env import QTDockerEnv - - -class QlibModelRunner(CachedRunner[ModelImplementation]): - """ - Docker run - Everything in a folder - - config.yaml - - Pytorch `model.py` - - results in `mlflow` - - https://github.com/microsoft/qlib/blob/main/qlib/contrib/model/pytorch_nn.py - - pt_model_uri: hard-code `model.py:Net` in the config - - let LLM modify model.py - """ - - def generate(self, exp: ModelExperiment) -> ModelExperiment: - - if RUNNER_SETTINGS.runner_cache_result: - cache_hit, result = self.get_cache_result(exp) - if cache_hit: - exp.result = result - return exp - TEMPLATE_PATH = Path(__file__).parent / "model_template" # Can be updated - - # To set the experiment level workspace and prepare the workspaces use the first task as the target task - exp.exp_ws = ModelImplementation(target_task=exp.sub_tasks[0]) - exp.exp_ws.prepare() - - # to copy_template_to_workspace - for file_path in TEMPLATE_PATH.iterdir(): - shutil.copyfile(file_path, exp.exp_ws.workspace_path / file_path.name) - - # to replace & inject code - exp.exp_ws.inject_code(**{"model.py": exp.sub_implementations[0].code_dict["model.py"]}) - - env_to_use = {} - - if exp.sub_tasks[0].model_type == "TimeSeries": - env_to_use = {"dataset_cls": "TSDatasetH", "step_len": 20, "num_timesteps": 20} - elif exp.sub_tasks[0].model_type == "Tabular": - env_to_use = {"dataset_cls": "DatasetH"} - # to execute - qtde = QTDockerEnv() - - # Preparing the Docker environment - qtde.prepare() - - # Run the Docker container with the specified entry - - execute_log = qtde.run( - local_path=exp.exp_ws.workspace_path, entry="qrun conf.yaml", env={"PYTHONPATH": "./", **env_to_use} - ) - - # Run the experiment analysis code - execute_log = qtde.run(local_path=exp.exp_ws.workspace_path, entry="python read_exp_res.py") - - csv_path = exp.exp_ws.workspace_path / "qlib_res.csv" - - if not csv_path.exists(): - logger.error(f"File {csv_path} does not exist.") - return None - - result = pd.read_csv(csv_path, index_col=0).iloc[:,0] - - exp.result = result - if RUNNER_SETTINGS.runner_cache_result: - self.dump_cache_result(exp, result) - - return exp diff --git a/rdagent/utils/env.py b/rdagent/utils/env.py index d90becf1..97265ad2 100644 --- a/rdagent/utils/env.py +++ b/rdagent/utils/env.py @@ -112,9 +112,9 @@ class LocalEnv(Env[LocalConf]): class DockerConf(BaseSettings): build_from_dockerfile: bool = False - dockerfile_folder_path: Optional[Path] = ( - None # the path to the dockerfile optional path provided when build_from_dockerfile is False - ) + dockerfile_folder_path: Optional[ + Path + ] = None # the path to the dockerfile optional path provided when build_from_dockerfile is False image: str # the image you want to build mount_path: str # the path in the docker image to mount the folder default_entry: str # the entry point of the image @@ -128,6 +128,9 @@ class DockerConf(BaseSettings): class QlibDockerConf(DockerConf): + class Config: + env_prefix = "QLIB_DOCKER_" # Use QLIB_DOCKER_ as prefix for environment variables + build_from_dockerfile: bool = True dockerfile_folder_path: Path = Path(__file__).parent.parent / "scenarios" / "qlib" / "docker" image: str = "local_qlib:latest" diff --git a/test/utils/env_tpl/read_exp.py b/test/utils/env_tpl/read_exp.py index b42d15b2..5a2c54c0 100644 --- a/test/utils/env_tpl/read_exp.py +++ b/test/utils/env_tpl/read_exp.py @@ -1,9 +1,11 @@ import qlib -from mlflow.tracking import MlflowClient from mlflow.entities import ViewType +from mlflow.tracking import MlflowClient + qlib.init() from qlib.workflow import R + # here is the documents of the https://qlib.readthedocs.io/en/latest/component/recorder.html # TODO: list all the recorder and metrics