From ef641332faa603949075dec43078ec129afdf5e9 Mon Sep 17 00:00:00 2001 From: you-n-g Date: Sun, 29 Sep 2024 18:43:17 +0800 Subject: [PATCH] feat: Dynamic scenario based on task (#392) * OAI api * more comments * llm utils * RD-Framework for dynamic scen * remove useless code * auto lint * fix dynamic scennario * Add parameter * auto lint * remove abstractmethod * fix lint --- .../coder/factor_coder/CoSTEER/evaluators.py | 18 ++++- .../factor_coder/CoSTEER/evolving_strategy.py | 6 +- .../coder/model_coder/CoSTEER/evaluators.py | 12 +++- rdagent/core/scenario.py | 25 +++++-- rdagent/oai/llm_utils.py | 9 +++ .../experiment/model_experiment.py | 3 +- rdagent/scenarios/general_model/scenario.py | 3 +- .../scenarios/kaggle/experiment/scenario.py | 3 +- .../qlib/experiment/factor_experiment.py | 9 +-- .../qlib/experiment/model_experiment.py | 3 +- rdagent/scenarios/qlib/experiment/utils.py | 66 ++++++++++++------- rdagent/utils/agent/tpl.py | 10 ++- 12 files changed, 121 insertions(+), 46 deletions(-) diff --git a/rdagent/components/coder/factor_coder/CoSTEER/evaluators.py b/rdagent/components/coder/factor_coder/CoSTEER/evaluators.py index 51ab645c..c46a2e62 100644 --- a/rdagent/components/coder/factor_coder/CoSTEER/evaluators.py +++ b/rdagent/components/coder/factor_coder/CoSTEER/evaluators.py @@ -89,7 +89,11 @@ class FactorCodeEvaluator(FactorEvaluator): system_prompt = ( Environment(undefined=StrictUndefined) .from_string(evaluate_prompts["evaluator_code_feedback_v1_system"]) - .render(scenario=self.scen.get_scenario_all_desc() if self.scen is not None else "No scenario description.") + .render( + scenario=self.scen.get_scenario_all_desc(target_task) + if self.scen is not None + else "No scenario description." + ) ) execution_feedback_to_render = execution_feedback @@ -167,7 +171,11 @@ class FactorOutputFormatEvaluator(FactorEvaluator): .from_string( evaluate_prompts["evaluator_output_format_system"], ) - .render(scenario=self.scen.get_scenario_all_desc() if self.scen is not None else "No scenario description.") + .render( + scenario=self.scen.get_scenario_all_desc(implementation.target_task) + if self.scen is not None + else "No scenario description." + ) ) # TODO: with retry_context(retry_n=3, except_list=[KeyError]): @@ -473,7 +481,11 @@ class FactorFinalDecisionEvaluator(Evaluator): system_prompt = ( Environment(undefined=StrictUndefined) .from_string(evaluate_prompts["evaluator_final_decision_v1_system"]) - .render(scenario=self.scen.get_scenario_all_desc() if self.scen is not None else "No scenario description.") + .render( + scenario=self.scen.get_scenario_all_desc(target_task) + if self.scen is not None + else "No scenario description." + ) ) execution_feedback_to_render = execution_feedback diff --git a/rdagent/components/coder/factor_coder/CoSTEER/evolving_strategy.py b/rdagent/components/coder/factor_coder/CoSTEER/evolving_strategy.py index 2dbfa9c5..71d4e584 100644 --- a/rdagent/components/coder/factor_coder/CoSTEER/evolving_strategy.py +++ b/rdagent/components/coder/factor_coder/CoSTEER/evolving_strategy.py @@ -133,7 +133,7 @@ class FactorEvolvingStrategy(MultiProcessEvolvingStrategy): implement_prompts["evolving_strategy_factor_implementation_v1_system"], ) .render( - scenario=self.scen.get_scenario_all_desc(), + scenario=self.scen.get_scenario_all_desc(target_task), queried_former_failed_knowledge=queried_former_failed_knowledge_to_render, ) ) @@ -226,7 +226,7 @@ class FactorEvolvingStrategyWithGraph(MultiProcessEvolvingStrategy): implement_prompts["evolving_strategy_factor_implementation_v1_system"], ) .render( - scenario=self.scen.get_scenario_all_desc(), + scenario=self.scen.get_scenario_all_desc(target_task), queried_former_failed_knowledge=queried_former_failed_knowledge_to_render, ) ) @@ -250,7 +250,7 @@ class FactorEvolvingStrategyWithGraph(MultiProcessEvolvingStrategy): Environment(undefined=StrictUndefined) .from_string(implement_prompts["evolving_strategy_error_summary_v2_system"]) .render( - scenario=self.scen.get_scenario_all_desc(), + scenario=self.scen.get_scenario_all_desc(target_task), factor_information_str=target_factor_task_information, code_and_feedback=queried_former_failed_knowledge_to_render[ -1 diff --git a/rdagent/components/coder/model_coder/CoSTEER/evaluators.py b/rdagent/components/coder/model_coder/CoSTEER/evaluators.py index 96a4ef68..f6381362 100644 --- a/rdagent/components/coder/model_coder/CoSTEER/evaluators.py +++ b/rdagent/components/coder/model_coder/CoSTEER/evaluators.py @@ -89,7 +89,11 @@ class ModelCodeEvaluator(Evaluator): system_prompt = ( Environment(undefined=StrictUndefined) .from_string(evaluate_prompts["evaluator_code_feedback"]["system"]) - .render(scenario=self.scen.get_scenario_all_desc() if self.scen is not None else "No scenario description.") + .render( + scenario=self.scen.get_scenario_all_desc(target_task) + if self.scen is not None + else "No scenario description." + ) ) execution_feedback_to_render = model_execution_feedback @@ -145,7 +149,11 @@ class ModelFinalEvaluator(Evaluator): system_prompt = ( Environment(undefined=StrictUndefined) .from_string(evaluate_prompts["evaluator_final_feedback"]["system"]) - .render(scenario=self.scen.get_scenario_all_desc() if self.scen is not None else "No scenario description.") + .render( + scenario=self.scen.get_scenario_all_desc(target_task) + if self.scen is not None + else "No scenario description." + ) ) execution_feedback_to_render = model_execution_feedback diff --git a/rdagent/core/scenario.py b/rdagent/core/scenario.py index 9943cc16..2d970b17 100644 --- a/rdagent/core/scenario.py +++ b/rdagent/core/scenario.py @@ -1,5 +1,7 @@ from abc import ABC, abstractmethod +from rdagent.core.experiment import Task + class Scenario(ABC): @property @@ -7,10 +9,21 @@ class Scenario(ABC): def background(self) -> str: """Background information""" + # TODO: We have to change all the sub classes to override get_source_data_desc instead of `source_data` + def get_source_data_desc(self, task: Task | None = None) -> str: # noqa: ARG002 + """ + Source data description + + The choice of data may vary based on the specific task at hand. + """ + return "" + @property - @abstractmethod def source_data(self) -> str: - """Source data description""" + """ + A convenient shortcut for describing source data + """ + return self.get_source_data_desc() @property @abstractmethod @@ -33,8 +46,12 @@ class Scenario(ABC): """Rich style description to present""" @abstractmethod - def get_scenario_all_desc(self) -> str: - """Combine all the description together""" + def get_scenario_all_desc(self, task: Task | None = None) -> str: + """ + Combine all descriptions together + + The scenario description varies based on the task being performed. + """ @property def experiment_setting(self) -> str | None: diff --git a/rdagent/oai/llm_utils.py b/rdagent/oai/llm_utils.py index d5ccd43e..22524ec8 100644 --- a/rdagent/oai/llm_utils.py +++ b/rdagent/oai/llm_utils.py @@ -235,6 +235,15 @@ class ChatSession: class APIBackend: + """ + This is a unified interface for different backends. + + (xiao) thinks integerate all kinds of API in a single class is not a good design. + So we should split them into different classes in `oai/backends/` in the future. + """ + + # FIXME: (xiao) We should avoid using self.xxxx. + # Instead, we can use self.cfg directly. If it's difficult to support different backend settings, we can split them into multiple BaseSettings. def __init__( # noqa: C901, PLR0912, PLR0915 self, *, diff --git a/rdagent/scenarios/data_mining/experiment/model_experiment.py b/rdagent/scenarios/data_mining/experiment/model_experiment.py index ad366663..f5cef548 100644 --- a/rdagent/scenarios/data_mining/experiment/model_experiment.py +++ b/rdagent/scenarios/data_mining/experiment/model_experiment.py @@ -5,6 +5,7 @@ from rdagent.components.coder.model_coder.model import ( ModelFBWorkspace, ModelTask, ) +from rdagent.core.experiment import Task from rdagent.core.prompts import Prompts from rdagent.core.scenario import Scenario from rdagent.scenarios.data_mining.experiment.workspace import DMFBWorkspace @@ -62,7 +63,7 @@ The demo showcases the iterative process of hypothesis generation, knowledge con To demonstrate the dynamic evolution of models through the R&D loop, emphasizing how each iteration enhances the model performance and reliability. The performane is measured by the AUROC score (Area Under the Receiver Operating Characteristic), which is a commonly used metric for binary classification. """ - def get_scenario_all_desc(self) -> str: + def get_scenario_all_desc(self, task: Task | None = None) -> str: return f"""Background of the scenario: {self.background} The interface you should follow to write the runnable code: diff --git a/rdagent/scenarios/general_model/scenario.py b/rdagent/scenarios/general_model/scenario.py index 9b7e9b12..9fe8c159 100644 --- a/rdagent/scenarios/general_model/scenario.py +++ b/rdagent/scenarios/general_model/scenario.py @@ -1,6 +1,7 @@ from copy import deepcopy from pathlib import Path +from rdagent.core.experiment import Task from rdagent.core.prompts import Prompts from rdagent.core.scenario import Scenario @@ -40,7 +41,7 @@ class GeneralModelScenario(Scenario): def rich_style_description(self) -> str: return self._rich_style_description - def get_scenario_all_desc(self) -> str: + def get_scenario_all_desc(self, task: Task | None = None) -> str: return f"""Background of the scenario: {self.background} The interface you should follow to write the runnable code: diff --git a/rdagent/scenarios/kaggle/experiment/scenario.py b/rdagent/scenarios/kaggle/experiment/scenario.py index a526913d..bf3bbe4a 100644 --- a/rdagent/scenarios/kaggle/experiment/scenario.py +++ b/rdagent/scenarios/kaggle/experiment/scenario.py @@ -8,6 +8,7 @@ import pandas as pd from jinja2 import Environment, StrictUndefined from rdagent.app.kaggle.conf import KAGGLE_IMPLEMENT_SETTING +from rdagent.core.experiment import Task from rdagent.core.prompts import Prompts from rdagent.core.scenario import Scenario from rdagent.oai.llm_utils import APIBackend @@ -203,7 +204,7 @@ The model code should follow the simulator: This is the Kaggle scenario for the competition: {self.competition} """ - def get_scenario_all_desc(self) -> str: + def get_scenario_all_desc(self, task: Task | None = None) -> str: return f"""Background of the scenario: {self._background} The source dataset you can use to generate the features: diff --git a/rdagent/scenarios/qlib/experiment/factor_experiment.py b/rdagent/scenarios/qlib/experiment/factor_experiment.py index 16b014c7..b45f22e6 100644 --- a/rdagent/scenarios/qlib/experiment/factor_experiment.py +++ b/rdagent/scenarios/qlib/experiment/factor_experiment.py @@ -6,6 +6,7 @@ from rdagent.components.coder.factor_coder.factor import ( FactorFBWorkspace, FactorTask, ) +from rdagent.core.experiment import Task from rdagent.core.prompts import Prompts from rdagent.core.scenario import Scenario from rdagent.scenarios.qlib.experiment.utils import get_data_folder_intro @@ -36,8 +37,7 @@ class QlibFactorScenario(Scenario): def background(self) -> str: return self._background - @property - def source_data(self) -> str: + def get_source_data_desc(self, task: Task | None = None) -> str: return self._source_data @property @@ -60,11 +60,12 @@ class QlibFactorScenario(Scenario): def experiment_setting(self) -> str: return self._experiment_setting - def get_scenario_all_desc(self) -> str: + def get_scenario_all_desc(self, task: Task | None = None) -> str: + """A static scenario describer""" return f"""Background of the scenario: {self.background} The source data you can use: -{self.source_data} +{self.get_source_data_desc(task)} The interface you should follow to write the runnable code: {self.interface} The output of your code should be in the format: diff --git a/rdagent/scenarios/qlib/experiment/model_experiment.py b/rdagent/scenarios/qlib/experiment/model_experiment.py index 9b88f427..cd9543df 100644 --- a/rdagent/scenarios/qlib/experiment/model_experiment.py +++ b/rdagent/scenarios/qlib/experiment/model_experiment.py @@ -6,6 +6,7 @@ from rdagent.components.coder.model_coder.model import ( ModelFBWorkspace, ModelTask, ) +from rdagent.core.experiment import Task from rdagent.core.prompts import Prompts from rdagent.core.scenario import Scenario from rdagent.scenarios.qlib.experiment.workspace import QlibFBWorkspace @@ -57,7 +58,7 @@ class QlibModelScenario(Scenario): def experiment_setting(self) -> str: return self._experiment_setting - def get_scenario_all_desc(self) -> str: + def get_scenario_all_desc(self, task: Task | None = None) -> str: return f"""Background of the scenario: {self.background} The interface you should follow to write the runnable code: diff --git a/rdagent/scenarios/qlib/experiment/utils.py b/rdagent/scenarios/qlib/experiment/utils.py index a39e590e..f2ff6375 100644 --- a/rdagent/scenarios/qlib/experiment/utils.py +++ b/rdagent/scenarios/qlib/experiment/utils.py @@ -1,3 +1,4 @@ +import re import shutil from pathlib import Path @@ -49,15 +50,31 @@ def generate_data_folder_from_qlib(): ) -def get_data_folder_intro(): - """Directly get the info of the data folder. +def get_data_folder_intro(fname_reg: str = ".*", flags=0) -> str: + """ + Directly get the info of the data folder. It is for preparing prompting message. + + Parameters + ---------- + fname_reg : str + a regular expression to filter the file name. + + flags: str + flags for re.match + + Returns + ------- + str + The description of the data folder. """ if ( not Path(FACTOR_IMPLEMENT_SETTINGS.data_folder).exists() or not Path(FACTOR_IMPLEMENT_SETTINGS.data_folder_debug).exists() ): + # FIXME: (xiao) I think this is writing in a hard-coded way. + # get data folder intro does not imply that we are generating the data folder. generate_data_folder_from_qlib() JJ_TPL = Environment(undefined=StrictUndefined).from_string( @@ -70,29 +87,30 @@ def get_data_folder_intro(): ) content_l = [] for p in Path(FACTOR_IMPLEMENT_SETTINGS.data_folder_debug).iterdir(): - if p.name.endswith(".h5"): - df = pd.read_hdf(p) - # get df.head() as string with full width - pd.set_option("display.max_columns", None) # or 1000 - pd.set_option("display.max_rows", None) # or 1000 - pd.set_option("display.max_colwidth", None) # or 199 - rendered = JJ_TPL.render( - file_name=p.name, - type_desc="generated by `pd.read_hdf(filename).head()`", - content=df.head().to_string(), - ) - content_l.append(rendered) - elif p.name.endswith(".md"): - with open(p) as f: - content = f.read() + if re.match(fname_reg, p.name, flags) is not None: + if p.name.endswith(".h5"): + df = pd.read_hdf(p) + # get df.head() as string with full width + pd.set_option("display.max_columns", None) # or 1000 + pd.set_option("display.max_rows", None) # or 1000 + pd.set_option("display.max_colwidth", None) # or 199 rendered = JJ_TPL.render( file_name=p.name, - type_desc="markdown", - content=content, + type_desc="generated by `pd.read_hdf(filename).head()`", + content=df.head().to_string(), ) content_l.append(rendered) - else: - raise NotImplementedError( - f"file type {p.name} is not supported. Please implement its description function.", - ) - return "\n ----------------- file splitter -------------\n".join(content_l) + elif p.name.endswith(".md"): + with open(p) as f: + content = f.read() + rendered = JJ_TPL.render( + file_name=p.name, + type_desc="markdown", + content=content, + ) + content_l.append(rendered) + else: + raise NotImplementedError( + f"file type {p.name} is not supported. Please implement its description function.", + ) + return "\n----------------- file splitter -------------\n".join(content_l) diff --git a/rdagent/utils/agent/tpl.py b/rdagent/utils/agent/tpl.py index a3047b9f..99df9b91 100644 --- a/rdagent/utils/agent/tpl.py +++ b/rdagent/utils/agent/tpl.py @@ -18,8 +18,11 @@ PROJ_PATH = DIRNAME.parent.parent # class T(SingletonBaseClass): TODO: singleton does not support args now. -class T: - """Use the simplest way to (C)reate a Template and (r)ender it!!""" +class RDAT: + """ + RD-Agent's Template + Use the simplest way to (C)reate a Template and (r)ender it!! + """ def __init__(self, uri: str): """ @@ -61,3 +64,6 @@ class T: Render the template with the given context. """ return Environment(undefined=StrictUndefined).from_string(self.template).render(**context) + + +T = RDAT # shortcuts