diff --git a/rdagent/components/coder/factor_coder/CoSTEER/__init__.py b/rdagent/components/coder/factor_coder/CoSTEER/__init__.py index 58bb8c39..f2edc1d3 100644 --- a/rdagent/components/coder/factor_coder/CoSTEER/__init__.py +++ b/rdagent/components/coder/factor_coder/CoSTEER/__init__.py @@ -88,7 +88,7 @@ class FactorCoSTEER(Developer[FactorExperiment]): self.rag = FactorGraphRAGStrategy(factor_knowledge_base) # init intermediate items - factor_experiment = FactorEvolvingItem(sub_tasks=exp.sub_tasks) + factor_experiment = FactorEvolvingItem.from_experiment(exp) self.evolve_agent = FactorRAGEvoAgent( max_loop=self.max_loop, diff --git a/rdagent/components/coder/factor_coder/CoSTEER/evolvable_subjects.py b/rdagent/components/coder/factor_coder/CoSTEER/evolvable_subjects.py index 0a0e4b6d..e45baed5 100644 --- a/rdagent/components/coder/factor_coder/CoSTEER/evolvable_subjects.py +++ b/rdagent/components/coder/factor_coder/CoSTEER/evolvable_subjects.py @@ -28,3 +28,10 @@ class FactorEvolvingItem(FactorExperiment, EvolvableSubjects): ) else: self.sub_gt_implementations = sub_gt_implementations + + @classmethod + def from_experiment(cls, exp: FactorExperiment) -> "FactorExperiment": + ei = cls(sub_tasks=exp.sub_tasks) + ei.based_experiments = exp.based_experiments + ei.experiment_workspace = exp.experiment_workspace + return ei diff --git a/rdagent/components/coder/model_coder/CoSTEER/__init__.py b/rdagent/components/coder/model_coder/CoSTEER/__init__.py index 91d8b7a2..8841d125 100644 --- a/rdagent/components/coder/model_coder/CoSTEER/__init__.py +++ b/rdagent/components/coder/model_coder/CoSTEER/__init__.py @@ -70,7 +70,7 @@ class ModelCoSTEER(Developer[ModelExperiment]): self.rag = ModelRAGStrategy(model_knowledge_base) # init intermediate items - model_experiment = ModelEvolvingItem(sub_tasks=exp.sub_tasks) + model_experiment = ModelEvolvingItem.from_experiment(exp) self.evolve_agent = ModelRAGEvoAgent( max_loop=self.max_loop, diff --git a/rdagent/components/coder/model_coder/CoSTEER/evolvable_subjects.py b/rdagent/components/coder/model_coder/CoSTEER/evolvable_subjects.py index 3729f751..2be2d826 100644 --- a/rdagent/components/coder/model_coder/CoSTEER/evolvable_subjects.py +++ b/rdagent/components/coder/model_coder/CoSTEER/evolvable_subjects.py @@ -27,3 +27,10 @@ class ModelEvolvingItem(ModelExperiment, EvolvableSubjects): ) else: self.sub_gt_implementations = sub_gt_implementations + + @classmethod + def from_experiment(cls, exp: ModelExperiment) -> "ModelEvolvingItem": + ei = cls(sub_tasks=exp.sub_tasks) + ei.based_experiments = exp.based_experiments + ei.experiment_workspace = exp.experiment_workspace + return ei diff --git a/rdagent/components/coder/model_coder/CoSTEER/evolving_strategy.py b/rdagent/components/coder/model_coder/CoSTEER/evolving_strategy.py index 2b9880d7..82476f01 100644 --- a/rdagent/components/coder/model_coder/CoSTEER/evolving_strategy.py +++ b/rdagent/components/coder/model_coder/CoSTEER/evolving_strategy.py @@ -30,23 +30,31 @@ class ModelCoderEvolvingStrategy(EvolvingStrategy): self, target_task: ModelTask, queried_knowledge: ModelQueriedKnowledge = None, - exp: ModelExperiment = None, # Add this parameter + current_exp: ModelExperiment = None, # Add this parameter ) -> str: model_information_str = target_task.get_task_information() model_type = target_task.model_type - # Get the current code from the experiment using build_from_SOTA - current_code = "" - if exp is not None: - self.build_from_SOTA(exp) - model_file_mapping = { - "XGBoost": "model_xgb.py", - "RandomForest": "model_rf.py", - "LightGBM": "model_lgb.py", - "NN": "model_nn.py", - } - if model_type in model_file_mapping: - current_code = exp.experiment_workspace.code_dict.get(model_file_mapping[model_type], "") + if len(current_exp.based_experiments) == 0: + current_code = None + else: + current_code = "" + sota_exp_code_dict = current_exp.based_experiments[-1].experiment_workspace.code_dict + if target_task.version == 2: + model_file_mapping = { + "XGBoost": "model/model_xgboost.py", + "RandomForest": "model/model_randomforest.py", + "LightGBM": "model/model_lightgbm.py", + "NN": "model/model_nn.py", + } + if model_type in model_file_mapping: + current_code = sota_exp_code_dict.get(model_file_mapping[model_type], None) + elif "model.py" in sota_exp_code_dict: + current_code = sota_exp_code_dict["model.py"] + else: + current_code = None + elif target_task.version == 1: + current_code = sota_exp_code_dict.get("model.py", None) if queried_knowledge is not None and model_information_str in queried_knowledge.success_task_to_knowledge_dict: return queried_knowledge.success_task_to_knowledge_dict[model_information_str].implementation @@ -74,7 +82,7 @@ class ModelCoderEvolvingStrategy(EvolvingStrategy): .render( scenario=self.scen.get_scenario_all_desc(), queried_former_failed_knowledge=queried_former_failed_knowledge_to_render, - current_code=current_code, # Add this line + current_code=current_code, ) ) @@ -87,7 +95,6 @@ class ModelCoderEvolvingStrategy(EvolvingStrategy): ) .render( model_information_str=model_information_str, - model_type=model_type, # Add model type to the prompt queried_similar_successful_knowledge=queried_similar_successful_knowledge_to_render, queried_former_failed_knowledge=queried_former_failed_knowledge_to_render, ) @@ -124,7 +131,7 @@ class ModelCoderEvolvingStrategy(EvolvingStrategy): queried_knowledge: ModelQueriedKnowledge | None = None, **kwargs, ) -> ModelEvolvingItem: - # 1. Find the models that need to be evolved + # 1.找出需要evolve的model to_be_finished_task_index = [] for index, target_model_task in enumerate(evo.sub_tasks): target_model_task_desc = target_model_task.get_task_information() @@ -140,7 +147,7 @@ class ModelCoderEvolvingStrategy(EvolvingStrategy): result = multiprocessing_wrapper( [ - (self.implement_one_model, (evo.sub_tasks[target_index], queried_knowledge)) + (self.implement_one_model, (evo.sub_tasks[target_index], queried_knowledge, evo)) for target_index in to_be_finished_task_index ], n=RD_AGENT_SETTINGS.multi_proc_n, diff --git a/rdagent/components/coder/model_coder/prompts.yaml b/rdagent/components/coder/model_coder/prompts.yaml index b58bb92d..4d12007f 100644 --- a/rdagent/components/coder/model_coder/prompts.yaml +++ b/rdagent/components/coder/model_coder/prompts.yaml @@ -52,9 +52,14 @@ evolving_strategy_model_coder: Your must write your code based on your former latest attempt below which consists of your former code and code feedback, you should read the former attempt carefully and must not modify the right part of your former code. - {% if current_code %} - --------------Current code in the workspace:--------------- You need to tune the model based on this! If it is not None, do not write from scratch. + {% if current_code is not none %} + User has write some code before. You should write the new code based on this code. Here is the latest code: + ```python {{ current_code }} + ``` + Your code should be very similar to the former code which means your code should be ninety more percent same as the former code! You should not modify the right part of the code. + {% else %} + User has not write any code before. You should write the new code from scratch. {% endif %} {% if queried_former_failed_knowledge|length != 0 %} diff --git a/rdagent/core/experiment.py b/rdagent/core/experiment.py index 7e86198c..e4c31849 100644 --- a/rdagent/core/experiment.py +++ b/rdagent/core/experiment.py @@ -184,15 +184,22 @@ ASpecificWSForExperiment = TypeVar("ASpecificWSForExperiment", bound=Workspace) ASpecificWSForSubTasks = TypeVar("ASpecificWSForSubTasks", bound=Workspace) -class Experiment(ABC, Generic[ASpecificTask, ASpecificWSForExperiment, ASpecificWSForSubTasks]): +class Experiment( + ABC, + Generic[ASpecificTask, ASpecificWSForExperiment, ASpecificWSForSubTasks], +): """ The experiment is a sequence of tasks and the implementations of the tasks after generated by the Developer. """ - def __init__(self, sub_tasks: Sequence[ASpecificTask]) -> None: + def __init__( + self, + sub_tasks: Sequence[ASpecificTask], + based_experiments: Sequence[ASpecificWSForExperiment] = [], + ) -> None: self.sub_tasks = sub_tasks self.sub_workspace_list: list[ASpecificWSForSubTasks | None] = [None] * len(self.sub_tasks) - self.based_experiments: Sequence[ASpecificWSForExperiment] = [] + self.based_experiments: Sequence[ASpecificWSForExperiment] = based_experiments self.result: object = None # The result of the experiment, can be different types in different scenarios. self.experiment_workspace: ASpecificWSForExperiment | None = None diff --git a/rdagent/scenarios/kaggle/developer/runner.py b/rdagent/scenarios/kaggle/developer/runner.py index f8c4078f..a6546efe 100644 --- a/rdagent/scenarios/kaggle/developer/runner.py +++ b/rdagent/scenarios/kaggle/developer/runner.py @@ -23,14 +23,6 @@ prompt_dict = Prompts(file_path=Path(__file__).parent.parent / "prompts.yaml") class KGCachedRunner(CachedRunner[ASpecificExp]): - def build_from_SOTA(self, exp: ASpecificExp) -> None: - if len(exp.based_experiments) > 0: - exp.experiment_workspace.inject_code(**exp.based_experiments[-1].experiment_workspace.code_dict) - exp.experiment_workspace.data_description = exp.based_experiments[-1].experiment_workspace.data_description - exp.experiment_workspace.model_description = exp.based_experiments[ - -1 - ].experiment_workspace.model_description.copy() - def get_cache_key(self, exp: ASpecificExp) -> str: codes = [] for f in sorted((exp.experiment_workspace.workspace_path / "feature").glob("*.py"), key=lambda x: x.name): @@ -44,7 +36,6 @@ class KGCachedRunner(CachedRunner[ASpecificExp]): """ For the initial development, the experiment serves as a benchmark for feature engineering. """ - self.build_from_SOTA(exp) if RUNNER_SETTINGS.cache_result: cache_hit, result = self.get_cache_result(exp) if cache_hit: @@ -94,7 +85,6 @@ class KGModelRunner(KGCachedRunner[KGModelExperiment]): def develop(self, exp: KGModelExperiment) -> KGModelExperiment: if exp.based_experiments and exp.based_experiments[-1].result is None: exp.based_experiments[-1] = self.init_develop(exp.based_experiments[-1]) - self.build_from_SOTA(exp) sub_ws = exp.sub_workspace_list[0] # TODO: There's a possibility of generating a hybrid model (lightgbm + xgboost), which results in having two items in the model_type list. @@ -175,7 +165,6 @@ class KGFactorRunner(KGCachedRunner[KGFactorExperiment]): def develop(self, exp: KGFactorExperiment) -> KGFactorExperiment: if exp.based_experiments and exp.based_experiments[-1].result is None: exp.based_experiments[-1] = self.init_develop(exp.based_experiments[-1]) - self.build_from_SOTA(exp) current_feature_file_count = len(list(exp.experiment_workspace.workspace_path.glob("feature/feature*.py"))) implemented_factor_count = 0 for sub_ws in exp.sub_workspace_list: diff --git a/rdagent/scenarios/kaggle/experiment/kaggle_experiment.py b/rdagent/scenarios/kaggle/experiment/kaggle_experiment.py index b905bf0d..562ea995 100644 --- a/rdagent/scenarios/kaggle/experiment/kaggle_experiment.py +++ b/rdagent/scenarios/kaggle/experiment/kaggle_experiment.py @@ -1,3 +1,4 @@ +from copy import deepcopy from pathlib import Path from rdagent.app.kaggle.conf import KAGGLE_IMPLEMENT_SETTING @@ -20,6 +21,14 @@ class KGModelExperiment(ModelExperiment[ModelTask, KGFBWorkspace, ModelFBWorkspa self.experiment_workspace = KGFBWorkspace( template_folder_path=Path(__file__).parent / f"{KAGGLE_IMPLEMENT_SETTING.competition}_template" ) + if len(self.based_experiments) > 0: + self.experiment_workspace.inject_code(**self.based_experiments[-1].experiment_workspace.code_dict) + self.experiment_workspace.data_description = deepcopy( + self.based_experiments[-1].experiment_workspace.data_description + ) + self.experiment_workspace.model_description = deepcopy( + self.based_experiments[-1].experiment_workspace.model_description + ) class KGFactorExperiment(FeatureExperiment[FactorTask, KGFBWorkspace, FactorFBWorkspace]): @@ -28,3 +37,11 @@ class KGFactorExperiment(FeatureExperiment[FactorTask, KGFBWorkspace, FactorFBWo self.experiment_workspace = KGFBWorkspace( template_folder_path=Path(__file__).parent / f"{KAGGLE_IMPLEMENT_SETTING.competition}_template" ) + if len(self.based_experiments) > 0: + self.experiment_workspace.inject_code(**self.based_experiments[-1].experiment_workspace.code_dict) + self.experiment_workspace.data_description = deepcopy( + self.based_experiments[-1].experiment_workspace.data_description + ) + self.experiment_workspace.model_description = deepcopy( + self.based_experiments[-1].experiment_workspace.model_description + ) diff --git a/rdagent/scenarios/kaggle/experiment/prompts.yaml b/rdagent/scenarios/kaggle/experiment/prompts.yaml index 770f90a5..828e6d5c 100644 --- a/rdagent/scenarios/kaggle/experiment/prompts.yaml +++ b/rdagent/scenarios/kaggle/experiment/prompts.yaml @@ -68,7 +68,7 @@ kg_background: |- For each loop, you need to help user decide which action item to choose and provide the corresponding code to implement the action item. - Most importantly, the output format & submission requirements are listed here: {submission_specifications} + So far, we only focus on Model tuning! So far, we only focus on Model tuning! So far, we only focus on Model tuning! kg_feature_interface: |- Your code should contain several parts: diff --git a/rdagent/scenarios/kaggle/knowledge_management/graph.py b/rdagent/scenarios/kaggle/knowledge_management/graph.py index 3c6d4220..614d8b09 100644 --- a/rdagent/scenarios/kaggle/knowledge_management/graph.py +++ b/rdagent/scenarios/kaggle/knowledge_management/graph.py @@ -29,7 +29,6 @@ class KGKnowledgeGraph(UndirectedGraph): documents.append(f.read()) self.load_from_documents(documents=documents, scenario=scenario) self.dump() - tmp = 1 def analyze_one_document(self, document_content: str, scenario: KGScenario) -> list: session_system_prompt = ( diff --git a/rdagent/scenarios/kaggle/proposal/proposal.py b/rdagent/scenarios/kaggle/proposal/proposal.py index fea50f0d..53584ad8 100644 --- a/rdagent/scenarios/kaggle/proposal/proposal.py +++ b/rdagent/scenarios/kaggle/proposal/proposal.py @@ -239,9 +239,9 @@ class KGHypothesisGen(ModelHypothesisGen): "hypothesis_output_format": Environment(undefined=StrictUndefined) .from_string(prompt_dict["hypothesis_output_format"]) .render(if_using_feature_selection=KAGGLE_IMPLEMENT_SETTING.if_using_feature_selection), - "hypothesis_specification": f"next experiment action is {action}" - if self.scen.if_action_choosing_based_on_UCB - else None, + "hypothesis_specification": ( + f"next experiment action is {action}" if self.scen.if_action_choosing_based_on_UCB else None + ), } return context_dict, True @@ -314,8 +314,9 @@ class KGHypothesis2Experiment(ModelHypothesis2Experiment): ) ) - exp = KGFactorExperiment(tasks) - exp.based_experiments = [KGFactorExperiment(sub_tasks=[])] + [t[1] for t in trace.hist if t[2]] + exp = KGFactorExperiment( + sub_tasks=tasks, based_experiments=([KGFactorExperiment(sub_tasks=[])] + [t[1] for t in trace.hist if t[2]]) + ) return exp def convert_model_experiment(self, response: str, trace: Trace) -> KGModelExperiment: @@ -331,8 +332,9 @@ class KGHypothesis2Experiment(ModelHypothesis2Experiment): version=2, ) ) - exp = KGModelExperiment(tasks) - exp.based_experiments = [KGModelExperiment(sub_tasks=[])] + [t[1] for t in trace.hist if t[2]] + exp = KGModelExperiment( + sub_tasks=tasks, based_experiments=([KGModelExperiment(sub_tasks=[])] + [t[1] for t in trace.hist if t[2]]) + ) return exp def convert_response(self, response: str, trace: Trace) -> ModelExperiment: