From 6da6fb5ee579cb83605eb300728f4bf279c1dfa8 Mon Sep 17 00:00:00 2001 From: Linlang <30293408+SunsetWolf@users.noreply.github.com> Date: Wed, 26 Mar 2025 22:48:01 +0800 Subject: [PATCH] refactor: remove unused code and update documentation in CoSTEER module (#720) * refactor: Remove unused code and update documentation in CoSTEER module * fix mypy error * fix black error --------- Co-authored-by: Young --- docs/scens/data_agent_fin.rst | 2 +- docs/scens/data_copilot_fin.rst | 2 +- docs/scens/kaggle_agent.rst | 2 +- rdagent/components/coder/CoSTEER/config.py | 2 -- .../components/coder/CoSTEER/evaluators.py | 9 ++++++- .../coder/CoSTEER/evolvable_subjects.py | 1 - .../coder/CoSTEER/evolving_strategy.py | 27 +++---------------- rdagent/components/coder/CoSTEER/scheduler.py | 25 ----------------- .../coder/factor_coder/evaluators.py | 1 - rdagent/core/evaluation.py | 7 +++++ rdagent/core/evolving_agent.py | 2 +- rdagent/core/experiment.py | 5 +++- 12 files changed, 27 insertions(+), 58 deletions(-) delete mode 100644 rdagent/components/coder/CoSTEER/scheduler.py diff --git a/docs/scens/data_agent_fin.rst b/docs/scens/data_agent_fin.rst index b2d8013d..b19996ba 100644 --- a/docs/scens/data_agent_fin.rst +++ b/docs/scens/data_agent_fin.rst @@ -133,6 +133,6 @@ The following environment variables can be set in the `.env` file to customize t .. autopydantic_settings:: rdagent.components.coder.factor_coder.config.FactorCoSTEERSettings :settings-show-field-summary: False - :members: coder_use_cache, data_folder, data_folder_debug, file_based_execution_timeout, select_method, select_threshold, max_loop, knowledge_base_path, new_knowledge_base_path + :members: coder_use_cache, data_folder, data_folder_debug, file_based_execution_timeout, select_method, max_loop, knowledge_base_path, new_knowledge_base_path :exclude-members: Config, fail_task_trial_limit, v1_query_former_trace_limit, v1_query_similar_success_limit, v2_query_component_limit, v2_query_error_limit, v2_query_former_trace_limit, v2_error_summary, v2_knowledge_sampler :no-index: diff --git a/docs/scens/data_copilot_fin.rst b/docs/scens/data_copilot_fin.rst index 28a34310..379c483f 100644 --- a/docs/scens/data_copilot_fin.rst +++ b/docs/scens/data_copilot_fin.rst @@ -159,6 +159,6 @@ The following environment variables can be set in the `.env` file to customize t .. autopydantic_settings:: rdagent.components.coder.factor_coder.config.FactorCoSTEERSettings :settings-show-field-summary: False - :members: coder_use_cache, data_folder, data_folder_debug, file_based_execution_timeout, select_method, select_threshold, max_loop, knowledge_base_path, new_knowledge_base_path + :members: coder_use_cache, data_folder, data_folder_debug, file_based_execution_timeout, select_method, max_loop, knowledge_base_path, new_knowledge_base_path :exclude-members: Config, python_bin, fail_task_trial_limit, v1_query_former_trace_limit, v1_query_similar_success_limit, v2_query_component_limit, v2_query_error_limit, v2_query_former_trace_limit, v2_error_summary, v2_knowledge_sampler :no-index: diff --git a/docs/scens/kaggle_agent.rst b/docs/scens/kaggle_agent.rst index df0435f2..9c53ee4c 100644 --- a/docs/scens/kaggle_agent.rst +++ b/docs/scens/kaggle_agent.rst @@ -268,5 +268,5 @@ The following environment variables can be set in the `.env` file to customize t .. autopydantic_settings:: rdagent.components.coder.factor_coder.config.FactorCoSTEERSettings :settings-show-field-summary: False :members: coder_use_cache, file_based_execution_timeout, select_method, max_loop - :exclude-members: Config, fail_task_trial_limit, v1_query_former_trace_limit, v1_query_similar_success_limit, v2_query_component_limit, v2_query_error_limit, v2_query_former_trace_limit, v2_error_summary, v2_knowledge_sampler, v2_add_fail_attempt_to_latest_successful_execution, new_knowledge_base_path, knowledge_base_path, data_folder, data_folder_debug, select_threshold + :exclude-members: Config, fail_task_trial_limit, v1_query_former_trace_limit, v1_query_similar_success_limit, v2_query_component_limit, v2_query_error_limit, v2_query_former_trace_limit, v2_error_summary, v2_knowledge_sampler, v2_add_fail_attempt_to_latest_successful_execution, new_knowledge_base_path, knowledge_base_path, data_folder, data_folder_debug :no-index: diff --git a/rdagent/components/coder/CoSTEER/config.py b/rdagent/components/coder/CoSTEER/config.py index cd1f2496..bcd21082 100644 --- a/rdagent/components/coder/CoSTEER/config.py +++ b/rdagent/components/coder/CoSTEER/config.py @@ -33,8 +33,6 @@ class CoSTEERSettings(ExtendedBaseSettings): new_knowledge_base_path: Union[str, None] = None """Path to the new knowledge base""" - select_threshold: int = 10 - max_seconds: int = 10**6 diff --git a/rdagent/components/coder/CoSTEER/evaluators.py b/rdagent/components/coder/CoSTEER/evaluators.py index 4c4f4dae..53f4b069 100644 --- a/rdagent/components/coder/CoSTEER/evaluators.py +++ b/rdagent/components/coder/CoSTEER/evaluators.py @@ -162,7 +162,14 @@ class CoSTEERMultiFeedback(Feedback): def __iter__(self): return iter(self.feedback_list) - def __bool__(self): + def finished(self) -> bool: + """ + In some implementations, tasks may fail multiple times, leading agents to skip the implementation. + This results in None feedback. However, we want to accept the correct parts and ignore None feedback. + """ + return all(feedback.final_decision for feedback in self.feedback_list if feedback is not None) + + def __bool__(self) -> bool: return all(feedback.final_decision for feedback in self.feedback_list) diff --git a/rdagent/components/coder/CoSTEER/evolvable_subjects.py b/rdagent/components/coder/CoSTEER/evolvable_subjects.py index 1d654dc0..87b3fa2a 100644 --- a/rdagent/components/coder/CoSTEER/evolvable_subjects.py +++ b/rdagent/components/coder/CoSTEER/evolvable_subjects.py @@ -14,7 +14,6 @@ class EvolvingItem(Experiment, EvolvableSubjects): sub_gt_implementations: list[FBWorkspace] = None, ): Experiment.__init__(self, sub_tasks=sub_tasks) - self.corresponding_selection: list = None if sub_gt_implementations is not None and len( sub_gt_implementations, ) != len(self.sub_tasks): diff --git a/rdagent/components/coder/CoSTEER/evolving_strategy.py b/rdagent/components/coder/CoSTEER/evolving_strategy.py index ac84d8eb..ffedafb8 100644 --- a/rdagent/components/coder/CoSTEER/evolving_strategy.py +++ b/rdagent/components/coder/CoSTEER/evolving_strategy.py @@ -12,7 +12,6 @@ from rdagent.components.coder.CoSTEER.evolvable_subjects import EvolvingItem from rdagent.components.coder.CoSTEER.knowledge_management import ( CoSTEERQueriedKnowledge, ) -from rdagent.components.coder.CoSTEER.scheduler import random_select from rdagent.core.conf import RD_AGENT_SETTINGS from rdagent.core.evolving_framework import EvolvingStrategy, EvoStep, QueriedKnowledge from rdagent.core.experiment import FBWorkspace, Task @@ -59,17 +58,6 @@ class MultiProcessEvolvingStrategy(EvolvingStrategy): """ raise NotImplementedError - def select_one_round_tasks( - self, - to_be_finished_task_index: list, - evo: EvolvingItem, - selected_num: int, - queried_knowledge: CoSTEERQueriedKnowledge, - scen: Scenario, - ) -> list: - """Since scheduler is not essential, we implement a simple random selection here.""" - return random_select(to_be_finished_task_index, evo, selected_num, queried_knowledge, scen) - @abstractmethod def assign_code_list_to_evo(self, code_list: list[dict], evo: EvolvingItem) -> None: """ @@ -92,10 +80,12 @@ class MultiProcessEvolvingStrategy(EvolvingStrategy): **kwargs, ) -> EvolvingItem: # 1.找出需要evolve的task - to_be_finished_task_index = [] + to_be_finished_task_index: list[int] = [] for index, target_task in enumerate(evo.sub_tasks): target_task_desc = target_task.get_task_information() if target_task_desc in queried_knowledge.success_task_to_knowledge_dict: + # NOTE: very weird logic: + # it depends on the knowledge to set the already finished task evo.sub_workspace_list[index] = queried_knowledge.success_task_to_knowledge_dict[ target_task_desc ].implementation @@ -105,19 +95,11 @@ class MultiProcessEvolvingStrategy(EvolvingStrategy): ): to_be_finished_task_index.append(index) - # 2. 选择selection方法 - # if the number of factors to be implemented is larger than the limit, we need to select some of them - - if self.settings.select_threshold < len(to_be_finished_task_index): - # Select a fixed number of factors if the total exceeds the threshold - to_be_finished_task_index = self.select_one_round_tasks( - to_be_finished_task_index, evo, self.settings.select_threshold, queried_knowledge, self.scen - ) - last_feedback = None if len(evolving_trace) > 0: last_feedback = evolving_trace[-1].feedback assert isinstance(last_feedback, CoSTEERMultiFeedback) + result = multiprocessing_wrapper( [ ( @@ -138,6 +120,5 @@ class MultiProcessEvolvingStrategy(EvolvingStrategy): code_list[target_index] = result[index] evo = self.assign_code_list_to_evo(code_list, evo) - evo.corresponding_selection = to_be_finished_task_index return evo diff --git a/rdagent/components/coder/CoSTEER/scheduler.py b/rdagent/components/coder/CoSTEER/scheduler.py deleted file mode 100644 index d8ea3e52..00000000 --- a/rdagent/components/coder/CoSTEER/scheduler.py +++ /dev/null @@ -1,25 +0,0 @@ -import random - -from rdagent.components.coder.CoSTEER.evolvable_subjects import EvolvingItem -from rdagent.components.coder.CoSTEER.knowledge_management import ( - CoSTEERQueriedKnowledge, -) -from rdagent.core.scenario import Scenario -from rdagent.log import rdagent_logger as logger - - -def random_select( - to_be_finished_task_index: list, - evo: EvolvingItem, - selected_num: int, - queried_knowledge: CoSTEERQueriedKnowledge, - scen: Scenario, -): - - to_be_finished_task_index = random.sample( - to_be_finished_task_index, - selected_num, - ) - - logger.info(f"The random selection is: {to_be_finished_task_index}") - return to_be_finished_task_index diff --git a/rdagent/components/coder/factor_coder/evaluators.py b/rdagent/components/coder/factor_coder/evaluators.py index 6b5b402d..464f4dd1 100644 --- a/rdagent/components/coder/factor_coder/evaluators.py +++ b/rdagent/components/coder/factor_coder/evaluators.py @@ -15,7 +15,6 @@ from rdagent.core.evolving_framework import QueriedKnowledge from rdagent.core.experiment import Workspace FactorSingleFeedback = CoSTEERSingleFeedbackDeprecated -FactorMultiFeedback = CoSTEERMultiFeedback class FactorEvaluatorForCoder(CoSTEEREvaluator): diff --git a/rdagent/core/evaluation.py b/rdagent/core/evaluation.py index e49ceca7..fa50b3b5 100644 --- a/rdagent/core/evaluation.py +++ b/rdagent/core/evaluation.py @@ -12,6 +12,13 @@ class Feedback: The building process of feedback will should be in evaluator """ + def finished(self) -> bool: + """ + In some implementations, tasks may fail multiple times, leading agents to skip the implementation. + So both skip and success indicate the task is finished. + """ + return self.__bool__() + def __bool__(self) -> bool: return True diff --git a/rdagent/core/evolving_agent.py b/rdagent/core/evolving_agent.py index 8369bc89..af87e346 100644 --- a/rdagent/core/evolving_agent.py +++ b/rdagent/core/evolving_agent.py @@ -102,6 +102,6 @@ class RAGEvoAgent(EvoAgent[RAGEvaluator]): yield evo # yield the control to caller for process control and logging. # 7. check if all tasks are completed - if self.with_feedback and es.feedback: + if self.with_feedback and es.feedback is not None and es.feedback.finished(): logger.info("All tasks in evolving subject have been completed.") break diff --git a/rdagent/core/experiment.py b/rdagent/core/experiment.py index 527937ab..55447138 100644 --- a/rdagent/core/experiment.py +++ b/rdagent/core/experiment.py @@ -10,7 +10,7 @@ from abc import ABC, abstractmethod from collections.abc import Sequence from copy import deepcopy from pathlib import Path -from typing import Any, Generic, TypeVar +from typing import Any, Generic, Literal, TypeVar from rdagent.core.conf import RD_AGENT_SETTINGS from rdagent.core.evaluation import Feedback @@ -297,6 +297,9 @@ class Experiment( ) -> None: self.hypothesis: Hypothesis | None = hypothesis # Experiment is optionally generated by hypothesis self.sub_tasks: Sequence[ASpecificTask] = sub_tasks + # None means + # - initialization placeholder before implementation + # - the developer actively skip the task; self.sub_workspace_list: list[ASpecificWSForSubTasks | None] = [None] * len(self.sub_tasks) # TODO: # It will be used in runner in history