From f603f2b20f52fde32390dcd0b0731e3aae63ff92 Mon Sep 17 00:00:00 2001 From: you-n-g Date: Sun, 20 Jul 2025 16:31:43 +0800 Subject: [PATCH] feat: add extra_eval config and import_class for custom evaluators (#1097) * feat: add extra_eval config and import_class for custom evaluators * lint * build: update litellm requirement to >=1.73 for get_valid_models * refactor: remove *args/**kwargs from _create_embedding_inner_function signature --- rdagent/components/coder/data_science/conf.py | 12 ++++++++++++ .../coder/data_science/pipeline/__init__.py | 5 +++++ rdagent/oai/backend/base.py | 4 +--- rdagent/oai/backend/deprec.py | 4 +--- rdagent/oai/backend/litellm.py | 6 +----- requirements.txt | 2 +- 6 files changed, 21 insertions(+), 12 deletions(-) diff --git a/rdagent/components/coder/data_science/conf.py b/rdagent/components/coder/data_science/conf.py index e5d9ba7f..21664441 100644 --- a/rdagent/components/coder/data_science/conf.py +++ b/rdagent/components/coder/data_science/conf.py @@ -23,6 +23,18 @@ class DSCoderCoSTEERSettings(CoSTEERSettings): env_type: str = "docker" # TODO: extract a function for env and conf. + extra_eval: list[str] = [] + """ + Extra evaluators + + The evaluator follows the following assumptions: + - It runs after previous evaluator (So the running results are alreadly there) + + It is not a complete feature due to it is only implemented in DS Pipeline & Coder. + + TODO: The complete version should be implemented in the CoSTEERSettings. + """ + def get_ds_env( conf_type: Literal["kaggle", "mlebench"] = "kaggle", diff --git a/rdagent/components/coder/data_science/pipeline/__init__.py b/rdagent/components/coder/data_science/pipeline/__init__.py index 64a1cab2..86c2c4e8 100644 --- a/rdagent/components/coder/data_science/pipeline/__init__.py +++ b/rdagent/components/coder/data_science/pipeline/__init__.py @@ -43,6 +43,7 @@ from rdagent.components.coder.data_science.share.eval import ModelDumpEvaluator from rdagent.core.exception import CoderError from rdagent.core.experiment import FBWorkspace from rdagent.core.scenario import Scenario +from rdagent.core.utils import import_class from rdagent.oai.llm_utils import APIBackend from rdagent.utils.agent.ret import PythonAgentOut from rdagent.utils.agent.tpl import T @@ -143,6 +144,10 @@ class PipelineCoSTEER(CoSTEER): if DS_RD_SETTING.enable_model_dump: eval_l.append(ModelDumpEvaluator(scen=scen, data_type="sample")) + for extra_eval in DSCoderCoSTEERSettings().extra_eval: + kls = import_class(extra_eval) + eval_l.append(kls(scen=scen)) + eva = CoSTEERMultiEvaluator( single_evaluator=eval_l, scen=scen ) # Please specify whether you agree running your eva in parallel or not diff --git a/rdagent/oai/backend/base.py b/rdagent/oai/backend/base.py index dd828880..e6c9e4c9 100644 --- a/rdagent/oai/backend/base.py +++ b/rdagent/oai/backend/base.py @@ -654,9 +654,7 @@ class APIBackend(ABC): raise NotImplementedError("Subclasses must implement this method") @abstractmethod - def _create_embedding_inner_function( # type: ignore[no-untyped-def] - self, input_content_list: list[str], *args, **kwargs - ) -> list[list[float]]: # noqa: ARG002 + def _create_embedding_inner_function(self, input_content_list: list[str]) -> list[list[float]]: """ Call the embedding function """ diff --git a/rdagent/oai/backend/deprec.py b/rdagent/oai/backend/deprec.py index 23501729..3fcae490 100644 --- a/rdagent/oai/backend/deprec.py +++ b/rdagent/oai/backend/deprec.py @@ -269,9 +269,7 @@ class DeprecBackend(APIBackend): """ return False - def _create_embedding_inner_function( # type: ignore[no-untyped-def] - self, input_content_list: list[str], *args, **kwargs - ) -> list[list[float]]: # noqa: ARG002 + def _create_embedding_inner_function(self, input_content_list: list[str]) -> list[list[float]]: content_to_embedding_dict = {} for sliced_filtered_input_content_list in [ input_content_list[i : i + LLM_SETTINGS.embedding_max_str_num] diff --git a/rdagent/oai/backend/litellm.py b/rdagent/oai/backend/litellm.py index 03039623..ce13d1c5 100644 --- a/rdagent/oai/backend/litellm.py +++ b/rdagent/oai/backend/litellm.py @@ -66,9 +66,7 @@ class LiteLLMAPIBackend(APIBackend): logger.info(f"{LogColors.CYAN}Token count: {LogColors.END} {num_tokens}", tag="debug_litellm_token") return num_tokens - def _create_embedding_inner_function( - self, input_content_list: list[str], *args: Any, **kwargs: Any - ) -> list[list[float]]: # noqa: ARG002 + def _create_embedding_inner_function(self, input_content_list: list[str]) -> list[list[float]]: """ Call the embedding function """ @@ -82,8 +80,6 @@ class LiteLLMAPIBackend(APIBackend): response = embedding( model=model_name, input=input_content_list, - *args, - **kwargs, ) response_list = [data["embedding"] for data in response.data] return response_list diff --git a/requirements.txt b/requirements.txt index 5cc0ea85..e94de46f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,7 +8,7 @@ loguru fire fuzzywuzzy openai -litellm==1.72.4 +litellm>=1.73 # to support `from litellm import get_valid_models` azure.identity pyarrow rich