feat: add extra_eval config and import_class for custom evaluators (#1097)

* feat: add extra_eval config and import_class for custom evaluators

* lint

* build: update litellm requirement to >=1.73 for get_valid_models

* refactor: remove *args/**kwargs from _create_embedding_inner_function signature

rdagent/components/coder/data_science/conf.py

@@ -23,6 +23,18 @@ class DSCoderCoSTEERSettings(CoSTEERSettings):
     env_type: str = "docker"
     # TODO: extract a function for env and conf.
 
+    extra_eval: list[str] = []
+    """
+    Extra evaluators
+
+    The evaluator follows the following assumptions:
+    - It runs after previous evaluator (So the running results are alreadly there)
+
+    It is not a complete feature due to it is only implemented in DS Pipeline & Coder.
+
+    TODO: The complete version should be implemented in the CoSTEERSettings.
+    """
+
 
 def get_ds_env(
     conf_type: Literal["kaggle", "mlebench"] = "kaggle",

rdagent/components/coder/data_science/pipeline/__init__.py

@@ -43,6 +43,7 @@ from rdagent.components.coder.data_science.share.eval import ModelDumpEvaluator
 from rdagent.core.exception import CoderError
 from rdagent.core.experiment import FBWorkspace
 from rdagent.core.scenario import Scenario
+from rdagent.core.utils import import_class
 from rdagent.oai.llm_utils import APIBackend
 from rdagent.utils.agent.ret import PythonAgentOut
 from rdagent.utils.agent.tpl import T
@@ -143,6 +144,10 @@ class PipelineCoSTEER(CoSTEER):
         if DS_RD_SETTING.enable_model_dump:
             eval_l.append(ModelDumpEvaluator(scen=scen, data_type="sample"))
 
+        for extra_eval in DSCoderCoSTEERSettings().extra_eval:
+            kls = import_class(extra_eval)
+            eval_l.append(kls(scen=scen))
+
         eva = CoSTEERMultiEvaluator(
             single_evaluator=eval_l, scen=scen
         )  # Please specify whether you agree running your eva in parallel or not