reporeformat V2 (#23)

* reformat factor implement process

* move some code to more reasonable place

* fix the bug

* add test function in factor_extract_and_implement.py

* change select factor number to ratio , add some factor implement setting and fix some bug while using knowledgebase

* change evoagent

* add abstract class EvoAgent

* add benchmark workflow

* fix some bug in llm_utils

* run wenjun's code

* fix the knowledgebase instance check

---------

Co-authored-by: xuyang1 <xuyang1@microsoft.com>

rdagent/core/conf.py

@@ -11,8 +11,10 @@ from pydantic_settings import BaseSettings
 # make sure that env variable is loaded while calling Config()
 load_dotenv(verbose=True, override=True)
 
+from pydantic_settings import BaseSettings
 
-class FincoSettings(BaseSettings):
+
+class RDAgentSettings(BaseSettings):
     use_azure: bool = True
     use_azure_token_provider: bool = False
     max_retry: int = 10
@@ -96,3 +98,4 @@ class FincoSettings(BaseSettings):
     # factor extraction conf
     max_input_duplicate_factor_group: int = 600
     max_output_duplicate_factor_group: int = 20
+

rdagent/core/evaluation.py

@@ -0,0 +1,16 @@
+from abc import ABC, abstractmethod
+from rdagent.core.task import (
+    TaskImplementation,
+    BaseTask,
+)
+
+class Evaluator(ABC):
+    @abstractmethod
+    def evaluate(
+        self,
+        target_task: BaseTask,
+        implementation: TaskImplementation,
+        gt_implementation: TaskImplementation,
+        **kwargs,
+    ):
+        raise NotImplementedError

rdagent/core/evolving_framework.py

@@ -91,6 +91,16 @@ class EvolvingStrategy(ABC):
         """
 
 
+class EvoAgent(ABC):
+    def __init__(self, max_loop, evolving_strategy) -> None:
+        self.max_loop = max_loop
+        self.evolving_strategy = evolving_strategy
+    
+    @abstractmethod
+    def multistep_evolve(self, evo: EvolvableSubjects, eva: Evaluator | Feedback, **kwargs: Any) -> EvolvableSubjects:
+        pass
+
+
 class RAGStrategy(ABC):
     """Retrival Augmentation Generation Strategy"""
 
@@ -119,66 +129,3 @@ class RAGStrategy(ABC):
 
         RAGStrategy should maintain the new knowledge all by itself.
         """
-
-
-class EvoAgent:
-    """It is responsible for driving the workflow."""
-
-    evolving_trace: list[EvoStep]
-
-    def __init__(
-        self,
-        evolving_strategy: EvolvingStrategy,
-        rag: RAGStrategy | None = None,
-    ) -> None:
-        self.evolving_trace = []
-        self.evolving_strategy = evolving_strategy
-        self.rag = rag
-
-    def step_evolving(
-        self,
-        evo: EvolvableSubjects,
-        eva: Evaluator | Feedback,
-        *,
-        with_knowledge: bool = False,
-        with_feedback: bool = True,
-        knowledge_self_gen: bool = False,
-    ) -> EvolvableSubjects:
-        """Common evolving mode are supported in this api .
-        - Interactive evolving:
-            - `with_feedback=True` and `eva` is a external Evaluator.
-
-        - Knowledge-driven evolving:
-            - `with_knowledge=True` and related knowledge are
-            queried based on `self.rag`
-
-        - Self-evolving: we have two ways to self-evolve.
-            - 1) self generating knowledge and then evolve
-                - `knowledge_self_gen=True` and `with_knowledge=True`
-            - 2) self evaluate to generate feedback and then evolve
-                - `with_feedback=True` and `eva` is a internal Evaluator.
-        """
-        # knowledge self-evolving
-        if knowledge_self_gen and self.rag is not None:
-            self.rag.generate_knowledge(self.evolving_trace)
-
-        # RAG
-        queried_knowledge = None
-        if with_knowledge and self.rag is not None:
-            queried_knowledge = self.rag.query(evo, self.evolving_trace)
-
-        # Evolve
-        evo = self.evolving_strategy.evolve(
-            evo=evo,
-            evolving_trace=self.evolving_trace,
-            queried_knowledge=queried_knowledge,
-        )
-        es = EvoStep(evo, queried_knowledge)
-
-        # Evaluate
-        if with_feedback:
-            es.feedback = eva if isinstance(eva, Feedback) else eva.evaluate(evo, queried_knowledge=queried_knowledge)
-
-        # Update trace
-        self.evolving_trace.append(es)
-        return evo

rdagent/core/exception.py

@@ -0,0 +1,26 @@
+class ImplementRunException(Exception):
+    """
+    Exceptions raised when Implementing and running code.
+    - start: FactorImplementationTask => FactorGenerator
+    - end: Get dataframe after execution
+
+    The more detailed evaluation in dataframe values are managed by the evaluator.
+    """
+
+
+class CodeFormatException(ImplementRunException):
+    """
+    The generated code is not found due format error.
+    """
+
+
+class RuntimeErrorException(ImplementRunException):
+    """
+    The generated code fail to execute the script.
+    """
+
+
+class NoOutputException(ImplementRunException):
+    """
+    The code fail to generate output file.
+    """

rdagent/core/implementation.py

@@ -0,0 +1,24 @@
+from abc import ABC, abstractmethod
+from typing import List
+
+from rdagent.core.task import (
+    TaskImplementation,
+)
+
+class TaskGenerator(ABC):
+    @abstractmethod
+    def generate(self, *args, **kwargs) -> List[TaskImplementation]:
+        raise NotImplementedError("generate method is not implemented.")
+
+    def collect_feedback(self, feedback_obj_l: List[object]):
+        """
+        When online evaluation.
+        The preivous feedbacks will be collected to support advanced factor generator
+
+        Parameters
+        ----------
+        feedback_obj_l : List[object]
+
+        """
+
+

rdagent/core/task.py

@@ -0,0 +1,28 @@
+from abc import ABC, abstractmethod
+from typing import Tuple
+import pandas as pd
+
+'''
+This file contains the all the data class for rdagent task.
+'''
+class BaseTask(ABC):
+    # 把name放在这里作为主键
+    pass
+
+class TaskImplementation(ABC):
+    def __init__(self, target_task: BaseTask) -> None:
+        self.target_task = target_task
+
+    @abstractmethod
+    def execute(self, *args, **kwargs) -> Tuple[str, pd.DataFrame]:
+        raise NotImplementedError("__call__ method is not implemented.")
+
+class TestCase:
+    def __init__(
+        self,
+        target_task: BaseTask,
+        ground_truth: TaskImplementation,
+    ):
+        self.ground_truth = ground_truth
+        self.target_task = target_task
+