feat: fallback to acceptable results (#1129)

* refactor: add is_acceptable, fallback logic and generify evolving agent

* refine lint

* small

* lint

* lint

* lint

* feat: add is_acceptable to CoSTEERMultiFeedback

* feat: add in-memory workspace checkpoint and recovery

* feat: preserve symbolic links in workspace checkpoints and recovery

* lint

* lint

* feat: limit workspace checkpoint to files under 100KB

* feat: add workspace checkpoint size limit setting

* prompt

* lint

rdagent/core/evolving_agent.py

@@ -2,24 +2,21 @@ from __future__ import annotations
 
 from abc import ABC, abstractmethod
 from collections.abc import Generator
-from typing import TYPE_CHECKING, Any, Generic, TypeVar
+from contextlib import nullcontext
+from typing import Any, Generic, TypeVar
 
 from filelock import FileLock
 from tqdm import tqdm
 
-if TYPE_CHECKING:
-    from rdagent.core.evolving_framework import EvolvableSubjects
-
-from contextlib import nullcontext
-
 from rdagent.core.evaluation import EvaluableObj, Evaluator, Feedback
-from rdagent.core.evolving_framework import EvolvingStrategy, EvoStep
+from rdagent.core.evolving_framework import EvolvableSubjects, EvolvingStrategy, EvoStep
 from rdagent.log import rdagent_logger as logger
 
 ASpecificEvaluator = TypeVar("ASpecificEvaluator", bound=Evaluator)
+ASpecificEvolvableSubjects = TypeVar("ASpecificEvolvableSubjects", bound=EvolvableSubjects)
 
 
-class EvoAgent(ABC, Generic[ASpecificEvaluator]):
+class EvoAgent(ABC, Generic[ASpecificEvaluator, ASpecificEvolvableSubjects]):
 
     def __init__(self, max_loop: int, evolving_strategy: EvolvingStrategy) -> None:
         self.max_loop = max_loop
@@ -28,9 +25,9 @@ class EvoAgent(ABC, Generic[ASpecificEvaluator]):
     @abstractmethod
     def multistep_evolve(
         self,
-        evo: EvolvableSubjects,
+        evo: ASpecificEvolvableSubjects,
         eva: ASpecificEvaluator | Feedback,
-    ) -> Generator[EvolvableSubjects, None, None]:
+    ) -> Generator[ASpecificEvolvableSubjects, None, None]:
         """
         yield EvolvableSubjects for caller for easier process control and logging.
         """
@@ -47,7 +44,7 @@ class RAGEvaluator(Evaluator):
         raise NotImplementedError
 
 
-class RAGEvoAgent(EvoAgent[RAGEvaluator]):
+class RAGEvoAgent(EvoAgent[RAGEvaluator, ASpecificEvolvableSubjects], Generic[ASpecificEvolvableSubjects]):
 
     def __init__(
         self,
@@ -63,7 +60,7 @@ class RAGEvoAgent(EvoAgent[RAGEvaluator]):
     ) -> None:
         super().__init__(max_loop, evolving_strategy)
         self.rag = rag
-        self.evolving_trace: list[EvoStep] = []
+        self.evolving_trace: list[EvoStep[ASpecificEvolvableSubjects]] = []
         self.with_knowledge = with_knowledge
         self.with_feedback = with_feedback
         self.knowledge_self_gen = knowledge_self_gen
@@ -72,9 +69,9 @@ class RAGEvoAgent(EvoAgent[RAGEvaluator]):
 
     def multistep_evolve(
         self,
-        evo: EvolvableSubjects,
+        evo: ASpecificEvolvableSubjects,
         eva: RAGEvaluator | Feedback,
-    ) -> Generator[EvolvableSubjects, None, None]:
+    ) -> Generator[ASpecificEvolvableSubjects, None, None]:
         for evo_loop_id in tqdm(range(self.max_loop), "Implementing"):
             with logger.tag(f"evo_loop_{evo_loop_id}"):
                 # 1. RAG
@@ -91,7 +88,7 @@ class RAGEvoAgent(EvoAgent[RAGEvaluator]):
                 )
 
                 # 3. Pack evolve results
-                es = EvoStep(evo, queried_knowledge)
+                es = EvoStep[ASpecificEvolvableSubjects](evo, queried_knowledge)
 
                 # 4. Evaluation
                 if self.with_feedback: