feat: fallback to acceptable results (#1129)

* refactor: add is_acceptable, fallback logic and generify evolving agent

* refine lint

* small

* lint

* lint

* lint

* feat: add is_acceptable to CoSTEERMultiFeedback

* feat: add in-memory workspace checkpoint and recovery

* feat: preserve symbolic links in workspace checkpoints and recovery

* lint

* lint

* feat: limit workspace checkpoint to files under 100KB

* feat: add workspace checkpoint size limit setting

* prompt

* lint

rdagent/components/coder/CoSTEER/__init__.py

@@ -1,4 +1,5 @@
 import pickle
+from copy import deepcopy
 from datetime import datetime
 from pathlib import Path
 
@@ -10,8 +11,7 @@ from rdagent.components.coder.CoSTEER.knowledge_management import (
     CoSTEERRAGStrategyV2,
 )
 from rdagent.core.developer import Developer
-from rdagent.core.evaluation import Evaluator
-from rdagent.core.evolving_agent import EvolvingStrategy, RAGEvoAgent
+from rdagent.core.evolving_agent import EvolvingStrategy, RAGEvaluator, RAGEvoAgent
 from rdagent.core.exception import CoderError
 from rdagent.core.experiment import Experiment
 from rdagent.log import rdagent_logger as logger
@@ -22,15 +22,13 @@ class CoSTEER(Developer[Experiment]):
     def __init__(
         self,
         settings: CoSTEERSettings,
-        eva: Evaluator,
+        eva: RAGEvaluator,
         es: EvolvingStrategy,
         *args,
         evolving_version: int = 2,
         max_seconds: int | None = None,
         with_knowledge: bool = True,
-        with_feedback: bool = True,
         knowledge_self_gen: bool = True,
-        filter_final_evo: bool = True,
         max_loop: int | None = None,
         **kwargs,
     ) -> None:
@@ -47,9 +45,7 @@ class CoSTEER(Developer[Experiment]):
         )
 
         self.with_knowledge = with_knowledge
-        self.with_feedback = with_feedback
         self.knowledge_self_gen = knowledge_self_gen
-        self.filter_final_evo = filter_final_evo
         self.evolving_strategy = es
         self.evaluator = eva
         self.evolving_version = evolving_version
@@ -71,25 +67,37 @@ class CoSTEER(Developer[Experiment]):
             )
         )
 
+    def _get_last_fb(self) -> CoSTEERMultiFeedback:
+        fb = self.evolve_agent.evolving_trace[-1].feedback
+        assert fb is not None, "feedback is None"
+        assert isinstance(fb, CoSTEERMultiFeedback), "feedback must be of type CoSTEERMultiFeedback"
+        return fb
+
     def develop(self, exp: Experiment) -> Experiment:
 
         # init intermediate items
         evo_exp = EvolvingItem.from_experiment(exp)
 
-        self.evolve_agent = RAGEvoAgent(
+        self.evolve_agent = RAGEvoAgent[EvolvingItem](
             max_loop=self.max_loop,
             evolving_strategy=self.evolving_strategy,
             rag=self.rag,
             with_knowledge=self.with_knowledge,
-            with_feedback=self.with_feedback,
+            with_feedback=True,
             knowledge_self_gen=self.knowledge_self_gen,
             enable_filelock=self.settings.enable_filelock,
             filelock_path=self.settings.filelock_path,
         )
 
+        # Evolving the solution
         start_datetime = datetime.now()
+        fallback_evo_exp = None
         for evo_exp in self.evolve_agent.multistep_evolve(evo_exp, self.evaluator):
             assert isinstance(evo_exp, Experiment)  # multiple inheritance
+            if self._get_last_fb().is_acceptable():
+                fallback_evo_exp = deepcopy(evo_exp)
+                fallback_evo_exp.create_ws_ckp()  # NOTE: creating checkpoints for saving files in the workspace to prevent inplace mutation.
+
             logger.log_object(evo_exp.sub_workspace_list, tag="evolving code")
             for sw in evo_exp.sub_workspace_list:
                 logger.info(f"evolving workspace: {sw}")
@@ -100,8 +108,16 @@ class CoSTEER(Developer[Experiment]):
                 logger.info("Global timer is timeout, stop evolving")
                 break
 
-        if self.with_feedback and self.filter_final_evo:
-            evo_exp = self._exp_postprocess_by_feedback(evo_exp, self.evolve_agent.evolving_trace[-1].feedback)
+        # if the final feedback is not finished(therefore acceptable), we will use the fallback solution.
+        try:
+            evo_exp = self._exp_postprocess_by_feedback(evo_exp, self._get_last_fb())
+        except CoderError:
+            if fallback_evo_exp is not None:
+                logger.info("Fallback to the fallback solution.")
+                evo_exp = fallback_evo_exp
+                evo_exp.recover_ws_ckp()  # NOTE: recovering checkpoints for restoring files in the workspace to prevent inplace mutation.
+            else:
+                raise
 
         exp.sub_workspace_list = evo_exp.sub_workspace_list
         exp.experiment_workspace = evo_exp.experiment_workspace

rdagent/components/coder/CoSTEER/evaluators.py

@@ -181,6 +181,9 @@ class CoSTEERMultiFeedback(Feedback):
     def __iter__(self):
         return iter(self.feedback_list)
 
+    def is_acceptable(self) -> bool:
+        return all(feedback.is_acceptable() for feedback in self.feedback_list)
+
     def finished(self) -> bool:
         """
         In some implementations, tasks may fail multiple times, leading agents to skip the implementation.

rdagent/components/coder/CoSTEER/evolvable_subjects.py

@@ -25,7 +25,7 @@ class EvolvingItem(Experiment, EvolvableSubjects):
             self.sub_gt_implementations = sub_gt_implementations
 
     @classmethod
-    def from_experiment(cls, exp: Experiment) -> Experiment:
+    def from_experiment(cls, exp: Experiment) -> "EvolvingItem":
         ei = cls(sub_tasks=exp.sub_tasks)
         ei.based_experiments = exp.based_experiments
         ei.experiment_workspace = exp.experiment_workspace

rdagent/core/conf.py

@@ -55,6 +55,11 @@ class RDAgentSettings(ExtendedBaseSettings):
 
     # workspace conf
     workspace_path: Path = Path.cwd() / "git_ignore_folder" / "RD-Agent_workspace"
+    workspace_ckp_size_limit: int = 0
+    """
+    the checkpoint for the workspace is a zip file.
+    0 (or any value <=0) means *no* size limit for files in workspace checkpoints
+    """
 
     # multi processing conf
     multi_proc_n: int = 1

rdagent/core/evaluation.py

@@ -12,6 +12,13 @@ class Feedback:
         The building process of feedback will should be in evaluator
     """
 
+    def is_acceptable(self) -> bool:
+        """
+        Sometimes, the solution is already acceptable, but we still want to refine it.
+        So we use different logic to determine whether the solution is acceptable or finished.
+        """
+        return self.__bool__()
+
     def finished(self) -> bool:
         """
         In some implementations, tasks may fail multiple times, leading agents to skip the implementation.

rdagent/core/evolving_agent.py

@@ -2,24 +2,21 @@ from __future__ import annotations
 
 from abc import ABC, abstractmethod
 from collections.abc import Generator
-from typing import TYPE_CHECKING, Any, Generic, TypeVar
+from contextlib import nullcontext
+from typing import Any, Generic, TypeVar
 
 from filelock import FileLock
 from tqdm import tqdm
 
-if TYPE_CHECKING:
-    from rdagent.core.evolving_framework import EvolvableSubjects
-
-from contextlib import nullcontext
-
 from rdagent.core.evaluation import EvaluableObj, Evaluator, Feedback
-from rdagent.core.evolving_framework import EvolvingStrategy, EvoStep
+from rdagent.core.evolving_framework import EvolvableSubjects, EvolvingStrategy, EvoStep
 from rdagent.log import rdagent_logger as logger
 
 ASpecificEvaluator = TypeVar("ASpecificEvaluator", bound=Evaluator)
+ASpecificEvolvableSubjects = TypeVar("ASpecificEvolvableSubjects", bound=EvolvableSubjects)
 
 
-class EvoAgent(ABC, Generic[ASpecificEvaluator]):
+class EvoAgent(ABC, Generic[ASpecificEvaluator, ASpecificEvolvableSubjects]):
 
     def __init__(self, max_loop: int, evolving_strategy: EvolvingStrategy) -> None:
         self.max_loop = max_loop
@@ -28,9 +25,9 @@ class EvoAgent(ABC, Generic[ASpecificEvaluator]):
     @abstractmethod
     def multistep_evolve(
         self,
-        evo: EvolvableSubjects,
+        evo: ASpecificEvolvableSubjects,
         eva: ASpecificEvaluator | Feedback,
-    ) -> Generator[EvolvableSubjects, None, None]:
+    ) -> Generator[ASpecificEvolvableSubjects, None, None]:
         """
         yield EvolvableSubjects for caller for easier process control and logging.
         """
@@ -47,7 +44,7 @@ class RAGEvaluator(Evaluator):
         raise NotImplementedError
 
 
-class RAGEvoAgent(EvoAgent[RAGEvaluator]):
+class RAGEvoAgent(EvoAgent[RAGEvaluator, ASpecificEvolvableSubjects], Generic[ASpecificEvolvableSubjects]):
 
     def __init__(
         self,
@@ -63,7 +60,7 @@ class RAGEvoAgent(EvoAgent[RAGEvaluator]):
     ) -> None:
         super().__init__(max_loop, evolving_strategy)
         self.rag = rag
-        self.evolving_trace: list[EvoStep] = []
+        self.evolving_trace: list[EvoStep[ASpecificEvolvableSubjects]] = []
         self.with_knowledge = with_knowledge
         self.with_feedback = with_feedback
         self.knowledge_self_gen = knowledge_self_gen
@@ -72,9 +69,9 @@ class RAGEvoAgent(EvoAgent[RAGEvaluator]):
 
     def multistep_evolve(
         self,
-        evo: EvolvableSubjects,
+        evo: ASpecificEvolvableSubjects,
         eva: RAGEvaluator | Feedback,
-    ) -> Generator[EvolvableSubjects, None, None]:
+    ) -> Generator[ASpecificEvolvableSubjects, None, None]:
         for evo_loop_id in tqdm(range(self.max_loop), "Implementing"):
             with logger.tag(f"evo_loop_{evo_loop_id}"):
                 # 1. RAG
@@ -91,7 +88,7 @@ class RAGEvoAgent(EvoAgent[RAGEvaluator]):
                 )
 
                 # 3. Pack evolve results
-                es = EvoStep(evo, queried_knowledge)
+                es = EvoStep[ASpecificEvolvableSubjects](evo, queried_knowledge)
 
                 # 4. Evaluation
                 if self.with_feedback:

rdagent/core/evolving_framework.py

@@ -3,7 +3,7 @@ from __future__ import annotations
 import copy
 from abc import ABC, abstractmethod
 from dataclasses import dataclass
-from typing import TYPE_CHECKING, Any
+from typing import TYPE_CHECKING, Any, Generic, TypeVar
 
 from rdagent.core.evaluation import EvaluableObj
 from rdagent.core.knowledge_base import KnowledgeBase
@@ -36,8 +36,11 @@ class EvolvableSubjects(EvaluableObj):
         return copy.deepcopy(self)
 
 
+ASpecificEvolvableSubjects = TypeVar("ASpecificEvolvableSubjects", bound=EvolvableSubjects)
+
+
 @dataclass
-class EvoStep:
+class EvoStep(Generic[ASpecificEvolvableSubjects]):
     """At a specific step,
     based on
     - previous trace
@@ -48,23 +51,24 @@ class EvoStep:
     (optional) After evaluation, we get feedback `feedback`.
     """
 
-    evolvable_subjects: EvolvableSubjects
+    evolvable_subjects: ASpecificEvolvableSubjects
+
     queried_knowledge: QueriedKnowledge | None = None
     feedback: Feedback | None = None
 
 
-class EvolvingStrategy(ABC):
+class EvolvingStrategy(ABC, Generic[ASpecificEvolvableSubjects]):
     def __init__(self, scen: Scenario) -> None:
         self.scen = scen
 
     @abstractmethod
     def evolve(
         self,
-        *evo: EvolvableSubjects,
-        evolving_trace: list[EvoStep] | None = None,
+        *evo: ASpecificEvolvableSubjects,
+        evolving_trace: list[EvoStep[ASpecificEvolvableSubjects]] | None = None,
         queried_knowledge: QueriedKnowledge | None = None,
         **kwargs: Any,
-    ) -> EvolvableSubjects:
+    ) -> ASpecificEvolvableSubjects:
         """The evolving trace is a list of (evolvable_subjects, feedback) ordered
         according to the time.
 
@@ -74,7 +78,7 @@ class EvolvingStrategy(ABC):
         """
 
 
-class RAGStrategy(ABC):
+class RAGStrategy(ABC, Generic[ASpecificEvolvableSubjects]):
     """Retrieval Augmentation Generation Strategy"""
 
     def __init__(self, *args: Any, **kwargs: Any) -> None:
@@ -91,7 +95,7 @@ class RAGStrategy(ABC):
     @abstractmethod
     def query(
         self,
-        evo: EvolvableSubjects,
+        evo: ASpecificEvolvableSubjects,
         evolving_trace: list[EvoStep],
         **kwargs: Any,
     ) -> QueriedKnowledge | None:
@@ -100,7 +104,7 @@ class RAGStrategy(ABC):
     @abstractmethod
     def generate_knowledge(
         self,
-        evolving_trace: list[EvoStep],
+        evolving_trace: list[EvoStep[ASpecificEvolvableSubjects]],
         *,
         return_knowledge: bool = False,
         **kwargs: Any,

rdagent/core/experiment.py

@@ -1,11 +1,13 @@
 from __future__ import annotations
 
+import io
 import os
 import platform
 import re
 import shutil
 import typing
 import uuid
+import zipfile
 from abc import ABC, abstractmethod
 from collections.abc import Sequence
 from copy import deepcopy
@@ -98,6 +100,19 @@ class Workspace(ABC, Generic[ASpecificTask, ASpecificFeedback]):
         Get all the code files in the workspace as a single string.
         """
 
+    # when the workspace is mutable inplace, provide support for creating checkpoints and recovering.
+    @abstractmethod
+    def create_ws_ckp(self) -> None:
+        """
+        Create an in-memory checkpoint of the workspace so it can be restored later.
+        """
+
+    @abstractmethod
+    def recover_ws_ckp(self) -> None:
+        """
+        Restore the workspace from the checkpoint created by :py:meth:`create_ws_ckp`.
+        """
+
 
 ASpecificWS = TypeVar("ASpecificWS", bound=Workspace)
 
@@ -138,6 +153,8 @@ class FBWorkspace(Workspace):
             {}
         )  # The code injected into the folder, store them in the variable to reproduce the former result
         self.workspace_path: Path = RD_AGENT_SETTINGS.workspace_path / uuid.uuid4().hex
+        # In-memory checkpoint data created by ``create_ws_ckp``.
+        self.ws_ckp: bytes | None = None
 
     @staticmethod
     def _format_code_dict(code_dict: dict[str, str]) -> str:
@@ -282,6 +299,58 @@ class FBWorkspace(Workspace):
         )
         return result
 
+    def create_ws_ckp(self) -> None:
+        """
+        Zip the contents of ``workspace_path`` and persist the archive on
+        ``self.ws_ckp`` for later restoration via :py:meth:`recover_ws_ckp`.
+        """
+        buf = io.BytesIO()
+        with zipfile.ZipFile(buf, "w", zipfile.ZIP_DEFLATED) as zf:
+            for file_path in self.workspace_path.rglob("*"):
+                # Only include regular files up to 100 KB so that the checkpoint
+                # remains lightweight. Larger files (for example, datasets) are
+                # expected to be recreated or mounted separately.
+                if file_path.is_symlink():
+                    # Preserve symbolic links within the archive
+                    zi = zipfile.ZipInfo(str(file_path.relative_to(self.workspace_path)))
+                    zi.create_system = 3  # indicates Unix
+                    zi.external_attr = 0o120777 << 16  # symlink file type + 0777 perms
+                    zf.writestr(zi, str(file_path.readlink()))
+                elif file_path.is_file():
+                    size_limit = RD_AGENT_SETTINGS.workspace_ckp_size_limit
+                    if size_limit <= 0 or file_path.stat().st_size <= size_limit:
+                        zf.write(file_path, file_path.relative_to(self.workspace_path))
+        self.ws_ckp = buf.getvalue()
+
+    def recover_ws_ckp(self) -> None:
+        """
+        Restore the workspace directory from the in-memory checkpoint created by
+        :py:meth:`create_ws_ckp`.
+        """
+        if self.ws_ckp is None:
+            msg = "Workspace checkpoint doesn't exist. Call `create_ws_ckp` first."
+            raise RuntimeError(msg)
+        shutil.rmtree(self.workspace_path, ignore_errors=True)
+        self.workspace_path.mkdir(parents=True, exist_ok=True)
+        buf = io.BytesIO(self.ws_ckp)
+        with zipfile.ZipFile(buf, "r") as zf:
+            for info in zf.infolist():
+                dest_path = self.workspace_path / info.filename
+                # File type bits (upper 4) are in high 16 bits of external_attr
+                mode = (info.external_attr >> 16) & 0o170000
+                symlink_mode = 0o120000  # Constant for symlink file type in Unix
+                if mode == symlink_mode:  # Symlink
+                    dest_path.parent.mkdir(parents=True, exist_ok=True)
+                    link_target = zf.read(info).decode()
+                    os.symlink(link_target, dest_path)
+                else:
+                    if info.is_dir():
+                        dest_path.mkdir(parents=True, exist_ok=True)
+                    else:
+                        dest_path.parent.mkdir(parents=True, exist_ok=True)
+                        with dest_path.open("wb") as f:
+                            f.write(zf.read(info))
+
     def __str__(self) -> str:
         return f"Workspace[{self.workspace_path=}" + (
             "]" if self.target_task is None else f",{self.target_task.name=}]"
@@ -355,6 +424,21 @@ class Experiment(
     def result(self, value: object) -> None:
         self.running_info.result = value
 
+    # when the workspace is mutable inplace, provide support for creating checkpoints and recovering.
+    def create_ws_ckp(self) -> None:
+        if self.experiment_workspace is not None:
+            self.experiment_workspace.create_ws_ckp()
+        for ws in self.sub_workspace_list:
+            if ws is not None:
+                ws.create_ws_ckp()
+
+    def recover_ws_ckp(self) -> None:
+        if self.experiment_workspace is not None:
+            self.experiment_workspace.recover_ws_ckp()
+        for ws in self.sub_workspace_list:
+            if ws is not None:
+                ws.recover_ws_ckp()
+
 
 ASpecificExp = TypeVar("ASpecificExp", bound=Experiment)
 ASpecificPlan = TypeVar("ASpecificPlan", bound=ExperimentPlan)

rdagent/scenarios/data_science/dev/runner/__init__.py

@@ -19,7 +19,7 @@ from rdagent.core.exception import RunnerError
 from rdagent.core.scenario import Scenario
 from rdagent.log import rdagent_logger as logger
 from rdagent.oai.llm_utils import APIBackend, md5_hash
-from rdagent.scenarios.data_science.dev.runner.eval import DSCoSTEERCoSTEEREvaluator
+from rdagent.scenarios.data_science.dev.runner.eval import DSRunnerEvaluator
 from rdagent.utils.agent.ret import PythonBatchEditOut, PythonBatchPatchOut
 from rdagent.utils.agent.tpl import T
 from rdagent.utils.workflow import wait_retry
@@ -128,7 +128,7 @@ class DSCoSTEERRunner(CoSTEER):
         **kwargs,
     ) -> None:
 
-        eval_l = [DSCoSTEERCoSTEEREvaluator(scen=scen)]
+        eval_l = [DSRunnerEvaluator(scen=scen)]
         if DS_RD_SETTING.enable_model_dump:
             eval_l.append(ModelDumpEvaluator(scen=scen, data_type="full"))
 

rdagent/scenarios/data_science/dev/runner/eval.py

@@ -1,5 +1,6 @@
 import json
 import re
+from dataclasses import dataclass
 from pathlib import Path
 
 import pandas as pd
@@ -26,21 +27,24 @@ from rdagent.utils.fmt import shrink_text
 DIRNAME = Path(__file__).absolute().resolve().parent
 
 
-class DSCoSTEEREvalFeedback(CoSTEERSingleFeedback):
+@dataclass
+class DSRunnerFeedback(CoSTEERSingleFeedback):
     """
     Feedback for Data Science CoSTEER evaluation.
     This feedback is used to evaluate the code and execution of the Data Science CoSTEER task.
     """
 
-    def __init__(
-        self, *args, hyperparameter_tuning_decision: bool = None, hyperparameter_tuning_suggestion: str = None, **kwargs
-    ):
-        super().__init__(*args, **kwargs)
-        self.hyperparameter_tuning_decision = hyperparameter_tuning_decision
-        self.hyperparameter_tuning_suggestion = hyperparameter_tuning_suggestion
+    acceptable: bool | None = None
+    hyperparameter_tuning_decision: bool | None = None
+    hyperparameter_tuning_suggestion: str | None = None
+
+    def is_acceptable(self) -> bool:
+        if self.acceptable is not None:
+            return self.acceptable
+        return super().is_acceptable()
 
 
-class DSCoSTEERCoSTEEREvaluator(CoSTEEREvaluator):
+class DSRunnerEvaluator(CoSTEEREvaluator):
 
     def evaluate(
         self,
@@ -49,7 +53,9 @@ class DSCoSTEERCoSTEEREvaluator(CoSTEEREvaluator):
         gt_implementation: FBWorkspace,
         queried_knowledge: QueriedKnowledge = None,
         **kwargs,
-    ) -> DSCoSTEEREvalFeedback:
+    ) -> DSRunnerFeedback:
+        # Only enalbe hyperparameter tuning on the first evaluation.
+        # Avoid too much time cunsumming.
         if len(queried_knowledge.task_to_former_failed_traces[target_task.get_task_information()][0]) == 0:
             enable_hyperparameter_tuning_check = True
         else:
@@ -158,10 +164,10 @@ class DSCoSTEERCoSTEEREvaluator(CoSTEEREvaluator):
         )
 
         feedback = build_cls_from_json_with_retry(
-            DSCoSTEEREvalFeedback,
+            DSRunnerFeedback,
             system_prompt=system_prompt,
             user_prompt=user_prompt,
-            init_kwargs_update_func=DSCoSTEEREvalFeedback.val_and_update_init_dict,
+            init_kwargs_update_func=DSRunnerFeedback.val_and_update_init_dict,
         )
 
         if feedback and not DS_RD_SETTING.coder_on_whole_pipeline:
@@ -180,7 +186,7 @@ class DSCoSTEERCoSTEEREvaluator(CoSTEEREvaluator):
                         break
 
                 if not use_one_model:
-                    feedback.final_decision = False
+                    feedback.acceptable = feedback.final_decision = False
                     logger.warning("No model script is used in `main.py`.")
                     feedback.code += "\n[Error] No model script is used in `main.py`."
 
@@ -196,7 +202,7 @@ class DSCoSTEERCoSTEEREvaluator(CoSTEEREvaluator):
                     logger.warning(f"Unused scripts: {unused_files}")
                     error_files = set(unused_files).intersection(set(must_have_files))
                     if error_files:
-                        feedback.final_decision = False
+                        feedback.acceptable = feedback.final_decision = False
                         logger.warning(f"{error_files} must be used in `main.py`.")
                         feedback.code += f"\n[Error] {error_files} must be used in `main.py`."
                     elif use_one_model:
@@ -204,9 +210,9 @@ class DSCoSTEERCoSTEEREvaluator(CoSTEEREvaluator):
                         implementation.inject_files(**{file: implementation.DEL_KEY for file in unused_files})
 
         if score_ret_code != 0:
-            feedback.final_decision = False
+            feedback.acceptable = feedback.final_decision = False
             feedback.return_checking += "\n" + score_check_text
         if submission_ret_code != 0:
-            feedback.final_decision = False
+            feedback.acceptable = feedback.final_decision = False
             feedback.return_checking += "\nSubmission file check failed."
         return feedback

rdagent/scenarios/data_science/dev/runner/prompts.yaml

@@ -24,10 +24,12 @@ DSCoSTEER_eval:
     2. Ensure the code does not contain any incorrect, fabricated, or deceptive operations, such as mocking data, scores, or results.
     3. Confirm that the prediction file (`submission.csv`) is generated using only the test dataset, and its format matches the sample submission.
     If the code does not satisfy the requirements:
+    - Set "acceptable" to false.
     - Set "final_decision" to false.
     {% if enable_hyperparameter_tuning_check %}- set "hyperparameter_tuning_decision" to false.
     - Set "hyperparameter_tuning_suggestion" to an empty string.
     If the code satisfy the requirements:
+    - Set "acceptable" to true.
     - Proceed to the next evaluation.
 
     # Evaluation 2: Hyperparameter
@@ -56,7 +58,8 @@ DSCoSTEER_eval:
     {
         "execution": "Describe whether the whole code base executed successfully and generating the final submission. Include any errors or issues encountered, and retain all error messages and traceback details.",
         "return_checking": "Verify the generated files, particularly the submission file. Ensure that its format matches the sample submission",
-        "code": "Provide feedback on code quality, readability, and adherence to the given specifications.",{% if enable_hyperparameter_tuning_check %}
+        "code": "Provide feedback on code quality, readability, and adherence to the given specifications.",
+        "acceptable": <true/false: if the solution has paased execution, return_checking, and code verification, then it is a valid solution and acceptable. Otherwise it is not acceptable.>,{% if enable_hyperparameter_tuning_check %}
         "hyperparameter_tuning_decision": <true/false>,
         "hyperparameter_tuning_suggestion": <suggestion in plain text for hyperparameter tuning>,{% endif %}
         "final_decision": <true/false>,
@@ -78,10 +81,12 @@ DSCoSTEER_eval:
     1. The code execution time or resource utilization suggest that there is room for improvement in the hyperparameters.
     2. The code must apply early stopping strategy already (in order to prevent overfitting).
     3. Your suggestion should have a strong chance of improving the model's performance. Focus on the most obvious and impactful opportunities for quick improvement by leveraging more training time. Don't explore hyperparameters with low confidence.  If there are no obvious and impactful opportunities and the code runs well, please accept it.
+
     If the code satisfy the requirements:
     - Set "hyperparameter_tuning_decision" to true.
-    - Set "final_decision" to false.
     - Provide a reasonable suggestion in "hyperparameter_tuning_suggestion". The "hyperparameter_tuning_suggestion" should begin with a clear observation, followed by your suggestion. For example: "[Observation] The maximum number of epochs was reached, but the validation loss is still going down and early stopping was not activated. Only 15% of the allowed time was used. [Suggestion] We recommend increasing epochs to 100 to avoid underfitting and further improve model performance."
+    - Set "final_decision" to false.
+
     If the code does not satisfy the requirements:
     - Set "hyperparameter_tuning_decision" to false.
     - Set "hyperparameter_tuning_suggestion" to an empty string.
@@ -93,6 +98,7 @@ DSCoSTEER_eval:
         "execution": "Describe whether the code executed successfully. Include any errors or issues encountered, and append all error messages and full traceback details without summarizing or omitting any information.",
         "return_checking": "Describe the expected file to be generated.",
         "code": "Provide feedback on code quality, readability, and adherence to the given specifications.",
+        "acceptable": <true/false: if the solution has paased execution, return_checking, and code verification, then it is a valid solution and acceptable. Otherwise it is not acceptable.>,
         {% if enable_hyperparameter_tuning_check %}"hyperparameter_tuning_decision": <true/false>,
         "hyperparameter_tuning_suggestion": <suggestion in plain text for hyperparameter tuning>,{% endif %}
         "final_decision": <true/false>,

test/utils/test_ws.py

@@ -0,0 +1,76 @@
+import os
+import tempfile
+import unittest
+from pathlib import Path
+
+from rdagent.core.experiment import FBWorkspace
+
+
+class TestFBWorkspace(unittest.TestCase):
+    """
+    Unit-tests for `FBWorkspace`.
+    """
+
+    def setUp(self) -> None:  # noqa: D401
+        """
+        Create an isolated temporary directory for each test case.
+        """
+        self._tmp_dir = tempfile.TemporaryDirectory()
+        self.tmp_path = Path(self._tmp_dir.name)
+
+    def tearDown(self) -> None:
+        """
+        Clean up the temporary directory created in :py:meth:`setUp`.
+        """
+        self._tmp_dir.cleanup()
+
+    def test_checkpoint_roundtrip(self) -> None:
+        """
+        Verify that ``create_ws_ckp`` captures the current workspace state and
+        ``recover_ws_ckp`` faithfully restores it.
+        """
+        # create a symbolic link inside workspace and ensure checkpoint preserves the link
+        external_file = self.tmp_path / "external.txt"
+        external_file.write_text("external data")
+        ws = FBWorkspace()
+        ws.workspace_path = self.tmp_path / "ws"
+        ws.prepare()
+        (ws.workspace_path / "sym.txt").symlink_to(external_file)
+        ws.inject_files(**{"foo.py": "print('hi')", "bar.py": "x = 1"})
+
+        # Snapshot current workspace
+        original_files = {
+            p.relative_to(ws.workspace_path): (os.readlink(p) if p.is_symlink() else p.read_text())
+            for p in ws.workspace_path.rglob("*")
+            if p.is_file() or p.is_symlink()
+        }
+        ws.create_ws_ckp()
+        self.assertIsNotNone(ws.ws_ckp, "Checkpoint data should have been generated")
+
+        # Mutate workspace
+        (ws.workspace_path / "foo.py").write_text("print('changed')")
+        (ws.workspace_path / "new.py").write_text("pass")
+        (ws.workspace_path / "sym.txt").unlink()
+
+        # Restore and verify equality with snapshot
+        ws.recover_ws_ckp()
+
+        # Ensure symbolic link still exists after recovery.
+        self.assertTrue((ws.workspace_path / "sym.txt").is_symlink())
+        recovered_files = {
+            p.relative_to(ws.workspace_path): (os.readlink(p) if p.is_symlink() else p.read_text())
+            for p in ws.workspace_path.rglob("*")
+            if p.is_file() or p.is_symlink()
+        }
+        self.assertEqual(recovered_files, original_files)
+
+        # Verify large files (>100 KB) are excluded when a size-limit is configured.
+        from rdagent.core.conf import RD_AGENT_SETTINGS as _SETTINGS
+
+        _SETTINGS.workspace_ckp_size_limit = 100 * 1024  # set limit temporarily for this test
+
+        large_file = ws.workspace_path / "large.bin"
+        large_file.write_bytes(b"0" * (110 * 1024))  # 110 KB dummy content
+        ws.create_ws_ckp()
+        ws.recover_ws_ckp()
+        self.assertFalse((ws.workspace_path / "large.bin").exists())