From 684e41f43a32f7650e7e4ef727643c70192eee2d Mon Sep 17 00:00:00 2001 From: you-n-g Date: Thu, 31 Jul 2025 17:53:18 +0800 Subject: [PATCH] feat: fallback to acceptable results (#1129) * refactor: add is_acceptable, fallback logic and generify evolving agent * refine lint * small * lint * lint * lint * feat: add is_acceptable to CoSTEERMultiFeedback * feat: add in-memory workspace checkpoint and recovery * feat: preserve symbolic links in workspace checkpoints and recovery * lint * lint * feat: limit workspace checkpoint to files under 100KB * feat: add workspace checkpoint size limit setting * prompt * lint --- rdagent/components/coder/CoSTEER/__init__.py | 38 ++++++--- .../components/coder/CoSTEER/evaluators.py | 3 + .../coder/CoSTEER/evolvable_subjects.py | 2 +- rdagent/core/conf.py | 5 ++ rdagent/core/evaluation.py | 7 ++ rdagent/core/evolving_agent.py | 27 +++--- rdagent/core/evolving_framework.py | 24 +++--- rdagent/core/experiment.py | 84 +++++++++++++++++++ .../data_science/dev/runner/__init__.py | 4 +- .../scenarios/data_science/dev/runner/eval.py | 36 ++++---- .../data_science/dev/runner/prompts.yaml | 10 ++- test/utils/test_ws.py | 76 +++++++++++++++++ 12 files changed, 260 insertions(+), 56 deletions(-) create mode 100644 test/utils/test_ws.py diff --git a/rdagent/components/coder/CoSTEER/__init__.py b/rdagent/components/coder/CoSTEER/__init__.py index 6fdfaf80..6e59dbdd 100644 --- a/rdagent/components/coder/CoSTEER/__init__.py +++ b/rdagent/components/coder/CoSTEER/__init__.py @@ -1,4 +1,5 @@ import pickle +from copy import deepcopy from datetime import datetime from pathlib import Path @@ -10,8 +11,7 @@ from rdagent.components.coder.CoSTEER.knowledge_management import ( CoSTEERRAGStrategyV2, ) from rdagent.core.developer import Developer -from rdagent.core.evaluation import Evaluator -from rdagent.core.evolving_agent import EvolvingStrategy, RAGEvoAgent +from rdagent.core.evolving_agent import EvolvingStrategy, RAGEvaluator, RAGEvoAgent from rdagent.core.exception import CoderError from rdagent.core.experiment import Experiment from rdagent.log import rdagent_logger as logger @@ -22,15 +22,13 @@ class CoSTEER(Developer[Experiment]): def __init__( self, settings: CoSTEERSettings, - eva: Evaluator, + eva: RAGEvaluator, es: EvolvingStrategy, *args, evolving_version: int = 2, max_seconds: int | None = None, with_knowledge: bool = True, - with_feedback: bool = True, knowledge_self_gen: bool = True, - filter_final_evo: bool = True, max_loop: int | None = None, **kwargs, ) -> None: @@ -47,9 +45,7 @@ class CoSTEER(Developer[Experiment]): ) self.with_knowledge = with_knowledge - self.with_feedback = with_feedback self.knowledge_self_gen = knowledge_self_gen - self.filter_final_evo = filter_final_evo self.evolving_strategy = es self.evaluator = eva self.evolving_version = evolving_version @@ -71,25 +67,37 @@ class CoSTEER(Developer[Experiment]): ) ) + def _get_last_fb(self) -> CoSTEERMultiFeedback: + fb = self.evolve_agent.evolving_trace[-1].feedback + assert fb is not None, "feedback is None" + assert isinstance(fb, CoSTEERMultiFeedback), "feedback must be of type CoSTEERMultiFeedback" + return fb + def develop(self, exp: Experiment) -> Experiment: # init intermediate items evo_exp = EvolvingItem.from_experiment(exp) - self.evolve_agent = RAGEvoAgent( + self.evolve_agent = RAGEvoAgent[EvolvingItem]( max_loop=self.max_loop, evolving_strategy=self.evolving_strategy, rag=self.rag, with_knowledge=self.with_knowledge, - with_feedback=self.with_feedback, + with_feedback=True, knowledge_self_gen=self.knowledge_self_gen, enable_filelock=self.settings.enable_filelock, filelock_path=self.settings.filelock_path, ) + # Evolving the solution start_datetime = datetime.now() + fallback_evo_exp = None for evo_exp in self.evolve_agent.multistep_evolve(evo_exp, self.evaluator): assert isinstance(evo_exp, Experiment) # multiple inheritance + if self._get_last_fb().is_acceptable(): + fallback_evo_exp = deepcopy(evo_exp) + fallback_evo_exp.create_ws_ckp() # NOTE: creating checkpoints for saving files in the workspace to prevent inplace mutation. + logger.log_object(evo_exp.sub_workspace_list, tag="evolving code") for sw in evo_exp.sub_workspace_list: logger.info(f"evolving workspace: {sw}") @@ -100,8 +108,16 @@ class CoSTEER(Developer[Experiment]): logger.info("Global timer is timeout, stop evolving") break - if self.with_feedback and self.filter_final_evo: - evo_exp = self._exp_postprocess_by_feedback(evo_exp, self.evolve_agent.evolving_trace[-1].feedback) + # if the final feedback is not finished(therefore acceptable), we will use the fallback solution. + try: + evo_exp = self._exp_postprocess_by_feedback(evo_exp, self._get_last_fb()) + except CoderError: + if fallback_evo_exp is not None: + logger.info("Fallback to the fallback solution.") + evo_exp = fallback_evo_exp + evo_exp.recover_ws_ckp() # NOTE: recovering checkpoints for restoring files in the workspace to prevent inplace mutation. + else: + raise exp.sub_workspace_list = evo_exp.sub_workspace_list exp.experiment_workspace = evo_exp.experiment_workspace diff --git a/rdagent/components/coder/CoSTEER/evaluators.py b/rdagent/components/coder/CoSTEER/evaluators.py index faa528b1..6f68da34 100644 --- a/rdagent/components/coder/CoSTEER/evaluators.py +++ b/rdagent/components/coder/CoSTEER/evaluators.py @@ -181,6 +181,9 @@ class CoSTEERMultiFeedback(Feedback): def __iter__(self): return iter(self.feedback_list) + def is_acceptable(self) -> bool: + return all(feedback.is_acceptable() for feedback in self.feedback_list) + def finished(self) -> bool: """ In some implementations, tasks may fail multiple times, leading agents to skip the implementation. diff --git a/rdagent/components/coder/CoSTEER/evolvable_subjects.py b/rdagent/components/coder/CoSTEER/evolvable_subjects.py index 87b3fa2a..0404ed1e 100644 --- a/rdagent/components/coder/CoSTEER/evolvable_subjects.py +++ b/rdagent/components/coder/CoSTEER/evolvable_subjects.py @@ -25,7 +25,7 @@ class EvolvingItem(Experiment, EvolvableSubjects): self.sub_gt_implementations = sub_gt_implementations @classmethod - def from_experiment(cls, exp: Experiment) -> Experiment: + def from_experiment(cls, exp: Experiment) -> "EvolvingItem": ei = cls(sub_tasks=exp.sub_tasks) ei.based_experiments = exp.based_experiments ei.experiment_workspace = exp.experiment_workspace diff --git a/rdagent/core/conf.py b/rdagent/core/conf.py index c006672a..b1efe7b6 100644 --- a/rdagent/core/conf.py +++ b/rdagent/core/conf.py @@ -55,6 +55,11 @@ class RDAgentSettings(ExtendedBaseSettings): # workspace conf workspace_path: Path = Path.cwd() / "git_ignore_folder" / "RD-Agent_workspace" + workspace_ckp_size_limit: int = 0 + """ + the checkpoint for the workspace is a zip file. + 0 (or any value <=0) means *no* size limit for files in workspace checkpoints + """ # multi processing conf multi_proc_n: int = 1 diff --git a/rdagent/core/evaluation.py b/rdagent/core/evaluation.py index fa50b3b5..07783067 100644 --- a/rdagent/core/evaluation.py +++ b/rdagent/core/evaluation.py @@ -12,6 +12,13 @@ class Feedback: The building process of feedback will should be in evaluator """ + def is_acceptable(self) -> bool: + """ + Sometimes, the solution is already acceptable, but we still want to refine it. + So we use different logic to determine whether the solution is acceptable or finished. + """ + return self.__bool__() + def finished(self) -> bool: """ In some implementations, tasks may fail multiple times, leading agents to skip the implementation. diff --git a/rdagent/core/evolving_agent.py b/rdagent/core/evolving_agent.py index ddda48fb..145bb87a 100644 --- a/rdagent/core/evolving_agent.py +++ b/rdagent/core/evolving_agent.py @@ -2,24 +2,21 @@ from __future__ import annotations from abc import ABC, abstractmethod from collections.abc import Generator -from typing import TYPE_CHECKING, Any, Generic, TypeVar +from contextlib import nullcontext +from typing import Any, Generic, TypeVar from filelock import FileLock from tqdm import tqdm -if TYPE_CHECKING: - from rdagent.core.evolving_framework import EvolvableSubjects - -from contextlib import nullcontext - from rdagent.core.evaluation import EvaluableObj, Evaluator, Feedback -from rdagent.core.evolving_framework import EvolvingStrategy, EvoStep +from rdagent.core.evolving_framework import EvolvableSubjects, EvolvingStrategy, EvoStep from rdagent.log import rdagent_logger as logger ASpecificEvaluator = TypeVar("ASpecificEvaluator", bound=Evaluator) +ASpecificEvolvableSubjects = TypeVar("ASpecificEvolvableSubjects", bound=EvolvableSubjects) -class EvoAgent(ABC, Generic[ASpecificEvaluator]): +class EvoAgent(ABC, Generic[ASpecificEvaluator, ASpecificEvolvableSubjects]): def __init__(self, max_loop: int, evolving_strategy: EvolvingStrategy) -> None: self.max_loop = max_loop @@ -28,9 +25,9 @@ class EvoAgent(ABC, Generic[ASpecificEvaluator]): @abstractmethod def multistep_evolve( self, - evo: EvolvableSubjects, + evo: ASpecificEvolvableSubjects, eva: ASpecificEvaluator | Feedback, - ) -> Generator[EvolvableSubjects, None, None]: + ) -> Generator[ASpecificEvolvableSubjects, None, None]: """ yield EvolvableSubjects for caller for easier process control and logging. """ @@ -47,7 +44,7 @@ class RAGEvaluator(Evaluator): raise NotImplementedError -class RAGEvoAgent(EvoAgent[RAGEvaluator]): +class RAGEvoAgent(EvoAgent[RAGEvaluator, ASpecificEvolvableSubjects], Generic[ASpecificEvolvableSubjects]): def __init__( self, @@ -63,7 +60,7 @@ class RAGEvoAgent(EvoAgent[RAGEvaluator]): ) -> None: super().__init__(max_loop, evolving_strategy) self.rag = rag - self.evolving_trace: list[EvoStep] = [] + self.evolving_trace: list[EvoStep[ASpecificEvolvableSubjects]] = [] self.with_knowledge = with_knowledge self.with_feedback = with_feedback self.knowledge_self_gen = knowledge_self_gen @@ -72,9 +69,9 @@ class RAGEvoAgent(EvoAgent[RAGEvaluator]): def multistep_evolve( self, - evo: EvolvableSubjects, + evo: ASpecificEvolvableSubjects, eva: RAGEvaluator | Feedback, - ) -> Generator[EvolvableSubjects, None, None]: + ) -> Generator[ASpecificEvolvableSubjects, None, None]: for evo_loop_id in tqdm(range(self.max_loop), "Implementing"): with logger.tag(f"evo_loop_{evo_loop_id}"): # 1. RAG @@ -91,7 +88,7 @@ class RAGEvoAgent(EvoAgent[RAGEvaluator]): ) # 3. Pack evolve results - es = EvoStep(evo, queried_knowledge) + es = EvoStep[ASpecificEvolvableSubjects](evo, queried_knowledge) # 4. Evaluation if self.with_feedback: diff --git a/rdagent/core/evolving_framework.py b/rdagent/core/evolving_framework.py index 692287f5..b0ae68d3 100644 --- a/rdagent/core/evolving_framework.py +++ b/rdagent/core/evolving_framework.py @@ -3,7 +3,7 @@ from __future__ import annotations import copy from abc import ABC, abstractmethod from dataclasses import dataclass -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, Generic, TypeVar from rdagent.core.evaluation import EvaluableObj from rdagent.core.knowledge_base import KnowledgeBase @@ -36,8 +36,11 @@ class EvolvableSubjects(EvaluableObj): return copy.deepcopy(self) +ASpecificEvolvableSubjects = TypeVar("ASpecificEvolvableSubjects", bound=EvolvableSubjects) + + @dataclass -class EvoStep: +class EvoStep(Generic[ASpecificEvolvableSubjects]): """At a specific step, based on - previous trace @@ -48,23 +51,24 @@ class EvoStep: (optional) After evaluation, we get feedback `feedback`. """ - evolvable_subjects: EvolvableSubjects + evolvable_subjects: ASpecificEvolvableSubjects + queried_knowledge: QueriedKnowledge | None = None feedback: Feedback | None = None -class EvolvingStrategy(ABC): +class EvolvingStrategy(ABC, Generic[ASpecificEvolvableSubjects]): def __init__(self, scen: Scenario) -> None: self.scen = scen @abstractmethod def evolve( self, - *evo: EvolvableSubjects, - evolving_trace: list[EvoStep] | None = None, + *evo: ASpecificEvolvableSubjects, + evolving_trace: list[EvoStep[ASpecificEvolvableSubjects]] | None = None, queried_knowledge: QueriedKnowledge | None = None, **kwargs: Any, - ) -> EvolvableSubjects: + ) -> ASpecificEvolvableSubjects: """The evolving trace is a list of (evolvable_subjects, feedback) ordered according to the time. @@ -74,7 +78,7 @@ class EvolvingStrategy(ABC): """ -class RAGStrategy(ABC): +class RAGStrategy(ABC, Generic[ASpecificEvolvableSubjects]): """Retrieval Augmentation Generation Strategy""" def __init__(self, *args: Any, **kwargs: Any) -> None: @@ -91,7 +95,7 @@ class RAGStrategy(ABC): @abstractmethod def query( self, - evo: EvolvableSubjects, + evo: ASpecificEvolvableSubjects, evolving_trace: list[EvoStep], **kwargs: Any, ) -> QueriedKnowledge | None: @@ -100,7 +104,7 @@ class RAGStrategy(ABC): @abstractmethod def generate_knowledge( self, - evolving_trace: list[EvoStep], + evolving_trace: list[EvoStep[ASpecificEvolvableSubjects]], *, return_knowledge: bool = False, **kwargs: Any, diff --git a/rdagent/core/experiment.py b/rdagent/core/experiment.py index 741f5fd1..78a7cbed 100644 --- a/rdagent/core/experiment.py +++ b/rdagent/core/experiment.py @@ -1,11 +1,13 @@ from __future__ import annotations +import io import os import platform import re import shutil import typing import uuid +import zipfile from abc import ABC, abstractmethod from collections.abc import Sequence from copy import deepcopy @@ -98,6 +100,19 @@ class Workspace(ABC, Generic[ASpecificTask, ASpecificFeedback]): Get all the code files in the workspace as a single string. """ + # when the workspace is mutable inplace, provide support for creating checkpoints and recovering. + @abstractmethod + def create_ws_ckp(self) -> None: + """ + Create an in-memory checkpoint of the workspace so it can be restored later. + """ + + @abstractmethod + def recover_ws_ckp(self) -> None: + """ + Restore the workspace from the checkpoint created by :py:meth:`create_ws_ckp`. + """ + ASpecificWS = TypeVar("ASpecificWS", bound=Workspace) @@ -138,6 +153,8 @@ class FBWorkspace(Workspace): {} ) # The code injected into the folder, store them in the variable to reproduce the former result self.workspace_path: Path = RD_AGENT_SETTINGS.workspace_path / uuid.uuid4().hex + # In-memory checkpoint data created by ``create_ws_ckp``. + self.ws_ckp: bytes | None = None @staticmethod def _format_code_dict(code_dict: dict[str, str]) -> str: @@ -282,6 +299,58 @@ class FBWorkspace(Workspace): ) return result + def create_ws_ckp(self) -> None: + """ + Zip the contents of ``workspace_path`` and persist the archive on + ``self.ws_ckp`` for later restoration via :py:meth:`recover_ws_ckp`. + """ + buf = io.BytesIO() + with zipfile.ZipFile(buf, "w", zipfile.ZIP_DEFLATED) as zf: + for file_path in self.workspace_path.rglob("*"): + # Only include regular files up to 100 KB so that the checkpoint + # remains lightweight. Larger files (for example, datasets) are + # expected to be recreated or mounted separately. + if file_path.is_symlink(): + # Preserve symbolic links within the archive + zi = zipfile.ZipInfo(str(file_path.relative_to(self.workspace_path))) + zi.create_system = 3 # indicates Unix + zi.external_attr = 0o120777 << 16 # symlink file type + 0777 perms + zf.writestr(zi, str(file_path.readlink())) + elif file_path.is_file(): + size_limit = RD_AGENT_SETTINGS.workspace_ckp_size_limit + if size_limit <= 0 or file_path.stat().st_size <= size_limit: + zf.write(file_path, file_path.relative_to(self.workspace_path)) + self.ws_ckp = buf.getvalue() + + def recover_ws_ckp(self) -> None: + """ + Restore the workspace directory from the in-memory checkpoint created by + :py:meth:`create_ws_ckp`. + """ + if self.ws_ckp is None: + msg = "Workspace checkpoint doesn't exist. Call `create_ws_ckp` first." + raise RuntimeError(msg) + shutil.rmtree(self.workspace_path, ignore_errors=True) + self.workspace_path.mkdir(parents=True, exist_ok=True) + buf = io.BytesIO(self.ws_ckp) + with zipfile.ZipFile(buf, "r") as zf: + for info in zf.infolist(): + dest_path = self.workspace_path / info.filename + # File type bits (upper 4) are in high 16 bits of external_attr + mode = (info.external_attr >> 16) & 0o170000 + symlink_mode = 0o120000 # Constant for symlink file type in Unix + if mode == symlink_mode: # Symlink + dest_path.parent.mkdir(parents=True, exist_ok=True) + link_target = zf.read(info).decode() + os.symlink(link_target, dest_path) + else: + if info.is_dir(): + dest_path.mkdir(parents=True, exist_ok=True) + else: + dest_path.parent.mkdir(parents=True, exist_ok=True) + with dest_path.open("wb") as f: + f.write(zf.read(info)) + def __str__(self) -> str: return f"Workspace[{self.workspace_path=}" + ( "]" if self.target_task is None else f",{self.target_task.name=}]" @@ -355,6 +424,21 @@ class Experiment( def result(self, value: object) -> None: self.running_info.result = value + # when the workspace is mutable inplace, provide support for creating checkpoints and recovering. + def create_ws_ckp(self) -> None: + if self.experiment_workspace is not None: + self.experiment_workspace.create_ws_ckp() + for ws in self.sub_workspace_list: + if ws is not None: + ws.create_ws_ckp() + + def recover_ws_ckp(self) -> None: + if self.experiment_workspace is not None: + self.experiment_workspace.recover_ws_ckp() + for ws in self.sub_workspace_list: + if ws is not None: + ws.recover_ws_ckp() + ASpecificExp = TypeVar("ASpecificExp", bound=Experiment) ASpecificPlan = TypeVar("ASpecificPlan", bound=ExperimentPlan) diff --git a/rdagent/scenarios/data_science/dev/runner/__init__.py b/rdagent/scenarios/data_science/dev/runner/__init__.py index 36fc946a..511f5eca 100644 --- a/rdagent/scenarios/data_science/dev/runner/__init__.py +++ b/rdagent/scenarios/data_science/dev/runner/__init__.py @@ -19,7 +19,7 @@ from rdagent.core.exception import RunnerError from rdagent.core.scenario import Scenario from rdagent.log import rdagent_logger as logger from rdagent.oai.llm_utils import APIBackend, md5_hash -from rdagent.scenarios.data_science.dev.runner.eval import DSCoSTEERCoSTEEREvaluator +from rdagent.scenarios.data_science.dev.runner.eval import DSRunnerEvaluator from rdagent.utils.agent.ret import PythonBatchEditOut, PythonBatchPatchOut from rdagent.utils.agent.tpl import T from rdagent.utils.workflow import wait_retry @@ -128,7 +128,7 @@ class DSCoSTEERRunner(CoSTEER): **kwargs, ) -> None: - eval_l = [DSCoSTEERCoSTEEREvaluator(scen=scen)] + eval_l = [DSRunnerEvaluator(scen=scen)] if DS_RD_SETTING.enable_model_dump: eval_l.append(ModelDumpEvaluator(scen=scen, data_type="full")) diff --git a/rdagent/scenarios/data_science/dev/runner/eval.py b/rdagent/scenarios/data_science/dev/runner/eval.py index 35ef6713..12ac0faf 100644 --- a/rdagent/scenarios/data_science/dev/runner/eval.py +++ b/rdagent/scenarios/data_science/dev/runner/eval.py @@ -1,5 +1,6 @@ import json import re +from dataclasses import dataclass from pathlib import Path import pandas as pd @@ -26,21 +27,24 @@ from rdagent.utils.fmt import shrink_text DIRNAME = Path(__file__).absolute().resolve().parent -class DSCoSTEEREvalFeedback(CoSTEERSingleFeedback): +@dataclass +class DSRunnerFeedback(CoSTEERSingleFeedback): """ Feedback for Data Science CoSTEER evaluation. This feedback is used to evaluate the code and execution of the Data Science CoSTEER task. """ - def __init__( - self, *args, hyperparameter_tuning_decision: bool = None, hyperparameter_tuning_suggestion: str = None, **kwargs - ): - super().__init__(*args, **kwargs) - self.hyperparameter_tuning_decision = hyperparameter_tuning_decision - self.hyperparameter_tuning_suggestion = hyperparameter_tuning_suggestion + acceptable: bool | None = None + hyperparameter_tuning_decision: bool | None = None + hyperparameter_tuning_suggestion: str | None = None + + def is_acceptable(self) -> bool: + if self.acceptable is not None: + return self.acceptable + return super().is_acceptable() -class DSCoSTEERCoSTEEREvaluator(CoSTEEREvaluator): +class DSRunnerEvaluator(CoSTEEREvaluator): def evaluate( self, @@ -49,7 +53,9 @@ class DSCoSTEERCoSTEEREvaluator(CoSTEEREvaluator): gt_implementation: FBWorkspace, queried_knowledge: QueriedKnowledge = None, **kwargs, - ) -> DSCoSTEEREvalFeedback: + ) -> DSRunnerFeedback: + # Only enalbe hyperparameter tuning on the first evaluation. + # Avoid too much time cunsumming. if len(queried_knowledge.task_to_former_failed_traces[target_task.get_task_information()][0]) == 0: enable_hyperparameter_tuning_check = True else: @@ -158,10 +164,10 @@ class DSCoSTEERCoSTEEREvaluator(CoSTEEREvaluator): ) feedback = build_cls_from_json_with_retry( - DSCoSTEEREvalFeedback, + DSRunnerFeedback, system_prompt=system_prompt, user_prompt=user_prompt, - init_kwargs_update_func=DSCoSTEEREvalFeedback.val_and_update_init_dict, + init_kwargs_update_func=DSRunnerFeedback.val_and_update_init_dict, ) if feedback and not DS_RD_SETTING.coder_on_whole_pipeline: @@ -180,7 +186,7 @@ class DSCoSTEERCoSTEEREvaluator(CoSTEEREvaluator): break if not use_one_model: - feedback.final_decision = False + feedback.acceptable = feedback.final_decision = False logger.warning("No model script is used in `main.py`.") feedback.code += "\n[Error] No model script is used in `main.py`." @@ -196,7 +202,7 @@ class DSCoSTEERCoSTEEREvaluator(CoSTEEREvaluator): logger.warning(f"Unused scripts: {unused_files}") error_files = set(unused_files).intersection(set(must_have_files)) if error_files: - feedback.final_decision = False + feedback.acceptable = feedback.final_decision = False logger.warning(f"{error_files} must be used in `main.py`.") feedback.code += f"\n[Error] {error_files} must be used in `main.py`." elif use_one_model: @@ -204,9 +210,9 @@ class DSCoSTEERCoSTEEREvaluator(CoSTEEREvaluator): implementation.inject_files(**{file: implementation.DEL_KEY for file in unused_files}) if score_ret_code != 0: - feedback.final_decision = False + feedback.acceptable = feedback.final_decision = False feedback.return_checking += "\n" + score_check_text if submission_ret_code != 0: - feedback.final_decision = False + feedback.acceptable = feedback.final_decision = False feedback.return_checking += "\nSubmission file check failed." return feedback diff --git a/rdagent/scenarios/data_science/dev/runner/prompts.yaml b/rdagent/scenarios/data_science/dev/runner/prompts.yaml index 2ac5d372..d2359574 100644 --- a/rdagent/scenarios/data_science/dev/runner/prompts.yaml +++ b/rdagent/scenarios/data_science/dev/runner/prompts.yaml @@ -24,10 +24,12 @@ DSCoSTEER_eval: 2. Ensure the code does not contain any incorrect, fabricated, or deceptive operations, such as mocking data, scores, or results. 3. Confirm that the prediction file (`submission.csv`) is generated using only the test dataset, and its format matches the sample submission. If the code does not satisfy the requirements: + - Set "acceptable" to false. - Set "final_decision" to false. {% if enable_hyperparameter_tuning_check %}- set "hyperparameter_tuning_decision" to false. - Set "hyperparameter_tuning_suggestion" to an empty string. If the code satisfy the requirements: + - Set "acceptable" to true. - Proceed to the next evaluation. # Evaluation 2: Hyperparameter @@ -56,7 +58,8 @@ DSCoSTEER_eval: { "execution": "Describe whether the whole code base executed successfully and generating the final submission. Include any errors or issues encountered, and retain all error messages and traceback details.", "return_checking": "Verify the generated files, particularly the submission file. Ensure that its format matches the sample submission", - "code": "Provide feedback on code quality, readability, and adherence to the given specifications.",{% if enable_hyperparameter_tuning_check %} + "code": "Provide feedback on code quality, readability, and adherence to the given specifications.", + "acceptable": ,{% if enable_hyperparameter_tuning_check %} "hyperparameter_tuning_decision": , "hyperparameter_tuning_suggestion": ,{% endif %} "final_decision": , @@ -78,10 +81,12 @@ DSCoSTEER_eval: 1. The code execution time or resource utilization suggest that there is room for improvement in the hyperparameters. 2. The code must apply early stopping strategy already (in order to prevent overfitting). 3. Your suggestion should have a strong chance of improving the model's performance. Focus on the most obvious and impactful opportunities for quick improvement by leveraging more training time. Don't explore hyperparameters with low confidence. If there are no obvious and impactful opportunities and the code runs well, please accept it. + If the code satisfy the requirements: - Set "hyperparameter_tuning_decision" to true. - - Set "final_decision" to false. - Provide a reasonable suggestion in "hyperparameter_tuning_suggestion". The "hyperparameter_tuning_suggestion" should begin with a clear observation, followed by your suggestion. For example: "[Observation] The maximum number of epochs was reached, but the validation loss is still going down and early stopping was not activated. Only 15% of the allowed time was used. [Suggestion] We recommend increasing epochs to 100 to avoid underfitting and further improve model performance." + - Set "final_decision" to false. + If the code does not satisfy the requirements: - Set "hyperparameter_tuning_decision" to false. - Set "hyperparameter_tuning_suggestion" to an empty string. @@ -93,6 +98,7 @@ DSCoSTEER_eval: "execution": "Describe whether the code executed successfully. Include any errors or issues encountered, and append all error messages and full traceback details without summarizing or omitting any information.", "return_checking": "Describe the expected file to be generated.", "code": "Provide feedback on code quality, readability, and adherence to the given specifications.", + "acceptable": , {% if enable_hyperparameter_tuning_check %}"hyperparameter_tuning_decision": , "hyperparameter_tuning_suggestion": ,{% endif %} "final_decision": , diff --git a/test/utils/test_ws.py b/test/utils/test_ws.py new file mode 100644 index 00000000..c19faac8 --- /dev/null +++ b/test/utils/test_ws.py @@ -0,0 +1,76 @@ +import os +import tempfile +import unittest +from pathlib import Path + +from rdagent.core.experiment import FBWorkspace + + +class TestFBWorkspace(unittest.TestCase): + """ + Unit-tests for `FBWorkspace`. + """ + + def setUp(self) -> None: # noqa: D401 + """ + Create an isolated temporary directory for each test case. + """ + self._tmp_dir = tempfile.TemporaryDirectory() + self.tmp_path = Path(self._tmp_dir.name) + + def tearDown(self) -> None: + """ + Clean up the temporary directory created in :py:meth:`setUp`. + """ + self._tmp_dir.cleanup() + + def test_checkpoint_roundtrip(self) -> None: + """ + Verify that ``create_ws_ckp`` captures the current workspace state and + ``recover_ws_ckp`` faithfully restores it. + """ + # create a symbolic link inside workspace and ensure checkpoint preserves the link + external_file = self.tmp_path / "external.txt" + external_file.write_text("external data") + ws = FBWorkspace() + ws.workspace_path = self.tmp_path / "ws" + ws.prepare() + (ws.workspace_path / "sym.txt").symlink_to(external_file) + ws.inject_files(**{"foo.py": "print('hi')", "bar.py": "x = 1"}) + + # Snapshot current workspace + original_files = { + p.relative_to(ws.workspace_path): (os.readlink(p) if p.is_symlink() else p.read_text()) + for p in ws.workspace_path.rglob("*") + if p.is_file() or p.is_symlink() + } + ws.create_ws_ckp() + self.assertIsNotNone(ws.ws_ckp, "Checkpoint data should have been generated") + + # Mutate workspace + (ws.workspace_path / "foo.py").write_text("print('changed')") + (ws.workspace_path / "new.py").write_text("pass") + (ws.workspace_path / "sym.txt").unlink() + + # Restore and verify equality with snapshot + ws.recover_ws_ckp() + + # Ensure symbolic link still exists after recovery. + self.assertTrue((ws.workspace_path / "sym.txt").is_symlink()) + recovered_files = { + p.relative_to(ws.workspace_path): (os.readlink(p) if p.is_symlink() else p.read_text()) + for p in ws.workspace_path.rglob("*") + if p.is_file() or p.is_symlink() + } + self.assertEqual(recovered_files, original_files) + + # Verify large files (>100 KB) are excluded when a size-limit is configured. + from rdagent.core.conf import RD_AGENT_SETTINGS as _SETTINGS + + _SETTINGS.workspace_ckp_size_limit = 100 * 1024 # set limit temporarily for this test + + large_file = ws.workspace_path / "large.bin" + large_file.write_bytes(b"0" * (110 * 1024)) # 110 KB dummy content + ws.create_ws_ckp() + ws.recover_ws_ckp() + self.assertFalse((ws.workspace_path / "large.bin").exists())