mirror of
https://github.com/NicolasBohn/NexQuant.git
synced 2026-07-28 16:07:46 +00:00
you-n-g
7fc09169bc
* refactor: add is_acceptable, fallback logic and generify evolving agent * refine lint * small * lint * lint * lint * feat: add is_acceptable to CoSTEERMultiFeedback * feat: add in-memory workspace checkpoint and recovery * feat: preserve symbolic links in workspace checkpoints and recovery * lint * lint * feat: limit workspace checkpoint to files under 100KB * feat: add workspace checkpoint size limit setting * prompt * lint
58 lines
1.5 KiB
Python
58 lines
1.5 KiB
Python
"""
|
|
It is expected to be shared among different frameworks.
|
|
"""
|
|
|
|
from abc import ABC, abstractmethod
|
|
|
|
|
|
class Feedback:
|
|
"""
|
|
Design Principle:
|
|
It will be more like a **dataclass**.
|
|
The building process of feedback will should be in evaluator
|
|
"""
|
|
|
|
def is_acceptable(self) -> bool:
|
|
"""
|
|
Sometimes, the solution is already acceptable, but we still want to refine it.
|
|
So we use different logic to determine whether the solution is acceptable or finished.
|
|
"""
|
|
return self.__bool__()
|
|
|
|
def finished(self) -> bool:
|
|
"""
|
|
In some implementations, tasks may fail multiple times, leading agents to skip the implementation.
|
|
So both skip and success indicate the task is finished.
|
|
"""
|
|
return self.__bool__()
|
|
|
|
def __bool__(self) -> bool:
|
|
return True
|
|
|
|
|
|
class EvaluableObj:
|
|
"""
|
|
A set of information that is evaluable. Following things can be included.
|
|
- Task
|
|
- Solution
|
|
- Ground Truth
|
|
"""
|
|
|
|
|
|
class Evaluator(ABC):
|
|
"""
|
|
Design Principle:
|
|
|
|
It should cover the building process of feedback from raw information.
|
|
Typically the building of feedback will be two phases.
|
|
1. raw information including stdout & workspace (feedback itself will handle this)
|
|
2. advanced/summarized feedback information. (evaluate will handle this)
|
|
"""
|
|
|
|
@abstractmethod
|
|
def evaluate(
|
|
self,
|
|
eo: EvaluableObj,
|
|
) -> Feedback:
|
|
raise NotImplementedError
|