chore: withdraw for policy error (#907)

* chore: withdraw for policy error

Co-authored-by: you-n-g <you-n-g@users.noreply.github.com>

* Apply suggestions from code review
This commit is contained in:
Tim
2025-05-29 17:23:06 +08:00
committed by GitHub
parent 1091d5ea29
commit a7c6fca8b4
4 changed files with 39 additions and 2 deletions
+29
View File
@@ -88,6 +88,9 @@ class LoopBase:
loop_trace: dict[int, list[LoopTrace]]
skip_loop_error: tuple[type[BaseException], ...] = () # you can define a list of error that will skip current loop
withdraw_loop_error: tuple[
type[BaseException], ...
] = () # you can define a list of error that will withdraw current loop
EXCEPTION_KEY = "_EXCEPTION"
@@ -183,6 +186,11 @@ class LoopBase:
self.step_idx = len(self.steps) - 1 # directly jump to the last step.
self.loop_prev_out[self.EXCEPTION_KEY] = e
continue
elif isinstance(e, self.withdraw_loop_error):
logger.warning(f"Withdraw loop {li} due to {e}")
# Back to previous loop
self.step_backward(li - 1)
continue
else:
raise
finally:
@@ -207,6 +215,27 @@ class LoopBase:
self.dump(self.session_folder / f"{li}" / f"{si}_{name}") # save a snapshot after the session
def step_backward(self, li: int) -> None:
prev_session_dir = self.session_folder / str(li)
prev_path = min(
(p for p in prev_session_dir.glob("*_*") if p.is_file()),
key=lambda item: int(item.name.split("_", 1)[0]),
default=None,
)
if prev_path:
loaded = type(self).load(
prev_path,
output_path=self.session_folder.parent,
do_truncate=False,
replace_timer=True,
)
logger.info(f"Load previous session from {prev_path}")
# Overwrite current instance state
self.__dict__ = loaded.__dict__
else:
logger.error(f"No previous dump found at {prev_session_dir}, cannot withdraw loop {li}")
raise
def dump(self, path: str | Path) -> None:
if RD_Agent_TIMER_wrapper.timer.started:
RD_Agent_TIMER_wrapper.timer.update_remain_time()