mirror of
https://github.com/NicolasBohn/NexQuant.git
synced 2026-08-08 20:47:44 +00:00
chore: withdraw for policy error (#907)
* chore: withdraw for policy error Co-authored-by: you-n-g <you-n-g@users.noreply.github.com> * Apply suggestions from code review
This commit is contained in:
@@ -88,6 +88,9 @@ class LoopBase:
|
||||
loop_trace: dict[int, list[LoopTrace]]
|
||||
|
||||
skip_loop_error: tuple[type[BaseException], ...] = () # you can define a list of error that will skip current loop
|
||||
withdraw_loop_error: tuple[
|
||||
type[BaseException], ...
|
||||
] = () # you can define a list of error that will withdraw current loop
|
||||
|
||||
EXCEPTION_KEY = "_EXCEPTION"
|
||||
|
||||
@@ -183,6 +186,11 @@ class LoopBase:
|
||||
self.step_idx = len(self.steps) - 1 # directly jump to the last step.
|
||||
self.loop_prev_out[self.EXCEPTION_KEY] = e
|
||||
continue
|
||||
elif isinstance(e, self.withdraw_loop_error):
|
||||
logger.warning(f"Withdraw loop {li} due to {e}")
|
||||
# Back to previous loop
|
||||
self.step_backward(li - 1)
|
||||
continue
|
||||
else:
|
||||
raise
|
||||
finally:
|
||||
@@ -207,6 +215,27 @@ class LoopBase:
|
||||
|
||||
self.dump(self.session_folder / f"{li}" / f"{si}_{name}") # save a snapshot after the session
|
||||
|
||||
def step_backward(self, li: int) -> None:
|
||||
prev_session_dir = self.session_folder / str(li)
|
||||
prev_path = min(
|
||||
(p for p in prev_session_dir.glob("*_*") if p.is_file()),
|
||||
key=lambda item: int(item.name.split("_", 1)[0]),
|
||||
default=None,
|
||||
)
|
||||
if prev_path:
|
||||
loaded = type(self).load(
|
||||
prev_path,
|
||||
output_path=self.session_folder.parent,
|
||||
do_truncate=False,
|
||||
replace_timer=True,
|
||||
)
|
||||
logger.info(f"Load previous session from {prev_path}")
|
||||
# Overwrite current instance state
|
||||
self.__dict__ = loaded.__dict__
|
||||
else:
|
||||
logger.error(f"No previous dump found at {prev_session_dir}, cannot withdraw loop {li}")
|
||||
raise
|
||||
|
||||
def dump(self, path: str | Path) -> None:
|
||||
if RD_Agent_TIMER_wrapper.timer.started:
|
||||
RD_Agent_TIMER_wrapper.timer.update_remain_time()
|
||||
|
||||
Reference in New Issue
Block a user