gavindiaz/NexQuant
1
0
Fork 0
mirror of https://github.com/NicolasBohn/NexQuant.git synced 2026-08-09 21:10:56 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: add restart and fix unzip (#538)

Browse Source 
* add restart function

* update

* fix ci

* update code

* fix restart

---------

Co-authored-by: yuanteli <1957922024@qq.com>
Co-authored-by: Xu Yang <xuyang1@microsoft.com>
This commit is contained in:
Xu Yang
2025-01-24 13:34:47 +08:00
committed by GitHub
parent 993f16a089
commit cb5127735f
4 changed files with 29 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
rdagent/app/data_science/loop.py
+13
View File
@@ -118,6 +118,19 @@ class DataScienceRDLoop(RDLoop):
ExperimentFeedback.from_exception(e),
)
)
if len(self.trace.hist) > DS_RD_SETTING.consecutive_errors:
cons_errors = 0
for _, feedback in reversed(self.trace.hist[-DS_RD_SETTING.consecutive_errors :]):
if feedback == ExperimentFeedback.from_exception(e) or feedback.reason.startswith(
"The experiment fails due to"
):
cons_errors += 1
else:
break
if cons_errors == DS_RD_SETTING.consecutive_errors + 1:
logger.error("Consecutive errors reached the limit. Dumping trace.")
logger.log_object(self.trace, tag="trace before restart")
self.trace = DSTrace(scen=self.trace.scen, knowledge_base=self.trace.knowledge_base)
logger.log_object(self.trace, tag="trace")
logger.log_object(self.trace.sota_experiment(), tag="SOTA experiment")