mirror of
https://github.com/NicolasBohn/NexQuant.git
synced 2026-08-09 13:00:56 +00:00
feat: idea pool integrated to exp_gen & add timer to RD-Agent & pause-resume to RD-loops (#795)
* update all code * update all code * dump knowledge base * rename the tag * add timer to RD-Agent * fix CI * fix CI * use batch embedding * fix a small bug * fix prompt bug * feat: add pause resume to handle K8S cluster pause (#804) * add resume to cluster running * fix non-pickle problem * fix a small bug * fix a small bug * avoid shutil move error * refine the logic * move knowledge base out of session * avoid mistake information to pipeline coding * avoid load and dump in steps * archive the right folder * small improvement * avoid restart when timer is already started * fix CI --------- Co-authored-by: Xu Yang <xuyang1@microsoft.com> --------- Co-authored-by: Xu Yang <peteryang@vip.qq.com> Co-authored-by: Xu Yang <xuyang1@microsoft.com> Co-authored-by: Xu <v-xuminrui@microsoft.com>
This commit is contained in:
Roland Minrui
@@ -1,5 +1,8 @@
|
||||
import shutil
|
||||
import subprocess
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from typing import Any, Optional, Union
|
||||
|
||||
import fire
|
||||
|
||||
@@ -28,10 +31,8 @@ from rdagent.scenarios.data_science.dev.feedback import DSExperiment2Feedback
|
||||
from rdagent.scenarios.data_science.dev.runner import DSCoSTEERRunner
|
||||
from rdagent.scenarios.data_science.experiment.experiment import DSExperiment
|
||||
from rdagent.scenarios.data_science.proposal.exp_gen import DSExpGen, DSTrace
|
||||
from rdagent.scenarios.data_science.proposal.exp_gen.select import (
|
||||
LatestCKPSelector,
|
||||
SOTAJumpCKPSelector,
|
||||
)
|
||||
from rdagent.scenarios.data_science.proposal.exp_gen.idea_pool import DSKnowledgeBase
|
||||
from rdagent.scenarios.data_science.proposal.exp_gen.select import LatestCKPSelector
|
||||
from rdagent.scenarios.kaggle.kaggle_crawler import download_data
|
||||
|
||||
|
||||
@@ -42,13 +43,6 @@ class DataScienceRDLoop(RDLoop):
|
||||
logger.log_object(PROP_SETTING.competition, tag="competition")
|
||||
scen: Scenario = import_class(PROP_SETTING.scen)(PROP_SETTING.competition)
|
||||
|
||||
### shared components in the workflow # TODO: check if
|
||||
knowledge_base = (
|
||||
import_class(PROP_SETTING.knowledge_base)(PROP_SETTING.knowledge_base_path, scen)
|
||||
if PROP_SETTING.knowledge_base != ""
|
||||
else None
|
||||
)
|
||||
|
||||
# 1) task generation from scratch
|
||||
# self.scratch_gen: tuple[HypothesisGen, Hypothesis2Experiment] = DummyHypothesisGen(scen),
|
||||
|
||||
@@ -70,8 +64,13 @@ class DataScienceRDLoop(RDLoop):
|
||||
# self.summarizer: Experiment2Feedback = import_class(PROP_SETTING.summarizer)(scen)
|
||||
# logger.log_object(self.summarizer, tag="summarizer")
|
||||
|
||||
# self.trace = KGTrace(scen=scen, knowledge_base=knowledge_base)
|
||||
self.trace = DSTrace(scen=scen)
|
||||
if DS_RD_SETTING.enable_knowledge_base and DS_RD_SETTING.knowledge_base_version == "v1":
|
||||
knowledge_base = DSKnowledgeBase(
|
||||
path=DS_RD_SETTING.knowledge_base_path, idea_pool_json_path=DS_RD_SETTING.idea_pool_json_path
|
||||
)
|
||||
self.trace = DSTrace(scen=scen, knowledge_base=knowledge_base)
|
||||
else:
|
||||
self.trace = DSTrace(scen=scen)
|
||||
self.summarizer = DSExperiment2Feedback(scen)
|
||||
super(RDLoop, self).__init__()
|
||||
|
||||
@@ -166,10 +165,70 @@ class DataScienceRDLoop(RDLoop):
|
||||
self.trace = DSTrace(scen=self.trace.scen, knowledge_base=self.trace.knowledge_base)
|
||||
logger.log_object(self.trace, tag="trace")
|
||||
logger.log_object(self.trace.sota_experiment(), tag="SOTA experiment")
|
||||
if DS_RD_SETTING.enable_knowledge_base and DS_RD_SETTING.knowledge_base_version == "v1":
|
||||
logger.log_object(self.trace.knowledge_base, tag="knowledge_base")
|
||||
self.trace.knowledge_base.dump()
|
||||
|
||||
if (
|
||||
DS_RD_SETTING.enable_log_archive
|
||||
and DS_RD_SETTING.log_archive_path is not None
|
||||
and Path(DS_RD_SETTING.log_archive_path).is_dir()
|
||||
):
|
||||
start_archive_datetime = datetime.now()
|
||||
logger.info(f"Archiving log folder after loop {self.loop_idx}")
|
||||
tar_path = (
|
||||
Path(
|
||||
DS_RD_SETTING.log_archive_temp_path
|
||||
if DS_RD_SETTING.log_archive_temp_path
|
||||
else DS_RD_SETTING.log_archive_path
|
||||
)
|
||||
/ "mid_log.tar"
|
||||
)
|
||||
subprocess.run(["tar", "-cf", str(tar_path), "-C", (Path().cwd() / "log"), "."], check=True)
|
||||
if DS_RD_SETTING.log_archive_temp_path is not None:
|
||||
shutil.move(tar_path, Path(DS_RD_SETTING.log_archive_path) / "mid_log.tar")
|
||||
tar_path = Path(DS_RD_SETTING.log_archive_path) / "mid_log.tar"
|
||||
shutil.copy(
|
||||
tar_path, Path(DS_RD_SETTING.log_archive_path) / "mid_log_bak.tar"
|
||||
) # backup when upper code line is killed when running
|
||||
self.timer.add_duration(datetime.now() - start_archive_datetime)
|
||||
|
||||
@classmethod
|
||||
def load(
|
||||
cls, path: Union[str, Path], output_path: Optional[Union[str, Path]] = None, do_truncate: bool = False
|
||||
) -> "LoopBase":
|
||||
session = super().load(path, output_path, do_truncate)
|
||||
if (
|
||||
DS_RD_SETTING.enable_knowledge_base
|
||||
and DS_RD_SETTING.knowledge_base_version == "v1"
|
||||
and Path(DS_RD_SETTING.knowledge_base_path).exists()
|
||||
):
|
||||
knowledge_base = DSKnowledgeBase(path=DS_RD_SETTING.knowledge_base_path)
|
||||
session.trace.knowledge_base = knowledge_base
|
||||
return session
|
||||
|
||||
def dump(self, path: str | Path) -> None:
|
||||
"""
|
||||
Since knowledge_base is big and we don't want to dump it every time
|
||||
So we remove it from the trace before dumping and restore it after.
|
||||
"""
|
||||
backup_knowledge_base = None
|
||||
if self.trace.knowledge_base is not None:
|
||||
backup_knowledge_base = self.trace.knowledge_base
|
||||
self.trace.knowledge_base = None
|
||||
super().dump(path)
|
||||
if backup_knowledge_base is not None:
|
||||
self.trace.knowledge_base = backup_knowledge_base
|
||||
|
||||
|
||||
def main(
|
||||
path=None, output_path=None, step_n=None, loop_n=None, competition="bms-molecular-translation", do_truncate=True
|
||||
path=None,
|
||||
output_path=None,
|
||||
step_n=None,
|
||||
loop_n=None,
|
||||
competition="bms-molecular-translation",
|
||||
do_truncate=True,
|
||||
timeout=None,
|
||||
):
|
||||
"""
|
||||
|
||||
@@ -213,7 +272,7 @@ def main(
|
||||
kaggle_loop = DataScienceRDLoop(DS_RD_SETTING)
|
||||
else:
|
||||
kaggle_loop = DataScienceRDLoop.load(path, output_path, do_truncate)
|
||||
kaggle_loop.run(step_n=step_n, loop_n=loop_n)
|
||||
kaggle_loop.run(step_n=step_n, loop_n=loop_n, all_duration=timeout)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
Reference in New Issue
Block a user