gavindiaz/NexQuant
1
0
Fork 0
mirror of https://github.com/NicolasBohn/NexQuant.git synced 2026-08-05 11:07:43 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix: move mlebench check into runner (#556)

Browse Source 
* abandon mlebench check in workflow

* move mlebench check to runner and put it into the exp
This commit is contained in:
Xu Yang
2025-02-06 15:37:41 +08:00
committed by GitHub
parent 0e447a3a19
commit b2d56e39e5
6 changed files with 41 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files
rdagent/components/coder/data_science/workflow/eval.py
+17 -16
View File
@@ -61,13 +61,13 @@ class WorkflowGeneralCaseSpecEvaluator(CoSTEEREvaluator):
}
de = DockerEnv(conf=ds_docker_conf)
# DockerEnv for MLEBench submission validation
mle_de_conf = MLEBDockerConf()
mle_de_conf.extra_volumes = {
f"{DS_RD_SETTING.local_data_path}/zip_files": "/mle/data",
}
mde = DockerEnv(conf=mle_de_conf)
mde.prepare()
# # DockerEnv for MLEBench submission validation
# mle_de_conf = MLEBDockerConf()
# mle_de_conf.extra_volumes = {
# f"{DS_RD_SETTING.local_data_path}/zip_files": "/mle/data",
# }
# mde = DockerEnv(conf=mle_de_conf)
# mde.prepare()
# Clean the scores.csv & submission.csv.
stdout = implementation.execute(env=de, entry=f"rm submission.csv scores.csv")
@@ -98,18 +98,19 @@ class WorkflowGeneralCaseSpecEvaluator(CoSTEEREvaluator):
else:
base_check_code = (DIRNAME / "eval_tests" / "submission_check.txt").read_text()
implementation.inject_files(**{"submission_check.py": base_check_code})
stdout += "----Submission Check 1-----\n"
# stdout += "----Submission Check 1-----\n"
stdout += implementation.execute(env=de, entry="python submission_check.py")
# MLEBench Check
mle_check_code = (
(DIRNAME / "eval_tests" / "mle_submission_check.txt")
.read_text()
.replace("<competition_id>", self.scen.competition)
)
implementation.inject_files(**{"mle_submission_check.py": mle_check_code})
stdout += "----Submission Check 2-----\n"
stdout += implementation.execute(env=mde, entry=f"python mle_submission_check.py")
# !!! Since we are running on a sampled dataset, mlebench check is not required.
# mle_check_code = (
# (DIRNAME / "eval_tests" / "mle_submission_check.txt")
# .read_text()
# .replace("<competition_id>", self.scen.competition)
# )
# implementation.inject_files(**{"mle_submission_check.py": mle_check_code})
# stdout += "----Submission Check 2-----\n"
# stdout += implementation.execute(env=mde, entry=f"python mle_submission_check.py")
system_prompt = T(".prompts:workflow_eval.system").r(
scenario=self.scen.get_scenario_all_desc(),