mirror of
https://github.com/NicolasBohn/NexQuant.git
synced 2026-07-27 23:47:46 +00:00
refactor: update MLEBench submission validation logic in runner and eval (#769)
* refactor: Update MLEBench submission validation logic in runner and eval * lint * Delete docs/scens/data_science.rst * fix: update variable name in feedback prompt * define a variable: submission_check_out * fix: update submission check handling for MLE data * refactor: reformat if condition for clarity --------- Co-authored-by: Linlang <Lv.Linlang@hotmail.com>
This commit is contained in:
you-n-g
@@ -111,28 +111,22 @@ class DSCoSTEERRunner(CoSTEER):
|
||||
description=f"The whole workflow of the solution has finished with some execution error, please check the error message and debug the whole code repo.\nCurrent code repo md5: {md5_hash(exp.experiment_workspace.all_codes)}",
|
||||
)
|
||||
]
|
||||
exp = super().develop(exp)
|
||||
exp = super().develop(exp) # run strategy(code implementation & evaluation loops)
|
||||
exp.sub_tasks = bak_sub_tasks
|
||||
|
||||
# NOTE: after running the loops, we expect some results are generated
|
||||
#
|
||||
# 1) scores of the models and ensemble
|
||||
score_fp = exp.experiment_workspace.workspace_path / "scores.csv"
|
||||
if not score_fp.exists():
|
||||
logger.error("Metrics file (scores.csv) is not generated.")
|
||||
raise RunnerError(f"Metrics file (scores.csv) is not generated")
|
||||
exp.result = pd.read_csv(score_fp, index_col=0)
|
||||
|
||||
# 2) if mle-bench, then the submission format checking will be used.
|
||||
# DockerEnv for MLEBench submission validation
|
||||
mde = get_ds_env(conf_type="mlebench")
|
||||
mde.conf.extra_volumes = {f"{DS_RD_SETTING.local_data_path}/zip_files": "/mle/data"}
|
||||
mde.prepare()
|
||||
# MLEBench Check
|
||||
mle_check_code = (
|
||||
(Path(__file__).absolute().resolve().parent / "eval_tests" / "mle_submission_format_test.txt")
|
||||
.read_text()
|
||||
.replace("<competition_id>", self.scen.competition)
|
||||
)
|
||||
exp.experiment_workspace.inject_files(**{"test/mle_submission_format_test.py": mle_check_code})
|
||||
exp.format_check_result = exp.experiment_workspace.execute(
|
||||
env=mde, entry=f"python test/mle_submission_format_test.py"
|
||||
)
|
||||
|
||||
if DS_RD_SETTING.if_using_mle_data:
|
||||
score_fp = exp.experiment_workspace.workspace_path / "test" / "mle_submission_format_test.output"
|
||||
with score_fp.open() as f:
|
||||
exp.format_check_result = f.read()
|
||||
return exp
|
||||
|
||||
@@ -91,37 +91,48 @@ class DSCoSTEERCoSTEEREvaluator(CoSTEEREvaluator):
|
||||
score_ret_code = 1
|
||||
|
||||
# DockerEnv for MLEBench submission validation
|
||||
mde = get_ds_env("mlebench")
|
||||
mde.conf.extra_volumes = {
|
||||
f"{DS_RD_SETTING.local_data_path}/zip_files": "/mle/data",
|
||||
}
|
||||
mde.prepare()
|
||||
# MLEBench Check
|
||||
mle_check_code = (
|
||||
(Path(__file__).absolute().resolve().parent / "eval_tests" / "mle_submission_format_test.txt")
|
||||
.read_text()
|
||||
.replace("<competition_id>", self.scen.competition)
|
||||
)
|
||||
implementation.inject_files(**{"test/mle_submission_format_test.py": mle_check_code})
|
||||
submission_check_out, submission_ret_code = implementation.execute_ret_code(
|
||||
env=mde, entry="python test/mle_submission_format_test.py"
|
||||
)
|
||||
submission_check_out = ""
|
||||
|
||||
if DS_RD_SETTING.if_using_mle_data:
|
||||
mde = get_ds_env("mlebench")
|
||||
mde.conf.extra_volumes = {
|
||||
f"{DS_RD_SETTING.local_data_path}/zip_files": "/mle/data",
|
||||
}
|
||||
mde.prepare()
|
||||
# MLEBench Check
|
||||
mle_check_code = (
|
||||
(Path(__file__).absolute().resolve().parent / "eval_tests" / "mle_submission_format_test.txt")
|
||||
.read_text()
|
||||
.replace("<competition_id>", self.scen.competition)
|
||||
)
|
||||
implementation.inject_files(**{"test/mle_submission_format_test.py": mle_check_code})
|
||||
submission_check_out, submission_ret_code = implementation.execute_ret_code(
|
||||
env=mde, entry="python test/mle_submission_format_test.py"
|
||||
)
|
||||
stdout += f"\nMLEBench submission check:\n{submission_check_out}\nIf MLEBench submission check returns a 'Submission is valid' or similar message, despite some warning messages, you should still consider the submission as valid and give a positive final decision. "
|
||||
implementation.inject_files(**{"test/mle_submission_format_test.output": submission_check_out})
|
||||
|
||||
if DS_RD_SETTING.rule_base_eval:
|
||||
if execute_ret_code == 0 and score_ret_code == 0 and submission_ret_code == 0:
|
||||
if DS_RD_SETTING.if_using_mle_data:
|
||||
score_check_text = score_check_text + "\n" + submission_check_out
|
||||
if (
|
||||
execute_ret_code == 0
|
||||
and score_ret_code == 0
|
||||
and (not DS_RD_SETTING.if_using_mle_data or submission_ret_code == 0)
|
||||
):
|
||||
return DSCoSTEEREvalFeedback(
|
||||
execution=stdout,
|
||||
return_checking=score_check_text + "\n" + submission_check_out,
|
||||
return_checking=score_check_text,
|
||||
code="Code evaluation is not available.",
|
||||
final_decision=True,
|
||||
)
|
||||
else:
|
||||
return DSCoSTEEREvalFeedback(
|
||||
execution=stdout,
|
||||
return_checking=score_check_text + "\n" + submission_check_out,
|
||||
return_checking=score_check_text,
|
||||
code="Code evaluation is not available.",
|
||||
final_decision=False,
|
||||
)
|
||||
stdout += f"\nMLEBench submission check:\n{submission_check_out}\nIf MLEBench submission check returns a 'Submission is valid' or similar message, despite some warning messages, you should still consider the submission as valid and give a positive final decision. "
|
||||
|
||||
system_prompt = T(".prompts:DSCoSTEER_eval.system").r(
|
||||
scenario=self.scen.get_scenario_all_desc(eda_output=implementation.file_dict.get("EDA.md", None)),
|
||||
@@ -181,7 +192,7 @@ class DSCoSTEERCoSTEEREvaluator(CoSTEEREvaluator):
|
||||
if score_ret_code != 0:
|
||||
feedback.final_decision = False
|
||||
feedback.return_checking += "\n" + score_check_text
|
||||
if submission_ret_code != 0:
|
||||
if DS_RD_SETTING.if_using_mle_data and submission_ret_code != 0:
|
||||
feedback.final_decision = False
|
||||
feedback.return_checking += "\nSubmission file check failed."
|
||||
return feedback
|
||||
|
||||
@@ -18,7 +18,10 @@ class DSExperiment(Experiment[Task, FBWorkspace, FBWorkspace]):
|
||||
# the initial workspace or the successful new version after coding
|
||||
self.experiment_workspace = FBWorkspace()
|
||||
self.pending_tasks_list = pending_tasks_list
|
||||
|
||||
self.format_check_result = None
|
||||
# this field is optional. It is not none only when we have a format checker. Currently, only following cases are supported.
|
||||
# - mle-bench
|
||||
|
||||
def is_ready_to_run(self) -> bool:
|
||||
"""
|
||||
|
||||
@@ -273,7 +273,7 @@ component_gen:
|
||||
|
||||
user: |-
|
||||
Here are the former experiments and their feedbacks:
|
||||
{{ exp_and_feedback_desc }}
|
||||
{{ exp_and_feedback_list_desc }}
|
||||
|
||||
Please choose the most proper component to focus on based on the information above. Please balance the exploration and exploitation.
|
||||
Avoid selecting the same component more than 5 times in a row to ensure that the chosen component is not overly repetitive.
|
||||
|
||||
Reference in New Issue
Block a user