mirror of
https://github.com/NicolasBohn/NexQuant.git
synced 2026-07-28 16:07:46 +00:00
you-n-g
20778a9b87
* use conda to run kaggle and mlebench code * refactor: Simplify environment configuration and execution logic * add setting to use local env in ds * refine dockerfile * fix: Move MLEBDockerEnv initialization inside conditionals & fix condaenv * refactor: reformat code for better readability and consistency * feat: add conda env to all envs. * fix: fix bugs when run loop * refactor: Simplify DockerEnv configuration in mle_summary.py * fix image bug * style: reformat code for better readability and consistency * change commit * feat: Add entrypoint script for sing_docker scenario in rdagent * refactor: add Any type hints and comments for clarity in env.py * feat: Create log directory if it doesn't exist in entrypoint script * feat: Add debug mode and list root directory in entrypoint script * fix: Remove specific branch checkout in Dockerfile for RD-Agent * fix: Add competition argument to loop.py script execution * fix: Correct directory navigation and dependency installation in entrypoint.sh * fix: Correct user ownership assignment in entrypoint script * refactor: Comment out redundant log copying to RD_OUTPUT_DIR * fix: Unset LOG_TRACE_PATH to prevent log contamination in entrypoint.sh --------- Co-authored-by: Xu Yang <peteryang@vip.qq.com>
94 lines
3.8 KiB
Python
94 lines
3.8 KiB
Python
"""
|
|
Beyond previous tests
|
|
-
|
|
"""
|
|
|
|
import json
|
|
import re
|
|
from pathlib import Path
|
|
|
|
from rdagent.app.data_science.conf import DS_RD_SETTING
|
|
from rdagent.components.coder.CoSTEER.evaluators import (
|
|
CoSTEEREvaluator,
|
|
CoSTEERSingleFeedback,
|
|
)
|
|
from rdagent.components.coder.data_science.conf import get_ds_env
|
|
from rdagent.core.evolving_framework import QueriedKnowledge
|
|
from rdagent.core.exception import CoderError
|
|
from rdagent.core.experiment import FBWorkspace, Task
|
|
from rdagent.oai.llm_utils import APIBackend
|
|
from rdagent.utils.agent.tpl import T
|
|
from rdagent.utils.agent.workflow import build_cls_from_json_with_retry
|
|
|
|
DIRNAME = Path(__file__).absolute().resolve().parent
|
|
ModelSingleFeedback = CoSTEERSingleFeedback
|
|
|
|
|
|
# Below are unit tests for testing the specification of the implemented model ------------------
|
|
class ModelGeneralCaseSpecEvaluator(CoSTEEREvaluator):
|
|
"""
|
|
Motivation case:
|
|
- Simplest case, we already split the data into train_data, valid_data, and test_data. We require the model to learn (optionally validate on valid data), and infer on test data.
|
|
|
|
Test workflow:
|
|
- Build train, valid, and test data to run it, and test the output (e.g., shape, etc.)
|
|
"""
|
|
|
|
def evaluate(
|
|
self,
|
|
target_task: Task,
|
|
implementation: FBWorkspace,
|
|
gt_implementation: FBWorkspace,
|
|
queried_knowledge: QueriedKnowledge = None,
|
|
**kwargs,
|
|
) -> ModelSingleFeedback:
|
|
target_task_information = target_task.get_task_information()
|
|
if (
|
|
queried_knowledge is not None
|
|
and target_task_information in queried_knowledge.success_task_to_knowledge_dict
|
|
):
|
|
return queried_knowledge.success_task_to_knowledge_dict[target_task_information].feedback
|
|
elif queried_knowledge is not None and target_task_information in queried_knowledge.failed_task_info_set:
|
|
return ModelSingleFeedback(
|
|
execution="This task has failed too many times, skip implementation.",
|
|
return_checking="This task has failed too many times, skip implementation.",
|
|
code="This task has failed too many times, skip implementation.",
|
|
final_decision=False,
|
|
)
|
|
|
|
env = get_ds_env()
|
|
env.conf.extra_volumes = {f"{DS_RD_SETTING.local_data_path}/sample/{self.scen.competition}": "/kaggle/input"}
|
|
|
|
fname = "test/model_test.py"
|
|
test_code = (
|
|
(DIRNAME / "eval_tests" / "model_test.txt").read_text().replace("model01", target_task.name)
|
|
) # only check the model changed this time
|
|
implementation.inject_files(**{fname: test_code})
|
|
stdout = implementation.execute(env=env, entry=f"python {fname}")
|
|
|
|
if stdout is None:
|
|
raise CoderError(
|
|
"The execution output contains too many progress bars and results in the LLM's token size exceeding the limit."
|
|
)
|
|
|
|
if "main.py" in implementation.file_dict:
|
|
workflow_stdout = implementation.execute(env=env, entry="python main.py")
|
|
workflow_stdout = re.sub(r"=== Start of EDA part ===(.*)=== End of EDA part ===", "", workflow_stdout)
|
|
else:
|
|
workflow_stdout = None
|
|
|
|
system_prompt = T(".prompts:model_eval.system").r(
|
|
task_desc=target_task.get_task_information(),
|
|
test_code=test_code,
|
|
code=implementation.file_dict[f"{target_task.name}.py"],
|
|
scenario=self.scen.get_scenario_all_desc(),
|
|
spec=implementation.file_dict["spec/model.md"],
|
|
workflow_stdout=workflow_stdout,
|
|
workflow_code=implementation.all_codes,
|
|
)
|
|
user_prompt = T(".prompts:model_eval.user").r(
|
|
stdout=stdout,
|
|
workflow_stdout=workflow_stdout,
|
|
)
|
|
return build_cls_from_json_with_retry(ModelSingleFeedback, system_prompt=system_prompt, user_prompt=user_prompt)
|