From 46dec7b624b94435043b93fee91fb8fe594b802e Mon Sep 17 00:00:00 2001 From: Tim Date: Tue, 6 May 2025 16:00:13 +0800 Subject: [PATCH] feat: custom data (#810) * custom data * fix: simplify competition check and log local description file * no sample data * feat: add test evaluation module with error handling support * fix: update eval path to use eval_sub_dir and add valid_check TODO * refactor: add MLETestEval check to conditionally run grading steps * avoid blank stdout * valid in testeval * rename test.csv to avoid conflict * Support Disabling sample submission * refactoring * fix: remove DS_KAGGLE_DATA and update prompt instructions * add try for grade * ignore submission * fix: remove tee from eval command and warn about pipeline exit code detection * optional to use raw description * support old data * add execution result to stdout * add metric to raw description * custom data explain * add debug_path * rst update --------- Co-authored-by: Young --- docs/scens/catalog.rst | 1 + docs/scens/data_science.rst | 60 ++++++++ rdagent/app/data_science/conf.py | 12 +- rdagent/app/data_science/loop.py | 10 +- rdagent/app/kaggle/conf.py | 2 + rdagent/components/coder/data_science/conf.py | 4 +- .../coder/data_science/ensemble/eval.py | 8 +- .../coder/data_science/feature/eval.py | 8 +- .../coder/data_science/model/eval.py | 8 +- .../coder/data_science/pipeline/eval.py | 63 +++++---- .../coder/data_science/pipeline/prompts.yaml | 23 ++- .../data_science/raw_data_loader/eval.py | 8 +- .../coder/data_science/workflow/eval.py | 8 +- rdagent/log/mle_summary.py | 68 +++++---- rdagent/log/ui/ds_trace.py | 5 +- .../scenarios/data_science/dev/runner/eval.py | 35 ++--- .../data_science/dev/runner/prompts.yaml | 20 ++- .../eval_tests/mle_submission_format_test.txt | 0 .../data_science/proposal/exp_gen/base.py | 2 +- .../scenarios/data_science/scen/__init__.py | 15 +- .../scenarios/data_science/scen/prompts.yaml | 10 ++ rdagent/scenarios/data_science/test_eval.py | 133 ++++++++++++++++++ rdagent/utils/env.py | 7 + 23 files changed, 376 insertions(+), 134 deletions(-) create mode 100644 docs/scens/data_science.rst rename rdagent/scenarios/data_science/{dev/runner => }/eval_tests/mle_submission_format_test.txt (100%) create mode 100644 rdagent/scenarios/data_science/test_eval.py diff --git a/docs/scens/catalog.rst b/docs/scens/catalog.rst index c7cf4e13..93fb2346 100644 --- a/docs/scens/catalog.rst +++ b/docs/scens/catalog.rst @@ -46,4 +46,5 @@ The supported scenarios are listed below: model_agent_med model_copilot_general kaggle_agent + data_science diff --git a/docs/scens/data_science.rst b/docs/scens/data_science.rst new file mode 100644 index 00000000..39112e7c --- /dev/null +++ b/docs/scens/data_science.rst @@ -0,0 +1,60 @@ +.. _data_science_agent: + +======================= +Data Science Agent +======================= + +**πŸ€– Automated Feature Engineering & Model Tuning Evolution** +------------------------------------------------------------------------------------------ +The Data Science Agent is an agent that can automatically perform feature engineering and model tuning. It can be used to solve various data science problems, such as image classification, time series forecasting, and text classification. + +🧭 Example Guide +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +- πŸ”§ **Set up RD-Agent Environment** + + - Before you start, please make sure you have installed RD-Agent and configured the environment for RD-Agent correctly. If you want to know how to install and configure the RD-Agent, please refer to the `documentation <../installation_and_configuration.html>`_. + +- πŸ”© **Setting the Environment variables at .env file** + + - Determine the path where the data will be stored and add it to the ``.env`` file. + + .. code-block:: sh + + dotenv set DS_LOCAL_DATA_PATH /ds_data + dotenv set DS_SCEN rdagent.scenarios.data_science.scen.DataScienceScen + +- πŸ“₯ **Prepare Competition Data** + + - Data Science competition data, contains three parts: competition description file (markdown file), competition dataset and competition evaluation files. + + - **Correct directory structure (Here is an example of competition data with id custom_data)** + + .. code-block:: text + + ds_data + └── eval + | └── custom_data + | └── grade.py + | └── valid.py + | └── test.csv + └── custom_data + └── train.csv + └── test.csv + └── sample_submission.csv + └── description.md + + - ``ds_data/custom_data/train.csv:`` Necessary training data in csv or parquet format, or training images. + + - ``ds_data/custom_data/description.md:`` (Optional) Competition description file. + + - ``ds_data/custom_data/sample_submission.csv:`` (Optional) Competition sample submission file. + + - ``ds_data/eval/custom_data/grade.py:`` (Optional) Competition grade script, in order to calculate the score for the submission. + + - ``ds_data/eval/custom_data/valid.py:`` (Optional) Competition validation script, in order to check if the submission format is correct. + + - ``ds_data/eval/custom_data/submission_test.csv:`` (Optional) Competition test label file. + + + diff --git a/rdagent/app/data_science/conf.py b/rdagent/app/data_science/conf.py index 9ec6c1a8..62998cf7 100644 --- a/rdagent/app/data_science/conf.py +++ b/rdagent/app/data_science/conf.py @@ -6,6 +6,7 @@ from rdagent.app.kaggle.conf import KaggleBasePropSetting class DataScienceBasePropSetting(KaggleBasePropSetting): + # TODO: Kaggle Setting should be the subclass of DataScience model_config = SettingsConfigDict(env_prefix="DS_", protected_namespaces=()) # Main components @@ -29,7 +30,7 @@ class DataScienceBasePropSetting(KaggleBasePropSetting): #### enable specification spec_enabled: bool = True - ### proposal related + #### proposal related proposal_version: str = "v1" coder_on_whole_pipeline: bool = False max_trace_hist: int = 3 @@ -38,8 +39,10 @@ class DataScienceBasePropSetting(KaggleBasePropSetting): runner_max_loop: int = 1 rule_base_eval: bool = False + sample_data: bool = True + use_raw_description: bool = False - ### model dump + #### model dump enable_model_dump: bool = False enable_doc_dev: bool = False model_dump_check_level: Literal["medium", "high"] = "medium" @@ -57,5 +60,10 @@ class DataScienceBasePropSetting(KaggleBasePropSetting): None # This is to store the mid tar file since writing the tar file is preferred in local storage then copy to target storage ) + #### Evaluation on Test related + eval_sub_dir: str = "eval" # TODO: fixme, this is not a good name + """We'll use f"{DS_RD_SETTING.local_data_path}/{DS_RD_SETTING.eval_sub_dir}/{competition}" + to find the scriipt to evaluate the submission on test""" + DS_RD_SETTING = DataScienceBasePropSetting() diff --git a/rdagent/app/data_science/loop.py b/rdagent/app/data_science/loop.py index d49aeba5..a20acb5f 100644 --- a/rdagent/app/data_science/loop.py +++ b/rdagent/app/data_science/loop.py @@ -304,15 +304,9 @@ def main( if competition is not None: DS_RD_SETTING.competition = competition - if DS_RD_SETTING.competition: - if DS_RD_SETTING.scen.endswith("KaggleScen"): - download_data(competition=DS_RD_SETTING.competition, settings=DS_RD_SETTING) - else: - if not Path(f"{DS_RD_SETTING.local_data_path}/{competition}").exists(): - logger.error(f"Please prepare data for competition {competition} first.") - return - else: + if not DS_RD_SETTING.competition: logger.error("Please specify competition name.") + if path is None: kaggle_loop = DataScienceRDLoop(DS_RD_SETTING) else: diff --git a/rdagent/app/kaggle/conf.py b/rdagent/app/kaggle/conf.py index 7f6f37e8..7d3ff473 100644 --- a/rdagent/app/kaggle/conf.py +++ b/rdagent/app/kaggle/conf.py @@ -46,9 +46,11 @@ class KaggleBasePropSetting(ExtendedBaseSettings): local_data_path: str = "" """Folder storing Kaggle competition data""" + # Evaluation on Test related if_using_mle_data: bool = False auto_submit: bool = False """Automatically upload and submit each experiment result to Kaggle platform""" + # Conditionally set the knowledge_base based on the use of graph RAG knowledge_base: str = "" """Knowledge base class, uses 'KGKnowledgeGraph' when advanced graph-based RAG is enabled, otherwise empty.""" diff --git a/rdagent/components/coder/data_science/conf.py b/rdagent/components/coder/data_science/conf.py index 049e20fc..789f8b2b 100644 --- a/rdagent/components/coder/data_science/conf.py +++ b/rdagent/components/coder/data_science/conf.py @@ -27,7 +27,9 @@ class DSCoderCoSTEERSettings(CoSTEERSettings): def get_ds_env( conf_type: Literal["kaggle", "mlebench"] = "kaggle", extra_volumes: dict = {}, - running_timeout_period: int = DS_RD_SETTING.debug_timeout, + running_timeout_period: int = ( + DS_RD_SETTING.debug_timeout if DS_RD_SETTING.sample_data else DS_RD_SETTING.full_timeout + ), ) -> Env: """ Retrieve the appropriate environment configuration based on the env_type setting. diff --git a/rdagent/components/coder/data_science/ensemble/eval.py b/rdagent/components/coder/data_science/ensemble/eval.py index 3dfb295e..04b9fc65 100644 --- a/rdagent/components/coder/data_science/ensemble/eval.py +++ b/rdagent/components/coder/data_science/ensemble/eval.py @@ -47,13 +47,7 @@ class EnsembleCoSTEEREvaluator(CoSTEEREvaluator): final_decision=False, ) - env = get_ds_env( - extra_volumes={ - f"{DS_RD_SETTING.local_data_path}/sample/{self.scen.competition}": T( - "scenarios.data_science.share:scen.input_path" - ).r() - } - ) + env = get_ds_env(extra_volumes={self.scen.debug_path: T("scenarios.data_science.share:scen.input_path").r()}) fname = "test/ensemble_test.txt" test_code = (DIRNAME / "eval_tests" / "ensemble_test.txt").read_text() diff --git a/rdagent/components/coder/data_science/feature/eval.py b/rdagent/components/coder/data_science/feature/eval.py index bef2266e..01fa857e 100644 --- a/rdagent/components/coder/data_science/feature/eval.py +++ b/rdagent/components/coder/data_science/feature/eval.py @@ -43,13 +43,7 @@ class FeatureCoSTEEREvaluator(CoSTEEREvaluator): final_decision=False, ) - env = get_ds_env( - extra_volumes={ - f"{DS_RD_SETTING.local_data_path}/sample/{self.scen.competition}": T( - "scenarios.data_science.share:scen.input_path" - ).r() - } - ) + env = get_ds_env(extra_volumes={self.scen.debug_path: T("scenarios.data_science.share:scen.input_path").r()}) # TODO: do we need to clean the generated temporary content? fname = "test/feature_test.py" diff --git a/rdagent/components/coder/data_science/model/eval.py b/rdagent/components/coder/data_science/model/eval.py index 3280043e..229512bb 100644 --- a/rdagent/components/coder/data_science/model/eval.py +++ b/rdagent/components/coder/data_science/model/eval.py @@ -57,13 +57,7 @@ class ModelGeneralCaseSpecEvaluator(CoSTEEREvaluator): final_decision=False, ) - env = get_ds_env( - extra_volumes={ - f"{DS_RD_SETTING.local_data_path}/sample/{self.scen.competition}": T( - "scenarios.data_science.share:scen.input_path" - ).r() - } - ) + env = get_ds_env(extra_volumes={self.scen.debug_path: T("scenarios.data_science.share:scen.input_path").r()}) if_model_removed = False diff --git a/rdagent/components/coder/data_science/pipeline/eval.py b/rdagent/components/coder/data_science/pipeline/eval.py index caed8b7d..5a4f4a0b 100644 --- a/rdagent/components/coder/data_science/pipeline/eval.py +++ b/rdagent/components/coder/data_science/pipeline/eval.py @@ -18,6 +18,7 @@ from rdagent.components.coder.CoSTEER.knowledge_management import ( from rdagent.components.coder.data_science.conf import get_clear_ws_cmd, get_ds_env from rdagent.components.coder.data_science.utils import remove_eda_part from rdagent.core.experiment import FBWorkspace, Task +from rdagent.scenarios.data_science.test_eval import get_test_eval from rdagent.utils.agent.tpl import T from rdagent.utils.agent.workflow import build_cls_from_json_with_retry @@ -52,18 +53,13 @@ class PipelineCoSTEEREvaluator(CoSTEEREvaluator): final_decision=False, ) - env = get_ds_env( - extra_volumes={ - f"{DS_RD_SETTING.local_data_path}/sample/{self.scen.competition}": T( - "scenarios.data_science.share:scen.input_path" - ).r() - } - ) + env = get_ds_env(extra_volumes={self.scen.debug_path: T("scenarios.data_science.share:scen.input_path").r()}) # Clean the scores.csv & submission.csv. implementation.execute(env=env, entry=get_clear_ws_cmd()) - stdout, execute_ret_code = implementation.execute_ret_code(env=env, entry=f"python main.py") + stdout, execute_ret_code = implementation.execute_ret_code(env=env, entry=f"python -m coverage run main.py") stdout = remove_eda_part(stdout) + stdout += f"The code executed {'successfully' if execute_ret_code == 0 else 'failed'}." score_fp = implementation.workspace_path / "scores.csv" score_ret_code = 0 @@ -101,35 +97,40 @@ class PipelineCoSTEEREvaluator(CoSTEEREvaluator): score_check_text += f"\n[Error] in checking the scores.csv file: {e}\nscores.csv's content:\n-----\n{score_fp.read_text()}\n-----" score_ret_code = 1 - # Check submission file - base_check_code = T(".eval_tests.submission_format_test", ftype="txt").r() - implementation.inject_files(**{"test/submission_format_test.py": base_check_code}) - # stdout += "----Submission Check 1-----\n" - submission_check_out, submission_ret_code = implementation.execute_ret_code( - env=env, entry="python test/submission_format_test.py" - ) - if DS_RD_SETTING.rule_base_eval: - if execute_ret_code == 0 and score_ret_code == 0 and submission_ret_code == 0: - return PipelineSingleFeedback( - execution=stdout, - return_checking=score_check_text + "\n" + submission_check_out, - code="Code evaluation is not available.", - final_decision=True, - ) - else: - return PipelineSingleFeedback( - execution=stdout, - return_checking=score_check_text + "\n" + submission_check_out, - code="Code evaluation is not available.", - final_decision=False, - ) - stdout += "\n" + submission_check_out + test_eval = get_test_eval() + if not test_eval.is_sub_enabled(self.scen.competition): + submission_ret_code = 0 + else: + # Check submission file + base_check_code = T(".eval_tests.submission_format_test", ftype="txt").r() + implementation.inject_files(**{"test/submission_format_test.py": base_check_code}) + # stdout += "----Submission Check 1-----\n" + submission_check_out, submission_ret_code = implementation.execute_ret_code( + env=env, entry="python test/submission_format_test.py" + ) + if DS_RD_SETTING.rule_base_eval: + if execute_ret_code == 0 and score_ret_code == 0 and submission_ret_code == 0: + return PipelineSingleFeedback( + execution=stdout, + return_checking=score_check_text + "\n" + submission_check_out, + code="Code evaluation is not available.", + final_decision=True, + ) + else: + return PipelineSingleFeedback( + execution=stdout, + return_checking=score_check_text + "\n" + submission_check_out, + code="Code evaluation is not available.", + final_decision=False, + ) + stdout += "\n" + submission_check_out eda_output = implementation.file_dict.get("EDA.md", None) system_prompt = T(".prompts:pipeline_eval.system").r( scenario=self.scen.get_scenario_all_desc(eda_output=eda_output), task_desc=target_task.get_task_information(), + is_sub_enabled=test_eval.is_sub_enabled(self.scen.competition), spec=T("scenarios.data_science.share:component_spec.Pipeline").r(), ) user_prompt = T(".prompts:pipeline_eval.user").r( diff --git a/rdagent/components/coder/data_science/pipeline/prompts.yaml b/rdagent/components/coder/data_science/pipeline/prompts.yaml index 1b694cdd..4331ce0b 100644 --- a/rdagent/components/coder/data_science/pipeline/prompts.yaml +++ b/rdagent/components/coder/data_science/pipeline/prompts.yaml @@ -111,7 +111,8 @@ pipeline_eval: The details on how to structure the code are given in the specification: {{ spec }} - + + {% if is_sub_enabled %} ## Evaluation Scope Your focus is to check whether the workflow code: Step 1: Executes successfully, correctly generating a final submission. @@ -146,6 +147,26 @@ pipeline_eval: "final_decision": } ``` + {% else %} + ## Evaluation Scope + Your focus is to check whether the workflow code executes successfully. + + You will be given the execution output (`stdout`) to determine correctness. + + [Note] + 1. Model performance is NOT a concern in this evaluationβ€”only correct execution and formatting matter. + + Please respond with your feedback in the following JSON format and order + ```json + { + "execution": "Describe whether the code executed successfully. Include any errors or issues encountered, and append all error messages and full traceback details without summarizing or omitting any information.", + "return_checking": "Describe the expected file to be generated.", + "code": "Provide feedback on code quality, readability, and adherence to the given specifications.", + "final_decision": + } + ``` + {% endif %} +# NOTE: when is_sub_enabled == False, we don't have any checking about the return. So it is just placeholder currently user: |- --------- code generated by user --------- diff --git a/rdagent/components/coder/data_science/raw_data_loader/eval.py b/rdagent/components/coder/data_science/raw_data_loader/eval.py index a977bd5c..26dc6758 100644 --- a/rdagent/components/coder/data_science/raw_data_loader/eval.py +++ b/rdagent/components/coder/data_science/raw_data_loader/eval.py @@ -46,13 +46,7 @@ class DataLoaderCoSTEEREvaluator(CoSTEEREvaluator): final_decision=False, ) - env = get_ds_env( - extra_volumes={ - f"{DS_RD_SETTING.local_data_path}/sample/{self.scen.competition}": T( - "scenarios.data_science.share:scen.input_path" - ).r() - } - ) + env = get_ds_env(extra_volumes={self.scen.debug_path: T("scenarios.data_science.share:scen.input_path").r()}) # TODO: do we need to clean the generated temporary content? fname = "test/data_loader_test.py" diff --git a/rdagent/components/coder/data_science/workflow/eval.py b/rdagent/components/coder/data_science/workflow/eval.py index 2b773a4f..927317ba 100644 --- a/rdagent/components/coder/data_science/workflow/eval.py +++ b/rdagent/components/coder/data_science/workflow/eval.py @@ -55,13 +55,7 @@ class WorkflowGeneralCaseSpecEvaluator(CoSTEEREvaluator): final_decision=False, ) - env = get_ds_env( - extra_volumes={ - f"{DS_RD_SETTING.local_data_path}/sample/{self.scen.competition}": T( - "scenarios.data_science.share:scen.input_path" - ).r() - } - ) + env = get_ds_env(extra_volumes={self.scen.debug_path: T("scenarios.data_science.share:scen.input_path").r()}) # # DockerEnv for MLEBench submission validation # mle_de_conf = MLEBDockerConf() diff --git a/rdagent/log/mle_summary.py b/rdagent/log/mle_summary.py index a6be5b84..a2f1f96f 100644 --- a/rdagent/log/mle_summary.py +++ b/rdagent/log/mle_summary.py @@ -13,11 +13,16 @@ from rdagent.core.experiment import FBWorkspace from rdagent.core.proposal import ExperimentFeedback from rdagent.log.storage import FileStorage from rdagent.scenarios.data_science.experiment.experiment import DSExperiment +from rdagent.scenarios.data_science.test_eval import ( + MLETestEval, + NoTestEvalError, + get_test_eval, +) from rdagent.scenarios.kaggle.kaggle_crawler import score_rank -from rdagent.utils.env import DockerEnv, MLEBDockerConf -de = get_ds_env(conf_type="mlebench", extra_volumes={f"{DS_RD_SETTING.local_data_path}/zip_files": "/mle/data"}) -de.prepare() +test_eval = get_test_eval() + +is_mle = isinstance(test_eval, MLETestEval) def extract_mle_json(log_content: str) -> dict | None: @@ -41,14 +46,15 @@ def save_grade_info(log_trace_path: Path): if "running" in msg.tag: if isinstance(msg.content, DSExperiment): - mle_score_str = msg.content.experiment_workspace.execute( - env=de, - entry=f"mlebench grade-sample submission.csv {competition} --data-dir /mle/data | tee mle_score.txt", - ) - msg.content.experiment_workspace.execute(env=de, entry="chmod 777 mle_score.txt") - trace_storage.log( - mle_score_str, name=f"{msg.tag}.mle_score.pid", save_type="pkl", timestamp=msg.timestamp - ) + # TODO: mle_score.txt is not a general name now. + # Please use a more general name like test_score.txt + try: + mle_score_str = test_eval.eval(competition, msg.content.experiment_workspace) + trace_storage.log( + mle_score_str, name=f"{msg.tag}.mle_score.pid", save_type="pkl", timestamp=msg.timestamp + ) + except Exception as e: + print(f"Error in {log_trace_path}: {e}") def is_valid_session(p: Path) -> bool: @@ -58,7 +64,10 @@ def is_valid_session(p: Path) -> bool: def save_all_grade_info(log_folder): for log_trace_path in log_folder.iterdir(): if is_valid_session(log_trace_path): - save_grade_info(log_trace_path) + try: + save_grade_info(log_trace_path) + except NoTestEvalError as e: + print(f"Error in {log_trace_path}: {e}") def summarize_folder(log_folder: Path, hours: int | None = None): @@ -106,16 +115,17 @@ def summarize_folder(log_folder: Path, hours: int | None = None): # get threshold scores workflowexp = FBWorkspace() - stdout = workflowexp.execute( - env=de, - entry=f"mlebench grade-sample None {stat[log_trace_path.name]['competition']} --data-dir /mle/data", - ) - grade_output = extract_mle_json(stdout) - if grade_output: - bronze_threshold = grade_output["bronze_threshold"] - silver_threshold = grade_output["silver_threshold"] - gold_threshold = grade_output["gold_threshold"] - median_threshold = grade_output["median_threshold"] + if is_mle: + stdout = workflowexp.execute( + env=test_eval.env, + entry=f"mlebench grade-sample None {stat[log_trace_path.name]['competition']} --data-dir /mle/data", + ) + grade_output = extract_mle_json(stdout) + if grade_output: + bronze_threshold = grade_output["bronze_threshold"] + silver_threshold = grade_output["silver_threshold"] + gold_threshold = grade_output["gold_threshold"] + median_threshold = grade_output["median_threshold"] if "direct_exp_gen" in msg.tag and isinstance(msg.content, DSExperiment): loop_num += 1 @@ -134,9 +144,10 @@ def summarize_folder(log_folder: Path, hours: int | None = None): if grade_output: if grade_output["score"] is not None: test_scores[loop_id + 1] = grade_output["score"] - _, test_ranks[loop_id + 1] = score_rank( - stat[log_trace_path.name]["competition"], grade_output["score"] - ) + if is_mle: + _, test_ranks[loop_id + 1] = score_rank( + stat[log_trace_path.name]["competition"], grade_output["score"] + ) if grade_output["valid_submission"]: valid_submission_num += 1 if grade_output["above_median"]: @@ -169,9 +180,10 @@ def summarize_folder(log_folder: Path, hours: int | None = None): sota_exp_stat = "made_submission" if grade_output["score"] is not None: sota_exp_score = grade_output["score"] - _, sota_exp_rank = score_rank( - stat[log_trace_path.name]["competition"], grade_output["score"] - ) + if is_mle: + _, sota_exp_rank = score_rank( + stat[log_trace_path.name]["competition"], grade_output["score"] + ) stat[log_trace_path.name].update( { diff --git a/rdagent/log/ui/ds_trace.py b/rdagent/log/ui/ds_trace.py index bf9a2872..71c6bf31 100644 --- a/rdagent/log/ui/ds_trace.py +++ b/rdagent/log/ui/ds_trace.py @@ -474,7 +474,10 @@ def summarize_data(): if "mle_score" in loop_data["running"]: mle_score_txt = loop_data["running"]["mle_score"] state.data[loop]["mle_score"] = extract_mle_json(mle_score_txt) - if state.data[loop]["mle_score"]["score"] is not None: + if ( + state.data[loop]["mle_score"] is not None + and state.data[loop]["mle_score"]["score"] is not None + ): df.loc[loop, "Running Score (test)"] = str(state.data[loop]["mle_score"]["score"]) else: state.data[loop]["mle_score"] = mle_score_txt diff --git a/rdagent/scenarios/data_science/dev/runner/eval.py b/rdagent/scenarios/data_science/dev/runner/eval.py index b00675e2..ba39b933 100644 --- a/rdagent/scenarios/data_science/dev/runner/eval.py +++ b/rdagent/scenarios/data_science/dev/runner/eval.py @@ -14,6 +14,11 @@ from rdagent.components.coder.data_science.utils import remove_eda_part from rdagent.core.evolving_framework import QueriedKnowledge from rdagent.core.experiment import FBWorkspace, Task from rdagent.log import rdagent_logger as logger +from rdagent.scenarios.data_science.test_eval import ( + MLETestEval, + NoTestEvalError, + get_test_eval, +) from rdagent.utils.agent.tpl import T from rdagent.utils.agent.workflow import build_cls_from_json_with_retry from rdagent.utils.fmt import shrink_text @@ -55,6 +60,7 @@ class DSCoSTEERCoSTEEREvaluator(CoSTEEREvaluator): eda_output = "No EDA output." implementation.inject_files(**{"EDA.md": eda_output}) stdout = remove_eda_part(stdout) + stdout += f"The code executed {'successfully' if execute_ret_code == 0 else 'failed'}. {'EDA output is emmitted. ' if eda_output else ''}" # Check score file score_fp = implementation.workspace_path / "scores.csv" @@ -100,28 +106,12 @@ class DSCoSTEERCoSTEEREvaluator(CoSTEEREvaluator): # DockerEnv for MLEBench submission validation submission_check_out = "" + submission_ret_code = 0 + test_eval = get_test_eval() - if DS_RD_SETTING.if_using_mle_data: - mde = get_ds_env( - conf_type="mlebench", - extra_volumes={ - f"{DS_RD_SETTING.local_data_path}/zip_files": "/mle/data", - }, - ) - mde.prepare() - # MLEBench Check - mle_check_code = ( - (Path(__file__).absolute().resolve().parent / "eval_tests" / "mle_submission_format_test.txt") - .read_text() - .replace("", self.scen.competition) - ) - implementation.inject_files(**{"test/mle_submission_format_test.py": mle_check_code}) - submission_check_out, submission_ret_code = implementation.execute_ret_code( - env=mde, entry="python test/mle_submission_format_test.py" - ) - stdout += f"\nMLEBench submission check:\n{submission_check_out}\nIf MLEBench submission check returns a 'Submission is valid' or similar message, despite some warning messages, you should still consider the submission as valid and give a positive final decision. " - implementation.inject_files(**{"test/mle_submission_format_test.output": submission_check_out}) - + if test_eval.enabled(self.scen.competition): + submission_check_out, submission_ret_code = test_eval.valid(self.scen.competition, implementation) + stdout += f"\nSubmission check:\n{submission_check_out}\nIf Submission check returns a 'Submission is valid' or similar message, despite some warning messages, you should still consider the submission as valid and give a positive final decision. " if DS_RD_SETTING.rule_base_eval: if DS_RD_SETTING.if_using_mle_data: score_check_text = score_check_text + "\n" + submission_check_out @@ -146,6 +136,7 @@ class DSCoSTEERCoSTEEREvaluator(CoSTEEREvaluator): system_prompt = T(".prompts:DSCoSTEER_eval.system").r( scenario=self.scen.get_scenario_all_desc(eda_output=implementation.file_dict.get("EDA.md", None)), + is_sub_enabled=test_eval.is_sub_enabled(self.scen.competition), task_desc=target_task.get_task_information(), ) user_prompt = T(".prompts:DSCoSTEER_eval.user").r( @@ -202,7 +193,7 @@ class DSCoSTEERCoSTEEREvaluator(CoSTEEREvaluator): if score_ret_code != 0: feedback.final_decision = False feedback.return_checking += "\n" + score_check_text - if DS_RD_SETTING.if_using_mle_data and submission_ret_code != 0: + if submission_ret_code != 0: feedback.final_decision = False feedback.return_checking += "\nSubmission file check failed." return feedback diff --git a/rdagent/scenarios/data_science/dev/runner/prompts.yaml b/rdagent/scenarios/data_science/dev/runner/prompts.yaml index a09be0d5..4fdd6e9a 100644 --- a/rdagent/scenarios/data_science/dev/runner/prompts.yaml +++ b/rdagent/scenarios/data_science/dev/runner/prompts.yaml @@ -15,6 +15,7 @@ DSCoSTEER_eval: - Model training - Ensembling + {% if is_sub_enabled %} The user will provide you the whole code base, some logs generated during the execution of the whole workflow. Your evaluation scope includes whether the workflow code: 1. Executes successfully, correctly organizing components and generating a final submission. 2. Generates predictions in the correct format, ensuring they align with the **sample submission** structure! @@ -29,7 +30,22 @@ DSCoSTEER_eval: "final_decision": } ``` - + {% else %} + The user will provide you the whole code base, some logs generated during the execution of the whole workflow. Your evaluation scope includes whether the workflow code executes successfully. + No need to check the detail of submission file. + + Please respond with your feedback in the following JSON format and order + ```json + { + "execution": "Describe whether the code executed successfully. Include any errors or issues encountered, and append all error messages and full traceback details without summarizing or omitting any information.", + "return_checking": "Describe the expected file to be generated.", + "code": "Provide feedback on code quality, readability, and adherence to the given specifications.", + "final_decision": + } + ``` + {% endif %} +# NOTE: when is_sub_enabled == False, we don't have any checking about the return. So it is just placeholder currently + user: |- --------- code base --------- {{ code }} @@ -62,4 +78,4 @@ DSCoSTEER_debugger: --------- code base --------- {{ code }} --------- feedback --------- - {{ feedback }} \ No newline at end of file + {{ feedback }} diff --git a/rdagent/scenarios/data_science/dev/runner/eval_tests/mle_submission_format_test.txt b/rdagent/scenarios/data_science/eval_tests/mle_submission_format_test.txt similarity index 100% rename from rdagent/scenarios/data_science/dev/runner/eval_tests/mle_submission_format_test.txt rename to rdagent/scenarios/data_science/eval_tests/mle_submission_format_test.txt diff --git a/rdagent/scenarios/data_science/proposal/exp_gen/base.py b/rdagent/scenarios/data_science/proposal/exp_gen/base.py index 613c09a6..efd7111a 100644 --- a/rdagent/scenarios/data_science/proposal/exp_gen/base.py +++ b/rdagent/scenarios/data_science/proposal/exp_gen/base.py @@ -34,7 +34,7 @@ class DSHypothesis(Hypothesis): if self.hypothesis == "": return f"No hypothesis available. Trying to construct the first runnable {self.component} component." lines = [] - if self.problem_name is not None and self.problem_desc is not None: + if hasattr(self, "problem_name") and self.problem_name is not None and self.problem_desc is not None: lines.append(f"Target Problem name: {self.problem_name}") lines.append(f"Target Problem: {self.problem_desc}") lines.extend( diff --git a/rdagent/scenarios/data_science/scen/__init__.py b/rdagent/scenarios/data_science/scen/__init__.py index fdb30c58..81ae91a3 100644 --- a/rdagent/scenarios/data_science/scen/__init__.py +++ b/rdagent/scenarios/data_science/scen/__init__.py @@ -1,4 +1,6 @@ import json +import os +import platform from pathlib import Path from typing import Dict @@ -32,8 +34,12 @@ class DataScienceScen(Scenario): raise FileNotFoundError(f"Cannot find {competition} in {DS_RD_SETTING.local_data_path}") local_path = DS_RD_SETTING.local_data_path - if not Path(f"{local_path}/sample/{competition}").exists(): - create_debug_data(competition, dataset_path=local_path) + if DS_RD_SETTING.sample_data: + self.debug_path = f"{local_path}/sample/{competition}" + if not Path(self.debug_path).exists(): + create_debug_data(competition, dataset_path=local_path) + else: + self.debug_path = f"{local_path}/{competition}" # 2) collect information of competition. self.metric_name: str | None = ( @@ -56,6 +62,7 @@ class DataScienceScen(Scenario): def _get_description(self): if (fp := Path(f"{DS_RD_SETTING.local_data_path}/{self.competition}/description.md")).exists(): + logger.info(f"{self.competition}/Found description.md, loading from local file.") return fp.read_text() elif (fp := Path(f"{DS_RD_SETTING.local_data_path}/{self.competition}.json")).exists(): logger.info(f"Found {self.competition}.json, loading from local file.") @@ -125,6 +132,8 @@ class DataScienceScen(Scenario): evaluation=self.metric_description, metric_name=self.metric_name, metric_direction=self.metric_direction, + raw_description=self.raw_description, + use_raw_description=DS_RD_SETTING.use_raw_description, time_limit=None, eda_output=None, ) @@ -139,6 +148,8 @@ class DataScienceScen(Scenario): evaluation=self.metric_description, metric_name=self.metric_name, metric_direction=self.metric_direction, + raw_description=self.raw_description, + use_raw_description=DS_RD_SETTING.use_raw_description, time_limit=f"{DS_RD_SETTING.full_timeout / 60 / 60 : .2f} hours", eda_output=eda_output, ) diff --git a/rdagent/scenarios/data_science/scen/prompts.yaml b/rdagent/scenarios/data_science/scen/prompts.yaml index 1a807c07..cbeca1c6 100644 --- a/rdagent/scenarios/data_science/scen/prompts.yaml +++ b/rdagent/scenarios/data_science/scen/prompts.yaml @@ -1,5 +1,13 @@ scenario_description: |- + {% if use_raw_description %} ------Background of the scenario------ + {{ raw_description }} + + The evaluation metrics used is directed as: + The metric is better when it is {% if metric_direction %}bigger{% else %}smaller{% endif %}. + + {% else %} + ------Background of the scenario------ {{ background }} ------ Guidelines for participating in the competition ---- @@ -10,6 +18,7 @@ scenario_description: |- ------The name of the evaluation metric used------ {{ metric_name }} + {% endif %} {% if time_limit %}------The time limit to your code------ You code running is limit to {{ time_limit }}, please change yor model type and model parameters to make sure your code can run within the time limit. @@ -30,6 +39,7 @@ competition_description_template: system: |- You are a data science assistant that extracts structured information from unstructured text. The user will provide you a Kaggle competition description, and you need to extract specific details from it. + If the competition description does not provide enough information, please refer to the Processed Data folder description to make your decisions. For the dataset, the competition may not include detailed information about the dataset. The user has read the dataset and provide you the relevant information. Please include it in your response. Please answer in Json format with the following schema: { diff --git a/rdagent/scenarios/data_science/test_eval.py b/rdagent/scenarios/data_science/test_eval.py new file mode 100644 index 00000000..40b8bbe6 --- /dev/null +++ b/rdagent/scenarios/data_science/test_eval.py @@ -0,0 +1,133 @@ +from abc import abstractmethod +from pathlib import Path + +from rdagent.app.data_science.conf import DS_RD_SETTING +from rdagent.components.coder.data_science.conf import get_ds_env +from rdagent.core.experiment import FBWorkspace + + +class NoTestEvalError(Exception): + """Test evaluation is not provided""" + + +class TestEvalBase: + """Evaluate a workspace on Test Dataset""" + + @abstractmethod + def eval(self, competition: str, workspace: FBWorkspace) -> str: + """eval the workspace as competition, and return the final evaluation result""" + + @abstractmethod + def valid(self, competition: str, workspace: FBWorkspace) -> tuple[str, int]: + """eval the workspace as competition, and return the final format check result""" + + @abstractmethod + def enabled(self, competition) -> bool: + """able to eval or not""" + + @abstractmethod + def is_sub_enabled(self, competition: str) -> bool: + """ + Is subsmiossion file enabled + + If a file like is provided; then we think inference from test data to submission file is enabled. + According test will be enabled as well. + + Why do not we merge `is_sub_enabled` and `enabled`, cases: + 1. The dataset provide evaluation. But we don't provide submission sample(llm will decide by himself) + 2. We proivde a sample submission. But we don't proivde strict evaluation. + + """ + input_dir = Path(f"{DS_RD_SETTING.local_data_path}/{competition}") + sample_submission_files = list(input_dir.glob("*sample_submission*.csv")) + list( + input_dir.glob("*sampleSubmission*.csv") + ) + return len(sample_submission_files) > 0 + + +class TestEval(TestEvalBase): + """The most basic version of evaluation for test data""" + + def __init__(self) -> None: + super().__init__() + self.env = get_ds_env() + + def eval(self, competition: str, workspace: FBWorkspace) -> str: + eval_path = Path(f"{DS_RD_SETTING.local_data_path}/{DS_RD_SETTING.eval_sub_dir}/{competition}") + if not eval_path.exists(): + err_msg = f"No Test Eval provided due to: {eval_path} not found" + raise NoTestEvalError(err_msg) + workspace.inject_files(**{"grade.py": (eval_path / "grade.py").read_text()}) + workspace.inject_files(**{"submission_test.csv": (eval_path / "submission_test.csv").read_text()}) + workspace.execute( + env=self.env, + entry=f"python grade.py {competition} | tee mle_score.txt", + ) + workspace.inject_files(**{file: workspace.DEL_KEY for file in ["grade.py", "submission_test.csv"]}) + workspace.execute(env=self.env, entry="chmod 777 mle_score.txt") + return (workspace.workspace_path / "mle_score.txt").read_text() + + def valid(self, competition: str, workspace: FBWorkspace) -> tuple[str, int]: + eval_path = Path(f"{DS_RD_SETTING.local_data_path}/{DS_RD_SETTING.eval_sub_dir}/{competition}") + if not eval_path.exists(): + err_msg = f"No Test Eval provided due to: {eval_path} not found" + raise NoTestEvalError(err_msg) + workspace.inject_files(**{"submission_format_valid.py": (eval_path / "valid.py").read_text()}) + workspace.inject_files(**{"submission_test.csv": (eval_path / "submission_test.csv").read_text()}) + submission_check_out, submission_ret_code = workspace.execute_ret_code( + env=self.env, + entry=f"python submission_format_valid.py {competition}", + ) + workspace.inject_files( + **{file: workspace.DEL_KEY for file in ["submission_format_valid.py", "submission_test.csv"]} + ) + workspace.inject_files(**{"test/mle_submission_format_test.output": submission_check_out}) + return submission_check_out, submission_ret_code + + def enabled(self, competition) -> bool: + return Path( + f"{DS_RD_SETTING.local_data_path}/{DS_RD_SETTING.eval_sub_dir}/{competition}/submission_test.csv" + ).exists() + + +class MLETestEval(TestEvalBase): + """Evaluation for test data for MLE-Bench competition""" + + def __init__(self) -> None: + super().__init__() + self.env = get_ds_env( + conf_type="mlebench", extra_volumes={f"{DS_RD_SETTING.local_data_path}/zip_files": "/mle/data"} + ) + self.env.prepare() + + def eval(self, competition: str, workspace: FBWorkspace) -> str: + workspace.execute( + env=self.env, + entry=f"mlebench grade-sample submission.csv {competition} --data-dir /mle/data | tee mle_score.txt", + ) + workspace.execute(env=self.env, entry="chmod 777 mle_score.txt") + return (workspace.workspace_path / "mle_score.txt").read_text() + + def valid(self, competition: str, workspace: FBWorkspace) -> tuple[str, int]: + mle_check_code = ( + (Path(__file__).absolute().resolve().parent / "eval_tests" / "mle_submission_format_test.txt") + .read_text() + .replace("", self.scen.competition) + ) + workspace.inject_files(**{"test/mle_submission_format_test.py": mle_check_code}) + submission_check_out, submission_ret_code = workspace.execute_ret_code( + env=mde, entry="python test/mle_submission_format_test.py" + ) + + workspace.inject_files(**{"test/mle_submission_format_test.output": submission_check_out}) + return submission_check_out, submission_ret_code + + def enabled(self, competition) -> bool: + return True + + +def get_test_eval() -> TestEvalBase: + """Get the test evaluation instance""" + if DS_RD_SETTING.if_using_mle_data: + return MLETestEval() + return TestEval() diff --git a/rdagent/utils/env.py b/rdagent/utils/env.py index 5c9c90f9..de81d988 100644 --- a/rdagent/utils/env.py +++ b/rdagent/utils/env.py @@ -205,6 +205,13 @@ class Env(Generic[ASpecificEnvConf]): if entry is None: entry = self.conf.default_entry + if "|" in entry: + logger.warning( + "You are using a command with a shell pipeline (i.e., '|'). " + "The exit code ($exit_code) will reflect the result of " + "the last command in the pipeline.", + ) + entry_add_timeout = ( f"/bin/sh -c 'timeout {self.conf.running_timeout_period} {entry}; " + "entry_exit_code=$?; "