gavindiaz/NexQuant
1
0
Fork 0
mirror of https://github.com/NicolasBohn/NexQuant.git synced 2026-07-29 08:27:43 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: custom data (#810)

Browse Source 
* custom data

* fix: simplify competition check and log local description file

* no sample data

* feat: add test evaluation module with error handling support

* fix: update eval path to use eval_sub_dir and add valid_check TODO

* refactor: add MLETestEval check to conditionally run grading steps

* avoid blank stdout

* valid in testeval

* rename test.csv to avoid conflict

* Support Disabling sample submission

* refactoring

* fix: remove DS_KAGGLE_DATA and update prompt instructions

* add try for grade

* ignore submission

* fix: remove tee from eval command and warn about pipeline exit code detection

* optional to use raw description

* support old data

* add execution result to stdout

* add metric to raw description

* custom data explain

* add debug_path

* rst update

---------

Co-authored-by: Young <afe.young@gmail.com>
This commit is contained in:
Tim
2025-05-06 16:00:13 +08:00
committed by GitHub
parent eda4a2a807
commit 46dec7b624
23 changed files with 376 additions and 134 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docs/scens/catalog.rst
+1
View File
@@ -46,4 +46,5 @@ The supported scenarios are listed below:
model_agent_med
model_copilot_general
kaggle_agent
data_science
docs/scens/data_science.rst
+60
View File
@@ -0,0 +1,60 @@
.. _data_science_agent:
=======================
Data Science Agent
=======================
**🤖 Automated Feature Engineering & Model Tuning Evolution**
------------------------------------------------------------------------------------------
The Data Science Agent is an agent that can automatically perform feature engineering and model tuning. It can be used to solve various data science problems, such as image classification, time series forecasting, and text classification.
🧭 Example Guide
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
- 🔧 **Set up RD-Agent Environment**
- Before you start, please make sure you have installed RD-Agent and configured the environment for RD-Agent correctly. If you want to know how to install and configure the RD-Agent, please refer to the `documentation <../installation_and_configuration.html>`_.
- 🔩 **Setting the Environment variables at .env file**
- Determine the path where the data will be stored and add it to the ``.env`` file.
.. code-block:: sh
dotenv set DS_LOCAL_DATA_PATH <your local directory>/ds_data
dotenv set DS_SCEN rdagent.scenarios.data_science.scen.DataScienceScen
- 📥 **Prepare Competition Data**
- Data Science competition data, contains three parts: competition description file (markdown file), competition dataset and competition evaluation files.
- **Correct directory structure (Here is an example of competition data with id custom_data)**
.. code-block:: text
ds_data
└── eval
| └── custom_data
| └── grade.py
| └── valid.py
| └── test.csv
└── custom_data
└── train.csv
└── test.csv
└── sample_submission.csv
└── description.md
- ``ds_data/custom_data/train.csv:`` Necessary training data in csv or parquet format, or training images.
- ``ds_data/custom_data/description.md:`` (Optional) Competition description file.
- ``ds_data/custom_data/sample_submission.csv:`` (Optional) Competition sample submission file.
- ``ds_data/eval/custom_data/grade.py:`` (Optional) Competition grade script, in order to calculate the score for the submission.
- ``ds_data/eval/custom_data/valid.py:`` (Optional) Competition validation script, in order to check if the submission format is correct.
- ``ds_data/eval/custom_data/submission_test.csv:`` (Optional) Competition test label file.
rdagent/app/data_science/conf.py
+10 -2
View File
@@ -6,6 +6,7 @@ from rdagent.app.kaggle.conf import KaggleBasePropSetting
class DataScienceBasePropSetting(KaggleBasePropSetting):
# TODO: Kaggle Setting should be the subclass of DataScience
model_config = SettingsConfigDict(env_prefix="DS_", protected_namespaces=())
# Main components
@@ -29,7 +30,7 @@ class DataScienceBasePropSetting(KaggleBasePropSetting):
#### enable specification
spec_enabled: bool = True
### proposal related
#### proposal related
proposal_version: str = "v1"
coder_on_whole_pipeline: bool = False
max_trace_hist: int = 3
@@ -38,8 +39,10 @@ class DataScienceBasePropSetting(KaggleBasePropSetting):
runner_max_loop: int = 1
rule_base_eval: bool = False
sample_data: bool = True
use_raw_description: bool = False
### model dump
#### model dump
enable_model_dump: bool = False
enable_doc_dev: bool = False
model_dump_check_level: Literal["medium", "high"] = "medium"
@@ -57,5 +60,10 @@ class DataScienceBasePropSetting(KaggleBasePropSetting):
None # This is to store the mid tar file since writing the tar file is preferred in local storage then copy to target storage
)
#### Evaluation on Test related
eval_sub_dir: str = "eval" # TODO: fixme, this is not a good name
"""We'll use f"{DS_RD_SETTING.local_data_path}/{DS_RD_SETTING.eval_sub_dir}/{competition}"
to find the scriipt to evaluate the submission on test"""
DS_RD_SETTING = DataScienceBasePropSetting()
rdagent/app/data_science/loop.py
+2 -8
View File
@@ -304,15 +304,9 @@ def main(
if competition is not None:
DS_RD_SETTING.competition = competition
if DS_RD_SETTING.competition:
if DS_RD_SETTING.scen.endswith("KaggleScen"):
download_data(competition=DS_RD_SETTING.competition, settings=DS_RD_SETTING)
else:
if not Path(f"{DS_RD_SETTING.local_data_path}/{competition}").exists():
logger.error(f"Please prepare data for competition {competition} first.")
return
else:
if not DS_RD_SETTING.competition:
logger.error("Please specify competition name.")
if path is None:
kaggle_loop = DataScienceRDLoop(DS_RD_SETTING)
else:
rdagent/app/kaggle/conf.py
+2
View File
@@ -46,9 +46,11 @@ class KaggleBasePropSetting(ExtendedBaseSettings):
local_data_path: str = ""
"""Folder storing Kaggle competition data"""
# Evaluation on Test related
if_using_mle_data: bool = False
auto_submit: bool = False
"""Automatically upload and submit each experiment result to Kaggle platform"""
# Conditionally set the knowledge_base based on the use of graph RAG
knowledge_base: str = ""
"""Knowledge base class, uses 'KGKnowledgeGraph' when advanced graph-based RAG is enabled, otherwise empty."""
rdagent/components/coder/data_science/conf.py
+3 -1
View File
@@ -27,7 +27,9 @@ class DSCoderCoSTEERSettings(CoSTEERSettings):
def get_ds_env(
conf_type: Literal["kaggle", "mlebench"] = "kaggle",
extra_volumes: dict = {},
running_timeout_period: int = DS_RD_SETTING.debug_timeout,
running_timeout_period: int = (
DS_RD_SETTING.debug_timeout if DS_RD_SETTING.sample_data else DS_RD_SETTING.full_timeout
),
) -> Env:
"""
Retrieve the appropriate environment configuration based on the env_type setting.
rdagent/components/coder/data_science/ensemble/eval.py
+1 -7
View File
@@ -47,13 +47,7 @@ class EnsembleCoSTEEREvaluator(CoSTEEREvaluator):
final_decision=False,
)
env = get_ds_env(
extra_volumes={
f"{DS_RD_SETTING.local_data_path}/sample/{self.scen.competition}": T(
"scenarios.data_science.share:scen.input_path"
).r()
}
)
env = get_ds_env(extra_volumes={self.scen.debug_path: T("scenarios.data_science.share:scen.input_path").r()})
fname = "test/ensemble_test.txt"
test_code = (DIRNAME / "eval_tests" / "ensemble_test.txt").read_text()
rdagent/components/coder/data_science/feature/eval.py
+1 -7
View File
@@ -43,13 +43,7 @@ class FeatureCoSTEEREvaluator(CoSTEEREvaluator):
final_decision=False,
)
env = get_ds_env(
extra_volumes={
f"{DS_RD_SETTING.local_data_path}/sample/{self.scen.competition}": T(
"scenarios.data_science.share:scen.input_path"
).r()
}
)
env = get_ds_env(extra_volumes={self.scen.debug_path: T("scenarios.data_science.share:scen.input_path").r()})
# TODO: do we need to clean the generated temporary content?
fname = "test/feature_test.py"
rdagent/components/coder/data_science/model/eval.py
+1 -7
View File
@@ -57,13 +57,7 @@ class ModelGeneralCaseSpecEvaluator(CoSTEEREvaluator):
final_decision=False,
)
env = get_ds_env(
extra_volumes={
f"{DS_RD_SETTING.local_data_path}/sample/{self.scen.competition}": T(
"scenarios.data_science.share:scen.input_path"
).r()
}
)
env = get_ds_env(extra_volumes={self.scen.debug_path: T("scenarios.data_science.share:scen.input_path").r()})
if_model_removed = False
rdagent/components/coder/data_science/pipeline/eval.py
+32 -31
View File
@@ -18,6 +18,7 @@ from rdagent.components.coder.CoSTEER.knowledge_management import (
from rdagent.components.coder.data_science.conf import get_clear_ws_cmd, get_ds_env
from rdagent.components.coder.data_science.utils import remove_eda_part
from rdagent.core.experiment import FBWorkspace, Task
from rdagent.scenarios.data_science.test_eval import get_test_eval
from rdagent.utils.agent.tpl import T
from rdagent.utils.agent.workflow import build_cls_from_json_with_retry
@@ -52,18 +53,13 @@ class PipelineCoSTEEREvaluator(CoSTEEREvaluator):
final_decision=False,
)
env = get_ds_env(
extra_volumes={
f"{DS_RD_SETTING.local_data_path}/sample/{self.scen.competition}": T(
"scenarios.data_science.share:scen.input_path"
).r()
}
)
env = get_ds_env(extra_volumes={self.scen.debug_path: T("scenarios.data_science.share:scen.input_path").r()})
# Clean the scores.csv & submission.csv.
implementation.execute(env=env, entry=get_clear_ws_cmd())
stdout, execute_ret_code = implementation.execute_ret_code(env=env, entry=f"python main.py")
stdout, execute_ret_code = implementation.execute_ret_code(env=env, entry=f"python -m coverage run main.py")
stdout = remove_eda_part(stdout)
stdout += f"The code executed {'successfully' if execute_ret_code == 0 else 'failed'}."
score_fp = implementation.workspace_path / "scores.csv"
score_ret_code = 0
@@ -101,35 +97,40 @@ class PipelineCoSTEEREvaluator(CoSTEEREvaluator):
score_check_text += f"\n[Error] in checking the scores.csv file: {e}\nscores.csv's content:\n-----\n{score_fp.read_text()}\n-----"
score_ret_code = 1
# Check submission file
base_check_code = T(".eval_tests.submission_format_test", ftype="txt").r()
implementation.inject_files(**{"test/submission_format_test.py": base_check_code})
# stdout += "----Submission Check 1-----\n"
submission_check_out, submission_ret_code = implementation.execute_ret_code(
env=env, entry="python test/submission_format_test.py"
)
if DS_RD_SETTING.rule_base_eval:
if execute_ret_code == 0 and score_ret_code == 0 and submission_ret_code == 0:
return PipelineSingleFeedback(
execution=stdout,
return_checking=score_check_text + "\n" + submission_check_out,
code="Code evaluation is not available.",
final_decision=True,
)
else:
return PipelineSingleFeedback(
execution=stdout,
return_checking=score_check_text + "\n" + submission_check_out,
code="Code evaluation is not available.",
final_decision=False,
)
stdout += "\n" + submission_check_out
test_eval = get_test_eval()
if not test_eval.is_sub_enabled(self.scen.competition):
submission_ret_code = 0
else:
# Check submission file
base_check_code = T(".eval_tests.submission_format_test", ftype="txt").r()
implementation.inject_files(**{"test/submission_format_test.py": base_check_code})
# stdout += "----Submission Check 1-----\n"
submission_check_out, submission_ret_code = implementation.execute_ret_code(
env=env, entry="python test/submission_format_test.py"
)
if DS_RD_SETTING.rule_base_eval:
if execute_ret_code == 0 and score_ret_code == 0 and submission_ret_code == 0:
return PipelineSingleFeedback(
execution=stdout,
return_checking=score_check_text + "\n" + submission_check_out,
code="Code evaluation is not available.",
final_decision=True,
)
else:
return PipelineSingleFeedback(
execution=stdout,
return_checking=score_check_text + "\n" + submission_check_out,
code="Code evaluation is not available.",
final_decision=False,
)
stdout += "\n" + submission_check_out
eda_output = implementation.file_dict.get("EDA.md", None)
system_prompt = T(".prompts:pipeline_eval.system").r(
scenario=self.scen.get_scenario_all_desc(eda_output=eda_output),
task_desc=target_task.get_task_information(),
is_sub_enabled=test_eval.is_sub_enabled(self.scen.competition),
spec=T("scenarios.data_science.share:component_spec.Pipeline").r(),
)
user_prompt = T(".prompts:pipeline_eval.user").r(
rdagent/components/coder/data_science/pipeline/prompts.yaml
+22 -1
View File
@@ -111,7 +111,8 @@ pipeline_eval:
The details on how to structure the code are given in the specification:
{{ spec }}
{% if is_sub_enabled %}
## Evaluation Scope
Your focus is to check whether the workflow code:
Step 1: Executes successfully, correctly generating a final submission.
@@ -146,6 +147,26 @@ pipeline_eval:
"final_decision": <true/false>
}
```
{% else %}
## Evaluation Scope
Your focus is to check whether the workflow code executes successfully.
You will be given the execution output (`stdout`) to determine correctness.
[Note]
1. Model performance is NOT a concern in this evaluation—only correct execution and formatting matter.
Please respond with your feedback in the following JSON format and order
```json
{
"execution": "Describe whether the code executed successfully. Include any errors or issues encountered, and append all error messages and full traceback details without summarizing or omitting any information.",
"return_checking": "Describe the expected file to be generated.",
"code": "Provide feedback on code quality, readability, and adherence to the given specifications.",
"final_decision": <true/false>
}
```
{% endif %}
# NOTE: when is_sub_enabled == False, we don't have any checking about the return. So it is just placeholder currently
user: |-
--------- code generated by user ---------
rdagent/components/coder/data_science/raw_data_loader/eval.py
+1 -7
View File
@@ -46,13 +46,7 @@ class DataLoaderCoSTEEREvaluator(CoSTEEREvaluator):
final_decision=False,
)
env = get_ds_env(
extra_volumes={
f"{DS_RD_SETTING.local_data_path}/sample/{self.scen.competition}": T(
"scenarios.data_science.share:scen.input_path"
).r()
}
)
env = get_ds_env(extra_volumes={self.scen.debug_path: T("scenarios.data_science.share:scen.input_path").r()})
# TODO: do we need to clean the generated temporary content?
fname = "test/data_loader_test.py"
rdagent/components/coder/data_science/workflow/eval.py
+1 -7
View File
@@ -55,13 +55,7 @@ class WorkflowGeneralCaseSpecEvaluator(CoSTEEREvaluator):
final_decision=False,
)
env = get_ds_env(
extra_volumes={
f"{DS_RD_SETTING.local_data_path}/sample/{self.scen.competition}": T(
"scenarios.data_science.share:scen.input_path"
).r()
}
)
env = get_ds_env(extra_volumes={self.scen.debug_path: T("scenarios.data_science.share:scen.input_path").r()})
# # DockerEnv for MLEBench submission validation
# mle_de_conf = MLEBDockerConf()
rdagent/log/mle_summary.py
+40 -28
View File
@@ -13,11 +13,16 @@ from rdagent.core.experiment import FBWorkspace
from rdagent.core.proposal import ExperimentFeedback
from rdagent.log.storage import FileStorage
from rdagent.scenarios.data_science.experiment.experiment import DSExperiment
from rdagent.scenarios.data_science.test_eval import (
MLETestEval,
NoTestEvalError,
get_test_eval,
)
from rdagent.scenarios.kaggle.kaggle_crawler import score_rank
from rdagent.utils.env import DockerEnv, MLEBDockerConf
de = get_ds_env(conf_type="mlebench", extra_volumes={f"{DS_RD_SETTING.local_data_path}/zip_files": "/mle/data"})
de.prepare()
test_eval = get_test_eval()
is_mle = isinstance(test_eval, MLETestEval)
def extract_mle_json(log_content: str) -> dict | None:
@@ -41,14 +46,15 @@ def save_grade_info(log_trace_path: Path):
if "running" in msg.tag:
if isinstance(msg.content, DSExperiment):
mle_score_str = msg.content.experiment_workspace.execute(
env=de,
entry=f"mlebench grade-sample submission.csv {competition} --data-dir /mle/data | tee mle_score.txt",
)
msg.content.experiment_workspace.execute(env=de, entry="chmod 777 mle_score.txt")
trace_storage.log(
mle_score_str, name=f"{msg.tag}.mle_score.pid", save_type="pkl", timestamp=msg.timestamp
)
# TODO: mle_score.txt is not a general name now.
# Please use a more general name like test_score.txt
try:
mle_score_str = test_eval.eval(competition, msg.content.experiment_workspace)
trace_storage.log(
mle_score_str, name=f"{msg.tag}.mle_score.pid", save_type="pkl", timestamp=msg.timestamp
)
except Exception as e:
print(f"Error in {log_trace_path}: {e}")
def is_valid_session(p: Path) -> bool:
@@ -58,7 +64,10 @@ def is_valid_session(p: Path) -> bool:
def save_all_grade_info(log_folder):
for log_trace_path in log_folder.iterdir():
if is_valid_session(log_trace_path):
save_grade_info(log_trace_path)
try:
save_grade_info(log_trace_path)
except NoTestEvalError as e:
print(f"Error in {log_trace_path}: {e}")
def summarize_folder(log_folder: Path, hours: int | None = None):
@@ -106,16 +115,17 @@ def summarize_folder(log_folder: Path, hours: int | None = None):
# get threshold scores
workflowexp = FBWorkspace()
stdout = workflowexp.execute(
env=de,
entry=f"mlebench grade-sample None {stat[log_trace_path.name]['competition']} --data-dir /mle/data",
)
grade_output = extract_mle_json(stdout)
if grade_output:
bronze_threshold = grade_output["bronze_threshold"]
silver_threshold = grade_output["silver_threshold"]
gold_threshold = grade_output["gold_threshold"]
median_threshold = grade_output["median_threshold"]
if is_mle:
stdout = workflowexp.execute(
env=test_eval.env,
entry=f"mlebench grade-sample None {stat[log_trace_path.name]['competition']} --data-dir /mle/data",
)
grade_output = extract_mle_json(stdout)
if grade_output:
bronze_threshold = grade_output["bronze_threshold"]
silver_threshold = grade_output["silver_threshold"]
gold_threshold = grade_output["gold_threshold"]
median_threshold = grade_output["median_threshold"]
if "direct_exp_gen" in msg.tag and isinstance(msg.content, DSExperiment):
loop_num += 1
@@ -134,9 +144,10 @@ def summarize_folder(log_folder: Path, hours: int | None = None):
if grade_output:
if grade_output["score"] is not None:
test_scores[loop_id + 1] = grade_output["score"]
_, test_ranks[loop_id + 1] = score_rank(
stat[log_trace_path.name]["competition"], grade_output["score"]
)
if is_mle:
_, test_ranks[loop_id + 1] = score_rank(
stat[log_trace_path.name]["competition"], grade_output["score"]
)
if grade_output["valid_submission"]:
valid_submission_num += 1
if grade_output["above_median"]:
@@ -169,9 +180,10 @@ def summarize_folder(log_folder: Path, hours: int | None = None):
sota_exp_stat = "made_submission"
if grade_output["score"] is not None:
sota_exp_score = grade_output["score"]
_, sota_exp_rank = score_rank(
stat[log_trace_path.name]["competition"], grade_output["score"]
)
if is_mle:
_, sota_exp_rank = score_rank(
stat[log_trace_path.name]["competition"], grade_output["score"]
)
stat[log_trace_path.name].update(
{
rdagent/log/ui/ds_trace.py
+4 -1
View File
@@ -474,7 +474,10 @@ def summarize_data():
if "mle_score" in loop_data["running"]:
mle_score_txt = loop_data["running"]["mle_score"]
state.data[loop]["mle_score"] = extract_mle_json(mle_score_txt)
if state.data[loop]["mle_score"]["score"] is not None:
if (
state.data[loop]["mle_score"] is not None
and state.data[loop]["mle_score"]["score"] is not None
):
df.loc[loop, "Running Score (test)"] = str(state.data[loop]["mle_score"]["score"])
else:
state.data[loop]["mle_score"] = mle_score_txt
rdagent/scenarios/data_science/dev/runner/eval.py
+13 -22
View File
@@ -14,6 +14,11 @@ from rdagent.components.coder.data_science.utils import remove_eda_part
from rdagent.core.evolving_framework import QueriedKnowledge
from rdagent.core.experiment import FBWorkspace, Task
from rdagent.log import rdagent_logger as logger
from rdagent.scenarios.data_science.test_eval import (
MLETestEval,
NoTestEvalError,
get_test_eval,
)
from rdagent.utils.agent.tpl import T
from rdagent.utils.agent.workflow import build_cls_from_json_with_retry
from rdagent.utils.fmt import shrink_text
@@ -55,6 +60,7 @@ class DSCoSTEERCoSTEEREvaluator(CoSTEEREvaluator):
eda_output = "No EDA output."
implementation.inject_files(**{"EDA.md": eda_output})
stdout = remove_eda_part(stdout)
stdout += f"The code executed {'successfully' if execute_ret_code == 0 else 'failed'}. {'EDA output is emmitted. ' if eda_output else ''}"
# Check score file
score_fp = implementation.workspace_path / "scores.csv"
@@ -100,28 +106,12 @@ class DSCoSTEERCoSTEEREvaluator(CoSTEEREvaluator):
# DockerEnv for MLEBench submission validation
submission_check_out = ""
submission_ret_code = 0
test_eval = get_test_eval()
if DS_RD_SETTING.if_using_mle_data:
mde = get_ds_env(
conf_type="mlebench",
extra_volumes={
f"{DS_RD_SETTING.local_data_path}/zip_files": "/mle/data",
},
)
mde.prepare()
# MLEBench Check
mle_check_code = (
(Path(__file__).absolute().resolve().parent / "eval_tests" / "mle_submission_format_test.txt")
.read_text()
.replace("<competition_id>", self.scen.competition)
)
implementation.inject_files(**{"test/mle_submission_format_test.py": mle_check_code})
submission_check_out, submission_ret_code = implementation.execute_ret_code(
env=mde, entry="python test/mle_submission_format_test.py"
)
stdout += f"\nMLEBench submission check:\n{submission_check_out}\nIf MLEBench submission check returns a 'Submission is valid' or similar message, despite some warning messages, you should still consider the submission as valid and give a positive final decision. "
implementation.inject_files(**{"test/mle_submission_format_test.output": submission_check_out})
if test_eval.enabled(self.scen.competition):
submission_check_out, submission_ret_code = test_eval.valid(self.scen.competition, implementation)
stdout += f"\nSubmission check:\n{submission_check_out}\nIf Submission check returns a 'Submission is valid' or similar message, despite some warning messages, you should still consider the submission as valid and give a positive final decision. "
if DS_RD_SETTING.rule_base_eval:
if DS_RD_SETTING.if_using_mle_data:
score_check_text = score_check_text + "\n" + submission_check_out
@@ -146,6 +136,7 @@ class DSCoSTEERCoSTEEREvaluator(CoSTEEREvaluator):
system_prompt = T(".prompts:DSCoSTEER_eval.system").r(
scenario=self.scen.get_scenario_all_desc(eda_output=implementation.file_dict.get("EDA.md", None)),
is_sub_enabled=test_eval.is_sub_enabled(self.scen.competition),
task_desc=target_task.get_task_information(),
)
user_prompt = T(".prompts:DSCoSTEER_eval.user").r(
@@ -202,7 +193,7 @@ class DSCoSTEERCoSTEEREvaluator(CoSTEEREvaluator):
if score_ret_code != 0:
feedback.final_decision = False
feedback.return_checking += "\n" + score_check_text
if DS_RD_SETTING.if_using_mle_data and submission_ret_code != 0:
if submission_ret_code != 0:
feedback.final_decision = False
feedback.return_checking += "\nSubmission file check failed."
return feedback
rdagent/scenarios/data_science/dev/runner/prompts.yaml
+18 -2
View File
@@ -15,6 +15,7 @@ DSCoSTEER_eval:
- Model training
- Ensembling
{% if is_sub_enabled %}
The user will provide you the whole code base, some logs generated during the execution of the whole workflow. Your evaluation scope includes whether the workflow code:
1. Executes successfully, correctly organizing components and generating a final submission.
2. Generates predictions in the correct format, ensuring they align with the **sample submission** structure!
@@ -29,7 +30,22 @@ DSCoSTEER_eval:
"final_decision": <true/false>
}
```
{% else %}
The user will provide you the whole code base, some logs generated during the execution of the whole workflow. Your evaluation scope includes whether the workflow code executes successfully.
No need to check the detail of submission file.
Please respond with your feedback in the following JSON format and order
```json
{
"execution": "Describe whether the code executed successfully. Include any errors or issues encountered, and append all error messages and full traceback details without summarizing or omitting any information.",
"return_checking": "Describe the expected file to be generated.",
"code": "Provide feedback on code quality, readability, and adherence to the given specifications.",
"final_decision": <true/false>
}
```
{% endif %}
# NOTE: when is_sub_enabled == False, we don't have any checking about the return. So it is just placeholder currently
user: |-
--------- code base ---------
{{ code }}
@@ -62,4 +78,4 @@ DSCoSTEER_debugger:
--------- code base ---------
{{ code }}
--------- feedback ---------
{{ feedback }}
{{ feedback }}
rdagent/scenarios/data_science/dev/runner/eval_tests/mle_submission_format_test.txt → rdagent/scenarios/data_science/eval_tests/mle_submission_format_test.txt
View File
rdagent/scenarios/data_science/proposal/exp_gen/base.py
+1 -1
View File
@@ -34,7 +34,7 @@ class DSHypothesis(Hypothesis):
if self.hypothesis == "":
return f"No hypothesis available. Trying to construct the first runnable {self.component} component."
lines = []
if self.problem_name is not None and self.problem_desc is not None:
if hasattr(self, "problem_name") and self.problem_name is not None and self.problem_desc is not None:
lines.append(f"Target Problem name: {self.problem_name}")
lines.append(f"Target Problem: {self.problem_desc}")
lines.extend(
rdagent/scenarios/data_science/scen/__init__.py
+13 -2
View File
@@ -1,4 +1,6 @@
import json
import os
import platform
from pathlib import Path
from typing import Dict
@@ -32,8 +34,12 @@ class DataScienceScen(Scenario):
raise FileNotFoundError(f"Cannot find {competition} in {DS_RD_SETTING.local_data_path}")
local_path = DS_RD_SETTING.local_data_path
if not Path(f"{local_path}/sample/{competition}").exists():
create_debug_data(competition, dataset_path=local_path)
if DS_RD_SETTING.sample_data:
self.debug_path = f"{local_path}/sample/{competition}"
if not Path(self.debug_path).exists():
create_debug_data(competition, dataset_path=local_path)
else:
self.debug_path = f"{local_path}/{competition}"
# 2) collect information of competition.
self.metric_name: str | None = (
@@ -56,6 +62,7 @@ class DataScienceScen(Scenario):
def _get_description(self):
if (fp := Path(f"{DS_RD_SETTING.local_data_path}/{self.competition}/description.md")).exists():
logger.info(f"{self.competition}/Found description.md, loading from local file.")
return fp.read_text()
elif (fp := Path(f"{DS_RD_SETTING.local_data_path}/{self.competition}.json")).exists():
logger.info(f"Found {self.competition}.json, loading from local file.")
@@ -125,6 +132,8 @@ class DataScienceScen(Scenario):
evaluation=self.metric_description,
metric_name=self.metric_name,
metric_direction=self.metric_direction,
raw_description=self.raw_description,
use_raw_description=DS_RD_SETTING.use_raw_description,
time_limit=None,
eda_output=None,
)
@@ -139,6 +148,8 @@ class DataScienceScen(Scenario):
evaluation=self.metric_description,
metric_name=self.metric_name,
metric_direction=self.metric_direction,
raw_description=self.raw_description,
use_raw_description=DS_RD_SETTING.use_raw_description,
time_limit=f"{DS_RD_SETTING.full_timeout / 60 / 60 : .2f} hours",
eda_output=eda_output,
)
rdagent/scenarios/data_science/scen/prompts.yaml
+10
View File
@@ -1,5 +1,13 @@
scenario_description: |-
{% if use_raw_description %}
------Background of the scenario------
{{ raw_description }}
The evaluation metrics used is directed as:
The metric is better when it is {% if metric_direction %}bigger{% else %}smaller{% endif %}.
{% else %}
------Background of the scenario------
{{ background }}
------ Guidelines for participating in the competition ----
@@ -10,6 +18,7 @@ scenario_description: |-
------The name of the evaluation metric used------
{{ metric_name }}
{% endif %}
{% if time_limit %}------The time limit to your code------
You code running is limit to {{ time_limit }}, please change yor model type and model parameters to make sure your code can run within the time limit.
@@ -30,6 +39,7 @@ competition_description_template:
system: |-
You are a data science assistant that extracts structured information from unstructured text.
The user will provide you a Kaggle competition description, and you need to extract specific details from it.
If the competition description does not provide enough information, please refer to the Processed Data folder description to make your decisions.
For the dataset, the competition may not include detailed information about the dataset. The user has read the dataset and provide you the relevant information. Please include it in your response.
Please answer in Json format with the following schema:
{
rdagent/scenarios/data_science/test_eval.py
+133
View File
@@ -0,0 +1,133 @@
from abc import abstractmethod
from pathlib import Path
from rdagent.app.data_science.conf import DS_RD_SETTING
from rdagent.components.coder.data_science.conf import get_ds_env
from rdagent.core.experiment import FBWorkspace
class NoTestEvalError(Exception):
"""Test evaluation is not provided"""
class TestEvalBase:
"""Evaluate a workspace on Test Dataset"""
@abstractmethod
def eval(self, competition: str, workspace: FBWorkspace) -> str:
"""eval the workspace as competition, and return the final evaluation result"""
@abstractmethod
def valid(self, competition: str, workspace: FBWorkspace) -> tuple[str, int]:
"""eval the workspace as competition, and return the final format check result"""
@abstractmethod
def enabled(self, competition) -> bool:
"""able to eval or not"""
@abstractmethod
def is_sub_enabled(self, competition: str) -> bool:
"""
Is subsmiossion file enabled
If a file like <sample submission csv> is provided; then we think inference from test data to submission file is enabled.
According test will be enabled as well.
Why do not we merge `is_sub_enabled` and `enabled`, cases:
1. The dataset provide evaluation. But we don't provide submission sample(llm will decide by himself)
2. We proivde a sample submission. But we don't proivde strict evaluation.
"""
input_dir = Path(f"{DS_RD_SETTING.local_data_path}/{competition}")
sample_submission_files = list(input_dir.glob("*sample_submission*.csv")) + list(
input_dir.glob("*sampleSubmission*.csv")
)
return len(sample_submission_files) > 0
class TestEval(TestEvalBase):
"""The most basic version of evaluation for test data"""
def __init__(self) -> None:
super().__init__()
self.env = get_ds_env()
def eval(self, competition: str, workspace: FBWorkspace) -> str:
eval_path = Path(f"{DS_RD_SETTING.local_data_path}/{DS_RD_SETTING.eval_sub_dir}/{competition}")
if not eval_path.exists():
err_msg = f"No Test Eval provided due to: {eval_path} not found"
raise NoTestEvalError(err_msg)
workspace.inject_files(**{"grade.py": (eval_path / "grade.py").read_text()})
workspace.inject_files(**{"submission_test.csv": (eval_path / "submission_test.csv").read_text()})
workspace.execute(
env=self.env,
entry=f"python grade.py {competition} | tee mle_score.txt",
)
workspace.inject_files(**{file: workspace.DEL_KEY for file in ["grade.py", "submission_test.csv"]})
workspace.execute(env=self.env, entry="chmod 777 mle_score.txt")
return (workspace.workspace_path / "mle_score.txt").read_text()
def valid(self, competition: str, workspace: FBWorkspace) -> tuple[str, int]:
eval_path = Path(f"{DS_RD_SETTING.local_data_path}/{DS_RD_SETTING.eval_sub_dir}/{competition}")
if not eval_path.exists():
err_msg = f"No Test Eval provided due to: {eval_path} not found"
raise NoTestEvalError(err_msg)
workspace.inject_files(**{"submission_format_valid.py": (eval_path / "valid.py").read_text()})
workspace.inject_files(**{"submission_test.csv": (eval_path / "submission_test.csv").read_text()})
submission_check_out, submission_ret_code = workspace.execute_ret_code(
env=self.env,
entry=f"python submission_format_valid.py {competition}",
)
workspace.inject_files(
**{file: workspace.DEL_KEY for file in ["submission_format_valid.py", "submission_test.csv"]}
)
workspace.inject_files(**{"test/mle_submission_format_test.output": submission_check_out})
return submission_check_out, submission_ret_code
def enabled(self, competition) -> bool:
return Path(
f"{DS_RD_SETTING.local_data_path}/{DS_RD_SETTING.eval_sub_dir}/{competition}/submission_test.csv"
).exists()
class MLETestEval(TestEvalBase):
"""Evaluation for test data for MLE-Bench competition"""
def __init__(self) -> None:
super().__init__()
self.env = get_ds_env(
conf_type="mlebench", extra_volumes={f"{DS_RD_SETTING.local_data_path}/zip_files": "/mle/data"}
)
self.env.prepare()
def eval(self, competition: str, workspace: FBWorkspace) -> str:
workspace.execute(
env=self.env,
entry=f"mlebench grade-sample submission.csv {competition} --data-dir /mle/data | tee mle_score.txt",
)
workspace.execute(env=self.env, entry="chmod 777 mle_score.txt")
return (workspace.workspace_path / "mle_score.txt").read_text()
def valid(self, competition: str, workspace: FBWorkspace) -> tuple[str, int]:
mle_check_code = (
(Path(__file__).absolute().resolve().parent / "eval_tests" / "mle_submission_format_test.txt")
.read_text()
.replace("<competition_id>", self.scen.competition)
)
workspace.inject_files(**{"test/mle_submission_format_test.py": mle_check_code})
submission_check_out, submission_ret_code = workspace.execute_ret_code(
env=mde, entry="python test/mle_submission_format_test.py"
)
workspace.inject_files(**{"test/mle_submission_format_test.output": submission_check_out})
return submission_check_out, submission_ret_code
def enabled(self, competition) -> bool:
return True
def get_test_eval() -> TestEvalBase:
"""Get the test evaluation instance"""
if DS_RD_SETTING.if_using_mle_data:
return MLETestEval()
return TestEval()
rdagent/utils/env.py
+7
View File
@@ -205,6 +205,13 @@ class Env(Generic[ASpecificEnvConf]):
if entry is None:
entry = self.conf.default_entry
if "|" in entry:
logger.warning(
"You are using a command with a shell pipeline (i.e., '|'). "
"The exit code ($exit_code) will reflect the result of "
"the last command in the pipeline.",
)
entry_add_timeout = (
f"/bin/sh -c 'timeout {self.conf.running_timeout_period} {entry}; "
+ "entry_exit_code=$?; "