gavindiaz/NexQuant
1
0
Fork 0
mirror of https://github.com/NicolasBohn/NexQuant.git synced 2026-07-29 08:27:43 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: make spec optional (#719)

Browse Source 
* feat: Add spec_enabled configuration for data science settings

* make spec alternative

* change spec logic in exp_gen

* remove some general texts

* align

---------

Co-authored-by: Young <afe.young@gmail.com>
Co-authored-by: yuanteli <1957922024@qq.com>
This commit is contained in:
XianBW
2025-03-27 17:17:41 +08:00
committed by GitHub
parent 031d264957
commit 7fc9e18a71
20 changed files with 334 additions and 92 deletions
Download Patch File Download Diff File Expand all files Collapse all files
rdagent/app/data_science/conf.py
+10
View File
@@ -19,5 +19,15 @@ class DataScienceBasePropSetting(KaggleBasePropSetting):
full_timeout: int = 3600
"""The timeout limit for running on full data"""
### specific feature
#### enable specification
spec_enabled: bool = True
# - [ ] rdagent/components/coder/data_science/raw_data_loader/__init__.py: make spec implementation optional
# - [ ] move spec responsibility into rdagent/scenarios/data_science/share.yaml
# - [ ] make all spec.md optional; but replace it with the test & responsibility. "spec/.*\.md".
# - [ ] replace yaml render with target test. "spec > .yaml data_science !out_spec !task_spec model_spec"
# - [ ] At the head of all tests, emphasis the function to be tested.
DS_RD_SETTING = DataScienceBasePropSetting()
rdagent/components/coder/data_science/ensemble/__init__.py
+23 -1
View File
@@ -12,8 +12,12 @@ File structure
"""
import json
from pathlib import Path
from typing import Dict
from jinja2 import Environment, StrictUndefined
from rdagent.app.data_science.conf import DS_RD_SETTING
from rdagent.components.coder.CoSTEER import CoSTEER
from rdagent.components.coder.CoSTEER.evaluators import (
CoSTEERMultiEvaluator,
@@ -35,6 +39,8 @@ from rdagent.oai.llm_utils import APIBackend
from rdagent.utils.agent.ret import PythonAgentOut
from rdagent.utils.agent.tpl import T
DIRNAME = Path(__file__).absolute().resolve().parent
class EnsembleMultiProcessEvolvingStrategy(MultiProcessEvolvingStrategy):
def implement_one_task(
@@ -79,8 +85,24 @@ class EnsembleMultiProcessEvolvingStrategy(MultiProcessEvolvingStrategy):
all_code=workspace.all_codes,
out_spec=PythonAgentOut.get_spec(),
)
if DS_RD_SETTING.spec_enabled:
code_spec = workspace.file_dict["spec/ensemble.md"]
else:
test_code = (
Environment(undefined=StrictUndefined)
.from_string((DIRNAME / "eval_tests" / "ensemble_test.txt").read_text())
.render(
model_names=[
fn[:-3] for fn in workspace.file_dict.keys() if fn.startswith("model_") and "test" not in fn
]
)
)
code_spec = T("scenarios.data_science.share:component_spec.general").r(
spec=T("scenarios.data_science.share:component_spec.Ensemble").r(), test_code=test_code
)
user_prompt = T(".prompts:ensemble_coder.user").r(
ensemble_spec=workspace.file_dict["spec/ensemble.md"],
code_spec=code_spec,
latest_code=workspace.file_dict.get("ensemble.py"),
latest_code_feedback=prev_task_feedback,
)
rdagent/components/coder/data_science/ensemble/eval_tests/ensemble_test.txt
+4 -2
View File
@@ -1,9 +1,11 @@
"""
A qualified ensemble implementation should:
- Successfully run
Tests for `ensemble_workflow` in ensemble.py
A qualified ensemble_workflow implementation should:
- Return predictions
- Have correct shapes for inputs and outputs
- Use validation data appropriately
- Generate a scores.csv file
"""
import numpy as np
rdagent/components/coder/data_science/ensemble/prompts.yaml
+2 -2
View File
@@ -51,8 +51,8 @@ ensemble_coder:
{% endif %}
user: |-
--------- Ensemble Specification ---------
{{ ensemble_spec }}
--------- Code Specification ---------
{{ code_spec }}
{% if latest_code %}
--------- Former code ---------
rdagent/components/coder/data_science/feature/__init__.py
+13 -1
View File
@@ -1,6 +1,8 @@
import json
from pathlib import Path
from typing import Dict
from rdagent.app.data_science.conf import DS_RD_SETTING
from rdagent.components.coder.CoSTEER import CoSTEER
from rdagent.components.coder.CoSTEER.evaluators import (
CoSTEERMultiEvaluator,
@@ -22,6 +24,8 @@ from rdagent.oai.llm_utils import APIBackend
from rdagent.utils.agent.ret import PythonAgentOut
from rdagent.utils.agent.tpl import T
DIRNAME = Path(__file__).absolute().resolve().parent
class FeatureMultiProcessEvolvingStrategy(MultiProcessEvolvingStrategy):
def implement_one_task(
@@ -64,8 +68,16 @@ class FeatureMultiProcessEvolvingStrategy(MultiProcessEvolvingStrategy):
queried_former_failed_knowledge=queried_former_failed_knowledge[0],
out_spec=PythonAgentOut.get_spec(),
)
code_spec = (
workspace.file_dict["spec/feature.md"]
if DS_RD_SETTING.spec_enabled
else T("scenarios.data_science.share:component_spec.general").r(
spec=T("scenarios.data_science.share:component_spec.FeatureEng").r(),
test_code=(DIRNAME / "eval_tests" / "feature_test.txt").read_text(),
)
)
user_prompt = T(".prompts:feature_coder.user").r(
feature_spec=workspace.file_dict["spec/feature.md"],
code_spec=code_spec,
latest_code=workspace.file_dict.get("feature.py"),
latest_code_feedback=prev_task_feedback,
)
rdagent/components/coder/data_science/feature/eval_tests/feature_test.txt
+1 -6
View File
@@ -1,10 +1,5 @@
"""
A qualified data loader should support following features
- successfully run
- len(test) == len(test_ids) == submission length
- len(train) == len(y)
Please make sure the stdout is rich enough to support informative feedback
Tests for `feat_eng` in feature.py
"""
import pickle
rdagent/components/coder/data_science/feature/prompts.yaml
+2 -2
View File
@@ -55,8 +55,8 @@ feature_coder:
{% endif %}
user: |-
--------- Feature Processing Specification ---------
{{ feature_spec }}
--------- Code Specification ---------
{{ code_spec }}
{% if latest_code %}
--------- Former code ---------
rdagent/components/coder/data_science/model/__init__.py
+13 -1
View File
@@ -1,5 +1,7 @@
from pathlib import Path
from typing import Dict
from rdagent.app.data_science.conf import DS_RD_SETTING
from rdagent.components.coder.CoSTEER import CoSTEER
from rdagent.components.coder.CoSTEER.evaluators import (
CoSTEERMultiEvaluator,
@@ -23,6 +25,8 @@ from rdagent.oai.llm_utils import APIBackend
from rdagent.utils.agent.ret import PythonBatchEditOut
from rdagent.utils.agent.tpl import T
DIRNAME = Path(__file__).absolute().resolve().parent
class ModelMultiProcessEvolvingStrategy(MultiProcessEvolvingStrategy):
def implement_one_task(
@@ -71,8 +75,16 @@ class ModelMultiProcessEvolvingStrategy(MultiProcessEvolvingStrategy):
# latest_code=workspace.file_dict.get(f"{target_task.name}.py", None),
# )
# We want to use a simpler way to
code_spec = (
workspace.file_dict["spec/model.md"]
if DS_RD_SETTING.spec_enabled
else T("scenarios.data_science.share:component_spec.general").r(
spec=T("scenarios.data_science.share:component_spec.Model").r(),
test_code=(DIRNAME / "eval_tests" / "model_test.txt").read_text().replace("model01", target_task.name),
)
)
user_prompt = T(".prompts:model_coder.user_general").r(
model_spec=workspace.file_dict["spec/model.md"],
code_spec=code_spec,
latest_model_code=workspace.get_codes(
r"^model_(?!test)\w+\.py$"
), # TODO: If we have high failure rate here, we should clean this step with less information.
rdagent/components/coder/data_science/model/eval.py
-2
View File
@@ -98,8 +98,6 @@ class ModelGeneralCaseSpecEvaluator(CoSTEEREvaluator):
task_desc=target_task.get_task_information(),
test_code=test_code,
code=implementation.file_dict[f"{target_task.name}.py"],
scenario=self.scen.get_scenario_all_desc(),
spec=implementation.file_dict["spec/model.md"],
workflow_stdout=workflow_stdout,
workflow_code=implementation.all_codes,
)
rdagent/components/coder/data_science/model/eval_tests/model_test.txt
+3
View File
@@ -1,3 +1,6 @@
"""
Tests for `model_workflow` in model01.py
"""
import time
from feature import feat_eng
rdagent/components/coder/data_science/model/prompts.yaml
+2 -2
View File
@@ -53,8 +53,8 @@ model_coder:
{% endif %}
user_general: |-
--------- Model Building Specification ---------
{{ model_spec }}
--------- Code Specification ---------
{{ code_spec }}
--------- Former model code ---------
{% if latest_model_code|length == 0 %}
rdagent/components/coder/data_science/raw_data_loader/__init__.py
+59 -39
View File
@@ -24,6 +24,7 @@ File structure
import json
import re
from pathlib import Path
from typing import Dict
from rdagent.app.data_science.conf import DS_RD_SETTING
@@ -53,6 +54,8 @@ from rdagent.oai.llm_utils import APIBackend
from rdagent.utils.agent.ret import PythonAgentOut
from rdagent.utils.agent.tpl import T
DIRNAME = Path(__file__).absolute().resolve().parent
class DataLoaderMultiProcessEvolvingStrategy(MultiProcessEvolvingStrategy):
def implement_one_task(
@@ -90,38 +93,41 @@ class DataLoaderMultiProcessEvolvingStrategy(MultiProcessEvolvingStrategy):
# 1. specifications
# TODO: We may move spec into a separated COSTEER task
if "spec/data_loader.md" not in workspace.file_dict: # Only generate the spec once
system_prompt = T(".prompts:spec.system").r(
runtime_environment=runtime_environment,
task_desc=data_loader_task_info,
competition_info=competition_info,
folder_spec=data_folder_info,
)
data_loader_prompt = T(".prompts:spec.user.data_loader").r(
latest_spec=workspace.file_dict.get("spec/data_loader.md")
)
feature_prompt = T(".prompts:spec.user.feature").r(latest_spec=workspace.file_dict.get("spec/feature.md"))
model_prompt = T(".prompts:spec.user.model").r(latest_spec=workspace.file_dict.get("spec/model.md"))
ensemble_prompt = T(".prompts:spec.user.ensemble").r(
latest_spec=workspace.file_dict.get("spec/ensemble.md")
)
workflow_prompt = T(".prompts:spec.user.workflow").r(
latest_spec=workspace.file_dict.get("spec/workflow.md")
)
if DS_RD_SETTING.spec_enabled:
if "spec/data_loader.md" not in workspace.file_dict: # Only generate the spec once
system_prompt = T(".prompts:spec.system").r(
runtime_environment=runtime_environment,
task_desc=data_loader_task_info,
competition_info=competition_info,
folder_spec=data_folder_info,
)
data_loader_prompt = T(".prompts:spec.user.data_loader").r(
latest_spec=workspace.file_dict.get("spec/data_loader.md")
)
feature_prompt = T(".prompts:spec.user.feature").r(
latest_spec=workspace.file_dict.get("spec/feature.md")
)
model_prompt = T(".prompts:spec.user.model").r(latest_spec=workspace.file_dict.get("spec/model.md"))
ensemble_prompt = T(".prompts:spec.user.ensemble").r(
latest_spec=workspace.file_dict.get("spec/ensemble.md")
)
workflow_prompt = T(".prompts:spec.user.workflow").r(
latest_spec=workspace.file_dict.get("spec/workflow.md")
)
spec_session = APIBackend().build_chat_session(session_system_prompt=system_prompt)
spec_session = APIBackend().build_chat_session(session_system_prompt=system_prompt)
data_loader_spec = spec_session.build_chat_completion(user_prompt=data_loader_prompt)
feature_spec = spec_session.build_chat_completion(user_prompt=feature_prompt)
model_spec = spec_session.build_chat_completion(user_prompt=model_prompt)
ensemble_spec = spec_session.build_chat_completion(user_prompt=ensemble_prompt)
workflow_spec = spec_session.build_chat_completion(user_prompt=workflow_prompt)
else:
data_loader_spec = workspace.file_dict["spec/data_loader.md"]
feature_spec = workspace.file_dict["spec/feature.md"]
model_spec = workspace.file_dict["spec/model.md"]
ensemble_spec = workspace.file_dict["spec/ensemble.md"]
workflow_spec = workspace.file_dict["spec/workflow.md"]
data_loader_spec = spec_session.build_chat_completion(user_prompt=data_loader_prompt)
feature_spec = spec_session.build_chat_completion(user_prompt=feature_prompt)
model_spec = spec_session.build_chat_completion(user_prompt=model_prompt)
ensemble_spec = spec_session.build_chat_completion(user_prompt=ensemble_prompt)
workflow_spec = spec_session.build_chat_completion(user_prompt=workflow_prompt)
else:
data_loader_spec = workspace.file_dict["spec/data_loader.md"]
feature_spec = workspace.file_dict["spec/feature.md"]
model_spec = workspace.file_dict["spec/model.md"]
ensemble_spec = workspace.file_dict["spec/ensemble.md"]
workflow_spec = workspace.file_dict["spec/workflow.md"]
# 2. code
system_prompt = T(".prompts:data_loader_coder.system").r(
@@ -130,9 +136,17 @@ class DataLoaderMultiProcessEvolvingStrategy(MultiProcessEvolvingStrategy):
queried_former_failed_knowledge=queried_former_failed_knowledge[0],
out_spec=PythonAgentOut.get_spec(),
)
code_spec = (
data_loader_spec
if DS_RD_SETTING.spec_enabled
else T("scenarios.data_science.share:component_spec.general").r(
spec=T("scenarios.data_science.share:component_spec.DataLoadSpec").r(),
test_code=(DIRNAME / "eval_tests" / "data_loader_test.txt").read_text(),
)
)
user_prompt = T(".prompts:data_loader_coder.user").r(
competition_info=competition_info,
data_loader_spec=data_loader_spec,
code_spec=code_spec,
folder_spec=data_folder_info,
latest_code=workspace.file_dict.get("load_data.py"),
latest_code_feedback=prev_task_feedback,
@@ -152,14 +166,20 @@ class DataLoaderMultiProcessEvolvingStrategy(MultiProcessEvolvingStrategy):
else:
raise CoderError("Failed to generate a new data loader code.")
return {
"spec/data_loader.md": data_loader_spec,
"spec/feature.md": feature_spec,
"spec/model.md": model_spec,
"spec/ensemble.md": ensemble_spec,
"spec/workflow.md": workflow_spec,
"load_data.py": data_loader_code,
}
return (
{
"spec/data_loader.md": data_loader_spec,
"spec/feature.md": feature_spec,
"spec/model.md": model_spec,
"spec/ensemble.md": ensemble_spec,
"spec/workflow.md": workflow_spec,
"load_data.py": data_loader_code,
}
if DS_RD_SETTING.spec_enabled
else {
"load_data.py": data_loader_code,
}
)
def assign_code_list_to_evo(self, code_list: list[dict[str, str]], evo):
"""
rdagent/components/coder/data_science/raw_data_loader/eval_tests/data_loader_test.txt
+1 -6
View File
@@ -1,10 +1,5 @@
"""
A qualified data loader should support following features
- successfully run
- len(test) == len(test_ids) == submission length
- len(train) == len(y)
Please make sure the stdout is rich enough to support informative feedback
Tests for `load_data` in load_data.py
"""
import pickle
rdagent/components/coder/data_science/raw_data_loader/prompts.yaml
+20 -19
View File
@@ -54,8 +54,9 @@ spec:
2. Precautions for Data Loading and Preprocessing:
- File Handling:
- Ensure proper file encoding and delimiters.
- Handle file encoding and delimiters appropriately.
- Combine or process multiple files if necessary.
- Avoid using the sample submission file to infer test indices. If a dedicated test index file is available, use that. If not, use the order in the test file as the test index.
- Data Preprocessing:
- Convert data types correctly (e.g., numeric, categorical, date parsing).
- Handle missing values appropriately.
@@ -64,14 +65,16 @@ spec:
- Apply competition-specific preprocessing steps as needed (e.g., text tokenization, image resizing).
3. Code Standards:
- Avoid using progress bars (e.g., `tqdm`) in the implementation.
- DO NOT use progress bars (e.g., `tqdm`).
- DO NOT use the sample submission file to extract test index information.
- DO NOT exclude features inadvertently during this process.
4. Notes:
- Update `DT` (data type) based on the specific competition dataset. This can include `pd.DataFrame`, `np.array`, `torch.Tensor`, etc.
- Never use sample submission as the test index, as it may not be the same as the test data. Use the test index file or test data source to get the test index.
- Only set the DT of variables without inferring the shape of these variables since you don't know the shape of the data.
5. Exploratory Data Analysis (EDA) part(Required):
5. Exploratory Data Analysis (EDA) [Required]:
- Before returning the data, you should always add an EDA part describing the data to help the following steps understand the data better.
- The EDA part should be drafted in plain text with certain format schema with no more than ten thousand characters.
- An evaluation agent will help to check whether the EDA part is added correctly.
@@ -104,18 +107,18 @@ spec:
- Inferred data shape to each input and output data variables. To uncertain dimension, use -1.
2. Precautions for Feature Engineering:
- Well handle the shape of the data
- Well handle the shape of the data:
- The sample size of the train data and the test data should be the same in all scenarios.
- To most of the scenario, the input shape and the output shape should be exactly the same.
- To some tabular data, you may add or remove some columns so your inferred column number may be unsure.
- Integration with Model Pipeline
- If feature engineering is strictly part of the model pipeline, state explicitly that it will be handled at the model stage.
- If integrated here, ensure this function applies all required transformations while avoiding data leakage.
- Integration with the Model Pipeline:
- If feature engineering is deferred to the model pipeline for better overall performance, state explicitly that it will be handled at the model stage.
- Otherwise, ensure this function applies all required transformations while avoiding data leakage.
- General Considerations:
- Ensure scalability for large datasets.
- Handle missing values and outliers appropriately (e.g., impute, remove, or replace).
- Ensure consistency between feature data types and transformations.
- Avoid data leakage: Only use features derived from training data, excluding information from test or validation sets.
- Prevent data leakage: Do not use information derived from the test set when transforming training data.
- Domain-Specific Features:
- Apply logic for competition-specific features (e.g., text vectorization, image augmentations, categorical encoding).
@@ -124,9 +127,7 @@ spec:
4. Notes:
- Align `DT` (data type) definitions with those in the Data Loader specification.
- Extend or adjust domain-specific transformations based on competition requirements.
- The device has GPU support, so you can use it for feature engineering if necessary to accelerate the process.
- Multi processing or parallel processing can be used to speed up the feature engineering process.
- GPU and multiprocessing are available and are encouraged to use for accelerating transformations.
- Only set the DT of variables without inferring the shape of these variables since you don't know the shape of the data.
{% if latest_spec %}
@@ -172,7 +173,7 @@ spec:
4. Notes:
- Align `DT` (data type) with the definitions used in Feature Engineering specifications.
- The device has GPU support, so you can use it for training if necessary to accelerate the process.
- The device has GPU support, so you are encouraged to use it for training if necessary to accelerate the process.
{% if latest_spec %}
5. Former Specification:
@@ -200,7 +201,7 @@ spec:
- Inferred data shape to each input and output data variables. To uncertain dimension, use -1.
2. Precautions:
- Validation of Inputs:
- Input Validation:
- Ensure all predictions in `test_preds_dict` and `val_preds_dict` have consistent shapes and dimensions.
- Verify that `val_label` is provided and matches the length of `val_preds_dict` predictions.
- Handle empty or invalid inputs gracefully with appropriate error messages.
@@ -222,7 +223,7 @@ spec:
scores_df = pd.DataFrame(scores.items(), columns=["Model", <metric_name>])
scores_df.to_csv("scores.csv", index=False)
```
- If there is only one model, still compute the ensemble score and store it under "ensemble".
- Even if only one model is present, compute the ensemble score and store it under `"ensemble"`.
3. Code Standards:
- Do not use progress bars (e.g., tqdm) in the code.
@@ -263,7 +264,7 @@ spec:
3. Dataset Splitting
- The dataset returned by `load_data` is not pre-split. After calling `feat_eng`, split the data into training and test sets.
- [Notice] If feasible, apply cross-validation (e.g. KFold) on the training set (`X_transformed`, `y_transformed`) to ensure a reliable assessment of model performance.
- [Notice] If feasible, apply cross-validation on the training set (`X_transformed`, `y_transformed`) to ensure a reliable assessment of model performance.
- Keep the test set (`X_test_transformed`) unchanged, as it is only used for generating the final predictions.
- Pseudocode logic for reference:
```
@@ -385,8 +386,8 @@ data_loader_coder:
--------- Competition Information ---------
{{ competition_info }}
--------- Data Loader Specification ---------
{{ data_loader_spec }}
--------- Code Specification ---------
{{ code_spec }}
--------- Data Folder Description (All path are relative to the data folder) ---------
{{ folder_spec }}
@@ -401,7 +402,7 @@ data_loader_coder:
The former code contains errors. You should correct the code based on the provided information, ensuring you do not repeat the same mistakes.
{% endif %}
You should strictly follow the function interface specifications provided by the specification to implement the function.
You should strictly follow the code specifications provided by the specification to implement the function.
data_loader_eval:
@@ -453,7 +454,7 @@ data_loader_eval:
- The number of unique values in each column.
- The distribution of the target variable.
- Any other information that you think is important for the following steps.
You will be given the EDA output, your job is to check whether the output contains the required and sufficient information. If no EDA output is provided, you should consider it as a failure. Put this evaluation result in the return_checking part.
You will be given the EDA output, your job is to check whether the output contains the required and sufficient information. If no EDA output is provided, you should consider it as a failure. Put this evaluation result in the return_checking part.
Your response must follow this structured JSON format:
```json
rdagent/components/coder/data_science/workflow/__init__.py
+6 -1
View File
@@ -1,6 +1,7 @@
import json
from typing import Dict
from rdagent.app.data_science.conf import DS_RD_SETTING
from rdagent.components.coder.CoSTEER import CoSTEER
from rdagent.components.coder.CoSTEER.evaluators import (
CoSTEERMultiEvaluator,
@@ -69,7 +70,11 @@ class WorkflowMultiProcessEvolvingStrategy(MultiProcessEvolvingStrategy):
model_codes=workspace.get_codes(r"^model_(?!test)\w+\.py$"),
ensemble_code=workspace.file_dict["ensemble.py"],
latest_code=workspace.file_dict.get("main.py"),
workflow_spec=workspace.file_dict["spec/workflow.md"],
code_spec=(
workspace.file_dict["spec/workflow.md"]
if DS_RD_SETTING.spec_enabled
else T("scenarios.data_science.share:component_spec.Workflow").r()
),
latest_code_feedback=prev_task_feedback,
)
rdagent/components/coder/data_science/workflow/eval.py
+5 -1
View File
@@ -115,7 +115,11 @@ class WorkflowGeneralCaseSpecEvaluator(CoSTEEREvaluator):
system_prompt = T(".prompts:workflow_eval.system").r(
scenario=self.scen.get_scenario_all_desc(),
task_desc=target_task.get_task_information(),
spec=implementation.file_dict["spec/workflow.md"],
spec=(
implementation.file_dict["spec/workflow.md"]
if DS_RD_SETTING.spec_enabled
else T("scenarios.data_science.share:component_spec.Workflow").r()
),
)
user_prompt = T(".prompts:workflow_eval.user").r(
stdout=stdout.strip(),
rdagent/components/coder/data_science/workflow/prompts.yaml
+4 -3
View File
@@ -52,8 +52,8 @@ workflow_coder:
{% endif %}
user: |-
--------- Workflow Specification ---------
{{ workflow_spec }}
--------- Code Specification ---------
{{ code_spec }}
--------- load data code ---------
file: load_data.py
@@ -96,9 +96,10 @@ workflow_eval:
The user provides workflow information and its components.
The details on how to structure the workflow are given in the specification file:
```python
```markdown
{{ spec }}
```
This workflow integrates multiple stages, including:
- Data loading
- Feature engineering
rdagent/scenarios/data_science/proposal/exp_gen.py
+11 -2
View File
@@ -3,6 +3,7 @@ from typing import Dict, Literal
import pandas as pd
from rdagent.app.data_science.conf import DS_RD_SETTING
from rdagent.components.coder.data_science.ensemble.exp import EnsembleTask
from rdagent.components.coder.data_science.feature.exp import FeatureTask
from rdagent.components.coder.data_science.model.exp import ModelTask
@@ -242,10 +243,14 @@ class DSExpGen(ExpGen):
else:
break
if DS_RD_SETTING.spec_enabled:
spec = last_successful_exp.experiment_workspace.file_dict[spec_file] if spec_file else None
else:
spec = T(f"scenarios.data_science.share:component_spec.{component}").r()
resp_dict = self._init_task_gen(
targets=component,
scenario_desc=scenario_desc,
spec=last_successful_exp.experiment_workspace.file_dict[spec_file] if spec_file else None,
spec=spec,
task_output_format=T(f".prompts:output_format.{component_prompt_key or component.lower()}").r(),
former_task=former_tasks_desc if former_tasks_desc else None,
)
@@ -384,13 +389,17 @@ class DSExpGen(ExpGen):
component_info = COMPONENT_TASK_MAPPING.get(component)
if component_info:
if DS_RD_SETTING.spec_enabled:
task_spec = sota_exp.experiment_workspace.file_dict[component_info["spec_file"]]
else:
task_spec = T(f"scenarios.data_science.share:component_spec.{component}").r()
system_prompt = T(".prompts:direct_exp_gen.system").r(
targets=component_info["target_name"],
component=component,
scenario=scenario_desc,
hypothesis_specification=T(".prompts:hypothesis_specification").r(),
hypothesis_output_format=T(".prompts:output_format.hypothesis").r(),
task_specification=sota_exp.experiment_workspace.file_dict[component_info["spec_file"]],
task_specification=task_spec,
task_output_format=component_info["task_output_format"],
workflow_check=(not component == "Workflow"),
)
rdagent/scenarios/data_science/proposal/prompts.yaml
+1 -1
View File
@@ -188,7 +188,7 @@ direct_exp_gen:
## Task Specification
The scope of the {{ targets }} can be described by a interface specification as follows:
```Python
```markdown
{{ task_specification }}
```
rdagent/scenarios/data_science/share.yaml
+154 -1
View File
@@ -72,4 +72,157 @@ component_description:
Ensemble: |-
Combines predictions from multiple models using ensemble strategies, evaluates their performance, and generates the final test predictions.
Workflow: |-
Integrates all pipeline components, from data loading to ensemble prediction, ensuring efficient execution and correct output formatting.
Integrates all pipeline components, from data loading to ensemble prediction, ensuring efficient execution and correct output formatting.
component_spec:
general: |-
{{ spec }}
Your code will be tested by the code below. You must ensure your implementation passes the test code:
```python
{{ test_code }}
```
DataLoadSpec: |-
1. File Handling:
- Handle file encoding and delimiters appropriately.
- Combine or process multiple files if necessary.
- Avoid using the sample submission file to infer test indices. If a dedicated test index file is available, use that. If not, use the order in the test file as the test index.
2. Data Preprocessing:
- Convert data types correctly (e.g., numeric, categorical, date parsing).
- Handle missing values appropriately.
- Optimize memory usage for large datasets using techniques like downcasting or reading data in chunks if necessary.
3. Code Standards:
- DO NOT use progress bars (e.g., `tqdm`).
- DO NOT use the sample submission file to extract test index information.
- DO NOT exclude features inadvertently during this process.
4. Exploratory Data Analysis (EDA) [Required]:
- Before returning the data, you should always add an EDA part describing the data to help the following steps understand the data better.
- The EDA part should be drafted in plain text with certain format schema with no more than ten thousand characters.
- An evaluation agent will help to check whether the EDA part is added correctly.
FeatureEng: |-
1. Well handle the shape of the data
- The sample size of the train data and the test data should be the same in all scenarios.
- To most of the scenario, the input shape and the output shape should be exactly the same.
- To some tabular data, you may add or remove some columns so your inferred column number may be unsure.
2. Integration with the Model Pipeline:
- If feature engineering is deferred to the model pipeline for better overall performance, state explicitly that it will be handled at the model stage.
- Otherwise, ensure this function applies all required transformations while avoiding data leakage.
3. General Considerations:
- Ensure scalability for large datasets.
- Handle missing values and outliers appropriately (e.g., impute, remove, or replace).
- Ensure consistency between feature data types and transformations.
- Prevent data leakage: Do not use information derived from the test set when transforming training data.
4. Code Standards:
- Avoid using progress bars (e.g., `tqdm`) in the implementation.
5. Notes:
- GPU and multiprocessing are available and are encouraged to use for accelerating transformations.
- Feature engineering should be executed **once** and reused across all models to ensure consistency: `X_transformed, y_transformed, X_test_transformed = feat_eng(X, y, X_test)`
Model: |-
- Do not use progress bars (e.g., `tqdm`) in the implementation.
- The device has GPU support, so you are encouraged to use it for training if necessary to accelerate the process.
Ensemble: |-
1. Input Validation:
- Handle empty or invalid inputs gracefully with appropriate error messages.
2. Metric Calculation and Storage:
- Calculate the metric (mentioned in the evaluation section of the competition information) for each model and ensemble strategy on valid, and save the results in `scores.csv`, e.g.:
```python
scores = {}
for model_name, val_pred in val_preds_dict.items():
scores[model_name] = calculate_metric(val_label, val_pred)
...
some code about ensemble strategy
...
ensemble_val_pred = ...
ensemble_score = calculate_metric(val_label, ensemble_val_pred)
scores["ensemble"] = ensemble_score # Ensure "ensemble" is explicitly stored
scores_df = pd.DataFrame(scores.items(), columns=["Model", <metric_name>])
scores_df.to_csv("scores.csv", index=False)
```
- Even if only one model is present, compute the ensemble score and store it under `"ensemble"`.
3. Code Standards:
- Do not use progress bars (e.g., tqdm) in the code.
4. Notes:
- Ensure flexibility to handle multiple ensemble strategies based on competition requirements.
Workflow: |-
Your task is to implement the main workflow script (`main.py`) for a Kaggle-style machine learning competition project.
Follow the provided project structure and specifications to ensure consistency and maintainability:
1. Workflow Integration:
- Integrate the following components into the workflow:
- Data loading (`load_data.py`).
- Feature engineering (`feature.py`).
- Model workflow for training and testing (`model_*.py`).
- Ensemble workflow that combines results from the model workflow to obtain the final prediction (`ensemble.py`).
- Treat each component as a modular and callable Python function.
- The workflow script should be flexible enough to handle either a single model or multiple models, with filenames (model_*.py) that are not determined at the outset.
For multiple model selection, utilize Python code to identify eligible models based on filenames, for example:
```python
available_models = [f for f in os.listdir('.') if f.startswith('model_') and 'test' not in f]
```
2. Feature Engineering
- The feature engineering should be called only once. For example:
`X_transformed, y_transformed, X_test_transformed = feat_eng(X, y, X_test)`
- It should be called before dataset splitting.
3. Dataset Splitting
- The dataset returned by `load_data` is not pre-split. After calling `feat_eng`, split the data into training and test sets.
- [Notice] If feasible, apply cross-validation on the training set (`X_transformed`, `y_transformed`) to ensure a reliable assessment of model performance.
- Keep the test set (`X_test_transformed`) unchanged, as it is only used for generating the final predictions.
- Pseudocode logic for reference:
```
Set number of splits and initialize KFold cross-validator.
Create dictionaries for validation and test predictions.
For each model file:
Import the model dynamically.
Initialize arrays for out-of-fold (OOF) and test predictions.
For each fold in KFold:
Split data into training and validation sets.
Run model workflow to get validation and test predictions.
Validate shapes.
Store validation and test predictions.
Compute average test predictions across folds.
Save OOF and averaged test predictions.
Ensemble predictions from all models and print the final shape.
```
4. Submission File:
- Save the final predictions as `submission.csv`, ensuring the format matches the competition requirements (refer to `sample_submission` in the Folder Description for the correct structure).
- Present the required submission format explicitly and ensure the output adheres to it.
5. Code Standards:
- Do not use progress bars (e.g., tqdm) in the code.
6. Ensemble Strategy:
Consolidate all model outputs into a dictionary, where each key is the model's filename (excluding the .py extension) and its corresponding value is the model's output.
Sample code:
{% raw %}
{% for model_name in model_names %}
model_module = __import__(model_name.replace('.py', ''))
val_pred, test_pred, _ = model_module.model_workflow(
X=train_X,
y=train_y,
val_X=val_X,
val_y=val_y,
test_X=X_test_transformed
)
val_preds_dict[model_module.__name__] = val_pred
test_preds_dict[model_module.__name__] = test_pred
{% endfor %}
final_pred = ensemble_workflow(test_preds_dict, val_preds_dict, val_y)
{% endraw %}