mirror of
https://github.com/NicolasBohn/NexQuant.git
synced 2026-08-04 18:57:44 +00:00
feat: make spec optional (#719)
* feat: Add spec_enabled configuration for data science settings * make spec alternative * change spec logic in exp_gen * remove some general texts * align --------- Co-authored-by: Young <afe.young@gmail.com> Co-authored-by: yuanteli <1957922024@qq.com>
This commit is contained in:
XianBW
@@ -24,6 +24,7 @@ File structure
|
||||
|
||||
import json
|
||||
import re
|
||||
from pathlib import Path
|
||||
from typing import Dict
|
||||
|
||||
from rdagent.app.data_science.conf import DS_RD_SETTING
|
||||
@@ -53,6 +54,8 @@ from rdagent.oai.llm_utils import APIBackend
|
||||
from rdagent.utils.agent.ret import PythonAgentOut
|
||||
from rdagent.utils.agent.tpl import T
|
||||
|
||||
DIRNAME = Path(__file__).absolute().resolve().parent
|
||||
|
||||
|
||||
class DataLoaderMultiProcessEvolvingStrategy(MultiProcessEvolvingStrategy):
|
||||
def implement_one_task(
|
||||
@@ -90,38 +93,41 @@ class DataLoaderMultiProcessEvolvingStrategy(MultiProcessEvolvingStrategy):
|
||||
|
||||
# 1. specifications
|
||||
# TODO: We may move spec into a separated COSTEER task
|
||||
if "spec/data_loader.md" not in workspace.file_dict: # Only generate the spec once
|
||||
system_prompt = T(".prompts:spec.system").r(
|
||||
runtime_environment=runtime_environment,
|
||||
task_desc=data_loader_task_info,
|
||||
competition_info=competition_info,
|
||||
folder_spec=data_folder_info,
|
||||
)
|
||||
data_loader_prompt = T(".prompts:spec.user.data_loader").r(
|
||||
latest_spec=workspace.file_dict.get("spec/data_loader.md")
|
||||
)
|
||||
feature_prompt = T(".prompts:spec.user.feature").r(latest_spec=workspace.file_dict.get("spec/feature.md"))
|
||||
model_prompt = T(".prompts:spec.user.model").r(latest_spec=workspace.file_dict.get("spec/model.md"))
|
||||
ensemble_prompt = T(".prompts:spec.user.ensemble").r(
|
||||
latest_spec=workspace.file_dict.get("spec/ensemble.md")
|
||||
)
|
||||
workflow_prompt = T(".prompts:spec.user.workflow").r(
|
||||
latest_spec=workspace.file_dict.get("spec/workflow.md")
|
||||
)
|
||||
if DS_RD_SETTING.spec_enabled:
|
||||
if "spec/data_loader.md" not in workspace.file_dict: # Only generate the spec once
|
||||
system_prompt = T(".prompts:spec.system").r(
|
||||
runtime_environment=runtime_environment,
|
||||
task_desc=data_loader_task_info,
|
||||
competition_info=competition_info,
|
||||
folder_spec=data_folder_info,
|
||||
)
|
||||
data_loader_prompt = T(".prompts:spec.user.data_loader").r(
|
||||
latest_spec=workspace.file_dict.get("spec/data_loader.md")
|
||||
)
|
||||
feature_prompt = T(".prompts:spec.user.feature").r(
|
||||
latest_spec=workspace.file_dict.get("spec/feature.md")
|
||||
)
|
||||
model_prompt = T(".prompts:spec.user.model").r(latest_spec=workspace.file_dict.get("spec/model.md"))
|
||||
ensemble_prompt = T(".prompts:spec.user.ensemble").r(
|
||||
latest_spec=workspace.file_dict.get("spec/ensemble.md")
|
||||
)
|
||||
workflow_prompt = T(".prompts:spec.user.workflow").r(
|
||||
latest_spec=workspace.file_dict.get("spec/workflow.md")
|
||||
)
|
||||
|
||||
spec_session = APIBackend().build_chat_session(session_system_prompt=system_prompt)
|
||||
spec_session = APIBackend().build_chat_session(session_system_prompt=system_prompt)
|
||||
|
||||
data_loader_spec = spec_session.build_chat_completion(user_prompt=data_loader_prompt)
|
||||
feature_spec = spec_session.build_chat_completion(user_prompt=feature_prompt)
|
||||
model_spec = spec_session.build_chat_completion(user_prompt=model_prompt)
|
||||
ensemble_spec = spec_session.build_chat_completion(user_prompt=ensemble_prompt)
|
||||
workflow_spec = spec_session.build_chat_completion(user_prompt=workflow_prompt)
|
||||
else:
|
||||
data_loader_spec = workspace.file_dict["spec/data_loader.md"]
|
||||
feature_spec = workspace.file_dict["spec/feature.md"]
|
||||
model_spec = workspace.file_dict["spec/model.md"]
|
||||
ensemble_spec = workspace.file_dict["spec/ensemble.md"]
|
||||
workflow_spec = workspace.file_dict["spec/workflow.md"]
|
||||
data_loader_spec = spec_session.build_chat_completion(user_prompt=data_loader_prompt)
|
||||
feature_spec = spec_session.build_chat_completion(user_prompt=feature_prompt)
|
||||
model_spec = spec_session.build_chat_completion(user_prompt=model_prompt)
|
||||
ensemble_spec = spec_session.build_chat_completion(user_prompt=ensemble_prompt)
|
||||
workflow_spec = spec_session.build_chat_completion(user_prompt=workflow_prompt)
|
||||
else:
|
||||
data_loader_spec = workspace.file_dict["spec/data_loader.md"]
|
||||
feature_spec = workspace.file_dict["spec/feature.md"]
|
||||
model_spec = workspace.file_dict["spec/model.md"]
|
||||
ensemble_spec = workspace.file_dict["spec/ensemble.md"]
|
||||
workflow_spec = workspace.file_dict["spec/workflow.md"]
|
||||
|
||||
# 2. code
|
||||
system_prompt = T(".prompts:data_loader_coder.system").r(
|
||||
@@ -130,9 +136,17 @@ class DataLoaderMultiProcessEvolvingStrategy(MultiProcessEvolvingStrategy):
|
||||
queried_former_failed_knowledge=queried_former_failed_knowledge[0],
|
||||
out_spec=PythonAgentOut.get_spec(),
|
||||
)
|
||||
code_spec = (
|
||||
data_loader_spec
|
||||
if DS_RD_SETTING.spec_enabled
|
||||
else T("scenarios.data_science.share:component_spec.general").r(
|
||||
spec=T("scenarios.data_science.share:component_spec.DataLoadSpec").r(),
|
||||
test_code=(DIRNAME / "eval_tests" / "data_loader_test.txt").read_text(),
|
||||
)
|
||||
)
|
||||
user_prompt = T(".prompts:data_loader_coder.user").r(
|
||||
competition_info=competition_info,
|
||||
data_loader_spec=data_loader_spec,
|
||||
code_spec=code_spec,
|
||||
folder_spec=data_folder_info,
|
||||
latest_code=workspace.file_dict.get("load_data.py"),
|
||||
latest_code_feedback=prev_task_feedback,
|
||||
@@ -152,14 +166,20 @@ class DataLoaderMultiProcessEvolvingStrategy(MultiProcessEvolvingStrategy):
|
||||
else:
|
||||
raise CoderError("Failed to generate a new data loader code.")
|
||||
|
||||
return {
|
||||
"spec/data_loader.md": data_loader_spec,
|
||||
"spec/feature.md": feature_spec,
|
||||
"spec/model.md": model_spec,
|
||||
"spec/ensemble.md": ensemble_spec,
|
||||
"spec/workflow.md": workflow_spec,
|
||||
"load_data.py": data_loader_code,
|
||||
}
|
||||
return (
|
||||
{
|
||||
"spec/data_loader.md": data_loader_spec,
|
||||
"spec/feature.md": feature_spec,
|
||||
"spec/model.md": model_spec,
|
||||
"spec/ensemble.md": ensemble_spec,
|
||||
"spec/workflow.md": workflow_spec,
|
||||
"load_data.py": data_loader_code,
|
||||
}
|
||||
if DS_RD_SETTING.spec_enabled
|
||||
else {
|
||||
"load_data.py": data_loader_code,
|
||||
}
|
||||
)
|
||||
|
||||
def assign_code_list_to_evo(self, code_list: list[dict[str, str]], evo):
|
||||
"""
|
||||
|
||||
+1
-6
@@ -1,10 +1,5 @@
|
||||
"""
|
||||
A qualified data loader should support following features
|
||||
- successfully run
|
||||
- len(test) == len(test_ids) == submission length
|
||||
- len(train) == len(y)
|
||||
|
||||
Please make sure the stdout is rich enough to support informative feedback
|
||||
Tests for `load_data` in load_data.py
|
||||
"""
|
||||
|
||||
import pickle
|
||||
|
||||
@@ -54,8 +54,9 @@ spec:
|
||||
|
||||
2. Precautions for Data Loading and Preprocessing:
|
||||
- File Handling:
|
||||
- Ensure proper file encoding and delimiters.
|
||||
- Handle file encoding and delimiters appropriately.
|
||||
- Combine or process multiple files if necessary.
|
||||
- Avoid using the sample submission file to infer test indices. If a dedicated test index file is available, use that. If not, use the order in the test file as the test index.
|
||||
- Data Preprocessing:
|
||||
- Convert data types correctly (e.g., numeric, categorical, date parsing).
|
||||
- Handle missing values appropriately.
|
||||
@@ -64,14 +65,16 @@ spec:
|
||||
- Apply competition-specific preprocessing steps as needed (e.g., text tokenization, image resizing).
|
||||
|
||||
3. Code Standards:
|
||||
- Avoid using progress bars (e.g., `tqdm`) in the implementation.
|
||||
- DO NOT use progress bars (e.g., `tqdm`).
|
||||
- DO NOT use the sample submission file to extract test index information.
|
||||
- DO NOT exclude features inadvertently during this process.
|
||||
|
||||
4. Notes:
|
||||
- Update `DT` (data type) based on the specific competition dataset. This can include `pd.DataFrame`, `np.array`, `torch.Tensor`, etc.
|
||||
- Never use sample submission as the test index, as it may not be the same as the test data. Use the test index file or test data source to get the test index.
|
||||
- Only set the DT of variables without inferring the shape of these variables since you don't know the shape of the data.
|
||||
|
||||
5. Exploratory Data Analysis (EDA) part(Required):
|
||||
5. Exploratory Data Analysis (EDA) [Required]:
|
||||
- Before returning the data, you should always add an EDA part describing the data to help the following steps understand the data better.
|
||||
- The EDA part should be drafted in plain text with certain format schema with no more than ten thousand characters.
|
||||
- An evaluation agent will help to check whether the EDA part is added correctly.
|
||||
@@ -104,18 +107,18 @@ spec:
|
||||
- Inferred data shape to each input and output data variables. To uncertain dimension, use -1.
|
||||
|
||||
2. Precautions for Feature Engineering:
|
||||
- Well handle the shape of the data
|
||||
- Well handle the shape of the data:
|
||||
- The sample size of the train data and the test data should be the same in all scenarios.
|
||||
- To most of the scenario, the input shape and the output shape should be exactly the same.
|
||||
- To some tabular data, you may add or remove some columns so your inferred column number may be unsure.
|
||||
- Integration with Model Pipeline
|
||||
- If feature engineering is strictly part of the model pipeline, state explicitly that it will be handled at the model stage.
|
||||
- If integrated here, ensure this function applies all required transformations while avoiding data leakage.
|
||||
- Integration with the Model Pipeline:
|
||||
- If feature engineering is deferred to the model pipeline for better overall performance, state explicitly that it will be handled at the model stage.
|
||||
- Otherwise, ensure this function applies all required transformations while avoiding data leakage.
|
||||
- General Considerations:
|
||||
- Ensure scalability for large datasets.
|
||||
- Handle missing values and outliers appropriately (e.g., impute, remove, or replace).
|
||||
- Ensure consistency between feature data types and transformations.
|
||||
- Avoid data leakage: Only use features derived from training data, excluding information from test or validation sets.
|
||||
- Prevent data leakage: Do not use information derived from the test set when transforming training data.
|
||||
- Domain-Specific Features:
|
||||
- Apply logic for competition-specific features (e.g., text vectorization, image augmentations, categorical encoding).
|
||||
|
||||
@@ -124,9 +127,7 @@ spec:
|
||||
|
||||
4. Notes:
|
||||
- Align `DT` (data type) definitions with those in the Data Loader specification.
|
||||
- Extend or adjust domain-specific transformations based on competition requirements.
|
||||
- The device has GPU support, so you can use it for feature engineering if necessary to accelerate the process.
|
||||
- Multi processing or parallel processing can be used to speed up the feature engineering process.
|
||||
- GPU and multiprocessing are available and are encouraged to use for accelerating transformations.
|
||||
- Only set the DT of variables without inferring the shape of these variables since you don't know the shape of the data.
|
||||
|
||||
{% if latest_spec %}
|
||||
@@ -172,7 +173,7 @@ spec:
|
||||
|
||||
4. Notes:
|
||||
- Align `DT` (data type) with the definitions used in Feature Engineering specifications.
|
||||
- The device has GPU support, so you can use it for training if necessary to accelerate the process.
|
||||
- The device has GPU support, so you are encouraged to use it for training if necessary to accelerate the process.
|
||||
|
||||
{% if latest_spec %}
|
||||
5. Former Specification:
|
||||
@@ -200,7 +201,7 @@ spec:
|
||||
- Inferred data shape to each input and output data variables. To uncertain dimension, use -1.
|
||||
|
||||
2. Precautions:
|
||||
- Validation of Inputs:
|
||||
- Input Validation:
|
||||
- Ensure all predictions in `test_preds_dict` and `val_preds_dict` have consistent shapes and dimensions.
|
||||
- Verify that `val_label` is provided and matches the length of `val_preds_dict` predictions.
|
||||
- Handle empty or invalid inputs gracefully with appropriate error messages.
|
||||
@@ -222,7 +223,7 @@ spec:
|
||||
scores_df = pd.DataFrame(scores.items(), columns=["Model", <metric_name>])
|
||||
scores_df.to_csv("scores.csv", index=False)
|
||||
```
|
||||
- If there is only one model, still compute the ensemble score and store it under "ensemble".
|
||||
- Even if only one model is present, compute the ensemble score and store it under `"ensemble"`.
|
||||
|
||||
3. Code Standards:
|
||||
- Do not use progress bars (e.g., tqdm) in the code.
|
||||
@@ -263,7 +264,7 @@ spec:
|
||||
|
||||
3. Dataset Splitting
|
||||
- The dataset returned by `load_data` is not pre-split. After calling `feat_eng`, split the data into training and test sets.
|
||||
- [Notice] If feasible, apply cross-validation (e.g. KFold) on the training set (`X_transformed`, `y_transformed`) to ensure a reliable assessment of model performance.
|
||||
- [Notice] If feasible, apply cross-validation on the training set (`X_transformed`, `y_transformed`) to ensure a reliable assessment of model performance.
|
||||
- Keep the test set (`X_test_transformed`) unchanged, as it is only used for generating the final predictions.
|
||||
- Pseudocode logic for reference:
|
||||
```
|
||||
@@ -385,8 +386,8 @@ data_loader_coder:
|
||||
--------- Competition Information ---------
|
||||
{{ competition_info }}
|
||||
|
||||
--------- Data Loader Specification ---------
|
||||
{{ data_loader_spec }}
|
||||
--------- Code Specification ---------
|
||||
{{ code_spec }}
|
||||
|
||||
--------- Data Folder Description (All path are relative to the data folder) ---------
|
||||
{{ folder_spec }}
|
||||
@@ -401,7 +402,7 @@ data_loader_coder:
|
||||
The former code contains errors. You should correct the code based on the provided information, ensuring you do not repeat the same mistakes.
|
||||
{% endif %}
|
||||
|
||||
You should strictly follow the function interface specifications provided by the specification to implement the function.
|
||||
You should strictly follow the code specifications provided by the specification to implement the function.
|
||||
|
||||
|
||||
data_loader_eval:
|
||||
@@ -453,7 +454,7 @@ data_loader_eval:
|
||||
- The number of unique values in each column.
|
||||
- The distribution of the target variable.
|
||||
- Any other information that you think is important for the following steps.
|
||||
You will be given the EDA output, your job is to check whether the output contains the required and sufficient information. If no EDA output is provided, you should consider it as a failure. Put this evaluation result in the return_checking part.
|
||||
You will be given the EDA output, your job is to check whether the output contains the required and sufficient information. If no EDA output is provided, you should consider it as a failure. Put this evaluation result in the return_checking part.
|
||||
|
||||
Your response must follow this structured JSON format:
|
||||
```json
|
||||
|
||||
Reference in New Issue
Block a user