diff --git a/rdagent/app/data_science/conf.py b/rdagent/app/data_science/conf.py index 875b9812..967c3631 100644 --- a/rdagent/app/data_science/conf.py +++ b/rdagent/app/data_science/conf.py @@ -19,5 +19,15 @@ class DataScienceBasePropSetting(KaggleBasePropSetting): full_timeout: int = 3600 """The timeout limit for running on full data""" + ### specific feature + + #### enable specification + spec_enabled: bool = True + # - [ ] rdagent/components/coder/data_science/raw_data_loader/__init__.py: make spec implementation optional + # - [ ] move spec responsibility into rdagent/scenarios/data_science/share.yaml + # - [ ] make all spec.md optional; but replace it with the test & responsibility. "spec/.*\.md". + # - [ ] replace yaml render with target test. "spec > .yaml data_science !out_spec !task_spec model_spec" + # - [ ] At the head of all tests, emphasis the function to be tested. + DS_RD_SETTING = DataScienceBasePropSetting() diff --git a/rdagent/components/coder/data_science/ensemble/__init__.py b/rdagent/components/coder/data_science/ensemble/__init__.py index b4366bbc..aa94277e 100644 --- a/rdagent/components/coder/data_science/ensemble/__init__.py +++ b/rdagent/components/coder/data_science/ensemble/__init__.py @@ -12,8 +12,12 @@ File structure """ import json +from pathlib import Path from typing import Dict +from jinja2 import Environment, StrictUndefined + +from rdagent.app.data_science.conf import DS_RD_SETTING from rdagent.components.coder.CoSTEER import CoSTEER from rdagent.components.coder.CoSTEER.evaluators import ( CoSTEERMultiEvaluator, @@ -35,6 +39,8 @@ from rdagent.oai.llm_utils import APIBackend from rdagent.utils.agent.ret import PythonAgentOut from rdagent.utils.agent.tpl import T +DIRNAME = Path(__file__).absolute().resolve().parent + class EnsembleMultiProcessEvolvingStrategy(MultiProcessEvolvingStrategy): def implement_one_task( @@ -79,8 +85,24 @@ class EnsembleMultiProcessEvolvingStrategy(MultiProcessEvolvingStrategy): all_code=workspace.all_codes, out_spec=PythonAgentOut.get_spec(), ) + + if DS_RD_SETTING.spec_enabled: + code_spec = workspace.file_dict["spec/ensemble.md"] + else: + test_code = ( + Environment(undefined=StrictUndefined) + .from_string((DIRNAME / "eval_tests" / "ensemble_test.txt").read_text()) + .render( + model_names=[ + fn[:-3] for fn in workspace.file_dict.keys() if fn.startswith("model_") and "test" not in fn + ] + ) + ) + code_spec = T("scenarios.data_science.share:component_spec.general").r( + spec=T("scenarios.data_science.share:component_spec.Ensemble").r(), test_code=test_code + ) user_prompt = T(".prompts:ensemble_coder.user").r( - ensemble_spec=workspace.file_dict["spec/ensemble.md"], + code_spec=code_spec, latest_code=workspace.file_dict.get("ensemble.py"), latest_code_feedback=prev_task_feedback, ) diff --git a/rdagent/components/coder/data_science/ensemble/eval_tests/ensemble_test.txt b/rdagent/components/coder/data_science/ensemble/eval_tests/ensemble_test.txt index 66d715b4..291f7e90 100644 --- a/rdagent/components/coder/data_science/ensemble/eval_tests/ensemble_test.txt +++ b/rdagent/components/coder/data_science/ensemble/eval_tests/ensemble_test.txt @@ -1,9 +1,11 @@ """ -A qualified ensemble implementation should: -- Successfully run +Tests for `ensemble_workflow` in ensemble.py + +A qualified ensemble_workflow implementation should: - Return predictions - Have correct shapes for inputs and outputs - Use validation data appropriately +- Generate a scores.csv file """ import numpy as np diff --git a/rdagent/components/coder/data_science/ensemble/prompts.yaml b/rdagent/components/coder/data_science/ensemble/prompts.yaml index cc353e08..65b17084 100644 --- a/rdagent/components/coder/data_science/ensemble/prompts.yaml +++ b/rdagent/components/coder/data_science/ensemble/prompts.yaml @@ -51,8 +51,8 @@ ensemble_coder: {% endif %} user: |- - --------- Ensemble Specification --------- - {{ ensemble_spec }} + --------- Code Specification --------- + {{ code_spec }} {% if latest_code %} --------- Former code --------- diff --git a/rdagent/components/coder/data_science/feature/__init__.py b/rdagent/components/coder/data_science/feature/__init__.py index 97e4ef0c..83e21f7b 100644 --- a/rdagent/components/coder/data_science/feature/__init__.py +++ b/rdagent/components/coder/data_science/feature/__init__.py @@ -1,6 +1,8 @@ import json +from pathlib import Path from typing import Dict +from rdagent.app.data_science.conf import DS_RD_SETTING from rdagent.components.coder.CoSTEER import CoSTEER from rdagent.components.coder.CoSTEER.evaluators import ( CoSTEERMultiEvaluator, @@ -22,6 +24,8 @@ from rdagent.oai.llm_utils import APIBackend from rdagent.utils.agent.ret import PythonAgentOut from rdagent.utils.agent.tpl import T +DIRNAME = Path(__file__).absolute().resolve().parent + class FeatureMultiProcessEvolvingStrategy(MultiProcessEvolvingStrategy): def implement_one_task( @@ -64,8 +68,16 @@ class FeatureMultiProcessEvolvingStrategy(MultiProcessEvolvingStrategy): queried_former_failed_knowledge=queried_former_failed_knowledge[0], out_spec=PythonAgentOut.get_spec(), ) + code_spec = ( + workspace.file_dict["spec/feature.md"] + if DS_RD_SETTING.spec_enabled + else T("scenarios.data_science.share:component_spec.general").r( + spec=T("scenarios.data_science.share:component_spec.FeatureEng").r(), + test_code=(DIRNAME / "eval_tests" / "feature_test.txt").read_text(), + ) + ) user_prompt = T(".prompts:feature_coder.user").r( - feature_spec=workspace.file_dict["spec/feature.md"], + code_spec=code_spec, latest_code=workspace.file_dict.get("feature.py"), latest_code_feedback=prev_task_feedback, ) diff --git a/rdagent/components/coder/data_science/feature/eval_tests/feature_test.txt b/rdagent/components/coder/data_science/feature/eval_tests/feature_test.txt index bb029aa0..d453fd60 100644 --- a/rdagent/components/coder/data_science/feature/eval_tests/feature_test.txt +++ b/rdagent/components/coder/data_science/feature/eval_tests/feature_test.txt @@ -1,10 +1,5 @@ """ -A qualified data loader should support following features -- successfully run -- len(test) == len(test_ids) == submission length -- len(train) == len(y) - -Please make sure the stdout is rich enough to support informative feedback +Tests for `feat_eng` in feature.py """ import pickle diff --git a/rdagent/components/coder/data_science/feature/prompts.yaml b/rdagent/components/coder/data_science/feature/prompts.yaml index 30c30795..56ce2d71 100644 --- a/rdagent/components/coder/data_science/feature/prompts.yaml +++ b/rdagent/components/coder/data_science/feature/prompts.yaml @@ -55,8 +55,8 @@ feature_coder: {% endif %} user: |- - --------- Feature Processing Specification --------- - {{ feature_spec }} + --------- Code Specification --------- + {{ code_spec }} {% if latest_code %} --------- Former code --------- diff --git a/rdagent/components/coder/data_science/model/__init__.py b/rdagent/components/coder/data_science/model/__init__.py index da960f8f..09bb211e 100644 --- a/rdagent/components/coder/data_science/model/__init__.py +++ b/rdagent/components/coder/data_science/model/__init__.py @@ -1,5 +1,7 @@ +from pathlib import Path from typing import Dict +from rdagent.app.data_science.conf import DS_RD_SETTING from rdagent.components.coder.CoSTEER import CoSTEER from rdagent.components.coder.CoSTEER.evaluators import ( CoSTEERMultiEvaluator, @@ -23,6 +25,8 @@ from rdagent.oai.llm_utils import APIBackend from rdagent.utils.agent.ret import PythonBatchEditOut from rdagent.utils.agent.tpl import T +DIRNAME = Path(__file__).absolute().resolve().parent + class ModelMultiProcessEvolvingStrategy(MultiProcessEvolvingStrategy): def implement_one_task( @@ -71,8 +75,16 @@ class ModelMultiProcessEvolvingStrategy(MultiProcessEvolvingStrategy): # latest_code=workspace.file_dict.get(f"{target_task.name}.py", None), # ) # We want to use a simpler way to + code_spec = ( + workspace.file_dict["spec/model.md"] + if DS_RD_SETTING.spec_enabled + else T("scenarios.data_science.share:component_spec.general").r( + spec=T("scenarios.data_science.share:component_spec.Model").r(), + test_code=(DIRNAME / "eval_tests" / "model_test.txt").read_text().replace("model01", target_task.name), + ) + ) user_prompt = T(".prompts:model_coder.user_general").r( - model_spec=workspace.file_dict["spec/model.md"], + code_spec=code_spec, latest_model_code=workspace.get_codes( r"^model_(?!test)\w+\.py$" ), # TODO: If we have high failure rate here, we should clean this step with less information. diff --git a/rdagent/components/coder/data_science/model/eval.py b/rdagent/components/coder/data_science/model/eval.py index 15e6d09d..98e1d7ed 100644 --- a/rdagent/components/coder/data_science/model/eval.py +++ b/rdagent/components/coder/data_science/model/eval.py @@ -98,8 +98,6 @@ class ModelGeneralCaseSpecEvaluator(CoSTEEREvaluator): task_desc=target_task.get_task_information(), test_code=test_code, code=implementation.file_dict[f"{target_task.name}.py"], - scenario=self.scen.get_scenario_all_desc(), - spec=implementation.file_dict["spec/model.md"], workflow_stdout=workflow_stdout, workflow_code=implementation.all_codes, ) diff --git a/rdagent/components/coder/data_science/model/eval_tests/model_test.txt b/rdagent/components/coder/data_science/model/eval_tests/model_test.txt index 5dea8be0..11fc11aa 100644 --- a/rdagent/components/coder/data_science/model/eval_tests/model_test.txt +++ b/rdagent/components/coder/data_science/model/eval_tests/model_test.txt @@ -1,3 +1,6 @@ +""" +Tests for `model_workflow` in model01.py +""" import time from feature import feat_eng diff --git a/rdagent/components/coder/data_science/model/prompts.yaml b/rdagent/components/coder/data_science/model/prompts.yaml index dc3b0685..0e50e782 100644 --- a/rdagent/components/coder/data_science/model/prompts.yaml +++ b/rdagent/components/coder/data_science/model/prompts.yaml @@ -53,8 +53,8 @@ model_coder: {% endif %} user_general: |- - --------- Model Building Specification --------- - {{ model_spec }} + --------- Code Specification --------- + {{ code_spec }} --------- Former model code --------- {% if latest_model_code|length == 0 %} diff --git a/rdagent/components/coder/data_science/raw_data_loader/__init__.py b/rdagent/components/coder/data_science/raw_data_loader/__init__.py index 6d82c8cc..5fb2fcfe 100644 --- a/rdagent/components/coder/data_science/raw_data_loader/__init__.py +++ b/rdagent/components/coder/data_science/raw_data_loader/__init__.py @@ -24,6 +24,7 @@ File structure import json import re +from pathlib import Path from typing import Dict from rdagent.app.data_science.conf import DS_RD_SETTING @@ -53,6 +54,8 @@ from rdagent.oai.llm_utils import APIBackend from rdagent.utils.agent.ret import PythonAgentOut from rdagent.utils.agent.tpl import T +DIRNAME = Path(__file__).absolute().resolve().parent + class DataLoaderMultiProcessEvolvingStrategy(MultiProcessEvolvingStrategy): def implement_one_task( @@ -90,38 +93,41 @@ class DataLoaderMultiProcessEvolvingStrategy(MultiProcessEvolvingStrategy): # 1. specifications # TODO: We may move spec into a separated COSTEER task - if "spec/data_loader.md" not in workspace.file_dict: # Only generate the spec once - system_prompt = T(".prompts:spec.system").r( - runtime_environment=runtime_environment, - task_desc=data_loader_task_info, - competition_info=competition_info, - folder_spec=data_folder_info, - ) - data_loader_prompt = T(".prompts:spec.user.data_loader").r( - latest_spec=workspace.file_dict.get("spec/data_loader.md") - ) - feature_prompt = T(".prompts:spec.user.feature").r(latest_spec=workspace.file_dict.get("spec/feature.md")) - model_prompt = T(".prompts:spec.user.model").r(latest_spec=workspace.file_dict.get("spec/model.md")) - ensemble_prompt = T(".prompts:spec.user.ensemble").r( - latest_spec=workspace.file_dict.get("spec/ensemble.md") - ) - workflow_prompt = T(".prompts:spec.user.workflow").r( - latest_spec=workspace.file_dict.get("spec/workflow.md") - ) + if DS_RD_SETTING.spec_enabled: + if "spec/data_loader.md" not in workspace.file_dict: # Only generate the spec once + system_prompt = T(".prompts:spec.system").r( + runtime_environment=runtime_environment, + task_desc=data_loader_task_info, + competition_info=competition_info, + folder_spec=data_folder_info, + ) + data_loader_prompt = T(".prompts:spec.user.data_loader").r( + latest_spec=workspace.file_dict.get("spec/data_loader.md") + ) + feature_prompt = T(".prompts:spec.user.feature").r( + latest_spec=workspace.file_dict.get("spec/feature.md") + ) + model_prompt = T(".prompts:spec.user.model").r(latest_spec=workspace.file_dict.get("spec/model.md")) + ensemble_prompt = T(".prompts:spec.user.ensemble").r( + latest_spec=workspace.file_dict.get("spec/ensemble.md") + ) + workflow_prompt = T(".prompts:spec.user.workflow").r( + latest_spec=workspace.file_dict.get("spec/workflow.md") + ) - spec_session = APIBackend().build_chat_session(session_system_prompt=system_prompt) + spec_session = APIBackend().build_chat_session(session_system_prompt=system_prompt) - data_loader_spec = spec_session.build_chat_completion(user_prompt=data_loader_prompt) - feature_spec = spec_session.build_chat_completion(user_prompt=feature_prompt) - model_spec = spec_session.build_chat_completion(user_prompt=model_prompt) - ensemble_spec = spec_session.build_chat_completion(user_prompt=ensemble_prompt) - workflow_spec = spec_session.build_chat_completion(user_prompt=workflow_prompt) - else: - data_loader_spec = workspace.file_dict["spec/data_loader.md"] - feature_spec = workspace.file_dict["spec/feature.md"] - model_spec = workspace.file_dict["spec/model.md"] - ensemble_spec = workspace.file_dict["spec/ensemble.md"] - workflow_spec = workspace.file_dict["spec/workflow.md"] + data_loader_spec = spec_session.build_chat_completion(user_prompt=data_loader_prompt) + feature_spec = spec_session.build_chat_completion(user_prompt=feature_prompt) + model_spec = spec_session.build_chat_completion(user_prompt=model_prompt) + ensemble_spec = spec_session.build_chat_completion(user_prompt=ensemble_prompt) + workflow_spec = spec_session.build_chat_completion(user_prompt=workflow_prompt) + else: + data_loader_spec = workspace.file_dict["spec/data_loader.md"] + feature_spec = workspace.file_dict["spec/feature.md"] + model_spec = workspace.file_dict["spec/model.md"] + ensemble_spec = workspace.file_dict["spec/ensemble.md"] + workflow_spec = workspace.file_dict["spec/workflow.md"] # 2. code system_prompt = T(".prompts:data_loader_coder.system").r( @@ -130,9 +136,17 @@ class DataLoaderMultiProcessEvolvingStrategy(MultiProcessEvolvingStrategy): queried_former_failed_knowledge=queried_former_failed_knowledge[0], out_spec=PythonAgentOut.get_spec(), ) + code_spec = ( + data_loader_spec + if DS_RD_SETTING.spec_enabled + else T("scenarios.data_science.share:component_spec.general").r( + spec=T("scenarios.data_science.share:component_spec.DataLoadSpec").r(), + test_code=(DIRNAME / "eval_tests" / "data_loader_test.txt").read_text(), + ) + ) user_prompt = T(".prompts:data_loader_coder.user").r( competition_info=competition_info, - data_loader_spec=data_loader_spec, + code_spec=code_spec, folder_spec=data_folder_info, latest_code=workspace.file_dict.get("load_data.py"), latest_code_feedback=prev_task_feedback, @@ -152,14 +166,20 @@ class DataLoaderMultiProcessEvolvingStrategy(MultiProcessEvolvingStrategy): else: raise CoderError("Failed to generate a new data loader code.") - return { - "spec/data_loader.md": data_loader_spec, - "spec/feature.md": feature_spec, - "spec/model.md": model_spec, - "spec/ensemble.md": ensemble_spec, - "spec/workflow.md": workflow_spec, - "load_data.py": data_loader_code, - } + return ( + { + "spec/data_loader.md": data_loader_spec, + "spec/feature.md": feature_spec, + "spec/model.md": model_spec, + "spec/ensemble.md": ensemble_spec, + "spec/workflow.md": workflow_spec, + "load_data.py": data_loader_code, + } + if DS_RD_SETTING.spec_enabled + else { + "load_data.py": data_loader_code, + } + ) def assign_code_list_to_evo(self, code_list: list[dict[str, str]], evo): """ diff --git a/rdagent/components/coder/data_science/raw_data_loader/eval_tests/data_loader_test.txt b/rdagent/components/coder/data_science/raw_data_loader/eval_tests/data_loader_test.txt index 427066de..86a6408f 100644 --- a/rdagent/components/coder/data_science/raw_data_loader/eval_tests/data_loader_test.txt +++ b/rdagent/components/coder/data_science/raw_data_loader/eval_tests/data_loader_test.txt @@ -1,10 +1,5 @@ """ -A qualified data loader should support following features -- successfully run -- len(test) == len(test_ids) == submission length -- len(train) == len(y) - -Please make sure the stdout is rich enough to support informative feedback +Tests for `load_data` in load_data.py """ import pickle diff --git a/rdagent/components/coder/data_science/raw_data_loader/prompts.yaml b/rdagent/components/coder/data_science/raw_data_loader/prompts.yaml index d7633d9d..2c6207fc 100644 --- a/rdagent/components/coder/data_science/raw_data_loader/prompts.yaml +++ b/rdagent/components/coder/data_science/raw_data_loader/prompts.yaml @@ -54,8 +54,9 @@ spec: 2. Precautions for Data Loading and Preprocessing: - File Handling: - - Ensure proper file encoding and delimiters. + - Handle file encoding and delimiters appropriately. - Combine or process multiple files if necessary. + - Avoid using the sample submission file to infer test indices. If a dedicated test index file is available, use that. If not, use the order in the test file as the test index. - Data Preprocessing: - Convert data types correctly (e.g., numeric, categorical, date parsing). - Handle missing values appropriately. @@ -64,14 +65,16 @@ spec: - Apply competition-specific preprocessing steps as needed (e.g., text tokenization, image resizing). 3. Code Standards: - - Avoid using progress bars (e.g., `tqdm`) in the implementation. + - DO NOT use progress bars (e.g., `tqdm`). + - DO NOT use the sample submission file to extract test index information. + - DO NOT exclude features inadvertently during this process. 4. Notes: - Update `DT` (data type) based on the specific competition dataset. This can include `pd.DataFrame`, `np.array`, `torch.Tensor`, etc. - Never use sample submission as the test index, as it may not be the same as the test data. Use the test index file or test data source to get the test index. - Only set the DT of variables without inferring the shape of these variables since you don't know the shape of the data. - 5. Exploratory Data Analysis (EDA) part(Required): + 5. Exploratory Data Analysis (EDA) [Required]: - Before returning the data, you should always add an EDA part describing the data to help the following steps understand the data better. - The EDA part should be drafted in plain text with certain format schema with no more than ten thousand characters. - An evaluation agent will help to check whether the EDA part is added correctly. @@ -104,18 +107,18 @@ spec: - Inferred data shape to each input and output data variables. To uncertain dimension, use -1. 2. Precautions for Feature Engineering: - - Well handle the shape of the data + - Well handle the shape of the data: - The sample size of the train data and the test data should be the same in all scenarios. - To most of the scenario, the input shape and the output shape should be exactly the same. - To some tabular data, you may add or remove some columns so your inferred column number may be unsure. - - Integration with Model Pipeline - - If feature engineering is strictly part of the model pipeline, state explicitly that it will be handled at the model stage. - - If integrated here, ensure this function applies all required transformations while avoiding data leakage. + - Integration with the Model Pipeline: + - If feature engineering is deferred to the model pipeline for better overall performance, state explicitly that it will be handled at the model stage. + - Otherwise, ensure this function applies all required transformations while avoiding data leakage. - General Considerations: - Ensure scalability for large datasets. - Handle missing values and outliers appropriately (e.g., impute, remove, or replace). - Ensure consistency between feature data types and transformations. - - Avoid data leakage: Only use features derived from training data, excluding information from test or validation sets. + - Prevent data leakage: Do not use information derived from the test set when transforming training data. - Domain-Specific Features: - Apply logic for competition-specific features (e.g., text vectorization, image augmentations, categorical encoding). @@ -124,9 +127,7 @@ spec: 4. Notes: - Align `DT` (data type) definitions with those in the Data Loader specification. - - Extend or adjust domain-specific transformations based on competition requirements. - - The device has GPU support, so you can use it for feature engineering if necessary to accelerate the process. - - Multi processing or parallel processing can be used to speed up the feature engineering process. + - GPU and multiprocessing are available and are encouraged to use for accelerating transformations. - Only set the DT of variables without inferring the shape of these variables since you don't know the shape of the data. {% if latest_spec %} @@ -172,7 +173,7 @@ spec: 4. Notes: - Align `DT` (data type) with the definitions used in Feature Engineering specifications. - - The device has GPU support, so you can use it for training if necessary to accelerate the process. + - The device has GPU support, so you are encouraged to use it for training if necessary to accelerate the process. {% if latest_spec %} 5. Former Specification: @@ -200,7 +201,7 @@ spec: - Inferred data shape to each input and output data variables. To uncertain dimension, use -1. 2. Precautions: - - Validation of Inputs: + - Input Validation: - Ensure all predictions in `test_preds_dict` and `val_preds_dict` have consistent shapes and dimensions. - Verify that `val_label` is provided and matches the length of `val_preds_dict` predictions. - Handle empty or invalid inputs gracefully with appropriate error messages. @@ -222,7 +223,7 @@ spec: scores_df = pd.DataFrame(scores.items(), columns=["Model", ]) scores_df.to_csv("scores.csv", index=False) ``` - - If there is only one model, still compute the ensemble score and store it under "ensemble". + - Even if only one model is present, compute the ensemble score and store it under `"ensemble"`. 3. Code Standards: - Do not use progress bars (e.g., tqdm) in the code. @@ -263,7 +264,7 @@ spec: 3. Dataset Splitting - The dataset returned by `load_data` is not pre-split. After calling `feat_eng`, split the data into training and test sets. - - [Notice] If feasible, apply cross-validation (e.g. KFold) on the training set (`X_transformed`, `y_transformed`) to ensure a reliable assessment of model performance. + - [Notice] If feasible, apply cross-validation on the training set (`X_transformed`, `y_transformed`) to ensure a reliable assessment of model performance. - Keep the test set (`X_test_transformed`) unchanged, as it is only used for generating the final predictions. - Pseudocode logic for reference: ``` @@ -385,8 +386,8 @@ data_loader_coder: --------- Competition Information --------- {{ competition_info }} - --------- Data Loader Specification --------- - {{ data_loader_spec }} + --------- Code Specification --------- + {{ code_spec }} --------- Data Folder Description (All path are relative to the data folder) --------- {{ folder_spec }} @@ -401,7 +402,7 @@ data_loader_coder: The former code contains errors. You should correct the code based on the provided information, ensuring you do not repeat the same mistakes. {% endif %} - You should strictly follow the function interface specifications provided by the specification to implement the function. + You should strictly follow the code specifications provided by the specification to implement the function. data_loader_eval: @@ -453,7 +454,7 @@ data_loader_eval: - The number of unique values in each column. - The distribution of the target variable. - Any other information that you think is important for the following steps. - You will be given the EDA output, your job is to check whether the output contains the required and sufficient information. If no EDA output is provided, you should consider it as a failure. Put this evaluation result in the return_checking part. + You will be given the EDA output, your job is to check whether the output contains the required and sufficient information. If no EDA output is provided, you should consider it as a failure. Put this evaluation result in the return_checking part. Your response must follow this structured JSON format: ```json diff --git a/rdagent/components/coder/data_science/workflow/__init__.py b/rdagent/components/coder/data_science/workflow/__init__.py index d9bdfaaa..72d9a2f2 100644 --- a/rdagent/components/coder/data_science/workflow/__init__.py +++ b/rdagent/components/coder/data_science/workflow/__init__.py @@ -1,6 +1,7 @@ import json from typing import Dict +from rdagent.app.data_science.conf import DS_RD_SETTING from rdagent.components.coder.CoSTEER import CoSTEER from rdagent.components.coder.CoSTEER.evaluators import ( CoSTEERMultiEvaluator, @@ -69,7 +70,11 @@ class WorkflowMultiProcessEvolvingStrategy(MultiProcessEvolvingStrategy): model_codes=workspace.get_codes(r"^model_(?!test)\w+\.py$"), ensemble_code=workspace.file_dict["ensemble.py"], latest_code=workspace.file_dict.get("main.py"), - workflow_spec=workspace.file_dict["spec/workflow.md"], + code_spec=( + workspace.file_dict["spec/workflow.md"] + if DS_RD_SETTING.spec_enabled + else T("scenarios.data_science.share:component_spec.Workflow").r() + ), latest_code_feedback=prev_task_feedback, ) diff --git a/rdagent/components/coder/data_science/workflow/eval.py b/rdagent/components/coder/data_science/workflow/eval.py index 6b9ea64d..6d1388c6 100644 --- a/rdagent/components/coder/data_science/workflow/eval.py +++ b/rdagent/components/coder/data_science/workflow/eval.py @@ -115,7 +115,11 @@ class WorkflowGeneralCaseSpecEvaluator(CoSTEEREvaluator): system_prompt = T(".prompts:workflow_eval.system").r( scenario=self.scen.get_scenario_all_desc(), task_desc=target_task.get_task_information(), - spec=implementation.file_dict["spec/workflow.md"], + spec=( + implementation.file_dict["spec/workflow.md"] + if DS_RD_SETTING.spec_enabled + else T("scenarios.data_science.share:component_spec.Workflow").r() + ), ) user_prompt = T(".prompts:workflow_eval.user").r( stdout=stdout.strip(), diff --git a/rdagent/components/coder/data_science/workflow/prompts.yaml b/rdagent/components/coder/data_science/workflow/prompts.yaml index 4c294b52..ed314fda 100644 --- a/rdagent/components/coder/data_science/workflow/prompts.yaml +++ b/rdagent/components/coder/data_science/workflow/prompts.yaml @@ -52,8 +52,8 @@ workflow_coder: {% endif %} user: |- - --------- Workflow Specification --------- - {{ workflow_spec }} + --------- Code Specification --------- + {{ code_spec }} --------- load data code --------- file: load_data.py @@ -96,9 +96,10 @@ workflow_eval: The user provides workflow information and its components. The details on how to structure the workflow are given in the specification file: - ```python + ```markdown {{ spec }} ``` + This workflow integrates multiple stages, including: - Data loading - Feature engineering diff --git a/rdagent/scenarios/data_science/proposal/exp_gen.py b/rdagent/scenarios/data_science/proposal/exp_gen.py index 8dbb672d..ca4b0bb6 100644 --- a/rdagent/scenarios/data_science/proposal/exp_gen.py +++ b/rdagent/scenarios/data_science/proposal/exp_gen.py @@ -3,6 +3,7 @@ from typing import Dict, Literal import pandas as pd +from rdagent.app.data_science.conf import DS_RD_SETTING from rdagent.components.coder.data_science.ensemble.exp import EnsembleTask from rdagent.components.coder.data_science.feature.exp import FeatureTask from rdagent.components.coder.data_science.model.exp import ModelTask @@ -242,10 +243,14 @@ class DSExpGen(ExpGen): else: break + if DS_RD_SETTING.spec_enabled: + spec = last_successful_exp.experiment_workspace.file_dict[spec_file] if spec_file else None + else: + spec = T(f"scenarios.data_science.share:component_spec.{component}").r() resp_dict = self._init_task_gen( targets=component, scenario_desc=scenario_desc, - spec=last_successful_exp.experiment_workspace.file_dict[spec_file] if spec_file else None, + spec=spec, task_output_format=T(f".prompts:output_format.{component_prompt_key or component.lower()}").r(), former_task=former_tasks_desc if former_tasks_desc else None, ) @@ -384,13 +389,17 @@ class DSExpGen(ExpGen): component_info = COMPONENT_TASK_MAPPING.get(component) if component_info: + if DS_RD_SETTING.spec_enabled: + task_spec = sota_exp.experiment_workspace.file_dict[component_info["spec_file"]] + else: + task_spec = T(f"scenarios.data_science.share:component_spec.{component}").r() system_prompt = T(".prompts:direct_exp_gen.system").r( targets=component_info["target_name"], component=component, scenario=scenario_desc, hypothesis_specification=T(".prompts:hypothesis_specification").r(), hypothesis_output_format=T(".prompts:output_format.hypothesis").r(), - task_specification=sota_exp.experiment_workspace.file_dict[component_info["spec_file"]], + task_specification=task_spec, task_output_format=component_info["task_output_format"], workflow_check=(not component == "Workflow"), ) diff --git a/rdagent/scenarios/data_science/proposal/prompts.yaml b/rdagent/scenarios/data_science/proposal/prompts.yaml index c3180d65..d6101958 100644 --- a/rdagent/scenarios/data_science/proposal/prompts.yaml +++ b/rdagent/scenarios/data_science/proposal/prompts.yaml @@ -188,7 +188,7 @@ direct_exp_gen: ## Task Specification The scope of the {{ targets }} can be described by a interface specification as follows: - ```Python + ```markdown {{ task_specification }} ``` diff --git a/rdagent/scenarios/data_science/share.yaml b/rdagent/scenarios/data_science/share.yaml index b0ed369c..810cc18a 100644 --- a/rdagent/scenarios/data_science/share.yaml +++ b/rdagent/scenarios/data_science/share.yaml @@ -72,4 +72,157 @@ component_description: Ensemble: |- Combines predictions from multiple models using ensemble strategies, evaluates their performance, and generates the final test predictions. Workflow: |- - Integrates all pipeline components, from data loading to ensemble prediction, ensuring efficient execution and correct output formatting. \ No newline at end of file + Integrates all pipeline components, from data loading to ensemble prediction, ensuring efficient execution and correct output formatting. + +component_spec: + general: |- + {{ spec }} + + Your code will be tested by the code below. You must ensure your implementation passes the test code: + ```python + {{ test_code }} + ``` + DataLoadSpec: |- + 1. File Handling: + - Handle file encoding and delimiters appropriately. + - Combine or process multiple files if necessary. + - Avoid using the sample submission file to infer test indices. If a dedicated test index file is available, use that. If not, use the order in the test file as the test index. + + 2. Data Preprocessing: + - Convert data types correctly (e.g., numeric, categorical, date parsing). + - Handle missing values appropriately. + - Optimize memory usage for large datasets using techniques like downcasting or reading data in chunks if necessary. + + 3. Code Standards: + - DO NOT use progress bars (e.g., `tqdm`). + - DO NOT use the sample submission file to extract test index information. + - DO NOT exclude features inadvertently during this process. + + 4. Exploratory Data Analysis (EDA) [Required]: + - Before returning the data, you should always add an EDA part describing the data to help the following steps understand the data better. + - The EDA part should be drafted in plain text with certain format schema with no more than ten thousand characters. + - An evaluation agent will help to check whether the EDA part is added correctly. + + FeatureEng: |- + 1. Well handle the shape of the data + - The sample size of the train data and the test data should be the same in all scenarios. + - To most of the scenario, the input shape and the output shape should be exactly the same. + - To some tabular data, you may add or remove some columns so your inferred column number may be unsure. + + 2. Integration with the Model Pipeline: + - If feature engineering is deferred to the model pipeline for better overall performance, state explicitly that it will be handled at the model stage. + - Otherwise, ensure this function applies all required transformations while avoiding data leakage. + + 3. General Considerations: + - Ensure scalability for large datasets. + - Handle missing values and outliers appropriately (e.g., impute, remove, or replace). + - Ensure consistency between feature data types and transformations. + - Prevent data leakage: Do not use information derived from the test set when transforming training data. + + 4. Code Standards: + - Avoid using progress bars (e.g., `tqdm`) in the implementation. + + 5. Notes: + - GPU and multiprocessing are available and are encouraged to use for accelerating transformations. + - Feature engineering should be executed **once** and reused across all models to ensure consistency: `X_transformed, y_transformed, X_test_transformed = feat_eng(X, y, X_test)` + + Model: |- + - Do not use progress bars (e.g., `tqdm`) in the implementation. + - The device has GPU support, so you are encouraged to use it for training if necessary to accelerate the process. + + Ensemble: |- + 1. Input Validation: + - Handle empty or invalid inputs gracefully with appropriate error messages. + + 2. Metric Calculation and Storage: + - Calculate the metric (mentioned in the evaluation section of the competition information) for each model and ensemble strategy on valid, and save the results in `scores.csv`, e.g.: + ```python + scores = {} + for model_name, val_pred in val_preds_dict.items(): + scores[model_name] = calculate_metric(val_label, val_pred) + + ... + some code about ensemble strategy + ... + ensemble_val_pred = ... + + ensemble_score = calculate_metric(val_label, ensemble_val_pred) + scores["ensemble"] = ensemble_score # Ensure "ensemble" is explicitly stored + + scores_df = pd.DataFrame(scores.items(), columns=["Model", ]) + scores_df.to_csv("scores.csv", index=False) + ``` + - Even if only one model is present, compute the ensemble score and store it under `"ensemble"`. + + 3. Code Standards: + - Do not use progress bars (e.g., tqdm) in the code. + + 4. Notes: + - Ensure flexibility to handle multiple ensemble strategies based on competition requirements. + + Workflow: |- + Your task is to implement the main workflow script (`main.py`) for a Kaggle-style machine learning competition project. + Follow the provided project structure and specifications to ensure consistency and maintainability: + 1. Workflow Integration: + - Integrate the following components into the workflow: + - Data loading (`load_data.py`). + - Feature engineering (`feature.py`). + - Model workflow for training and testing (`model_*.py`). + - Ensemble workflow that combines results from the model workflow to obtain the final prediction (`ensemble.py`). + - Treat each component as a modular and callable Python function. + - The workflow script should be flexible enough to handle either a single model or multiple models, with filenames (model_*.py) that are not determined at the outset. + For multiple model selection, utilize Python code to identify eligible models based on filenames, for example: + ```python + available_models = [f for f in os.listdir('.') if f.startswith('model_') and 'test' not in f] + ``` + 2. Feature Engineering + - The feature engineering should be called only once. For example: + `X_transformed, y_transformed, X_test_transformed = feat_eng(X, y, X_test)` + - It should be called before dataset splitting. + + 3. Dataset Splitting + - The dataset returned by `load_data` is not pre-split. After calling `feat_eng`, split the data into training and test sets. + - [Notice] If feasible, apply cross-validation on the training set (`X_transformed`, `y_transformed`) to ensure a reliable assessment of model performance. + - Keep the test set (`X_test_transformed`) unchanged, as it is only used for generating the final predictions. + - Pseudocode logic for reference: + ``` + Set number of splits and initialize KFold cross-validator. + Create dictionaries for validation and test predictions. + For each model file: + Import the model dynamically. + Initialize arrays for out-of-fold (OOF) and test predictions. + For each fold in KFold: + Split data into training and validation sets. + Run model workflow to get validation and test predictions. + Validate shapes. + Store validation and test predictions. + Compute average test predictions across folds. + Save OOF and averaged test predictions. + Ensemble predictions from all models and print the final shape. + ``` + + 4. Submission File: + - Save the final predictions as `submission.csv`, ensuring the format matches the competition requirements (refer to `sample_submission` in the Folder Description for the correct structure). + - Present the required submission format explicitly and ensure the output adheres to it. + + 5. Code Standards: + - Do not use progress bars (e.g., tqdm) in the code. + + 6. Ensemble Strategy: + Consolidate all model outputs into a dictionary, where each key is the model's filename (excluding the .py extension) and its corresponding value is the model's output. + Sample code: + {% raw %} + {% for model_name in model_names %} + model_module = __import__(model_name.replace('.py', '')) + val_pred, test_pred, _ = model_module.model_workflow( + X=train_X, + y=train_y, + val_X=val_X, + val_y=val_y, + test_X=X_test_transformed + ) + val_preds_dict[model_module.__name__] = val_pred + test_preds_dict[model_module.__name__] = test_pred + {% endfor %} + final_pred = ensemble_workflow(test_preds_dict, val_preds_dict, val_y) + {% endraw %} \ No newline at end of file