gavindiaz/NexQuant
1
0
Fork 0
mirror of https://github.com/NicolasBohn/NexQuant.git synced 2026-08-01 09:27:43 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: enable drafting with knowledge (#998)

Browse Source 
* add pipeline for drafting v2

* fix the pipeline and add general knowledge

* debug

* fix bug

* fix bug

* change draft version1

* add function calling to task gen

* fix circular import bug

* change draft version3

* exp1_test

* feat: add DraftRouterExpGen and make summarizer configurable

* Update rdagent/scenarios/data_science/proposal/exp_gen/proposal.py

* change code structure

* stashed changes

* test

* test1

* revert conf.py

* add runtime enviornment info to general knowledge

* remove redundant code

* clean code

* remove files

* reformat

* fix bug

* fix bug

* simplify code

* fix minor bug

* fix bug and reformat

* revert config

* remove unused prompt

* add general knowledge

* fix ci

---------

Co-authored-by: Xu <v-xuminrui@microsoft.com>
Co-authored-by: jingyuanlm <842442862@qq.com>
Co-authored-by: Young <afe.young@gmail.com>
Co-authored-by: you-n-g <you-n-g@users.noreply.github.com>
This commit is contained in:
Roland Minrui
2025-07-09 18:36:42 +08:00
committed by GitHub
parent 12d4e8a850
commit 1c35004782
12 changed files with 725 additions and 282 deletions
Download Patch File Download Diff File Expand all files Collapse all files
rdagent/app/data_science/conf.py
+4
View File
@@ -21,6 +21,10 @@ class DataScienceBasePropSetting(KaggleBasePropSetting):
hypothesis_gen: str = "rdagent.scenarios.data_science.proposal.exp_gen.proposal.DSProposalV2ExpGen"
"""Hypothesis generation class"""
summarizer: str = "rdagent.scenarios.data_science.dev.feedback.DSExperiment2Feedback"
summarizer_init_kwargs: dict = {
"version": "exp_feedback",
}
## Workflow Related
consecutive_errors: int = 5
rdagent/scenarios/data_science/dev/coder.py
View File
rdagent/scenarios/data_science/dev/feedback.py
+8 -2
View File
@@ -9,6 +9,7 @@ from rdagent.core.proposal import (
ExperimentFeedback,
HypothesisFeedback,
)
from rdagent.core.scenario import Scenario
from rdagent.log.utils import dict_get_with_warning
from rdagent.oai.llm_utils import APIBackend
from rdagent.scenarios.data_science.experiment.experiment import DSExperiment
@@ -20,6 +21,10 @@ from rdagent.utils.repo.diff import generate_diff_from_dict
class DSExperiment2Feedback(Experiment2Feedback):
def __init__(self, scen: Scenario, version: str = "exp_feedback") -> None:
super().__init__(scen)
self.version = version
def generate_feedback(self, exp: DSExperiment, trace: DSTrace) -> ExperimentFeedback:
# 用哪些信息来生成feedback
# 1. pending_tasks_list[0][0] 任务的描述
@@ -63,10 +68,11 @@ class DSExperiment2Feedback(Experiment2Feedback):
)
eda_output = exp.experiment_workspace.file_dict.get("EDA.md", None)
system_prompt = T(".prompts:exp_feedback.system").r(
system_prompt = T(f".prompts:{self.version}.system").r(
scenario=self.scen.get_scenario_all_desc(eda_output=eda_output)
)
user_prompt = T(".prompts:exp_feedback.user").r(
user_prompt = T(f".prompts:{self.version}.user").r(
sota_desc=sota_desc,
cur_exp=exp,
diff_edition=diff_edition,
rdagent/scenarios/data_science/dev/prompts.yaml
+143 -120
View File
@@ -80,6 +80,148 @@ exp_feedback:
user: |-
We are currently in a process of validating hypotheses to iteratively improve our models for Kaggle competitions. Each round aims explicitly to confirm or reject hypotheses based on experiment results.
## SOTA Solution
{{ sota_desc }}
## Current Solution
### Task of Current Solution
{{ cur_exp.pending_tasks_list[0][0].get_task_information() }}
{% if cur_exp.hypothesis %}
The experiment was designed based on the following hypothesis:
{{ cur_exp.hypothesis }}
Modified code according to hypothesis:
{% else %}
Modified code:
{% endif %}
{% for de in diff_edition %}
{{ de }}
{% endfor %}
### Final Results of the Current Solution
1. Pay close attention to the `ensemble` score, as it represents the final evaluation metric for this iteration.
2. If any individual model significantly outperforms the ensemble, this may indicate an issue in the ensemble method. But if the final `ensemble` score surpasses the current SOTA, you should update the SOTA record. However, it seems that there are noticeable issues in the ensemble component, be sure to highlight them explicitly.
Below are the results and running time for this experiment:
Running time: {{ cur_exp.running_info.running_time }} seconds.
Results: {{ cur_exp.result }}
{% if cur_vs_sota_score is not none %}
Below is the comparison of the current `ensemble` performance with the SOTA results:
{{ cur_vs_sota_score }}
{% endif %}
{% if cur_exp.format_check_result is not none %}
### Submission format check to current solution:
{{ cur_exp.format_check_result }}
{% endif %}
### Complete Code of Current Solution
{{ cur_exp.experiment_workspace.all_codes }}
## Feedback of past experiments
{{ feedback_desc or "There has not been any experiments yet." }}
Please refer to these hypotheses and feedback to help you recommend new experiment and hypothesis
Tips:
- Step 1: If submission format has issues, prioritize fixing them before proceeding. If the format is correct and it's the first valid submission ever (there has never been valid submissions in the past), set `"Replace Best Result": "yes"`. If the format is correct and this is not the first valid submission, proceed to Step 2.
- Step 2: If evaluation alignment issues are identified (validation approach does not follow competition requirements), address these methodological discrepancies immediately.
- Step 3: If new results significantly worse than SOTA, or repeated hyperparameter adjustments yield no improvement, it might be time to rethink or shift focus.
exp_feedback_draft:
system: |-
You are an advanced assistant analyzing results in data-driven R&D.
Below is a detailed description of the current Kaggle competition scenario:
{{ scenario }}
Your task is to analyze the current experiment's hypothesis, implementation (code and its changes), and results, explicitly comparing them with previous best SOTA result step by step.
# Step-by-step Analysis Process:
Step 1: Verify Submission Format
- If the submission format check fails:
- Identify and clearly specify code or workflow issues.
- Recommend corrective actions explicitly.
- Set `"Replace Best Result": "no"`.
- Begin your `reasoning` with `[Submission format error]`, clearly stating the issues causing experiment failure.
- If submission passes the submission format check:
- If this is the first valid submission ever, set `"Replace Best Result": "yes"`.
- Otherwise, proceed to Step 2.
Step 2: Evaluate Alignment with Competition Requirements (if format correct)
- GOAL: CAREFULLY ANALYZE WHETHER THE EXPERIMENTAL SETUP AND CODE MAY CAUSE MISALIGNMENT BETWEEN VALIDATION AND TEST PERFORMANCE.
- Confirm strict adherence to the competition's evaluation rules listed in `scenario`:
- Exact match between validation metric and official Kaggle metric.
- Consistent prediction methodologies between validation and test datasets.
- No shortcuts or fold-specific strategies applied inconsistently.
- Rigorous checks for corner-case consistency.
- If the validation score appears unreliable, provide concrete evidence from the scenario description or code implementation. Do not rely on assumptions without direct supporting evidence.
- Additionally, detect whether the setup introduces structural risks, such as overfitting-prone finetuning strategies or domain adaptation on insufficient data.
- If overfitting is detected, provide a detailed analysis explaining how and why it occurs, referencing scenario description, code implementation, and validation scores to support your findings.
- If such discrepancies or risks are found:
- Clearly document these issues in `Reasoning`, referencing both scenario description and code implementation—not just validation scores.
- Set `"Evaluation Aligned With Task": "no"` and `"Replace Best Result": "no"`.
- Begin your `reasoning` with `[Evaluation error]`, explicitly stating the evaluation alignment issues causing experiment failure.
- If evaluation alignment passes, set `"Evaluation Aligned With Task": "yes"`, and then proceed to Step 3.
Step 3: Analyze Experimental Results (if format and evaluation alignment correct)
- Explicitly confirm or refute the hypothesis with precise data points or performance trends.
- Directly compare the current `ensemble` validation score to the SOTA `ensemble` validation score. Do not focus on individual models unless anomalies are significant.
- Based on the metric used in the competition, the comparison should fit into the following categories:
- If the current `ensemble` validation score is obviously worse than the SOTA `ensemble` validation score, set `"Replace Best Result": "no"`.
- If the current `ensemble` validation score is obviously better than the SOTA `ensemble` validation score, set `"Replace Best Result": "yes"`.
- If the current `ensemble` validation score is similar to the SOTA `ensemble` validation score or both reach the ceiling performance, proceed to Step 4.
- Begin your `reasoning` with `[Experiment Analysis]`, clearly stating why the current experiment's result surpasses or falls short compared to the SOTA.
- NOTES:
- The experiments focus on the comparison of the final ensemble results (Don't reject the results because they are still not perfect)
- If the `ensemble` score does not exceed the best individual mode or single fold, it is still acceptable unless the gap is significant.
Step 4: Analyze Code With Similar validation Results
- If the current `ensemble` validation score is similar to the SOTA `ensemble` validation score, give the decision based on the comparison between the current experiment and SOTA.
- The current code should replace the best result if the code is:
- Less potential overfitting and no data leakage. The code should not modify the validation and test set distributions.
- Using best practices and modeling techniques. The code should has a more reasonable and efficient choice of every component based on the scenario.
- Interpretable and domain alignment. The code should be tied to solid domain knowledge and be interpretable.
- More resource efficiency. The code should be more efficient in terms of time and space complexity.
- Please examine the code carefully based on the above criteria and provide a detailed analysis of the code.
- Begin your `reasoning` with `[Code Analysis]`, clearly stating why the current code is better or worse than SOTA, based on the analysis of code implementation.
- If the current code is not better than SOTA, set `"Replace Best Result": "no"`. Otherwise, set `"Replace Best Result": "yes"`.
Step 5: EDA improvement analysis (if needed)
- The user might provide Data Overview in EDA format which is the output of the EDA code. You should analyze the EDA result and provide feedback on how it can be improved.
- The improvement might include some addons or modifications or deletions to some part of the EDA code.
- You should provide your feedback based on the current code and SOTA code. Especially focus on the feature engineering part.
- For example, if the code truncate the line with N words, you can suggest to print the mean, median or quantile of the length of the line for better understanding of the data in the next rounds of experiments.
Provide detailed and constructive feedback structured as follows without anything else:
{
"Submission Format Check": "yes or no",
"First Valid Submission": "yes or no",
"Code Change Summary": "Clearly summarize the changes made to the code (please cover the most important changes while being concise); during development, extra modifications may be made beyond the intent of the hypothesis, so these changes should also be included to provide complete information",
"Observations": "Clearly summarize current and SOTA ensemble results with exact scores and notable patterns. Limit to no more than three concise, data-focused sentences. Your observation must be grounded by explicit evidence from scenario description or code implementation, not just validation scores.",
"Feedback for Hypothesis": Explicitly confirm or refute the hypothesis based on specific data points or performance trends. Limit to two sentences.",
"Evaluation Aligned With Task": "yes or no",
"Replace Best Result": "yes or no",
"Refine Decision": "yes or no",
"Reasoning": "Clearly explain the reason for success or failure of the experiment. Begin explicitly with [Submission format error], [Evaluation error], [Experiment Analysis] or [Code Analysis] depending on the step at which issues arose. Reference specific scores and methodological differences with SOTA. Limit to three sentences.",
"EDA Improvement": "improvement suggestion for EDA code, if needed, otherwise set to 'no'. If there is no EDA code, set to 'no'."
}
user: |-
We are currently in a process of validating hypotheses to iteratively improve our models for Kaggle competitions. Each round aims explicitly to confirm or reject hypotheses based on experiment results.
We prioritize minimal, incremental code changes that lead to measurable improvements.**
- Once a pipeline can run end-to-end and produce valid outputs with reasonable validation results, **future iterations should avoid large-scale rewrites**.
- Instead, apply **small, controlled changes** to gradually improve performance. Examples include:
- Increasing `max_epoch` or adjusting early stopping to allow better convergence.
- Slightly modifying model architecture (e.g., unfreezing layers, switching backbone).
- Tuning hyperparameters like learning rate, batch size, or dropout.
- Introducing one new augmentation or feature at a time.
- This approach ensures that each change is **testable**, **traceable**, and **reversible**, and it avoids the risk of silently breaking a previously working pipeline.
## SOTA Solution
{{ sota_desc }}
@@ -126,128 +268,9 @@ exp_feedback:
{{ feedback_desc or "There has not been any experiments yet." }}
Please refer to these hypotheses and feedback to help you recommend new experiment and hypothesis
Tips:
- Step 1: If submission format has issues, prioritize fixing them before proceeding. If the format is correct and it's the first valid submission ever (there has never been valid submissions in the past), set `"Replace Best Result": "yes"`. If the format is correct and this is not the first valid submission, proceed to Step 2.
- Step 2: If evaluation alignment issues are identified (validation approach does not follow competition requirements), address these methodological discrepancies immediately.
- Step 3: If new results significantly worse than SOTA, or repeated hyperparameter adjustments yield no improvement, it might be time to rethink or shift focus.
exp_feedback_v3:
system: |-
You are an advanced assistant analyzing results in data-driven R&D.
Below is a detailed description of the current Kaggle competition scenario:
{{ scenario }}
Your task is to analyze the current experiment's hypothesis, implementation (code), and results, explicitly comparing them with previous experiments and the best previous result (SOTA).
Step-by-step Analysis Process:
Step 1: Verify Submission Format
- If the submission format check fails:
- Identify and clearly specify code or workflow issues.
- Recommend corrective actions explicitly.
- Set `"Replace Best Result": "no"`.
- Begin your `reasoning` with `[Submission format error]`, clearly stating the issues causing experiment failure.
- If submission passes the submission format check:
- If this is the first valid submission ever, set `"Replace Best Result": "yes"`.
- Otherwise, proceed to Step 2.
Step 2: Evaluate Alignment with Competition Requirements (if format correct)
- GOAL: CAREFULLY ANALYZE WHETHER THE EXPERIMENTAL SETUP AND CODE MAY CAUSE MISALIGNMENT BETWEEN VALIDATION AND TEST PERFORMANCE.
- Confirm strict adherence to the competition's evaluation rules listed in `scenario`:
- Exact match between validation metric and official Kaggle metric.
- Consistent prediction methodologies between validation and test datasets.
- No shortcuts or fold-specific strategies applied inconsistently.
- Rigorous checks for corner-case consistency.
- Additionally, detect whether the setup introduces structural risks, such as overfitting-prone finetuning strategies or domain adaptation on insufficient data.
- If such discrepancies or risks are found:
- Clearly document these issues in `Reasoning`.
- Set `"Evaluation Aligned With Task": "no"` and `"Replace Best Result": "no"`.
- Begin your `reasoning` with `[Evaluation error]`, explicitly stating the evaluation alignment issues causing experiment failure.
- If evaluation alignment passes, set `"Evaluation Aligned With Task": "yes"`, and then proceed to Step 3.
Step 3: Analyze Experimental Results (if format and evaluation alignment correct)
- Explicitly confirm or refute the hypothesis with precise data points or performance trends.
- Directly compare the current `ensemble` validation score to the SOTA `ensemble` validation score. Do not focus on individual models unless anomalies are significant.
- Based on the metric used in the competition, the comparison should fit into the following categories:
- If the current `ensemble` validation score is obviously worse than the SOTA `ensemble` validation score, set `"Replace Best Result": "no"`.
- If the current `ensemble` validation score is obviously better than the SOTA `ensemble` validation score, set `"Replace Best Result": "yes"`.
- If the current `ensemble` validation score is similar to the SOTA `ensemble` validation score or both reach the ceiling performance, proceed to Step 4.
- Begin your `reasoning` with `[Experiment Analysis]`, clearly stating why the current experiment's result surpasses or falls short compared to the SOTA.
- NOTES:
- The experiments focus on the comparison of the final ensemble results (Don't reject the results because they are still not perfect)
- If the `ensemble` score does not exceed the best individual mode or single fold, it is still acceptable unless the gap is significant.
Step 4: Analyze Code With Similar validation Results
- If the current `ensemble` validation score is similar to the SOTA `ensemble` validation score, give the decision based on the comparison between the current experiment and SOTA.
- The current code should replace the best result if the code is:
- Less potential overfitting and no data leakage. The code should not modify the validation and test set distributions.
- Using best practices and modeling techniques. The code should has a more reasonable and efficient choice of every component based on the scenario.
- Interpretable and domain alignment. The code should be tied to solid domain knowledge and be interpretable.
- More resource efficiency. The code should be more efficient in terms of time and space complexity.
- Please examine the code carefully based on the above criteria and provide a detailed analysis of the code.
- Begin your `reasoning` with `[Code Analysis]`, clearly stating why the current code is better or worse than SOTA.
- If the current code is not better than SOTA, set `"Replace Best Result": "no"`. Otherwise, set `"Replace Best Result": "yes"`.
Provide detailed and constructive feedback structured as follows:
Example JSON Structure for Result Analysis:
{
"Submission Format Check": "yes or no",
"First Valid Submission": "yes or no",
"Observations": "Clearly summarize current and SOTA ensemble results with exact scores and notable patterns. Limit to no more than three concise, data-focused sentences.",
"Feedback for Hypothesis": Explicitly confirm or refute the hypothesis based on specific data points or performance trends. Limit to two sentences.",
"Evaluation Aligned With Task": "yes or no",
"Replace Best Result": "yes or no",
"Reasoning": "Clearly explain the reason for success or failure of the experiment. Begin explicitly with [Submission format error], [Evaluation error], [Experiment Analysis] or [Code Analysis] depending on the step at which issues arose. Reference specific scores and methodological differences with SOTA. Limit to three sentences."
}
user: |-
We are currently in a process of validating hypotheses to iteratively improve our models for Kaggle competitions. Each round aims explicitly to confirm or reject hypotheses based on experiment results.
## SOTA Solution
{{ sota_desc }}
## Current Solution
### Task of Current Solution
{{ cur_exp.pending_tasks_list[0][0].get_task_information() }}
{% if cur_exp.hypothesis %}
The experiment was designed based on the following hypothesis:
{{ cur_exp.hypothesis }}
Modified code according to hypothesis:
{% else %}
Modified code:
{% endif %}
{% for de in diff_edition %}
{{ de }}
{% endfor %}
### Final Results of the Current Solution
1. Pay close attention to the `ensemble` score, as it represents the final evaluation metric for this iteration.
2. If any individual model significantly outperforms the ensemble, this may indicate an issue in the ensemble method. But if the final `ensemble` score surpasses the current SOTA, you should update the SOTA record. However, it seems that there are noticeable issues in the ensemble component, be sure to highlight them explicitly.
Below are the results for this experiment:
{{ cur_exp.result }}
{% if cur_vs_sota_score is not none %}
Below is the comparison of the current `ensemble` performance with the SOTA results:
{{ cur_vs_sota_score }}
{% endif %}
{% if cur_exp.format_check_result is not none %}
### Submission format check to current solution:
{{ cur_exp.format_check_result }}
{% endif %}
### Complete Code of Current Solution
{{ cur_exp.experiment_workspace.all_codes }}
## Feedback of past experiments
{{ feedback_desc or "There has not been any experiments yet." }}
Please refer to these hypotheses and feedback to help you recommend new experiment and hypothesis
Tips:
- Step 1: If submission format has issues, prioritize fixing them before proceeding. If the format is correct and it's the first valid submission ever (there has never been valid submissions in the past), set `"Replace Best Result": "yes"`. If the format is correct and this is not the first valid submission, proceed to Step 2.
- Step 2: If evaluation alignment issues are identified (validation approach does not follow competition requirements), address these methodological discrepancies immediately.
- Step 3: If new results significantly worse than SOTA, or repeated hyperparameter adjustments yield no improvement, it might be time to rethink or shift focus.
- Step 4: If the result is only slightly better than the SOTA, but the code modifications are extensive (e.g., low modification score or too many critical changes), reject the update. Prefer small-step improvements with minimal changes. Set `"Replace Best Result": "no"` and explain in `"Reasoning"` starting with `[Code Change Too Large]`.
rdagent/scenarios/data_science/loop.py
+4 -4
View File
@@ -30,7 +30,7 @@ from rdagent.log import rdagent_logger as logger
from rdagent.scenarios.data_science.dev.feedback import DSExperiment2Feedback
from rdagent.scenarios.data_science.dev.runner import DSCoSTEERRunner
from rdagent.scenarios.data_science.experiment.experiment import DSExperiment
from rdagent.scenarios.data_science.proposal.exp_gen import DSExpGen, DSTrace
from rdagent.scenarios.data_science.proposal.exp_gen import DSTrace
from rdagent.scenarios.data_science.proposal.exp_gen.idea_pool import DSKnowledgeBase
from rdagent.scenarios.data_science.proposal.exp_gen.proposal import DSProposalV2ExpGen
from rdagent.utils.workflow.misc import wait_retry
@@ -112,8 +112,6 @@ class DataScienceRDLoop(RDLoop):
self.runner = DSCoSTEERRunner(scen)
if DS_RD_SETTING.enable_doc_dev:
self.docdev = DocDev(scen)
# self.summarizer: Experiment2Feedback = import_class(PROP_SETTING.summarizer)(scen)
# logger.log_object(self.summarizer, tag="summarizer")
if DS_RD_SETTING.enable_knowledge_base and DS_RD_SETTING.knowledge_base_version == "v1":
knowledge_base = DSKnowledgeBase(
@@ -122,7 +120,9 @@ class DataScienceRDLoop(RDLoop):
self.trace = DSTrace(scen=scen, knowledge_base=knowledge_base)
else:
self.trace = DSTrace(scen=scen)
self.summarizer = DSExperiment2Feedback(scen)
self.summarizer = import_class(PROP_SETTING.summarizer)(scen=scen, **PROP_SETTING.summarizer_init_kwargs)
super(RDLoop, self).__init__()
async def direct_exp_gen(self, prev_out: dict[str, Any]):
rdagent/scenarios/data_science/proposal/exp_gen/__init__.py
+2 -37
View File
@@ -1,38 +1,3 @@
from rdagent.app.data_science.conf import DS_RD_SETTING
from rdagent.core.proposal import ExpGen
from rdagent.core.scenario import Scenario
from rdagent.log import rdagent_logger as logger
from rdagent.oai.llm_utils import APIBackend, md5_hash
from rdagent.scenarios.data_science.experiment.experiment import DSExperiment
from rdagent.scenarios.data_science.proposal.exp_gen.base import DSHypothesis, DSTrace
from rdagent.scenarios.data_science.proposal.exp_gen.draft import DSDraftExpGen
from rdagent.scenarios.data_science.proposal.exp_gen.proposal import (
DSProposalV1ExpGen,
DSProposalV2ExpGen,
)
from rdagent.scenarios.data_science.scen import DataScienceScen
from rdagent.utils.agent.tpl import T
from rdagent.scenarios.data_science.proposal.exp_gen.base import DSTrace
class DSExpGen(ExpGen):
"""
Data Science Task Generator.
This is a experiment router generator;
"""
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
def gen(self, trace: DSTrace) -> DSExperiment:
# sota_exp = trace.sota_experiment()
# # Draft
# # TODO: draft here
# if sota_exp is None:
# pass
# Propose
if DS_RD_SETTING.proposal_version == "v1":
return DSProposalV1ExpGen(scen=self.scen).gen(trace=trace)
if DS_RD_SETTING.proposal_version == "v2":
return DSProposalV2ExpGen(scen=self.scen).gen(trace=trace)
__all__ = ["DSTrace"]
rdagent/scenarios/data_science/proposal/exp_gen/draft.py
-118
View File
@@ -1,118 +0,0 @@
import json
from typing import TYPE_CHECKING
from rdagent.app.data_science.conf import DS_RD_SETTING
from rdagent.components.coder.data_science.ensemble.exp import EnsembleTask
from rdagent.components.coder.data_science.feature.exp import FeatureTask
from rdagent.components.coder.data_science.model.exp import ModelTask
from rdagent.components.coder.data_science.raw_data_loader.exp import DataLoaderTask
from rdagent.components.coder.data_science.workflow.exp import WorkflowTask
from rdagent.core.proposal import ExpGen, Hypothesis
from rdagent.oai.llm_utils import APIBackend
from rdagent.scenarios.data_science.experiment.experiment import COMPONENT, DSExperiment
from rdagent.scenarios.data_science.proposal.exp_gen.base import DSHypothesis, DSTrace
from rdagent.utils.agent.tpl import T
class DSDraftExpGen(ExpGen):
def _init_task_gen(
self,
targets: str,
scenario_desc: str,
task_output_format: str,
workspace_code: str | None = None,
spec: str = None,
hypothesis: Hypothesis | None = None,
exp_and_feedback_desc: str | None = None,
former_task: str | None = None,
) -> dict:
system_prompt = T(".prompts:task_gen.system").r(
targets=targets,
scenario=scenario_desc,
task_specification=spec,
hypothesis=hypothesis,
task_output_format=task_output_format,
)
user_prompt = T(".prompts:task_gen.user").r(
targets=targets,
hypothesis=hypothesis,
workspace_code=workspace_code,
exp_and_feedback_desc=exp_and_feedback_desc,
former_task_desc=former_task,
)
resp_dict = json.loads(
APIBackend().build_messages_and_create_chat_completion(
user_prompt=user_prompt, system_prompt=system_prompt, json_mode=True, json_target_type=dict
)
)
return resp_dict
def gen(
self,
component: COMPONENT,
trace: DSTrace,
) -> DSExperiment:
"""Handle any component using a unified approach.
Args:
component: Name of the component (e.g. "DataLoadSpec")
task_cls: The task class to instantiate (e.g. DataLoaderTask)
scenario_desc: Description of the current scenario
last_successful_exp: Last successful experiment or None
spec_file: Path to specification file if needed
selection: The selection of the node to generate the task
"""
last_successful_exp = trace.last_successful_exp()
# typecheck on the last successful exp, should be DSExperiment
if not isinstance(last_successful_exp, DSExperiment):
eda_output = None
else:
eda_output = last_successful_exp.experiment_workspace.file_dict.get("EDA.md", None)
scenario_desc = trace.scen.get_scenario_all_desc(eda_output=eda_output)
init_component_config = {
"DataLoadSpec": {"task_cls": DataLoaderTask, "spec_file": None, "component_prompt_key": "data_loader"},
"FeatureEng": {"task_cls": FeatureTask, "spec_file": "spec/feature.md", "component_prompt_key": "feature"},
"Model": {"task_cls": ModelTask, "spec_file": "spec/model.md", "component_prompt_key": "model"},
"Ensemble": {"task_cls": EnsembleTask, "spec_file": "spec/ensemble.md", "component_prompt_key": "ensemble"},
"Workflow": {"task_cls": WorkflowTask, "spec_file": "spec/workflow.md", "component_prompt_key": "workflow"},
}
task_cls = init_component_config[component]["task_cls"]
spec_file = init_component_config[component].get("spec_file")
component_prompt_key = init_component_config[component].get("component_prompt_key")
former_tasks_desc = ""
search_list = trace.retrieve_search_list()
if len(search_list) > 0:
for exp, fb in reversed(search_list):
if exp is not last_successful_exp:
former_task_desc = exp.pending_tasks_list[0][0].get_task_information()
former_task_desc += f"\n\nYou have tried to implement the same component and got the following exception: \n{fb.exception}\n Please try different methods to avoid the same errors and results in an infinite loop"
former_tasks_desc += former_task_desc
else:
break
if DS_RD_SETTING.spec_enabled:
spec = last_successful_exp.experiment_workspace.file_dict[spec_file] if spec_file else None
else:
spec = T(f"scenarios.data_science.share:component_spec.{component}").r()
resp_dict = self._init_task_gen(
targets=component,
scenario_desc=scenario_desc,
spec=spec,
task_output_format=T(f".prompts:output_format.{component_prompt_key or component.lower()}").r(),
former_task=former_tasks_desc if former_tasks_desc else None,
)
task = task_cls(
name=component if component != "Model" else resp_dict.pop("model_name"),
description=resp_dict.get("description", f"{component} description not provided"),
)
exp = DSExperiment(pending_tasks_list=[[task]], hypothesis=DSHypothesis(component))
if last_successful_exp:
# exp.experiment_workspace.inject_code_from_folder(last_successful_exp.experiment_workspace.workspace_path)
exp.experiment_workspace.inject_code_from_file_dict(last_successful_exp.experiment_workspace)
return exp
rdagent/scenarios/data_science/proposal/exp_gen/draft/draft.py
+283
View File
@@ -0,0 +1,283 @@
import json
from typing import TYPE_CHECKING, Any, Dict, List
from pydantic import BaseModel, Field
from rdagent.app.data_science.conf import DS_RD_SETTING
from rdagent.components.coder.data_science.ensemble.exp import EnsembleTask
from rdagent.components.coder.data_science.feature.exp import FeatureTask
from rdagent.components.coder.data_science.model.exp import ModelTask
from rdagent.components.coder.data_science.pipeline.exp import PipelineTask
from rdagent.components.coder.data_science.raw_data_loader.exp import DataLoaderTask
from rdagent.components.coder.data_science.workflow.exp import WorkflowTask
from rdagent.core.proposal import ExpGen, Hypothesis
from rdagent.log import rdagent_logger as logger
from rdagent.oai.llm_utils import APIBackend
from rdagent.scenarios.data_science.experiment.experiment import COMPONENT, DSExperiment
from rdagent.scenarios.data_science.proposal.exp_gen.base import DSHypothesis, DSTrace
from rdagent.scenarios.data_science.proposal.exp_gen.utils import (
CodingSketch,
get_component,
)
from rdagent.utils.agent.tpl import T
class DSDraftExpGen(ExpGen):
def _init_task_gen(
self,
targets: str,
scenario_desc: str,
task_output_format: str,
workspace_code: str | None = None,
spec: str = None,
hypothesis: Hypothesis | None = None,
exp_and_feedback_desc: str | None = None,
former_task: str | None = None,
) -> dict:
system_prompt = T(".prompts:task_gen.system").r(
targets=targets,
scenario=scenario_desc,
task_specification=spec,
hypothesis=hypothesis,
task_output_format=task_output_format,
)
user_prompt = T(".prompts:task_gen.user").r(
targets=targets,
hypothesis=hypothesis,
workspace_code=workspace_code,
exp_and_feedback_desc=exp_and_feedback_desc,
former_task_desc=former_task,
)
resp_dict = json.loads(
APIBackend().build_messages_and_create_chat_completion(
user_prompt=user_prompt, system_prompt=system_prompt, json_mode=True, json_target_type=dict
)
)
return resp_dict
def gen(
self,
component: COMPONENT,
trace: DSTrace,
) -> DSExperiment:
"""Handle any component using a unified approach.
Args:
component: Name of the component (e.g. "DataLoadSpec")
task_cls: The task class to instantiate (e.g. DataLoaderTask)
scenario_desc: Description of the current scenario
last_successful_exp: Last successful experiment or None
spec_file: Path to specification file if needed
selection: The selection of the node to generate the task
"""
last_successful_exp = trace.last_successful_exp()
# typecheck on the last successful exp, should be DSExperiment
if not isinstance(last_successful_exp, DSExperiment):
eda_output = None
else:
eda_output = last_successful_exp.experiment_workspace.file_dict.get("EDA.md", None)
scenario_desc = trace.scen.get_scenario_all_desc(eda_output=eda_output)
init_component_config = {
"DataLoadSpec": {"task_cls": DataLoaderTask, "spec_file": None, "component_prompt_key": "data_loader"},
"FeatureEng": {"task_cls": FeatureTask, "spec_file": "spec/feature.md", "component_prompt_key": "feature"},
"Model": {"task_cls": ModelTask, "spec_file": "spec/model.md", "component_prompt_key": "model"},
"Ensemble": {"task_cls": EnsembleTask, "spec_file": "spec/ensemble.md", "component_prompt_key": "ensemble"},
"Workflow": {"task_cls": WorkflowTask, "spec_file": "spec/workflow.md", "component_prompt_key": "workflow"},
}
task_cls = init_component_config[component]["task_cls"]
spec_file = init_component_config[component].get("spec_file")
component_prompt_key = init_component_config[component].get("component_prompt_key")
former_tasks_desc = ""
search_list = trace.retrieve_search_list()
if len(search_list) > 0:
for exp, fb in reversed(search_list):
if exp is not last_successful_exp:
former_task_desc = exp.pending_tasks_list[0][0].get_task_information()
former_task_desc += f"\n\nYou have tried to implement the same component and got the following exception: \n{fb.exception}\n Please try different methods to avoid the same errors and results in an infinite loop"
former_tasks_desc += former_task_desc
else:
break
if DS_RD_SETTING.spec_enabled:
spec = last_successful_exp.experiment_workspace.file_dict[spec_file] if spec_file else None
else:
spec = T(f"scenarios.data_science.share:component_spec.{component}").r()
resp_dict = self._init_task_gen(
targets=component,
scenario_desc=scenario_desc,
spec=spec,
task_output_format=T(f".prompts:output_format.{component_prompt_key or component.lower()}").r(),
former_task=former_tasks_desc if former_tasks_desc else None,
)
task = task_cls(
name=component if component != "Model" else resp_dict.pop("model_name"),
description=resp_dict.get("description", f"{component} description not provided"),
)
exp = DSExperiment(pending_tasks_list=[[task]], hypothesis=DSHypothesis(component))
if last_successful_exp:
# exp.experiment_workspace.inject_code_from_folder(last_successful_exp.experiment_workspace.workspace_path)
exp.experiment_workspace.inject_code_from_file_dict(last_successful_exp.experiment_workspace)
return exp
class DSDraftExpGenV2(ExpGen):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.support_function_calling = APIBackend().support_function_calling()
def tag_gen(self, scenario_desc: str) -> str:
sys_prompt = T(".prompts_draft:tag_gen.system").r(tag_desc=T(".prompts_draft:description.tag_description").r())
user_prompt = T(".prompts_draft:tag_gen.user").r(
scenario_desc=scenario_desc,
)
response = APIBackend().build_messages_and_create_chat_completion(
user_prompt=user_prompt,
system_prompt=sys_prompt,
json_mode=True,
json_target_type=Dict[str, str],
)
return json.loads(response)["tag"].lower()
def knowledge_gen(self) -> str:
runtime_environment = self.scen.get_runtime_environment()
general_knowledge = T(".prompts_draft:knowledge.general").r(
runtime_environment=runtime_environment,
component_desc=T(".prompts_draft:description.component_description").r(),
)
return f"{general_knowledge}"
def hypothesis_gen(
self,
knowledge: str,
component_desc: str,
scenario_desc: str,
failed_exp_feedback_list_desc: str,
) -> DSHypothesis:
sys_prompt = T(".prompts_draft:hypothesis_draft.system").r(component_desc=component_desc)
user_prompt = T(".prompts_draft:hypothesis_draft.user").r(
scenario_desc=scenario_desc,
knowledge=knowledge,
failed_exp_feedback_list_desc=failed_exp_feedback_list_desc,
)
response = APIBackend().build_messages_and_create_chat_completion(
user_prompt=user_prompt,
system_prompt=sys_prompt,
json_mode=True,
json_target_type=Dict[str, str],
)
resp_dict = json.loads(response)
return DSHypothesis(
component=resp_dict.get("component", "Model"),
hypothesis=resp_dict.get("hypothesis", "Hypothesis not provided"),
reason=resp_dict.get("reason", "Reason not provided"),
)
def task_gen(
self,
component_desc: str,
scenario_desc: str,
hypothesis: DSHypothesis,
pipeline: bool,
knowledge: str,
failed_exp_feedback_list_desc: str,
) -> DSExperiment:
if pipeline:
component_info = get_component("Pipeline")
else:
component_info = get_component(hypothesis.component)
data_folder_info = self.scen.processed_data_folder_description
sys_prompt = T(".prompts_draft:task_gen.system").r(
task_output_format=component_info["task_output_format"] if not self.support_function_calling else None,
component_desc=component_desc,
workflow_check=not pipeline and hypothesis.component != "Workflow",
)
user_prompt = T(".prompts_draft:task_gen.user").r(
scenario_desc=scenario_desc,
knowledge=knowledge,
data_folder_info=data_folder_info,
hypothesis=hypothesis,
failed_exp_and_feedback_list_desc=failed_exp_feedback_list_desc,
)
response = APIBackend().build_messages_and_create_chat_completion(
user_prompt=user_prompt,
system_prompt=sys_prompt,
response_format=CodingSketch if self.support_function_calling else {"type": "json_object"},
json_target_type=Dict[str, str | Dict[str, str]] if not self.support_function_calling else None,
)
task_dict = json.loads(response)
task_design = (
task_dict.get("task_design", {}) if not self.support_function_calling else task_dict.get("sketch", {})
)
logger.info(f"Task design:\n{task_design}")
task_name = hypothesis.component
description = (
task_design
if isinstance(task_design, str)
else task_design.get("description", f"{component_info['target_name']} description not provided")
)
task_class = component_info["task_class"]
task = task_class(
name=task_name,
description=description,
)
new_workflow_desc = task_dict.get("workflow_update", "No update needed")
exp = DSExperiment(pending_tasks_list=[[task]], hypothesis=hypothesis)
if not pipeline and new_workflow_desc != "No update needed":
workflow_task = WorkflowTask(
name="Workflow",
description=new_workflow_desc,
)
exp.pending_tasks_list.append([workflow_task])
return exp
def gen(self, trace: DSTrace) -> DSExperiment:
# Step 0: Prepare
pipeline = DS_RD_SETTING.coder_on_whole_pipeline
if pipeline:
component_desc = T("scenarios.data_science.share:component_description_in_pipeline").r()
else:
component_desc = "\n".join(
[
f"[{key}] {value}"
for key, value in T("scenarios.data_science.share:component_description").template.items()
]
)
last_exp = trace.last_exp()
if not isinstance(last_exp, DSExperiment):
eda_output = None
else:
eda_output = last_exp.experiment_workspace.file_dict.get("EDA.md", None)
scenario_desc = trace.scen.get_scenario_all_desc(eda_output=eda_output)
failed_exp_feedback_list_desc = T("scenarios.data_science.share:describe.trace").r(
exp_and_feedback_list=trace.experiment_and_feedback_list_after_init(return_type="failed"),
type="failed",
pipeline=pipeline,
)
# Step 1: Retrieve Knowledge
knowledge = self.knowledge_gen()
# Step 2: Generate Hypothesis based on General Knowledge
hypothesis = self.hypothesis_gen(
knowledge=knowledge,
component_desc=component_desc,
scenario_desc=scenario_desc,
failed_exp_feedback_list_desc=failed_exp_feedback_list_desc,
)
# Step 3: Design Task
return self.task_gen(
component_desc=component_desc,
scenario_desc=scenario_desc,
hypothesis=hypothesis,
failed_exp_feedback_list_desc=failed_exp_feedback_list_desc,
knowledge=knowledge,
pipeline=pipeline,
)
rdagent/scenarios/data_science/proposal/exp_gen/draft/prompts_draft.yaml
+252
View File
@@ -0,0 +1,252 @@
description:
tag_description: |-
[NLP]: Tasks involving natural language processing, such as text classification, sentiment analysis, or language modeling.
[CV]: Tasks involving computer vision, such as image classification, object detection, or segmentation.
[Tabular]: Tasks involving structured/tabular data, such as regression, classification, or time series forecasting.
component_description: |-
[DataPreprocess]: Loads raw data, handles missing values, type conversions, normalization, and ensures consistency. Includes validation, outlier detection, and cleaning for feature engineering.
[EDA]: Performs exploratory analysis to uncover data distributions, patterns, anomalies, and relationships. Generates summary statistics, visualizations, and initial hypotheses to guide processing.
[FeatureEngineer]: Transforms raw data into meaningful features via encoding, scaling, feature creation, and selection. Ensures reproducibility and robustness for modeling.
[Model]: Handles model selection, architecture design, training, validation, and evaluation. Ensures generalization and suitability for the problem.
[Ensemble]: Combines predictions from multiple models (averaging, stacking, blending) to improve robustness and generalization. Ensures model diversity and evaluates ensemble performance.
[Tuning]: Optimizes model and pipeline parameters using grid/random search or Bayesian methods. Maximizes validation performance while preventing overfitting.
knowledge:
general: |-
This is general techniques for data science tasks, aiming to ensure the pipeline runs **correctly, robustly, and reproducibly**.
## Runtime Environment
{{ runtime_environment }}
## Component Description
The following components are used to describe the task. Each component has a specific role in the data science pipeline, and they should be used to structure the task effectively.
{{ component_desc }}
## Component Guidelines
1. [DataPreprocess]
- This is the **foundation of the pipeline** and must be executed **first**.
- Ensure all **raw data is correctly loaded**, without missing files, broken paths, or incorrect dtypes.
- **Do not generate or fabricate synthetic data** unless explicitly allowed by competition rules.
- You must **traverse directories** to validate the presence of required files (such as images and data tables).
- Handle missing values, type casting, ID normalization, and consistent formats **before any modeling**.
- If this step fails or is skipped, all downstream steps are invalid.
2. [EDA] (Exploratory Data Analysis)
- Perform essential statistical summaries and visualization to understand **label distribution**, **feature correlation**, and **data quality**.
- Detect issues such as class imbalance, high-cardinality features, duplicates, or corrupted samples.
- Use EDA findings to form modeling hypotheses and choose sampling strategies.
3. [FeatureEngineer]
- Build features in **modular, traceable steps**.
- Begin with basic and interpretable features; add complex ones only when justified.
- Ensure reproducibility—avoid in-place mutation or random feature engineering without seeds.
4. [Model]
- Choose models suitable for the **data modality**, **dataset size**, and **available compute resources**.
- You may use larger models **as long as they can finish training within the time constraint**.
- Start with a **simple but realistic baseline** to verify pipeline correctness.
- Estimate optimal batch size via dry-runs or heuristics based on available resources.
- Ensure training time is acceptable with early stopping.
- Save best model checkpoints, log key metrics, and visualize learning curves.
5. [Tuning]
- Tune **only after verifying the pipeline and model correctness**.
- Use a small subset or minimal cross-validation to debug tuning logic before scaling up.
- Dynamically set parameters (such as batch size or epochs) based on observed resource usage.
- Set training duration to allow convergence without overfitting.
6. [Ensemble]
- Ensemble **only after** all base models are fully trained and validated.
- Prefer diverse models (e.g., different seeds, architectures, folds) to improve ensemble effectiveness.
- Keep the ensembling method **simple, reproducible**.
- Ensemble logic must not bypass earlier validation steps.
hypothesis_draft:
system: |-
{% include "scenarios.data_science.share:scen.role" %}
The user is about to draft the very first implementation for a Kaggle competition. There is no existing State-of-the-Art (SOTA) implementation yet—this is the initial baseline. The user will also be provided with a template implementation, which is distilled from successful approaches in other competitions and by GrandMasters.
You will be provided with:
1. A detailed competition scenario description.
2. A template implementation and guidelines, representing best practices and acknowledged knowledge from other top solutions.
3. A history of previous failed experiments and their associated feedbacks, chronologically ordered, where each failed experiment did not surpass the SOTA that was current at the time of its execution. The failed experiments are based on the current SOTA implementation and are used to propose hypotheses for further performance improvements.
Your task is to propose one specific, actionable, and testable hypothesis that will guide the creation of the first end-to-end implementation, leveraging the provided template as a starting point.
# Hypothesis Proposal for First Implementation
## Steps to Hypothesize
1. **Understand the Competition Context**:
- Carefully analyze the competition scenario description.
- Review the provided template implementation and guidelines, and identify any necessary adaptations for this specific competition.
- Refer to the template and guidelines for best practices and ensure alignment with recommended approaches.
- Prioritize hypotheses that ensure a successful, end-to-end runnable pipeline.
2. **Drafting the First Implementation**:
- Your hypothesis must focus on building the simplest possible, yet correct and runnable, baseline pipeline, using the provided template and guidelines as a foundation.
- Explicitly reference the template and guidelines when proposing adaptations or changes.
- The goal is to ensure the pipeline can execute end-to-end, generate a valid submission, and produce a baseline score.
- Avoid complex or multi-step solutions; do not combine unrelated techniques.
- Prioritize correctness, runnability, and adherence to competition requirements over performance or sophistication.
3. **Actionable and Testable**:
- The hypothesis must propose a clear, concrete action or adaptation that can be directly implemented and tested, especially in the context of the provided template and guidelines.
- It should specify the core model type, minimal preprocessing, and essential steps to produce a valid submission.
- If resource constraints are a concern, propose measures to ensure the pipeline completes within limits (e.g., use a lightweight model, reduce data size, limit epochs or folds).
## Guidelines for Writing Hypotheses
1. **Be Specific and Decisive**:
- Clearly state the exact change or approach for the first implementation, especially how the provided template and guidelines should be adapted.
- Reference specific sections or recommendations from the template and guidelines where relevant.
- Avoid vague statements or alternatives.
- The hypothesis must be more informative than simply restating the competition description or the template.
2. **Ensure Testability and Actionability**:
- The hypothesis should describe an action that can be implemented and validated in the first run.
- The expected outcome is a runnable, correct, and valid baseline pipeline.
3. **Align with Competition Requirements**:
- The hypothesis must directly address the competition's requirements.
- It should ensure the output files (e.g., submission.csv, scores.csv) are generated in the correct format.
4. **Maintain Singular Focus**:
- Propose only one core idea or change for the first implementation.
- Do not bundle unrelated ideas.
5. **Prioritize Runnability and Correctness**:
- The main goal is to get a working pipeline that produces a valid submission.
- Performance improvements can be addressed in future iterations.
## Component Tag
After proposing the hypothesis, assign a single component tag to the hypothesis.
Choose the **single most relevant** tag from the list below, even if the hypothesis appears to touch upon multiple areas. Use the following detailed descriptions to understand the scope and boundaries of each component.
{{ component_desc }}
## Final Output Format in JSON Schema:
For each of the identified problem, you should propose a hypothesis strictly following to the JSON schema. Your final output should be a dict containing all the proposed hypothesis.
{
"component": "The component tag of the hypothesis. Must be one of ('DataLoadSpec', 'FeatureEng', 'Model', 'Ensemble', 'Workflow').",
"hypothesis": "A concise, testable statement derived from previous experimental outcomes.",
"reason": "Provide a clear, logical progression from problem identification to hypothesis formulation, grounded in evidence (e.g., trace history, domain principles, or competition constraints). Refer to the Hypothesis Guidelines for better understanding.",
}
user: |-
# Scenario Description
{{ scenario_desc }}
# Template Implementation & Guidelines
{{ knowledge }}
# Previous Failed Experiments and Feedbacks
{{ failed_exp_feedback_list_desc }}
task_gen:
system: |-
{% include "scenarios.data_science.share:scen.role" %}
The user is about to draft the very first implementation for a Kaggle competition. There is no existing State-of-the-Art (SOTA) implementation yet—this is the initial baseline. The user will also be provided with a template implementation, which is distilled from successful approaches in other competitions and by GrandMasters.
You will be provided with:
1. A detailed competition scenario description.
2. A template implementation and guidelines, representing best practices and acknowledged knowledge from top solutions.
3. A history of previous failed experiments and their associated feedbacks, chronologically ordered, where each failed experiment did not surpass the SOTA that was current at the time of its execution. The failed experiments are based on the current SOTA implementation and are used to propose hypotheses for further performance improvements.
4. A proposed hypothesis, which aimed at forming the basis of an initial SOTA.
Your primary goal is to generate a detailed, step-by-step **sketch or refinement plan** for a new data processing and modeling pipeline, specifically for the main workflow script (`main.py`), that effectively implements the `Proposed Hypothesis`. This sketch will guide a developer to write the code correctly.
# Pipeline Implementation Standards & Constraints
The `main.py` sketch you generate should lead to a pipeline implementation that adheres to the following standards. These are guiding principles for the final *outcome* of your sketch:
1. **Program Execution**: The resulting `main.py` script must be executable via `python main.py` without command-line parameters. Configurations should be hardcoded for simplicity.
2. **File Handling**:
- Implement robust handling of file encodings and delimiters.
- Input files are under `{% include "scenarios.data_science.share:scen.input_path" %}`. The sketch must detail how they are loaded and, if multiple, combined or processed.
- Test indices must be determined from a dedicated test index file (if available) or by the order in the test data file. **Crucially, DO NOT use the sample submission file to infer test indices or the number of test samples.**
- Ensure actual data (not just filenames) is loaded during the data loading phase.
- If data is in zip files, the sketch should advise on robust loading, e.g., pre-extraction or careful handling if using multiprocessing in data loaders.
3. **Data Preprocessing**:
- Convert data to correct types (numeric, categorical, parse dates).
- Optimize memory usage (e.g., downcasting, chunk processing if essential and the hypothesis supports it).
- Implement domain-specific preprocessing relevant to the hypothesis (e.g., text tokenization, image resizing/augmentation).
4. **Code Standards**:
- The pipeline must **NOT** use progress bars (e.g., `tqdm`) in the submission code.
- Reiterate: **DO NOT** use the sample submission file to extract test indices or any other information beyond the required column names and format for the output file.
- Ensure no features are inadvertently excluded during processing.
5. **Preferred Technologies & Methodological Notes**:
- Tabular tasks: Default to LightGBM (LGB) as first choice. Use XGBoost (XGB) or CatBoost if the dataset involves time dependencies, sparse features, or heavy categorical interactions. Neural models (e.g., TabNet, FT-Transformer) can be added if the hypothesis explicitly requires them, but are not default.
- NLP tasks: Default to deBERTa V3 (Base or Large) if no other model is mandated by hypothesis. For classification or regression, prefer fine-tuning pretrained deBERTa models. Use lighter models (e.g., RoBERTa-base, BERT-base) if compute is limited. Use generative models (e.g., T5, GPT-style) only when required (e.g., summarization, generation).
- CV tasks: Use Swin Transformer (Base or Large) as the default choice for image-based tasks. If efficiency is a concern, prefer EfficientNetV2 or ConvNeXt-Tiny. Always use ImageNet pretrained weights and augmentations (e.g., RandAugment, CutMix) unless the hypothesis overrides them.
- If no SOTA is given and hypothesis is unclear, design the simplest working pipeline using these defaults to ensure a valid end-to-end run. Baselines must prioritize correctness, simplicity, and trainability over complexity.
- Once a correct and runnable pipeline is in place (i.e., no bugs, correct outputs, clean structure), all further development effort should focus on model selection, feature engineering, hyperparameter tuning, and ensemble strategies. These are the core levers of competitive performance.
6. **General Data Science Considerations**:
- Design for scalability.
- Handle missing values and outliers appropriately as guided by the hypothesis or SOTA.
- Ensure consistency between feature data types and any transformations applied.
- Prevent data leakage from test/validation sets into any training stage.
7. **Resource Utilization**: Leverage GPU and multiprocessing where appropriate and beneficial, if consistent with the hypothesis and efficiency goals.
8. **Metric Calculation and Storage (`scores.csv`)**:
- Calculate the official competition metric on a proper validation set (e.g., K-fold CV, typically 3-5 folds unless efficiency dictates fewer). Save results to `scores.csv`.
- The sketch must ensure this step is included. A successful run should always produce scores.
- `scores.csv` must have an index with model names and the literal string "ensemble" (lowercase). Columns should be "Model" (the name of the model or the ensemble strategy), and the exact metric name (e.g., "AUC").
- When only one model is used, its score should be present, and an "ensemble" score (which would be the same as the single model's score in this case) must also be recorded.
- Ensure validation metrics and processes are consistent across all parts of the pipeline. Avoid changes that would alter how validation metrics are calculated unless that is part of the hypothesis.
9. **Submission File (`submission.csv`)**: Generate `submission.csv` in the **exact format** required (column names, order, data types), as detailed by `sample_submission.csv` in the `Competition Scenario Description`. This is a critical step.
# Guidelines for Sketching the `main.py` Workflow
YOUR TASK IS TO create a conceptual sketch for drafting or updating the `main.py` workflow. This is a plan, not code.
1. **No Code**: The sketch **MUST NOT** contain any programming code, specific library calls, or pseudo-code. Describe steps conceptually (e.g., "Load training data from {% include "scenarios.data_science.share:scen.input_path" %}/train.csv"). List specific algorithm names where appropriate (e.g., "Apply XGBoost classifier," "Use Isotonic Regression for calibration").
2. **Structure and Conciseness**:
- If SOTA exists, understand its structure first.
- If no SOTA, outline a clear, logical sequence of steps for the new `main.py`.
3. **Leverage SOTA or Design a New One**:
- **If a `Current SOTA Implementation` is provided**: Your sketch must primarily detail the **minimal and targeted changes, additions, or replacements** needed to integrate the `Proposed Hypothesis` into that SOTA. Focus only on what needs to change.
- **If NO `Current SOTA Implementation` is provided (Initial Version)**: This is critical. Your sketch **MUST** describe a **COMPLETE, END-TO-END, YET SIMPLEST POSSIBLE baseline pipeline**.
- It must cover: Data loading (from specified paths), essential preprocessing (as per hypothesis or minimal viable), a basic model implementation (as per hypothesis), a simple validation strategy (e.g., a single train-validation split or fewer folds if CV is too complex initially), generation of `scores.csv`, and `submission.csv` in the correct format.
- The overriding goal for this initial sketch is **RUNNABILITY and CORRECTNESS of the pipeline structure**. Prioritize getting a valid submission out, even with a very basic model. Avoid any complexity not absolutely mandated by the core hypothesis or competition basics.
4. **Learn from Past Failures**:
- If `Previous Failed Experiments & Feedback` are provided, analyze them meticulously. Design the sketch to explicitly avoid repeating similar mistakes, especially if failures relate to the current hypothesis, data handling, submission format, or resource usage (timeouts).
- If a hypothesis aims to fix a past failure, the sketch should detail precisely how the fix is implemented.
5. **Specificity and Clarity**:
- Be unambiguous. Instead of "select model," if the hypothesis implies "Train an EfficientNet-B0 model," state that.
- The sketch must be definitive. No open-ended options or phrases like "for example," or "e.g.," within a step's action.
6. **Resource Constraints & Efficiency**:
- Always design the workflow to execute within the competition `Time Limit`.
- If `Previous Failed Experiments` explicitly state time/memory constraint issues, your sketch **MUST** make efficiency the **TOP PRIORITY**. Clearly state `[EFFICIENCY AS TOP PRIORITY]` at the beginning of your sketch.
- The sketch must then detail *specific measures* to achieve this (e.g., "Reduce CV folds to 2," "Limit training to 3 epochs," "Use a smaller pre-trained model like MobileNetV2," "Subsample training data to 50% if full dataset causes timeout").
- Even if the `Proposed Hypothesis` is not about efficiency, if past experiments failed due to timeouts or the dataset/model is complex, the sketch **must still incorporate measures to improve overall pipeline efficiency**. This might involve simplifying aspects unrelated to the core hypothesis (e.g., reducing image resolution, simpler feature engineering) to ensure the hypothesis can be tested within limits.
- The goal is a workflow that successfully implements and validates the `Proposed Hypothesis` effectively, balancing performance with strict resource constraints. An experiment that times out provides no information.
- If you plan to prioritize efficiency, you can modify the parts which is not related to the hypothesis. Which means your task should still able to validate the hypothesis.
- Add [EFFICIENCY AS PRIORITY] tag in the task description to indicate that the task takes efficiency as a priority.
- Although the task should prioritize efficiency, it should not be the only focus. The task should also be aligned with the proposed hypothesis and the current SOTA implementation.
7. **Reminders of Common Mistakes (Especially for New `main.py`)**: At the end of your sketch, include a "Key Reminders for Developer" section. Add the following reminders if appropriate.
- Ensure all input files are loaded from their exact paths under `{% include "scenarios.data_science.share:scen.input_path" %}` (e.g., `{% include "scenarios.data_science.share:scen.input_path" %}<competition_name>/train.csv`)."
- Verify `submission.csv` strictly adheres to format: columns, correct data types, and no extra index.
- "Implement correct label mapping for classification tasks (e.g., 0-indexed, contiguous integers for loss functions like PyTorch's CrossEntropyLoss) to prevent runtime errors."
- Handle file I/O robustly, especially for zipped data or large files, to prevent `FileNotFoundError` or `BadZipFile` issues.
- Confirm no `tqdm` or other progress bars are in the final script.
- Double-check that validation scores are saved correctly to `scores.csv` with specified 'Model' and metric columns, even for a single model run (include 'ensemble' row).
{% if task_output_format is not none %}
## [Partial Response Format 1] Task Output Format:
{{ task_output_format }}
{% if workflow_check %}
# Step 2: Workflow Update
Since components have dependencies, your second task is to update the workflow to reflect the changes made to the target component. Please also decide whether the workflow needs to be updated and provide a brief description of the change task.
{{ component_desc }}
[Partial Response Format 2] Your generated workflow description should be a simple text and the following agent will do the implementation. If you think the workflow should not be updated, just respond with "No update needed".
{% endif %}
Your final output should strictly adhere to the following JSON format.
{
"task_design": ---The dict corresponding to task output format---,
{% if workflow_check %}"workflow_update": ---A string corresponding to workflow description--- {% endif %}
}
{% endif %}
user: |-
# Competition Scenario Description
{{ scenario_desc }}
# Template Implementation & Guidelines
{{ knowledge }}
# Template Implementation & Guidelines
{{ knowledge }}
# Data Folder Structure (All files are under {% include "scenarios.data_science.share:scen.input_path" %})
{{ data_folder_info }}
# Proposed Hypothesis
This sketch should implement the following hypotheses:
Hypothesis: {{ hypothesis.hypothesis }}
Reason: {{ hypothesis.reason }}
# Previous Failed Experiments & Feedback
{{ failed_exp_and_feedback_list_desc }}
rdagent/scenarios/data_science/proposal/exp_gen/proposal.py
+3 -1
View File
@@ -19,7 +19,9 @@ from rdagent.oai.llm_utils import APIBackend, md5_hash
from rdagent.scenarios.data_science.dev.feedback import ExperimentFeedback
from rdagent.scenarios.data_science.experiment.experiment import DSExperiment
from rdagent.scenarios.data_science.proposal.exp_gen.base import DSHypothesis, DSTrace
from rdagent.scenarios.data_science.proposal.exp_gen.draft import DSDraftExpGen
from rdagent.scenarios.data_science.proposal.exp_gen.draft.draft import (
DSDraftExpGen, # TODO: DSDraftExpGen should be moved to router in the further
)
from rdagent.scenarios.data_science.proposal.exp_gen.idea_pool import DSIdea
from rdagent.utils.agent.tpl import T
from rdagent.utils.repo.diff import generate_diff_from_dict
rdagent/scenarios/data_science/proposal/exp_gen/router/__init__.py
+25
View File
@@ -0,0 +1,25 @@
from rdagent.app.data_science.conf import DS_RD_SETTING
from rdagent.core.proposal import ExpGen
from rdagent.scenarios.data_science.experiment.experiment import DSExperiment
from rdagent.scenarios.data_science.proposal.exp_gen.base import DSTrace
from rdagent.scenarios.data_science.proposal.exp_gen.draft.draft import DSDraftExpGenV2
from rdagent.scenarios.data_science.proposal.exp_gen.proposal import DSProposalV2ExpGen
class DraftRouterExpGen(ExpGen):
"""
A intelligent router for drafting and proposing.
"""
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.draft_exp_gen = DSDraftExpGenV2(self.scen)
self.base_exp_gen = DSProposalV2ExpGen(self.scen)
def gen(self, trace: DSTrace) -> DSExperiment:
pipeline = DS_RD_SETTING.coder_on_whole_pipeline
sota_exp = trace.sota_experiment()
if sota_exp is None and pipeline:
return self.draft_exp_gen.gen(trace)
return self.base_exp_gen.gen(trace)
rdagent/scenarios/data_science/proposal/exp_gen/utils.py
+1
View File
@@ -8,6 +8,7 @@ from rdagent.components.coder.data_science.model.exp import ModelTask
from rdagent.components.coder.data_science.pipeline.exp import PipelineTask
from rdagent.components.coder.data_science.raw_data_loader.exp import DataLoaderTask
from rdagent.components.coder.data_science.workflow.exp import WorkflowTask
from rdagent.utils.agent.tpl import T
_COMPONENT_META: Dict[str, Dict[str, Any]] = {
"DataLoadSpec": {