mirror of
https://github.com/NicolasBohn/NexQuant.git
synced 2026-08-01 01:27:42 +00:00
feat: new exp gen v2 implementation (#725)
* first framework commit * idea proposal v2 Co-authored-by: Roland Minrui <RolandMinrui@users.noreply.github.com> * fix a small bug in v1 * fix a small bug * add problem to DShypothesis * use exp gen as unified interface * merge yuante's code into pr * fix a small bug in draft * update all minrui's code * small update * fix small bug & remove useless code * fix return type * fix CI --------- Co-authored-by: Xu Yang <peteryang@vip.qq.com> Co-authored-by: Roland Minrui <RolandMinrui@users.noreply.github.com> Co-authored-by: Xu <v-xuminrui@microsoft.com> Co-authored-by: Xu Yang <xuyang1@microsoft.com>
This commit is contained in:
Roland Minrui
@@ -24,5 +24,7 @@ class DataScienceBasePropSetting(KaggleBasePropSetting):
|
||||
#### enable specification
|
||||
spec_enabled: bool = True
|
||||
|
||||
proposal_version: str = "v1"
|
||||
|
||||
|
||||
DS_RD_SETTING = DataScienceBasePropSetting()
|
||||
|
||||
@@ -80,6 +80,7 @@ class LiteLLMAPIBackend(APIBackend):
|
||||
if json_mode and supports_response_schema(model=LITELLM_SETTINGS.chat_model):
|
||||
kwargs["response_format"] = {"type": "json_object"}
|
||||
|
||||
logger.info(self._build_log_messages(messages), tag="llm_messages")
|
||||
# Call LiteLLM completion
|
||||
response = completion(
|
||||
model=LITELLM_SETTINGS.chat_model,
|
||||
@@ -93,7 +94,6 @@ class LiteLLMAPIBackend(APIBackend):
|
||||
f"{LogColors.GREEN}Using chat model{LogColors.END} {LITELLM_SETTINGS.chat_model}", tag="llm_messages"
|
||||
)
|
||||
|
||||
logger.info(self._build_log_messages(messages), tag="llm_messages")
|
||||
if LITELLM_SETTINGS.chat_stream:
|
||||
logger.info(f"{LogColors.BLUE}assistant:{LogColors.END}", tag="llm_messages")
|
||||
content = ""
|
||||
|
||||
@@ -77,6 +77,8 @@ class DSExperiment2Feedback(Experiment2Feedback):
|
||||
)
|
||||
)
|
||||
|
||||
# Currently, we do not use `observations`, `hypothesis_evaluation`, and `new_hypothesis` in the framework.
|
||||
# `new_hypothesis` should not exist in the feedback.
|
||||
return HypothesisFeedback(
|
||||
observations=resp_dict.get("Observations", "No observations provided"),
|
||||
hypothesis_evaluation=resp_dict.get("Feedback for Hypothesis", "No feedback provided"),
|
||||
|
||||
@@ -15,16 +15,14 @@ exp_feedback:
|
||||
Your feedback should:
|
||||
1. Confirm if the current result supports or refutes the hypothesis.
|
||||
2. Compare with previous best results.
|
||||
3. Suggest improvements or new directions. Stay innovative and adaptive.
|
||||
4. SOTA results are the best outcomes we have achieved in this scenario. If we do not have complete experiment available (i.e., results that are runnable and can generate evaluation outcomes), **please replace it as the best result/SOTA**.
|
||||
3. SOTA results are the best outcomes we have achieved in this scenario.
|
||||
|
||||
Please provide detailed and constructive feedback.
|
||||
Example JSON Structure for Result Analysis:
|
||||
{
|
||||
"Observations": "Your overall observations here",
|
||||
"Feedback for Hypothesis": "Observations related to the hypothesis",
|
||||
"New Hypothesis": "Your new hypothesis here",
|
||||
"Reasoning": "Reasoning for the new hypothesis",
|
||||
"Observations": "A detailed summary of the experimental results, including the description and scores for both SOTA and the current experiment. Limit this field to no more than three sentences, focusing on concrete data rather than general statements.",
|
||||
"Feedback for Hypothesis": "A brief evaluation of the original hypothesis that highlights specific data points or trends which support or contradict it. Limit this field to two sentences.",
|
||||
"Reasoning": "A clear explanation of why the current result performs better or worse than SOTA. This should reference both the SOTA description score and the current experiment's description score, providing insight into the factors contributing to the observed differences. Limit this field to one to three sentences.",
|
||||
"Replace Best Result": "yes or no"
|
||||
}
|
||||
|
||||
|
||||
@@ -1,475 +0,0 @@
|
||||
import json
|
||||
from typing import Dict, Literal
|
||||
|
||||
import pandas as pd
|
||||
|
||||
from rdagent.app.data_science.conf import DS_RD_SETTING
|
||||
from rdagent.components.coder.data_science.ensemble.exp import EnsembleTask
|
||||
from rdagent.components.coder.data_science.feature.exp import FeatureTask
|
||||
from rdagent.components.coder.data_science.model.exp import ModelTask
|
||||
from rdagent.components.coder.data_science.raw_data_loader.exp import DataLoaderTask
|
||||
from rdagent.components.coder.data_science.workflow.exp import WorkflowTask
|
||||
from rdagent.core.knowledge_base import KnowledgeBase
|
||||
from rdagent.core.proposal import ExperimentFeedback, ExpGen, Hypothesis, Trace
|
||||
from rdagent.oai.llm_utils import APIBackend
|
||||
from rdagent.scenarios.data_science.experiment.experiment import COMPONENT, DSExperiment
|
||||
from rdagent.scenarios.data_science.scen import DataScienceScen
|
||||
from rdagent.utils.agent.tpl import T
|
||||
from rdagent.utils.repo.diff import generate_diff_from_dict
|
||||
from rdagent.utils.workflow import wait_retry
|
||||
|
||||
|
||||
class DSHypothesis(Hypothesis):
|
||||
def __init__(
|
||||
self,
|
||||
component: COMPONENT,
|
||||
hypothesis: str = "",
|
||||
reason: str = "",
|
||||
concise_reason: str = "",
|
||||
concise_observation: str = "",
|
||||
concise_justification: str = "",
|
||||
concise_knowledge: str = "",
|
||||
) -> None:
|
||||
super().__init__(
|
||||
hypothesis, reason, concise_reason, concise_observation, concise_justification, concise_knowledge
|
||||
)
|
||||
self.component = component
|
||||
|
||||
def __str__(self) -> str:
|
||||
if self.hypothesis == "":
|
||||
return f"No hypothesis available. Trying to construct the first runnable {self.component} component."
|
||||
return f"""Chosen Component: {self.component}
|
||||
Hypothesis: {self.hypothesis}
|
||||
Reason: {self.reason}
|
||||
Concise Reason & Knowledge: {self.concise_reason}
|
||||
Concise Observation: {self.concise_observation}
|
||||
Concise Justification: {self.concise_justification}
|
||||
Concise Knowledge: {self.concise_knowledge}
|
||||
"""
|
||||
|
||||
|
||||
COMPONENT_TASK_MAPPING = {
|
||||
"DataLoadSpec": {
|
||||
"target_name": "Data loader and specification generation",
|
||||
"spec_file": "spec/data_loader.md",
|
||||
"task_output_format": T(".prompts:output_format.data_loader").r(),
|
||||
"task_class": DataLoaderTask,
|
||||
},
|
||||
"FeatureEng": {
|
||||
"target_name": "Feature engineering",
|
||||
"spec_file": "spec/feature.md",
|
||||
"task_output_format": T(".prompts:output_format.feature").r(),
|
||||
"task_class": FeatureTask,
|
||||
},
|
||||
"Model": {
|
||||
"target_name": "Model",
|
||||
"spec_file": "spec/model.md",
|
||||
"task_output_format": T(".prompts:output_format.model").r(),
|
||||
"task_class": ModelTask,
|
||||
},
|
||||
"Ensemble": {
|
||||
"target_name": "Ensemble",
|
||||
"spec_file": "spec/ensemble.md",
|
||||
"task_output_format": T(".prompts:output_format.ensemble").r(),
|
||||
"task_class": EnsembleTask,
|
||||
},
|
||||
"Workflow": {
|
||||
"target_name": "Workflow",
|
||||
"spec_file": "spec/workflow.md",
|
||||
"task_output_format": T(".prompts:output_format.workflow").r(),
|
||||
"task_class": WorkflowTask,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
class DSTrace(Trace[DataScienceScen, KnowledgeBase]):
|
||||
def __init__(self, scen: DataScienceScen, knowledge_base: KnowledgeBase | None = None) -> None:
|
||||
self.scen: DataScienceScen = scen
|
||||
self.hist: list[tuple[DSExperiment, ExperimentFeedback]] = []
|
||||
self.knowledge_base = knowledge_base
|
||||
|
||||
COMPLETE_ORDER = ("DataLoadSpec", "FeatureEng", "Model", "Ensemble", "Workflow")
|
||||
|
||||
def next_incomplete_component(self) -> COMPONENT | None:
|
||||
"""
|
||||
NOTE:
|
||||
- A component will be complete until get True decision feedback !!!
|
||||
"""
|
||||
for c in self.COMPLETE_ORDER:
|
||||
if not self.has_component(c):
|
||||
return c
|
||||
return None
|
||||
|
||||
def has_component(self, component: COMPONENT) -> bool:
|
||||
for exp, fb in self.hist:
|
||||
assert isinstance(exp.hypothesis, DSHypothesis), "Hypothesis should be DSHypothesis (and not None)"
|
||||
if exp.hypothesis.component == component and fb:
|
||||
return True
|
||||
return False
|
||||
|
||||
def experiment_and_feedback_list_after_init(
|
||||
self, return_type: Literal["sota", "failed", "all"]
|
||||
) -> list[tuple[DSExperiment, ExperimentFeedback]]:
|
||||
"""
|
||||
Retrieve a list of experiments and feedbacks based on the return_type.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
return_type : str
|
||||
One of "sota", "failed", "all".
|
||||
|
||||
Returns
|
||||
-------
|
||||
list[tuple[DSExperiment, ExperimentFeedback]]
|
||||
List of experiments and feedbacks.
|
||||
"""
|
||||
|
||||
final_component = self.COMPLETE_ORDER[-1]
|
||||
has_final_component = False
|
||||
exp_and_feedback_list = []
|
||||
for exp, fb in self.hist:
|
||||
if has_final_component:
|
||||
if return_type == "all":
|
||||
exp_and_feedback_list.append((exp, fb))
|
||||
elif return_type == "failed" and not fb.decision:
|
||||
exp_and_feedback_list.append((exp, fb))
|
||||
elif return_type == "sota" and fb.decision:
|
||||
exp_and_feedback_list.append((exp, fb))
|
||||
if exp.hypothesis.component == final_component and fb:
|
||||
has_final_component = True
|
||||
return exp_and_feedback_list
|
||||
|
||||
def sota_experiment(self) -> DSExperiment | None:
|
||||
"""
|
||||
Returns
|
||||
-------
|
||||
Experiment or None
|
||||
The experiment result if found, otherwise None.
|
||||
"""
|
||||
if self.next_incomplete_component() is None:
|
||||
for exp, ef in self.hist[::-1]:
|
||||
# the sota exp should be accepted decision and all required components are completed.
|
||||
if ef.decision:
|
||||
return exp
|
||||
return None
|
||||
|
||||
def last_successful_exp(self) -> DSExperiment | None:
|
||||
"""
|
||||
Access the last successful experiment even part of the components are not completed.
|
||||
"""
|
||||
for exp, ef in self.hist[::-1]:
|
||||
if ef.decision:
|
||||
return exp
|
||||
return None
|
||||
|
||||
def last_runnable_exp_fb(self) -> tuple[DSExperiment, ExperimentFeedback] | None:
|
||||
"""
|
||||
Access the last runnable experiment (no exception, usually not all task failed) and feedback
|
||||
"""
|
||||
for exp, ef in self.hist[::-1]:
|
||||
if ef.exception is None:
|
||||
return exp, ef
|
||||
return None
|
||||
|
||||
|
||||
class DSExpGen(ExpGen):
|
||||
"""Data Science Task Generator."""
|
||||
|
||||
def __init__(self, scen: DataScienceScen, max_trace_hist: int = 3) -> None:
|
||||
self.max_trace_hist = max_trace_hist # max number of historical trace to know when propose new experiment
|
||||
super().__init__(scen)
|
||||
|
||||
def _init_task_gen(
|
||||
self,
|
||||
targets: str,
|
||||
scenario_desc: str,
|
||||
task_output_format: str,
|
||||
workspace_code: str | None = None,
|
||||
spec: str = None,
|
||||
hypothesis: Hypothesis | None = None,
|
||||
exp_and_feedback_desc: str | None = None,
|
||||
former_task: str | None = None,
|
||||
) -> dict:
|
||||
system_prompt = T(".prompts:task_gen.system").r(
|
||||
targets=targets,
|
||||
scenario=scenario_desc,
|
||||
task_specification=spec,
|
||||
hypothesis=hypothesis,
|
||||
task_output_format=task_output_format,
|
||||
)
|
||||
user_prompt = T(".prompts:task_gen.user").r(
|
||||
targets=targets,
|
||||
hypothesis=hypothesis,
|
||||
workspace_code=workspace_code,
|
||||
exp_and_feedback_desc=exp_and_feedback_desc,
|
||||
former_task_desc=former_task,
|
||||
)
|
||||
|
||||
resp_dict = json.loads(
|
||||
APIBackend().build_messages_and_create_chat_completion(
|
||||
user_prompt=user_prompt, system_prompt=system_prompt, json_mode=True, json_target_type=dict
|
||||
)
|
||||
)
|
||||
|
||||
return resp_dict
|
||||
|
||||
def _handle_missing_component(
|
||||
self,
|
||||
component: COMPONENT,
|
||||
task_cls: type,
|
||||
scenario_desc: str,
|
||||
trace: Trace,
|
||||
last_successful_exp: DSExperiment | None,
|
||||
spec_file: str | None = None,
|
||||
component_prompt_key: str | None = None,
|
||||
) -> DSExperiment:
|
||||
"""Handle any component using a unified approach.
|
||||
|
||||
Args:
|
||||
component: Name of the component (e.g. "DataLoadSpec")
|
||||
task_cls: The task class to instantiate (e.g. DataLoaderTask)
|
||||
scenario_desc: Description of the current scenario
|
||||
last_successful_exp: Last successful experiment or None
|
||||
spec_file: Path to specification file if needed
|
||||
"""
|
||||
|
||||
former_tasks_desc = ""
|
||||
if len(trace.hist) > 0:
|
||||
for exp, fb in reversed(trace.hist):
|
||||
if exp is not last_successful_exp:
|
||||
former_task_desc = exp.pending_tasks_list[0][0].get_task_information()
|
||||
former_task_desc += f"\n\nYou have tried to implement the same component and got the following exception: \n{fb.exception}\n Please try different methods to avoid the same errors and results in an infinite loop"
|
||||
former_tasks_desc += former_task_desc
|
||||
else:
|
||||
break
|
||||
|
||||
if DS_RD_SETTING.spec_enabled:
|
||||
spec = last_successful_exp.experiment_workspace.file_dict[spec_file] if spec_file else None
|
||||
else:
|
||||
spec = T(f"scenarios.data_science.share:component_spec.{component}").r()
|
||||
resp_dict = self._init_task_gen(
|
||||
targets=component,
|
||||
scenario_desc=scenario_desc,
|
||||
spec=spec,
|
||||
task_output_format=T(f".prompts:output_format.{component_prompt_key or component.lower()}").r(),
|
||||
former_task=former_tasks_desc if former_tasks_desc else None,
|
||||
)
|
||||
|
||||
task = task_cls(
|
||||
name=component if component != "Model" else resp_dict.pop("model_name"),
|
||||
description=resp_dict.get("description", f"{component} description not provided"),
|
||||
)
|
||||
|
||||
exp = DSExperiment(pending_tasks_list=[[task]], hypothesis=DSHypothesis(component))
|
||||
if last_successful_exp:
|
||||
# exp.experiment_workspace.inject_code_from_folder(last_successful_exp.experiment_workspace.workspace_path)
|
||||
exp.experiment_workspace.inject_code_from_file_dict(last_successful_exp.experiment_workspace)
|
||||
return exp
|
||||
|
||||
def gen(self, trace: DSTrace) -> DSExperiment:
|
||||
scenario_desc = trace.scen.get_scenario_all_desc()
|
||||
last_successful_exp = trace.last_successful_exp()
|
||||
|
||||
next_missing_component = trace.next_incomplete_component()
|
||||
|
||||
init_component_config = {
|
||||
"DataLoadSpec": {"task_cls": DataLoaderTask, "spec_file": None, "component_prompt_key": "data_loader"},
|
||||
"FeatureEng": {"task_cls": FeatureTask, "spec_file": "spec/feature.md", "component_prompt_key": "feature"},
|
||||
"Model": {"task_cls": ModelTask, "spec_file": "spec/model.md", "component_prompt_key": "model"},
|
||||
"Ensemble": {"task_cls": EnsembleTask, "spec_file": "spec/ensemble.md", "component_prompt_key": "ensemble"},
|
||||
"Workflow": {"task_cls": WorkflowTask, "spec_file": "spec/workflow.md", "component_prompt_key": "workflow"},
|
||||
}
|
||||
|
||||
if next_missing_component in init_component_config:
|
||||
# TODO: we may merge the if else logic in the future.
|
||||
# the current
|
||||
config = init_component_config[next_missing_component]
|
||||
return self._handle_missing_component(
|
||||
component=next_missing_component,
|
||||
task_cls=config["task_cls"],
|
||||
scenario_desc=scenario_desc,
|
||||
last_successful_exp=last_successful_exp,
|
||||
spec_file=config.get("spec_file"),
|
||||
trace=trace,
|
||||
component_prompt_key=config.get("component_prompt_key"),
|
||||
)
|
||||
else: # propose new component by LLM
|
||||
# Guidelines:
|
||||
# System prompts: Shared condition you are facing
|
||||
# - scenario description: `scenario_desc`
|
||||
# - expected output format
|
||||
# User prompts: Task Specific information
|
||||
# - Previous Feedback
|
||||
# - Current sota implementation (encourage change based on it)
|
||||
# - Extra RAG
|
||||
sota_exp = trace.sota_experiment()
|
||||
assert sota_exp is not None, "SOTA experiment is not provided."
|
||||
exp_and_feedback = trace.hist[-1]
|
||||
last_exp = exp_and_feedback[0]
|
||||
|
||||
# Step 1: Generate component
|
||||
# Describe current best solution using shared template
|
||||
sota_exp_desc = T("scenarios.data_science.share:describe.exp").r(
|
||||
exp=sota_exp, heading="Best of previous exploration of the scenario"
|
||||
)
|
||||
last_exp_diff = "\n".join(
|
||||
generate_diff_from_dict(
|
||||
sota_exp.experiment_workspace.file_dict, last_exp.experiment_workspace.file_dict
|
||||
)
|
||||
) # we use file_dict for hitting the cache when replicate the experiment in another machine.
|
||||
|
||||
sota_exp_feedback_list = trace.experiment_and_feedback_list_after_init(return_type="sota")
|
||||
failed_exp_feedback_list = trace.experiment_and_feedback_list_after_init(return_type="failed")[
|
||||
-self.max_trace_hist :
|
||||
]
|
||||
all_exp_feedback_list = trace.experiment_and_feedback_list_after_init(return_type="all")
|
||||
trace_component_to_feedback_df = pd.DataFrame(columns=["component", "hypothesis", "decision"])
|
||||
for index, (exp, fb) in enumerate(all_exp_feedback_list):
|
||||
trace_component_to_feedback_df.loc[f"trial {index + 1}"] = [
|
||||
exp.hypothesis.component,
|
||||
exp.hypothesis.hypothesis,
|
||||
fb.decision,
|
||||
]
|
||||
|
||||
sota_exp_feedback_list_desc = T("scenarios.data_science.share:describe.trace").r(
|
||||
exp_and_feedback_list=sota_exp_feedback_list,
|
||||
success=True,
|
||||
)
|
||||
failed_exp_feedback_list_desc = T("scenarios.data_science.share:describe.trace").r(
|
||||
exp_and_feedback_list=failed_exp_feedback_list,
|
||||
success=False,
|
||||
)
|
||||
|
||||
# Generate component using template with proper context
|
||||
component_sys_prompt = T(".prompts:component_gen.system").r(
|
||||
scenario=scenario_desc,
|
||||
sota_exp_desc=sota_exp_desc,
|
||||
last_exp_diff=last_exp_diff,
|
||||
component_desc="\n".join(
|
||||
[
|
||||
f"[{key}] {value}"
|
||||
for key, value in T("scenarios.data_science.share:component_description").template.items()
|
||||
]
|
||||
),
|
||||
component_output_format=T(".prompts:output_format.component").r(),
|
||||
)
|
||||
|
||||
component_user_prompt = T(".prompts:component_gen.user").r(
|
||||
sota_exp_and_feedback_list_desc=sota_exp_feedback_list_desc,
|
||||
failed_exp_and_feedback_list_desc=failed_exp_feedback_list_desc,
|
||||
component_and_feedback_df=(
|
||||
trace_component_to_feedback_df.to_string()
|
||||
if len(trace_component_to_feedback_df) > 0
|
||||
else "No experiment and feedback provided"
|
||||
),
|
||||
)
|
||||
|
||||
resp_dict_component: dict = json.loads(
|
||||
APIBackend().build_messages_and_create_chat_completion(
|
||||
component_user_prompt, component_sys_prompt, json_mode=True, json_target_type=Dict[str, str]
|
||||
)
|
||||
)
|
||||
|
||||
component = resp_dict_component.get("component", "Component not provided")
|
||||
component_reason = resp_dict_component.get("reason", "Reason not provided")
|
||||
sota_exp_model_file_count = len(
|
||||
[
|
||||
k
|
||||
for k in sota_exp.experiment_workspace.file_dict.keys()
|
||||
if k.endswith(".py") and "test" not in k and k.startswith("model")
|
||||
]
|
||||
)
|
||||
if sota_exp_model_file_count <= 1 and component == "Ensemble":
|
||||
component = "Model"
|
||||
|
||||
# Why we should split component selection and steps after?
|
||||
# - after we know the selected component, we can use RAG.
|
||||
|
||||
# Step 2: Generate the rest of the hypothesis & task
|
||||
component_info = COMPONENT_TASK_MAPPING.get(component)
|
||||
|
||||
if component_info:
|
||||
if DS_RD_SETTING.spec_enabled:
|
||||
task_spec = sota_exp.experiment_workspace.file_dict[component_info["spec_file"]]
|
||||
else:
|
||||
task_spec = T(f"scenarios.data_science.share:component_spec.{component}").r()
|
||||
system_prompt = T(".prompts:direct_exp_gen.system").r(
|
||||
targets=component_info["target_name"],
|
||||
component=component,
|
||||
scenario=scenario_desc,
|
||||
hypothesis_specification=T(".prompts:hypothesis_specification").r(),
|
||||
hypothesis_output_format=T(".prompts:output_format.hypothesis").r(),
|
||||
task_specification=task_spec,
|
||||
task_output_format=component_info["task_output_format"],
|
||||
workflow_check=(not component == "Workflow"),
|
||||
)
|
||||
|
||||
user_prompt = T(".prompts:direct_exp_gen.user").r(
|
||||
targets=component_info["target_name"],
|
||||
sota_exp_desc=sota_exp_desc,
|
||||
sota_exp_and_feedback_list_desc=sota_exp_feedback_list_desc,
|
||||
failed_exp_and_feedback_list_desc=failed_exp_feedback_list_desc,
|
||||
last_exp_diff=last_exp_diff,
|
||||
)
|
||||
|
||||
def _append_retry(args: tuple, kwargs: dict) -> tuple[tuple, dict]:
|
||||
# Only modify the user_prompt on retries (i > 0)
|
||||
user_prompt = args[0]
|
||||
user_prompt += "\n\nretrying..."
|
||||
return (user_prompt,), kwargs
|
||||
|
||||
@wait_retry(retry_n=5, transform_args_fn=_append_retry)
|
||||
def _f(user_prompt):
|
||||
resp_dict = json.loads(
|
||||
APIBackend().build_messages_and_create_chat_completion(
|
||||
user_prompt=user_prompt,
|
||||
system_prompt=system_prompt,
|
||||
json_mode=True,
|
||||
# NOTE: corner cases.
|
||||
# workflow_update may be a string
|
||||
# model could have 2 level nested dict.
|
||||
json_target_type=dict[str, dict[str, str | dict] | str],
|
||||
)
|
||||
)
|
||||
assert "hypothesis_proposal" in resp_dict, "Hypothesis proposal not provided."
|
||||
assert "task_design" in resp_dict, "Task design not provided."
|
||||
task_class = component_info["task_class"]
|
||||
hypothesis_proposal = resp_dict.get("hypothesis_proposal", {})
|
||||
hypothesis = DSHypothesis(
|
||||
component=component,
|
||||
hypothesis=hypothesis_proposal.get("hypothesis", ""),
|
||||
reason=component_reason + "\n" + hypothesis_proposal.get("reason", ""),
|
||||
concise_reason=hypothesis_proposal.get("concise_reason", ""),
|
||||
concise_observation=hypothesis_proposal.get("concise_observation", ""),
|
||||
concise_justification=hypothesis_proposal.get("concise_justification", ""),
|
||||
concise_knowledge=hypothesis_proposal.get("concise_knowledge", ""),
|
||||
)
|
||||
|
||||
task_design = resp_dict.get("task_design", {})
|
||||
task_name = task_design["model_name"] if component == "Model" else component
|
||||
description = task_design.get(
|
||||
"description", f"{component_info['target_name']} description not provided"
|
||||
)
|
||||
task = task_class(
|
||||
name=task_name,
|
||||
description=description,
|
||||
**{k: task_design.get(k, v) for k, v in component_info.get("extra_params", {}).items()},
|
||||
)
|
||||
new_workflow_desc = resp_dict.get("workflow_update", "No update needed")
|
||||
return hypothesis, task, new_workflow_desc
|
||||
|
||||
hypothesis, task, new_workflow_desc = _f(user_prompt)
|
||||
|
||||
exp = DSExperiment(pending_tasks_list=[[task]], hypothesis=hypothesis)
|
||||
# exp.experiment_workspace.inject_code_from_folder(sota_exp.experiment_workspace.workspace_path)
|
||||
exp.experiment_workspace.inject_code_from_file_dict(sota_exp.experiment_workspace)
|
||||
|
||||
if new_workflow_desc != "No update needed":
|
||||
workflow_task = WorkflowTask(
|
||||
name="Workflow",
|
||||
description=new_workflow_desc,
|
||||
)
|
||||
exp.pending_tasks_list.append([workflow_task])
|
||||
return exp
|
||||
else:
|
||||
raise ValueError(f"Unknown component: {component}")
|
||||
@@ -0,0 +1,36 @@
|
||||
from rdagent.app.data_science.conf import DS_RD_SETTING
|
||||
from rdagent.core.proposal import ExpGen
|
||||
from rdagent.scenarios.data_science.experiment.experiment import DSExperiment
|
||||
from rdagent.scenarios.data_science.proposal.exp_gen.base import DSTrace
|
||||
from rdagent.scenarios.data_science.proposal.exp_gen.draft import DSDraftExpGen
|
||||
from rdagent.scenarios.data_science.proposal.exp_gen.proposal import (
|
||||
DSProposalV1ExpGen,
|
||||
DSProposalV2ExpGen,
|
||||
)
|
||||
from rdagent.scenarios.data_science.scen import DataScienceScen
|
||||
|
||||
|
||||
class DSExpGen(ExpGen):
|
||||
"""Data Science Task Generator."""
|
||||
|
||||
def __init__(self, scen: DataScienceScen, max_trace_hist: int = 3) -> None:
|
||||
self.max_trace_hist = max_trace_hist # max number of historical trace to know when propose new experiment
|
||||
super().__init__(scen)
|
||||
|
||||
def gen(self, trace: DSTrace) -> DSExperiment:
|
||||
next_missing_component = trace.next_incomplete_component()
|
||||
if next_missing_component is not None:
|
||||
return DSDraftExpGen(scen=self.scen).gen(
|
||||
component=next_missing_component,
|
||||
trace=trace,
|
||||
)
|
||||
if DS_RD_SETTING.proposal_version == "v1":
|
||||
return DSProposalV1ExpGen(scen=self.scen).gen(
|
||||
trace=trace,
|
||||
max_trace_hist=self.max_trace_hist,
|
||||
)
|
||||
if DS_RD_SETTING.proposal_version == "v2":
|
||||
return DSProposalV2ExpGen(scen=self.scen).gen(
|
||||
trace=trace,
|
||||
max_trace_hist=self.max_trace_hist,
|
||||
)
|
||||
@@ -0,0 +1,127 @@
|
||||
from abc import abstractmethod
|
||||
from typing import Literal
|
||||
|
||||
from rdagent.core.evolving_framework import KnowledgeBase
|
||||
from rdagent.core.proposal import ExperimentFeedback, Hypothesis, Trace
|
||||
from rdagent.scenarios.data_science.experiment.experiment import COMPONENT, DSExperiment
|
||||
from rdagent.scenarios.data_science.scen import DataScienceScen
|
||||
|
||||
|
||||
class DSHypothesis(Hypothesis):
|
||||
def __init__(
|
||||
self,
|
||||
component: COMPONENT,
|
||||
hypothesis: str = "",
|
||||
reason: str = "",
|
||||
concise_reason: str = "",
|
||||
concise_observation: str = "",
|
||||
concise_justification: str = "",
|
||||
concise_knowledge: str = "",
|
||||
problem: str = "",
|
||||
) -> None:
|
||||
super().__init__(
|
||||
hypothesis, reason, concise_reason, concise_observation, concise_justification, concise_knowledge
|
||||
)
|
||||
self.component = component
|
||||
self.problem = problem
|
||||
|
||||
def __str__(self) -> str:
|
||||
if self.hypothesis == "":
|
||||
return f"No hypothesis available. Trying to construct the first runnable {self.component} component."
|
||||
lines = []
|
||||
if self.problem is not None:
|
||||
lines.append(f"Target Problem: {self.problem}")
|
||||
lines.extend(
|
||||
[f"Chosen Component: {self.component}", f"Hypothesis: {self.hypothesis}", f"Reason: {self.reason}"]
|
||||
)
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
class DSTrace(Trace[DataScienceScen, KnowledgeBase]):
|
||||
def __init__(self, scen: DataScienceScen, knowledge_base: KnowledgeBase | None = None) -> None:
|
||||
self.scen: DataScienceScen = scen
|
||||
self.hist: list[tuple[DSExperiment, ExperimentFeedback]] = []
|
||||
self.knowledge_base = knowledge_base
|
||||
|
||||
COMPLETE_ORDER = ("DataLoadSpec", "FeatureEng", "Model", "Ensemble", "Workflow")
|
||||
|
||||
def next_incomplete_component(self) -> COMPONENT | None:
|
||||
"""
|
||||
NOTE:
|
||||
- A component will be complete until get True decision feedback !!!
|
||||
"""
|
||||
for c in self.COMPLETE_ORDER:
|
||||
if not self.has_component(c):
|
||||
return c
|
||||
return None
|
||||
|
||||
def has_component(self, component: COMPONENT) -> bool:
|
||||
for exp, fb in self.hist:
|
||||
assert isinstance(exp.hypothesis, DSHypothesis), "Hypothesis should be DSHypothesis (and not None)"
|
||||
if exp.hypothesis.component == component and fb:
|
||||
return True
|
||||
return False
|
||||
|
||||
def experiment_and_feedback_list_after_init(
|
||||
self, return_type: Literal["sota", "failed", "all"]
|
||||
) -> list[tuple[DSExperiment, ExperimentFeedback]]:
|
||||
"""
|
||||
Retrieve a list of experiments and feedbacks based on the return_type.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
return_type : str
|
||||
One of "sota", "failed", "all".
|
||||
|
||||
Returns
|
||||
-------
|
||||
list[tuple[DSExperiment, ExperimentFeedback]]
|
||||
List of experiments and feedbacks.
|
||||
"""
|
||||
|
||||
final_component = self.COMPLETE_ORDER[-1]
|
||||
has_final_component = False
|
||||
exp_and_feedback_list = []
|
||||
for exp, fb in self.hist:
|
||||
if has_final_component:
|
||||
if return_type == "all":
|
||||
exp_and_feedback_list.append((exp, fb))
|
||||
elif return_type == "failed" and not fb.decision:
|
||||
exp_and_feedback_list.append((exp, fb))
|
||||
elif return_type == "sota" and fb.decision:
|
||||
exp_and_feedback_list.append((exp, fb))
|
||||
if exp.hypothesis.component == final_component and fb:
|
||||
has_final_component = True
|
||||
return exp_and_feedback_list
|
||||
|
||||
def sota_experiment(self) -> DSExperiment | None:
|
||||
"""
|
||||
Returns
|
||||
-------
|
||||
Experiment or None
|
||||
The experiment result if found, otherwise None.
|
||||
"""
|
||||
if self.next_incomplete_component() is None:
|
||||
for exp, ef in self.hist[::-1]:
|
||||
# the sota exp should be accepted decision and all required components are completed.
|
||||
if ef.decision:
|
||||
return exp
|
||||
return None
|
||||
|
||||
def last_successful_exp(self) -> DSExperiment | None:
|
||||
"""
|
||||
Access the last successful experiment even part of the components are not completed.
|
||||
"""
|
||||
for exp, ef in self.hist[::-1]:
|
||||
if ef.decision:
|
||||
return exp
|
||||
return None
|
||||
|
||||
def last_runnable_exp_fb(self) -> tuple[DSExperiment, ExperimentFeedback] | None:
|
||||
"""
|
||||
Access the last runnable experiment (no exception, usually not all task failed) and feedback
|
||||
"""
|
||||
for exp, ef in self.hist[::-1]:
|
||||
if ef.exception is None:
|
||||
return exp, ef
|
||||
return None
|
||||
@@ -0,0 +1,111 @@
|
||||
import json
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from rdagent.app.data_science.conf import DS_RD_SETTING
|
||||
from rdagent.components.coder.data_science.ensemble.exp import EnsembleTask
|
||||
from rdagent.components.coder.data_science.feature.exp import FeatureTask
|
||||
from rdagent.components.coder.data_science.model.exp import ModelTask
|
||||
from rdagent.components.coder.data_science.raw_data_loader.exp import DataLoaderTask
|
||||
from rdagent.components.coder.data_science.workflow.exp import WorkflowTask
|
||||
from rdagent.core.proposal import ExpGen, Hypothesis
|
||||
from rdagent.oai.llm_utils import APIBackend
|
||||
from rdagent.scenarios.data_science.experiment.experiment import COMPONENT, DSExperiment
|
||||
from rdagent.scenarios.data_science.proposal.exp_gen.base import DSHypothesis, DSTrace
|
||||
from rdagent.utils.agent.tpl import T
|
||||
|
||||
|
||||
class DSDraftExpGen(ExpGen):
|
||||
|
||||
def _init_task_gen(
|
||||
self,
|
||||
targets: str,
|
||||
scenario_desc: str,
|
||||
task_output_format: str,
|
||||
workspace_code: str | None = None,
|
||||
spec: str = None,
|
||||
hypothesis: Hypothesis | None = None,
|
||||
exp_and_feedback_desc: str | None = None,
|
||||
former_task: str | None = None,
|
||||
) -> dict:
|
||||
system_prompt = T(".prompts:task_gen.system").r(
|
||||
targets=targets,
|
||||
scenario=scenario_desc,
|
||||
task_specification=spec,
|
||||
hypothesis=hypothesis,
|
||||
task_output_format=task_output_format,
|
||||
)
|
||||
user_prompt = T(".prompts:task_gen.user").r(
|
||||
targets=targets,
|
||||
hypothesis=hypothesis,
|
||||
workspace_code=workspace_code,
|
||||
exp_and_feedback_desc=exp_and_feedback_desc,
|
||||
former_task_desc=former_task,
|
||||
)
|
||||
|
||||
resp_dict = json.loads(
|
||||
APIBackend().build_messages_and_create_chat_completion(
|
||||
user_prompt=user_prompt, system_prompt=system_prompt, json_mode=True, json_target_type=dict
|
||||
)
|
||||
)
|
||||
|
||||
return resp_dict
|
||||
|
||||
def gen(
|
||||
self,
|
||||
component: COMPONENT,
|
||||
trace: DSTrace,
|
||||
) -> DSExperiment:
|
||||
"""Handle any component using a unified approach.
|
||||
|
||||
Args:
|
||||
component: Name of the component (e.g. "DataLoadSpec")
|
||||
task_cls: The task class to instantiate (e.g. DataLoaderTask)
|
||||
scenario_desc: Description of the current scenario
|
||||
last_successful_exp: Last successful experiment or None
|
||||
spec_file: Path to specification file if needed
|
||||
"""
|
||||
scenario_desc = trace.scen.get_scenario_all_desc()
|
||||
last_successful_exp = trace.last_successful_exp()
|
||||
init_component_config = {
|
||||
"DataLoadSpec": {"task_cls": DataLoaderTask, "spec_file": None, "component_prompt_key": "data_loader"},
|
||||
"FeatureEng": {"task_cls": FeatureTask, "spec_file": "spec/feature.md", "component_prompt_key": "feature"},
|
||||
"Model": {"task_cls": ModelTask, "spec_file": "spec/model.md", "component_prompt_key": "model"},
|
||||
"Ensemble": {"task_cls": EnsembleTask, "spec_file": "spec/ensemble.md", "component_prompt_key": "ensemble"},
|
||||
"Workflow": {"task_cls": WorkflowTask, "spec_file": "spec/workflow.md", "component_prompt_key": "workflow"},
|
||||
}
|
||||
task_cls = init_component_config[component]["task_cls"]
|
||||
spec_file = init_component_config[component].get("spec_file")
|
||||
component_prompt_key = init_component_config[component].get("component_prompt_key")
|
||||
|
||||
former_tasks_desc = ""
|
||||
if len(trace.hist) > 0:
|
||||
for exp, fb in reversed(trace.hist):
|
||||
if exp is not last_successful_exp:
|
||||
former_task_desc = exp.pending_tasks_list[0][0].get_task_information()
|
||||
former_task_desc += f"\n\nYou have tried to implement the same component and got the following exception: \n{fb.exception}\n Please try different methods to avoid the same errors and results in an infinite loop"
|
||||
former_tasks_desc += former_task_desc
|
||||
else:
|
||||
break
|
||||
|
||||
if DS_RD_SETTING.spec_enabled:
|
||||
spec = last_successful_exp.experiment_workspace.file_dict[spec_file] if spec_file else None
|
||||
else:
|
||||
spec = T(f"scenarios.data_science.share:component_spec.{component}").r()
|
||||
resp_dict = self._init_task_gen(
|
||||
targets=component,
|
||||
scenario_desc=scenario_desc,
|
||||
spec=spec,
|
||||
task_output_format=T(f".prompts:output_format.{component_prompt_key or component.lower()}").r(),
|
||||
former_task=former_tasks_desc if former_tasks_desc else None,
|
||||
)
|
||||
|
||||
task = task_cls(
|
||||
name=component if component != "Model" else resp_dict.pop("model_name"),
|
||||
description=resp_dict.get("description", f"{component} description not provided"),
|
||||
)
|
||||
|
||||
exp = DSExperiment(pending_tasks_list=[[task]], hypothesis=DSHypothesis(component))
|
||||
if last_successful_exp:
|
||||
# exp.experiment_workspace.inject_code_from_folder(last_successful_exp.experiment_workspace.workspace_path)
|
||||
exp.experiment_workspace.inject_code_from_file_dict(last_successful_exp.experiment_workspace)
|
||||
return exp
|
||||
+2
-5
@@ -292,7 +292,6 @@ component_gen:
|
||||
Please choose the most proper component to focus on based on the information above. Please balance the exploration and exploitation.
|
||||
Avoid selecting the same component more than 5 times in a row to ensure that the chosen component is not overly repetitive.
|
||||
|
||||
|
||||
exp_and_feedback: |-
|
||||
{% for experiment, feedback in trace.hist[-10:] %}
|
||||
## Experiment {{ loop.index }}
|
||||
@@ -320,8 +319,8 @@ output_format:
|
||||
The output should follow JSON format. The schema is as follows:
|
||||
{
|
||||
"component": "If "hypothesis_specification" provides the component you need to take, please follow "hypothesis_specification" to choose the component. Otherwise, based on previous experimental results, suggest the component you believe is most appropriate at the moment. It should be one of ["DataLoadSpec", "FeatureEng", "Model", "Ensemble", "Workflow"]",
|
||||
"hypothesis": "The new hypothesis generated based on the information provided.",
|
||||
"reason": "The reason why you generate this hypothesis. It should be comprehensive and logical. It should cover the other keys below and extend them.",
|
||||
"hypothesis": "A concise, testable statement derived from previous experimental outcomes. Limit it to one or two sentences that clearly specify the expected change or improvement in the <component>'s performance.",
|
||||
"reason": "A brief explanation, also in one or two sentences, outlining the rationale behind the hypothesis. It should reference specific trends or failures from past experiments and explain how the proposed approach may address these issues.",
|
||||
"concise_reason": "Two-line summary. First line focuses on a concise justification for the change. Second line generalizes a knowledge statement.",
|
||||
"concise_observation": "One line summary. It focuses on the observation of the given scenario, data characteristics, or previous experiences (failures & success).",
|
||||
"concise_justification": "One line summary. Justify the hypothesis based on theoretical principles or initial assumptions.",
|
||||
@@ -332,14 +331,12 @@ output_format:
|
||||
The output should follow JSON format. The schema is as follows:
|
||||
{
|
||||
"description": "A precise and comprehensive description of the overall data loader for the data science workflow",
|
||||
# Don't add ellipsis (...) or any filler text that might cause JSON parsing errors here!
|
||||
}
|
||||
feature: |-
|
||||
Design a specific and detailed feature engineering task based on the given hypothesis. The output should be detailed enough to directly implement the corresponding code.
|
||||
The output should follow JSON format. The schema is as follows:
|
||||
{
|
||||
"description": "A precise and comprehensive description of feature engineering task",
|
||||
# Don't add ellipsis (...) or any filler text that might cause JSON parsing errors here!
|
||||
}
|
||||
model: |-
|
||||
Design a specific and detailed model task based on the given hypothesis. The output should be detailed enough to directly implement the corresponding code.
|
||||
@@ -0,0 +1,194 @@
|
||||
scenario_problem:
|
||||
system: |-
|
||||
You are a Kaggle Grandmaster and expert ML engineer with deep expertise in statistics, machine learning, and competition optimization.
|
||||
You will be given scenario and competition description and the current SOTA implementation and feedback.
|
||||
Your task is to analyze the given information and extract the **Scenario Problems** from the given materials.
|
||||
|
||||
## Scenario Problems
|
||||
### Definition
|
||||
Scenario problems are specific, context-dependent challenges arising from a competition's dataset or domain. They fall into two categories:
|
||||
1. Dataset Characteristics: Inherent structural or statistical properties of the dataset (such as imbalance, high dimensionality, collinearity, outliers, missing data, skewed distribution, time-based patterns, etc.).
|
||||
2. Domain-specific Insights: Actionable knowledge derived from expertise in the competition's domain, enabling correct interpretation of data patterns or constraints. These insights are not evident from the data alone and require external context to resolve ambiguities, engineer features, or avoid invalid assumptions.
|
||||
|
||||
### Specification
|
||||
{{ problem_spec }}
|
||||
|
||||
### Core Analysis Dimensions
|
||||
1. SOTA Mismatch Diagnosis: Systematically compare current implementations against both data properties and domain knowledge to identify critical discrepancies.
|
||||
2. Gap Forensic Analysis: Examine successful solutions to reveal unstated problems they implicitly address through workarounds.
|
||||
3. Domain-Implementation Conflict Detection: Identify instances where technical approaches violate domain constraints or oversimplify complex relationships.
|
||||
|
||||
### Output Format
|
||||
{{ problem_output_format }}
|
||||
|
||||
user: |-
|
||||
# Scenario Description
|
||||
{{ scenario_desc }}
|
||||
|
||||
# Competition Description
|
||||
{{ competition_desc }}
|
||||
|
||||
# Current SOTA Implementation
|
||||
{{ sota_exp_desc }}
|
||||
|
||||
feedback_problem:
|
||||
system: |-
|
||||
You are a Kaggle Grandmaster and expert ML engineer with deep expertise in statistics, machine learning, and competition optimization.
|
||||
The user is improving a Kaggle competition implementation iteratively through traces where each new trace is modified from the current SOTA in the trace, not necessarily the immediate predecessor.
|
||||
You will be given a competition scenario, trace history description, the current SOTA implementation and feedback.
|
||||
Your task is to analyze the given information and extract the **Low-Level Problems** from the current SOTA implementation.
|
||||
|
||||
## Low-Level Problems
|
||||
### Definition
|
||||
Low-level problems are specific and fine-grained technical, or methodological issues within one or more of the five components ('DataLoadSpec', 'FeatureEng', 'Model', 'Ensemble', 'Workflow') in the implementation.
|
||||
|
||||
### Specification
|
||||
{{ problem_spec }}
|
||||
|
||||
### Output Format
|
||||
{{ problem_output_format }}
|
||||
|
||||
user: |-
|
||||
# Scenario Description
|
||||
{{ scenario_desc }}
|
||||
|
||||
Here's the former SOTA experiments and their feedbacks:
|
||||
{{ sota_exp_and_feedback_list_desc }}
|
||||
|
||||
Also, here's the former failed experiments and their feedbacks:
|
||||
{{ failed_exp_and_feedback_list_desc }}
|
||||
|
||||
# Current SOTA Implementation
|
||||
{{ sota_exp_desc }}
|
||||
|
||||
hypothesis_gen:
|
||||
system: |-
|
||||
You are a Kaggle Grandmaster and expert ML engineer with deep expertise in statistics, machine learning, and competition optimization.
|
||||
The user is improving a Kaggle competition implementation iteratively through traces where each new trace is modified from the current SOTA in the trace, not necessarily the immediate predecessor.
|
||||
You will be given a competition scenario, trace history description, the current SOTA implementation, and a list of identified problems.
|
||||
Your role involves two tasks:
|
||||
1. **Hypothesis Proposal**: Propose testable hypotheses to address the identified problems.
|
||||
2. **Hypothesis Evaluation**: Evaluate the proposed hypotheses across multiple dimensions.
|
||||
|
||||
# Task 1: Hypothesis Proposal
|
||||
For each identified problem, you are required to propose a hypothesis aimed at improving the current SOTA implementation.
|
||||
A hypothesis is a precise, testable, and actionable statement that proposes a specific modification or improvement to address an identified problem in a Kaggle competition implementation.
|
||||
Each hypothesis should focus on one of the following 5 components of an implementation:
|
||||
{{ component_desc }}
|
||||
|
||||
## Hypothesis Specification
|
||||
1. The hypothesis should be precise, testable, and directly actionable. Avoid general or vague statements. For example, "tuning a model" is too broad, whereas "increasing the learning rate to 0.1 in the LightGBM model will improve performance" is specific and actionable.
|
||||
2. Each hypothesis should focus on a single direction per experiment. Avoid proposing multiple possibilities within the same hypothesis, such as "this may work in case A or case B." Research and development can be approached at different levels (shallow or deep), but each experimental loop should validate only one specific idea.
|
||||
3. The hypothesis should based on current SOTA solution. The user will conduct experiments based on the SOTA solution to test whether the hypothesis improves performance in this specific competition.
|
||||
|
||||
|
||||
# Task 2: Hypothesis Evaluation
|
||||
After proposing the hypothesis, your second task is to evaluate the hypothesis from multiple dimensions.
|
||||
|
||||
## Evaluation Instruction
|
||||
Please score the proposed hypothesis from 1 to 10 for each of the following dimensions (where 1 means lowest and 10 means highest):
|
||||
1. Problem-Hypothesis Alignment: How well the hypothesis addresses the identified problem.
|
||||
2. Expected Impact: The estimated improvement after applying the hypothesis to current SOTA implementation.
|
||||
3. Novelty: Degree of innovation compared to previous attempts.
|
||||
4. Feasibility: The ease of implementing the proposed hypothesis in the current SOTA implementation.
|
||||
5. Risk-Reward Balance: The exploration-exploitation balance of the proposed hypothesis.
|
||||
|
||||
## Final Output Format in JSON Schema:
|
||||
{{ hypothesis_output_format }}
|
||||
|
||||
|
||||
user: |-
|
||||
# Scenario Description
|
||||
{{ scenario_desc }}
|
||||
|
||||
Here's the former SOTA experiments and their feedbacks:
|
||||
{{ sota_exp_and_feedback_list_desc }}
|
||||
|
||||
Also, here's the former failed experiments and their feedbacks:
|
||||
{{ failed_exp_and_feedback_list_desc }}
|
||||
|
||||
# Current SOTA Implementation
|
||||
{{ sota_exp_desc }}
|
||||
|
||||
# Identified Problems
|
||||
{{ problems }}
|
||||
|
||||
|
||||
task_gen:
|
||||
system: |-
|
||||
You are a Kaggle Grandmaster and expert ML engineer with deep expertise in statistics, machine learning, and competition optimization.
|
||||
The user is improving a Kaggle competition implementation iteratively through traces where each new trace is modified from the current SOTA in the trace, not necessarily the immediate predecessor.
|
||||
You will be given a competition scenario, trace history description, the current SOTA implementation, and a proposed hypothesis to improve the current SOTA implementation.
|
||||
|
||||
# Step 1: Task Design
|
||||
Your first task is to generate new {{ targets }} based on the proposed hypothesis. Your task should very detailed with specific steps and instructions. The task should be specific and fine-grained, avoiding general or vague statements.
|
||||
|
||||
## Specification
|
||||
{{ task_specification }}
|
||||
|
||||
## [Partial Response Format 1] Task Output Format
|
||||
{{ task_output_format }}
|
||||
|
||||
{% if workflow_check %}
|
||||
# Step 2: Workflow Update
|
||||
Since components have dependencies, your second task is to update the workflow to reflect the changes made to the target component. Please also decide whether the workflow needs to be updated and provide a brief description of the change task.
|
||||
{{ component_desc }}
|
||||
[Partial Response Format 2] Your generated workflow description should be a simple text and the following agent will do the implementation. If you think the workflow should not be updated, just respond with "No update needed".
|
||||
{% endif %}
|
||||
|
||||
Your final output should strictly adhere to the following JSON format.
|
||||
{
|
||||
"task_design": [Partial Response Format 1],
|
||||
{% if workflow_check %}"workflow_update": [Partial Response Format 2], {% endif %}
|
||||
}
|
||||
|
||||
user: |-
|
||||
# Scenario Description
|
||||
{{ scenario_desc }}
|
||||
|
||||
# Current SOTA Implementation
|
||||
{{ sota_exp_desc }}
|
||||
|
||||
# Proposed Hypothesis you should strictly follow:
|
||||
{{ hypothesis }}
|
||||
|
||||
specification:
|
||||
problem: |-
|
||||
1. The problem should be specific and fine-grained. Avoid general or vague statements.
|
||||
2. The problem should technical or methodological. Focus on design and implementation flaws, not runtime errors.
|
||||
hypothesis: |-
|
||||
1. The hypothesis should be precise, testable, and directly actionable. Avoid general or vague statements. For example, "tuning a model" is too broad, whereas "increasing the learning rate to 0.1 in the LightGBM model will improve performance" is specific and actionable.
|
||||
2. Each hypothesis should focus on a single direction per experiment. Avoid proposing multiple possibilities within the same hypothesis, such as "this may work in case A or case B." Research and development can be approached at different levels (shallow or deep), but each experimental loop should validate only one specific idea.
|
||||
3. The hypothesis should based on current SOTA solution. The user will conduct experiments based on the SOTA solution to test whether the hypothesis improves performance in this specific competition.
|
||||
|
||||
|
||||
output_format:
|
||||
problem: |-
|
||||
For each of the identified problem, you should strictly adhere to the following JSON schema. Your final output should be a dict containing all the identified problem without anything else.
|
||||
{
|
||||
"problem name 1": {
|
||||
"problem": "Description of the first issue",
|
||||
"reason": "Brief explanation of why this is a problem, based on the feedback or inferred from provided materials."
|
||||
},
|
||||
"problem name 2": {
|
||||
"problem": "Description of the second issue",
|
||||
"reason": "Brief explanation of why this is a problem, based on the feedback or inferred from provided materials."
|
||||
}
|
||||
}
|
||||
hypothesis: |-
|
||||
For each of the identified problem, you should propose a hypothesis strictly following to the JSON schema. Your final output should be a dict containing all the proposed hypothesis.
|
||||
{
|
||||
"problem name 1": {
|
||||
"observation": "The observation of the given scenario, data characteristics, or trace history.",
|
||||
"component": "The component name that the hypothesis focus on. Must be one of ('DataLoadSpec', 'FeatureEng', 'Model', 'Ensemble', 'Workflow').",
|
||||
"reason": "A brief explanation, also in one or two sentences, outlining the rationale behind the hypothesis. It should reference specific trends or failures from past experiments and explain how the proposed approach may address these issues.",
|
||||
"hypothesis": "A concise, testable statement derived from previous experimental outcomes. Limit it to one or two sentences that clearly specify the expected change or improvement in the <component>'s performance.",
|
||||
"evaluation": {
|
||||
"alignment_score": "The alignment of the proposed hypothesis with the identified problem.",
|
||||
"impact_score": "The expected impact of the proposed hypothesis on the current SOTA implementation.",
|
||||
"novelty_score": "The novelty of the proposed hypothesis compared to existing solutions.",
|
||||
"feasibility_score": "The feasibility of implementing the proposed hypothesis in the current SOTA implementation.",
|
||||
"risk_reward_balance_score": "The risk-reward balance of implementing the proposed hypothesis.",
|
||||
}
|
||||
},
|
||||
}
|
||||
@@ -0,0 +1,460 @@
|
||||
import json
|
||||
from typing import Dict, List
|
||||
|
||||
import pandas as pd
|
||||
|
||||
from rdagent.app.data_science.conf import DS_RD_SETTING
|
||||
from rdagent.components.coder.data_science.ensemble.exp import EnsembleTask
|
||||
from rdagent.components.coder.data_science.feature.exp import FeatureTask
|
||||
from rdagent.components.coder.data_science.model.exp import ModelTask
|
||||
from rdagent.components.coder.data_science.raw_data_loader.exp import DataLoaderTask
|
||||
from rdagent.components.coder.data_science.workflow.exp import WorkflowTask
|
||||
from rdagent.core.proposal import ExpGen
|
||||
from rdagent.oai.llm_utils import APIBackend
|
||||
from rdagent.scenarios.data_science.experiment.experiment import DSExperiment
|
||||
from rdagent.scenarios.data_science.proposal.exp_gen.base import DSHypothesis, DSTrace
|
||||
from rdagent.utils.agent.tpl import T
|
||||
from rdagent.utils.repo.diff import generate_diff_from_dict
|
||||
from rdagent.utils.workflow import wait_retry
|
||||
|
||||
COMPONENT_TASK_MAPPING = {
|
||||
"DataLoadSpec": {
|
||||
"target_name": "Data loader and specification generation",
|
||||
"spec_file": "spec/data_loader.md",
|
||||
"task_output_format": T(".prompts:output_format.data_loader").r(),
|
||||
"task_class": DataLoaderTask,
|
||||
},
|
||||
"FeatureEng": {
|
||||
"target_name": "Feature engineering",
|
||||
"spec_file": "spec/feature.md",
|
||||
"task_output_format": T(".prompts:output_format.feature").r(),
|
||||
"task_class": FeatureTask,
|
||||
},
|
||||
"Model": {
|
||||
"target_name": "Model",
|
||||
"spec_file": "spec/model.md",
|
||||
"task_output_format": T(".prompts:output_format.model").r(),
|
||||
"task_class": ModelTask,
|
||||
},
|
||||
"Ensemble": {
|
||||
"target_name": "Ensemble",
|
||||
"spec_file": "spec/ensemble.md",
|
||||
"task_output_format": T(".prompts:output_format.ensemble").r(),
|
||||
"task_class": EnsembleTask,
|
||||
},
|
||||
"Workflow": {
|
||||
"target_name": "Workflow",
|
||||
"spec_file": "spec/workflow.md",
|
||||
"task_output_format": T(".prompts:output_format.workflow").r(),
|
||||
"task_class": WorkflowTask,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
class DSProposalV1ExpGen(ExpGen):
|
||||
def gen(self, trace: DSTrace, max_trace_hist: int) -> DSExperiment:
|
||||
# Guidelines:
|
||||
# System prompts: Shared condition you are facing
|
||||
# - scenario description: `scenario_desc`
|
||||
# - expected output format
|
||||
# User prompts: Task Specific information
|
||||
# - Previous Feedback
|
||||
# - Current sota implementation (encourage change based on it)
|
||||
# - Extra RAG
|
||||
|
||||
scenario_desc = trace.scen.get_scenario_all_desc()
|
||||
sota_exp = trace.sota_experiment()
|
||||
assert sota_exp is not None, "SOTA experiment is not provided."
|
||||
exp_and_feedback = trace.hist[-1]
|
||||
last_exp = exp_and_feedback[0]
|
||||
|
||||
# Step 1: Generate component
|
||||
# Describe current best solution using shared template
|
||||
sota_exp_desc = T("scenarios.data_science.share:describe.exp").r(
|
||||
exp=sota_exp, heading="Best of previous exploration of the scenario"
|
||||
)
|
||||
last_exp_diff = "\n".join(
|
||||
generate_diff_from_dict(sota_exp.experiment_workspace.file_dict, last_exp.experiment_workspace.file_dict)
|
||||
) # we use file_dict for hitting the cache when replicate the experiment in another machine.
|
||||
|
||||
sota_exp_feedback_list = trace.experiment_and_feedback_list_after_init(return_type="sota")
|
||||
failed_exp_feedback_list = trace.experiment_and_feedback_list_after_init(return_type="failed")[-max_trace_hist:]
|
||||
all_exp_feedback_list = trace.experiment_and_feedback_list_after_init(return_type="all")
|
||||
trace_component_to_feedback_df = pd.DataFrame(columns=["component", "hypothesis", "decision"])
|
||||
for index, (exp, fb) in enumerate(all_exp_feedback_list):
|
||||
trace_component_to_feedback_df.loc[f"trial {index + 1}"] = [
|
||||
exp.hypothesis.component,
|
||||
exp.hypothesis.hypothesis,
|
||||
fb.decision,
|
||||
]
|
||||
|
||||
sota_exp_feedback_list_desc = T("scenarios.data_science.share:describe.trace").r(
|
||||
exp_and_feedback_list=sota_exp_feedback_list,
|
||||
success=True,
|
||||
)
|
||||
failed_exp_feedback_list_desc = T("scenarios.data_science.share:describe.trace").r(
|
||||
exp_and_feedback_list=failed_exp_feedback_list,
|
||||
success=False,
|
||||
)
|
||||
|
||||
# Generate component using template with proper context
|
||||
component_sys_prompt = T(".prompts:component_gen.system").r(
|
||||
scenario=scenario_desc,
|
||||
sota_exp_desc=sota_exp_desc,
|
||||
last_exp_diff=last_exp_diff,
|
||||
component_desc="\n".join(
|
||||
[
|
||||
f"[{key}] {value}"
|
||||
for key, value in T("scenarios.data_science.share:component_description").template.items()
|
||||
]
|
||||
),
|
||||
component_output_format=T(".prompts:output_format.component").r(),
|
||||
)
|
||||
|
||||
component_user_prompt = T(".prompts:component_gen.user").r(
|
||||
sota_exp_and_feedback_list_desc=sota_exp_feedback_list_desc,
|
||||
failed_exp_and_feedback_list_desc=failed_exp_feedback_list_desc,
|
||||
component_and_feedback_df=(
|
||||
trace_component_to_feedback_df.to_string()
|
||||
if len(trace_component_to_feedback_df) > 0
|
||||
else "No experiment and feedback provided"
|
||||
),
|
||||
)
|
||||
|
||||
resp_dict_component: dict = json.loads(
|
||||
APIBackend().build_messages_and_create_chat_completion(
|
||||
component_user_prompt, component_sys_prompt, json_mode=True, json_target_type=Dict[str, str]
|
||||
)
|
||||
)
|
||||
|
||||
component = resp_dict_component.get("component", "Component not provided")
|
||||
component_reason = resp_dict_component.get("reason", "Reason not provided")
|
||||
sota_exp_model_file_count = len(
|
||||
[
|
||||
k
|
||||
for k in sota_exp.experiment_workspace.file_dict.keys()
|
||||
if k.endswith(".py") and "test" not in k and k.startswith("model")
|
||||
]
|
||||
)
|
||||
if sota_exp_model_file_count <= 1 and component == "Ensemble":
|
||||
component = "Model"
|
||||
|
||||
# Why we should split component selection and steps after?
|
||||
# - after we know the selected component, we can use RAG.
|
||||
|
||||
# Step 2: Generate the rest of the hypothesis & task
|
||||
component_info = COMPONENT_TASK_MAPPING.get(component)
|
||||
|
||||
if component_info:
|
||||
if DS_RD_SETTING.spec_enabled:
|
||||
task_spec = sota_exp.experiment_workspace.file_dict[component_info["spec_file"]]
|
||||
else:
|
||||
task_spec = T(f"scenarios.data_science.share:component_spec.{component}").r()
|
||||
system_prompt = T(".prompts:direct_exp_gen.system").r(
|
||||
targets=component_info["target_name"],
|
||||
component=component,
|
||||
scenario=scenario_desc,
|
||||
hypothesis_specification=T(".prompts:hypothesis_specification").r(),
|
||||
hypothesis_output_format=T(".prompts:output_format.hypothesis").r(),
|
||||
task_specification=task_spec,
|
||||
task_output_format=component_info["task_output_format"],
|
||||
workflow_check=(not component == "Workflow"),
|
||||
)
|
||||
|
||||
user_prompt = T(".prompts:direct_exp_gen.user").r(
|
||||
targets=component_info["target_name"],
|
||||
sota_exp_desc=sota_exp_desc,
|
||||
sota_exp_and_feedback_list_desc=sota_exp_feedback_list_desc,
|
||||
failed_exp_and_feedback_list_desc=failed_exp_feedback_list_desc,
|
||||
last_exp_diff=last_exp_diff,
|
||||
)
|
||||
|
||||
def _append_retry(args: tuple, kwargs: dict) -> tuple[tuple, dict]:
|
||||
# Only modify the user_prompt on retries (i > 0)
|
||||
user_prompt = args[0]
|
||||
user_prompt += "\n\nretrying..."
|
||||
return (user_prompt,), kwargs
|
||||
|
||||
@wait_retry(retry_n=5, transform_args_fn=_append_retry)
|
||||
def _f(user_prompt):
|
||||
resp_dict = json.loads(
|
||||
APIBackend().build_messages_and_create_chat_completion(
|
||||
user_prompt=user_prompt,
|
||||
system_prompt=system_prompt,
|
||||
json_mode=True,
|
||||
# NOTE: corner cases.
|
||||
# workflow_update may be a string
|
||||
# model could have 2 level nested dict.
|
||||
json_target_type=dict[str, dict[str, str | dict] | str],
|
||||
)
|
||||
)
|
||||
assert "hypothesis_proposal" in resp_dict, "Hypothesis proposal not provided."
|
||||
assert "task_design" in resp_dict, "Task design not provided."
|
||||
task_class = component_info["task_class"]
|
||||
hypothesis_proposal = resp_dict.get("hypothesis_proposal", {})
|
||||
hypothesis = DSHypothesis(
|
||||
component=component,
|
||||
hypothesis=hypothesis_proposal.get("hypothesis", ""),
|
||||
reason=component_reason + "\n" + hypothesis_proposal.get("reason", ""),
|
||||
concise_reason=hypothesis_proposal.get("concise_reason", ""),
|
||||
concise_observation=hypothesis_proposal.get("concise_observation", ""),
|
||||
concise_justification=hypothesis_proposal.get("concise_justification", ""),
|
||||
concise_knowledge=hypothesis_proposal.get("concise_knowledge", ""),
|
||||
)
|
||||
|
||||
task_design = resp_dict.get("task_design", {})
|
||||
task_name = task_design["model_name"] if component == "Model" else component
|
||||
description = task_design.get(
|
||||
"description", f"{component_info['target_name']} description not provided"
|
||||
)
|
||||
task = task_class(
|
||||
name=task_name,
|
||||
description=description,
|
||||
**{k: task_design.get(k, v) for k, v in component_info.get("extra_params", {}).items()},
|
||||
)
|
||||
new_workflow_desc = resp_dict.get("workflow_update", "No update needed")
|
||||
return hypothesis, task, new_workflow_desc
|
||||
|
||||
hypothesis, task, new_workflow_desc = _f(user_prompt)
|
||||
|
||||
exp = DSExperiment(pending_tasks_list=[[task]], hypothesis=hypothesis)
|
||||
# exp.experiment_workspace.inject_code_from_folder(sota_exp.experiment_workspace.workspace_path)
|
||||
exp.experiment_workspace.inject_code_from_file_dict(sota_exp.experiment_workspace)
|
||||
|
||||
if new_workflow_desc != "No update needed":
|
||||
workflow_task = WorkflowTask(
|
||||
name="Workflow",
|
||||
description=new_workflow_desc,
|
||||
)
|
||||
exp.pending_tasks_list.append([workflow_task])
|
||||
return exp
|
||||
else:
|
||||
raise ValueError(f"Unknown component: {component}")
|
||||
|
||||
|
||||
class DSProposalV2ExpGen(ExpGen):
|
||||
def identify_scenario_problem(self, scenario_desc: str, competition_desc: str, sota_exp_desc: str) -> Dict:
|
||||
sys_prompt = T(".prompts_v2:scenario_problem.system").r(
|
||||
problem_spec=T(".prompts_v2:specification.problem").r(),
|
||||
problem_output_format=T(".prompts_v2:output_format.problem").r(),
|
||||
)
|
||||
user_prompt = T(".prompts_v2:scenario_problem.user").r(
|
||||
scenario_desc=scenario_desc,
|
||||
competition_desc=competition_desc,
|
||||
sota_exp_desc=sota_exp_desc,
|
||||
)
|
||||
response = APIBackend().build_messages_and_create_chat_completion(
|
||||
user_prompt=user_prompt,
|
||||
system_prompt=sys_prompt,
|
||||
json_mode=True,
|
||||
json_target_type=Dict[str, Dict[str, str]],
|
||||
)
|
||||
return json.loads(response)
|
||||
|
||||
def identify_feedback_problem(
|
||||
self,
|
||||
scenario_desc: str,
|
||||
sota_exp_feedback_list_desc: str,
|
||||
failed_exp_feedback_list_desc: str,
|
||||
sota_exp_desc: str,
|
||||
) -> Dict:
|
||||
sys_prompt = T(".prompts_v2:scenario_problem.system").r(
|
||||
problem_spec=T(".prompts_v2:specification.problem").r(),
|
||||
problem_output_format=T(".prompts_v2:output_format.problem").r(),
|
||||
)
|
||||
user_prompt = T(".prompts_v2:feedback_problem.user").r(
|
||||
scenario_desc=scenario_desc,
|
||||
sota_exp_and_feedback_list_desc=sota_exp_feedback_list_desc,
|
||||
failed_exp_and_feedback_list_desc=failed_exp_feedback_list_desc,
|
||||
sota_exp_desc=sota_exp_desc,
|
||||
)
|
||||
response = APIBackend().build_messages_and_create_chat_completion(
|
||||
user_prompt=user_prompt,
|
||||
system_prompt=sys_prompt,
|
||||
json_mode=True,
|
||||
json_target_type=Dict[str, Dict[str, str]],
|
||||
)
|
||||
return json.loads(response)
|
||||
|
||||
def hypothesis_gen(
|
||||
self,
|
||||
component_desc: str,
|
||||
scenario_desc: str,
|
||||
sota_exp_feedback_list_desc: str,
|
||||
failed_exp_feedback_list_desc: str,
|
||||
sota_exp_desc: str,
|
||||
problems: list,
|
||||
) -> Dict:
|
||||
sys_prompt = T(".prompts_v2:hypothesis_gen.system").r(
|
||||
component_desc=component_desc,
|
||||
hypothesis_spec=T(".prompts_v2:specification.hypothesis").r(),
|
||||
hypothesis_output_format=T(".prompts_v2:output_format.hypothesis").r(),
|
||||
)
|
||||
user_prompt = T(".prompts_v2:hypothesis_gen.user").r(
|
||||
scenario_desc=scenario_desc,
|
||||
sota_exp_and_feedback_list_desc=sota_exp_feedback_list_desc,
|
||||
failed_exp_and_feedback_list_desc=failed_exp_feedback_list_desc,
|
||||
sota_exp_desc=sota_exp_desc,
|
||||
problems=json.dumps(problems, indent=2),
|
||||
)
|
||||
response = APIBackend().build_messages_and_create_chat_completion(
|
||||
user_prompt=user_prompt,
|
||||
system_prompt=sys_prompt,
|
||||
json_mode=True,
|
||||
json_target_type=Dict[str, Dict[str, str | Dict[str, str | int]]],
|
||||
)
|
||||
return json.loads(response)
|
||||
|
||||
def hypothesis_rank(self, hypothesis_dict: dict, problem_dict: dict) -> DSHypothesis:
|
||||
# TODO use rule base or llm to rank the hypothesis
|
||||
|
||||
max_score_problem_name = (
|
||||
pd.DataFrame(
|
||||
{problem_name: hypothesis_dict[problem_name]["evaluation"] for problem_name in hypothesis_dict}
|
||||
)
|
||||
.sum()
|
||||
.idxmax(axis=0)
|
||||
)
|
||||
problem = problem_dict.get(max_score_problem_name, {}).get("problem", "Problem not provided")
|
||||
|
||||
return DSHypothesis(
|
||||
component=hypothesis_dict[max_score_problem_name]["component"],
|
||||
hypothesis=hypothesis_dict[max_score_problem_name]["hypothesis"],
|
||||
reason=hypothesis_dict[max_score_problem_name]["reason"],
|
||||
problem=problem,
|
||||
)
|
||||
|
||||
def task_gen(
|
||||
self,
|
||||
component_desc: str,
|
||||
scenario_desc: str,
|
||||
sota_exp_desc: str,
|
||||
sota_exp: DSExperiment,
|
||||
hypothesis: DSHypothesis,
|
||||
) -> DSExperiment:
|
||||
component_info = COMPONENT_TASK_MAPPING.get(hypothesis.component)
|
||||
if DS_RD_SETTING.spec_enabled:
|
||||
task_spec = sota_exp.experiment_workspace.file_dict[component_info["spec_file"]]
|
||||
else:
|
||||
task_spec = T(f"scenarios.data_science.share:component_spec.{hypothesis.component}").r()
|
||||
sys_prompt = T(".prompts_v2:task_gen.system").r(
|
||||
targets=component_info["target_name"],
|
||||
task_specification=task_spec,
|
||||
task_output_format=component_info["task_output_format"],
|
||||
component_desc=component_desc,
|
||||
workflow_check=(not hypothesis.component == "Workflow"),
|
||||
)
|
||||
user_prompt = T(".prompts_v2:task_gen.user").r(
|
||||
scenario_desc=scenario_desc, sota_exp_desc=sota_exp_desc, hypothesis=str(hypothesis)
|
||||
)
|
||||
response = APIBackend().build_messages_and_create_chat_completion(
|
||||
user_prompt=user_prompt,
|
||||
system_prompt=sys_prompt,
|
||||
json_mode=True,
|
||||
json_target_type=Dict[str, str | Dict[str, str]],
|
||||
)
|
||||
task_dict = json.loads(response)
|
||||
task_design = task_dict.get("task_design", {})
|
||||
task_name = task_design["model_name"] if hypothesis.component == "Model" else hypothesis.component
|
||||
description = (
|
||||
task_design
|
||||
if isinstance(task_design, str)
|
||||
else task_design.get("description", f"{component_info['target_name']} description not provided")
|
||||
)
|
||||
task_class = COMPONENT_TASK_MAPPING[hypothesis.component]["task_class"]
|
||||
task = task_class(
|
||||
name=task_name,
|
||||
description=description,
|
||||
)
|
||||
new_workflow_desc = task_dict.get("workflow_update", "No update needed")
|
||||
exp = DSExperiment(pending_tasks_list=[[task]], hypothesis=hypothesis)
|
||||
# exp.experiment_workspace.inject_code_from_folder(sota_exp.experiment_workspace.workspace_path)
|
||||
exp.experiment_workspace.inject_code_from_file_dict(sota_exp.experiment_workspace)
|
||||
|
||||
if new_workflow_desc != "No update needed":
|
||||
workflow_task = WorkflowTask(
|
||||
name="Workflow",
|
||||
description=new_workflow_desc,
|
||||
)
|
||||
exp.pending_tasks_list.append([workflow_task])
|
||||
return exp
|
||||
|
||||
def gen(self, trace: DSTrace, max_trace_hist: int) -> DSExperiment:
|
||||
component_desc = "\n".join(
|
||||
[
|
||||
f"[{key}] {value}"
|
||||
for key, value in T("scenarios.data_science.share:component_description").template.items()
|
||||
]
|
||||
)
|
||||
|
||||
sota_exp = trace.sota_experiment()
|
||||
scenario_desc = trace.scen.get_scenario_all_desc()
|
||||
competition_desc = trace.scen.get_competition_full_desc()
|
||||
|
||||
sota_exp_desc = T("scenarios.data_science.share:describe.exp").r(
|
||||
exp=sota_exp, heading="Best of previous exploration of the scenario"
|
||||
)
|
||||
|
||||
sota_exp_feedback_list = trace.experiment_and_feedback_list_after_init(return_type="sota")
|
||||
failed_exp_feedback_list = trace.experiment_and_feedback_list_after_init(return_type="failed")[-max_trace_hist:]
|
||||
|
||||
sota_exp_feedback_list_desc = T("scenarios.data_science.share:describe.trace").r(
|
||||
exp_and_feedback_list=sota_exp_feedback_list,
|
||||
success=True,
|
||||
)
|
||||
failed_exp_feedback_list_desc = T("scenarios.data_science.share:describe.trace").r(
|
||||
exp_and_feedback_list=failed_exp_feedback_list,
|
||||
success=False,
|
||||
)
|
||||
|
||||
# Step 1: Identify problems
|
||||
scen_problems = self.identify_scenario_problem(
|
||||
scenario_desc=scenario_desc,
|
||||
competition_desc=competition_desc,
|
||||
sota_exp_desc=sota_exp_desc,
|
||||
)
|
||||
fb_problems = self.identify_feedback_problem(
|
||||
scenario_desc=scenario_desc,
|
||||
sota_exp_feedback_list_desc=sota_exp_feedback_list_desc,
|
||||
failed_exp_feedback_list_desc=failed_exp_feedback_list_desc,
|
||||
sota_exp_desc=sota_exp_desc,
|
||||
)
|
||||
all_problems = {**scen_problems, **fb_problems}
|
||||
|
||||
# Step 2: Propose hypothesis based on the identified problems
|
||||
hypothesis_dict = self.hypothesis_gen(
|
||||
component_desc=component_desc,
|
||||
scenario_desc=scenario_desc,
|
||||
sota_exp_feedback_list_desc=sota_exp_feedback_list_desc,
|
||||
failed_exp_feedback_list_desc=failed_exp_feedback_list_desc,
|
||||
sota_exp_desc=sota_exp_desc,
|
||||
problems=all_problems,
|
||||
)
|
||||
sota_exp_model_file_count = len(
|
||||
[
|
||||
k
|
||||
for k in sota_exp.experiment_workspace.file_dict.keys()
|
||||
if k.endswith(".py") and "test" not in k and k.startswith("model")
|
||||
]
|
||||
)
|
||||
if sota_exp_model_file_count <= 1:
|
||||
pop_names = []
|
||||
for problem_name in hypothesis_dict:
|
||||
if hypothesis_dict[problem_name].get("component", "") == "Ensemble":
|
||||
pop_names.append(problem_name)
|
||||
for name in pop_names:
|
||||
hypothesis_dict.pop(name)
|
||||
|
||||
# Step 3: Select the best hypothesis
|
||||
new_hypothesis = self.hypothesis_rank(
|
||||
hypothesis_dict=hypothesis_dict,
|
||||
problem_dict=all_problems,
|
||||
)
|
||||
|
||||
return self.task_gen(
|
||||
component_desc=component_desc,
|
||||
scenario_desc=scenario_desc,
|
||||
sota_exp_desc=sota_exp_desc,
|
||||
sota_exp=sota_exp,
|
||||
hypothesis=new_hypothesis,
|
||||
)
|
||||
@@ -18,8 +18,7 @@ scenario_description: |-
|
||||
{% else %} The metric is better when it is smaller.
|
||||
{% endif %}
|
||||
|
||||
{% if eda_output is not none %}
|
||||
------Data Overview(EDA)------
|
||||
{% if eda_output is not none %}------Data Overview(EDA)------
|
||||
{{ eda_output }}
|
||||
{% endif %}
|
||||
|
||||
|
||||
@@ -56,6 +56,11 @@ describe: # some template to describe some object
|
||||
The experiment is designed based on hypothesis: {{ exp_and_feedback[0].hypothesis }}
|
||||
### Task of experiment
|
||||
{{ exp_and_feedback[0].pending_tasks_list[0][0].get_task_information() }}
|
||||
{% if exp_and_feedback[0].result is none %}
|
||||
Experiment score: Running buggy
|
||||
{% else %}
|
||||
Experiment score: {{ exp_and_feedback[0].result.loc["ensemble"].iloc[0] }}
|
||||
{% endif %}
|
||||
Experiment feedback decision: {{ exp_and_feedback[1].decision }}
|
||||
Reason: {{ exp_and_feedback[1].reason }}
|
||||
{% endfor %}
|
||||
|
||||
Reference in New Issue
Block a user