diff --git a/rdagent/scenarios/data_science/proposal/exp_gen/__init__.py b/rdagent/scenarios/data_science/proposal/exp_gen/__init__.py index 4675e787..10760642 100644 --- a/rdagent/scenarios/data_science/proposal/exp_gen/__init__.py +++ b/rdagent/scenarios/data_science/proposal/exp_gen/__init__.py @@ -3,10 +3,7 @@ from rdagent.core.proposal import ExpGen from rdagent.core.utils import import_class from rdagent.scenarios.data_science.experiment.experiment import DSExperiment from rdagent.scenarios.data_science.proposal.exp_gen.base import DSHypothesis, DSTrace -from rdagent.scenarios.data_science.proposal.exp_gen.draft import ( - DSDraftExpGen, - DSDraftV2ExpGen, -) +from rdagent.scenarios.data_science.proposal.exp_gen.draft import DSDraftExpGen from rdagent.scenarios.data_science.proposal.exp_gen.proposal import ( DSProposalV1ExpGen, DSProposalV2ExpGen, @@ -32,9 +29,6 @@ class DSExpGen(ExpGen): if DS_RD_SETTING.proposal_version not in ["v1", "v2"]: return import_class(DS_RD_SETTING.proposal_version)(scen=self.scen).gen(trace=trace) - if trace.sota_experiment() is None: - return DSDraftV2ExpGen(scen=self.scen).gen(trace=trace) - if DS_RD_SETTING.coder_on_whole_pipeline: return DSProposalV2ExpGen(scen=self.scen).gen(trace=trace, pipeline=True) diff --git a/rdagent/scenarios/data_science/proposal/exp_gen/base.py b/rdagent/scenarios/data_science/proposal/exp_gen/base.py index efd7111a..6e37ddbd 100644 --- a/rdagent/scenarios/data_science/proposal/exp_gen/base.py +++ b/rdagent/scenarios/data_science/proposal/exp_gen/base.py @@ -12,7 +12,7 @@ class DSHypothesis(Hypothesis): def __init__( self, component: COMPONENT, - hypothesis: str = "", + hypothesis: str | None = None, reason: str | None = None, concise_reason: str | None = None, concise_observation: str | None = None, @@ -31,15 +31,18 @@ class DSHypothesis(Hypothesis): self.problem_label = problem_label def __str__(self) -> str: - if self.hypothesis == "": + if self.hypothesis is None: return f"No hypothesis available. Trying to construct the first runnable {self.component} component." + lines = [] - if hasattr(self, "problem_name") and self.problem_name is not None and self.problem_desc is not None: - lines.append(f"Target Problem name: {self.problem_name}") + if self.problem_name is not None: + lines.append(f"Target Problem Name: {self.problem_name}") + if self.problem_desc is not None: lines.append(f"Target Problem: {self.problem_desc}") - lines.extend( - [f"Chosen Component: {self.component}", f"Hypothesis: {self.hypothesis}", f"Reason: {self.reason}"] - ) + lines.append(f"Chosen Component: {self.component}") + lines.append(f"Hypothesis: {self.hypothesis}") + if self.reason is not None: + lines.append(f"Reason: {self.reason}") return "\n".join(lines) @@ -203,18 +206,25 @@ class DSTrace(Trace[DataScienceScen, KnowledgeBase]): final_component = self.COMPLETE_ORDER[-1] has_final_component = True if DS_RD_SETTING.coder_on_whole_pipeline else False - exp_and_feedback_list = [] - for exp, fb in search_list: + SOTA_exp_and_feedback_list = [] + failed_exp_and_feedback_list = [] + for exp, fb in enumerate(search_list): if has_final_component: - if return_type == "all": - exp_and_feedback_list.append((exp, fb)) - elif return_type == "failed" and not fb.decision: - exp_and_feedback_list.append((exp, fb)) - elif return_type == "sota" and fb.decision: - exp_and_feedback_list.append((exp, fb)) + if fb.decision: + SOTA_exp_and_feedback_list.append((exp, fb)) + failed_exp_and_feedback_list = [] + else: + failed_exp_and_feedback_list.append((exp, fb)) if exp.hypothesis.component == final_component and fb: has_final_component = True - return exp_and_feedback_list + if return_type == "all": + return SOTA_exp_and_feedback_list + failed_exp_and_feedback_list + elif return_type == "failed": + return failed_exp_and_feedback_list + elif return_type == "sota": + return SOTA_exp_and_feedback_list + else: + raise ValueError("Invalid return_type. Must be 'sota', 'failed', or 'all'.") def sota_experiment_fb( self, diff --git a/rdagent/scenarios/data_science/proposal/exp_gen/draft.py b/rdagent/scenarios/data_science/proposal/exp_gen/draft.py index 58702095..719ff8f0 100644 --- a/rdagent/scenarios/data_science/proposal/exp_gen/draft.py +++ b/rdagent/scenarios/data_science/proposal/exp_gen/draft.py @@ -1,11 +1,10 @@ import json -from typing import TYPE_CHECKING, Dict +from typing import TYPE_CHECKING from rdagent.app.data_science.conf import DS_RD_SETTING from rdagent.components.coder.data_science.ensemble.exp import EnsembleTask from rdagent.components.coder.data_science.feature.exp import FeatureTask from rdagent.components.coder.data_science.model.exp import ModelTask -from rdagent.components.coder.data_science.pipeline.exp import PipelineTask from rdagent.components.coder.data_science.raw_data_loader.exp import DataLoaderTask from rdagent.components.coder.data_science.workflow.exp import WorkflowTask from rdagent.core.proposal import ExpGen, Hypothesis @@ -117,75 +116,3 @@ class DSDraftExpGen(ExpGen): # exp.experiment_workspace.inject_code_from_folder(last_successful_exp.experiment_workspace.workspace_path) exp.experiment_workspace.inject_code_from_file_dict(last_successful_exp.experiment_workspace) return exp - - -class DSDraftV2ExpGen(ExpGen): - def task_gen( - self, - scenario_desc: str, - scen_problems: dict, - drafting_trace_desc: str, - ) -> DSExperiment: - scen_problems_text = "" - for i, (problem_name, problem_dict) in enumerate(scen_problems.items()): - scen_problems_text += f"## Problem Name: {problem_name}\n" - scen_problems_text += f"- Problem Description: {problem_dict['problem']}\n\n" - sys_prompt = T(".prompts_drafting:task_draft.system").r( - task_spec=T(f"scenarios.data_science.share:component_spec.Pipeline").r(), - ) - user_prompt = T(".prompts_drafting:task_draft.user").r( - scenario_desc=scenario_desc, - scen_problems=scen_problems_text, - drafting_trace_desc=drafting_trace_desc, - ) - response = APIBackend().build_messages_and_create_chat_completion( - user_prompt=user_prompt, - system_prompt=sys_prompt, - json_mode=True, - json_target_type=Dict[str, str], - ) - task_dict = json.loads(response) - task_design = task_dict.get("task_design", "Description not provided") - task = PipelineTask(name="Workflow", description=task_design) - - # we use a pesudo hypothesis here - pseudo_hypothesis = DSHypothesis( - component="Workflow", - hypothesis="This is a pseudo hypothesis for drafting the first competition implementation. Your result should not be influenced by this hypothesis.", - problem_name="This is a pseudo problem name for drafting. The corresponding problem description includes several problem together.", - problem_desc=scen_problems_text, - ) - exp = DSExperiment(pending_tasks_list=[[task]], hypothesis=pseudo_hypothesis) - return exp - - def gen(self, trace: DSTrace) -> DSExperiment: - # Prepare - last_exp = trace.last_exp() - if not isinstance(last_exp, DSExperiment): - eda_output = None - else: - eda_output = last_exp.experiment_workspace.file_dict.get("EDA.md", None) - - component_desc = T("scenarios.data_science.share:component_description_in_pipeline").r() - scenario_desc = trace.scen.get_scenario_all_desc(eda_output=eda_output) - drafting_trace_desc = T("scenarios.data_science.share:describe.drafting_trace").r( - exp_and_feedback_list=trace.experiment_and_feedback_list_after_init(return_type="all"), - ) - - # Step 1: Identify Scenario Problems - sys_prompt = T(".prompts_drafting:scenario_problem.system").r() - user_prompt = T(".prompts_drafting:scenario_problem.user").r(scenario_desc=scenario_desc) - response = APIBackend().build_messages_and_create_chat_completion( - user_prompt=user_prompt, - system_prompt=sys_prompt, - json_mode=True, - json_target_type=Dict[str, Dict[str, str]], - ) - scen_problems = json.loads(response) - - # Step 2: Design Task - return self.task_gen( - scenario_desc=scenario_desc, - scen_problems=scen_problems, - drafting_trace_desc=drafting_trace_desc, - ) diff --git a/rdagent/scenarios/data_science/proposal/exp_gen/prompts_drafting.yaml b/rdagent/scenarios/data_science/proposal/exp_gen/prompts_drafting.yaml deleted file mode 100644 index 0ae68366..00000000 --- a/rdagent/scenarios/data_science/proposal/exp_gen/prompts_drafting.yaml +++ /dev/null @@ -1,72 +0,0 @@ -scenario_problem: - system: |- - {% include "scenarios.data_science.share:scen.role" %} - The user is creating a Kaggle competition implementation iteratively and this is the first iteration. You will be given the Kaggle competition scenario. - Your task is to analyze the given information and extract the **Scenario Problems** from the given materials to aid the implementation. - - ## Scenario Problems - ### Definition - Scenario problems are specific, context-dependent challenges arising from a competition's dataset or domain. They fall into two categories: - 1. Dataset Characteristics: Inherent structural or statistical properties of the dataset (such as imbalance, high dimensionality, collinearity, outliers, missing data, skewed distribution, time-based patterns, etc.). - 2. Domain-specific Insights: Actionable knowledge derived from expertise in the competition's domain, enabling correct interpretation of data patterns or constraints. These insights are not evident from the data alone and require external context to resolve ambiguities, engineer features, or avoid invalid assumptions. - - ### Specification - 1. The problem should be specific and fine-grained. Avoid general or vague statements. - 2. The problem should technical or methodological. Focus on design and implementation flaws. - 3. The problem should be strictly aligned with the improvement of target metric. **IF THE PROBLEM IS SOLVED, THEN THE TARGET METRIC WILL IMPROVE**. - - ### Output Format - For each of the identified problem, you should strictly adhere to the following JSON schema. Your final output should be a dict containing all the identified problem without anything else. - { - "problem name 1": { - "problem": "Description of the first issue in no more than three sentences.", - "reason": "Brief explanation of why this is a problem, based on evidence from provided materials in no more than three sentences." - }, - "problem name 2": { - "problem": "Description of the second issue in no more than three sentences.", - "reason": "Brief explanation of why this is a problem, based on evidence from provided materials in no more than three sentences." - } - } - - user: |- - # Scenario Description - {{ scenario_desc }} - - -task_draft: - system: |- - {% include "scenarios.data_science.share:scen.role" %} - The user is creating a Kaggle competition implementation iteratively and this is the first iteration. - You will be given a competition scenario and a list of identified scenario problems from the given competition scenario. - In addition, if there are any previous failed experiments, you will receive the task designs and failures. Please read them carefully to have a better understanding. - Your role is to design a very detailed task with specific steps and instructions to implement competition solution and address identified scenario problems. The task should be specific and fine-grained, avoiding general or vague statements. - - # Task Design - ## Task Specification - {{ task_spec }} - - ## Task Design Guidelines - Here are guidelines **YOU MUST FOLLOW** in your task design: - 1. The task should be concise with several steps each only in a few sentences. - 2. DO NOT write any code in the task description. - 3. DO NOT use any phrases like "for example" or "eg.," in the task description. Clearly give a decision (such as the specific method or model name) in the task description. - 4. DO NOT use vague statements like "choose a proper model" or "optimize the pipeline". Instead, specify the exact step and task to be made. - 5. Your task design should try to cover **ALL** the identified scenario problems. DO NOT include any conflicting ideas in the task design. If there are conflicting ideas due to conflicting identified problems, prioritize the most impactful or feasible option. If multiple solutions exist for a problem, select the most impactful or feasible option only. DO NOT include any conflicting ideas in the task description. - 6. Carefully read and analyze the previous failed experiments if any so that no similar mistakes will be made in your task design. Remember to put the lessons you learned from previous experiments in the new task design. - - ## Task Output Format: - Design a specific and detailed Pipeline task based on the given competition scenario and scenario problems. The output should be detailed enough to directly implement the corresponding code. - The output should follow JSON format. The schema is as follows: - { - "task_design": "A precise and comprehensive description of the main workflow script (`main.py`).", - } - - user: |- - # Scenario Description - {{ scenario_desc }} - - # Identified Scenario Problems - {{ scen_problems }} - - # Previous Failed Experiments - {{ drafting_trace_desc }} diff --git a/rdagent/scenarios/data_science/proposal/exp_gen/prompts_v2.yaml b/rdagent/scenarios/data_science/proposal/exp_gen/prompts_v2.yaml index 2597a493..22759707 100644 --- a/rdagent/scenarios/data_science/proposal/exp_gen/prompts_v2.yaml +++ b/rdagent/scenarios/data_science/proposal/exp_gen/prompts_v2.yaml @@ -1,13 +1,61 @@ +scenario_problem: + system: |- + {% include "scenarios.data_science.share:scen.role" %} + The user is improving a Kaggle competition implementation iteratively through traces where each new trace is modified from the current SOTA in the trace. If new trace surpasses the current SOTA, it will be the new SOTA. If not, it will be a failed experiment. + You will be given the scenario description and the current SOTA implementation and feedback. + Your task is to analyze the given information and extract the **Scenario Problems** from the given materials. + + ## Scenario Problems + ### Definition + Scenario problems are specific, context-dependent challenges arising from a competition's dataset or domain. They fall into two categories: + 1. Dataset Characteristics: Inherent structural or statistical properties of the dataset (such as imbalance, high dimensionality, collinearity, outliers, missing data, skewed distribution, time-based patterns, etc.). + 2. Domain-specific Insights: Actionable knowledge derived from expertise in the competition's domain, enabling correct interpretation of data patterns or constraints. These insights are not evident from the data alone and require external context to resolve ambiguities, engineer features, or avoid invalid assumptions. + + ### Specification + {{ problem_spec }} + + ### Core Analysis Dimensions + 1. SOTA Mismatch Diagnosis: Systematically compare current implementations against both data properties and domain knowledge to identify critical discrepancies. + 2. Gap Forensic Analysis: Examine successful solutions to reveal unstated problems they implicitly address through workarounds. + 3. Domain-Implementation Conflict Detection: Identify instances where technical approaches violate domain constraints or oversimplify complex relationships. + 4. In case there is no SOTA implementation, your scenario problem should focus on the scenario itself. + + ### Output Format + {{ problem_output_format }} + + user: |- + # Scenario Description + {{ scenario_desc }} + + # Current SOTA Implementation + {{ sota_exp_desc }} + feedback_problem: system: |- {% include "scenarios.data_science.share:scen.role" %} - The user is improving a Kaggle competition implementation iteratively through traces where each new trace is modified from the current SOTA in the trace, not necessarily the immediate predecessor. + The user is improving a Kaggle competition implementation iteratively through traces where each new trace is modified from the current SOTA in the trace. If new trace surpasses the current SOTA, it will be the new SOTA. If not, it will be a failed experiment. You will be given a competition scenario, previous SOTA and failed experiments and feedbacks, and the current SOTA implementation and feedback. - Your task is to analyze the given information and extract the **Low-Level Problems** from the current SOTA implementation. + Your task is to analyze the given information and extract the **Feedback Problems** from the previous experiments or the current SOTA implementation. - ## Low-Level Problems + ## Feedback Problems ### Definition - Low-level problems are specific and fine-grained technical, or methodological issues within the implementation. + Feedback problems are specific and fine-grained technical, or methodological issues within the previous experiments or the current SOTA implementation. + + ### Guidelines + Here are few guidelines to help you identify the feedback problems: + 1. Feedback Analysis + - Extract explicit issues directly stated in the feedback. + - Infer implicit issues from feedback context. + 2. Code Review + - Feature Engineering. Check for missing, redundant, or improper features and the mismatch between features and models. + - Model Architecture. Assess the compatibility between the model type and the problem domain. Verify model architecture and hyperparameters. + - Ensemble. Check if ensemble methods are optimized. Identify underperforming base models to remove from the ensemble. + - Training. Validate hyperparameter (e.g., learning rate, batch size), loss functions, and regularization. Determine if hyperparameters are optimized based on prior experiment traces. + 3. Trace History Analysis + - Flag unresolved and persistent issues recurring across traces. + - Highlight partial fixes (e.g., inappropriate feature engineering). + - Identify unexplored directions (e.g. new features, new model structures) from prior traces. + - Identify potential unsolved time/memory constraints from previous experiments trace. ### Specification {{ problem_spec }} @@ -19,7 +67,7 @@ feedback_problem: # Scenario Description {{ scenario_desc }} - # Previous Experiments and Feedbacks: + # Previous Experiments and Feedbacks {{ exp_and_feedback_list_desc }} # Current SOTA Implementation @@ -28,7 +76,7 @@ feedback_problem: hypothesis_gen: system: |- {% include "scenarios.data_science.share:scen.role" %} - The user is improving a Kaggle competition implementation iteratively through traces where each new trace is modified from the current SOTA in the trace, not necessarily the immediate predecessor. + The user is improving a Kaggle competition implementation iteratively through traces where each new trace is modified from the current SOTA in the trace. If new trace surpasses the current SOTA, it will be the new SOTA. If not, it will be a failed experiment. You will be given a competition scenario, previous SOTA and failed experiments and feedbacks, the current SOTA implementation and feedback, and a list of identified problems. Your role involves two tasks: 1. **Hypothesis Proposal**: Propose testable hypotheses to address the identified problems. @@ -58,8 +106,10 @@ hypothesis_gen: - If the problem relates to hyperparameter tuning, recommend a specific method or strategy for tuning. 4. Priority Note on Time/Memory Constraints - If time/memory constraints exist, they must be prioritized above all other problems. In such cases, do not response any other problems in the response dictionary. + 5. Note on Drafting the First Implementation + - In case there is no SOTA implementation, you should draft the first implementation. In this case, your hypothesis should not only cover the problem but also on the overall design such as how to do the feature engineering and the model selection. {% if enable_idea_pool %} - 5. Idea Reference + 6. Idea Reference - Each idea is a method, technique or trick that contributes to high performance from other competition implementation under similar problem. You are free to use them as an inspiration for your hypothesis proposal. {% endif %} @@ -117,6 +167,11 @@ task_gen: 5. Specific and Non-Vague - Avoid vague statements like "choose a proper model" Instead, specify the exact task to be made. - No phrases like "for example" or "eg.," should be used in the task. Give a clear decision in the task. + 6. Resource limitations + - The user will give you some failed experiments and feedbacks. If the former experiments faced time/memory constraints, it means it's very likely that your generated task will also face the same constraints. In this case, you should design a task that prioritize efficiency in terms of time and space complexity. + - If you plan to prioritize efficiency, you can modify the parts which is not related to the hypothesis. Which means your task should still able to validate the hypothesis. + - Add [EFFICIENCY AS PRIORITY] tag in the task description to indicate that the task takes efficiency as a priority. + - Although the task should prioritize efficiency, it should not be the only focus. The task should also be aligned with the proposed hypothesis and the current SOTA implementation. ## [Partial Response Format 1] Task Output Format: {{ task_output_format }} diff --git a/rdagent/scenarios/data_science/proposal/exp_gen/proposal.py b/rdagent/scenarios/data_science/proposal/exp_gen/proposal.py index 8118a80d..de084ad4 100644 --- a/rdagent/scenarios/data_science/proposal/exp_gen/proposal.py +++ b/rdagent/scenarios/data_science/proposal/exp_gen/proposal.py @@ -11,6 +11,7 @@ from rdagent.components.coder.data_science.pipeline.exp import PipelineTask from rdagent.components.coder.data_science.raw_data_loader.exp import DataLoaderTask from rdagent.components.coder.data_science.workflow.exp import WorkflowTask from rdagent.core.proposal import ExpGen +from rdagent.log import rdagent_logger as logger from rdagent.oai.llm_utils import APIBackend, md5_hash from rdagent.scenarios.data_science.experiment.experiment import DSExperiment from rdagent.scenarios.data_science.proposal.exp_gen.base import DSHypothesis, DSTrace @@ -224,6 +225,23 @@ class DSProposalV1ExpGen(ExpGen): class DSProposalV2ExpGen(ExpGen): + def identify_scenario_problem(self, scenario_desc: str, sota_exp_desc: str) -> Dict: + sys_prompt = T(".prompts_v2:scenario_problem.system").r( + problem_spec=T(".prompts_v2:specification.problem").r(), + problem_output_format=T(".prompts_v2:output_format.problem").r(), + ) + user_prompt = T(".prompts_v2:scenario_problem.user").r( + scenario_desc=scenario_desc, + sota_exp_desc=sota_exp_desc, + ) + response = APIBackend().build_messages_and_create_chat_completion( + user_prompt=user_prompt, + system_prompt=sys_prompt, + json_mode=True, + json_target_type=Dict[str, Dict[str, str]], + ) + return json.loads(response) + def identify_feedback_problem(self, scenario_desc: str, exp_feedback_list_desc: str, sota_exp_desc: str) -> Dict: sys_prompt = T(".prompts_v2:feedback_problem.system").r( problem_spec=T(".prompts_v2:specification.problem").r(), @@ -287,7 +305,12 @@ class DSProposalV2ExpGen(ExpGen): resp_dict = json.loads(response) return resp_dict - def hypothesis_rank(self, hypothesis_dict: dict, problem_dict: dict, pipeline: bool) -> Tuple[str, DSHypothesis]: + def hypothesis_rank( + self, + hypothesis_dict: dict, + problem_dict: dict, + trace: DSTrace, + ) -> Tuple[str, DSHypothesis]: weights = { "alignment_score": 0.2, "impact_score": 0.4, @@ -313,16 +336,19 @@ class DSProposalV2ExpGen(ExpGen): scores = pd.DataFrame(scores_dict) scores_sorted = scores.sum().sort_values(ascending=False) - if len(scores_sorted) > 5: - scores_sorted = scores_sorted[: len(scores_sorted) // 2] + scores_sorted = scores_sorted[:5] # Select top 5 hypotheses - # Increase the weight of the hypothesis that is inspired by the idea pool + # Increase the weight of the hypothesis that is inspired by the idea pool to 3x. + # Linear decay the weight of the scenario problem from 3x to 1x. index_to_pick_pool_list = [] for j, problem_name in enumerate(scores_sorted.index): if hypothesis_dict[problem_name].get("inspired", False): - index_to_pick_pool_list.extend([j] * 3) + index_to_pick_pool_list.extend([j] * 4) + elif problem_dict.get(problem_name, {}).get("label", "") == "SCENARIO_PROBLEM": + index_to_pick_pool_list.extend([j] * (3 - len(trace.hist) // 3)) else: - index_to_pick_pool_list.append(j) + index_to_pick_pool_list.extend([j] * 2) + logger.info(f"index_to_pick_pool_list: {index_to_pick_pool_list}") # Create a random but reproducible integer reproducible_int = int.from_bytes(bytes.fromhex(md5_hash(scores_sorted.to_string())), byteorder="big") % len( @@ -433,21 +459,34 @@ class DSProposalV2ExpGen(ExpGen): exp_feedback_list_desc = T("scenarios.data_science.share:describe.trace").r( exp_and_feedback_list=trace.experiment_and_feedback_list_after_init(return_type="all"), type="all", + pipeline=pipeline, ) failed_exp_feedback_list_desc = T("scenarios.data_science.share:describe.trace").r( exp_and_feedback_list=trace.experiment_and_feedback_list_after_init(return_type="failed"), type="failed", + pipeline=pipeline, ) # Step 1: Identify problems - fb_problems = self.identify_feedback_problem( - scenario_desc=scenario_desc, - exp_feedback_list_desc=exp_feedback_list_desc, - sota_exp_desc=sota_exp_desc, - ) - for problem_name in fb_problems: - fb_problems[problem_name]["label"] = "FEEDBACK_PROBLEM" - all_problems = fb_problems + all_problems = {} + if len(trace.hist) >= 3: + fb_problems = self.identify_feedback_problem( + scenario_desc=scenario_desc, + exp_feedback_list_desc=exp_feedback_list_desc, + sota_exp_desc=sota_exp_desc, + ) + for problem_name in fb_problems: + fb_problems[problem_name]["label"] = "FEEDBACK_PROBLEM" + all_problems[problem_name] = fb_problems[problem_name] + + if len(trace.hist) < 9: + scen_problems = self.identify_scenario_problem( + scenario_desc=scenario_desc, + sota_exp_desc=sota_exp_desc, + ) + for problem_name in scen_problems: + scen_problems[problem_name]["label"] = "SCENARIO_PROBLEM" + all_problems[problem_name] = scen_problems[problem_name] # Step 1.5: Sample ideas from idea pool if DS_RD_SETTING.enable_knowledge_base: @@ -489,7 +528,7 @@ class DSProposalV2ExpGen(ExpGen): pickled_problem_name, new_hypothesis = self.hypothesis_rank( hypothesis_dict=hypothesis_dict, problem_dict=all_problems, - pipeline=pipeline, + trace=trace, ) # Step 3.5: Update knowledge base with the picked problem if DS_RD_SETTING.enable_knowledge_base: diff --git a/rdagent/scenarios/data_science/share.yaml b/rdagent/scenarios/data_science/share.yaml index dac88bae..6118f03f 100644 --- a/rdagent/scenarios/data_science/share.yaml +++ b/rdagent/scenarios/data_science/share.yaml @@ -40,47 +40,21 @@ describe: # some template to describe some object trace: |- {% if exp_and_feedback_list|length == 0 %} - No previous {% if type == "success" %}successful{% elif type == "failure" %}failed{% else %}successful or failed{% endif %} trial available. + No previous {% if type == "success" %}SOTA{% elif type == "failure" %}failed{% endif %} experiments available. {% else %} - {% if type == "success" %} - ## {{ heading | default('Trace of the successful trial') }} - {% elif type == "failure" %} - ## {{ heading | default('Trace of the failed trial') }} - {% else %} - ## {{ heading | default('Trace of all trials') }} - {% endif %} - - Before current trial, several {% if type == "success" %}successful{% elif type == "failure" %}failed{% else %}successful or failed{% endif %} trials are listed below. - {% if type == "success" %} - {{success_trial_desc | default('The current SOTA method is the combination of the best solutions of these trials.')}} - {% endif %} - - The trace order is from the earliest to the latest. Please focus more on the later trials. - {% for exp_and_feedback in exp_and_feedback_list %} - ### Experiment index: {{ loop.index }} - The experiment is designed based on hypothesis: {{ exp_and_feedback[0].hypothesis }} - + ## Experiment Index: {{ loop.index }} + Target Problem: {{ exp_and_feedback[0].hypothesis.problem_desc }} + {% if not pipeline %}Chosen Component: {{ exp_and_feedback[0].hypothesis.component }}{% endif %} + Proposed Hypothesis: {{ exp_and_feedback[0].hypothesis.hypothesis }} + Surpass Previous SOTA: {{ exp_and_feedback[1].decision }} {% if exp_and_feedback[0].result is none %} - Experiment score: Running buggy + Experiment Score: Running buggy + Experiment Error: {{ exp_and_feedback[1].reason }} {% else %} - Experiment score: {{ exp_and_feedback[0].result.loc["ensemble"].iloc[0] }} + Experiment Score: {{ exp_and_feedback[0].result.loc["ensemble"].iloc[0] }} + Experiment Feedback: {{ exp_and_feedback[1].reason }} {% endif %} - - Experiment feedback decision: {{ exp_and_feedback[1].decision }} - Reason: {{ exp_and_feedback[1].reason }} - {% endfor %} - {% endif %} - - drafting_trace: |- - {% if exp_and_feedback_list|length == 0 %} - No previous drafting experiments available. - {% else %} - {% for exp_and_feedback in exp_and_feedback_list %} - ## Drafting Experiment {{ loop.index }} - Task Design: {{ exp_and_feedback[0].pending_tasks_list[0][0].get_task_information() }} - Failure: {{ exp_and_feedback[1].reason }} - {% endfor %} {% endif %}