fix: merge datascience v3 and v2 (#974)

* add coder version

* merge cooder and feedback prompts

* align v2 and v3 proposal prompts

* fix a small bug

* fix a bug

* fix another bug

* support both function calling and json mode in v2 proposal

* fix minor bug

* reformat

* remove proposal v3

* fix a small bug in json mode

* fix CI

* remove tmp file

* remove v3 check

---------

Co-authored-by: Xu Yang <xuyang1@microsoft.com>

rdagent/components/coder/data_science/pipeline/__init__.py

@@ -98,21 +98,12 @@ class PipelineMultiProcessEvolvingStrategy(MultiProcessEvolvingStrategy):
             spec=T("scenarios.data_science.share:component_spec.Pipeline").r(),
             enable_model_dump=DS_RD_SETTING.enable_model_dump,
         )
-        if DS_RD_SETTING.proposal_version == "v3":
-            # FIXME: A temporary patch for BUILD
-            user_prompt = T(".prompts:pipeline_coder.user_v3").r(
-                competition_info=competition_info,
-                folder_spec=data_folder_info,
-                latest_code=workspace.file_dict.get("main.py"),
-                latest_code_feedback=prev_task_feedback,
-            )
-        else:
-            user_prompt = T(".prompts:pipeline_coder.user").r(
-                competition_info=competition_info,
-                folder_spec=data_folder_info,
-                latest_code=workspace.file_dict.get("main.py"),
-                latest_code_feedback=prev_task_feedback,
-            )
+        user_prompt = T(".prompts:pipeline_coder.user").r(
+            competition_info=competition_info,
+            folder_spec=data_folder_info,
+            latest_code=workspace.file_dict.get("main.py"),
+            latest_code_feedback=prev_task_feedback,
+        )
 
         for _ in range(5):
             pipeline_code = PythonAgentOut.extract_output(

rdagent/components/coder/data_science/pipeline/prompts.yaml

@@ -83,27 +83,6 @@ pipeline_coder:
     --------- Data Folder Description (All path are relative to the data folder) ---------
     {{ folder_spec }}
     
-    {% if latest_code %}
-    --------- Former code ---------
-    {{ latest_code }}
-    {% if latest_code_feedback is not none %}
-    --------- Feedback to former code ---------
-    {{ latest_code_feedback }}
-    The former code contains errors. You should correct the code based on the provided information, ensuring you do not repeat the same mistakes.
-    {% else %}
-    The former code is correct. You should try to improve the code based on the provided task while not changing the irrelevant parts.
-    {% endif %}
-    {% endif %} 
-
-    You should strictly follow the code specifications provided by the specification to implement the function.
-
-  user_v3: |-
-    --------- Competition Information ---------
-    {{ competition_info }}
-
-    --------- Data Folder Description (All path are relative to the data folder) ---------
-    {{ folder_spec }}
-    
     {% if latest_code %}
     --------- Former code ---------
     {{ latest_code }}

rdagent/oai/backend/base.py

@@ -499,6 +499,13 @@ class APIBackend(ABC):
                 self.cache.embedding_set(content_to_embedding_dict)
         return [content_to_embedding_dict[content] for content in input_content_list]  # type: ignore[misc]
 
+    @abstractmethod
+    def support_function_calling(self) -> bool:
+        """
+        Check if the backend supports function calling
+        """
+        raise NotImplementedError("Subclasses must implement this method")
+
     @abstractmethod
     def _calculate_token_from_messages(self, messages: list[dict[str, Any]]) -> int:
         """

rdagent/oai/backend/deprec.py

@@ -261,6 +261,13 @@ class DeprecBackend(APIBackend):
                     raise
         return encoding
 
+    def support_function_calling(self) -> bool:
+        """
+        Check if the backend supports function calling.
+        Currently, deprec backend does not support function calling so it returns False. #FIXME: maybe a mapping to the backend class is needed.
+        """
+        return False
+
     def _create_embedding_inner_function(  # type: ignore[no-untyped-def]
         self, input_content_list: list[str], *args, **kwargs
     ) -> list[list[float]]:  # noqa: ARG002

rdagent/oai/backend/litellm.py

@@ -7,6 +7,7 @@ from litellm import (
     completion,
     completion_cost,
     embedding,
+    supports_function_calling,
     supports_response_schema,
     token_counter,
 )
@@ -93,6 +94,12 @@ class LiteLLMAPIBackend(APIBackend):
         """
         if json_mode and supports_response_schema(model=LITELLM_SETTINGS.chat_model):
             kwargs["response_format"] = {"type": "json_object"}
+        elif not supports_response_schema(model=LITELLM_SETTINGS.chat_model) and "response_format" in kwargs:
+            logger.warning(
+                f"{LogColors.RED}Model {LITELLM_SETTINGS.chat_model} does not support response schema, ignoring response_format argument.{LogColors.END}",
+                tag="llm_messages",
+            )
+            kwargs.pop("response_format")
 
         if LITELLM_SETTINGS.log_llm_chat_content:
             logger.info(self._build_log_messages(messages), tag="llm_messages")
@@ -183,3 +190,9 @@ class LiteLLMAPIBackend(APIBackend):
             tag="token_cost",
         )
         return content, finish_reason
+
+    def support_function_calling(self) -> bool:
+        """
+        Check if the backend supports function calling
+        """
+        return supports_function_calling(model=LITELLM_SETTINGS.chat_model) and LITELLM_SETTINGS.enable_function_call

rdagent/oai/llm_conf.py

@@ -16,6 +16,9 @@ class LLMSettings(ExtendedBaseSettings):
     embedding_model: str = "text-embedding-3-small"
 
     reasoning_effort: Literal["low", "medium", "high"] | None = None
+    enable_function_call: bool = (
+        True  # Whether to enable function calling in chat models. may not work for models that do not support it.
+    )
 
     # Handling format
     reasoning_think_rm: bool = False

rdagent/scenarios/data_science/dev/feedback.py

@@ -87,29 +87,16 @@ class DSExperiment2Feedback(Experiment2Feedback):
                     )
 
         eda_output = exp.experiment_workspace.file_dict.get("EDA.md", None)
-        if DS_RD_SETTING.proposal_version == "v3":
-            # FIXME: Some minor changes. Did not have time to test the full.
-            system_prompt = T(".prompts:exp_feedback_v3.system").r(
-                scenario=self.scen.get_scenario_all_desc(eda_output=eda_output)
-            )
-            user_prompt = T(".prompts:exp_feedback_v3.user").r(
-                sota_desc=sota_desc,
-                cur_exp=exp,
-                diff_edition=diff_edition,
-                feedback_desc=feedback_desc,
-                cur_vs_sota_score=cur_vs_sota_score,
-            )
-        else:
-            system_prompt = T(".prompts:exp_feedback.system").r(
-                scenario=self.scen.get_scenario_all_desc(eda_output=eda_output)
-            )
-            user_prompt = T(".prompts:exp_feedback.user").r(
-                sota_desc=sota_desc,
-                cur_exp=exp,
-                diff_edition=diff_edition,
-                feedback_desc=feedback_desc,
-                cur_vs_sota_score=cur_vs_sota_score,
-            )
+        system_prompt = T(".prompts:exp_feedback.system").r(
+            scenario=self.scen.get_scenario_all_desc(eda_output=eda_output)
+        )
+        user_prompt = T(".prompts:exp_feedback.user").r(
+            sota_desc=sota_desc,
+            cur_exp=exp,
+            diff_edition=diff_edition,
+            feedback_desc=feedback_desc,
+            cur_vs_sota_score=cur_vs_sota_score,
+        )
 
         resp_dict = json.loads(
             APIBackend().build_messages_and_create_chat_completion(

rdagent/scenarios/data_science/dev/prompts.yaml

@@ -15,7 +15,9 @@ exp_feedback:
       - Recommend corrective actions explicitly.
       - Set `"Replace Best Result": "no"`.
       - Begin your `reasoning` with `[Submission format error]`, clearly stating the issues causing experiment failure.
-    - If submission passes, proceed to Step 2.
+    - If submission passes the submission format check:
+      - If this is the first valid submission ever, set `"Replace Best Result": "yes"`.
+      - Otherwise, proceed to Step 2.
 
     Step 2: Evaluate Alignment with Competition Requirements (if format correct)
     - GOAL: CAREFULLY ANALYZE WHETHER THE EXPERIMENTAL SETUP AND CODE MAY CAUSE MISALIGNMENT BETWEEN VALIDATION AND TEST PERFORMANCE.
@@ -59,6 +61,8 @@ exp_feedback:
     Provide detailed and constructive feedback structured as follows:
     Example JSON Structure for Result Analysis:
     {
+      "Submission Format Check": "yes or no",
+      "First Valid Submission": "yes or no",
       "Observations": "Clearly summarize current and SOTA ensemble results with exact scores and notable patterns. Limit to no more than three concise, data-focused sentences. Your observation must be grounded by explicit evidence from scenario description or code implementation, not just validation scores.",
       "Feedback for Hypothesis": Explicitly confirm or refute the hypothesis based on specific data points or performance trends. Limit to two sentences.",
       "Evaluation Aligned With Task": "yes or no",
@@ -110,11 +114,11 @@ exp_feedback:
     {{ cur_exp.experiment_workspace.all_codes }}
 
     ## Feedback of past experiments
-    {{ feedback_desc }}
+    {{ feedback_desc or "There has not been any experiments yet." }}
     Please refer to these hypotheses and feedback to help you recommend new experiment and hypothesis
 
     Tips:
-    - Step 1: If submission format has issues, prioritize fixing them before proceeding.
+    - Step 1: If submission format has issues, prioritize fixing them before proceeding. If the format is correct and it's the first valid submission ever (there has never been valid submissions in the past), set `"Replace Best Result": "yes"`. If the format is correct and this is not the first valid submission, proceed to Step 2.
     - Step 2: If evaluation alignment issues are identified (validation approach does not follow competition requirements), address these methodological discrepancies immediately.
     - Step 3: If new results significantly worse than SOTA, or repeated hyperparameter adjustments yield no improvement, it might be time to rethink or shift focus.
 

rdagent/scenarios/data_science/loop.py

@@ -90,14 +90,11 @@ class DataScienceRDLoop(RDLoop):
         from rdagent.scenarios.data_science.proposal.exp_gen.proposal import (
             DSProposalV1ExpGen,
             DSProposalV2ExpGen,
-            DSProposalV3ExpGen,
         )
 
         if class_uri == "rdagent.scenarios.data_science.proposal.exp_gen.DSExpGen":
-            if DS_RD_SETTING.proposal_version not in ["v1", "v2", "v3"]:
+            if DS_RD_SETTING.proposal_version not in ["v1", "v2"]:
                 return import_class(DS_RD_SETTING.proposal_version)(scen=scen)
-            if DS_RD_SETTING.proposal_version == "v3":
-                return DSProposalV3ExpGen(scen=scen)
             if DS_RD_SETTING.proposal_version == "v1":
                 return DSProposalV1ExpGen(scen=scen)
             if DS_RD_SETTING.proposal_version == "v2":

rdagent/scenarios/data_science/proposal/exp_gen/prompts_v2.yaml

@@ -1,30 +1,36 @@
 scenario_problem:
   system: |-
     {% include "scenarios.data_science.share:scen.role" %}
-    The user is improving a Kaggle competition implementation iteratively through traces where each new trace is modified from the current SOTA in the trace. If new trace surpasses the current SOTA, it will be the new SOTA. If not, it will be a failed experiment.
-    You will be provide with: 
-      1. A detailed competition scenario description;
-      2. Previous SOTA experiments and feedbacks, which are past SOTA experiments indexed from oldest to newest;
-      3. Previous failed experiments and feedbacks, which are ordered attempts that did not surpass the current SOTA implementation;
-    Your task is to analyze the given information and extract the **Scenario Problems** from the given materials.
+    The user is improving a Kaggle competition implementation iteratively. Each new iteration (trace) is typically a modification of the current overall State-of-the-Art (SOTA) solution. If a new trace's performance surpasses the current SOTA, it establishes a new SOTA. Otherwise, it is considered a failed experiment.
 
-    ## Scenario Problems
-    ### Definition
-    Scenario problems are specific, context-dependent challenges arising from a competition's dataset or domain. They fall into two categories:
-    1. Dataset Characteristics: Inherent structural or statistical properties of the dataset (such as imbalance, high dimensionality, collinearity, outliers, missing data, skewed distribution, time-based patterns, etc.).
-    2. Domain-specific Insights: Actionable knowledge derived from expertise in the competition's domain, enabling correct interpretation of data patterns or constraints. These insights are not evident from the data alone and require external context to resolve ambiguities, engineer features, or avoid invalid assumptions.
+    You will be provided with:
+    1. A detailed competition scenario description;
+    2. A history of previous SOTA experiments and their associated feedbacks, typically indexed or ordered from oldest to newest;
+    3. A history of previous failed experiments and their associated feedbacks, chronologically ordered, where each failed experiment did not surpass the SOTA that was current at the time of its execution;
 
-    ### Specification
-    {{ problem_spec }}
+    Your task is to analyze the provided information (primarily the scenario and current SOTA, if available) and identify a concise list of **Key Challenges** or **Core Problems** relevant to achieving success in this competition and improving the target metric. Aim for **FEWER BUT BETTER** challenges (e.g., 2-3 critical challenges), focusing on the most impactful aspects that can be methodically addressed.
+
+    ### Core Analysis Dimensions for Identifying Challenges
+    1. **SOTA Alignment Analysis**: (If SOTA is provided) Systematically compare the current SOTA implementation against dataset properties and domain knowledge to identify discrepancies or areas representing core challenges to overcome for enhancement.
+    2. **Gap Identification**: (If successful past solutions or common winning strategies are known/inferred) Examine what implicitly addressed problems or unexploited avenues these successful approaches highlight. These gaps can represent current challenges.
+    3. **Domain-Implementation Coherence Check**: Identify instances where technical approaches might violate domain constraints, oversimplify complex relationships, or miss domain-specific nuances. These incoherencies are challenges.
+    4. **Scenario-First Focus (No SOTA)**: If no SOTA implementation is available, the **primary identified challenge** should be foundational. It should focus on establishing a **simple, robust, and efficient baseline** that directly addresses the core task and evaluation metric. Avoid overly complex initial challenges.
+
+    ## Key Challenges / Core Problems
+    You **MUST** categorize each identified challenge into one of the following two types. This categorization should be based on the primary driver or nature of the challenge:
+    1. **Dataset-Driven Challenge**: Challenges primarily derived from addressing or leveraging inherent structural or statistical properties of the dataset (e.g., mitigating imbalance, managing high dimensionality, specific feature engineering needs for data types like text or time-series, handling missing data, transforming skewed distributions, accounting for collinearity or outliers).
+    2. **Domain-Informed Challenge**: Challenges primarily derived from correctly applying actionable knowledge specific to the competition's domain. This includes the correct interpretation of data patterns based on domain context, domain-specific feature engineering, adhering to known domain constraints, or avoiding invalid assumptions that data analysis alone might not reveal.
+
+    ### Specification for each Identified Challenge
+    1. The challenge should be specific and fine-grained. Avoid general or vague statements.
+    2. The challenge should be technical or methodological. Focus on design and implementation strategies that need to be solved, not simple runtime bugs (unless the bug points to a deeper architectural challenge or a persistent efficiency problem).
+    3. The challenge must be strictly aligned with the improvement of the target metric.
+    4. If no SOTA is available, at least one identified challenge must guide the creation of the simplest possible, yet potentially competitive, baseline model that can run to completion.
     
-    ### Core Analysis Dimensions
-    1. SOTA Mismatch Diagnosis: Systematically compare current implementations against both data properties and domain knowledge to identify critical discrepancies.
-    2. Gap Forensic Analysis: Examine successful solutions to reveal unstated problems they implicitly address through workarounds.
-    3. Domain-Implementation Conflict Detection: Identify instances where technical approaches violate domain constraints or oversimplify complex relationships.
-    4. In case there is no SOTA implementation, your scenario problem should focus on the scenario itself.
-
+    {% if problem_output_format is not none %}
     ### Output Format
     {{ problem_output_format }}
+    {% endif %}
 
   user: |-
     # Scenario Description
@@ -36,13 +42,17 @@ scenario_problem:
 feedback_problem:
   system: |-
     {% include "scenarios.data_science.share:scen.role" %}
-    The user is improving a Kaggle competition implementation iteratively through traces where each new trace is modified from the current SOTA in the trace. If new trace surpasses the current SOTA, it will be the new SOTA. If not, it will be a failed experiment.
-    You will be provided with: 
-      1. A detailed competition scenario description;
-      2. Previous SOTA experiments and feedbacks, which are past SOTA experiments indexed from oldest to newest;
-      3. Previous failed experiments and feedbacks, which are ordered attempts that did not surpass the current SOTA implementation;
-      4. The current SOTA implementation and feedback, which is the latest SOTA experiments from the previous experiments;
-    Your task is to analyze the given information and extract the **Feedback Problems** from the previous experiments or the current SOTA implementation.
+    The user is improving a Kaggle competition implementation iteratively through traces. Each new trace is a modification of the State-of-the-Art (SOTA) implementation that was current at the time that trace was initiated. If a new trace's performance surpasses the SOTA it aimed to improve upon, it becomes the new SOTA. If not, it is considered a failed experiment.
+
+    You will be provided with:
+    1. A detailed competition scenario description;
+    2. A history of previous SOTA experiments and their associated feedbacks, typically indexed or ordered from oldest to newest;
+    3. A history of previous failed experiments and their associated feedbacks, chronologically ordered, where each failed experiment did not surpass the SOTA that was current at the time of its execution;
+    4. The overall current SOTA implementation and its associated feedback, which represents the best-performing experiment from the entire history provided up to this point.
+
+    Your task is to analyze all this provided historical information and extract **Key Learnings and Unresolved Challenges** from the experiment history. These should guide concrete improvements in subsequent iterations.
+
+    ## Key Learnings and Unresolved Challenges
 
     
     {% if inject_diverse %}
@@ -53,31 +63,38 @@ feedback_problem:
     3. Do not do incremental exploration on the previous problems.
     {% endif %}
 
-    ## Feedback Problems
     ### Definition
-    Feedback problems are specific and fine-grained technical, or methodological issues within the previous experiments or the current SOTA implementation.
+    Key Learnings and Unresolved Challenges are specific, fine-grained technical or methodological observations, persistent issues, or patterns identified within previous experiments or the current SOTA implementation. These are primarily derived from explicit feedback, code analysis, or patterns in the trace history, and should highlight problems that need solving or learnings that should inform future hypotheses.
 
-    ### Guidelines
-    Here are few guidelines to help you identify the feedback problems:
-    1. Feedback Analysis
-      - Extract explicit issues directly stated in the feedback.
-      - Infer implicit issues from feedback context.
-    2. Code Review
-      - Feature Engineering. Check for missing, redundant, or improper features and the mismatch between features and models.
-      - Model Architecture. Assess the compatibility between the model type and the problem domain. Verify model architecture and hyperparameters.
-      - Ensemble. Check if ensemble methods are optimized. Identify underperforming base models to remove from the ensemble.
-      - Training. Validate hyperparameter (e.g., learning rate, batch size), loss functions, and regularization. Determine if hyperparameters are optimized based on prior experiment traces.
-    3. Trace History Analysis
-      - Flag unresolved and persistent issues recurring across traces.
-      - Highlight partial fixes (e.g., inappropriate feature engineering).
-      - Identify unexplored directions (e.g. new features, new model structures) from prior traces.
-      - Identify potential unsolved time/memory constraints from previous experiments trace.
+    ### Guidelines for Identification
+    Here are guidelines to help you identify these Learnings and Challenges:
+
+    1. **Feedback Analysis**:
+      - **Explicit Issues/Suggestions as Challenges**: Extract critical issues, errors (especially those pointing to deeper problems like resource limits or incorrect submission formats if not easily fixed), or direct suggestions from feedback that represent unresolved problems.
+      - **Implicit Gaps as Challenges**: Infer unaddressed points, shortcomings, or areas for improvement implied by feedback that constitute ongoing challenges.
+      - **Time/Memory Constraints as Critical Challenges**: If previous experiments indicate failures due to time/memory limitations, or inefficient resource usage, this **MUST** be listed as a critical challenge. This includes identifying if the current SOTA or failed experiments are too complex for the given time limits.
+
+    2. **Implementation Review (of SOTA or relevant past experiments)**:
+      - **Suboptimal Design as Challenges**: Identify potentially suboptimal feature selection, model architecture, hyperparameters, ensemble strategy, training/validation processes that appear as recurring problems or limit performance, framing them as challenges to be addressed.
+      - **Common Implementation Issues**: Note the coding issues that are blocking for receiving a reasonable result. For example, the submission format was repeatedly incorrect despite attempts to fix it, this is an unresolved challenge related to the implementation.
+
+    3. **Trace History Analysis (Trends & Patterns as Challenges)**:
+      - **Persistent Issues/Errors as Challenges**: Flag unresolved negative patterns, errors (e.g., recurrent `zipfile.BadZipFile`, CUDA label errors, submission format mismatches if they persist after attempts to fix), or suboptimal outcomes that recur across multiple experiment traces. These represent core unresolved challenges.
+      - **Ineffective/Partial Fixes**: Highlight if previous changes intended to solve a problem were only partially successful or ineffective, meaning the core challenge remains.
+      - **Unexplored Promising Directions**: Identify potentially valuable approaches (e.g., alternative feature sets, different model families, advanced optimization techniques) that were hinted at by feedback, briefly tried without full exploration, or represent logical next steps given the trajectory of past experiments.
+      - **Constraint Violations/Inefficiencies as Challenges**: Explicitly note any unaddressed time or memory constraint violations or significant computational inefficiencies as critical challenges that need strategic solutions.
+
+    ### Specification for each Learning/Challenge
+    1. The Learning/Challenge must be specific, actionable, and evidence-based (tied to feedback, code, or trace history).
+    2. It should focus on technical or methodological problems that need solving.
+    3. Clearly state the learning or articulate the challenge.
+    4. Addressing the challenge or applying the learning should have a plausible positive impact on the target metric or successful execution.
+    5. The challenge must be strictly aligned with the improvement of the target metric.
     
-    ### Specification
-    {{ problem_spec }}
-
+    {% if problem_output_format is not none %}
     ### Output Format
     {{ problem_output_format }}
+    {% endif %}
 
   user: |-
     # Scenario Description
@@ -89,68 +106,140 @@ feedback_problem:
     # Current SOTA Implementation
     {{ sota_exp_desc }}
 
+scenario_description: |-
+  {% if use_raw_description -%}
+  ====== Background ======
+  {{ raw_description }}
+
+  {% else %}
+  ====== Background ======
+  {{ background }}
+
+  {% if eda_output is not none %}
+  ====== Data Overview (EDA) ======
+  {{ eda_output }}
+  {% endif %}
+
+  ====== Submission Format ======
+  Please ensure your submission adheres to the following specifications:
+  {{ submission_specifications }}
+
+  ====== Important Guidelines ======
+  Before submitting your results, please note the following:
+  - We have numerous tests in place to check your code.
+  - Ensure your submission is genuine.
+  - Do not manipulate data or return values solely to pass preliminary tests, as this will not lead to successful final evaluation.
+
+  {% endif %}
+
+  ====== Evaluation ======
+  {% if not use_raw_description and metric_name %}
+  The primary evaluation metric for this task is: **{{ metric_name }}**.
+  {% endif %}
+  This metric is considered better when it is **{% if metric_direction %}larger{% else %}smaller{% endif %}**.
+
+  {% if evaluation is not none %}
+  Additional Evaluation Details:
+  {{ evaluation }}
+  {% endif %}
+
+  {% if time_limit %}
+  ====== Time Limit ======
+  Your code's execution is limited to **{{ time_limit }}**.
+  Please optimize your model and parameters to ensure your code runs within this specified time constraint.
+  {% endif %}
+
 hypothesis_gen:
   system: |-
     {% include "scenarios.data_science.share:scen.role" %}
-    The user is improving a Kaggle competition implementation iteratively through traces where each new trace is modified from the current SOTA in the trace. If new trace surpasses the current SOTA, it will be the new SOTA. If not, it will be a failed experiment.
-    You will be provided with: 
-      1. A detailed competition scenario description;
-      2. Previous SOTA experiments and feedbacks, which are past SOTA experiments indexed from oldest to newest;
-      3. Previous failed experiments and feedbacks, which are ordered attempts that did not surpass the current SOTA implementation;
-      4. The current SOTA implementation and feedback, which is the latest SOTA experiments from the previous experiments;
-      5. A list of identified problems, which are specific technical or methodological issues within the previous experiments;
-    Your task is to:
-      1. **Hypothesis Proposal**: Propose testable hypotheses to address the identified problems.
-      2. **Hypothesis Evaluation**: Evaluate the proposed hypotheses across multiple dimensions.
+    The user is iteratively improving a Kaggle competition implementation. Each new iteration (trace) is a modification of the current State-of-the-Art (SOTA). If a new trace surpasses the current SOTA, it becomes the new SOTA. Otherwise, it's a failed experiment.
+    You will be provided with:
+    1. A detailed competition scenario description.
+    2. Previous SOTA experiments and feedback (chronologically ordered, oldest to newest).
+    3. Previous failed experiments and feedback (ordered attempts that did not improve SOTA).
+    4. The current SOTA implementation and feedback (the latest successful experiment).
+    5. A list of identified **Challenges** from history), which we will refer to as "Identified Challenges" below.
+
+    Your task is to perform two main steps:
+    1. **Hypothesis Proposal**: For each relevant Identified Challenge, propose one specific, testable hypothesis.
+    2. **Hypothesis Evaluation**: Evaluate each proposed hypothesis across multiple dimensions.
 
     {% if enable_idea_pool %}
-    In order to assist you in the hypothesis proposal, the user has sampled a list of ideas for each of the identified problems.
-    The ideas are extracted methods or techniques from previous SOTA implementations of other competitions.
-    These ideas can potentially tackle the identified problems and improve the current SOTA implementation but you should decide whether to use them or not.
-    To specific problem, if you choose to use the given idea, you should modify it to a proper hypothesis and also mark the inspired flag as True.
+    To help you propose hypotheses, the user may provide a list of ideas for each Identified Challenge. These ideas are methods or techniques from successful SOTA implementations in other competitions.
+    Evaluate these ideas: they might help address the Identified Challenges and improve the current SOTA. You must decide whether to use them. If you adapt a provided idea for a specific Challenge into your hypothesis, ensure you clearly state this by setting the 'inspired' flag to True for that hypothesis.
     {% endif %}
 
     # Task 1: Hypothesis Proposal
-    For each identified problem, propose a hypothesis to improve the current SOTA implementation.
+    First note that the user might provide a list of challenges containing duplicates. You should only propose one hypothesis for each unique challenge. If a challenge is a duplicate of a previous one, you can skip it.
+    For each Identified Challenge, propose one hypothesis corresponding to the Challenge, aimed at improving the current SOTA implementation or establishing a robust initial SOTA.
 
-    ## Hypothesis Guidelines
-    Here are few guidelines to help you formulate hypotheses:
-    1. Problem Impact Analysis
-      - Quantify how the problem degrades performance.
-    2. Previous Experiments Analysis
-      - For previous SOTA experiments, analyze insights and implicit patterns that can be leveraged to improve the current SOTA implementation.
-      - For failed experiments, think about the persistent problems they facing. If these experiments consistently failed due to time/memory constraints, prioritize changes on efficiency.
-    3. Actionable Changes
-      - If the problem relates to time/memory constraints, consider smaller model sizes or alternative algorithms with reduced complexity.
-      - If the problem involves underperforming models, propose removing or replacing models of significantly worse performance.
-      - If the problem relates to hyperparameter tuning, recommend a specific method or strategy for tuning.
-    4. Note on Time/Memory Constraints
-      - If prior experiments failed due to time/memory limitations, assume your new hypothesis will face the same constraints. In this case, prioritize efficiency and **ONLY** response to the problems related to time/memory constraints in your response dictionary.
-      - Besides, do not compromise performance merely for efficiency since the current SOTA implementation do not encounter the constraints. You should think about how to balance the efficiency and performance so that your new hypothesis can be executed successfully and achieve satisfactory performance. 
-    5. Note on Drafting the First Implementation
-      - In case there is no SOTA implementation, you should draft the first implementation. In this case, your hypothesis should not only cover the problem but also on the overall design such as how to do the feature engineering and the model selection.
+    ## 1.1. Steps to Hypothesize
+    Follow these steps to formulate effective hypotheses:
+
+    1. **Understanding the Challenge**:
+      - Analyze the Identified Challenge to understand its root cause and potential impact on the competition's target metric or successful execution.
+      - If the Challenge stems from past experiments (SOTA or failed), review the specifics of those experiments to ensure the proposed hypothesis offers a novel, more effective, or correctly implemented solution.
+      - If the Challenge relates to persistent problems from failed experiments (e.g., experiments consistently failed due to time/memory constraints, or recurrent errors like incorrect data loading or submission formats), your hypothesis MUST propose a direct and robust tentative solution.
+    2. **Drafting the First Implementation (if no SOTA exists)**:
+      - If there is no SOTA implementation yet (i.e., you are drafting the first implementation based on a foundational Challenge identified in the previous step), your primary hypothesis **MUST** focus on a plan creating the **simplest possible, yet potentially competitive, baseline model** that directly addresses this foundational Challenge and can run to completion reliably.
+      - This initial hypothesis should define the core data processing, feature engineering, model choice, and submission generation steps in their most straightforward form. Avoid proposing complex, multi-faceted solutions or combining multiple distinct techniques for the very first run.
+    3. **Actionable Changes**:
+      - If a Challenge relates to **time/memory constraints**, the hypothesis must propose concrete changes (e.g., simpler model architecture, reduced number of cross-validation folds or training epochs, data sub-sampling, more efficient data loading, optimized code). The aim is to create a solution that can run to completion reliably. While performance is key, a non-completing experiment has zero performance. Strive for the best possible performance *within* the operational constraints.
+      - If a Challenge involves underperforming models (e.g., in an ensemble), propose specific actions like removing or replacing those models.
+      - If a Challenge relates to hyperparameter tuning, recommend a specific method or strategy (e.g., "Implement Bayesian optimization for RandomForest's learning_rate and num_leaves to address the 'suboptimal hyperparameter' challenge").
+      - If a Challenge points to data loading, preprocessing, or submission format errors, the hypothesis must detail the exact changes required to rectify these issues.
     {% if enable_idea_pool %}
-    6. Idea Reference
-      - Each idea is a method, technique or trick that contributes to high performance from other competition implementation under similar problem. You are free to use them as an inspiration for your hypothesis proposal.
+    4. **Idea Reference**: Provided ideas are methods, techniques, or tricks from high-performing implementations in other competitions addressing similar problems. Use them as inspiration if you find them suitable for the current Challenge.
     {% endif %}
 
-    ## Hypothesis Specification
-    {{ hypothesis_spec }}
+    ## 1.2. Guidelines for Writing Hypotheses
 
+    1. **Be Specific and Decisive**:
+      - Clearly state the exact, unambiguous change(s) being proposed. Avoid vague goals like "improve the model" or "optimize the pipeline."
+      - The hypothesis must propose a single, clear course of action. Do not suggest alternatives (e.g., "try method A or method B").
+      - The hypothesis statement must be direct and definitive, without phrases like "for example," "e.g.," or "might involve."
+      - The hypothesis must be more informative and decisive than the Challenge it addresses. It should not simply restate the Challenge or suggest a general approach without specifics.
+    2. **Ensure Testability and Actionability**:
+      - The hypothesis must describe an action or change that can be practically implemented and tested.
+      - If the hypothesis is about improving SOTA, it should clearly state the expected improvement, typically related to a measurable performance metric or successful execution.
+        - *Good Example (Optimizer)*: "Changing the optimizer from Adam to AdamW and applying a learning rate of 1e-4 for the BERT-based text classifier will increase F1-score by at least 2% on the validation set.""
+        - *Good Example (Format)*: "To address the 'incorrect submission format' challenge, modify the submission generation step to create a CSV file with columns 'Id' and 'Category', mapping predicted class indices to original category labels, which is expected to pass submission validation."
+        - *Good Example (Efficiency)*: "To resolve the 'timeout during training' challenge, reduce `NUM_EPOCHS` from 5 to 2 and `N_SPLITS` for cross-validation from 5 to 3 in the main training loop, aiming to complete execution within the 1-hour limit while minimizing impact on the F1-score."
+        - *Poor Example*: "Tune the model for better results."
+      - If the hypothesis is about establishing the first solution, it should clearly outline the expected outcome -- RUNNABILITY and CORRECTNESS. Prioritize getting a valid submission out, even with a very basic model or pipeline.
+        - *Good Example*: "Implement a simple RandomForest classifier with default parameters, using 5-fold cross-validation for model evaluation. This will lead to a decent baseline model that can run to completion and generate a valid submission file."
+    3. **Align with Current SOTA and Identified Challenges**:
+      - The hypothesis must be directly relevant to improving the *current* State-of-the-Art (SOTA) implementation or establishing a new SOTA if none exists.
+      - It must directly address one of the `Identified Challenges` provided as input.
+    4. **Maintain Singular Focus within Hypothesis**:
+      - If a hypothesis involves multiple adjustments, these must be tightly correlated and contribute to a single, unified conceptual change addressing the core of the Identified Challenge.
+      - Avoid bundling multiple independent or unrelated ideas into a single hypothesis. Each hypothesis should test one core concept.
+    5. **Address the Overall Pipeline (for Pipeline-Focused Tasks)**:
+      - The hypothesis should address improvements to the end-to-end pipeline.
+      - It can propose coordinated changes across multiple parts of the SOTA implementation if these are necessary to achieve a significant pipeline-level improvement to address the Challenge. (Note: Even for pipeline-focused hypotheses, you will still select the single *most relevant* primary component tag during the evaluation task.)
 
     # Task 2: Hypothesis Evaluation
-    After proposing the hypothesis, your second task is to evaluate the hypothesis from multiple dimensions.
+    After proposing one hypothesis for each relevant Identified Challenge, evaluate each one.
 
-    ## Evaluation Instruction
-    Firstly, you should tag the hypothesis with one of the following components. If the hypothesis is related to multiple components, you should choose the most relevant one.
-    {{ component_desc }}
+    ## 2.1. Evaluation Instruction
+    For each individual hypothesis you proposed in Task 1, perform the following two evaluation steps:
 
-    Secondly, please score the proposed hypothesis from 1 to 10 for each of the following dimensions (where 1 means lowest and 10 means highest):
-    1. Problem-Hypothesis Alignment: How well the hypothesis addresses the identified problem.
-    2. Expected Impact: The estimated improvement after applying the hypothesis to current SOTA implementation.
-    3. Novelty: Degree of innovation compared to previous attempts. If the proposed hypothesis is similar to previous experiments' hypothesis, assign novelty score to one.
-    4. Feasibility: The ease of implementing the proposed hypothesis in the current SOTA implementation.
-    5. Risk-Reward Balance: The exploration-exploitation balance of the proposed hypothesis.
+    1. **Assign a Component Tag:** Assign a single component tag to the hypothesis. Choose the **single most relevant** tag from the official list below, even if the hypothesis appears to touch upon multiple areas. Use the following detailed descriptions to understand the scope and boundaries of each component.
+
+      - **`DataLoadSpec`**: Responsible for loading raw competition data, ensuring data is converted to the correct types, and potentially providing an initial exploratory data analysis (EDA) summary. (e.g., fixing `zipfile.BadZipFile` by improving loading logic).
+      - **`FeatureEng`**: Focuses on transforming raw data into meaningful features suitable for model consumption. Key responsibilities include maintaining data shape consistency, preventing data leakage during feature creation, and optimizing features for model performance. Feature engineering should be model-agnostic.
+      - **`Model`**: Involves model building (developing new models to address the problem), model tuning (optimizing existing models for better performance), or model removal. This component also handles data operations or augmentations closely tied to a specific model framework (e.g., PyTorch `Datasets` & `DataLoaders`, TensorFlow `tf.data`, or fixing CUDA label errors by ensuring correct label mapping before loss calculation).
+      - **`Ensemble`**: Combines predictions from multiple models using various ensemble strategies.
+      - **`Workflow`**: Integrates all pipeline components, orchestrating the flow from data loading through to final output generation (e.g., correcting `submission.csv` column names or structure, managing overall pipeline execution logic for efficiency).
+
+    2. **Score the Hypothesis:** For each hypothesis, provide a score from 1 (lowest/worst) to 10 (highest/best) on each of the following five dimensions. Base your scores on all provided information.
+
+      - **Challenge-Hypothesis Alignment (Score: 1-10):** How directly and effectively does the hypothesis address the core issues of the `Identified Challenge` it targets? A higher score means a stronger, more direct alignment.
+      - **Expected Impact (Score: 1-10):** What is the estimated magnitude of improvement (e.g., in the primary competition metric, efficiency, robustness, or successful execution) if this hypothesis is successfully implemented? Higher scores for greater positive impact.
+      - **Novelty (Score: 1-10):** How innovative or original is this hypothesis when compared to the approaches and ideas evident in the `previous SOTA experiments` and `previous failed experiments`? Assign a score of 1 if the hypothesis is a repeat or substantially similar to a previously attempted hypothesis (whether successful or failed), UNLESS the previous attempt clearly failed due to a trivial implementation bug and the current hypothesis proposes the correct implementation of the same core idea.
+      - **Feasibility (Score: 1-10):** How easily and practically can this hypothesis be implemented and *run to completion* within the existing SOTA codebase and operational constraints (e.g., allowed time for training/inference, available compute resources, overall complexity)? Higher scores for easier implementation and higher likelihood of successful execution.
+      - **Risk-Reward Balance (Score: 1-10):** Considering the potential for significant improvement (reward) versus the probability of failure, negative side-effects, or excessive resource consumption (risk), how optimal is this balance? A high score indicates a favorable balance.
+      - **Prioritization for Critical Challenges:** If a hypothesis directly and credibly addresses a **critical Challenge that caused prior experiment failures** (e.g., timeout, persistent data loading errors, incorrect submission format preventing any score), its **Expected Impact** and **Risk-Reward Balance** should generally be scored highly (e.g., 8-10), and **Feasibility** should also be high if the proposed solution is indeed simpler, more direct, or more efficient. This ensures such critical hypotheses are prioritized.
 
     {% if inject_diverse %}
     # Focus on Diversity!!
@@ -160,10 +249,11 @@ hypothesis_gen:
     3. Think out of the box and explore the hypothesis that are not covered by the previous experiments and feedbacks, but are reasonable and aligned with the identified problems. 
     4. Do not do incremental exploration on the previous problems, like lightgbm -> xgboost, or 1dCNN -> 2dCNN. Totally different hypothesis on model\data\feature\ensemble\workflow level are welcomed.
     {% endif %}
-
-
+    
+    {% if hypothesis_output_format is not none %}
     ## Final Output Format in JSON Schema:
     {{ hypothesis_output_format }}
+    {% endif %}
     
   user: |-
     # Scenario Description
@@ -175,40 +265,98 @@ hypothesis_gen:
     # Current SOTA Implementation
     {{ sota_exp_desc }}
 
-    # Identified Problems{% if enable_idea_pool %} with Sampled Ideas{% endif %}
+    # Identified Challenges{% if enable_idea_pool %} with Sampled Ideas{% endif %}
     {{ problems }}
 
 task_gen:
   system: |-
     {% include "scenarios.data_science.share:scen.role" %}
-    The user is improving a Kaggle competition implementation iteratively through traces where each new trace is modified from the current SOTA in the trace, not necessarily the immediate predecessor.
-    You will be provided with: 
-      1. A detailed competition scenario description;
-      2. Previous SOTA experiments and feedbacks, which are past SOTA experiments indexed from oldest to newest;
-      3. Previous failed experiments and feedbacks, which are ordered attempts that did not surpass the current SOTA implementation;
-      4. The current SOTA implementation and feedback, which is the latest SOTA experiments from the previous experiments;
-      5. A proposed hypothesis to improve the current SOTA implementation;
+    The user is iteratively developing a Kaggle competition solution. Each new iteration aims to improve upon the current State-of-the-Art (SOTA) implementation by applying a specific hypothesis that addresses an identified challenge. The new trace is based on the current SOTA; the SOTA itself evolves.
 
-    # Step 1: Task Design
-    Your first task is to generate new {{ targets }} based on the proposed hypothesis. Your task should very detailed with specific steps and instructions. The task should be specific and fine-grained, avoiding general or vague statements.
+    You will be provided with the following inputs:
+    1. **Competition Scenario Description**: Details about the competition (task type, data, evaluation metric, time limits, etc.).
+    2. **Previous SOTA Experiments & Feedback**: (If available) A history of successful implementations, ordered chronologically.
+    3. **Previous Failed Experiments & Feedback**: (If available) A history of unsuccessful attempts, which are crucial for learning.
+    4. **Current SOTA Implementation & Feedback**: (If available) Details of the best-performing solution so far. **If no SOTA implementation is provided, your primary task is to sketch the initial, simplest possible, end-to-end `main.py` workflow.**
+    5. **Proposed Hypothesis**: One, or more specific hypotheses aimed at improving the current SOTA or forming the basis of an initial SOTA. This hypothesis directly addresses an "Identified Challenge" from a previous analysis step.
 
-    ## Specification
-    {{ task_specification }}
+    Your primary goal is to generate a detailed, step-by-step **sketch or refinement plan** for a new data processing and modeling pipeline, specifically for the main workflow script (`main.py`), that effectively implements the `Proposed Hypothesis`. This sketch will guide a developer to write the code correctly.
 
-    ## Task Design Guidelines
-    1. The task should be concise with several steps each only in a few sentences. 
-    2. DO NOT repeat the details which has already included in the SOTA code. If the SOTA code has covered the steps perfectly, you should not repeat the steps in detail. 
-    3. DO NOT write any code in the task description!
-    4. Observe reasons from failed experiments and feedback to prevent repeating similar mistakes in analogous situations.
-    5. Specific and Non-Vague
-      - Avoid vague statements like "choose a proper model" Instead, specify the exact task to be made.
-      - No phrases like "for example" or "eg.," should be used in the task. Give a clear decision in the task.
-    6. Resource limitations
-      - The user will give you some failed experiments and feedbacks. If the former experiments faced time/memory constraints, it means it's very likely that your generated task will also face the same constraints. In this case, you should design a task that prioritize efficiency in terms of time and space complexity.
+    ### BACKGROUND CONTEXT: Pipeline Implementation Standards & Constraints ###
+
+    The `main.py` sketch you generate should lead to a pipeline implementation that adheres to the following standards. These are guiding principles for the final *outcome* of your sketch:
+
+    1. **Program Execution**: The resulting `main.py` script must be executable via `python main.py` without command-line parameters. Configurations should be hardcoded for simplicity.
+    2. **File Handling**:
+      - Implement robust handling of file encodings and delimiters.
+      - Input files are under `{% include "scenarios.data_science.share:scen.input_path" %}`. The sketch must detail how they are loaded and, if multiple, combined or processed.
+      - Test indices must be determined from a dedicated test index file (if available) or by the order in the test data file. **Crucially, DO NOT use the sample submission file to infer test indices or the number of test samples.**
+      - Ensure actual data (not just filenames) is loaded during the data loading phase.
+      - If data is in zip files, the sketch should advise on robust loading, e.g., pre-extraction or careful handling if using multiprocessing in data loaders.
+    3. **Data Preprocessing**:
+      - Convert data to correct types (numeric, categorical, parse dates).
+      - Optimize memory usage (e.g., downcasting, chunk processing if essential and the hypothesis supports it).
+      - Implement domain-specific preprocessing relevant to the hypothesis (e.g., text tokenization, image resizing/augmentation).
+    4. **Code Standards**:
+      - The pipeline must **NOT** use progress bars (e.g., `tqdm`) in the submission code.
+      - Reiterate: **DO NOT** use the sample submission file to extract test indices or any other information beyond the required column names and format for the output file.
+      - Ensure no features are inadvertently excluded during processing.
+    5. **Preferred Technologies & Methodological Notes**:
+      - NN models: Prefer PyTorch (over TensorFlow) if no SOTA or hypothesis dictates otherwise. Prioritize fine-tuning pre-trained models.
+      - Decision Tree models: Prefer XGBoost or RandomForest over LightGBM unless SOTA or hypothesis dictates otherwise.
+    6. **General Data Science Considerations**:
+      - Design for scalability.
+      - Handle missing values and outliers appropriately as guided by the hypothesis or SOTA.
+      - Ensure consistency between feature data types and any transformations applied.
+      - Prevent data leakage from test/validation sets into any training stage.
+    7. **Resource Utilization**: Leverage GPU and multiprocessing where appropriate and beneficial, if consistent with the hypothesis and efficiency goals.
+    8. **Metric Calculation and Storage (`scores.csv`)**:
+      - Calculate the official competition metric on a proper validation set (e.g., K-fold CV, typically 3-5 folds unless efficiency dictates fewer). Save results to `scores.csv`.
+      - The sketch must ensure this step is included. A successful run should always produce scores.
+      - `scores.csv` must have an index with model names and the literal string "ensemble" (lowercase). Columns should be "Model" (the name of the model or the ensemble strategy), and the exact metric name (e.g., "AUC").
+      - When only one model is used, its score should be present, and an "ensemble" score (which would be the same as the single model's score in this case) must also be recorded.
+      - Ensure validation metrics and processes are consistent across all parts of the pipeline. Avoid changes that would alter how validation metrics are calculated unless that is part of the hypothesis.
+    9. **Submission File (`submission.csv`)**: Generate `submission.csv` in the **exact format** required (column names, order, data types), as detailed by `sample_submission.csv` in the `Competition Scenario Description`. This is a critical step.
+
+    ### END OF BACKGROUND CONTEXT ###
+
+    # Guidelines for Sketching the `main.py` Workflow
+
+    YOUR TASK IS TO create a conceptual sketch for drafting or updating the `main.py` workflow. This is a plan, not code.
+
+    1. **No Code**: The sketch **MUST NOT** contain any programming code, specific library calls, or pseudo-code. Describe steps conceptually (e.g., "Load training data from {% include "scenarios.data_science.share:scen.input_path" %}/train.csv"). List specific algorithm names where appropriate (e.g., "Apply XGBoost classifier," "Use Isotonic Regression for calibration").
+    2. **Structure and Conciseness**:
+      - If SOTA exists, understand its structure first.
+      - If no SOTA, outline a clear, logical sequence of steps for the new `main.py`.
+    3. **Leverage SOTA or Design a New One**:
+      - **If a `Current SOTA Implementation` is provided**: Your sketch must primarily detail the **minimal and targeted changes, additions, or replacements** needed to integrate the `Proposed Hypothesis` into that SOTA. Focus only on what needs to change.
+      - **If NO `Current SOTA Implementation` is provided (Initial Version)**: This is critical. Your sketch **MUST** describe a **COMPLETE, END-TO-END, YET SIMPLEST POSSIBLE baseline pipeline**.
+        - It must cover: Data loading (from specified paths), essential preprocessing (as per hypothesis or minimal viable), a basic model implementation (as per hypothesis), a simple validation strategy (e.g., a single train-validation split or fewer folds if CV is too complex initially), generation of `scores.csv`, and `submission.csv` in the correct format.
+        - The overriding goal for this initial sketch is **RUNNABILITY and CORRECTNESS of the pipeline structure**. Prioritize getting a valid submission out, even with a very basic model. Avoid any complexity not absolutely mandated by the core hypothesis or competition basics.
+    4. **Learn from Past Failures**:
+      - If `Previous Failed Experiments & Feedback` are provided, analyze them meticulously. Design the sketch to explicitly avoid repeating similar mistakes, especially if failures relate to the current hypothesis, data handling, submission format, or resource usage (timeouts).
+      - If a hypothesis aims to fix a past failure, the sketch should detail precisely how the fix is implemented.
+    5. **Specificity and Clarity**:
+      - Be unambiguous. Instead of "select model," if the hypothesis implies "Train an EfficientNet-B0 model," state that.
+      - The sketch must be definitive. No open-ended options or phrases like "for example," or "e.g.," within a step's action.
+    6. **Resource Constraints & Efficiency**:
+      - Always design the workflow to execute within the competition `Time Limit`.
+      - If `Previous Failed Experiments` explicitly state time/memory constraint issues, your sketch **MUST** make efficiency the **TOP PRIORITY**. Clearly state `[EFFICIENCY AS TOP PRIORITY]` at the beginning of your sketch.
+      - The sketch must then detail *specific measures* to achieve this (e.g., "Reduce CV folds to 2," "Limit training to 3 epochs," "Use a smaller pre-trained model like MobileNetV2," "Subsample training data to 50% if full dataset causes timeout").
+      - Even if the `Proposed Hypothesis` is not about efficiency, if past experiments failed due to timeouts or the dataset/model is complex, the sketch **must still incorporate measures to improve overall pipeline efficiency**. This might involve simplifying aspects unrelated to the core hypothesis (e.g., reducing image resolution, simpler feature engineering) to ensure the hypothesis can be tested within limits.
+      - The goal is a workflow that successfully implements and validates the `Proposed Hypothesis` effectively, balancing performance with strict resource constraints. An experiment that times out provides no information.
       - If you plan to prioritize efficiency, you can modify the parts which is not related to the hypothesis. Which means your task should still able to validate the hypothesis.
       - Add [EFFICIENCY AS PRIORITY] tag in the task description to indicate that the task takes efficiency as a priority.
       - Although the task should prioritize efficiency, it should not be the only focus. The task should also be aligned with the proposed hypothesis and the current SOTA implementation.
-
+    7. **Reminders of Common Mistakes (Especially for New `main.py`)**: At the end of your sketch, include a "Key Reminders for Developer" section. Add the following reminders if appropriate.
+      - Ensure all input files are loaded from their exact paths under `{% include "scenarios.data_science.share:scen.input_path" %}` (e.g., `{% include "scenarios.data_science.share:scen.input_path" %}<competition_name>/train.csv`)."
+      - Verify `submission.csv` strictly adheres to format: columns, correct data types, and no extra index.
+      - "Implement correct label mapping for classification tasks (e.g., 0-indexed, contiguous integers for loss functions like PyTorch's CrossEntropyLoss) to prevent runtime errors."
+      - Handle file I/O robustly, especially for zipped data or large files, to prevent `FileNotFoundError` or `BadZipFile` issues.
+      - Confirm no `tqdm` or other progress bars are in the final script.
+      - Double-check that validation scores are saved correctly to `scores.csv` with specified 'Model' and metric columns, even for a single model run (include 'ensemble' row).
+    
+    {% if task_output_format is not none %}
     ## [Partial Response Format 1] Task Output Format:
     {{ task_output_format }}
 
@@ -224,17 +372,27 @@ task_gen:
       "task_design": ---The dict corresponding to task output format---,
       {% if workflow_check %}"workflow_update": ---A string corresponding to workflow description--- {% endif %}
     }
+    {% endif %}
     
   user: |-
-    # Scenario Description
+    # Competition Scenario Description
     {{ scenario_desc }}
 
+    # Data Folder Structure (All files are under {% include "scenarios.data_science.share:scen.input_path" %})
+    {{ data_folder_info }}
+
     # Current SOTA Implementation
     {{ sota_exp_desc }}
 
-    # Proposed Hypothesis you should strictly follow:
-    {{ hypothesis }}
+    # Proposed Hypothesis
+    This sketch should implement the following hypotheses:
 
+    {% for hypothesis in hypotheses %}
+    ## {{ hypothesis.problem_name }}
+    **Why:** {{ hypothesis.problem_desc }}
+    **Hypothesis:** {{ hypothesis.hypothesis }}
+
+    {% endfor %}
     # Feedback from Previous Failed Experiments (e.g., experiments that did not pass evaluation, encountered bugs, or failed to surpass SOTA performance):
     {{ failed_exp_and_feedback_list_desc }}
 
@@ -268,11 +426,6 @@ idea_sample:
     {{ problem_ideas }}
 
 specification:
-  problem: |-
-    1. The problem should be specific and fine-grained. Avoid general or vague statements. 
-    2. The problem should technical or methodological. Focus on design and implementation flaws, not runtime errors.
-    3. The problem should be strictly aligned with the improvement of target metric. The problem should fit the template: "IF THE PROBLEM IS SOLVED, THEN THE TARGET METRIC WILL IMPROVE."
-  
   hypothesis: |-
     1. Each hypothesis should be specific and non-vague.
       - Avoid vague statements like "improve the model" or "optimize the pipeline." Instead, specify the exact changes to be made. Do not use ambiguous changes like "try method A or method B". 

rdagent/scenarios/data_science/proposal/exp_gen/proposal.py

@@ -454,10 +454,15 @@ class DSProposalV1ExpGen(ExpGen):
 
 
 class DSProposalV2ExpGen(ExpGen):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.support_function_calling = APIBackend().support_function_calling()
+
     def identify_scenario_problem(self, scenario_desc: str, sota_exp_desc: str) -> Dict:
         sys_prompt = T(".prompts_v2:scenario_problem.system").r(
-            problem_spec=T(".prompts_v2:specification.problem").r(),
-            problem_output_format=T(".prompts_v2:output_format.problem").r(),
+            problem_output_format=(
+                T(".prompts_v2:output_format.problem").r() if not self.support_function_calling else None
+            ),
         )
         user_prompt = T(".prompts_v2:scenario_problem.user").r(
             scenario_desc=scenario_desc,
@@ -466,17 +471,26 @@ class DSProposalV2ExpGen(ExpGen):
         response = APIBackend().build_messages_and_create_chat_completion(
             user_prompt=user_prompt,
             system_prompt=sys_prompt,
-            json_mode=True,
-            json_target_type=Dict[str, Dict[str, str]],
+            response_format=ScenarioChallenges if self.support_function_calling else {"type": "json_object"},
+            json_target_type=Dict[str, Dict[str, str]] if not self.support_function_calling else None,
         )
-        return json.loads(response)
+        if self.support_function_calling:
+            challenges = ScenarioChallenges(**json.loads(response))
+            # Translate to problems
+            problems = {o.caption: {"problem": o.statement, "reason": o.reasoning} for o in challenges.challenges}
+            logger.info(f"Identified scenario problems:\n" + json.dumps(problems))
+        else:
+            problems = json.loads(response)
+            logger.info(f"Identified scenario problems:\n" + json.dumps(problems))
+        return problems
 
     def identify_feedback_problem(
         self, scenario_desc: str, exp_feedback_list_desc: str, sota_exp_desc: str, inject_diverse: bool = False
     ) -> Dict:
         sys_prompt = T(".prompts_v2:feedback_problem.system").r(
-            problem_spec=T(".prompts_v2:specification.problem").r(),
-            problem_output_format=T(".prompts_v2:output_format.problem").r(),
+            problem_output_format=(
+                T(".prompts_v2:output_format.problem").r() if not self.support_function_calling else None
+            ),
             inject_diverse=inject_diverse,
         )
         user_prompt = T(".prompts_v2:feedback_problem.user").r(
@@ -487,10 +501,18 @@ class DSProposalV2ExpGen(ExpGen):
         response = APIBackend().build_messages_and_create_chat_completion(
             user_prompt=user_prompt,
             system_prompt=sys_prompt,
-            json_mode=True,
-            json_target_type=Dict[str, Dict[str, str]],
+            response_format=TraceChallenges if self.support_function_calling else {"type": "json_object"},
+            json_target_type=Dict[str, Dict[str, str]] if not self.support_function_calling else None,
         )
-        return json.loads(response)
+        if self.support_function_calling:
+            challenges = TraceChallenges(**json.loads(response))
+            # Translate to problems
+            problems = {o.caption: {"problem": o.statement, "reason": o.reasoning} for o in challenges.challenges}
+            logger.info(f"Identified feedback problems:\n" + json.dumps(problems))
+        else:
+            problems = json.loads(response)
+            logger.info(f"Identified feedback problems:\n" + json.dumps(problems))
+        return problems
 
     def identify_problem(
         self, current_sub_trace, scenario_desc, sota_exp_desc, exp_feedback_list_desc, inject_diverse
@@ -535,21 +557,19 @@ class DSProposalV2ExpGen(ExpGen):
         inject_diverse: bool = False,
     ) -> Dict:
         problem_formatted_str = ""
-        for problem_name, problem_dict in problems.items():
-            if "problem" not in problem_dict:
-                continue
-            problem_formatted_str += f"Problem Name: {problem_name}\n"
-            problem_formatted_str += f"- Problem Description: {problem_dict['problem']}\n"
+        for i, (problem_name, problem_dict) in enumerate(problems.items()):
+            problem_formatted_str += f"## {i+1}. {problem_name}\n"
+            problem_formatted_str += f"{problem_dict['problem']}\n"
             if "idea" in problem_dict:
                 idea_formatted_str = DSIdea(problem_dict["idea"]).to_formatted_str()
-                problem_formatted_str += f"- Sampled Idea by user: \n{idea_formatted_str}\n"
+                problem_formatted_str += f"Sampled Idea by user: \n{idea_formatted_str}\n"
             problem_formatted_str += "\n\n"
 
         sys_prompt = T(".prompts_v2:hypothesis_gen.system").r(
-            component_desc=component_desc,
-            hypothesis_spec=T(".prompts_v2:specification.hypothesis").r(pipeline=pipeline),
-            hypothesis_output_format=T(".prompts_v2:output_format.hypothesis").r(
-                pipeline=pipeline, enable_idea_pool=enable_idea_pool
+            hypothesis_output_format=(
+                T(".prompts_v2:output_format.hypothesis").r(pipeline=pipeline, enable_idea_pool=enable_idea_pool)
+                if not self.support_function_calling
+                else None
             ),
             pipeline=pipeline,
             enable_idea_pool=enable_idea_pool,
@@ -565,10 +585,31 @@ class DSProposalV2ExpGen(ExpGen):
         response = APIBackend().build_messages_and_create_chat_completion(
             user_prompt=user_prompt,
             system_prompt=sys_prompt,
-            json_mode=True,
-            json_target_type=Dict[str, Dict[str, str | Dict[str, str | int]]],
+            response_format=HypothesisList if self.support_function_calling else {"type": "json_object"},
+            json_target_type=(
+                Dict[str, Dict[str, str | Dict[str, str | int]]] if not self.support_function_calling else None
+            ),
         )
-        resp_dict = json.loads(response)
+        if self.support_function_calling:
+            hypotheses = HypothesisList(**json.loads(response))
+            resp_dict = {
+                h.caption: {
+                    "reason": h.challenge,
+                    "component": h.component,
+                    "hypothesis": h.hypothesis,
+                    "evaluation": {
+                        "alignment_score": h.evaluation.alignment.score,
+                        "impact_score": h.evaluation.impact.score,
+                        "novelty_score": h.evaluation.novelty.score,
+                        "feasibility_score": h.evaluation.feasibility.score,
+                        "risk_reward_balance_score": h.evaluation.risk_reward_balance.score,
+                    },
+                }
+                for h in hypotheses.hypotheses
+            }
+        else:
+            resp_dict = json.loads(response)
+        logger.info(f"Generated hypotheses:\n" + json.dumps(resp_dict, indent=2))
 
         # make sure the problem name is aligned
         problem_keys = set(problems.keys())
@@ -675,44 +716,40 @@ class DSProposalV2ExpGen(ExpGen):
         scenario_desc: str,
         sota_exp_desc: str,
         sota_exp: DSExperiment,
-        hypothesis: DSHypothesis,
+        hypotheses: list[DSHypothesis],
         pipeline: bool,
         failed_exp_feedback_list_desc: str,
     ) -> DSExperiment:
         if pipeline:
             component_info = get_component("Pipeline")
         else:
-            component_info = get_component(hypothesis.component)
-        if pipeline:
-            task_spec = T(f"scenarios.data_science.share:component_spec.Pipeline").r()
-        elif DS_RD_SETTING.spec_enabled and sota_exp is not None:
-            task_spec = sota_exp.experiment_workspace.file_dict[component_info["spec_file"]]
-        else:
-            task_spec = T(f"scenarios.data_science.share:component_spec.{hypothesis.component}").r()
+            component_info = get_component(hypotheses[0].component)
+        data_folder_info = self.scen.processed_data_folder_description
         sys_prompt = T(".prompts_v2:task_gen.system").r(
-            targets=component_info["target_name"],
-            task_specification=task_spec,
-            task_output_format=component_info["task_output_format"],
+            task_output_format=component_info["task_output_format"] if not self.support_function_calling else None,
+            # task_output_format=component_info["task_output_format"],
             component_desc=component_desc,
-            workflow_check=not pipeline and hypothesis.component != "Workflow",
+            workflow_check=not pipeline and hypotheses[0].component != "Workflow",
         )
-        user_prompt = T(".prompts_v2:task_gen.user").r(
+        user_prompt = T(".prompts_v3:task_gen.user").r(
             scenario_desc=scenario_desc,
+            data_folder_info=data_folder_info,
             sota_exp_desc=sota_exp_desc,
-            hypothesis=str(hypothesis),
+            hypotheses=hypotheses,
             failed_exp_and_feedback_list_desc=failed_exp_feedback_list_desc,
         )
         response = APIBackend().build_messages_and_create_chat_completion(
             user_prompt=user_prompt,
             system_prompt=sys_prompt,
-            json_mode=True,
-            json_target_type=Dict[str, str | Dict[str, str]],
+            response_format=CodingSketch if self.support_function_calling else {"type": "json_object"},
+            json_target_type=Dict[str, str | Dict[str, str]] if not self.support_function_calling else None,
         )
         task_dict = json.loads(response)
-        task_design = task_dict.get("task_design", {})
-        task_name = (
-            task_design["model_name"] if (hypothesis.component == "Model" and not pipeline) else hypothesis.component
+        task_design = (
+            task_dict.get("task_design", {}) if not self.support_function_calling else task_dict.get("sketch", {})
         )
+        logger.info(f"Task design:\n{task_design}")
+        task_name = hypotheses[0].component
         description = (
             task_design
             if isinstance(task_design, str)
@@ -724,7 +761,7 @@ class DSProposalV2ExpGen(ExpGen):
             description=description,
         )
         new_workflow_desc = task_dict.get("workflow_update", "No update needed")
-        exp = DSExperiment(pending_tasks_list=[[task]], hypothesis=hypothesis)
+        exp = DSExperiment(pending_tasks_list=[[task]], hypothesis=hypotheses[0])
         # exp.experiment_workspace.inject_code_from_folder(sota_exp.experiment_workspace.workspace_path)
         if sota_exp is not None:
             exp.experiment_workspace.inject_code_from_file_dict(sota_exp.experiment_workspace)
@@ -736,6 +773,35 @@ class DSProposalV2ExpGen(ExpGen):
             exp.pending_tasks_list.append([workflow_task])
         return exp
 
+    def get_scenario_all_desc(self, trace: DSTrace, eda_output=None) -> str:
+        return T(".prompts_v3:scenario_description").r(
+            background=trace.scen.background,
+            submission_specifications=trace.scen.submission_specifications,
+            evaluation=trace.scen.metric_description,
+            metric_name=trace.scen.metric_name,
+            metric_direction=trace.scen.metric_direction,
+            raw_description=trace.scen.raw_description,
+            use_raw_description=DS_RD_SETTING.use_raw_description,
+            time_limit=f"{DS_RD_SETTING.full_timeout / 60 / 60 : .2f} hours",
+            eda_output=eda_output,
+        )
+
+    def get_all_hypotheses(self, problem_dict: dict, hypothesis_dict: dict) -> list[DSHypothesis]:
+        result = []
+        for name, data in hypothesis_dict.items():
+            problem_data = problem_dict.get(name, {})
+            result.append(
+                DSHypothesis(
+                    component=data.get("component", "Model"),
+                    hypothesis=data.get("hypothesis", "Hypothesis not provided"),
+                    reason=data.get("reason", "Reason not provided"),
+                    problem_name=name,
+                    problem_desc=problem_data.get("problem", "Problem description not provided"),
+                    problem_label=problem_data.get("label", "FEEDBACK_PROBLEM"),
+                )
+            )
+        return result
+
     def gen(self, trace: DSTrace) -> DSExperiment:
         pipeline = DS_RD_SETTING.coder_on_whole_pipeline
         if not pipeline and (draft_exp := draft_exp_in_decomposition(self.scen, trace)):
@@ -756,7 +822,7 @@ class DSProposalV2ExpGen(ExpGen):
             eda_output = None
         else:
             eda_output = sota_exp.experiment_workspace.file_dict.get("EDA.md", None)
-        scenario_desc = trace.scen.get_scenario_all_desc(eda_output=eda_output)
+        scenario_desc = self.get_scenario_all_desc(trace, eda_output=eda_output)
 
         sota_exp_desc = T("scenarios.data_science.share:describe.exp").r(
             exp=sota_exp, heading="Best of previous exploration of the scenario"
@@ -838,374 +904,6 @@ class DSProposalV2ExpGen(ExpGen):
         if DS_RD_SETTING.enable_knowledge_base:
             trace.knowledge_base.update_pickled_problem(all_problems, pickled_problem_name)
 
-        return self.task_gen(
-            component_desc=component_desc,
-            scenario_desc=scenario_desc,
-            sota_exp_desc=sota_exp_desc,
-            sota_exp=sota_exp,
-            hypothesis=new_hypothesis,
-            pipeline=pipeline,
-            failed_exp_feedback_list_desc=failed_exp_feedback_list_desc,
-        )
-
-
-class DSProposalV3ExpGen(DSProposalV2ExpGen):
-    def identify_scenario_problem(self, scenario_desc: str, sota_exp_desc: str) -> Dict:
-        sys_prompt = T(".prompts_v3:scenario_problem.system").r()
-        user_prompt = T(".prompts_v3:scenario_problem.user").r(
-            scenario_desc=scenario_desc,
-            sota_exp_desc=sota_exp_desc,
-        )
-        response = APIBackend().build_messages_and_create_chat_completion(
-            user_prompt=user_prompt,
-            system_prompt=sys_prompt,
-            response_format=ScenarioChallenges,
-            # json_mode=True,
-            # json_target_type=Dict[str, Dict[str, str]],
-        )
-        challenges = ScenarioChallenges(**json.loads(response))
-        # Translate to problems
-        problems = {o.caption: {"problem": o.statement, "reason": o.reasoning} for o in challenges.challenges}
-        logger.info(f"Identified scenario problems:\n" + json.dumps(problems))
-        return problems
-
-    def identify_feedback_problem(self, scenario_desc: str, exp_feedback_list_desc: str, sota_exp_desc: str) -> Dict:
-        sys_prompt = T(".prompts_v3:feedback_problem.system").r()
-        user_prompt = T(".prompts_v3:feedback_problem.user").r(
-            scenario_desc=scenario_desc,
-            exp_and_feedback_list_desc=exp_feedback_list_desc,
-            sota_exp_desc=sota_exp_desc,
-        )
-        response = APIBackend().build_messages_and_create_chat_completion(
-            user_prompt=user_prompt,
-            system_prompt=sys_prompt,
-            response_format=TraceChallenges,
-            # json_mode=True,
-            # json_target_type=Dict[str, Dict[str, str]],
-        )
-        challenges = TraceChallenges(**json.loads(response))
-        # Translate to problems
-        problems = {o.caption: {"problem": o.statement, "reason": o.reasoning} for o in challenges.challenges}
-        logger.info(f"Identified feedback problems:\n" + json.dumps(problems))
-        return problems
-
-    def get_scenario_all_desc_v3(self, trace: DSTrace, eda_output=None) -> str:
-        return T(".prompts_v3:scenario_description").r(
-            background=trace.scen.background,
-            submission_specifications=trace.scen.submission_specifications,
-            evaluation=trace.scen.metric_description,
-            metric_name=trace.scen.metric_name,
-            metric_direction=trace.scen.metric_direction,
-            raw_description=trace.scen.raw_description,
-            use_raw_description=DS_RD_SETTING.use_raw_description,
-            time_limit=f"{DS_RD_SETTING.full_timeout / 60 / 60 : .2f} hours",
-            eda_output=eda_output,
-        )
-
-    @wait_retry(retry_n=5)
-    def hypothesis_gen(
-        self,
-        component_desc: str,
-        scenario_desc: str,
-        exp_feedback_list_desc: str,
-        sota_exp_desc: str,
-        problems: dict,
-        pipeline: bool,
-        enable_idea_pool: bool,
-    ) -> Dict:
-        problem_formatted_str = ""
-        for i, (problem_name, problem_dict) in enumerate(problems.items()):
-            problem_formatted_str += f"## {i+1}. {problem_name}\n"
-            problem_formatted_str += f"{problem_dict['problem']}\n"
-            if "idea" in problem_dict:
-                idea_formatted_str = DSIdea(problem_dict["idea"]).to_formatted_str()
-                problem_formatted_str += f"Sampled Idea by user: \n{idea_formatted_str}\n"
-            problem_formatted_str += "\n\n"
-
-        sys_prompt = T(".prompts_v3:hypothesis_gen.system").r(
-            pipeline=pipeline,
-            enable_idea_pool=enable_idea_pool,
-        )
-        user_prompt = T(".prompts_v3:hypothesis_gen.user").r(
-            scenario_desc=scenario_desc,
-            exp_and_feedback_list_desc=exp_feedback_list_desc,
-            sota_exp_desc=sota_exp_desc,
-            problems=problem_formatted_str,
-            enable_idea_pool=enable_idea_pool,
-        )
-        response = APIBackend().build_messages_and_create_chat_completion(
-            user_prompt=user_prompt, system_prompt=sys_prompt, response_format=HypothesisList
-        )
-        hypotheses = HypothesisList(**json.loads(response))
-        resp_dict = {
-            h.caption: {
-                "reason": h.challenge,
-                "component": h.component,
-                "hypothesis": h.hypothesis,
-                "evaluation": {
-                    "alignment_score": h.evaluation.alignment.score,
-                    "impact_score": h.evaluation.impact.score,
-                    "novelty_score": h.evaluation.novelty.score,
-                    "feasibility_score": h.evaluation.feasibility.score,
-                    "risk_reward_balance_score": h.evaluation.risk_reward_balance.score,
-                },
-            }
-            for h in hypotheses.hypotheses
-        }
-
-        logger.info(f"Generated hypotheses:\n" + json.dumps(resp_dict, indent=2))
-
-        if len(resp_dict) == 0:
-            logger.error("No hypothesis generated. Retrying...")
-            raise ValueError("No hypothesis generated.")
-
-        # make sure the problem name is aligned
-        problem_keys = set(problems.keys())
-        resp_keys = set(resp_dict.keys())
-        if not resp_keys.issubset(problem_keys):
-            logger.error("Problem names are not fully aligned. Retrying...")
-            raise ValueError("Problem names are not fully aligned.")
-
-        return resp_dict
-
-    def task_gen(
-        self,
-        component_desc: str,
-        scenario_desc: str,
-        sota_exp_desc: str,
-        sota_exp: DSExperiment,
-        hypotheses: list[DSHypothesis],
-        pipeline: bool,
-        failed_exp_feedback_list_desc: str,
-    ) -> DSExperiment:
-        if pipeline:
-            component_info = get_component("Pipeline")
-        else:
-            component_info = get_component(hypotheses[0].component)
-        data_folder_info = self.scen.processed_data_folder_description
-        sys_prompt = T(".prompts_v3:task_gen.system").r(
-            # targets=component_info["target_name"],
-            # task_output_format=component_info["task_output_format"],
-            # component_desc=component_desc,
-            # workflow_check=not pipeline and hypothesis.component != "Workflow",
-        )
-        user_prompt = T(".prompts_v3:task_gen.user").r(
-            scenario_desc=scenario_desc,
-            data_folder_info=data_folder_info,
-            sota_exp_desc=sota_exp_desc,
-            hypotheses=hypotheses,
-            failed_exp_and_feedback_list_desc=failed_exp_feedback_list_desc,
-        )
-        response = APIBackend().build_messages_and_create_chat_completion(
-            user_prompt=user_prompt,
-            system_prompt=sys_prompt,
-            response_format=CodingSketch,
-            # json_mode=True,
-            # json_target_type=Dict[str, str | Dict[str, str]],
-        )
-        task_dict = json.loads(response)
-        task_design = task_dict.get("sketch", {})
-        logger.info("Task design:\n" + task_design)
-        task_name = hypotheses[0].component
-        description = (
-            task_design
-            if isinstance(task_design, str)
-            else task_design.get("description", f"{component_info['target_name']} description not provided")
-        )
-        task_class = component_info["task_class"]
-        task = task_class(
-            name=task_name,
-            description=description,
-        )
-        new_workflow_desc = task_dict.get("workflow_update", "No update needed")
-        exp = DSExperiment(pending_tasks_list=[[task]], hypothesis=hypotheses[0])
-        # exp.experiment_workspace.inject_code_from_folder(sota_exp.experiment_workspace.workspace_path)
-        if sota_exp is not None:
-            exp.experiment_workspace.inject_code_from_file_dict(sota_exp.experiment_workspace)
-        if not pipeline and new_workflow_desc != "No update needed":
-            workflow_task = WorkflowTask(
-                name="Workflow",
-                description=new_workflow_desc,
-            )
-            exp.pending_tasks_list.append([workflow_task])
-        return exp
-
-    def get_all_hypotheses(self, problem_dict: dict, hypothesis_dict: dict) -> list[DSHypothesis]:
-        result = []
-        for name, data in hypothesis_dict.items():
-            problem_data = problem_dict.get(name, {})
-            result.append(
-                DSHypothesis(
-                    component=data.get("component", "Model"),
-                    hypothesis=data.get("hypothesis", "Hypothesis not provided"),
-                    reason=data.get("reason", "Reason not provided"),
-                    problem_name=name,
-                    problem_desc=problem_data.get("problem", "Problem description not provided"),
-                    problem_label=problem_data.get("label", "FEEDBACK_PROBLEM"),
-                )
-            )
-        return result
-
-    # FIXME: remove this, dump solution, should be merged into identify_problem in V2
-    def identify_problems_v3(
-        self, trace: DSTrace, scenario_desc: str, sota_exp_desc: str, exp_feedback_list_desc: str
-    ) -> Dict:
-        sub_trace = trace.get_parent_exps()
-        trace_length = len(trace.hist)
-        all_problems = {}
-
-        # 阶段一：探索期（主要场景问题）
-        if trace_length <= 3:
-            scen_problems = self.identify_scenario_problem(scenario_desc, sota_exp_desc)
-            for problem_name in scen_problems:
-                scen_problems[problem_name]["label"] = "SCENARIO_PROBLEM"
-                all_problems[problem_name] = scen_problems[problem_name]
-            self.scen_prob_multiplier = 3
-
-        # 阶段二：混合期（两种问题都考虑）
-        elif trace_length <= 6:
-            # 优先场景问题，但也考虑反馈
-            scen_problems = self.identify_scenario_problem(scenario_desc, sota_exp_desc)
-            for problem_name in scen_problems:
-                scen_problems[problem_name]["label"] = "SCENARIO_PROBLEM"
-                all_problems[problem_name] = scen_problems[problem_name]
-
-            fb_problems = self.identify_feedback_problem(scenario_desc, exp_feedback_list_desc, sota_exp_desc)
-            for problem_name in fb_problems:
-                fb_problems[problem_name]["label"] = "FEEDBACK_PROBLEM"
-                all_problems[problem_name] = fb_problems[problem_name]
-            self.scen_prob_multiplier = 2
-
-        # 阶段三：优化期（主要反馈问题）
-        else:
-            fb_problems = self.identify_feedback_problem(scenario_desc, exp_feedback_list_desc, sota_exp_desc)
-            for problem_name in fb_problems:
-                fb_problems[problem_name]["label"] = "FEEDBACK_PROBLEM"
-                all_problems[problem_name] = fb_problems[problem_name]
-            self.scen_prob_multiplier = 1
-
-        return all_problems
-
-    def gen(self, trace: DSTrace) -> DSExperiment:
-        pipeline = DS_RD_SETTING.coder_on_whole_pipeline
-        if not pipeline and (draft_exp := draft_exp_in_decomposition(self.scen, trace)):
-            return draft_exp
-
-        if pipeline:
-            component_desc = T("scenarios.data_science.share:component_description_in_pipeline").r()
-        else:
-            component_desc = "\n".join(
-                [
-                    f"[{key}] {value}"
-                    for key, value in T("scenarios.data_science.share:component_description").template.items()
-                ]
-            )
-
-        sota_exp = trace.sota_experiment()
-        if not isinstance(sota_exp, DSExperiment):
-            eda_output = None
-        else:
-            eda_output = sota_exp.experiment_workspace.file_dict.get("EDA.md", None)
-        scenario_desc = self.get_scenario_all_desc_v3(trace, eda_output=eda_output)
-
-        sota_exp_desc = T("scenarios.data_science.share:describe.exp").r(
-            exp=sota_exp, heading="Best of previous exploration of the scenario"
-        )
-
-        exp_feedback_list_desc = T("scenarios.data_science.share:describe.trace").r(
-            exp_and_feedback_list=trace.experiment_and_feedback_list_after_init(return_type="all"),
-            type="all",
-            pipeline=pipeline,
-        )
-        failed_exp_feedback_list_desc = T("scenarios.data_science.share:describe.trace").r(
-            exp_and_feedback_list=trace.experiment_and_feedback_list_after_init(return_type="failed"),
-            type="failed",
-            pipeline=pipeline,
-        )
-
-        if DS_RD_SETTING.enable_inject_diverse and len(trace.hist) > 0:
-            if len(trace.current_selection) == 0:
-                # start a new sub-trace, and inject diverse problems.
-                inject_diverse = True
-                logger.info("Start a new sub-trace, and inject diverse problems.")
-            else:
-                inject_diverse = False
-        else:
-            inject_diverse = False
-        # Step 1: Identify problems
-        all_problems = {}
-
-        all_problems = self.identify_problems_v3(
-            trace=trace,
-            scenario_desc=scenario_desc,
-            sota_exp_desc=sota_exp_desc,
-            exp_feedback_list_desc=exp_feedback_list_desc,
-        )
-
-        # if len(trace.hist) > 3:
-        #     fb_problems = self.identify_feedback_problem(
-        #         scenario_desc=scenario_desc,
-        #         exp_feedback_list_desc=exp_feedback_list_desc,
-        #         sota_exp_desc=sota_exp_desc,
-        #     )
-        #     for problem_name in fb_problems:
-        #         fb_problems[problem_name]["label"] = "FEEDBACK_PROBLEM"
-        #         all_problems[problem_name] = fb_problems[problem_name]
-
-        # if len(trace.hist) < 9:
-        #     scen_problems = self.identify_scenario_problem(
-        #         scenario_desc=scenario_desc,
-        #         sota_exp_desc=sota_exp_desc,
-        #     )
-        #     for problem_name in scen_problems:
-        #         scen_problems[problem_name]["label"] = "SCENARIO_PROBLEM"
-        #         all_problems[problem_name] = scen_problems[problem_name]
-
-        # Step 1.5: Sample ideas from idea pool
-        if DS_RD_SETTING.enable_knowledge_base:
-            all_problems = trace.knowledge_base.sample_ideas(
-                problems=all_problems,
-                scenario_desc=scenario_desc,
-                exp_feedback_list_desc=exp_feedback_list_desc,
-                sota_exp_desc=sota_exp_desc,
-                competition_desc=self.scen.get_competition_full_desc(),
-            )
-
-        # Step 2: Propose hypothesis based on the identified problems (and sampled ideas)
-        hypothesis_dict = self.hypothesis_gen(
-            component_desc=component_desc,
-            scenario_desc=scenario_desc,
-            exp_feedback_list_desc=exp_feedback_list_desc,
-            sota_exp_desc=sota_exp_desc,
-            problems=all_problems,
-            pipeline=pipeline,
-            enable_idea_pool=DS_RD_SETTING.enable_knowledge_base,
-        )
-        if not pipeline:
-            sota_exp_model_file_count = len(
-                [
-                    k
-                    for k in sota_exp.experiment_workspace.file_dict.keys()
-                    if k.endswith(".py") and "test" not in k and k.startswith("model")
-                ]
-            )
-            if sota_exp_model_file_count <= 1:
-                pop_names = []
-                for problem_name in hypothesis_dict:
-                    if hypothesis_dict[problem_name].get("component", "") == "Ensemble":
-                        pop_names.append(problem_name)
-                for name in pop_names:
-                    hypothesis_dict.pop(name)
-
-        # Step 3: Select the best hypothesis
-        pickled_problem_name, new_hypothesis = self.hypothesis_rank(
-            hypothesis_dict=hypothesis_dict,
-            problem_dict=all_problems,
-        )
-        # Step 3.5: Update knowledge base with the picked problem
-        if DS_RD_SETTING.enable_knowledge_base:
-            trace.knowledge_base.update_pickled_problem(all_problems, pickled_problem_name)
-
         return self.task_gen(
             component_desc=component_desc,
             scenario_desc=scenario_desc,