feat: add pipeline coder (#742)

* init commit

* limit problem numbers

* ensemble lower case

* add runtime and spec to coder

* submission check notice

* sub EDA in sample execution

* avoid lightgbm

* add time limit to scenario

* rephrase the submission check

* give positive feedback when facing warning in check

* ENABLE FEEDBACK

* fix feedback bug

---------

Co-authored-by: Xu Yang <peteryang@vip.qq.com>

rdagent/app/data_science/conf.py

@@ -25,6 +25,7 @@ class DataScienceBasePropSetting(KaggleBasePropSetting):
     spec_enabled: bool = True
 
     proposal_version: str = "v1"
+    coder_on_whole_pipeline: bool = False
 
     coder_max_loop: int = 10
     runner_max_loop: int = 3

rdagent/app/data_science/loop.py

@@ -10,6 +10,8 @@ from rdagent.components.coder.data_science.feature import FeatureCoSTEER
 from rdagent.components.coder.data_science.feature.exp import FeatureTask
 from rdagent.components.coder.data_science.model import ModelCoSTEER
 from rdagent.components.coder.data_science.model.exp import ModelTask
+from rdagent.components.coder.data_science.pipeline import PipelineCoSTEER
+from rdagent.components.coder.data_science.pipeline.exp import PipelineTask
 from rdagent.components.coder.data_science.raw_data_loader import DataLoaderCoSTEER
 from rdagent.components.coder.data_science.raw_data_loader.exp import DataLoaderTask
 from rdagent.components.coder.data_science.workflow import WorkflowCoSTEER
@@ -54,6 +56,8 @@ class DataScienceRDLoop(RDLoop):
         self.ensemble_coder = EnsembleCoSTEER(scen)
         self.workflow_coder = WorkflowCoSTEER(scen)
 
+        self.pipeline_coder = PipelineCoSTEER(scen)
+
         self.runner = DSCoSTEERRunner(scen)
         # self.summarizer: Experiment2Feedback = import_class(PROP_SETTING.summarizer)(scen)
         # logger.log_object(self.summarizer, tag="summarizer")
@@ -86,6 +90,8 @@ class DataScienceRDLoop(RDLoop):
                     exp = self.ensemble_coder.develop(exp)
                 elif isinstance(exp.sub_tasks[0], WorkflowTask):
                     exp = self.workflow_coder.develop(exp)
+                elif isinstance(exp.sub_tasks[0], PipelineTask):
+                    exp = self.pipeline_coder.develop(exp)
                 else:
                     raise NotImplementedError(f"Unsupported component in DataScienceRDLoop: {exp.hypothesis.component}")
             exp.sub_tasks = []
@@ -106,7 +112,7 @@ class DataScienceRDLoop(RDLoop):
         - If we come to feedback phase, the previous development steps are successful.
         """
         exp: DSExperiment = prev_out["running"]
-        if self.trace.next_incomplete_component() is None:
+        if self.trace.next_incomplete_component() is None or DS_RD_SETTING.coder_on_whole_pipeline:
             # we have alreadly completed components in previous trace. So current loop is focusing on a new proposed idea.
             # So we need feedback for the proposal.
             feedback = self.summarizer.generate_feedback(exp, self.trace)
@@ -130,7 +136,11 @@ class DataScienceRDLoop(RDLoop):
                     ExperimentFeedback.from_exception(e),
                 )
             )
-            if self.trace.sota_experiment() is None and len(self.trace.hist) >= DS_RD_SETTING.consecutive_errors:
+            if (
+                self.trace.sota_experiment() is None
+                and len(self.trace.hist) >= DS_RD_SETTING.consecutive_errors
+                and not DS_RD_SETTING.coder_on_whole_pipeline
+            ):
                 # if {in inital/drafting stage} and {tried enough times}
                 for _, fb in self.trace.hist[-DS_RD_SETTING.consecutive_errors :]:
                     if fb:

rdagent/components/coder/data_science/pipeline/__init__.py

@@ -0,0 +1,163 @@
+"""
+
+Loop should not large change exclude
+- Action Choice[current data loader & spec]
+- other should share
+    - Propose[choice] => Task[Choice] => CoSTEER =>
+        -
+
+Extra feature:
+- cache
+
+
+File structure
+- ___init__.py: the entrance/agent of coder
+- evaluator.py
+- conf.py
+- exp.py: everything under the experiment, e.g.
+    - Task
+    - Experiment
+    - Workspace
+- test.py
+    - Each coder could be tested.
+"""
+
+import json
+import re
+from pathlib import Path
+from typing import Dict
+
+from rdagent.app.data_science.conf import DS_RD_SETTING
+from rdagent.components.coder.CoSTEER import CoSTEER
+from rdagent.components.coder.CoSTEER.evaluators import (
+    CoSTEERMultiEvaluator,
+    CoSTEERSingleFeedback,
+)
+from rdagent.components.coder.CoSTEER.evolving_strategy import (
+    MultiProcessEvolvingStrategy,
+)
+from rdagent.components.coder.CoSTEER.knowledge_management import (
+    CoSTEERQueriedKnowledge,
+)
+from rdagent.components.coder.data_science.conf import (
+    DSCoderCoSTEERSettings,
+    get_ds_env,
+)
+from rdagent.components.coder.data_science.pipeline.eval import PipelineCoSTEEREvaluator
+from rdagent.components.coder.data_science.raw_data_loader.eval import (
+    DataLoaderCoSTEEREvaluator,
+)
+from rdagent.components.coder.data_science.raw_data_loader.exp import DataLoaderTask
+from rdagent.core.exception import CoderError
+from rdagent.core.experiment import FBWorkspace
+from rdagent.core.scenario import Scenario
+from rdagent.oai.llm_utils import APIBackend
+from rdagent.utils.agent.ret import PythonAgentOut
+from rdagent.utils.agent.tpl import T
+
+DIRNAME = Path(__file__).absolute().resolve().parent
+
+
+class PipelineMultiProcessEvolvingStrategy(MultiProcessEvolvingStrategy):
+    def implement_one_task(
+        self,
+        target_task: DataLoaderTask,
+        queried_knowledge: CoSTEERQueriedKnowledge | None = None,
+        workspace: FBWorkspace | None = None,
+        prev_task_feedback: CoSTEERSingleFeedback | None = None,
+    ) -> dict[str, str]:
+        competition_info = self.scen.get_scenario_all_desc()
+        runtime_environment = self.scen.get_runtime_environment()
+        data_folder_info = self.scen.processed_data_folder_description
+        pipeline_task_info = target_task.get_task_information()
+
+        queried_similar_successful_knowledge = (
+            queried_knowledge.task_to_similar_task_successful_knowledge[pipeline_task_info]
+            if queried_knowledge is not None
+            else []
+        )
+        queried_former_failed_knowledge = (
+            queried_knowledge.task_to_former_failed_traces[pipeline_task_info] if queried_knowledge is not None else []
+        )
+        queried_former_failed_knowledge = (
+            [
+                knowledge
+                for knowledge in queried_former_failed_knowledge[0]
+                if knowledge.implementation.file_dict.get("main.py") != workspace.file_dict.get("main.py")
+            ],
+            queried_former_failed_knowledge[1],
+        )
+
+        system_prompt = T(".prompts:pipeline_coder.system").r(
+            task_desc=pipeline_task_info,
+            queried_similar_successful_knowledge=queried_similar_successful_knowledge,
+            queried_former_failed_knowledge=queried_former_failed_knowledge[0],
+            out_spec=PythonAgentOut.get_spec(),
+            runtime_environment=runtime_environment,
+            spec=T("scenarios.data_science.share:component_spec.Pipeline").r(),
+        )
+        user_prompt = T(".prompts:pipeline_coder.user").r(
+            competition_info=competition_info,
+            folder_spec=data_folder_info,
+            latest_code=workspace.file_dict.get("main.py"),
+            latest_code_feedback=prev_task_feedback,
+        )
+
+        for _ in range(5):
+            pipeline_code = PythonAgentOut.extract_output(
+                APIBackend().build_messages_and_create_chat_completion(
+                    user_prompt=user_prompt,
+                    system_prompt=system_prompt,
+                )
+            )
+            if pipeline_code != workspace.file_dict.get("main.py"):
+                break
+            else:
+                user_prompt = user_prompt + "\nPlease avoid generating same code to former code!"
+        else:
+            raise CoderError("Failed to generate a new pipeline code.")
+
+        return {
+            "main.py": pipeline_code,
+        }
+
+    def assign_code_list_to_evo(self, code_list: list[dict[str, str]], evo):
+        """
+        Assign the code list to the evolving item.
+
+        The code list is aligned with the evolving item's sub-tasks.
+        If a task is not implemented, put a None in the list.
+        """
+        for index in range(len(evo.sub_tasks)):
+            if code_list[index] is None:
+                continue
+            if evo.sub_workspace_list[index] is None:
+                # evo.sub_workspace_list[index] = FBWorkspace(target_task=evo.sub_tasks[index])
+                evo.sub_workspace_list[index] = evo.experiment_workspace
+            evo.sub_workspace_list[index].inject_files(**code_list[index])
+        return evo
+
+
+class PipelineCoSTEER(CoSTEER):
+    def __init__(
+        self,
+        scen: Scenario,
+        *args,
+        **kwargs,
+    ) -> None:
+        settings = DSCoderCoSTEERSettings()
+        eva = CoSTEERMultiEvaluator(
+            PipelineCoSTEEREvaluator(scen=scen), scen=scen
+        )  # Please specify whether you agree running your eva in parallel or not
+        es = PipelineMultiProcessEvolvingStrategy(scen=scen, settings=settings)
+
+        super().__init__(
+            *args,
+            settings=settings,
+            eva=eva,
+            es=es,
+            evolving_version=2,
+            scen=scen,
+            max_loop=DS_RD_SETTING.coder_max_loop,
+            **kwargs,
+        )

rdagent/components/coder/data_science/pipeline/eval.py

@@ -0,0 +1,119 @@
+# tess successfully running.
+# (GPT) if it aligns with the spec & rationality of the spec.
+import json
+import re
+from pathlib import Path
+
+import pandas as pd
+
+from rdagent.app.data_science.conf import DS_RD_SETTING
+from rdagent.components.coder.CoSTEER import CoSTEERMultiFeedback
+from rdagent.components.coder.CoSTEER.evaluators import (
+    CoSTEEREvaluator,
+    CoSTEERSingleFeedback,
+)
+from rdagent.components.coder.CoSTEER.knowledge_management import (
+    CoSTEERQueriedKnowledgeV2,
+)
+from rdagent.components.coder.data_science.conf import get_ds_env
+from rdagent.core.experiment import FBWorkspace, Task
+from rdagent.utils.agent.tpl import T
+from rdagent.utils.agent.workflow import build_cls_from_json_with_retry
+
+DIRNAME = Path(__file__).absolute().resolve().parent
+
+PipelineSingleFeedback = CoSTEERSingleFeedback
+PipelineMultiFeedback = CoSTEERMultiFeedback
+
+
+class PipelineCoSTEEREvaluator(CoSTEEREvaluator):
+
+    def evaluate(
+        self,
+        target_task: Task,
+        implementation: FBWorkspace,
+        gt_implementation: FBWorkspace,
+        queried_knowledge: CoSTEERQueriedKnowledgeV2 = None,
+        **kwargs,
+    ) -> PipelineSingleFeedback:
+
+        target_task_information = target_task.get_task_information()
+        if (
+            queried_knowledge is not None
+            and target_task_information in queried_knowledge.success_task_to_knowledge_dict
+        ):
+            return queried_knowledge.success_task_to_knowledge_dict[target_task_information].feedback
+        elif queried_knowledge is not None and target_task_information in queried_knowledge.failed_task_info_set:
+            return PipelineSingleFeedback(
+                execution="This task has failed too many times, skip implementation.",
+                return_checking="This task has failed too many times, skip implementation.",
+                code="This task has failed too many times, skip implementation.",
+                final_decision=False,
+            )
+
+        env = get_ds_env()
+        env.conf.extra_volumes = {f"{DS_RD_SETTING.local_data_path}/sample/{self.scen.competition}": "/kaggle/input"}
+
+        # Clean the scores.csv & submission.csv.
+        implementation.execute(env=env, entry=f"rm submission.csv scores.csv")
+        stdout = implementation.execute(env=env, entry=f"python main.py")
+        stdout = re.sub(r"=== Start of EDA part ===(.*)=== End of EDA part ===", "", stdout)
+
+        score_fp = implementation.workspace_path / "scores.csv"
+        score_ret_code = 0
+        score_check_text = ""
+        if not score_fp.exists():
+            score_check_text = "[Error] Metrics file (scores.csv) is not generated!"
+            score_ret_code = 1
+        else:
+            try:
+                score_df = pd.read_csv(score_fp, index_col=0)
+                model_set_in_scores = set(score_df.index)
+
+                # Check model names (index)
+                if "ensemble" not in model_set_in_scores:
+                    score_check_text += (
+                        f"\n[Error] The score dataframe doesn't contain the ensemble model.\nscore_df is:\n{score_df}"
+                    )
+                    score_ret_code = 1
+
+                # Check metric name (columns)
+                if score_df.columns.tolist() != [self.scen.metric_name]:
+                    score_check_text += f"\n[Error] The scores dataframe does not contain the correct column names.\nCorrect columns is: ['{self.scen.metric_name}']\nBut got: {score_df.columns.tolist()}"
+                    score_ret_code = 1
+
+            except Exception as e:
+                score_check_text += f"\n[Error] in checking the scores.csv file: {e}\nscores.csv's content:\n-----\n{score_fp.read_text()}\n-----"
+                score_ret_code = 1
+
+        # Check submission file
+        base_check_code = (DIRNAME / "eval_tests" / "submission_format_test.txt").read_text()
+        implementation.inject_files(**{"test/submission_format_test.py": base_check_code})
+        # stdout += "----Submission Check 1-----\n"
+        submission_check_out, submission_ret_code = implementation.execute_ret_code(
+            env=env, entry="python test/submission_format_test.py"
+        )
+        stdout += "\n" + submission_check_out
+
+        system_prompt = T(".prompts:pipeline_eval.system").r(
+            scenario=self.scen.get_scenario_all_desc(),
+            task_desc=target_task.get_task_information(),
+            spec=T("scenarios.data_science.share:component_spec.Pipeline").r(),
+        )
+        user_prompt = T(".prompts:pipeline_eval.user").r(
+            stdout=stdout.strip(),
+            code=implementation.file_dict["main.py"],
+        )
+        wfb = build_cls_from_json_with_retry(
+            PipelineSingleFeedback,
+            system_prompt=system_prompt,
+            user_prompt=user_prompt,
+            init_kwargs_update_func=PipelineSingleFeedback.val_and_update_init_dict,
+        )
+        if score_ret_code != 0:
+            wfb.final_decision = False
+            wfb.return_checking += "\n" + score_check_text
+        if submission_ret_code != 0:
+            wfb.final_decision = False
+            wfb.return_checking += "\nSubmission file check failed."
+        return wfb

rdagent/components/coder/data_science/pipeline/eval_tests/submission_format_test.txt

@@ -0,0 +1,77 @@
+from pathlib import Path
+import pandas as pd
+import hashlib
+
+def calculate_md5(file_path):
+    with open(file_path, "rb") as f:
+        file_hash = hashlib.md5(f.read()).hexdigest()
+    return file_hash
+
+file_md5 = calculate_md5("scores.csv")
+
+"""
+find . | grep -i sample | grep -i submission | grep -v sample_submission.csv | grep -v zip_files  | grep -v 'sample/'
+./denoising-dirty-documents/sampleSubmission.csv
+./the-icml-2013-whale-challenge-right-whale-redux/sampleSubmission.csv
+./text-normalization-challenge-russian-language/ru_sample_submission_2.csv.zip
+./text-normalization-challenge-russian-language/ru_sample_submission_2.csv
+./random-acts-of-pizza/sampleSubmission.csv
+./text-normalization-challenge-english-language/en_sample_submission_2.csv.zip
+./text-normalization-challenge-english-language/en_sample_submission_2.csv
+./detecting-insults-in-social-commentary/sample_submission_null.csv
+"""
+
+# Find sample submission file dynamically
+input_dir = Path("/kaggle/input")
+# Look for common variations of sample submission filenames
+sample_submission_files = list(input_dir.glob("*sample_submission*.csv")) + \
+                         list(input_dir.glob("*sampleSubmission*.csv"))
+
+assert sample_submission_files, "Error: No sample submission file found in /kaggle/input/"
+
+# Use first matching file
+sample_submission_name = sample_submission_files[0].name
+SAMPLE_SUBMISSION_PATH = str(sample_submission_files[0])
+print(f"Using sample submission file: {sample_submission_name}")
+
+# Check if the sample submission file exists
+assert Path(SAMPLE_SUBMISSION_PATH).exists(), f"Error: {sample_submission_name} not found at {SAMPLE_SUBMISSION_PATH}"
+
+# Check if our submission file exists
+assert Path('submission.csv').exists(), "Error: submission.csv not found"
+
+sample_submission = pd.read_csv(SAMPLE_SUBMISSION_PATH)
+our_submission = pd.read_csv('submission.csv')
+
+success = True
+# Print the columns of the sample submission file
+print(f"Columns in {sample_submission_name}:", sample_submission.columns)
+print("Columns in our_submission.csv:", our_submission.columns)
+
+for col in sample_submission.columns:
+    if col not in our_submission.columns:
+        success = False
+        print(f'Column {col} not found in submission.csv')
+
+if success:
+    print(f'submission.csv\'s columns aligns with {sample_submission_name} .')
+
+
+# Print the first 5 rows of the two submission files, with columns separated by commas.
+def print_first_rows(file_path, file_name, num_rows=5):
+    print(f"\nFirst {num_rows} rows of {file_name}:")
+    try:
+        with open(file_path, 'r') as file:
+            for i, line in enumerate(file):
+                if i < num_rows:
+                    print(line.strip())
+                else:
+                    break
+    except FileNotFoundError:
+        print(f"Error: {file_name} not found.")
+
+print_first_rows(SAMPLE_SUBMISSION_PATH, sample_submission_name)
+print_first_rows('submission.csv', 'submission.csv')
+
+assert calculate_md5("scores.csv") == file_md5, "scores.csv should not be rewritten"
+print(f"\nPlease Checked the content of the submission file(submission.csv should has the same format with {sample_submission_name} but might not the same index with {sample_submission_name}). ")

rdagent/components/coder/data_science/pipeline/exp.py

@@ -0,0 +1,6 @@
+from rdagent.components.coder.CoSTEER.task import CoSTEERTask
+
+
+# Because we use isinstance to distinguish between different types of tasks, we need to use sub classes to represent different types of tasks
+class PipelineTask(CoSTEERTask):
+    pass

rdagent/components/coder/data_science/pipeline/prompts.yaml

@@ -0,0 +1,134 @@
+pipeline_coder:
+  system: |-
+    You are a world-class data scientist and machine learning engineer with deep expertise in statistics, mathematics, and computer science.
+    Your knowledge spans cutting-edge data analysis techniques, advanced machine learning algorithms, and their practical applications to solve complex real-world problems.
+
+    ## Task Description
+    {{ task_desc }}
+
+    ## The runtime environment your code will running on
+    {{ runtime_environment }}
+
+    ## Specification your code should follow
+    {{ spec }}
+
+    {% if queried_similar_successful_knowledge|length != 0 or queried_former_failed_knowledge|length != 0 %}
+    ## Relevant Information for This Task
+    {% endif %}
+    
+    {% if queried_similar_successful_knowledge|length != 0 %}
+    --------- Successful Implementations for Similar Models ---------
+    ====={% for similar_successful_knowledge in queried_similar_successful_knowledge %} Model {{ loop.index }}:=====
+    {{ similar_successful_knowledge.target_task.get_task_information() }}
+    =====Code:=====
+    {{ similar_successful_knowledge.implementation.all_codes }}
+    {% endfor %} 
+    {% endif %}
+
+    {% if queried_former_failed_knowledge|length != 0 %}
+    --------- Previous Failed Attempts ---------
+    {% for former_failed_knowledge in queried_former_failed_knowledge %} Attempt {{ loop.index }}:
+    =====Code:=====
+    {{ former_failed_knowledge.implementation.all_codes }}
+    =====Feedback:=====
+    {{ former_failed_knowledge.feedback }}
+    {% endfor %}
+    {% endif %}
+
+
+    ## Guidelines
+    1. Ensure that the dataset is loaded strictly from `/kaggle/input/`, following the exact folder structure described in the **Data Folder Description**, and do not attempt to load data from the current directory (`./`).
+    2. You should avoid using logging module to output information in your generated code, and instead use the print() function.
+    
+    ## Exploratory Data Analysis (EDA) part(Required):
+    - Before returning the data, you should always add an EDA part describing the data to help the following steps understand the data better.
+    - The EDA part should include but not limited in the following information in plain text:
+      - The shape of the data.
+      - The first 5 rows of the data.
+      - The data types of each column.
+      - The number of missing values in each column.
+      - The number of unique values in each column.
+      - The distribution of the target variable.
+      - Any other information that you think is important for the following steps.
+    - The EDA part should be drafted in plain text sending to standard output with command print or other similar functions with no more than ten thousand characters in the following schema: 
+      === Start of EDA part ===
+      { You EDA output content }
+      === End of EDA part ===
+      User will use the following code to match: re.search(r"(.*?)=== Start of EDA part ===(.*)=== End of EDA part ===", stdout, re.DOTALL).groups()[1]
+    - An evaluation agent will help to check whether the EDA part is added correctly.
+    - During the EDA part, you should try to avoid any irrelevant information sending to the standard output.
+
+    ## Output Format
+    {% if out_spec %}
+    {{ out_spec }}
+    {% else %}
+    Please response the code in the following json format. Here is an example structure for the JSON output:
+    {
+        "code": "The Python code as a string."
+    }
+    {% endif %}
+
+  user: |-
+    --------- Competition Information ---------
+    {{ competition_info }}
+
+    --------- Data Folder Description (All path are relative to the data folder) ---------
+    {{ folder_spec }}
+    
+    {% if latest_code %}
+    --------- Former code ---------
+    {{ latest_code }}
+    {% if latest_code_feedback is not none %}
+    --------- Feedback to former code ---------
+    {{ latest_code_feedback }}
+    The former code contains errors. You should correct the code based on the provided information, ensuring you do not repeat the same mistakes.
+    {% else %}
+    The former code is correct. You should try to improve the code based on the provided task while not changing the irrelevant parts.
+    {% endif %}
+    {% endif %} 
+
+    You should strictly follow the code specifications provided by the specification to implement the function.
+
+
+pipeline_eval:
+  system: |-
+    You are a data scientist responsible for evaluating code generation.
+
+    ## Task Description
+    The user is trying to build a code in the following scenario:
+    {{ scenario }}
+
+    The main code generation task is as follows:
+    {{ task_desc }}
+
+    The details on how to structure the code are given in the specification:
+    {{ spec }}
+
+    ## Evaluation Scope
+    Your focus is to check whether the workflow code:
+    1. Executes successfully, correctly generating a final submission.
+    2. Generates predictions in the correct format, ensuring they align with the submission structure!
+
+    ## Evaluation Criteria
+    You will be given the execution output (`stdout`) to determine correctness.  
+
+    [Note] 
+    1. Model performance is NOT a concern in this evaluation—only correct execution and formatting matter.
+    2. You only check the format of the submission since we only feed you part of the data, so the submission might has different index to the sample submission data.
+    
+    Please respond with your feedback in the following JSON format and order
+    ```json
+    {
+        "execution": "Describe whether the code executed successfully, correctly integrating all components and generating the final submission. Include any errors or issues encountered, and append all error messages and full traceback details without summarizing or omitting any information.",
+        "return_checking": "Verify the generated files, particularly the submission file. Ensure that its format matches the sample submission, checking the index, column names, and CSV content.",
+        "code": "Provide feedback on code quality, readability, and adherence to the given specifications.",
+        "final_decision": <true/false>
+    }
+    ```
+  
+  user: |-
+    --------- code generated by user ---------
+    {{ code }}
+
+    --------- code running stdout ---------
+    {{ stdout }}

rdagent/oai/backend/base.py

@@ -386,6 +386,7 @@ class APIBackend(ABC):
             cache_result = self.cache.chat_get(input_content_json)
             if cache_result is not None:
                 if LLM_SETTINGS.log_llm_chat_content:
+                    logger.info(self._build_log_messages(messages), tag="llm_messages")
                     logger.info(f"{LogColors.CYAN}Response:{cache_result}{LogColors.END}", tag="llm_messages")
                 return cache_result
 

rdagent/scenarios/data_science/dev/runner/eval.py

@@ -9,17 +9,12 @@ from rdagent.components.coder.CoSTEER.evaluators import (
     CoSTEEREvaluator,
     CoSTEERSingleFeedback,
 )
-from rdagent.components.coder.data_science.conf import (
-    DSCoderCoSTEERSettings,
-    get_ds_env,
-)
+from rdagent.components.coder.data_science.conf import get_ds_env
 from rdagent.core.evolving_framework import QueriedKnowledge
 from rdagent.core.experiment import FBWorkspace, Task
 from rdagent.log import rdagent_logger as logger
-from rdagent.oai.llm_utils import APIBackend
 from rdagent.utils.agent.tpl import T
 from rdagent.utils.agent.workflow import build_cls_from_json_with_retry
-from rdagent.utils.env import DockerEnv, MLEBDockerConf
 from rdagent.utils.fmt import shrink_text
 
 DIRNAME = Path(__file__).absolute().resolve().parent
@@ -48,6 +43,9 @@ class DSCoSTEERCoSTEEREvaluator(CoSTEEREvaluator):
 
         # execute workflow
         stdout = implementation.execute(env=env, entry="python -m coverage run main.py")
+        match = re.search(r"(.*?)=== Start of EDA part ===(.*)=== End of EDA part ===", stdout, re.DOTALL)
+        eda_output = match.groups()[1] if match else None
+        self.scen.eda_output = eda_output
         stdout = re.sub(r"=== Start of EDA part ===(.*)=== End of EDA part ===", "", stdout)
 
         # Check score file
@@ -67,9 +65,15 @@ class DSCoSTEERCoSTEEREvaluator(CoSTEEREvaluator):
                 )
 
                 # Check model names (index)
-                if model_set_in_scores != model_set_in_folder.union({"ensemble"}):
-                    score_check_text += f"\n[Error] The scores dataframe does not contain the correct model names as index.\ncorrect model names are: {model_set_in_folder.union({'ensemble'})}\nscore_df is:\n{score_df}"
-                    score_ret_code = 1
+                # in Pipeline task, we only check ensemble in scores.csv
+                if DS_RD_SETTING.coder_on_whole_pipeline:
+                    if "ensemble" not in model_set_in_scores:
+                        score_check_text += f"\n[Error] The score dataframe doesn't contain the ensemble model.\nscore_df is:\n{score_df}"
+                        score_ret_code = 1
+                else:
+                    if model_set_in_scores != model_set_in_folder.union({"ensemble"}):
+                        score_check_text += f"\n[Error] The scores dataframe does not contain the correct model names as index.\ncorrect model names are: {model_set_in_folder.union({'ensemble'})}\nscore_df is:\n{score_df}"
+                        score_ret_code = 1
 
                 # Check metric name (columns)
                 if score_df.columns.tolist() != [self.scen.metric_name]:
@@ -97,7 +101,7 @@ class DSCoSTEERCoSTEEREvaluator(CoSTEEREvaluator):
         submission_check_out, submission_ret_code = implementation.execute_ret_code(
             env=mde, entry="python test/mle_submission_format_test.py"
         )
-        stdout += f"\nMLEBench submission check:\n{submission_check_out}"
+        stdout += f"\nMLEBench submission check:\n{submission_check_out}\nIf MLEBench submission check returns a 'Submission is valid' or similar message, despite some warning messages, you should still consider the submission as valid and give a positive final decision. "
 
         system_prompt = T(".prompts:DSCoSTEER_eval.system").r(
             scenario=self.scen.get_scenario_all_desc(),
@@ -115,7 +119,7 @@ class DSCoSTEERCoSTEEREvaluator(CoSTEEREvaluator):
             init_kwargs_update_func=DSCoSTEEREvalFeedback.val_and_update_init_dict,
         )
 
-        if feedback:
+        if feedback and not DS_RD_SETTING.coder_on_whole_pipeline:
             # remove unused files
             implementation.execute(env=env, entry="python -m coverage json -o coverage.json")
             coverage_report_path = implementation.workspace_path / "coverage.json"

rdagent/scenarios/data_science/proposal/exp_gen/__init__.py

@@ -18,6 +18,12 @@ class DSExpGen(ExpGen):
         super().__init__(scen)
 
     def gen(self, trace: DSTrace) -> DSExperiment:
+        if DS_RD_SETTING.coder_on_whole_pipeline:
+            return DSProposalV2ExpGen(scen=self.scen).gen(
+                trace=trace,
+                max_trace_hist=self.max_trace_hist,
+                pipeline=True,
+            )
         next_missing_component = trace.next_incomplete_component()
         if next_missing_component is not None:
             return DSDraftExpGen(scen=self.scen).gen(

rdagent/scenarios/data_science/proposal/exp_gen/base.py

@@ -1,6 +1,7 @@
 from abc import abstractmethod
 from typing import Literal
 
+from rdagent.app.data_science.conf import DS_RD_SETTING
 from rdagent.core.evolving_framework import KnowledgeBase
 from rdagent.core.proposal import ExperimentFeedback, Hypothesis, Trace
 from rdagent.scenarios.data_science.experiment.experiment import COMPONENT, DSExperiment
@@ -80,7 +81,7 @@ class DSTrace(Trace[DataScienceScen, KnowledgeBase]):
         """
 
         final_component = self.COMPLETE_ORDER[-1]
-        has_final_component = False
+        has_final_component = True if DS_RD_SETTING.coder_on_whole_pipeline else False
         exp_and_feedback_list = []
         for exp, fb in self.hist:
             if has_final_component:
@@ -101,7 +102,7 @@ class DSTrace(Trace[DataScienceScen, KnowledgeBase]):
         Experiment or None
             The experiment result if found, otherwise None.
         """
-        if self.next_incomplete_component() is None:
+        if DS_RD_SETTING.coder_on_whole_pipeline or self.next_incomplete_component() is None:
             for exp, ef in self.hist[::-1]:
                 # the sota exp should be accepted decision and all required components are completed.
                 if ef.decision:

rdagent/scenarios/data_science/proposal/exp_gen/prompts.yaml

@@ -357,3 +357,9 @@ output_format:
     {
         "description": "A precise and comprehensive description of the main workflow script (`main.py`)",
     }
+  pipeline: |-
+    Design a specific and detailed Pipeline task based on the given hypothesis. The output should be detailed enough to directly implement the corresponding code.
+    The output should follow JSON format. The schema is as follows:
+    {
+        "description": "A precise and comprehensive description of the main workflow script (`main.py`)",
+    }

rdagent/scenarios/data_science/proposal/exp_gen/prompts_v2.yaml

@@ -38,9 +38,11 @@ feedback_problem:
     You will be given a competition scenario, trace history description, the current SOTA implementation and feedback.
     Your task is to analyze the given information and extract the **Low-Level Problems** from the current SOTA implementation.
 
+    {% if not pipeline %}
     ## Low-Level Problems
     ### Definition
     Low-level problems are specific and fine-grained technical, or methodological issues within one or more of the five components ('DataLoadSpec', 'FeatureEng', 'Model', 'Ensemble', 'Workflow') in the implementation.
+    {% endif %}
     
     ### Specification
     {{ problem_spec }}
@@ -73,13 +75,18 @@ hypothesis_gen:
     # Task 1: Hypothesis Proposal
     For each identified problem, you are required to propose a hypothesis aimed at improving the current SOTA implementation. 
     A hypothesis is a precise, testable, and actionable statement that proposes a specific modification or improvement to address an identified problem in a Kaggle competition implementation.
+    {% if not pipeline %}
     Each hypothesis should focus on one of the following 5 components of an implementation:
     {{ component_desc }}
+    {% else %}
+    Each hypothesis should focus on the whole pipeline.
+    {% endif %}
 
     ## Hypothesis Specification
     1. The hypothesis should be precise, testable, and directly actionable. Avoid general or vague statements. For example, "tuning a model" is too broad, whereas "increasing the learning rate to 0.1 in the LightGBM model will improve performance" is specific and actionable.
     2. Each hypothesis should focus on a single direction per experiment. Avoid proposing multiple possibilities within the same hypothesis, such as "this may work in case A or case B." Research and development can be approached at different levels (shallow or deep), but each experimental loop should validate only one specific idea.
     3. The hypothesis should based on current SOTA solution. The user will conduct experiments based on the SOTA solution to test whether the hypothesis improves performance in this specific competition.
+    4. For problems which you think are covered by the current SOTA implementation or by the former hypothesis, you should ignore that problem and not include it in your response. But you should not respond an empty hypothesis list.
 
 
     # Task 2: Hypothesis Evaluation
@@ -164,7 +171,9 @@ specification:
 
 output_format:
   problem: |-
-    For each of the identified problem, you should strictly adhere to the following JSON schema. Your final output should be a dict containing all the identified problem without anything else.
+    For each of the identified problem, you should strictly adhere to the following JSON schema. 
+    Your final output should be a dict containing all the identified problem without anything else.
+    Please respond at most five problems considering the most valuable and recently not explored.
     {
       "problem name 1": {
         "problem": "Description of the first issue",
@@ -180,7 +189,9 @@ output_format:
     {
       "problem name 1": {
         "observation": "The observation of the given scenario, data characteristics, or trace history.",
-        "component": "The component name that the hypothesis focus on. Must be one of ('DataLoadSpec', 'FeatureEng', 'Model', 'Ensemble', 'Workflow').",
+        {% if not pipeline %}"component": "The component name that the hypothesis focus on. Must be one of ('DataLoadSpec', 'FeatureEng', 'Model', 'Ensemble', 'Workflow').",
+        {% else %}"component": "The component name that the hypothesis focus on. Must be 'Pipeline'.",
+        {% endif %}
         "reason": "A brief explanation, also in one or two sentences, outlining the rationale behind the hypothesis. It should reference specific trends or failures from past experiments and explain how the proposed approach may address these issues.",
         "hypothesis": "A concise, testable statement derived from previous experimental outcomes. Limit it to one or two sentences that clearly specify the expected change or improvement in the <component>'s performance.",
         "evaluation": {

rdagent/scenarios/data_science/proposal/exp_gen/proposal.py

@@ -7,6 +7,7 @@ from rdagent.app.data_science.conf import DS_RD_SETTING
 from rdagent.components.coder.data_science.ensemble.exp import EnsembleTask
 from rdagent.components.coder.data_science.feature.exp import FeatureTask
 from rdagent.components.coder.data_science.model.exp import ModelTask
+from rdagent.components.coder.data_science.pipeline.exp import PipelineTask
 from rdagent.components.coder.data_science.raw_data_loader.exp import DataLoaderTask
 from rdagent.components.coder.data_science.workflow.exp import WorkflowTask
 from rdagent.core.proposal import ExpGen
@@ -48,6 +49,11 @@ COMPONENT_TASK_MAPPING = {
         "task_output_format": T(".prompts:output_format.workflow").r(),
         "task_class": WorkflowTask,
     },
+    "Pipeline": {
+        "target_name": "Pipeline",
+        "task_output_format": T(".prompts:output_format.pipeline").r(),
+        "task_class": PipelineTask,
+    },
 }
 
 
@@ -257,10 +263,12 @@ class DSProposalV2ExpGen(ExpGen):
         sota_exp_feedback_list_desc: str,
         failed_exp_feedback_list_desc: str,
         sota_exp_desc: str,
+        pipeline: bool,
     ) -> Dict:
         sys_prompt = T(".prompts_v2:scenario_problem.system").r(
             problem_spec=T(".prompts_v2:specification.problem").r(),
             problem_output_format=T(".prompts_v2:output_format.problem").r(),
+            pipeline=pipeline,
         )
         user_prompt = T(".prompts_v2:feedback_problem.user").r(
             scenario_desc=scenario_desc,
@@ -284,11 +292,13 @@ class DSProposalV2ExpGen(ExpGen):
         failed_exp_feedback_list_desc: str,
         sota_exp_desc: str,
         problems: list,
+        pipeline: bool,
     ) -> Dict:
         sys_prompt = T(".prompts_v2:hypothesis_gen.system").r(
             component_desc=component_desc,
             hypothesis_spec=T(".prompts_v2:specification.hypothesis").r(),
-            hypothesis_output_format=T(".prompts_v2:output_format.hypothesis").r(),
+            hypothesis_output_format=T(".prompts_v2:output_format.hypothesis").r(pipeline=pipeline),
+            pipeline=pipeline,
         )
         user_prompt = T(".prompts_v2:hypothesis_gen.user").r(
             scenario_desc=scenario_desc,
@@ -305,8 +315,10 @@ class DSProposalV2ExpGen(ExpGen):
         )
         return json.loads(response)
 
-    def hypothesis_rank(self, hypothesis_dict: dict, problem_dict: dict) -> DSHypothesis:
+    def hypothesis_rank(self, hypothesis_dict: dict, problem_dict: dict, pipeline: bool) -> DSHypothesis:
         # TODO use rule base or llm to rank the hypothesis
+        if pipeline:
+            problem_dict = {k: v for k, v in hypothesis_dict.items() if v.get("component", "") == "Pipeline"}
 
         max_score_problem_name = (
             pd.DataFrame(
@@ -331,9 +343,10 @@ class DSProposalV2ExpGen(ExpGen):
         sota_exp_desc: str,
         sota_exp: DSExperiment,
         hypothesis: DSHypothesis,
+        pipeline: bool,
     ) -> DSExperiment:
         component_info = COMPONENT_TASK_MAPPING.get(hypothesis.component)
-        if DS_RD_SETTING.spec_enabled:
+        if not pipeline and DS_RD_SETTING.spec_enabled and sota_exp is not None:
             task_spec = sota_exp.experiment_workspace.file_dict[component_info["spec_file"]]
         else:
             task_spec = T(f"scenarios.data_science.share:component_spec.{hypothesis.component}").r()
@@ -342,7 +355,7 @@ class DSProposalV2ExpGen(ExpGen):
             task_specification=task_spec,
             task_output_format=component_info["task_output_format"],
             component_desc=component_desc,
-            workflow_check=(not hypothesis.component == "Workflow"),
+            workflow_check=not pipeline and hypothesis.component != "Workflow",
         )
         user_prompt = T(".prompts_v2:task_gen.user").r(
             scenario_desc=scenario_desc, sota_exp_desc=sota_exp_desc, hypothesis=str(hypothesis)
@@ -361,7 +374,7 @@ class DSProposalV2ExpGen(ExpGen):
             if isinstance(task_design, str)
             else task_design.get("description", f"{component_info['target_name']} description not provided")
         )
-        task_class = COMPONENT_TASK_MAPPING[hypothesis.component]["task_class"]
+        task_class = component_info["task_class"]
         task = task_class(
             name=task_name,
             description=description,
@@ -369,9 +382,10 @@ class DSProposalV2ExpGen(ExpGen):
         new_workflow_desc = task_dict.get("workflow_update", "No update needed")
         exp = DSExperiment(pending_tasks_list=[[task]], hypothesis=hypothesis)
         # exp.experiment_workspace.inject_code_from_folder(sota_exp.experiment_workspace.workspace_path)
-        exp.experiment_workspace.inject_code_from_file_dict(sota_exp.experiment_workspace)
+        if sota_exp is not None:
+            exp.experiment_workspace.inject_code_from_file_dict(sota_exp.experiment_workspace)
 
-        if new_workflow_desc != "No update needed":
+        if not pipeline and new_workflow_desc != "No update needed":
             workflow_task = WorkflowTask(
                 name="Workflow",
                 description=new_workflow_desc,
@@ -379,7 +393,7 @@ class DSProposalV2ExpGen(ExpGen):
             exp.pending_tasks_list.append([workflow_task])
         return exp
 
-    def gen(self, trace: DSTrace, max_trace_hist: int) -> DSExperiment:
+    def gen(self, trace: DSTrace, max_trace_hist: int, pipeline: bool = False) -> DSExperiment:
         component_desc = "\n".join(
             [
                 f"[{key}] {value}"
@@ -418,6 +432,7 @@ class DSProposalV2ExpGen(ExpGen):
             sota_exp_feedback_list_desc=sota_exp_feedback_list_desc,
             failed_exp_feedback_list_desc=failed_exp_feedback_list_desc,
             sota_exp_desc=sota_exp_desc,
+            pipeline=pipeline,
         )
         all_problems = {**scen_problems, **fb_problems}
 
@@ -429,26 +444,29 @@ class DSProposalV2ExpGen(ExpGen):
             failed_exp_feedback_list_desc=failed_exp_feedback_list_desc,
             sota_exp_desc=sota_exp_desc,
             problems=all_problems,
+            pipeline=pipeline,
         )
-        sota_exp_model_file_count = len(
-            [
-                k
-                for k in sota_exp.experiment_workspace.file_dict.keys()
-                if k.endswith(".py") and "test" not in k and k.startswith("model")
-            ]
-        )
-        if sota_exp_model_file_count <= 1:
-            pop_names = []
-            for problem_name in hypothesis_dict:
-                if hypothesis_dict[problem_name].get("component", "") == "Ensemble":
-                    pop_names.append(problem_name)
-            for name in pop_names:
-                hypothesis_dict.pop(name)
+        if not pipeline:
+            sota_exp_model_file_count = len(
+                [
+                    k
+                    for k in sota_exp.experiment_workspace.file_dict.keys()
+                    if k.endswith(".py") and "test" not in k and k.startswith("model")
+                ]
+            )
+            if sota_exp_model_file_count <= 1:
+                pop_names = []
+                for problem_name in hypothesis_dict:
+                    if hypothesis_dict[problem_name].get("component", "") == "Ensemble":
+                        pop_names.append(problem_name)
+                for name in pop_names:
+                    hypothesis_dict.pop(name)
 
         # Step 3: Select the best hypothesis
         new_hypothesis = self.hypothesis_rank(
             hypothesis_dict=hypothesis_dict,
             problem_dict=all_problems,
+            pipeline=pipeline,
         )
 
         return self.task_gen(
@@ -457,4 +475,5 @@ class DSProposalV2ExpGen(ExpGen):
             sota_exp_desc=sota_exp_desc,
             sota_exp=sota_exp,
             hypothesis=new_hypothesis,
+            pipeline=pipeline,
         )

rdagent/scenarios/data_science/scen/__init__.py

@@ -114,6 +114,7 @@ class DataScienceScen(Scenario):
             metric_name=self.metric_name,
             metric_direction=self.metric_direction,
             eda_output=self.eda_output,
+            time_limit=f"{DS_RD_SETTING.full_timeout / 60 / 60 : .2f} hours",
         )
 
     def get_runtime_environment(self) -> str:

rdagent/scenarios/data_science/scen/prompts.yaml

@@ -8,6 +8,12 @@ scenario_description: |-
   ------The expected output & submission format specifications------
   {{ submission_specifications }}
 
+  ------The name of the evaluation metric used------
+  {{ metric_name }}
+
+  ------The time limit to your code------
+  You code running is limit to {{ time_limit }}, please change yor model type and model parameters to make sure your code can run within the time limit.
+
   {% if evaluation is not none %}
   ------Evaluation------
   {{ evaluation }}

rdagent/scenarios/data_science/share.yaml

@@ -240,3 +240,49 @@ component_spec:
       {% endfor %}
       final_pred = ensemble_workflow(test_preds_dict, val_preds_dict, val_y)
       {% endraw %}
+    
+  Pipeline: |-
+    1. File Handling:
+      - Handle file encoding and delimiters appropriately.
+      - Combine or process multiple files if necessary.
+      - Avoid using the sample submission file to infer test indices. If a dedicated test index file is available, use that. If not, use the order in the test file as the test index.
+      - Ensure you load the actual data from the files, not just the filenames or paths. Do not postpone data loading to later steps.
+
+    2. Data Preprocessing:
+      - Convert data types correctly (e.g., numeric, categorical, date parsing).
+      - Optimize memory usage for large datasets using techniques like downcasting or reading data in chunks if necessary.
+      - Domain-Specific Handling: 
+        - Apply competition-specific preprocessing steps as needed (e.g., text tokenization, image resizing).
+
+    3. Code Standards:
+      - DO NOT use progress bars (e.g., `tqdm`).
+      - DO NOT use the sample submission file to extract test index information.
+      - DO NOT exclude features inadvertently during this process.
+
+    4. NOTES
+      - Never use sample submission as the test index, as it may not be the same as the test data. Use the test index file or test data source to get the test index.
+      - For neural network models, use pytorch rather than tensorflow as the backend if possible.
+      - For decision tree models, use xgboost or RandomForest rather than lightgbm as the backend if possible.
+
+    5. General Considerations:
+      - Ensure scalability for large datasets.
+      - Handle missing values and outliers appropriately (e.g., impute, remove, or replace).
+      - Ensure consistency between feature data types and transformations.
+      - Prevent data leakage: Do not use information derived from the test set when transforming training data.     
+
+    6. Notes:
+      - GPU and multiprocessing are available and are encouraged to use for accelerating transformations.
+      - Feature engineering should be executed **once** and reused across all models to ensure consistency: `X_transformed, y_transformed, X_test_transformed = feat_eng(X, y, X_test)`
+  
+    7. Metric Calculation and Storage:
+      - Calculate the metric (mentioned in the evaluation section of the competition information) for each model and ensemble strategy on valid, and save the results in `scores.csv`
+      - The evaluation should be based on 5-fold cross-validation but only if that's an appropriate evaluation for the task at hand.
+      - Even if only one model is present, compute the ensemble score and store it under `"ensemble"`.
+      - The index of `scores.csv` should include the model name and the "ensemble" strategy. "ensemble" should be exactly in the index with all lower case letters.
+      - The column names in `scores.csv` should be:
+        - Model: The name of the model or ensemble strategy.
+        - <metric_name>: The calculated metric value for that model or ensemble strategy. The metric name can be found in the scenario description. The metric name should be exactly the same as the one in the scenario description since user will use it to check the result.
+
+    8. Submission File:
+      - Save the final predictions as `submission.csv`, ensuring the format matches the competition requirements (refer to `sample_submission` in the Folder Description for the correct structure).
+      - Present the required submission format explicitly and ensure the output adheres to it.