feat: out spec change for o1-preview (#666)

* add not_json batcheditout * ensemble out_spec change * feature out_spec change * model out_spec change * workflow out_spec change * runner debugger out_spec change * filter_progress_bar return format fix * data_loader and spec out_spec change * show finish_reason in llm log * json_mode fix * remove hardcode * fix CI * fix grammer * complete PythonBatchEditOut logic --------- Co-authored-by: yuanteli <1957922024@qq.com>
2026-07-29 08:27:43 +00:00 · 2025-03-18 17:43:44 +08:00
parent d32d8754d9
commit e21b334741
15 changed files with 160 additions and 126 deletions
@@ -32,6 +32,7 @@ from rdagent.core.exception import CoderError
 from rdagent.core.experiment import FBWorkspace
 from rdagent.core.scenario import Scenario
 from rdagent.oai.llm_utils import APIBackend
+from rdagent.utils.agent.ret import PythonAgentOut
 from rdagent.utils.agent.tpl import T


@@ -76,6 +77,7 @@ class EnsembleMultiProcessEvolvingStrategy(MultiProcessEvolvingStrategy):
                queried_former_failed_knowledge[0] if queried_former_failed_knowledge else None
            ),
            all_code=workspace.all_codes,
+            out_spec=PythonAgentOut.get_spec(),
        )
        user_prompt = T(".prompts:ensemble_coder.user").r(
            ensemble_spec=workspace.file_dict["spec/ensemble.md"],
@@ -84,14 +86,12 @@ class EnsembleMultiProcessEvolvingStrategy(MultiProcessEvolvingStrategy):
        )

        for _ in range(5):
-            ensemble_code = json.loads(
+            ensemble_code = PythonAgentOut.extract_output(
                APIBackend().build_messages_and_create_chat_completion(
                    user_prompt=user_prompt,
                    system_prompt=system_prompt,
-                    json_mode=True,
-                    json_target_type=Dict[str, str],
                )
-            )["code"]
+            )
            if ensemble_code != workspace.file_dict.get("ensemble.py"):
                break
            else:
@@ -41,10 +41,14 @@ ensemble_coder:
    2. You should avoid using logging module to output information in your generated code, and instead use the print() function.

    ## Output Format
+    {% if out_spec %}
+    {{ out_spec }}
+    {% else %}
    Please response the code in the following json format. Here is an example structure for the JSON output:
    {
        "code": "The Python code as a string."
    }
+    {% endif %}

  user: |-
    --------- Ensemble Specification ---------
@@ -19,6 +19,7 @@ from rdagent.core.exception import CoderError
 from rdagent.core.experiment import FBWorkspace
 from rdagent.core.scenario import Scenario
 from rdagent.oai.llm_utils import APIBackend
+from rdagent.utils.agent.ret import PythonAgentOut
 from rdagent.utils.agent.tpl import T


@@ -61,6 +62,7 @@ class FeatureMultiProcessEvolvingStrategy(MultiProcessEvolvingStrategy):
            data_loader_code=workspace.file_dict.get("load_data.py"),
            queried_similar_successful_knowledge=queried_similar_successful_knowledge,
            queried_former_failed_knowledge=queried_former_failed_knowledge[0],
+            out_spec=PythonAgentOut.get_spec(),
        )
        user_prompt = T(".prompts:feature_coder.user").r(
            feature_spec=workspace.file_dict["spec/feature.md"],
@@ -69,14 +71,12 @@ class FeatureMultiProcessEvolvingStrategy(MultiProcessEvolvingStrategy):
        )

        for _ in range(5):
-            feature_code = json.loads(
+            feature_code = PythonAgentOut.extract_output(
                APIBackend().build_messages_and_create_chat_completion(
                    user_prompt=user_prompt,
                    system_prompt=system_prompt,
-                    json_mode=True,
-                    json_target_type=Dict[str, str],
                )
-            )["code"]
+            )
            if feature_code != workspace.file_dict.get("feature.py"):
                break
            else:
@@ -45,10 +45,14 @@ feature_coder:
      - You should avoid using logging module to output information in your generated code, and instead use the print() function.

    ## Output Format
+    {% if out_spec %}
+    {{ out_spec }}
+    {% else %}
    Please response the code in the following json format. Here is an example structure for the JSON output:
    {
        "code": "The Python code as a string."
    }
+    {% endif %}
  
  user: |-
    --------- Feature Processing Specification ---------
@@ -20,7 +20,7 @@ from rdagent.core.exception import CoderError
 from rdagent.core.experiment import FBWorkspace
 from rdagent.core.scenario import Scenario
 from rdagent.oai.llm_utils import APIBackend
-from rdagent.utils.agent.ret import BatchEditOut
+from rdagent.utils.agent.ret import PythonBatchEditOut
 from rdagent.utils.agent.tpl import T


@@ -63,7 +63,7 @@ class ModelMultiProcessEvolvingStrategy(MultiProcessEvolvingStrategy):
            feature_code=workspace.file_dict["feature.py"],
            queried_similar_successful_knowledge=queried_similar_successful_knowledge,
            queried_former_failed_knowledge=queried_former_failed_knowledge[0],
-            out_spec=BatchEditOut.get_spec(),
+            out_spec=PythonBatchEditOut.get_spec(),
        )
        # user_prompt = T(".prompts:model_coder.user").r(
        #     model_spec=workspace.file_dict["spec/model.md"],
@@ -80,12 +80,10 @@ class ModelMultiProcessEvolvingStrategy(MultiProcessEvolvingStrategy):
        )

        for _ in range(5):
-            batch_edit = BatchEditOut.extract_output(
+            batch_edit = PythonBatchEditOut.extract_output(
                APIBackend().build_messages_and_create_chat_completion(
                    user_prompt=user_prompt,
                    system_prompt=system_prompt,
-                    json_mode=BatchEditOut.json_mode,
-                    json_target_type=Dict[str, str],
                )
            )

@@ -50,6 +50,7 @@ from rdagent.core.exception import CoderError
 from rdagent.core.experiment import FBWorkspace
 from rdagent.core.scenario import Scenario
 from rdagent.oai.llm_utils import APIBackend
+from rdagent.utils.agent.ret import PythonAgentOut
 from rdagent.utils.agent.tpl import T


@@ -110,31 +111,11 @@ class DataLoaderMultiProcessEvolvingStrategy(MultiProcessEvolvingStrategy):

            spec_session = APIBackend().build_chat_session(session_system_prompt=system_prompt)

-            data_loader_spec = json.loads(
-                spec_session.build_chat_completion(
-                    user_prompt=data_loader_prompt, json_mode=True, json_target_type=Dict[str, str]
-                )
-            )["spec"]
-            feature_spec = json.loads(
-                spec_session.build_chat_completion(
-                    user_prompt=feature_prompt, json_mode=True, json_target_type=Dict[str, str]
-                )
-            )["spec"]
-            model_spec = json.loads(
-                spec_session.build_chat_completion(
-                    user_prompt=model_prompt, json_mode=True, json_target_type=Dict[str, str]
-                )
-            )["spec"]
-            ensemble_spec = json.loads(
-                spec_session.build_chat_completion(
-                    user_prompt=ensemble_prompt, json_mode=True, json_target_type=Dict[str, str]
-                )
-            )["spec"]
-            workflow_spec = json.loads(
-                spec_session.build_chat_completion(
-                    user_prompt=workflow_prompt, json_mode=True, json_target_type=Dict[str, str]
-                )
-            )["spec"]
+            data_loader_spec = spec_session.build_chat_completion(user_prompt=data_loader_prompt)
+            feature_spec = spec_session.build_chat_completion(user_prompt=feature_prompt)
+            model_spec = spec_session.build_chat_completion(user_prompt=model_prompt)
+            ensemble_spec = spec_session.build_chat_completion(user_prompt=ensemble_prompt)
+            workflow_spec = spec_session.build_chat_completion(user_prompt=workflow_prompt)
        else:
            data_loader_spec = workspace.file_dict["spec/data_loader.md"]
            feature_spec = workspace.file_dict["spec/feature.md"]
@@ -147,6 +128,7 @@ class DataLoaderMultiProcessEvolvingStrategy(MultiProcessEvolvingStrategy):
            task_desc=data_loader_task_info,
            queried_similar_successful_knowledge=queried_similar_successful_knowledge,
            queried_former_failed_knowledge=queried_former_failed_knowledge[0],
+            out_spec=PythonAgentOut.get_spec(),
        )
        user_prompt = T(".prompts:data_loader_coder.user").r(
            competition_info=competition_info,
@@ -157,14 +139,12 @@ class DataLoaderMultiProcessEvolvingStrategy(MultiProcessEvolvingStrategy):
        )

        for _ in range(5):
-            data_loader_code = json.loads(
+            data_loader_code = PythonAgentOut.extract_output(
                APIBackend().build_messages_and_create_chat_completion(
                    user_prompt=user_prompt,
                    system_prompt=system_prompt,
-                    json_mode=True,
-                    json_target_type=Dict[str, str],
                )
-            )["code"]
+            )
            if data_loader_code != workspace.file_dict.get("load_data.py"):
                break
            else:
@@ -82,10 +82,8 @@ spec:
        You should follow the provided specifications to improve this task.
      {% endif %}

-      Please respond with a JSON structure as follows:
-      {
-          "spec": "The function definition in code format, tailored to the Competition Information, with detailed explanations provided in the docstring."
-      }
+      ## Output Format
+      You should return the specification in markdown format directly, while the **function definition** within it should be in code format, tailored to the Competition Information, with detailed explanations provided in the docstring.

    feature: |-
      Feature engineering specification text should adhere to the following requirements:
@@ -137,10 +135,8 @@ spec:
        You should follow the provided specifications to improve this task.
      {% endif %}

-      Please respond with a JSON structure as follows:
-      {
-          "spec": "The function definition in code format, tailored to the Competition Information, with detailed explanations provided in the docstring."
-      }
+      ## Output Format
+      You should return the specification in markdown format directly, while the **function definition** within it should be in code format, tailored to the Competition Information, with detailed explanations provided in the docstring.

    model: |-
      Model building specification text should adhere to the following requirements:
@@ -184,10 +180,8 @@ spec:
        You should follow the provided specifications to improve this task.
      {% endif %}

-      Please respond in the following JSON format:
-      {
-          "spec": "The function definition in code format, tailored to the Competition Information, with detailed explanations provided in the docstring."
-      }
+      ## Output Format
+      You should return the specification in markdown format directly, while the **function definition** within it should be in code format, tailored to the Competition Information, with detailed explanations provided in the docstring.

    ensemble: |-
      Ensemble specification text adhere to the following requirements:
@@ -244,72 +238,69 @@ spec:
        You should follow the provided specifications to improve this task.
      {% endif %}

-      Please respond in the following JSON format:
-      {
-          "spec": "The function definition in code format, tailored to the Competition Information, with detailed explanations provided in the docstring."
-      }
+      ## Output Format
+      You should return the specification in markdown format directly, while the **function definition** within it should be in code format, tailored to the Competition Information, with detailed explanations provided in the docstring.

    workflow: |-
      Your task is to implement the main workflow script (`main.py`) for a Kaggle-style machine learning competition project. 
      Follow the provided project structure and specifications to ensure consistency and maintainability:
-        1. Workflow Integration:
-          - Integrate the following components into the workflow:
-            - Data loading (`load_data.py`).
-            - Feature engineering (`feature.py`).
-            - Model workflow for training and testing (`model_*.py`). 
-            - Ensemble workflow that combines results from the model workflow to obtain the final prediction (`ensemble.py`).
-          - Treat each component as a modular and callable Python function.
-          - The workflow script should be flexible enough to handle either a single model or multiple models, with filenames (model_*.py) that are not determined at the outset.
-            For multiple model selection, utilize Python code to identify eligible models based on filenames, for example:
-            ```python
-            available_models = [f for f in os.listdir('.') if f.startswith('model_') and 'test' not in f]
-            ```
-        2. Feature Engineering
-          - The feature engineering should be called only once. For example:
-            `X_transformed, y_transformed, X_test_transformed = feat_eng(X, y, X_test)`
-          - It should be called before dataset splitting.
+      1. Workflow Integration:
+        - Integrate the following components into the workflow:
+          - Data loading (`load_data.py`).
+          - Feature engineering (`feature.py`).
+          - Model workflow for training and testing (`model_*.py`). 
+          - Ensemble workflow that combines results from the model workflow to obtain the final prediction (`ensemble.py`).
+        - Treat each component as a modular and callable Python function.
+        - The workflow script should be flexible enough to handle either a single model or multiple models, with filenames (model_*.py) that are not determined at the outset.
+          For multiple model selection, utilize Python code to identify eligible models based on filenames, for example:
+          ```python
+          available_models = [f for f in os.listdir('.') if f.startswith('model_') and 'test' not in f]
+          ```
+      2. Feature Engineering
+        - The feature engineering should be called only once. For example:
+          `X_transformed, y_transformed, X_test_transformed = feat_eng(X, y, X_test)`
+        - It should be called before dataset splitting.

-        3. Dataset Splitting
-          - The dataset returned by `load_data` is not pre-split. After calling `feat_eng`, split the data into training and test sets.
-          - If feasible, apply cross-validation on the training set (`X_transformed`, `y_transformed`) to ensure a reliable assessment of model performance.
-          - Keep the test set (`X_test_transformed`) unchanged, as it is only used for generating the final predictions.
+      3. Dataset Splitting
+        - The dataset returned by `load_data` is not pre-split. After calling `feat_eng`, split the data into training and test sets.
+        - If feasible, apply cross-validation on the training set (`X_transformed`, `y_transformed`) to ensure a reliable assessment of model performance.
+        - Keep the test set (`X_test_transformed`) unchanged, as it is only used for generating the final predictions.

-        4. Submission File:
-          - Save the final predictions as `submission.csv`, ensuring the format matches the competition requirements (refer to `sample_submission` in the Folder Description for the correct structure).
-          - Present the required submission format explicitly and ensure the output adheres to it.
+      4. Submission File:
+        - Save the final predictions as `submission.csv`, ensuring the format matches the competition requirements (refer to `sample_submission` in the Folder Description for the correct structure).
+        - Present the required submission format explicitly and ensure the output adheres to it.

-        5. Code Standards:
-          - Do not use progress bars (e.g., tqdm) in the code.
+      5. Code Standards:
+        - Do not use progress bars (e.g., tqdm) in the code.

-        6. Ensemble Strategy:
-          Consolidate all model outputs into a dictionary, where each key is the model's filename (excluding the .py extension) and its corresponding value is the model's output.
-          Sample code:
-          {% raw %}
-          {% for model_name in model_names %}
-          model_module = __import__(model_name.replace('.py', ''))
-          val_pred, test_pred, _ = model_module.model_workflow(
-            X=train_X,
-            y=train_y,
-            val_X=val_X,
-            val_y=val_y,
-            test_X=X_test_transformed
-          )
-          val_preds_dict[model_module.__name__] = val_pred
-          test_preds_dict[model_module.__name__] = test_pred
-          {% endfor %}
-          final_pred = ensemble_workflow(test_preds_dict, val_preds_dict, val_y)
-          {% endraw %}
+      6. Ensemble Strategy:
+        Consolidate all model outputs into a dictionary, where each key is the model's filename (excluding the .py extension) and its corresponding value is the model's output.
+        Sample code:
+        {% raw %}
+        {% for model_name in model_names %}
+        model_module = __import__(model_name.replace('.py', ''))
+        val_pred, test_pred, _ = model_module.model_workflow(
+          X=train_X,
+          y=train_y,
+          val_X=val_X,
+          val_y=val_y,
+          test_X=X_test_transformed
+        )
+        val_preds_dict[model_module.__name__] = val_pred
+        test_preds_dict[model_module.__name__] = test_pred
+        {% endfor %}
+        final_pred = ensemble_workflow(test_preds_dict, val_preds_dict, val_y)
+        {% endraw %}

-        {% if latest_spec %}
-        7. Former Specification:
-          {{ latest_spec }}
-          You should follow the provided specifications to improve this task.
-        {% endif %}
+      {% if latest_spec %}
+      7. Former Specification:
+        {{ latest_spec }}
+        You should follow the provided specifications to improve this task.
+      {% endif %}

-        Please response the specification in the following json format. Here is an example structure for the JSON output:
-        {
-            "spec": "The corresponding specification string as described above. You should create the rules based on the competition information instead of copying the requirements."
-        }
+      ## Output Format
+      You should return the specification in markdown format directly.
+      You should create the rules based on the competition information instead of copying the requirements.

 data_loader_coder:
  system: |-
@@ -365,10 +356,14 @@ data_loader_coder:
    - During the EDA part, you should try to avoid any irrelevant information sending to the standard output.

    ## Output Format
+    {% if out_spec %}
+    {{ out_spec }}
+    {% else %}
    Please response the code in the following json format. Here is an example structure for the JSON output:
    {
        "code": "The Python code as a string."
    }
+    {% endif %}

  user: |-
    --------- Competition Information ---------
@@ -21,6 +21,7 @@ from rdagent.core.exception import CoderError
 from rdagent.core.experiment import FBWorkspace
 from rdagent.core.scenario import Scenario
 from rdagent.oai.llm_utils import APIBackend
+from rdagent.utils.agent.ret import PythonAgentOut
 from rdagent.utils.agent.tpl import T


@@ -60,6 +61,7 @@ class WorkflowMultiProcessEvolvingStrategy(MultiProcessEvolvingStrategy):
            competition_info=self.scen.get_scenario_all_desc(),
            queried_similar_successful_knowledge=queried_similar_successful_knowledge,
            queried_former_failed_knowledge=queried_former_failed_knowledge[0],
+            out_spec=PythonAgentOut.get_spec(),
        )
        user_prompt = T(".prompts:workflow_coder.user").r(
            load_data_code=workspace.file_dict["load_data.py"],
@@ -72,14 +74,12 @@ class WorkflowMultiProcessEvolvingStrategy(MultiProcessEvolvingStrategy):
        )

        for _ in range(5):
-            workflow_code = json.loads(
+            workflow_code = PythonAgentOut.extract_output(
                APIBackend().build_messages_and_create_chat_completion(
                    user_prompt=user_prompt,
                    system_prompt=system_prompt,
-                    json_mode=True,
-                    json_target_type=Dict[str, str],
                )
-            )["code"]
+            )
            if workflow_code != workspace.file_dict.get("main.py"):
                break
            else:
@@ -42,10 +42,14 @@ workflow_coder:
    5. You should avoid using logging module to output information in your generated code, and instead use the print() function.

    ## Output Format
+    {% if out_spec %}
+    {{ out_spec }}
+    {% else %}
    Please response the code in the following json format. Here is an example structure for the JSON output:
    {
        "code": "The Python code as a string."
    }
+    {% endif %}
  
  user: |-
    --------- Workflow Specification ---------