feat: add model removal and adjust some framework logic (#681)

* prune model task * add component_description * add model removal logic to component, hypo, and task gen * fix ci * adjust coder to meet the requirement of model removal * fix and refine the logic of model removal * add model removal logic in model_eval * fix ci * fix ci * prune some unnecessary codes
2026-08-01 09:27:43 +00:00 · 2025-03-18 14:42:53 +08:00
parent 229866123b
commit 3e9ba73c2f
9 changed files with 137 additions and 79 deletions
@@ -103,6 +103,10 @@ class ModelMultiProcessEvolvingStrategy(MultiProcessEvolvingStrategy):
                f"{target_task.name}.py"
            ] != workspace.file_dict.get(f"{target_task.name}.py"):
                break
+
+            # If the task involves model removal, assume it can only process one model at a time.
+            if len(batch_edit) == 1 and batch_edit[f"{target_task.name}.py"] == "__DEL__":
+                break
        else:
            raise CoderError("Failed to generate a new model code.")

@@ -59,17 +59,23 @@ class ModelGeneralCaseSpecEvaluator(CoSTEEREvaluator):
        env = get_ds_env()
        env.conf.extra_volumes = {f"{DS_RD_SETTING.local_data_path}/sample/{self.scen.competition}": "/kaggle/input"}

-        fname = "test/model_test.py"
-        test_code = (
-            (DIRNAME / "eval_tests" / "model_test.txt").read_text().replace("model01", target_task.name)
-        )  # only check the model changed this time
-        implementation.inject_files(**{fname: test_code})
-        stdout = implementation.execute(env=env, entry=f"python {fname}")
+        if_model_removed = False

-        if stdout is None:
-            raise CoderError(
-                "The execution output contains too many progress bars and results in the LLM's token size exceeding the limit."
-            )
+        if f"{target_task.name}.py" in implementation.file_dict:
+            fname = "test/model_test.py"
+            test_code = (
+                (DIRNAME / "eval_tests" / "model_test.txt").read_text().replace("model01", target_task.name)
+            )  # only check the model changed this time
+            implementation.inject_files(**{fname: test_code})
+            stdout = implementation.execute(env=env, entry=f"python {fname}")
+
+            if stdout is None:
+                raise CoderError(
+                    "The execution output contains too many progress bars and results in the LLM's token size exceeding the limit."
+                )
+        else:
+            if_model_removed = True
+            stdout = f"Model {target_task.name} removal succeeded."

        if "main.py" in implementation.file_dict:
            workflow_stdout = implementation.execute(env=env, entry="python main.py")
@@ -77,19 +83,31 @@ class ModelGeneralCaseSpecEvaluator(CoSTEEREvaluator):
        else:
            workflow_stdout = None

-        system_prompt = T(".prompts:model_eval.system").r(
-            task_desc=target_task.get_task_information(),
-            test_code=test_code,
-            code=implementation.file_dict[f"{target_task.name}.py"],
-            scenario=self.scen.get_scenario_all_desc(),
-            spec=implementation.file_dict["spec/model.md"],
-            workflow_stdout=workflow_stdout,
-            workflow_code=implementation.all_codes,
-        )
-        user_prompt = T(".prompts:model_eval.user").r(
-            stdout=stdout,
-            workflow_stdout=workflow_stdout,
-        )
+        if if_model_removed:
+            system_prompt = T(".prompts:model_eval_rm.system").r(
+                task_desc=target_task.get_task_information(),
+                workflow_stdout=workflow_stdout,
+                workflow_code=implementation.all_codes,
+            )
+            user_prompt = T(".prompts:model_eval_rm.user").r(
+                stdout=stdout,
+                workflow_stdout=workflow_stdout,
+            )
+        else:
+            system_prompt = T(".prompts:model_eval.system").r(
+                task_desc=target_task.get_task_information(),
+                test_code=test_code,
+                code=implementation.file_dict[f"{target_task.name}.py"],
+                scenario=self.scen.get_scenario_all_desc(),
+                spec=implementation.file_dict["spec/model.md"],
+                workflow_stdout=workflow_stdout,
+                workflow_code=implementation.all_codes,
+            )
+            user_prompt = T(".prompts:model_eval.user").r(
+                stdout=stdout,
+                workflow_stdout=workflow_stdout,
+            )
+
        return build_cls_from_json_with_retry(
            ModelSingleFeedback,
            system_prompt=system_prompt,
@@ -9,28 +9,13 @@ class ModelTask(CoSTEERTask):
        self,
        name: str,
        description: str,
-        architecture: str = "",
        *args,
-        hyperparameters: Dict[str, str] = {},
-        model_type: Optional[str] = None,
        **kwargs,
    ) -> None:
-        self.architecture: str = architecture
-        self.hyperparameters: str = hyperparameters
-        self.model_type: str | None = (
-            model_type  # Tabular for tabular model, TimesSeries for time series model, Graph for graph model, XGBoost for XGBoost model
-            # TODO: More Models Supported
-        )
        super().__init__(name=name, description=description, *args, **kwargs)

    def get_task_information(self):
        task_desc = f"""name: {self.name}
 description: {self.description}
 """
-        if self.architecture:
-            task_desc += f"architecture: {self.architecture}\n"
-        if self.hyperparameters:
-            task_desc += f"hyperparameters: {self.hyperparameters}\n"
-        if self.model_type:
-            task_desc += f"model_type: {self.model_type}\n"
        return task_desc
@@ -138,3 +138,43 @@ model_eval:
    --------- Whole workflow test stdout ---------
    {{ workflow_stdout }}
    {% endif %}
+
+model_eval_rm:
+  system: |-
+    You are a data scientist responsible for evaluating model removal process.
+
+    ## Task Description
+    {{ task_desc }}
+
+    {% if workflow_stdout is not none %}
+    ## Whole Workflow Consideration
+    The model building code is part of the whole workflow. The user has executed the entire pipeline and provided additional stdout.
+
+    **Workflow Code:**
+    ```python
+    {{ workflow_code }}
+    ```
+
+    You should evaluate both the model removal test results and the overall workflow results. **Approve the code only if both tests pass.**
+    {% endif %}
+    
+    ## Evaluation Criteria
+    You will be given the standard output (`stdout`) from the model removal test and, if applicable, the workflow test.
+
+    Please respond with your feedback in the following JSON format and order
+    ```json
+    {
+        "execution": "Describe how well the model removal executed, including any errors or issues encountered. Append all error messages and full traceback details without summarizing or omitting any information.",
+        "return_checking": "Check the generated value, including whether the value is generated and comparing the shape of the model output with the requirement in spec.md.",
+        "code": "Assess code quality, readability, and adherence to specifications.",
+        "final_decision": <true/false>
+    }
+    ```
+
+  user: |-
+    --------- Model removal test stdout ---------
+    {{ stdout }}   
+    {% if workflow_stdout is not none %}
+    --------- Whole workflow test stdout ---------
+    {{ workflow_stdout }}
+    {% endif %}