feat: dump model (#776)

* feat: add model dump flag and multi-evaluator support

* tmp code

* refactor: update evaluator feedback and FBWorkspace types

* feat: add get_clear_ws_cmd and CPU count in Docker environment

* feat: Add model dump check level and enhance evaluator functionality

fix data type bug

* fix: Ensure required files exist before model dump evaluation

* refactor: streamline prompt and file checks in model dump evaluation

* fix: add assertions and reorder file reads in model dump evaluator

* feat: remove EDA part from evaluation output

* docs: update dump_model guidelines and eval prompt to include template

* style: reformat multiline dicts and lists in conf and eval files

* fix: add DOTALL flag to EDA removal regex

rdagent/components/coder/data_science/pipeline/__init__.py

@@ -48,6 +48,7 @@ from rdagent.components.coder.data_science.raw_data_loader.eval import (
     DataLoaderCoSTEEREvaluator,
 )
 from rdagent.components.coder.data_science.raw_data_loader.exp import DataLoaderTask
+from rdagent.components.coder.data_science.share.eval import ModelDumpEvaluator
 from rdagent.core.exception import CoderError
 from rdagent.core.experiment import FBWorkspace
 from rdagent.core.scenario import Scenario
@@ -95,6 +96,7 @@ class PipelineMultiProcessEvolvingStrategy(MultiProcessEvolvingStrategy):
             out_spec=PythonAgentOut.get_spec(),
             runtime_environment=runtime_environment,
             spec=T("scenarios.data_science.share:component_spec.Pipeline").r(),
+            enable_model_dump=DS_RD_SETTING.enable_model_dump,
         )
         user_prompt = T(".prompts:pipeline_coder.user").r(
             competition_info=competition_info,
@@ -146,8 +148,12 @@ class PipelineCoSTEER(CoSTEER):
         **kwargs,
     ) -> None:
         settings = DSCoderCoSTEERSettings()
+        eval_l = [PipelineCoSTEEREvaluator(scen=scen)]
+        if DS_RD_SETTING.enable_model_dump:
+            eval_l.append(ModelDumpEvaluator(scen=scen, data_type="sample"))
+
         eva = CoSTEERMultiEvaluator(
-            PipelineCoSTEEREvaluator(scen=scen), scen=scen
+            single_evaluator=eval_l, scen=scen
         )  # Please specify whether you agree running your eva in parallel or not
         es = PipelineMultiProcessEvolvingStrategy(scen=scen, settings=settings)
 

rdagent/components/coder/data_science/pipeline/eval.py

@@ -15,7 +15,7 @@ from rdagent.components.coder.CoSTEER.evaluators import (
 from rdagent.components.coder.CoSTEER.knowledge_management import (
     CoSTEERQueriedKnowledgeV2,
 )
-from rdagent.components.coder.data_science.conf import get_ds_env
+from rdagent.components.coder.data_science.conf import get_clear_ws_cmd, get_ds_env
 from rdagent.core.experiment import FBWorkspace, Task
 from rdagent.utils.agent.tpl import T
 from rdagent.utils.agent.workflow import build_cls_from_json_with_retry
@@ -55,7 +55,7 @@ class PipelineCoSTEEREvaluator(CoSTEEREvaluator):
         env.conf.extra_volumes = {f"{DS_RD_SETTING.local_data_path}/sample/{self.scen.competition}": "/kaggle/input"}
 
         # Clean the scores.csv & submission.csv.
-        implementation.execute(env=env, entry=f"rm submission.csv scores.csv")
+        implementation.execute(env=env, entry=get_clear_ws_cmd())
         stdout, execute_ret_code = implementation.execute_ret_code(env=env, entry=f"python main.py")
         stdout = re.sub(r"=== Start of EDA part ===(.*)=== End of EDA part ===", "", stdout)
 

rdagent/components/coder/data_science/pipeline/prompts.yaml

@@ -58,6 +58,13 @@ pipeline_coder:
     - An evaluation agent will help to check whether the EDA part is added correctly.
     - During the EDA part, you should try to avoid any irrelevant information sending to the standard output.
 
+    {% if enable_model_dump %}
+    ## Model Dumping
+    {% include "components.coder.data_science.share.prompts:dump_model_coder.guideline" %}
+    {% endif %}
+
+
+
     ## Output Format
     {% if out_spec %}
     {{ out_spec }}
@@ -125,10 +132,10 @@ pipeline_eval:
         "final_decision": <true/false>
     }
     ```
-  
+
   user: |-
     --------- code generated by user ---------
     {{ code }}
 
     --------- code running stdout ---------
-    {{ stdout }}
+    {{ stdout }}