several bug fix (#621)

rdagent/components/coder/data_science/feature/eval_tests/feature_test.txt

@@ -63,13 +63,15 @@ if isinstance(X, pd.DataFrame) and isinstance(X_test, pd.DataFrame):
 if isinstance(X, pd.DataFrame):
     X_dtypes_unique_sorted = sorted([str(dt) for dt in X.dtypes.unique()])
     X_loaded_dtypes_unique_sorted = sorted([str(dt) for dt in X_loaded.dtypes.unique()])
+
+    X_dtypes_unique_sorted_new = [
+        dt for dt in X_dtypes_unique_sorted if dt not in X_loaded_dtypes_unique_sorted and dt != "object"
+    ]
     assert (
-        len(X_loaded_dtypes_unique_sorted) == 1
-        and X_loaded_dtypes_unique_sorted[0] in {np.float64, np.float32}
-    ) or (
-        X_dtypes_unique_sorted == X_loaded_dtypes_unique_sorted
+        np.dtypes.ObjectDType in X_loaded_dtypes_unique_sorted or len(X_dtypes_unique_sorted_new) == 0
     ), f"feature engineering has produced new data types which is not allowed, data loader data types are {X_loaded_dtypes_unique_sorted} and feature engineering data types are {X_dtypes_unique_sorted}"
 
+
 print(
     "Feature Engineering test passed successfully. All checks including length, width, and data types have been validated."
 )

rdagent/components/coder/data_science/model/prompts.yaml

@@ -37,6 +37,7 @@ model_coder:
     {{ feature_code }}
     2. You should avoid using logging module to output information in your generated code, and instead use the print() function.
     3. You can decide whether to use AutoML based on the characteristics of the task.
+    4. If the model can both be implemented by PyTorch and Tensorflow, please use pytorch for broader compatibility.
 
     ## Output Format
     {% if out_spec %}

rdagent/scenarios/data_science/dev/runner/eval.py

@@ -71,25 +71,6 @@ class DSCoSTEERCoSTEEREvaluator(CoSTEEREvaluator):
             stdout += f"\n MLEBench submission check:"
             stdout += implementation.execute(env=mde, entry="python test/mle_submission_format_test.py")
 
-        # remove unused files
-        implementation.execute(env=de, entry="coverage json -o coverage.json")
-        if Path(implementation.workspace_path / "coverage.json").exists():
-            with open(implementation.workspace_path / "coverage.json") as f:
-                used_files = set(json.load(f)["files"].keys())
-                logger.info("All used scripts: {}".format(used_files))
-                all_python_files = set(Path(implementation.workspace_path).rglob("*.py"))
-                unused_files = [
-                    py_file
-                    for py_file in all_python_files
-                    if not (py_file.name in used_files or py_file.name.endswith("test.py"))
-                ]
-                if unused_files:
-                    logger.warning(f"Unused scripts: {unused_files}")
-                    implementation.inject_files(
-                        **{file_path.name: implementation.DEL_KEY for file_path in unused_files}
-                    )
-            os.remove(implementation.workspace_path / "coverage.json")
-
         system_prompt = T(".prompts:DSCoSTEER_eval.system").r(
             scenario=self.scen.get_scenario_all_desc(),
             task_desc=target_task.get_task_information(),
@@ -99,6 +80,28 @@ class DSCoSTEERCoSTEEREvaluator(CoSTEEREvaluator):
             stdout=shrink_text(stdout),
         )
 
-        return build_cls_from_json_with_retry(
+        feedback = build_cls_from_json_with_retry(
             DSCoSTEEREvalFeedback, system_prompt=system_prompt, user_prompt=user_prompt
         )
+
+        if feedback:
+            # remove unused files
+            implementation.execute(env=de, entry="coverage json -o coverage.json")
+            if Path(implementation.workspace_path / "coverage.json").exists():
+                with open(implementation.workspace_path / "coverage.json") as f:
+                    used_files = set(json.load(f)["files"].keys())
+                    logger.info("All used scripts: {}".format(used_files))
+                    all_python_files = set(Path(implementation.workspace_path).rglob("*.py"))
+                    unused_files = [
+                        py_file
+                        for py_file in all_python_files
+                        if not (py_file.name in used_files or py_file.name.endswith("test.py"))
+                    ]
+                    if unused_files:
+                        logger.warning(f"Unused scripts: {unused_files}")
+                        implementation.inject_files(
+                            **{file_path.name: implementation.DEL_KEY for file_path in unused_files}
+                        )
+                os.remove(implementation.workspace_path / "coverage.json")
+
+        return feedback

rdagent/scenarios/data_science/proposal/exp_gen.py

@@ -98,17 +98,30 @@ class DSTrace(Trace[DataScienceScen, KnowledgeBase]):
         - A component will be complete until get True decision feedback !!!
         """
         for c in self.COMPLETE_ORDER:
-            if not self.has_compponent(c):
+            if not self.has_component(c):
                 return c
         return None
 
-    def has_compponent(self, component: COMPONENT) -> bool:
+    def has_component(self, component: COMPONENT) -> bool:
         for exp, fb in self.hist:
             assert isinstance(exp.hypothesis, DSHypothesis), "Hypothesis should be DSHypothesis (and not None)"
             if exp.hypothesis.component == component and fb:
                 return True
         return False
 
+    def experiment_and_feedback_list_after_init(
+        self, return_success=True
+    ) -> list[tuple[DSExperiment, ExperimentFeedback]]:
+        final_component = self.COMPLETE_ORDER[-1]
+        has_final_component = False
+        exp_and_feedback_list = []
+        for exp, fb in self.hist:
+            if has_final_component and bool(fb) == return_success:
+                exp_and_feedback_list.append((exp, fb))
+            if exp.hypothesis.component == final_component and fb:
+                has_final_component = True
+        return exp_and_feedback_list
+
     def sota_experiment(self) -> DSExperiment | None:
         """
         Returns
@@ -293,8 +306,19 @@ class DSExpGen(ExpGen):
                     sota_exp.experiment_workspace.file_dict, last_exp.experiment_workspace.file_dict
                 )
             )  # we use file_dict for hitting the cache when replicate the experiment in another machine.
-            exp_and_feedback_desc = T("scenarios.data_science.share:describe.feedback").r(
-                exp_and_feedback=exp_and_feedback
+
+            sota_exp_feedback_list = trace.experiment_and_feedback_list_after_init(return_success=True)
+            failed_exp_feedback_list = trace.experiment_and_feedback_list_after_init(return_success=False)[
+                -self.max_trace_hist :
+            ]
+
+            sota_exp_feedback_list_desc = T("scenarios.data_science.share:describe.trace").r(
+                exp_and_feedback_list=sota_exp_feedback_list,
+                success=True,
+            )
+            failed_exp_feedback_list_desc = T("scenarios.data_science.share:describe.trace").r(
+                exp_and_feedback_list=failed_exp_feedback_list,
+                success=False,
             )
 
             # Generate component using template with proper context
@@ -306,7 +330,8 @@ class DSExpGen(ExpGen):
             )
 
             component_user_prompt = T(".prompts:component_gen.user").r(
-                exp_and_feedback_desc=exp_and_feedback_desc,
+                sota_exp_and_feedback_list_desc=sota_exp_feedback_list_desc,
+                failed_exp_and_feedback_list_desc=failed_exp_feedback_list_desc,
             )
 
             resp_dict_component: dict = json.loads(
@@ -316,6 +341,15 @@ class DSExpGen(ExpGen):
             )
 
             component = resp_dict_component.get("component", "Component not provided")
+            sota_exp_model_file_count = len(
+                [
+                    k
+                    for k in sota_exp.experiment_workspace.file_dict.keys()
+                    if k.endswith(".py") and "test" not in k and k.startswith("model")
+                ]
+            )
+            if sota_exp_model_file_count <= 1 and component == "Ensemble":
+                component = "Model"
 
             # Why we should split component selection and steps after?
             # - after we know the selected component, we can use RAG.
@@ -336,21 +370,10 @@ class DSExpGen(ExpGen):
                     workflow_check=(not component == "Workflow"),
                 )
 
-                recent_trace_desc = []
-                for i in range(self.max_trace_hist):
-                    if i < len(trace.hist):
-                        eaf = trace.hist[-i - 1]
-                        if eaf[1].decision:
-                            # we only add failed direction incase of trying same invalid direction
-                            break
-                        recent_trace_desc.insert(
-                            0, T("scenarios.data_science.share:describe.feedback").r(exp_and_feedback=eaf)
-                        )
                 user_prompt = T(".prompts:direct_exp_gen.user").r(
-                    exp_and_feedback_desc=exp_and_feedback_desc,
-                    sota_exp_desc=sota_exp_desc,
+                    sota_exp_and_feedback_list_desc=sota_exp_feedback_list_desc,
+                    failed_exp_and_feedback_list_desc=failed_exp_feedback_list_desc,
                     last_exp_diff=last_exp_diff,
-                    recent_trace_desc="\n".join(recent_trace_desc),
                 )
 
                 def _append_retry(args: tuple, kwargs: dict) -> tuple[tuple, dict]:

rdagent/scenarios/data_science/proposal/prompts.yaml

@@ -213,24 +213,20 @@ direct_exp_gen:
     {% endif %}
 
   user: |-
-    # The detailed description of current best experiments
-    {{ sota_exp_desc }}
+    # All former successful experiments and their feedbacks, the current SOTA solution is the combination of the best solutions of these trials:
+    {{ sota_exp_and_feedback_list_desc }}
 
-    ## The according feedbacks for the best experiments
-    {{ exp_and_feedback_desc }}
-
-    {% if recent_trace_desc %}
+    {% if failed_exp_and_feedback_list_desc %}
     # Several trials after the best experiments
     The user has made several hypothesis on this scenario and did several evaluation on them.
     The former hypothesis and the corresponding feedbacks are as follows (focus on the last one & the new hypothesis that it provides and reasoning to see if you agree):
-    {{ recent_trace_desc }}
+    {{ failed_exp_and_feedback_list_desc }}
 
     {% if last_exp_diff %}
     # Here are the differences between the latest version of implementation and the current best version of implementation
     It is presented in diff format, highlighting changes from the best version to the latest version.
     {{ last_exp_diff }}
     {% endif %}
-
     {% endif %}
 
 
@@ -275,7 +271,11 @@ component_gen:
     {{ component_output_format }}
 
   user: |-
-    {{ exp_and_feedback_desc }}
+    Here's the former SOTA experiments and their feedbacks:
+    {{ sota_exp_and_feedback_list_desc }}
+
+    Also, here's the former failed experiments and their feedbacks:
+    {{ failed_exp_and_feedback_list_desc }}
 
 
 exp_and_feedback: |-

rdagent/scenarios/data_science/share.yaml

@@ -40,3 +40,23 @@ describe: # some template to describe some object
     feedback decision: {{ exp_and_feedback[1].decision }}
     reason: {{ exp_and_feedback[1].reason }}
     {% endif %}
+
+  trace: |-
+    {% if exp_and_feedback_list|length == 0 %}
+    No previous {% if success %}successful{% else %}failed{% endif %} trial available.
+    {% else %}
+    {% if success %}
+    ## {{ heading | default('Trace of the successful trial') }}
+    {% else %}
+    ## {{ heading | default('Trace of the failed trial') }}
+    {% endif %}
+    Before current trial, several {% if success %}successful{% else %}failed{% endif %} trials are listed below. {% if success %}The current SOTA method is the combination of the best solutions of these trials.{% endif %} The trace order is from the earliest to the latest please focus more on the later trials.
+    {% for exp_and_feedback in exp_and_feedback_list %}
+    ### Experiment index: {{ loop.index }}
+    The experiment is designed based on hypothesis: {{ exp_and_feedback[0].hypothesis }}
+    ### Task of experiment
+    {{ exp_and_feedback[0].pending_tasks_list[0][0].get_task_information() }}
+    Experiment feedback decision: {{ exp_and_feedback[1].decision }}
+    Reason: {{ exp_and_feedback[1].reason }}
+    {% endfor %}
+    {% endif %}