fix score bug and refine restart strategy (#540)

rdagent/app/data_science/loop.py

@@ -118,16 +118,11 @@ class DataScienceRDLoop(RDLoop):
                     ExperimentFeedback.from_exception(e),
                 )
             )
-            if len(self.trace.hist) > DS_RD_SETTING.consecutive_errors:
-                cons_errors = 0
-                for _, feedback in reversed(self.trace.hist[-DS_RD_SETTING.consecutive_errors :]):
-                    if feedback == ExperimentFeedback.from_exception(e) or feedback.reason.startswith(
-                        "The experiment fails due to"
-                    ):
-                        cons_errors += 1
-                    else:
-                        break
-                if cons_errors == DS_RD_SETTING.consecutive_errors + 1:
+            if len(self.trace.hist) >= DS_RD_SETTING.consecutive_errors:
+                trace_exp_next_component_list = [
+                    exp.next_component_required() for exp, _ in self.trace.hist[-DS_RD_SETTING.consecutive_errors :]
+                ]
+                if None not in trace_exp_next_component_list and len(set(trace_exp_next_component_list)) == 1:
                     logger.error("Consecutive errors reached the limit. Dumping trace.")
                     logger.log_object(self.trace, tag="trace before restart")
                     self.trace = DSTrace(scen=self.trace.scen, knowledge_base=self.trace.knowledge_base)

rdagent/log/mle_summary.py

@@ -35,7 +35,7 @@ def save_grade_info(log_trace_path: Path):
             if isinstance(msg.content, DSExperiment):
                 msg.content.experiment_workspace.execute(
                     env=de,
-                    entry=f"bash -c 'mlebench grade-sample submission.csv {competition} --data-dir /mle/data > mle_score.txt 2>&1'",
+                    entry=f"mlebench grade-sample submission.csv {competition} --data-dir /mle/data > mle_score.txt 2>&1",
                 )
                 msg.content.experiment_workspace.execute(env=de, entry="chmod 777 mle_score.txt")
 
@@ -77,7 +77,7 @@ def summarize_folder(log_folder: Path):
                                 f"mle_score.txt in {grade_output_path} not found, genarate it first!"
                             )
                         grade_output = extract_mle_json(grade_output_path.read_text())
-                        if grade_output["score"] is not None:
+                        if grade_output and grade_output["score"] is not None:
                             test_scores[loop_num - 1] = grade_output["score"]
                             if grade_output["any_medal"]:
                                 medal = (