diff --git a/rdagent/app/data_science/loop.py b/rdagent/app/data_science/loop.py index a2b5566d..8f1478d7 100644 --- a/rdagent/app/data_science/loop.py +++ b/rdagent/app/data_science/loop.py @@ -118,16 +118,11 @@ class DataScienceRDLoop(RDLoop): ExperimentFeedback.from_exception(e), ) ) - if len(self.trace.hist) > DS_RD_SETTING.consecutive_errors: - cons_errors = 0 - for _, feedback in reversed(self.trace.hist[-DS_RD_SETTING.consecutive_errors :]): - if feedback == ExperimentFeedback.from_exception(e) or feedback.reason.startswith( - "The experiment fails due to" - ): - cons_errors += 1 - else: - break - if cons_errors == DS_RD_SETTING.consecutive_errors + 1: + if len(self.trace.hist) >= DS_RD_SETTING.consecutive_errors: + trace_exp_next_component_list = [ + exp.next_component_required() for exp, _ in self.trace.hist[-DS_RD_SETTING.consecutive_errors :] + ] + if None not in trace_exp_next_component_list and len(set(trace_exp_next_component_list)) == 1: logger.error("Consecutive errors reached the limit. Dumping trace.") logger.log_object(self.trace, tag="trace before restart") self.trace = DSTrace(scen=self.trace.scen, knowledge_base=self.trace.knowledge_base) diff --git a/rdagent/log/mle_summary.py b/rdagent/log/mle_summary.py index 43954185..558fadd7 100644 --- a/rdagent/log/mle_summary.py +++ b/rdagent/log/mle_summary.py @@ -35,7 +35,7 @@ def save_grade_info(log_trace_path: Path): if isinstance(msg.content, DSExperiment): msg.content.experiment_workspace.execute( env=de, - entry=f"bash -c 'mlebench grade-sample submission.csv {competition} --data-dir /mle/data > mle_score.txt 2>&1'", + entry=f"mlebench grade-sample submission.csv {competition} --data-dir /mle/data > mle_score.txt 2>&1", ) msg.content.experiment_workspace.execute(env=de, entry="chmod 777 mle_score.txt") @@ -77,7 +77,7 @@ def summarize_folder(log_folder: Path): f"mle_score.txt in {grade_output_path} not found, genarate it first!" ) grade_output = extract_mle_json(grade_output_path.read_text()) - if grade_output["score"] is not None: + if grade_output and grade_output["score"] is not None: test_scores[loop_num - 1] = grade_output["score"] if grade_output["any_medal"]: medal = (