From 1737aff7c9016db3495321d2f9eb53cee6aa3fb8 Mon Sep 17 00:00:00 2001
From: you-n-g <you-n-g@users.noreply.github.com>
Date: Wed, 19 Feb 2025 01:00:25 +0800
Subject: [PATCH] fix: Runnable on first complete & Rename method to
 next_incomplete_component for clarity (#615)

---
 rdagent/app/data_science/loop.py              | 14 +++++++---
 .../data_science/experiment/experiment.py     |  7 +++++
 .../data_science/proposal/exp_gen.py          | 28 ++++++++-----------
 3 files changed, 28 insertions(+), 21 deletions(-)

diff --git a/rdagent/app/data_science/loop.py b/rdagent/app/data_science/loop.py
index 93fae288..30393960 100644
--- a/rdagent/app/data_science/loop.py
+++ b/rdagent/app/data_science/loop.py
@@ -93,18 +93,24 @@ class DataScienceRDLoop(RDLoop):
 
     def running(self, prev_out: dict[str, Any]):
         exp: DSExperiment = prev_out["coding"]
-        if self.trace.next_component_required() is None:
+        if exp.is_ready_to_run():
             new_exp = self.runner.develop(exp)
             logger.log_object(new_exp)
             return new_exp
-        else:
-            return exp
+        return exp
 
     def feedback(self, prev_out: dict[str, Any]) -> ExperimentFeedback:
+        """
+        Assumption:
+        - If we come to feedback phase, the previous development steps are successful.
+        """
         exp: DSExperiment = prev_out["running"]
-        if self.trace.next_component_required() is None:
+        if self.trace.next_incomplete_component() is None:
+            # we have alreadly completed components in previous trace. So current loop is focusing on a new proposed idea.
+            # So we need feedback for the proposal.
             feedback = self.summarizer.generate_feedback(exp, self.trace)
         else:
+            # Otherwise, it is on drafting stage, don't need complicated feedbacks.
             feedback = ExperimentFeedback(
                 reason=f"{exp.hypothesis.component} is completed.",
                 decision=True,
diff --git a/rdagent/scenarios/data_science/experiment/experiment.py b/rdagent/scenarios/data_science/experiment/experiment.py
index d071807d..7e5201e0 100644
--- a/rdagent/scenarios/data_science/experiment/experiment.py
+++ b/rdagent/scenarios/data_science/experiment/experiment.py
@@ -19,3 +19,10 @@ class DSExperiment(Experiment[Task, FBWorkspace, FBWorkspace]):
         self.experiment_workspace = FBWorkspace()
         self.pending_tasks_list = pending_tasks_list
         self.format_check_result = None
+
+    def is_ready_to_run(self) -> bool:
+        """
+        ready to run does not indicate the experiment is runnable
+        (so it is different from `trace.next_incomplete_component`.)
+        """
+        return self.experiment_workspace is not None and "main.py" in self.experiment_workspace.file_dict
diff --git a/rdagent/scenarios/data_science/proposal/exp_gen.py b/rdagent/scenarios/data_science/proposal/exp_gen.py
index 16fef349..9f8c55ab 100644
--- a/rdagent/scenarios/data_science/proposal/exp_gen.py
+++ b/rdagent/scenarios/data_science/proposal/exp_gen.py
@@ -92,7 +92,11 @@ class DSTrace(Trace[DataScienceScen, KnowledgeBase]):
 
     COMPLETE_ORDER = ("DataLoadSpec", "FeatureEng", "Model", "Ensemble", "Workflow")
 
-    def next_component_required(self) -> COMPONENT | None:
+    def next_incomplete_component(self) -> COMPONENT | None:
+        """
+        NOTE:
+        - A component will be complete until get True decision feedback !!!
+        """
         for c in self.COMPLETE_ORDER:
             if not self.has_compponent(c):
                 return c
@@ -105,27 +109,17 @@ class DSTrace(Trace[DataScienceScen, KnowledgeBase]):
                 return True
         return False
 
-    def sota_experiment(self, last_n: int = -1) -> DSExperiment | None:
+    def sota_experiment(self) -> DSExperiment | None:
         """
-        Access the last experiment result.
-
-        Parameters
-        ----------
-        last_n : int
-            The index from the last experiment result to access.
-            Use -1 for the most recent experiment, -2 for the second most recent, and so on.
-
         Returns
         -------
         Experiment or None
             The experiment result if found, otherwise None.
         """
-        assert last_n < 0
-        for exp, ef in self.hist[::-1]:
-            # the sota exp should be accepted decision and all required components are completed.
-            if ef.decision and self.next_component_required() is None:
-                last_n += 1
-                if last_n == 0:
+        if self.next_incomplete_component() is None:
+            for exp, ef in self.hist[::-1]:
+                # the sota exp should be accepted decision and all required components are completed.
+                if ef.decision:
                     return exp
         return None
 
@@ -252,7 +246,7 @@ class DSExpGen(ExpGen):
         scenario_desc = trace.scen.get_scenario_all_desc()
         last_successful_exp = trace.last_successful_exp()
 
-        next_missing_component = trace.next_component_required()
+        next_missing_component = trace.next_incomplete_component()
 
         init_component_config = {
             "DataLoadSpec": {"task_cls": DataLoaderTask, "spec_file": None, "component_prompt_key": "data_loader"},