diff --git a/rdagent/scenarios/data_science/dev/feedback.py b/rdagent/scenarios/data_science/dev/feedback.py index 119943aa..38e2d453 100644 --- a/rdagent/scenarios/data_science/dev/feedback.py +++ b/rdagent/scenarios/data_science/dev/feedback.py @@ -55,6 +55,34 @@ class DSExperiment2Feedback(Experiment2Feedback): else: diff_edition = [] + sota_exp_and_feedback = trace.experiment_and_feedback_list_after_init(return_type="sota", with_index=True) + sota_valid_score_df = pd.DataFrame( + index=[index for index, exp, fb in sota_exp_and_feedback], columns=["score", "improvement"] + ) + for row_index, (index, sota_exp, sota_fb) in enumerate(sota_exp_and_feedback): + sota_valid_score_df.loc[index, "score"] = ( + sota_exp.result.loc["ensemble"].iloc[0] if sota_exp.result is not None else "N/A" + ) + if row_index > 0: + improve_percent = ( + ( + ((sota_valid_score_df.loc[index, "score"] - sota_valid_score_df.iloc[row_index - 1]["score"])) + / sota_valid_score_df.iloc[row_index - 1]["score"] + * 100 + ) + if self.scen.evaluation_metric_direction + else ( + ((sota_valid_score_df.iloc[row_index - 1]["score"] - sota_valid_score_df.loc[index, "score"])) + / sota_valid_score_df.iloc[row_index - 1]["score"] + * 100 + ) + ) + sota_valid_score_df.loc[index, "improvement"] = f"{improve_percent:.2f}%" + sota_valid_score_df["improvement"] = sota_valid_score_df["score"].diff() + all_sota_results_desc = T("scenarios.data_science.share:describe.sota_results").r( + sota_valid_score_dict=sota_valid_score_df.to_dict(orient="index"), + ) + # assumption: # The feedback should focus on experiment **improving**. # Assume that all the the sota exp is based on the previous sota experiment @@ -78,6 +106,7 @@ class DSExperiment2Feedback(Experiment2Feedback): diff_edition=diff_edition, feedback_desc=feedback_desc, cur_vs_sota_score=cur_vs_sota_score, + all_sota_results_desc=all_sota_results_desc, ) resp_dict = json.loads( diff --git a/rdagent/scenarios/data_science/dev/prompts.yaml b/rdagent/scenarios/data_science/dev/prompts.yaml index 2b95301a..8914b73b 100644 --- a/rdagent/scenarios/data_science/dev/prompts.yaml +++ b/rdagent/scenarios/data_science/dev/prompts.yaml @@ -40,15 +40,23 @@ exp_feedback: - Directly compare the current `ensemble` validation score to the SOTA `ensemble` validation score. Do not focus on individual models unless anomalies are significant. - Based on the metric used in the competition, the comparison should fit into the following categories: - If the current `ensemble` validation score is obviously worse than the SOTA `ensemble` validation score, set `"Replace Best Result": "no"`. - - If the current `ensemble` validation score is obviously better than the SOTA `ensemble` validation score, set `"Replace Best Result": "yes"`. - - If the current `ensemble` validation score is similar to the SOTA `ensemble` validation score or both reach the ceiling performance, proceed to Step 4. + - If the current `ensemble` validation score is obviously better than the SOTA `ensemble` validation score, proceed to Step 4. + - If the current `ensemble` validation score is similar to the SOTA `ensemble` validation score or both reach the ceiling performance, proceed to Step 5. - Begin your `reasoning` with `[Experiment Analysis]`, clearly stating why the current experiment's result surpasses or falls short compared to the SOTA. - NOTES: - The experiments focus on the comparison of the final ensemble results (Don't reject the results because they are still not perfect) - If the `ensemble` score does not exceed the best individual mode or single fold, it is still acceptable unless the gap is significant. - Step 4: Analyze Code With Similar validation Results - - If the current `ensemble` validation score is similar to the SOTA `ensemble` validation score, give the decision based on the comparison between the current experiment and SOTA. + Step 4: Identify validation score improvement source + - The user also provides you all the ensemble validation scores of the SOTA experiments you can get the improve pattern of the validation score numbers. + - Since the current validation score is obviously better than the SOTA `ensemble` validation score, you should analyze whether the current number based on all the history numbers following the patterns: + - If the sota validation score has been stabilized or joggling for several rounds, you should be very careful to accept the current score as a valid improvement because it is likely that this validation score improvement is the cause of over-fit. + - If the code manipulate the validation dataset or modified the validation process. please set `"Replace Best Result": "no"` and explain in `"Reasoning"` starting with `[Validation Score Manipulation]`. + - If the improvement is based a big innovated code change, please proceed to Step 5 to check the code quality and give decision based on both the validation score and code quality. + - If no much sota validation score is provided, you should proceed to Step 5 to check the code quality and give decision based on both the validation score and code quality. + + Step 5: Analyze Code To get Better validation results + - If the current `ensemble` validation score is similar to the SOTA `ensemble` validation score or suspicious validation improvement after several trials, give the decision based on the comparison between the current experiment and SOTA. - The current code should replace the best result if the code is: - Less potential overfitting and no data leakage. The code should not modify the validation and test set distributions. - Using best practices and modeling techniques. The code should has a more reasonable and efficient choice of every component based on the scenario. diff --git a/rdagent/scenarios/data_science/loop.py b/rdagent/scenarios/data_science/loop.py index 5ff1fee1..700bd57a 100644 --- a/rdagent/scenarios/data_science/loop.py +++ b/rdagent/scenarios/data_science/loop.py @@ -324,6 +324,11 @@ class DataScienceRDLoop(RDLoop): ) # backup when upper code line is killed when running self.timer.add_duration(datetime.now() - start_archive_datetime) + def _check_exit_conditions_on_step(self, loop_id: Optional[int] = None, step_id: Optional[int] = None): + if step_id not in [self.steps.index("running"), self.steps.index("feedback")]: + # pass the check for running and feedbacks since they are very likely to be finished soon. + super()._check_exit_conditions_on_step(loop_id=loop_id, step_id=step_id) + @classmethod def load( cls, diff --git a/rdagent/scenarios/data_science/proposal/exp_gen/base.py b/rdagent/scenarios/data_science/proposal/exp_gen/base.py index 1e9001f3..4e13e609 100644 --- a/rdagent/scenarios/data_science/proposal/exp_gen/base.py +++ b/rdagent/scenarios/data_science/proposal/exp_gen/base.py @@ -171,6 +171,7 @@ class DSTrace(Trace[DataScienceScen, KnowledgeBase]): search_type: Literal["all", "ancestors"] = "ancestors", selection: tuple[int, ...] | None = None, max_retrieve_num: int | None = None, + with_index: bool = False, ) -> list[tuple[DSExperiment, ExperimentFeedback]]: """ Retrieve a list of experiments and feedbacks based on the return_type. @@ -180,13 +181,13 @@ class DSTrace(Trace[DataScienceScen, KnowledgeBase]): has_final_component = True if DS_RD_SETTING.coder_on_whole_pipeline else False SOTA_exp_and_feedback_list = [] failed_exp_and_feedback_list_after_sota = [] - for exp, fb in search_list: + for index, (exp, fb) in enumerate(search_list): if has_final_component: if fb.decision: - SOTA_exp_and_feedback_list.append((exp, fb)) + SOTA_exp_and_feedback_list.append((index, exp, fb) if with_index else (exp, fb)) failed_exp_and_feedback_list_after_sota = [] else: - failed_exp_and_feedback_list_after_sota.append((exp, fb)) + failed_exp_and_feedback_list_after_sota.append((index, exp, fb) if with_index else (exp, fb)) if exp.hypothesis.component == final_component and fb: has_final_component = True if max_retrieve_num is not None and (SOTA_exp_and_feedback_list or failed_exp_and_feedback_list_after_sota): diff --git a/rdagent/scenarios/data_science/share.yaml b/rdagent/scenarios/data_science/share.yaml index 4f860533..054bfe27 100644 --- a/rdagent/scenarios/data_science/share.yaml +++ b/rdagent/scenarios/data_science/share.yaml @@ -65,6 +65,12 @@ describe: # some template to describe some object {% endif %} {% endfor %} {% endif %} + + sota_results: |- + All validation scores of the SOTA experiments: + {% for index, data in sota_valid_score_dict.items() %} + ## Trial index: {{ index }}, validation score: {{ data.score }}, validation score improvement: {{ data.improvement }} + {% endfor %} scen: # customizable role: |- diff --git a/rdagent/utils/workflow/loop.py b/rdagent/utils/workflow/loop.py index 24a31ad8..94f2d685 100644 --- a/rdagent/utils/workflow/loop.py +++ b/rdagent/utils/workflow/loop.py @@ -164,7 +164,7 @@ class LoopBase: self._pbar.close() del self._pbar - def _check_exit_conditions_on_step(self) -> None: + def _check_exit_conditions_on_step(self, loop_id: Optional[int] = None, step_id: Optional[int] = None) -> None: """Check if the loop should continue or terminate. Raises @@ -270,7 +270,7 @@ class LoopBase: step_index=next_step, step_name=self.steps[next_step], ) - self._check_exit_conditions_on_step() + self._check_exit_conditions_on_step(loop_id=li, step_id=si) else: logger.warning(f"Step forward {si} of loop {li} is skipped.")