add rule-based eval to speed up the whole process (#768)

rdagent/app/data_science/conf.py

@@ -32,5 +32,7 @@ class DataScienceBasePropSetting(KaggleBasePropSetting):
     coder_max_loop: int = 10
     runner_max_loop: int = 3
 
+    rule_base_eval: bool = False
+
 
 DS_RD_SETTING = DataScienceBasePropSetting()

rdagent/components/coder/data_science/pipeline/eval.py

@@ -56,7 +56,7 @@ class PipelineCoSTEEREvaluator(CoSTEEREvaluator):
 
         # Clean the scores.csv & submission.csv.
         implementation.execute(env=env, entry=f"rm submission.csv scores.csv")
-        stdout = implementation.execute(env=env, entry=f"python main.py")
+        stdout, execute_ret_code = implementation.execute_ret_code(env=env, entry=f"python main.py")
         stdout = re.sub(r"=== Start of EDA part ===(.*)=== End of EDA part ===", "", stdout)
 
         score_fp = implementation.workspace_path / "scores.csv"
@@ -102,6 +102,21 @@ class PipelineCoSTEEREvaluator(CoSTEEREvaluator):
         submission_check_out, submission_ret_code = implementation.execute_ret_code(
             env=env, entry="python test/submission_format_test.py"
         )
+        if DS_RD_SETTING.rule_base_eval:
+            if execute_ret_code == 0 and score_ret_code == 0 and submission_ret_code == 0:
+                return PipelineSingleFeedback(
+                    execution=stdout,
+                    return_checking=score_check_text + "\n" + submission_check_out,
+                    code="Code evaluation is not available.",
+                    final_decision=True,
+                )
+            else:
+                return PipelineSingleFeedback(
+                    execution=stdout,
+                    return_checking=score_check_text + "\n" + submission_check_out,
+                    code="Code evaluation is not available.",
+                    final_decision=False,
+                )
         stdout += "\n" + submission_check_out
 
         system_prompt = T(".prompts:pipeline_eval.system").r(

rdagent/components/coder/data_science/pipeline/eval_tests/submission_format_test.txt

@@ -1,12 +1,15 @@
-from pathlib import Path
-import pandas as pd
 import hashlib
+from pathlib import Path
+
+import pandas as pd
+
 
 def calculate_md5(file_path):
     with open(file_path, "rb") as f:
         file_hash = hashlib.md5(f.read()).hexdigest()
     return file_hash
 
+
 file_md5 = calculate_md5("scores.csv")
 
 """
@@ -24,8 +27,9 @@ find . | grep -i sample | grep -i submission | grep -v sample_submission.csv | g
 # Find sample submission file dynamically
 input_dir = Path("/kaggle/input")
 # Look for common variations of sample submission filenames
-sample_submission_files = list(input_dir.glob("*sample_submission*.csv")) + \
-                         list(input_dir.glob("*sampleSubmission*.csv"))
+sample_submission_files = list(input_dir.glob("*sample_submission*.csv")) + list(
+    input_dir.glob("*sampleSubmission*.csv")
+)
 
 assert sample_submission_files, "Error: No sample submission file found in /kaggle/input/"
 
@@ -38,10 +42,10 @@ print(f"Using sample submission file: {sample_submission_name}")
 assert Path(SAMPLE_SUBMISSION_PATH).exists(), f"Error: {sample_submission_name} not found at {SAMPLE_SUBMISSION_PATH}"
 
 # Check if our submission file exists
-assert Path('submission.csv').exists(), "Error: submission.csv not found"
+assert Path("submission.csv").exists(), "Error: submission.csv not found"
 
 sample_submission = pd.read_csv(SAMPLE_SUBMISSION_PATH)
-our_submission = pd.read_csv('submission.csv')
+our_submission = pd.read_csv("submission.csv")
 
 success = True
 # Print the columns of the sample submission file
@@ -51,17 +55,19 @@ print("Columns in our_submission.csv:", our_submission.columns)
 for col in sample_submission.columns:
     if col not in our_submission.columns:
         success = False
-        print(f'Column {col} not found in submission.csv')
+        print(f"Column {col} not found in submission.csv")
 
 if success:
-    print(f'submission.csv\'s columns aligns with {sample_submission_name} .')
+    print(f"submission.csv's columns aligns with {sample_submission_name} .")
+else:
+    raise AssertionError(f"submission.csv's columns does not align with {sample_submission_name} .")
 
 
 # Print the first 5 rows of the two submission files, with columns separated by commas.
 def print_first_rows(file_path, file_name, num_rows=5):
     print(f"\nFirst {num_rows} rows of {file_name}:")
     try:
-        with open(file_path, 'r') as file:
+        with open(file_path, "r") as file:
             for i, line in enumerate(file):
                 if i < num_rows:
                     print(line.strip())
@@ -70,8 +76,11 @@ def print_first_rows(file_path, file_name, num_rows=5):
     except FileNotFoundError:
         print(f"Error: {file_name} not found.")
 
+
 print_first_rows(SAMPLE_SUBMISSION_PATH, sample_submission_name)
-print_first_rows('submission.csv', 'submission.csv')
+print_first_rows("submission.csv", "submission.csv")
 
 assert calculate_md5("scores.csv") == file_md5, "scores.csv should not be rewritten"
-print(f"\nPlease Checked the content of the submission file(submission.csv should has the same format with {sample_submission_name} but might not the same index with {sample_submission_name}). ")
+print(
+    f"\nPlease Checked the content of the submission file(submission.csv should has the same format with {sample_submission_name} but might not the same index with {sample_submission_name}). "
+)

rdagent/scenarios/data_science/dev/feedback.py

@@ -3,6 +3,7 @@ from typing import Dict
 
 import pandas as pd
 
+from rdagent.app.data_science.conf import DS_RD_SETTING
 from rdagent.core.proposal import (
     Experiment2Feedback,
     ExperimentFeedback,
@@ -58,6 +59,32 @@ class DSExperiment2Feedback(Experiment2Feedback):
                 f"The current score is {cur_score}, while the SOTA score is {sota_score}. "
                 f"{'In this competition, higher is better.' if self.scen.metric_direction else 'In this competition, lower is better.'}"
             )
+        if DS_RD_SETTING.rule_base_eval:
+            if sota_exp:
+                if cur_score > sota_score:
+                    return ExperimentFeedback(
+                        observations="The current score bigger than the SOTA score.",
+                        hypothesis_evaluation="The current score is bigger than the SOTA score.",
+                        new_hypothesis="No new hypothesis provided",
+                        reason="The current score is bigger than the SOTA score.",
+                        decision=True if self.scen.metric_direction else False,
+                    )
+                elif cur_score < sota_score:
+                    return ExperimentFeedback(
+                        observations="The current score smaller than the SOTA score.",
+                        hypothesis_evaluation="The current score is smaller than the SOTA score.",
+                        new_hypothesis="No new hypothesis provided",
+                        reason="The current score is smaller than the SOTA score.",
+                        decision=False if self.scen.metric_direction else True,
+                    )
+                else:
+                    return ExperimentFeedback(
+                        observations="The current score equals to the SOTA score.",
+                        hypothesis_evaluation="The current score equals to the SOTA score.",
+                        new_hypothesis="No new hypothesis provided",
+                        reason="The current score equals to the SOTA score.",
+                        decision=False,
+                    )
 
         system_prompt = T(".prompts:exp_feedback.system").r(scenario=self.scen.get_scenario_all_desc())
         user_prompt = T(".prompts:exp_feedback.user").r(

rdagent/scenarios/data_science/dev/prompts.yaml

@@ -14,8 +14,8 @@ exp_feedback:
     Step 2: Analyze Experiment Results (If Submission is Correct)
     Your feedback should:
     1. Confirm if the current result supports or refutes the hypothesis.
-    2. Compare with previous best results.
-    3. SOTA results are the best outcomes we have achieved in this scenario.
+    2. Compare the validation score named "ensemble" with previous "ensemble" best results. You don't need to compare individual model scores.
+    3. SOTA results are the best outcomes we have achieved in this scenario. 
 
     Step 3: Review of Evaluation Alignment with Competition Requirements (listed in scenario)
     As a reviewer, you should verify whether the current experiment strictly adheres to the evaluation requirements specified in the competition scenario description:

rdagent/scenarios/data_science/dev/runner/eval.py

@@ -42,7 +42,7 @@ class DSCoSTEERCoSTEEREvaluator(CoSTEEREvaluator):
         )  # Remove previous submission and scores files generated by worklfow.
 
         # execute workflow
-        stdout = implementation.execute(env=env, entry="python -m coverage run main.py")
+        stdout, execute_ret_code = implementation.execute_ret_code(env=env, entry="python -m coverage run main.py")
         match = re.search(r"(.*?)=== Start of EDA part ===(.*)=== End of EDA part ===", stdout, re.DOTALL)
         eda_output = match.groups()[1] if match else None
         self.scen.eda_output = eda_output
@@ -106,6 +106,21 @@ class DSCoSTEERCoSTEEREvaluator(CoSTEEREvaluator):
         submission_check_out, submission_ret_code = implementation.execute_ret_code(
             env=mde, entry="python test/mle_submission_format_test.py"
         )
+        if DS_RD_SETTING.rule_base_eval:
+            if execute_ret_code == 0 and score_ret_code == 0 and submission_ret_code == 0:
+                return DSCoSTEEREvalFeedback(
+                    execution=stdout,
+                    return_checking=score_check_text + "\n" + submission_check_out,
+                    code="Code evaluation is not available.",
+                    final_decision=True,
+                )
+            else:
+                return DSCoSTEEREvalFeedback(
+                    execution=stdout,
+                    return_checking=score_check_text + "\n" + submission_check_out,
+                    code="Code evaluation is not available.",
+                    final_decision=False,
+                )
         stdout += f"\nMLEBench submission check:\n{submission_check_out}\nIf MLEBench submission check returns a 'Submission is valid' or similar message, despite some warning messages, you should still consider the submission as valid and give a positive final decision. "
 
         system_prompt = T(".prompts:DSCoSTEER_eval.system").r(

rdagent/scenarios/data_science/dev/runner/eval_tests/mle_submission_format_test.txt

@@ -4,7 +4,7 @@ from mlebench.grade import validate_submission
 from mlebench.registry import registry
 
 # Check if our submission file exists
-assert Path('submission.csv').exists(), "Error: submission.csv not found"
+assert Path("submission.csv").exists(), "Error: submission.csv not found"
 
 COMPETITION_ID = "<competition_id>"
 new_registry = registry.set_data_dir(Path("/mle/data"))
@@ -12,4 +12,7 @@ competition = new_registry.get_competition(COMPETITION_ID)
 
 is_valid, message = validate_submission(Path("submission.csv"), competition)
 
-print(message)
+print(message)
+
+if not is_valid:
+    raise AssertionError("Submission is invalid")

rdagent/scenarios/data_science/share.yaml

@@ -301,9 +301,9 @@ component_spec:
   
     7. Metric Calculation and Storage:
       - Calculate the metric (mentioned in the evaluation section of the competition information) for each model and ensemble strategy on valid, and save the results in `scores.csv`
-      - The evaluation should be based on 5-fold cross-validation but only if that's an appropriate evaluation for the task at hand.
+      - The evaluation should be based on 5-fold cross-validation but only if that's an appropriate evaluation for the task at hand. Store the mean validation score of 5-fold cross-validation in `scores.csv` on each model.
       - Even if only one model is present, compute the ensemble score and store it under `"ensemble"`.
-      - The index of `scores.csv` should include the model name and the "ensemble" strategy. "ensemble" should be exactly in the index with all lower case letters.
+      - The index of `scores.csv` should include the model name and the "ensemble" strategy. "ensemble" should be exactly in the index with all lower case letters. Ensemble is the result from several models. If only one model is present, the ensemble score should be the same as the model score.
       - The column names in `scores.csv` should be:
         - Model: The name of the model or ensemble strategy.
         - <metric_name>: The calculated metric value for that model or ensemble strategy. The metric name can be found in the scenario description. The metric name should be exactly the same as the one in the scenario description since user will use it to check the result.