From 92e7932126275f994cfb6ca90e660c1bbb465001 Mon Sep 17 00:00:00 2001 From: Xu Yang Date: Tue, 8 Apr 2025 03:27:21 -0600 Subject: [PATCH] add rule-based eval to speed up the whole process (#768) --- rdagent/app/data_science/conf.py | 2 ++ .../coder/data_science/pipeline/eval.py | 17 +++++++++- .../eval_tests/submission_format_test.txt | 31 ++++++++++++------- .../scenarios/data_science/dev/feedback.py | 27 ++++++++++++++++ .../scenarios/data_science/dev/prompts.yaml | 4 +-- .../scenarios/data_science/dev/runner/eval.py | 17 +++++++++- .../eval_tests/mle_submission_format_test.txt | 7 +++-- rdagent/scenarios/data_science/share.yaml | 4 +-- 8 files changed, 90 insertions(+), 19 deletions(-) diff --git a/rdagent/app/data_science/conf.py b/rdagent/app/data_science/conf.py index b728ab4e..6ebf4ff8 100644 --- a/rdagent/app/data_science/conf.py +++ b/rdagent/app/data_science/conf.py @@ -32,5 +32,7 @@ class DataScienceBasePropSetting(KaggleBasePropSetting): coder_max_loop: int = 10 runner_max_loop: int = 3 + rule_base_eval: bool = False + DS_RD_SETTING = DataScienceBasePropSetting() diff --git a/rdagent/components/coder/data_science/pipeline/eval.py b/rdagent/components/coder/data_science/pipeline/eval.py index fa28efd2..0f4d6cff 100644 --- a/rdagent/components/coder/data_science/pipeline/eval.py +++ b/rdagent/components/coder/data_science/pipeline/eval.py @@ -56,7 +56,7 @@ class PipelineCoSTEEREvaluator(CoSTEEREvaluator): # Clean the scores.csv & submission.csv. implementation.execute(env=env, entry=f"rm submission.csv scores.csv") - stdout = implementation.execute(env=env, entry=f"python main.py") + stdout, execute_ret_code = implementation.execute_ret_code(env=env, entry=f"python main.py") stdout = re.sub(r"=== Start of EDA part ===(.*)=== End of EDA part ===", "", stdout) score_fp = implementation.workspace_path / "scores.csv" @@ -102,6 +102,21 @@ class PipelineCoSTEEREvaluator(CoSTEEREvaluator): submission_check_out, submission_ret_code = implementation.execute_ret_code( env=env, entry="python test/submission_format_test.py" ) + if DS_RD_SETTING.rule_base_eval: + if execute_ret_code == 0 and score_ret_code == 0 and submission_ret_code == 0: + return PipelineSingleFeedback( + execution=stdout, + return_checking=score_check_text + "\n" + submission_check_out, + code="Code evaluation is not available.", + final_decision=True, + ) + else: + return PipelineSingleFeedback( + execution=stdout, + return_checking=score_check_text + "\n" + submission_check_out, + code="Code evaluation is not available.", + final_decision=False, + ) stdout += "\n" + submission_check_out system_prompt = T(".prompts:pipeline_eval.system").r( diff --git a/rdagent/components/coder/data_science/pipeline/eval_tests/submission_format_test.txt b/rdagent/components/coder/data_science/pipeline/eval_tests/submission_format_test.txt index cf1299d1..c7190b53 100644 --- a/rdagent/components/coder/data_science/pipeline/eval_tests/submission_format_test.txt +++ b/rdagent/components/coder/data_science/pipeline/eval_tests/submission_format_test.txt @@ -1,12 +1,15 @@ -from pathlib import Path -import pandas as pd import hashlib +from pathlib import Path + +import pandas as pd + def calculate_md5(file_path): with open(file_path, "rb") as f: file_hash = hashlib.md5(f.read()).hexdigest() return file_hash + file_md5 = calculate_md5("scores.csv") """ @@ -24,8 +27,9 @@ find . | grep -i sample | grep -i submission | grep -v sample_submission.csv | g # Find sample submission file dynamically input_dir = Path("/kaggle/input") # Look for common variations of sample submission filenames -sample_submission_files = list(input_dir.glob("*sample_submission*.csv")) + \ - list(input_dir.glob("*sampleSubmission*.csv")) +sample_submission_files = list(input_dir.glob("*sample_submission*.csv")) + list( + input_dir.glob("*sampleSubmission*.csv") +) assert sample_submission_files, "Error: No sample submission file found in /kaggle/input/" @@ -38,10 +42,10 @@ print(f"Using sample submission file: {sample_submission_name}") assert Path(SAMPLE_SUBMISSION_PATH).exists(), f"Error: {sample_submission_name} not found at {SAMPLE_SUBMISSION_PATH}" # Check if our submission file exists -assert Path('submission.csv').exists(), "Error: submission.csv not found" +assert Path("submission.csv").exists(), "Error: submission.csv not found" sample_submission = pd.read_csv(SAMPLE_SUBMISSION_PATH) -our_submission = pd.read_csv('submission.csv') +our_submission = pd.read_csv("submission.csv") success = True # Print the columns of the sample submission file @@ -51,17 +55,19 @@ print("Columns in our_submission.csv:", our_submission.columns) for col in sample_submission.columns: if col not in our_submission.columns: success = False - print(f'Column {col} not found in submission.csv') + print(f"Column {col} not found in submission.csv") if success: - print(f'submission.csv\'s columns aligns with {sample_submission_name} .') + print(f"submission.csv's columns aligns with {sample_submission_name} .") +else: + raise AssertionError(f"submission.csv's columns does not align with {sample_submission_name} .") # Print the first 5 rows of the two submission files, with columns separated by commas. def print_first_rows(file_path, file_name, num_rows=5): print(f"\nFirst {num_rows} rows of {file_name}:") try: - with open(file_path, 'r') as file: + with open(file_path, "r") as file: for i, line in enumerate(file): if i < num_rows: print(line.strip()) @@ -70,8 +76,11 @@ def print_first_rows(file_path, file_name, num_rows=5): except FileNotFoundError: print(f"Error: {file_name} not found.") + print_first_rows(SAMPLE_SUBMISSION_PATH, sample_submission_name) -print_first_rows('submission.csv', 'submission.csv') +print_first_rows("submission.csv", "submission.csv") assert calculate_md5("scores.csv") == file_md5, "scores.csv should not be rewritten" -print(f"\nPlease Checked the content of the submission file(submission.csv should has the same format with {sample_submission_name} but might not the same index with {sample_submission_name}). ") +print( + f"\nPlease Checked the content of the submission file(submission.csv should has the same format with {sample_submission_name} but might not the same index with {sample_submission_name}). " +) diff --git a/rdagent/scenarios/data_science/dev/feedback.py b/rdagent/scenarios/data_science/dev/feedback.py index 4462288c..238d2e23 100644 --- a/rdagent/scenarios/data_science/dev/feedback.py +++ b/rdagent/scenarios/data_science/dev/feedback.py @@ -3,6 +3,7 @@ from typing import Dict import pandas as pd +from rdagent.app.data_science.conf import DS_RD_SETTING from rdagent.core.proposal import ( Experiment2Feedback, ExperimentFeedback, @@ -58,6 +59,32 @@ class DSExperiment2Feedback(Experiment2Feedback): f"The current score is {cur_score}, while the SOTA score is {sota_score}. " f"{'In this competition, higher is better.' if self.scen.metric_direction else 'In this competition, lower is better.'}" ) + if DS_RD_SETTING.rule_base_eval: + if sota_exp: + if cur_score > sota_score: + return ExperimentFeedback( + observations="The current score bigger than the SOTA score.", + hypothesis_evaluation="The current score is bigger than the SOTA score.", + new_hypothesis="No new hypothesis provided", + reason="The current score is bigger than the SOTA score.", + decision=True if self.scen.metric_direction else False, + ) + elif cur_score < sota_score: + return ExperimentFeedback( + observations="The current score smaller than the SOTA score.", + hypothesis_evaluation="The current score is smaller than the SOTA score.", + new_hypothesis="No new hypothesis provided", + reason="The current score is smaller than the SOTA score.", + decision=False if self.scen.metric_direction else True, + ) + else: + return ExperimentFeedback( + observations="The current score equals to the SOTA score.", + hypothesis_evaluation="The current score equals to the SOTA score.", + new_hypothesis="No new hypothesis provided", + reason="The current score equals to the SOTA score.", + decision=False, + ) system_prompt = T(".prompts:exp_feedback.system").r(scenario=self.scen.get_scenario_all_desc()) user_prompt = T(".prompts:exp_feedback.user").r( diff --git a/rdagent/scenarios/data_science/dev/prompts.yaml b/rdagent/scenarios/data_science/dev/prompts.yaml index dc573a76..c695df1a 100644 --- a/rdagent/scenarios/data_science/dev/prompts.yaml +++ b/rdagent/scenarios/data_science/dev/prompts.yaml @@ -14,8 +14,8 @@ exp_feedback: Step 2: Analyze Experiment Results (If Submission is Correct) Your feedback should: 1. Confirm if the current result supports or refutes the hypothesis. - 2. Compare with previous best results. - 3. SOTA results are the best outcomes we have achieved in this scenario. + 2. Compare the validation score named "ensemble" with previous "ensemble" best results. You don't need to compare individual model scores. + 3. SOTA results are the best outcomes we have achieved in this scenario. Step 3: Review of Evaluation Alignment with Competition Requirements (listed in scenario) As a reviewer, you should verify whether the current experiment strictly adheres to the evaluation requirements specified in the competition scenario description: diff --git a/rdagent/scenarios/data_science/dev/runner/eval.py b/rdagent/scenarios/data_science/dev/runner/eval.py index c6503701..308ed384 100644 --- a/rdagent/scenarios/data_science/dev/runner/eval.py +++ b/rdagent/scenarios/data_science/dev/runner/eval.py @@ -42,7 +42,7 @@ class DSCoSTEERCoSTEEREvaluator(CoSTEEREvaluator): ) # Remove previous submission and scores files generated by worklfow. # execute workflow - stdout = implementation.execute(env=env, entry="python -m coverage run main.py") + stdout, execute_ret_code = implementation.execute_ret_code(env=env, entry="python -m coverage run main.py") match = re.search(r"(.*?)=== Start of EDA part ===(.*)=== End of EDA part ===", stdout, re.DOTALL) eda_output = match.groups()[1] if match else None self.scen.eda_output = eda_output @@ -106,6 +106,21 @@ class DSCoSTEERCoSTEEREvaluator(CoSTEEREvaluator): submission_check_out, submission_ret_code = implementation.execute_ret_code( env=mde, entry="python test/mle_submission_format_test.py" ) + if DS_RD_SETTING.rule_base_eval: + if execute_ret_code == 0 and score_ret_code == 0 and submission_ret_code == 0: + return DSCoSTEEREvalFeedback( + execution=stdout, + return_checking=score_check_text + "\n" + submission_check_out, + code="Code evaluation is not available.", + final_decision=True, + ) + else: + return DSCoSTEEREvalFeedback( + execution=stdout, + return_checking=score_check_text + "\n" + submission_check_out, + code="Code evaluation is not available.", + final_decision=False, + ) stdout += f"\nMLEBench submission check:\n{submission_check_out}\nIf MLEBench submission check returns a 'Submission is valid' or similar message, despite some warning messages, you should still consider the submission as valid and give a positive final decision. " system_prompt = T(".prompts:DSCoSTEER_eval.system").r( diff --git a/rdagent/scenarios/data_science/dev/runner/eval_tests/mle_submission_format_test.txt b/rdagent/scenarios/data_science/dev/runner/eval_tests/mle_submission_format_test.txt index 72fb2dff..fd7241bb 100644 --- a/rdagent/scenarios/data_science/dev/runner/eval_tests/mle_submission_format_test.txt +++ b/rdagent/scenarios/data_science/dev/runner/eval_tests/mle_submission_format_test.txt @@ -4,7 +4,7 @@ from mlebench.grade import validate_submission from mlebench.registry import registry # Check if our submission file exists -assert Path('submission.csv').exists(), "Error: submission.csv not found" +assert Path("submission.csv").exists(), "Error: submission.csv not found" COMPETITION_ID = "" new_registry = registry.set_data_dir(Path("/mle/data")) @@ -12,4 +12,7 @@ competition = new_registry.get_competition(COMPETITION_ID) is_valid, message = validate_submission(Path("submission.csv"), competition) -print(message) \ No newline at end of file +print(message) + +if not is_valid: + raise AssertionError("Submission is invalid") diff --git a/rdagent/scenarios/data_science/share.yaml b/rdagent/scenarios/data_science/share.yaml index c0fe6541..f8fc7b63 100644 --- a/rdagent/scenarios/data_science/share.yaml +++ b/rdagent/scenarios/data_science/share.yaml @@ -301,9 +301,9 @@ component_spec: 7. Metric Calculation and Storage: - Calculate the metric (mentioned in the evaluation section of the competition information) for each model and ensemble strategy on valid, and save the results in `scores.csv` - - The evaluation should be based on 5-fold cross-validation but only if that's an appropriate evaluation for the task at hand. + - The evaluation should be based on 5-fold cross-validation but only if that's an appropriate evaluation for the task at hand. Store the mean validation score of 5-fold cross-validation in `scores.csv` on each model. - Even if only one model is present, compute the ensemble score and store it under `"ensemble"`. - - The index of `scores.csv` should include the model name and the "ensemble" strategy. "ensemble" should be exactly in the index with all lower case letters. + - The index of `scores.csv` should include the model name and the "ensemble" strategy. "ensemble" should be exactly in the index with all lower case letters. Ensemble is the result from several models. If only one model is present, the ensemble score should be the same as the model score. - The column names in `scores.csv` should be: - Model: The name of the model or ensemble strategy. - : The calculated metric value for that model or ensemble strategy. The metric name can be found in the scenario description. The metric name should be exactly the same as the one in the scenario description since user will use it to check the result.