From 5d04badc6a38ab01c03b7470db9bfebd3e73fa62 Mon Sep 17 00:00:00 2001 From: you-n-g Date: Wed, 22 Jan 2025 21:51:34 +0800 Subject: [PATCH] feat: Enhance eval script with file cleanup and detailed submission checks (#529) --- .../coder/data_science/workflow/eval.py | 3 +++ .../workflow/eval_tests/submission_check.txt | 27 +++++++++++++++++-- .../coder/data_science/workflow/prompts.yaml | 6 ++--- 3 files changed, 31 insertions(+), 5 deletions(-) diff --git a/rdagent/components/coder/data_science/workflow/eval.py b/rdagent/components/coder/data_science/workflow/eval.py index 584075ed..7f2e9d8e 100644 --- a/rdagent/components/coder/data_science/workflow/eval.py +++ b/rdagent/components/coder/data_science/workflow/eval.py @@ -58,6 +58,9 @@ class WorkflowGeneralCaseSpecEvaluator(CoSTEEREvaluator): f"{DS_RD_SETTING.local_data_path}/sample/{self.scen.competition}": "/kaggle/input" } de = DockerEnv(conf=ds_docker_conf) + # Clean the scores.csv & submission.csv. + stdout = implementation.execute(env=de, entry=f"rm submission.csv scores.csv") + fname = "main.py" stdout = implementation.execute(env=de, entry=f"python {fname}") diff --git a/rdagent/components/coder/data_science/workflow/eval_tests/submission_check.txt b/rdagent/components/coder/data_science/workflow/eval_tests/submission_check.txt index abca1553..c2c16709 100644 --- a/rdagent/components/coder/data_science/workflow/eval_tests/submission_check.txt +++ b/rdagent/components/coder/data_science/workflow/eval_tests/submission_check.txt @@ -1,5 +1,5 @@ -import pandas as pd from pathlib import Path +import pandas as pd # Check if the sample submission file exists if not Path("/kaggle/input/sample_submission.csv").exists(): @@ -9,10 +9,33 @@ sample_submission = pd.read_csv('/kaggle/input/sample_submission.csv') our_submission = pd.read_csv('submission.csv') success = True +# Print the columns of the sample submission file +print("Columns in sample_submission.csv:", sample_submission.columns) +print("Columns in our_submission.csv:", our_submission.columns) + for col in sample_submission.columns: if col not in our_submission.columns: success = False print(f'Column {col} not found in submission.csv') if success: - print('submission.csv is valid.') \ No newline at end of file + print('submission.csv\'s columns aligns with sample_submission.csv .') + + +# Print the first 5 rows of the two submission files, with columns separated by commas. +def print_first_rows(file_path, file_name, num_rows=5): + print(f"\nFirst {num_rows} rows of {file_name}:") + try: + with open(file_path, 'r') as file: + for i, line in enumerate(file): + if i < num_rows: + print(line.strip()) + else: + break + except FileNotFoundError: + print(f"Error: {file_name} not found.") + +print_first_rows('/kaggle/input/sample_submission.csv', 'sample_submission.csv') +print_first_rows('submission.csv', 'submission.csv') + +print("\nPlease Checked the content of the submission file(submission.csv should align with sample_submission.csv). ") diff --git a/rdagent/components/coder/data_science/workflow/prompts.yaml b/rdagent/components/coder/data_science/workflow/prompts.yaml index fd41e8bd..4110510f 100644 --- a/rdagent/components/coder/data_science/workflow/prompts.yaml +++ b/rdagent/components/coder/data_science/workflow/prompts.yaml @@ -100,9 +100,9 @@ workflow_eval: Please respond with your feedback in the following JSON format and order: ```json { - "execution": "Describe whether the model executed successfully, including any errors or issues encountered. Please keep the error message and tracking information", - "return_checking": "Check the generated value, including whether the value is generated and comparing the shape of the model output with the requirement in the specification. You also need to check whether the hyperparameters used for retraining are correctly returned during the test execution of the model.", - "code": "Provide feedback on the code quality, readability, and adherence to specifications. Check whether the hyperparameters from the previous run are used in the model code, compare the parameter names in stdout and if they are used in the retraining part of the code.", + "execution": "Describe whether the main code executed successfully(well organizing different components and generating the final submission), including any errors or issues encountered. Please keep the error message and tracking information", + "return_checking": "Review the generated files, specifically the submission file, to ensure the format matches the sample submission. Pay attention to the index, columns, and content of the CSV files in the stdout." + "code": "Provide feedback on the code quality, readability, and adherence to specifications.", "final_decision": } ```