feat: Enhance eval script with file cleanup and detailed submission checks (#529)

rdagent/components/coder/data_science/workflow/eval.py

@@ -58,6 +58,9 @@ class WorkflowGeneralCaseSpecEvaluator(CoSTEEREvaluator):
             f"{DS_RD_SETTING.local_data_path}/sample/{self.scen.competition}": "/kaggle/input"
         }
         de = DockerEnv(conf=ds_docker_conf)
+        # Clean the scores.csv & submission.csv.
+        stdout = implementation.execute(env=de, entry=f"rm submission.csv scores.csv")
+
         fname = "main.py"
         stdout = implementation.execute(env=de, entry=f"python {fname}")
 

rdagent/components/coder/data_science/workflow/eval_tests/submission_check.txt

@@ -1,5 +1,5 @@
-import pandas as pd
 from pathlib import Path
+import pandas as pd
 
 # Check if the sample submission file exists
 if not Path("/kaggle/input/sample_submission.csv").exists():
@@ -9,10 +9,33 @@ sample_submission = pd.read_csv('/kaggle/input/sample_submission.csv')
 our_submission = pd.read_csv('submission.csv')
 
 success = True
+# Print the columns of the sample submission file
+print("Columns in sample_submission.csv:", sample_submission.columns)
+print("Columns in our_submission.csv:", our_submission.columns)
+
 for col in sample_submission.columns:
     if col not in our_submission.columns:
         success = False
         print(f'Column {col} not found in submission.csv')
 
 if success:
-    print('submission.csv is valid.')
+    print('submission.csv\'s columns aligns with sample_submission.csv .')
+
+
+# Print the first 5 rows of the two submission files, with columns separated by commas.
+def print_first_rows(file_path, file_name, num_rows=5):
+    print(f"\nFirst {num_rows} rows of {file_name}:")
+    try:
+        with open(file_path, 'r') as file:
+            for i, line in enumerate(file):
+                if i < num_rows:
+                    print(line.strip())
+                else:
+                    break
+    except FileNotFoundError:
+        print(f"Error: {file_name} not found.")
+
+print_first_rows('/kaggle/input/sample_submission.csv', 'sample_submission.csv')
+print_first_rows('submission.csv', 'submission.csv')
+
+print("\nPlease Checked the content of the submission file(submission.csv should align with sample_submission.csv). ")

rdagent/components/coder/data_science/workflow/prompts.yaml

@@ -100,9 +100,9 @@ workflow_eval:
     Please respond with your feedback in the following JSON format and order:
     ```json
     {
-        "execution": "Describe whether the model executed successfully, including any errors or issues encountered. Please keep the error message and tracking information",
-        "return_checking": "Check the generated value, including whether the value is generated and comparing the shape of the model output with the requirement in the specification. You also need to check whether the hyperparameters used for retraining are correctly returned during the test execution of the model.",
-        "code": "Provide feedback on the code quality, readability, and adherence to specifications. Check whether the hyperparameters from the previous run are used in the model code, compare the parameter names in stdout and if they are used in the retraining part of the code.",
+        "execution": "Describe whether the main code executed successfully(well organizing different components and generating the final submission), including any errors or issues encountered. Please keep the error message and tracking information",
+        "return_checking": "Review the generated files, specifically the submission file, to ensure the format matches the sample submission. Pay attention to the index, columns, and content of the CSV files in the stdout."
+        "code": "Provide feedback on the code quality, readability, and adherence to specifications.",
         "final_decision": <true/false>
     }
     ```