diff --git a/rdagent/components/coder/data_science/workflow/eval.py b/rdagent/components/coder/data_science/workflow/eval.py index 584075ed..7f2e9d8e 100644 --- a/rdagent/components/coder/data_science/workflow/eval.py +++ b/rdagent/components/coder/data_science/workflow/eval.py @@ -58,6 +58,9 @@ class WorkflowGeneralCaseSpecEvaluator(CoSTEEREvaluator): f"{DS_RD_SETTING.local_data_path}/sample/{self.scen.competition}": "/kaggle/input" } de = DockerEnv(conf=ds_docker_conf) + # Clean the scores.csv & submission.csv. + stdout = implementation.execute(env=de, entry=f"rm submission.csv scores.csv") + fname = "main.py" stdout = implementation.execute(env=de, entry=f"python {fname}") diff --git a/rdagent/components/coder/data_science/workflow/eval_tests/submission_check.txt b/rdagent/components/coder/data_science/workflow/eval_tests/submission_check.txt index abca1553..c2c16709 100644 --- a/rdagent/components/coder/data_science/workflow/eval_tests/submission_check.txt +++ b/rdagent/components/coder/data_science/workflow/eval_tests/submission_check.txt @@ -1,5 +1,5 @@ -import pandas as pd from pathlib import Path +import pandas as pd # Check if the sample submission file exists if not Path("/kaggle/input/sample_submission.csv").exists(): @@ -9,10 +9,33 @@ sample_submission = pd.read_csv('/kaggle/input/sample_submission.csv') our_submission = pd.read_csv('submission.csv') success = True +# Print the columns of the sample submission file +print("Columns in sample_submission.csv:", sample_submission.columns) +print("Columns in our_submission.csv:", our_submission.columns) + for col in sample_submission.columns: if col not in our_submission.columns: success = False print(f'Column {col} not found in submission.csv') if success: - print('submission.csv is valid.') \ No newline at end of file + print('submission.csv\'s columns aligns with sample_submission.csv .') + + +# Print the first 5 rows of the two submission files, with columns separated by commas. +def print_first_rows(file_path, file_name, num_rows=5): + print(f"\nFirst {num_rows} rows of {file_name}:") + try: + with open(file_path, 'r') as file: + for i, line in enumerate(file): + if i < num_rows: + print(line.strip()) + else: + break + except FileNotFoundError: + print(f"Error: {file_name} not found.") + +print_first_rows('/kaggle/input/sample_submission.csv', 'sample_submission.csv') +print_first_rows('submission.csv', 'submission.csv') + +print("\nPlease Checked the content of the submission file(submission.csv should align with sample_submission.csv). ") diff --git a/rdagent/components/coder/data_science/workflow/prompts.yaml b/rdagent/components/coder/data_science/workflow/prompts.yaml index fd41e8bd..4110510f 100644 --- a/rdagent/components/coder/data_science/workflow/prompts.yaml +++ b/rdagent/components/coder/data_science/workflow/prompts.yaml @@ -100,9 +100,9 @@ workflow_eval: Please respond with your feedback in the following JSON format and order: ```json { - "execution": "Describe whether the model executed successfully, including any errors or issues encountered. Please keep the error message and tracking information", - "return_checking": "Check the generated value, including whether the value is generated and comparing the shape of the model output with the requirement in the specification. You also need to check whether the hyperparameters used for retraining are correctly returned during the test execution of the model.", - "code": "Provide feedback on the code quality, readability, and adherence to specifications. Check whether the hyperparameters from the previous run are used in the model code, compare the parameter names in stdout and if they are used in the retraining part of the code.", + "execution": "Describe whether the main code executed successfully(well organizing different components and generating the final submission), including any errors or issues encountered. Please keep the error message and tracking information", + "return_checking": "Review the generated files, specifically the submission file, to ensure the format matches the sample submission. Pay attention to the index, columns, and content of the CSV files in the stdout." + "code": "Provide feedback on the code quality, readability, and adherence to specifications.", "final_decision": } ```