From d0afa599a7885aed8f5857cbd6fd17e828dc27f3 Mon Sep 17 00:00:00 2001 From: Xu Yang Date: Fri, 4 Jul 2025 14:30:40 +0800 Subject: [PATCH] feat: try coder on whole data (#1017) * commit all code * fix feedback bug * add debug mode in ds * update prompt * prioritize performance than time lit * use run instead * store running time * move all data running into running phase * fix a bug * use sample data as default --------- Co-authored-by: Xu Yang --- rdagent/app/data_science/conf.py | 2 +- rdagent/components/coder/data_science/conf.py | 2 +- .../coder/data_science/pipeline/__init__.py | 1 + .../coder/data_science/pipeline/eval.py | 31 ++++- .../eval_tests/submission_format_test.txt | 106 ++++++++++-------- .../coder/data_science/pipeline/prompts.yaml | 39 ++++++- rdagent/oai/backend/base.py | 2 +- .../scenarios/data_science/dev/feedback.py | 11 +- .../proposal/exp_gen/prompts_v2.yaml | 4 +- .../scenarios/data_science/scen/__init__.py | 2 +- .../scenarios/data_science/scen/prompts.yaml | 4 +- 11 files changed, 135 insertions(+), 69 deletions(-) diff --git a/rdagent/app/data_science/conf.py b/rdagent/app/data_science/conf.py index a0c88319..e6539d66 100644 --- a/rdagent/app/data_science/conf.py +++ b/rdagent/app/data_science/conf.py @@ -46,7 +46,7 @@ class DataScienceBasePropSetting(KaggleBasePropSetting): runner_max_loop: int = 1 rule_base_eval: bool = False - sample_data: bool = True + sample_data_by_LLM: bool = False use_raw_description: bool = False show_nan_columns: bool = False diff --git a/rdagent/components/coder/data_science/conf.py b/rdagent/components/coder/data_science/conf.py index 789f8b2b..a31a841d 100644 --- a/rdagent/components/coder/data_science/conf.py +++ b/rdagent/components/coder/data_science/conf.py @@ -28,7 +28,7 @@ def get_ds_env( conf_type: Literal["kaggle", "mlebench"] = "kaggle", extra_volumes: dict = {}, running_timeout_period: int = ( - DS_RD_SETTING.debug_timeout if DS_RD_SETTING.sample_data else DS_RD_SETTING.full_timeout + DS_RD_SETTING.debug_timeout if not DS_RD_SETTING.sample_data_by_LLM else DS_RD_SETTING.full_timeout ), ) -> Env: """ diff --git a/rdagent/components/coder/data_science/pipeline/__init__.py b/rdagent/components/coder/data_science/pipeline/__init__.py index cdaa2cf2..7bdb20c6 100644 --- a/rdagent/components/coder/data_science/pipeline/__init__.py +++ b/rdagent/components/coder/data_science/pipeline/__init__.py @@ -97,6 +97,7 @@ class PipelineMultiProcessEvolvingStrategy(MultiProcessEvolvingStrategy): runtime_environment=runtime_environment, spec=T("scenarios.data_science.share:component_spec.Pipeline").r(), enable_model_dump=DS_RD_SETTING.enable_model_dump, + enable_debug_mode=DS_RD_SETTING.sample_data_by_LLM, ) user_prompt = T(".prompts:pipeline_coder.user").r( competition_info=competition_info, diff --git a/rdagent/components/coder/data_science/pipeline/eval.py b/rdagent/components/coder/data_science/pipeline/eval.py index 19b25bf4..6fa54052 100644 --- a/rdagent/components/coder/data_science/pipeline/eval.py +++ b/rdagent/components/coder/data_science/pipeline/eval.py @@ -55,13 +55,33 @@ class PipelineCoSTEEREvaluator(CoSTEEREvaluator): env = get_ds_env(extra_volumes={self.scen.debug_path: T("scenarios.data_science.share:scen.input_path").r()}) - # Clean the scores.csv & submission.csv. + stdout = "" implementation.execute(env=env, entry=get_clear_ws_cmd()) - result = implementation.run(env=env, entry=f"python -m coverage run main.py") - implementation.running_info.running_time = result.running_time + if DS_RD_SETTING.sample_data_by_LLM: + # Because coder runs on full data, we need to run debug mode in advance to save time + result = implementation.run(env=env, entry=f"python -m coverage run main.py --debug") + else: + result = implementation.run(env=env, entry=f"python -m coverage run main.py") execute_ret_code = result.exit_code - stdout = remove_eda_part(result.stdout) - stdout += f"The code executed {'successfully' if execute_ret_code == 0 else 'failed'}." + result.stdout = remove_eda_part(result.stdout) + if result.exit_code != 0: + stdout += f"Code failed to run. Please check the stdout:\n Following the stdout of the debug mode run:\n{result.stdout.strip()}\n" + else: + stdout += f"Code ran successfully.\n Following the stdout of the debug mode run:\n{result.stdout.strip()}\n" + if DS_RD_SETTING.sample_data_by_LLM: + debug_time, full_estimated_time = None, None + if match := re.search(r"debug_time:\s*(\d+(?:.\d+)?)", result.stdout, re.DOTALL): + debug_time = float(match.group(1)) + if match := re.search(r"estimated_time:\s*(\d+(?:.\d+)?)", result.stdout, re.DOTALL): + full_estimated_time = float(match.group(1)) + if debug_time is not None and full_estimated_time is not None: + stdout += f"Debug mode ran in {debug_time:.2f} seconds, estimated full run time is {full_estimated_time:.2f} seconds.\n" + if full_estimated_time < env.conf.running_timeout_period * 3: + stdout += "The estimated full run time is less than three times the timeout period.\n" + else: + stdout += f"The estimated full run time is more than three times the timeout period.\n" + else: + stdout += "Debug mode did not provide debug_time or estimated_time, it's a buggy implementation.\n" score_fp = implementation.workspace_path / "scores.csv" score_ret_code = 0 @@ -137,6 +157,7 @@ class PipelineCoSTEEREvaluator(CoSTEEREvaluator): task_desc=target_task.get_task_information(), is_sub_enabled=test_eval.is_sub_enabled(self.scen.competition), spec=T("scenarios.data_science.share:component_spec.Pipeline").r(), + debug_mode=DS_RD_SETTING.sample_data_by_LLM, ) user_prompt = T(".prompts:pipeline_eval.user").r( stdout=stdout.strip(), diff --git a/rdagent/components/coder/data_science/pipeline/eval_tests/submission_format_test.txt b/rdagent/components/coder/data_science/pipeline/eval_tests/submission_format_test.txt index 7439e2d4..88c559f0 100644 --- a/rdagent/components/coder/data_science/pipeline/eval_tests/submission_format_test.txt +++ b/rdagent/components/coder/data_science/pipeline/eval_tests/submission_format_test.txt @@ -1,6 +1,5 @@ import hashlib from pathlib import Path - import pandas as pd @@ -10,8 +9,12 @@ def calculate_md5(file_path): return file_hash -file_md5 = calculate_md5("scores.csv") - +if Path("scores.csv").exists(): + file_md5 = calculate_md5("scores.csv") +else: + print("Warning: scores.csv does not exist. MD5 check will be skipped.") + file_md5 = None + """ find . | grep -i sample | grep -i submission | grep -v sample_submission.csv | grep -v zip_files | grep -v 'sample/' ./denoising-dirty-documents/sampleSubmission.csv @@ -25,62 +28,67 @@ find . | grep -i sample | grep -i submission | grep -v sample_submission.csv | g """ # Find sample submission file dynamically -input_dir = Path("{% include "scenarios.data_science.share:scen.input_path" %}") -# Look for common variations of sample submission filenames +input_dir = Path('{% include "scenarios.data_science.share:scen.input_path" %}') sample_submission_files = list(input_dir.glob("*sample_submission*.csv")) + list( input_dir.glob("*sampleSubmission*.csv") ) -assert sample_submission_files, "Error: No sample submission file found in {% include "scenarios.data_science.share:scen.input_path" %}" - -# Use first matching file -sample_submission_name = sample_submission_files[0].name -SAMPLE_SUBMISSION_PATH = str(sample_submission_files[0]) -print(f"Using sample submission file: {sample_submission_name}") - -# Check if the sample submission file exists -assert Path(SAMPLE_SUBMISSION_PATH).exists(), f"Error: {sample_submission_name} not found at {SAMPLE_SUBMISSION_PATH}" - -# Check if our submission file exists -assert Path("submission.csv").exists(), "Error: submission.csv not found" - -sample_submission = pd.read_csv(SAMPLE_SUBMISSION_PATH) -our_submission = pd.read_csv("submission.csv") - -success = True -# Print the columns of the sample submission file -print(f"Columns in {sample_submission_name}:", sample_submission.columns) -print("Columns in our_submission.csv:", our_submission.columns) - -for col in sample_submission.columns: - if col not in our_submission.columns: - success = False - print(f"Column {col} not found in submission.csv") - -if success: - print(f"submission.csv's columns aligns with {sample_submission_name} .") +if not sample_submission_files: + print(f'Error: No sample submission file found in {% include "scenarios.data_science.share:scen.input_path" %}') + sample_submission_name = None + SAMPLE_SUBMISSION_PATH = None else: - raise AssertionError(f"submission.csv's columns does not align with {sample_submission_name} .") + sample_submission_name = sample_submission_files[0].name + SAMPLE_SUBMISSION_PATH = str(sample_submission_files[0]) + print(f"Using sample submission file: {sample_submission_name}") + +if SAMPLE_SUBMISSION_PATH is not None and not Path(SAMPLE_SUBMISSION_PATH).exists(): + print(f"Error: {sample_submission_name} not found at {SAMPLE_SUBMISSION_PATH}") + +if not Path("submission.csv").exists(): + print("Error: submission.csv not found") + +if SAMPLE_SUBMISSION_PATH is not None and Path(SAMPLE_SUBMISSION_PATH).exists() and Path("submission.csv").exists(): + sample_submission = pd.read_csv(SAMPLE_SUBMISSION_PATH) + our_submission = pd.read_csv("submission.csv") + + success = True + print(f"Columns in {sample_submission_name}:", sample_submission.columns) + print("Columns in our_submission.csv:", our_submission.columns) + + for col in sample_submission.columns: + if col not in our_submission.columns: + success = False + print(f"Column {col} not found in submission.csv") + + if success: + print(f"submission.csv's columns aligns with {sample_submission_name} .") + else: + print(f"submission.csv's columns does not align with {sample_submission_name} .") -# Print the first 5 rows of the two submission files, with columns separated by commas. -def print_first_rows(file_path, file_name, num_rows=5): - print(f"\nFirst {num_rows} rows of {file_name}:") - try: - with open(file_path, "r") as file: - for i, line in enumerate(file): - if i < num_rows: - print(line.strip()) - else: - break - except FileNotFoundError: - print(f"Error: {file_name} not found.") + def print_first_rows(file_path, file_name, num_rows=5): + print(f"\nFirst {num_rows} rows of {file_name}:") + try: + with open(file_path, "r") as file: + for i, line in enumerate(file): + if i < num_rows: + print(line.strip()) + else: + break + except FileNotFoundError: + print(f"Error: {file_name} not found.") -print_first_rows(SAMPLE_SUBMISSION_PATH, sample_submission_name) -print_first_rows("submission.csv", "submission.csv") + print_first_rows(SAMPLE_SUBMISSION_PATH, sample_submission_name) + print_first_rows("submission.csv", "submission.csv") + + if file_md5 is not None: + if calculate_md5("scores.csv") != file_md5: + print("Warning: scores.csv has been rewritten in the test script!") +else: + print("Skipping comparison and preview due to missing files.") -assert calculate_md5("scores.csv") == file_md5, "scores.csv should not be rewritten" print( f"\nPlease Checked the content of the submission file(submission.csv should has the same format with {sample_submission_name} but might not the same index with {sample_submission_name}). " ) diff --git a/rdagent/components/coder/data_science/pipeline/prompts.yaml b/rdagent/components/coder/data_science/pipeline/prompts.yaml index a29f141b..3a807d74 100644 --- a/rdagent/components/coder/data_science/pipeline/prompts.yaml +++ b/rdagent/components/coder/data_science/pipeline/prompts.yaml @@ -39,6 +39,7 @@ pipeline_coder: ## Guidelines 1. Ensure that the dataset is loaded strictly from `{% include "scenarios.data_science.share:scen.input_path" %}`, following the exact folder structure described in the **Data Folder Description**, and do not attempt to load data from the current directory (`./`). 2. You should avoid using logging module to output information in your generated code, and instead use the print() function. + 3. You should be very careful about the try catch block in your code. You may use it to handle missing files in data reading, but you should not use it to handle the errors in your code. Especially use it to bypass the errors in your code. Directly solve the errors in your code instead of using try catch block to bypass them. ## Exploratory Data Analysis (EDA) part(Required): - Before returning the data, you should always add an EDA part describing the data to help the following steps understand the data better. @@ -64,7 +65,30 @@ pipeline_coder: {% include "components.coder.data_science.share.prompts:dump_model_coder.guideline" %} {% endif %} - + {% if enable_debug_mode %} + Your code will be executed in a debug mode with following command: + ```bash + python main.py --debug + ``` + In debug mode, you should only sample ten percent of the data and run the minimum epochs to quickly test the correctness of the code. + In debug mode, you should implement a timer to measure the time taken for your debug configuration and estimate the time required for the full run. + For example, you can sample ten percent of the data and run for one epoch, then the full run with ten epochs will take one hundred times the time taken for the debug run. The scale is calculated by yourself depending on the data sampling and epoch number you choose. If your full run enables early stopping, the scale should be smaller considering the early stopping will stop the training earlier than the full epochs. + Your debug code should run exactly the same as the full run, except for the data sampling and epoch number, to ensure the correctness of the code. + You should print total time and estimated time in standard output using print function in the following schema: + === Start of Debug Information === + debug_time: time_taken_for_debug_run_in_seconds (e.g., 'debug_time: 10.0') + estimated_time: estimated_time_for_full_run_in_seconds (e.g., 'estimated_time: 100.0') + === End of Debug Information === + User will use the following code to match: re.search(r"(.*?)=== Start of Debug Information ===(.*)=== End of Debug Information ===", stdout, re.DOTALL).groups()[1] + Notice, data sampling should only be applied in debug mode. Always use the full data in the full run! + Example code: + ```python + if args.debug: + sample_size = int(0.1 * len(train_dataset)) # 10% for debug + else: + sample_size = len(train_dataset) + ``` + {% endif %} ## Output Format {% if out_spec %} @@ -117,9 +141,11 @@ pipeline_eval: Your focus is to check whether the workflow code: Step 1: Executes successfully without any errors. Please distinguish between the errors and warnings. - Step 2: Correctly generates a final submission in the correct format, ensuring: they align with the submission structure, the index names and column names should match the sample, and the items should not be empty or apparently incorrect. - - Step 3: Aligns with the competition requirements. This includes: + Step 2: Correctly generates a final submission in the correct format, ensuring: they align with the submission structure, the index names and column names should match the sample, and the items should not be empty or apparently incorrect. The final submission and validation metrics should not be generated with mock data or dummy values. They should only the results of the actual code execution. Once you found the coding containing any data mock or dummy values, you should emphasize it in your feedback and reject this implementation. + + Step 3: No data sampling is applied in the full run, and the code runs on the full dataset. The code should not contain any data sampling or data reduction logic in the full run. Once you found any data sampling or reduction logic in the full run, you should emphasize it in your feedback and reject this implementation. + + Step 4: Aligns with the competition requirements. This includes: - CAREFULLY ANALYZE WHETHER THE EXPERIMENTAL SETUP AND CODE MAY CAUSE MISALIGNMENT BETWEEN VALIDATION AND TEST PERFORMANCE. - Confirm strict adherence to the competition's evaluation rules listed in `scenario`: - Exact match between the implementation code of metric and the requirements of the scenario. The metric number is not the focus. @@ -131,6 +157,11 @@ pipeline_eval: - Begin your `code` with `[Evaluation error]`, explicitly stating the evaluation alignment issues causing experiment failure. - If no issues are found, begin your `code` with `[Code analysis]`, providing a detailed analysis of the code quality, readability, and adherence to specifications. + {% if debug_mode %} + Step 5: The code is executed in a debug mode with the command `python main.py --debug`. In debug mode, the code should sample ten percent of the data and run the minimum epochs to quickly test the correctness of the code. You should check whether the code follows this requirements. If not, you should emphasize it in your feedback and reject this implementation. + You will also be given the execution time and estimated time for the full run. If not time information is provided or the estimated full run time is more than three times the timeout period, you should reject the implementation because the full run will not finish in time. + {% endif %} + ## Evaluation Criteria You will be given the execution output (`stdout`) to determine correctness. diff --git a/rdagent/oai/backend/base.py b/rdagent/oai/backend/base.py index 6daff0da..e9ef0880 100644 --- a/rdagent/oai/backend/base.py +++ b/rdagent/oai/backend/base.py @@ -479,7 +479,7 @@ class APIBackend(ABC): if json_target_type is not None: TypeAdapter(json_target_type).validate_json(all_response) if (response_format := kwargs.get("response_format")) is not None: - if issubclass(response_format, BaseModel): + if not isinstance(response_format, dict) and issubclass(response_format, BaseModel): # It may raise TypeError if initialization fails response_format(**json.loads(all_response)) else: diff --git a/rdagent/scenarios/data_science/dev/feedback.py b/rdagent/scenarios/data_science/dev/feedback.py index 753f053c..7ce46280 100644 --- a/rdagent/scenarios/data_science/dev/feedback.py +++ b/rdagent/scenarios/data_science/dev/feedback.py @@ -109,7 +109,7 @@ class DSExperiment2Feedback(Experiment2Feedback): ) ) - if resp_dict.get("Evaluation Aligned With Task", "no") == "no": + if evaluation_not_aligned := dict_get_with_warning(resp_dict, "Evaluation Aligned With Task", "no") == "no": exp.result = None # Currently, we do not use `observations`, `hypothesis_evaluation`, and `new_hypothesis` in the framework. @@ -118,11 +118,16 @@ class DSExperiment2Feedback(Experiment2Feedback): observations=dict_get_with_warning(resp_dict, "Observations", "No observations provided"), hypothesis_evaluation=dict_get_with_warning(resp_dict, "Feedback for Hypothesis", "No feedback provided"), new_hypothesis=dict_get_with_warning(resp_dict, "New Hypothesis", "No new hypothesis provided"), - reason=dict_get_with_warning(resp_dict, "Reasoning", "No reasoning provided"), + reason=dict_get_with_warning(resp_dict, "Reasoning", "No reasoning provided") + + ("\nRejected because evaluation code not aligned with task." if evaluation_not_aligned else ""), code_change_summary=dict_get_with_warning( resp_dict, "Code Change Summary", "No code change summary provided" ), - decision=convert2bool(dict_get_with_warning(resp_dict, "Replace Best Result", "no")), + decision=( + False + if evaluation_not_aligned + else convert2bool(dict_get_with_warning(resp_dict, "Replace Best Result", "no")) + ), ) if hypothesis_feedback and DS_RD_SETTING.enable_knowledge_base: diff --git a/rdagent/scenarios/data_science/proposal/exp_gen/prompts_v2.yaml b/rdagent/scenarios/data_science/proposal/exp_gen/prompts_v2.yaml index 8836f321..e75e26a9 100644 --- a/rdagent/scenarios/data_science/proposal/exp_gen/prompts_v2.yaml +++ b/rdagent/scenarios/data_science/proposal/exp_gen/prompts_v2.yaml @@ -144,8 +144,8 @@ scenario_description: |- {% if time_limit %} ====== Time Limit ====== - Your code's execution is limited to **{{ time_limit }}**. - Please optimize your model and parameters to ensure your code runs within this specified time constraint. + Your code's execution is limited to **{{ time_limit }}**. After this time limit, your code will be terminated. But remember your main target is to achieve the best performance and you have several times to modify your code. So please be bold to make the best use of all the time limit and don't be too conservative. + During this time limit, you have all the resources available to you. Please fully leverage all the computational resources(CPUs and GPUs) to achieve the best performance like choose a powerful model, use a large batch size, enable data sampler with big parallel. {% endif %} hypothesis_gen: diff --git a/rdagent/scenarios/data_science/scen/__init__.py b/rdagent/scenarios/data_science/scen/__init__.py index fac00486..329c0255 100644 --- a/rdagent/scenarios/data_science/scen/__init__.py +++ b/rdagent/scenarios/data_science/scen/__init__.py @@ -30,7 +30,7 @@ class DataScienceScen(Scenario): raise FileNotFoundError(f"Cannot find {competition} in {DS_RD_SETTING.local_data_path}") local_path = DS_RD_SETTING.local_data_path - if DS_RD_SETTING.sample_data: + if not DS_RD_SETTING.sample_data_by_LLM: self.debug_path = f"{local_path}/sample/{competition}" if not Path(self.debug_path).exists(): sample_py_path = Path(local_path) / competition / "sample.py" diff --git a/rdagent/scenarios/data_science/scen/prompts.yaml b/rdagent/scenarios/data_science/scen/prompts.yaml index cbeca1c6..49fbfd44 100644 --- a/rdagent/scenarios/data_science/scen/prompts.yaml +++ b/rdagent/scenarios/data_science/scen/prompts.yaml @@ -21,8 +21,8 @@ scenario_description: |- {% endif %} {% if time_limit %}------The time limit to your code------ - You code running is limit to {{ time_limit }}, please change yor model type and model parameters to make sure your code can run within the time limit. - + You code running is limit to {{ time_limit }}, after this time limit, your code will be terminated. But remember your main target is to achieve the best performance and you have several times to modify your code. So please be bold to make the best use of all the time limit and don't be too conservative. + During this time limit, you have all the resources available to you. Please fully leverage all the computational resources(CPUs and GPUs) to achieve the best performance like choose a powerful model, use a large batch size, enable data sampler with big parallel. {% endif %} {% if evaluation is not none %}------Evaluation------ {{ evaluation }}