feat: try coder on whole data (#1017)

* commit all code

* fix feedback bug

* add debug mode in ds

* update prompt

* prioritize performance than time lit

* use run instead

* store running time

* move all data running into running phase

* fix a bug

* use sample data as default

---------

Co-authored-by: Xu Yang <xuyang1@microsoft.com>

rdagent/app/data_science/conf.py

@@ -46,7 +46,7 @@ class DataScienceBasePropSetting(KaggleBasePropSetting):
     runner_max_loop: int = 1
 
     rule_base_eval: bool = False
-    sample_data: bool = True
+    sample_data_by_LLM: bool = False
     use_raw_description: bool = False
     show_nan_columns: bool = False
 

rdagent/components/coder/data_science/conf.py

@@ -28,7 +28,7 @@ def get_ds_env(
     conf_type: Literal["kaggle", "mlebench"] = "kaggle",
     extra_volumes: dict = {},
     running_timeout_period: int = (
-        DS_RD_SETTING.debug_timeout if DS_RD_SETTING.sample_data else DS_RD_SETTING.full_timeout
+        DS_RD_SETTING.debug_timeout if not DS_RD_SETTING.sample_data_by_LLM else DS_RD_SETTING.full_timeout
     ),
 ) -> Env:
     """

rdagent/components/coder/data_science/pipeline/__init__.py

@@ -97,6 +97,7 @@ class PipelineMultiProcessEvolvingStrategy(MultiProcessEvolvingStrategy):
             runtime_environment=runtime_environment,
             spec=T("scenarios.data_science.share:component_spec.Pipeline").r(),
             enable_model_dump=DS_RD_SETTING.enable_model_dump,
+            enable_debug_mode=DS_RD_SETTING.sample_data_by_LLM,
         )
         user_prompt = T(".prompts:pipeline_coder.user").r(
             competition_info=competition_info,

rdagent/components/coder/data_science/pipeline/eval.py

@@ -55,13 +55,33 @@ class PipelineCoSTEEREvaluator(CoSTEEREvaluator):
 
         env = get_ds_env(extra_volumes={self.scen.debug_path: T("scenarios.data_science.share:scen.input_path").r()})
 
-        # Clean the scores.csv & submission.csv.
+        stdout = ""
         implementation.execute(env=env, entry=get_clear_ws_cmd())
-        result = implementation.run(env=env, entry=f"python -m coverage run main.py")
-        implementation.running_info.running_time = result.running_time
+        if DS_RD_SETTING.sample_data_by_LLM:
+            # Because coder runs on full data, we need to run debug mode in advance to save time
+            result = implementation.run(env=env, entry=f"python -m coverage run main.py --debug")
+        else:
+            result = implementation.run(env=env, entry=f"python -m coverage run main.py")
         execute_ret_code = result.exit_code
-        stdout = remove_eda_part(result.stdout)
-        stdout += f"The code executed {'successfully' if execute_ret_code == 0 else 'failed'}."
+        result.stdout = remove_eda_part(result.stdout)
+        if result.exit_code != 0:
+            stdout += f"Code failed to run. Please check the stdout:\n Following the stdout of the debug mode run:\n{result.stdout.strip()}\n"
+        else:
+            stdout += f"Code ran successfully.\n Following the stdout of the debug mode run:\n{result.stdout.strip()}\n"
+        if DS_RD_SETTING.sample_data_by_LLM:
+            debug_time, full_estimated_time = None, None
+            if match := re.search(r"debug_time:\s*(\d+(?:.\d+)?)", result.stdout, re.DOTALL):
+                debug_time = float(match.group(1))
+            if match := re.search(r"estimated_time:\s*(\d+(?:.\d+)?)", result.stdout, re.DOTALL):
+                full_estimated_time = float(match.group(1))
+            if debug_time is not None and full_estimated_time is not None:
+                stdout += f"Debug mode ran in {debug_time:.2f} seconds, estimated full run time is {full_estimated_time:.2f} seconds.\n"
+                if full_estimated_time < env.conf.running_timeout_period * 3:
+                    stdout += "The estimated full run time is less than three times the timeout period.\n"
+                else:
+                    stdout += f"The estimated full run time is more than three times the timeout period.\n"
+            else:
+                stdout += "Debug mode did not provide debug_time or estimated_time, it's a buggy implementation.\n"
 
         score_fp = implementation.workspace_path / "scores.csv"
         score_ret_code = 0
@@ -137,6 +157,7 @@ class PipelineCoSTEEREvaluator(CoSTEEREvaluator):
             task_desc=target_task.get_task_information(),
             is_sub_enabled=test_eval.is_sub_enabled(self.scen.competition),
             spec=T("scenarios.data_science.share:component_spec.Pipeline").r(),
+            debug_mode=DS_RD_SETTING.sample_data_by_LLM,
         )
         user_prompt = T(".prompts:pipeline_eval.user").r(
             stdout=stdout.strip(),

rdagent/components/coder/data_science/pipeline/eval_tests/submission_format_test.txt

@@ -1,6 +1,5 @@
 import hashlib
 from pathlib import Path
-
 import pandas as pd
 
 
@@ -10,8 +9,12 @@ def calculate_md5(file_path):
     return file_hash
 
 
-file_md5 = calculate_md5("scores.csv")
-
+if Path("scores.csv").exists():
+    file_md5 = calculate_md5("scores.csv")
+else:
+    print("Warning: scores.csv does not exist. MD5 check will be skipped.")
+    file_md5 = None
+    
 """
 find . | grep -i sample | grep -i submission | grep -v sample_submission.csv | grep -v zip_files  | grep -v 'sample/'
 ./denoising-dirty-documents/sampleSubmission.csv
@@ -25,62 +28,67 @@ find . | grep -i sample | grep -i submission | grep -v sample_submission.csv | g
 """
 
 # Find sample submission file dynamically
-input_dir = Path("{% include "scenarios.data_science.share:scen.input_path" %}")
-# Look for common variations of sample submission filenames
+input_dir = Path('{% include "scenarios.data_science.share:scen.input_path" %}')
 sample_submission_files = list(input_dir.glob("*sample_submission*.csv")) + list(
     input_dir.glob("*sampleSubmission*.csv")
 )
 
-assert sample_submission_files, "Error: No sample submission file found in {% include "scenarios.data_science.share:scen.input_path" %}"
-
-# Use first matching file
-sample_submission_name = sample_submission_files[0].name
-SAMPLE_SUBMISSION_PATH = str(sample_submission_files[0])
-print(f"Using sample submission file: {sample_submission_name}")
-
-# Check if the sample submission file exists
-assert Path(SAMPLE_SUBMISSION_PATH).exists(), f"Error: {sample_submission_name} not found at {SAMPLE_SUBMISSION_PATH}"
-
-# Check if our submission file exists
-assert Path("submission.csv").exists(), "Error: submission.csv not found"
-
-sample_submission = pd.read_csv(SAMPLE_SUBMISSION_PATH)
-our_submission = pd.read_csv("submission.csv")
-
-success = True
-# Print the columns of the sample submission file
-print(f"Columns in {sample_submission_name}:", sample_submission.columns)
-print("Columns in our_submission.csv:", our_submission.columns)
-
-for col in sample_submission.columns:
-    if col not in our_submission.columns:
-        success = False
-        print(f"Column {col} not found in submission.csv")
-
-if success:
-    print(f"submission.csv's columns aligns with {sample_submission_name} .")
+if not sample_submission_files:
+    print(f'Error: No sample submission file found in {% include "scenarios.data_science.share:scen.input_path" %}')
+    sample_submission_name = None
+    SAMPLE_SUBMISSION_PATH = None
 else:
-    raise AssertionError(f"submission.csv's columns does not align with {sample_submission_name} .")
+    sample_submission_name = sample_submission_files[0].name
+    SAMPLE_SUBMISSION_PATH = str(sample_submission_files[0])
+    print(f"Using sample submission file: {sample_submission_name}")
+
+if SAMPLE_SUBMISSION_PATH is not None and not Path(SAMPLE_SUBMISSION_PATH).exists():
+    print(f"Error: {sample_submission_name} not found at {SAMPLE_SUBMISSION_PATH}")
+
+if not Path("submission.csv").exists():
+    print("Error: submission.csv not found")
+
+if SAMPLE_SUBMISSION_PATH is not None and Path(SAMPLE_SUBMISSION_PATH).exists() and Path("submission.csv").exists():
+    sample_submission = pd.read_csv(SAMPLE_SUBMISSION_PATH)
+    our_submission = pd.read_csv("submission.csv")
+
+    success = True
+    print(f"Columns in {sample_submission_name}:", sample_submission.columns)
+    print("Columns in our_submission.csv:", our_submission.columns)
+
+    for col in sample_submission.columns:
+        if col not in our_submission.columns:
+            success = False
+            print(f"Column {col} not found in submission.csv")
+
+    if success:
+        print(f"submission.csv's columns aligns with {sample_submission_name} .")
+    else:
+        print(f"submission.csv's columns does not align with {sample_submission_name} .")
 
 
-# Print the first 5 rows of the two submission files, with columns separated by commas.
-def print_first_rows(file_path, file_name, num_rows=5):
-    print(f"\nFirst {num_rows} rows of {file_name}:")
-    try:
-        with open(file_path, "r") as file:
-            for i, line in enumerate(file):
-                if i < num_rows:
-                    print(line.strip())
-                else:
-                    break
-    except FileNotFoundError:
-        print(f"Error: {file_name} not found.")
+    def print_first_rows(file_path, file_name, num_rows=5):
+        print(f"\nFirst {num_rows} rows of {file_name}:")
+        try:
+            with open(file_path, "r") as file:
+                for i, line in enumerate(file):
+                    if i < num_rows:
+                        print(line.strip())
+                    else:
+                        break
+        except FileNotFoundError:
+            print(f"Error: {file_name} not found.")
 
 
-print_first_rows(SAMPLE_SUBMISSION_PATH, sample_submission_name)
-print_first_rows("submission.csv", "submission.csv")
+    print_first_rows(SAMPLE_SUBMISSION_PATH, sample_submission_name)
+    print_first_rows("submission.csv", "submission.csv")
+
+    if file_md5 is not None:
+        if calculate_md5("scores.csv") != file_md5:
+            print("Warning: scores.csv has been rewritten in the test script!")
+else:
+    print("Skipping comparison and preview due to missing files.")
 
-assert calculate_md5("scores.csv") == file_md5, "scores.csv should not be rewritten"
 print(
     f"\nPlease Checked the content of the submission file(submission.csv should has the same format with {sample_submission_name} but might not the same index with {sample_submission_name}). "
 )

rdagent/components/coder/data_science/pipeline/prompts.yaml

@@ -39,6 +39,7 @@ pipeline_coder:
     ## Guidelines
     1. Ensure that the dataset is loaded strictly from `{% include "scenarios.data_science.share:scen.input_path" %}`, following the exact folder structure described in the **Data Folder Description**, and do not attempt to load data from the current directory (`./`).
     2. You should avoid using logging module to output information in your generated code, and instead use the print() function.
+    3. You should be very careful about the try catch block in your code. You may use it to handle missing files in data reading, but you should not use it to handle the errors in your code. Especially use it to bypass the errors in your code. Directly solve the errors in your code instead of using try catch block to bypass them.
     
     ## Exploratory Data Analysis (EDA) part(Required):
     - Before returning the data, you should always add an EDA part describing the data to help the following steps understand the data better.
@@ -64,7 +65,30 @@ pipeline_coder:
     {% include "components.coder.data_science.share.prompts:dump_model_coder.guideline" %}
     {% endif %}
 
-
+    {% if enable_debug_mode %}
+    Your code will be executed in a debug mode with following command: 
+    ```bash
+    python main.py --debug
+    ```
+    In debug mode, you should only sample ten percent of the data and run the minimum epochs to quickly test the correctness of the code.
+    In debug mode, you should implement a timer to measure the time taken for your debug configuration and estimate the time required for the full run.
+    For example, you can sample ten percent of the data and run for one epoch, then the full run with ten epochs will take one hundred times the time taken for the debug run. The scale is calculated by yourself depending on the data sampling and epoch number you choose. If your full run enables early stopping, the scale should be smaller considering the early stopping will stop the training earlier than the full epochs.
+    Your debug code should run exactly the same as the full run, except for the data sampling and epoch number, to ensure the correctness of the code.
+    You should print total time and estimated time in standard output using print function in the following schema:
+    === Start of Debug Information ===
+    debug_time: time_taken_for_debug_run_in_seconds (e.g., 'debug_time: 10.0')
+    estimated_time: estimated_time_for_full_run_in_seconds (e.g., 'estimated_time: 100.0')
+    === End of Debug Information ===
+    User will use the following code to match: re.search(r"(.*?)=== Start of Debug Information ===(.*)=== End of Debug Information ===", stdout, re.DOTALL).groups()[1]
+    Notice, data sampling should only be applied in debug mode. Always use the full data in the full run!
+    Example code:
+    ```python
+    if args.debug:
+      sample_size = int(0.1 * len(train_dataset))  # 10% for debug
+    else:
+      sample_size = len(train_dataset)
+    ```
+    {% endif %}
 
     ## Output Format
     {% if out_spec %}
@@ -117,9 +141,11 @@ pipeline_eval:
     Your focus is to check whether the workflow code:
     Step 1: Executes successfully without any errors. Please distinguish between the errors and warnings.
 
-    Step 2: Correctly generates a final submission in the correct format, ensuring: they align with the submission structure, the index names and column names should match the sample, and the items should not be empty or apparently incorrect.
-    
-    Step 3: Aligns with the competition requirements. This includes:
+    Step 2: Correctly generates a final submission in the correct format, ensuring: they align with the submission structure, the index names and column names should match the sample, and the items should not be empty or apparently incorrect. The final submission and validation metrics should not be generated with mock data or dummy values. They should only the results of the actual code execution. Once you found the coding containing any data mock or dummy values, you should emphasize it in your feedback and reject this implementation.
+
+    Step 3: No data sampling is applied in the full run, and the code runs on the full dataset. The code should not contain any data sampling or data reduction logic in the full run. Once you found any data sampling or reduction logic in the full run, you should emphasize it in your feedback and reject this implementation.
+
+    Step 4: Aligns with the competition requirements. This includes:
     - CAREFULLY ANALYZE WHETHER THE EXPERIMENTAL SETUP AND CODE MAY CAUSE MISALIGNMENT BETWEEN VALIDATION AND TEST PERFORMANCE.
     - Confirm strict adherence to the competition's evaluation rules listed in `scenario`:
       - Exact match between the implementation code of metric and the requirements of the scenario. The metric number is not the focus.
@@ -131,6 +157,11 @@ pipeline_eval:
       - Begin your `code` with `[Evaluation error]`, explicitly stating the evaluation alignment issues causing experiment failure.
     - If no issues are found, begin your `code` with `[Code analysis]`, providing a detailed analysis of the code quality, readability, and adherence to specifications.
 
+    {% if debug_mode %}
+    Step 5: The code is executed in a debug mode with the command `python main.py --debug`. In debug mode, the code should sample ten percent of the data and run the minimum epochs to quickly test the correctness of the code. You should check whether the code follows this requirements. If not, you should emphasize it in your feedback and reject this implementation.
+    You will also be given the execution time and estimated time for the full run. If not time information is provided or the estimated full run time is more than three times the timeout period, you should reject the implementation because the full run will not finish in time. 
+    {% endif %}
+
     ## Evaluation Criteria
     You will be given the execution output (`stdout`) to determine correctness.  
 

rdagent/oai/backend/base.py

@@ -479,7 +479,7 @@ class APIBackend(ABC):
         if json_target_type is not None:
             TypeAdapter(json_target_type).validate_json(all_response)
         if (response_format := kwargs.get("response_format")) is not None:
-            if issubclass(response_format, BaseModel):
+            if not isinstance(response_format, dict) and issubclass(response_format, BaseModel):
                 # It may raise TypeError if initialization fails
                 response_format(**json.loads(all_response))
             else:

rdagent/scenarios/data_science/dev/feedback.py

@@ -109,7 +109,7 @@ class DSExperiment2Feedback(Experiment2Feedback):
             )
         )
 
-        if resp_dict.get("Evaluation Aligned With Task", "no") == "no":
+        if evaluation_not_aligned := dict_get_with_warning(resp_dict, "Evaluation Aligned With Task", "no") == "no":
             exp.result = None
 
         # Currently, we do not use `observations`, `hypothesis_evaluation`, and `new_hypothesis` in the framework.
@@ -118,11 +118,16 @@ class DSExperiment2Feedback(Experiment2Feedback):
             observations=dict_get_with_warning(resp_dict, "Observations", "No observations provided"),
             hypothesis_evaluation=dict_get_with_warning(resp_dict, "Feedback for Hypothesis", "No feedback provided"),
             new_hypothesis=dict_get_with_warning(resp_dict, "New Hypothesis", "No new hypothesis provided"),
-            reason=dict_get_with_warning(resp_dict, "Reasoning", "No reasoning provided"),
+            reason=dict_get_with_warning(resp_dict, "Reasoning", "No reasoning provided")
+            + ("\nRejected because evaluation code not aligned with task." if evaluation_not_aligned else ""),
             code_change_summary=dict_get_with_warning(
                 resp_dict, "Code Change Summary", "No code change summary provided"
             ),
-            decision=convert2bool(dict_get_with_warning(resp_dict, "Replace Best Result", "no")),
+            decision=(
+                False
+                if evaluation_not_aligned
+                else convert2bool(dict_get_with_warning(resp_dict, "Replace Best Result", "no"))
+            ),
         )
 
         if hypothesis_feedback and DS_RD_SETTING.enable_knowledge_base:

rdagent/scenarios/data_science/proposal/exp_gen/prompts_v2.yaml

@@ -144,8 +144,8 @@ scenario_description: |-
 
   {% if time_limit %}
   ====== Time Limit ======
-  Your code's execution is limited to **{{ time_limit }}**.
-  Please optimize your model and parameters to ensure your code runs within this specified time constraint.
+  Your code's execution is limited to **{{ time_limit }}**. After this time limit, your code will be terminated. But remember your main target is to achieve the best performance and you have several times to modify your code. So please be bold to make the best use of all the time limit and don't be too conservative.
+  During this time limit, you have all the resources available to you. Please fully leverage all the computational resources(CPUs and GPUs) to achieve the best performance like choose a powerful model, use a large batch size, enable data sampler with big parallel.
   {% endif %}
 
 hypothesis_gen:

rdagent/scenarios/data_science/scen/__init__.py

@@ -30,7 +30,7 @@ class DataScienceScen(Scenario):
             raise FileNotFoundError(f"Cannot find {competition} in {DS_RD_SETTING.local_data_path}")
 
         local_path = DS_RD_SETTING.local_data_path
-        if DS_RD_SETTING.sample_data:
+        if not DS_RD_SETTING.sample_data_by_LLM:
             self.debug_path = f"{local_path}/sample/{competition}"
             if not Path(self.debug_path).exists():
                 sample_py_path = Path(local_path) / competition / "sample.py"

rdagent/scenarios/data_science/scen/prompts.yaml

@@ -21,8 +21,8 @@ scenario_description: |-
   {% endif %}
 
   {% if time_limit %}------The time limit to your code------
-  You code running is limit to {{ time_limit }}, please change yor model type and model parameters to make sure your code can run within the time limit.
-
+  You code running is limit to {{ time_limit }}, after this time limit, your code will be terminated. But remember your main target is to achieve the best performance and you have several times to modify your code. So please be bold to make the best use of all the time limit and don't be too conservative.
+  During this time limit, you have all the resources available to you. Please fully leverage all the computational resources(CPUs and GPUs) to achieve the best performance like choose a powerful model, use a large batch size, enable data sampler with big parallel.
   {% endif %}
   {% if evaluation is not none %}------Evaluation------
   {{ evaluation }}