feat: add eda to data science scenario (#639)

* add eda to data science scenario

* fix CI

rdagent/components/coder/data_science/ensemble/__init__.py

@@ -74,6 +74,7 @@ class EnsembleMultiProcessEvolvingStrategy(MultiProcessEvolvingStrategy):
             queried_former_failed_knowledge=(
                 queried_former_failed_knowledge[0] if queried_former_failed_knowledge else None
             ),
+            all_code=workspace.all_codes,
         )
         user_prompt = T(".prompts:ensemble_coder.user").r(
             ensemble_spec=workspace.file_dict["spec/ensemble.md"],

rdagent/components/coder/data_science/ensemble/eval.py

@@ -1,4 +1,5 @@
 import json
+import re
 from pathlib import Path
 
 from jinja2 import Environment, StrictUndefined
@@ -69,6 +70,7 @@ class EnsembleCoSTEEREvaluator(CoSTEEREvaluator):
 
         if "main.py" in implementation.file_dict:
             workflow_stdout = implementation.execute(env=de, entry="python main.py")
+            workflow_stdout = re.sub(r"=== Start of EDA part ===(.*)=== End of EDA part ===", "", workflow_stdout)
         else:
             workflow_stdout = None
 

rdagent/components/coder/data_science/ensemble/prompts.yaml

@@ -35,7 +35,10 @@ ensemble_coder:
     {% endfor %}
     {% endif %}
 
-    You should avoid using logging module to output information in your generated code, and instead use the print() function.
+    ## Guidelines
+    1. The function's code is associated with several other functions including a data loader, feature engineering, and model training. all codes are as follows:
+    {{ all_code }}
+    2. You should avoid using logging module to output information in your generated code, and instead use the print() function.
 
     ## Output Format
     Please response the code in the following json format. Here is an example structure for the JSON output:

rdagent/components/coder/data_science/feature/__init__.py

@@ -55,6 +55,7 @@ class FeatureMultiProcessEvolvingStrategy(MultiProcessEvolvingStrategy):
 
         # 2. code
         system_prompt = T(".prompts:feature_coder.system").r(
+            competition_info=self.scen.get_scenario_all_desc(),
             task_desc=feature_information_str,
             data_loader_code=workspace.file_dict.get("load_data.py"),
             queried_similar_successful_knowledge=queried_similar_successful_knowledge,

rdagent/components/coder/data_science/feature/eval.py

@@ -1,4 +1,5 @@
 import json
+import re
 from pathlib import Path
 
 from rdagent.app.data_science.conf import DS_RD_SETTING
@@ -60,6 +61,7 @@ class FeatureCoSTEEREvaluator(CoSTEEREvaluator):
 
         if "main.py" in implementation.file_dict:
             workflow_stdout = implementation.execute(env=de, entry="python main.py")
+            workflow_stdout = re.sub(r"=== Start of EDA part ===(.*)=== End of EDA part ===", "", workflow_stdout)
         else:
             workflow_stdout = None
 

rdagent/components/coder/data_science/feature/prompts.yaml

@@ -5,6 +5,9 @@ feature_coder:
 
     ## Task Description
     {{ task_desc }}
+    
+    ## Competition Information for This Task
+    {{ competition_info }}
 
     {% if queried_similar_successful_knowledge|length != 0 or queried_former_failed_knowledge|length != 0 %}
     ## Relevant Information for This Task
@@ -31,7 +34,8 @@ feature_coder:
 
     ## Guidelines
     1. If feature engineering is unnecessary or should be combined with model training, you may skip this step.
-    2. The function input is the output of the following data loader:
+    2. Be cautious of any column drop in the code. Dropping a column easily without any more attempts, it may not be a good practice.
+    3. The function input is the output of the following data loader:
     ```python
     {{ data_loader_code }}
     ```

rdagent/components/coder/data_science/model/__init__.py

@@ -56,6 +56,7 @@ class ModelMultiProcessEvolvingStrategy(MultiProcessEvolvingStrategy):
         # 2. code
         system_prompt = T(".prompts:model_coder.system").r(
             task_desc=model_information_str,
+            competition_info=self.scen.get_scenario_all_desc(),
             data_loader_code=workspace.file_dict.get("load_data.py"),
             feature_code=workspace.file_dict["feature.py"],
             queried_similar_successful_knowledge=queried_similar_successful_knowledge,

rdagent/components/coder/data_science/model/eval.py

@@ -76,6 +76,7 @@ class ModelGeneralCaseSpecEvaluator(CoSTEEREvaluator):
 
         if "main.py" in implementation.file_dict:
             workflow_stdout = implementation.execute(env=de, entry="python main.py")
+            workflow_stdout = re.sub(r"=== Start of EDA part ===(.*)=== End of EDA part ===", "", workflow_stdout)
         else:
             workflow_stdout = None
 

rdagent/components/coder/data_science/model/prompts.yaml

@@ -5,6 +5,9 @@ model_coder:
 
     ## Task Description
     {{ task_desc }}
+    
+    ## Competition Information for This Task
+    {{ competition_info }}
 
     {% if queried_similar_successful_knowledge|length != 0 or queried_former_failed_knowledge|length != 0 %}
     ## Relevant Information for This Task

rdagent/components/coder/data_science/raw_data_loader/__init__.py

@@ -23,7 +23,9 @@ File structure
 """
 
 import json
+import re
 
+from rdagent.app.data_science.conf import DS_RD_SETTING
 from rdagent.components.coder.CoSTEER import CoSTEER
 from rdagent.components.coder.CoSTEER.evaluators import (
     CoSTEERMultiEvaluator,
@@ -45,6 +47,7 @@ from rdagent.core.experiment import FBWorkspace
 from rdagent.core.scenario import Scenario
 from rdagent.oai.llm_utils import APIBackend
 from rdagent.utils.agent.tpl import T
+from rdagent.utils.env import DockerEnv, DSDockerConf
 
 
 class DataLoaderMultiProcessEvolvingStrategy(MultiProcessEvolvingStrategy):
@@ -193,3 +196,15 @@ class DataLoaderCoSTEER(CoSTEER):
         es = DataLoaderMultiProcessEvolvingStrategy(scen=scen, settings=settings)
 
         super().__init__(*args, settings=settings, eva=eva, es=es, evolving_version=2, scen=scen, **kwargs)
+
+    def develop(self, exp):
+        new_exp = super().develop(exp)
+
+        ds_docker_conf = DSDockerConf()
+        ds_docker_conf.extra_volumes = {f"{DS_RD_SETTING.local_data_path}/{self.scen.competition}": "/kaggle/input"}
+        de = DockerEnv(conf=ds_docker_conf)
+        stdout = new_exp.experiment_workspace.execute(env=de, entry=f"python test/data_loader_test.py")
+        match = re.search(r"(.*?)=== Start of EDA part ===(.*)=== End of EDA part ===", stdout, re.DOTALL)
+        eda_output = match.groups()[1] if match else None
+        self.scen.eda_output = eda_output
+        return new_exp

rdagent/components/coder/data_science/raw_data_loader/eval.py

@@ -1,6 +1,7 @@
 # tess successfully running.
 # (GPT) if it aligns with the spec & rationality of the spec.
 import json
+import re
 from pathlib import Path
 
 from rdagent.app.data_science.conf import DS_RD_SETTING
@@ -58,9 +59,15 @@ class DataLoaderCoSTEEREvaluator(CoSTEEREvaluator):
         test_code = (DIRNAME / "eval_tests" / "data_loader_test.txt").read_text()
         implementation.inject_files(**{fname: test_code})
         stdout = implementation.execute(env=de, entry=f"python {fname}")
+        match = re.search(r"(.*?)=== Start of EDA part ===(.*)=== End of EDA part ===(.*)", stdout, re.DOTALL)
+        stdout_part_1, eda_output, stdout_part_2 = match.groups() if match else (stdout, None, "")
+        stdout = stdout_part_1 + stdout_part_2
+        if eda_output is not None and len(eda_output.split(" ")) > 10000:
+            eda_output += "Length of EDA output is too long, truncated. Please reject this implementation and motivate it to reduce the length of EDA output."
 
         if "main.py" in implementation.file_dict:
             workflow_stdout = implementation.execute(env=de, entry="python main.py")
+            workflow_stdout = re.sub(r"=== Start of EDA part ===(.*)=== End of EDA part ===", "", workflow_stdout)
         else:
             workflow_stdout = None
 
@@ -73,6 +80,7 @@ class DataLoaderCoSTEEREvaluator(CoSTEEREvaluator):
         )
         user_prompt = T(".prompts:data_loader_eval.user").r(
             stdout=stdout,
+            eda_output=eda_output,
             workflow_stdout=workflow_stdout,
         )
 

rdagent/components/coder/data_science/raw_data_loader/prompts.yaml

@@ -28,7 +28,7 @@ spec:
     
     Your specifications should include only the function definition and docstring, without any code implementation or inline comments.
 
-    ----------- Competition Information -----------
+    ## Competition Information for This Task
     {{ competition_info }}
 
     ----------- Folder Description (All path are relative to the data folder) ---------
@@ -69,9 +69,15 @@ spec:
       4. Notes:
         - Update `DT` (data type) based on the specific competition dataset. This can include `pd.DataFrame`, `np.array`, `torch.Tensor`, etc.
         - Never use sample submission as the test index, as it may not be the same as the test data. Use the test index file or test data source to get the test index.
+        - Only set the DT of variables without inferring the shape of these variables since you don't know the shape of the data.
+      
+      5. Exploratory Data Analysis (EDA) part(Required):
+        - Before returning the data, you should always add an EDA part describing the data to help the following steps understand the data better.
+        - The EDA part should be drafted in plain text with certain format schema with no more than ten thousand characters.
+        - An evaluation agent will help to check whether the EDA part is added correctly.
 
       {% if latest_spec %}
-      5. Former Specification:
+      6. Former Specification:
         {{ latest_spec }}
         You should follow the provided specifications to improve this task.
       {% endif %}
@@ -123,6 +129,7 @@ spec:
         - Extend or adjust domain-specific transformations based on competition requirements.
         - The device has GPU support, so you can use it for feature engineering if necessary to accelerate the process.
         - Multi processing or parallel processing can be used to speed up the feature engineering process.
+        - Only set the DT of variables without inferring the shape of these variables since you don't know the shape of the data.
       
       {% if latest_spec %}
       5. Former Specification:
@@ -229,6 +236,7 @@ spec:
       4. Notes:
         - Align `DT` (data type) definitions with those used in model specifications.
         - Ensure flexibility to handle multiple ensemble strategies based on competition requirements.
+        - Only set the DT of variables without inferring the shape of these variables since you don't know the shape of the data.
 
       {% if latest_spec %}
       5. Former Specification:
@@ -334,7 +342,25 @@ data_loader_coder:
 
     ## Guidelines
     1. Ensure that the dataset is loaded strictly from `/kaggle/input/`, following the exact folder structure described in the **Data Folder Description**, and do not attempt to load data from the current directory (`./`).
-    3. You should avoid using logging module to output information in your generated code, and instead use the print() function.
+    2. You should avoid using logging module to output information in your generated code, and instead use the print() function.
+    
+    ## Exploratory Data Analysis (EDA) part(Required):
+    - Before returning the data, you should always add an EDA part describing the data to help the following steps understand the data better.
+    - The EDA part should include but not limited in the following information in plain text:
+      - The shape of the data.
+      - The first 5 rows of the data.
+      - The data types of each column.
+      - The number of missing values in each column.
+      - The number of unique values in each column.
+      - The distribution of the target variable.
+      - Any other information that you think is important for the following steps.
+    - The EDA part should be drafted in plain text sending to standard output with command print or other similar functions with no more than ten thousand characters in the following schema: 
+      === Start of EDA part ===
+      { You EDA output content }
+      === End of EDA part ===
+      User will use the following code to match: re.search(r"(.*?)=== Start of EDA part ===(.*)=== End of EDA part ===", stdout, re.DOTALL).groups()[1]
+    - An evaluation agent will help to check whether the EDA part is added correctly.
+    - During the EDA part, you should try to avoid any irrelevant information sending to the standard output.
 
     ## Output Format
     Please response the code in the following json format. Here is an example structure for the JSON output:
@@ -398,6 +424,23 @@ data_loader_eval:
     
     ## Evaluation Criteria
     You will be given the standard output (`stdout`) from the data loader test and, if applicable, the workflow test.
+
+    ## Exploratory Data Analysis (EDA) Part evaluation
+    - The code has also generated some EDA output to help understand the data better. 
+    - The EDA part should be drafted in plain text sending to standard output with command print or other similar functions with no more than ten thousand characters in the following schema: 
+      === Start of EDA part ===
+      { You EDA output content }
+      === End of EDA part ===
+      User will use the following code to match: re.search(r"(.*?)=== Start of EDA part ===(.*)=== End of EDA part ===", stdout, re.DOTALL).groups()[1]
+    - The EDA part should include but not limited in the following information in plain text:
+      - The shape of the data.
+      - The first 5 rows of the data.
+      - The data types of each column.
+      - The number of missing values in each column.
+      - The number of unique values in each column.
+      - The distribution of the target variable.
+      - Any other information that you think is important for the following steps.
+     You will be given the EDA output, your job is to check whether the output contains the required and sufficient information. If no EDA output is provided, you should consider it as a failure. Put this evaluation result in the return_checking part.
     
     Your response must follow this structured JSON format:
     ```json
@@ -412,6 +455,12 @@ data_loader_eval:
   user: |-
     --------- Data loader test stdout ---------
     {{ stdout }}   
+    --------- Data loader EDA stdout ---------
+    {% if eda_output is not none %}
+    {{ eda_output }}
+    {% else %}
+    No EDA output is provided.
+    {% endif %}
     {% if workflow_stdout is not none %}
     --------- Whole workflow test stdout ---------
     {{ workflow_stdout }}

rdagent/components/coder/data_science/workflow/__init__.py

@@ -31,7 +31,6 @@ class WorkflowMultiProcessEvolvingStrategy(MultiProcessEvolvingStrategy):
         workspace: FBWorkspace | None = None,
         prev_task_feedback: CoSTEERSingleFeedback | None = None,
     ) -> dict[str, str]:
-        # competition_info = self.scen.competition_descriptions
         workflow_information_str = target_task.get_task_information()
 
         # 1. query
@@ -57,7 +56,7 @@ class WorkflowMultiProcessEvolvingStrategy(MultiProcessEvolvingStrategy):
         # 2. code
         system_prompt = T(".prompts:workflow_coder.system").r(
             task_desc=workflow_information_str,
-            competition_info=self.scen.get_competition_full_desc(),
+            competition_info=self.scen.get_scenario_all_desc(),
             queried_similar_successful_knowledge=queried_similar_successful_knowledge,
             queried_former_failed_knowledge=queried_former_failed_knowledge[0],
         )

rdagent/components/coder/data_science/workflow/eval.py

@@ -70,10 +70,10 @@ class WorkflowGeneralCaseSpecEvaluator(CoSTEEREvaluator):
         # mde.prepare()
 
         # Clean the scores.csv & submission.csv.
-        stdout = implementation.execute(env=de, entry=f"rm submission.csv scores.csv")
+        implementation.execute(env=de, entry=f"rm submission.csv scores.csv")
 
-        fname = "main.py"
-        stdout = implementation.execute(env=de, entry=f"python {fname}")
+        stdout = implementation.execute(env=de, entry=f"python main.py")
+        stdout = re.sub(r"=== Start of EDA part ===(.*)=== End of EDA part ===", "", stdout)
 
         # Check score file
         score_fp = implementation.workspace_path / "scores.csv"