diff --git a/rdagent/components/coder/data_science/ensemble/__init__.py b/rdagent/components/coder/data_science/ensemble/__init__.py index bba8f437..25515e75 100644 --- a/rdagent/components/coder/data_science/ensemble/__init__.py +++ b/rdagent/components/coder/data_science/ensemble/__init__.py @@ -74,6 +74,7 @@ class EnsembleMultiProcessEvolvingStrategy(MultiProcessEvolvingStrategy): queried_former_failed_knowledge=( queried_former_failed_knowledge[0] if queried_former_failed_knowledge else None ), + all_code=workspace.all_codes, ) user_prompt = T(".prompts:ensemble_coder.user").r( ensemble_spec=workspace.file_dict["spec/ensemble.md"], diff --git a/rdagent/components/coder/data_science/ensemble/eval.py b/rdagent/components/coder/data_science/ensemble/eval.py index 6abe6e72..96830882 100644 --- a/rdagent/components/coder/data_science/ensemble/eval.py +++ b/rdagent/components/coder/data_science/ensemble/eval.py @@ -1,4 +1,5 @@ import json +import re from pathlib import Path from jinja2 import Environment, StrictUndefined @@ -69,6 +70,7 @@ class EnsembleCoSTEEREvaluator(CoSTEEREvaluator): if "main.py" in implementation.file_dict: workflow_stdout = implementation.execute(env=de, entry="python main.py") + workflow_stdout = re.sub(r"=== Start of EDA part ===(.*)=== End of EDA part ===", "", workflow_stdout) else: workflow_stdout = None diff --git a/rdagent/components/coder/data_science/ensemble/prompts.yaml b/rdagent/components/coder/data_science/ensemble/prompts.yaml index 544fcaab..73ba14b6 100644 --- a/rdagent/components/coder/data_science/ensemble/prompts.yaml +++ b/rdagent/components/coder/data_science/ensemble/prompts.yaml @@ -35,7 +35,10 @@ ensemble_coder: {% endfor %} {% endif %} - You should avoid using logging module to output information in your generated code, and instead use the print() function. + ## Guidelines + 1. The function's code is associated with several other functions including a data loader, feature engineering, and model training. all codes are as follows: + {{ all_code }} + 2. You should avoid using logging module to output information in your generated code, and instead use the print() function. ## Output Format Please response the code in the following json format. Here is an example structure for the JSON output: diff --git a/rdagent/components/coder/data_science/feature/__init__.py b/rdagent/components/coder/data_science/feature/__init__.py index 10695d58..3871215e 100644 --- a/rdagent/components/coder/data_science/feature/__init__.py +++ b/rdagent/components/coder/data_science/feature/__init__.py @@ -55,6 +55,7 @@ class FeatureMultiProcessEvolvingStrategy(MultiProcessEvolvingStrategy): # 2. code system_prompt = T(".prompts:feature_coder.system").r( + competition_info=self.scen.get_scenario_all_desc(), task_desc=feature_information_str, data_loader_code=workspace.file_dict.get("load_data.py"), queried_similar_successful_knowledge=queried_similar_successful_knowledge, diff --git a/rdagent/components/coder/data_science/feature/eval.py b/rdagent/components/coder/data_science/feature/eval.py index db9b042e..818da0d9 100644 --- a/rdagent/components/coder/data_science/feature/eval.py +++ b/rdagent/components/coder/data_science/feature/eval.py @@ -1,4 +1,5 @@ import json +import re from pathlib import Path from rdagent.app.data_science.conf import DS_RD_SETTING @@ -60,6 +61,7 @@ class FeatureCoSTEEREvaluator(CoSTEEREvaluator): if "main.py" in implementation.file_dict: workflow_stdout = implementation.execute(env=de, entry="python main.py") + workflow_stdout = re.sub(r"=== Start of EDA part ===(.*)=== End of EDA part ===", "", workflow_stdout) else: workflow_stdout = None diff --git a/rdagent/components/coder/data_science/feature/prompts.yaml b/rdagent/components/coder/data_science/feature/prompts.yaml index fd2183c1..ccb02edf 100644 --- a/rdagent/components/coder/data_science/feature/prompts.yaml +++ b/rdagent/components/coder/data_science/feature/prompts.yaml @@ -5,6 +5,9 @@ feature_coder: ## Task Description {{ task_desc }} + + ## Competition Information for This Task + {{ competition_info }} {% if queried_similar_successful_knowledge|length != 0 or queried_former_failed_knowledge|length != 0 %} ## Relevant Information for This Task @@ -31,7 +34,8 @@ feature_coder: ## Guidelines 1. If feature engineering is unnecessary or should be combined with model training, you may skip this step. - 2. The function input is the output of the following data loader: + 2. Be cautious of any column drop in the code. Dropping a column easily without any more attempts, it may not be a good practice. + 3. The function input is the output of the following data loader: ```python {{ data_loader_code }} ``` diff --git a/rdagent/components/coder/data_science/model/__init__.py b/rdagent/components/coder/data_science/model/__init__.py index 8d55242c..7bd41eeb 100644 --- a/rdagent/components/coder/data_science/model/__init__.py +++ b/rdagent/components/coder/data_science/model/__init__.py @@ -56,6 +56,7 @@ class ModelMultiProcessEvolvingStrategy(MultiProcessEvolvingStrategy): # 2. code system_prompt = T(".prompts:model_coder.system").r( task_desc=model_information_str, + competition_info=self.scen.get_scenario_all_desc(), data_loader_code=workspace.file_dict.get("load_data.py"), feature_code=workspace.file_dict["feature.py"], queried_similar_successful_knowledge=queried_similar_successful_knowledge, diff --git a/rdagent/components/coder/data_science/model/eval.py b/rdagent/components/coder/data_science/model/eval.py index a3d425d9..df03341d 100644 --- a/rdagent/components/coder/data_science/model/eval.py +++ b/rdagent/components/coder/data_science/model/eval.py @@ -76,6 +76,7 @@ class ModelGeneralCaseSpecEvaluator(CoSTEEREvaluator): if "main.py" in implementation.file_dict: workflow_stdout = implementation.execute(env=de, entry="python main.py") + workflow_stdout = re.sub(r"=== Start of EDA part ===(.*)=== End of EDA part ===", "", workflow_stdout) else: workflow_stdout = None diff --git a/rdagent/components/coder/data_science/model/prompts.yaml b/rdagent/components/coder/data_science/model/prompts.yaml index 80ae4102..8979201e 100644 --- a/rdagent/components/coder/data_science/model/prompts.yaml +++ b/rdagent/components/coder/data_science/model/prompts.yaml @@ -5,6 +5,9 @@ model_coder: ## Task Description {{ task_desc }} + + ## Competition Information for This Task + {{ competition_info }} {% if queried_similar_successful_knowledge|length != 0 or queried_former_failed_knowledge|length != 0 %} ## Relevant Information for This Task diff --git a/rdagent/components/coder/data_science/raw_data_loader/__init__.py b/rdagent/components/coder/data_science/raw_data_loader/__init__.py index 5a1aaf5e..2d30ec3d 100644 --- a/rdagent/components/coder/data_science/raw_data_loader/__init__.py +++ b/rdagent/components/coder/data_science/raw_data_loader/__init__.py @@ -23,7 +23,9 @@ File structure """ import json +import re +from rdagent.app.data_science.conf import DS_RD_SETTING from rdagent.components.coder.CoSTEER import CoSTEER from rdagent.components.coder.CoSTEER.evaluators import ( CoSTEERMultiEvaluator, @@ -45,6 +47,7 @@ from rdagent.core.experiment import FBWorkspace from rdagent.core.scenario import Scenario from rdagent.oai.llm_utils import APIBackend from rdagent.utils.agent.tpl import T +from rdagent.utils.env import DockerEnv, DSDockerConf class DataLoaderMultiProcessEvolvingStrategy(MultiProcessEvolvingStrategy): @@ -193,3 +196,15 @@ class DataLoaderCoSTEER(CoSTEER): es = DataLoaderMultiProcessEvolvingStrategy(scen=scen, settings=settings) super().__init__(*args, settings=settings, eva=eva, es=es, evolving_version=2, scen=scen, **kwargs) + + def develop(self, exp): + new_exp = super().develop(exp) + + ds_docker_conf = DSDockerConf() + ds_docker_conf.extra_volumes = {f"{DS_RD_SETTING.local_data_path}/{self.scen.competition}": "/kaggle/input"} + de = DockerEnv(conf=ds_docker_conf) + stdout = new_exp.experiment_workspace.execute(env=de, entry=f"python test/data_loader_test.py") + match = re.search(r"(.*?)=== Start of EDA part ===(.*)=== End of EDA part ===", stdout, re.DOTALL) + eda_output = match.groups()[1] if match else None + self.scen.eda_output = eda_output + return new_exp diff --git a/rdagent/components/coder/data_science/raw_data_loader/eval.py b/rdagent/components/coder/data_science/raw_data_loader/eval.py index c023bf23..335022f4 100644 --- a/rdagent/components/coder/data_science/raw_data_loader/eval.py +++ b/rdagent/components/coder/data_science/raw_data_loader/eval.py @@ -1,6 +1,7 @@ # tess successfully running. # (GPT) if it aligns with the spec & rationality of the spec. import json +import re from pathlib import Path from rdagent.app.data_science.conf import DS_RD_SETTING @@ -58,9 +59,15 @@ class DataLoaderCoSTEEREvaluator(CoSTEEREvaluator): test_code = (DIRNAME / "eval_tests" / "data_loader_test.txt").read_text() implementation.inject_files(**{fname: test_code}) stdout = implementation.execute(env=de, entry=f"python {fname}") + match = re.search(r"(.*?)=== Start of EDA part ===(.*)=== End of EDA part ===(.*)", stdout, re.DOTALL) + stdout_part_1, eda_output, stdout_part_2 = match.groups() if match else (stdout, None, "") + stdout = stdout_part_1 + stdout_part_2 + if eda_output is not None and len(eda_output.split(" ")) > 10000: + eda_output += "Length of EDA output is too long, truncated. Please reject this implementation and motivate it to reduce the length of EDA output." if "main.py" in implementation.file_dict: workflow_stdout = implementation.execute(env=de, entry="python main.py") + workflow_stdout = re.sub(r"=== Start of EDA part ===(.*)=== End of EDA part ===", "", workflow_stdout) else: workflow_stdout = None @@ -73,6 +80,7 @@ class DataLoaderCoSTEEREvaluator(CoSTEEREvaluator): ) user_prompt = T(".prompts:data_loader_eval.user").r( stdout=stdout, + eda_output=eda_output, workflow_stdout=workflow_stdout, ) diff --git a/rdagent/components/coder/data_science/raw_data_loader/prompts.yaml b/rdagent/components/coder/data_science/raw_data_loader/prompts.yaml index 21629690..34257f14 100644 --- a/rdagent/components/coder/data_science/raw_data_loader/prompts.yaml +++ b/rdagent/components/coder/data_science/raw_data_loader/prompts.yaml @@ -28,7 +28,7 @@ spec: Your specifications should include only the function definition and docstring, without any code implementation or inline comments. - ----------- Competition Information ----------- + ## Competition Information for This Task {{ competition_info }} ----------- Folder Description (All path are relative to the data folder) --------- @@ -69,9 +69,15 @@ spec: 4. Notes: - Update `DT` (data type) based on the specific competition dataset. This can include `pd.DataFrame`, `np.array`, `torch.Tensor`, etc. - Never use sample submission as the test index, as it may not be the same as the test data. Use the test index file or test data source to get the test index. + - Only set the DT of variables without inferring the shape of these variables since you don't know the shape of the data. + + 5. Exploratory Data Analysis (EDA) part(Required): + - Before returning the data, you should always add an EDA part describing the data to help the following steps understand the data better. + - The EDA part should be drafted in plain text with certain format schema with no more than ten thousand characters. + - An evaluation agent will help to check whether the EDA part is added correctly. {% if latest_spec %} - 5. Former Specification: + 6. Former Specification: {{ latest_spec }} You should follow the provided specifications to improve this task. {% endif %} @@ -123,6 +129,7 @@ spec: - Extend or adjust domain-specific transformations based on competition requirements. - The device has GPU support, so you can use it for feature engineering if necessary to accelerate the process. - Multi processing or parallel processing can be used to speed up the feature engineering process. + - Only set the DT of variables without inferring the shape of these variables since you don't know the shape of the data. {% if latest_spec %} 5. Former Specification: @@ -229,6 +236,7 @@ spec: 4. Notes: - Align `DT` (data type) definitions with those used in model specifications. - Ensure flexibility to handle multiple ensemble strategies based on competition requirements. + - Only set the DT of variables without inferring the shape of these variables since you don't know the shape of the data. {% if latest_spec %} 5. Former Specification: @@ -334,7 +342,25 @@ data_loader_coder: ## Guidelines 1. Ensure that the dataset is loaded strictly from `/kaggle/input/`, following the exact folder structure described in the **Data Folder Description**, and do not attempt to load data from the current directory (`./`). - 3. You should avoid using logging module to output information in your generated code, and instead use the print() function. + 2. You should avoid using logging module to output information in your generated code, and instead use the print() function. + + ## Exploratory Data Analysis (EDA) part(Required): + - Before returning the data, you should always add an EDA part describing the data to help the following steps understand the data better. + - The EDA part should include but not limited in the following information in plain text: + - The shape of the data. + - The first 5 rows of the data. + - The data types of each column. + - The number of missing values in each column. + - The number of unique values in each column. + - The distribution of the target variable. + - Any other information that you think is important for the following steps. + - The EDA part should be drafted in plain text sending to standard output with command print or other similar functions with no more than ten thousand characters in the following schema: + === Start of EDA part === + { You EDA output content } + === End of EDA part === + User will use the following code to match: re.search(r"(.*?)=== Start of EDA part ===(.*)=== End of EDA part ===", stdout, re.DOTALL).groups()[1] + - An evaluation agent will help to check whether the EDA part is added correctly. + - During the EDA part, you should try to avoid any irrelevant information sending to the standard output. ## Output Format Please response the code in the following json format. Here is an example structure for the JSON output: @@ -398,6 +424,23 @@ data_loader_eval: ## Evaluation Criteria You will be given the standard output (`stdout`) from the data loader test and, if applicable, the workflow test. + + ## Exploratory Data Analysis (EDA) Part evaluation + - The code has also generated some EDA output to help understand the data better. + - The EDA part should be drafted in plain text sending to standard output with command print or other similar functions with no more than ten thousand characters in the following schema: + === Start of EDA part === + { You EDA output content } + === End of EDA part === + User will use the following code to match: re.search(r"(.*?)=== Start of EDA part ===(.*)=== End of EDA part ===", stdout, re.DOTALL).groups()[1] + - The EDA part should include but not limited in the following information in plain text: + - The shape of the data. + - The first 5 rows of the data. + - The data types of each column. + - The number of missing values in each column. + - The number of unique values in each column. + - The distribution of the target variable. + - Any other information that you think is important for the following steps. + You will be given the EDA output, your job is to check whether the output contains the required and sufficient information. If no EDA output is provided, you should consider it as a failure. Put this evaluation result in the return_checking part. Your response must follow this structured JSON format: ```json @@ -412,6 +455,12 @@ data_loader_eval: user: |- --------- Data loader test stdout --------- {{ stdout }} + --------- Data loader EDA stdout --------- + {% if eda_output is not none %} + {{ eda_output }} + {% else %} + No EDA output is provided. + {% endif %} {% if workflow_stdout is not none %} --------- Whole workflow test stdout --------- {{ workflow_stdout }} diff --git a/rdagent/components/coder/data_science/workflow/__init__.py b/rdagent/components/coder/data_science/workflow/__init__.py index 0cc1708f..78179560 100644 --- a/rdagent/components/coder/data_science/workflow/__init__.py +++ b/rdagent/components/coder/data_science/workflow/__init__.py @@ -31,7 +31,6 @@ class WorkflowMultiProcessEvolvingStrategy(MultiProcessEvolvingStrategy): workspace: FBWorkspace | None = None, prev_task_feedback: CoSTEERSingleFeedback | None = None, ) -> dict[str, str]: - # competition_info = self.scen.competition_descriptions workflow_information_str = target_task.get_task_information() # 1. query @@ -57,7 +56,7 @@ class WorkflowMultiProcessEvolvingStrategy(MultiProcessEvolvingStrategy): # 2. code system_prompt = T(".prompts:workflow_coder.system").r( task_desc=workflow_information_str, - competition_info=self.scen.get_competition_full_desc(), + competition_info=self.scen.get_scenario_all_desc(), queried_similar_successful_knowledge=queried_similar_successful_knowledge, queried_former_failed_knowledge=queried_former_failed_knowledge[0], ) diff --git a/rdagent/components/coder/data_science/workflow/eval.py b/rdagent/components/coder/data_science/workflow/eval.py index abc888c6..255bf694 100644 --- a/rdagent/components/coder/data_science/workflow/eval.py +++ b/rdagent/components/coder/data_science/workflow/eval.py @@ -70,10 +70,10 @@ class WorkflowGeneralCaseSpecEvaluator(CoSTEEREvaluator): # mde.prepare() # Clean the scores.csv & submission.csv. - stdout = implementation.execute(env=de, entry=f"rm submission.csv scores.csv") + implementation.execute(env=de, entry=f"rm submission.csv scores.csv") - fname = "main.py" - stdout = implementation.execute(env=de, entry=f"python {fname}") + stdout = implementation.execute(env=de, entry=f"python main.py") + stdout = re.sub(r"=== Start of EDA part ===(.*)=== End of EDA part ===", "", stdout) # Check score file score_fp = implementation.workspace_path / "scores.csv" diff --git a/rdagent/scenarios/data_science/dev/runner/eval.py b/rdagent/scenarios/data_science/dev/runner/eval.py index 405c378b..9aceed34 100644 --- a/rdagent/scenarios/data_science/dev/runner/eval.py +++ b/rdagent/scenarios/data_science/dev/runner/eval.py @@ -1,5 +1,6 @@ import json import os +import re from pathlib import Path from rdagent.app.data_science.conf import DS_RD_SETTING @@ -43,6 +44,7 @@ class DSCoSTEERCoSTEEREvaluator(CoSTEEREvaluator): # execute workflow stdout = implementation.execute(env=de, entry="coverage run main.py") + stdout = re.sub(r"=== Start of EDA part ===(.*)=== End of EDA part ===", "", stdout) score_fp = implementation.workspace_path / "scores.csv" if not score_fp.exists(): diff --git a/rdagent/scenarios/data_science/scen/__init__.py b/rdagent/scenarios/data_science/scen/__init__.py index f53c3c1e..586ac8b4 100644 --- a/rdagent/scenarios/data_science/scen/__init__.py +++ b/rdagent/scenarios/data_science/scen/__init__.py @@ -232,6 +232,7 @@ class DataScienceScen(Scenario): self.processed_data_folder_description = self._get_data_folder_description() self._analysis_competition_description() self.metric_direction = self._get_direction() + self.eda_output = None def _get_description(self): if (fp := Path(f"{DS_RD_SETTING.local_data_path}/{self.competition}.json")).exists(): @@ -306,6 +307,7 @@ class DataScienceScen(Scenario): submission_specifications=self.submission_specifications, evaluation=self.target_description, metric_direction=self.metric_direction, + eda_output=self.eda_output, ) def get_runtime_environment(self) -> str: diff --git a/rdagent/scenarios/data_science/scen/prompts.yaml b/rdagent/scenarios/data_science/scen/prompts.yaml index 40fa4b0d..af3c3abb 100644 --- a/rdagent/scenarios/data_science/scen/prompts.yaml +++ b/rdagent/scenarios/data_science/scen/prompts.yaml @@ -15,6 +15,10 @@ scenario_description: |- {% else %}The metric is better when it is smaller. {% endif %} + {% if eda_output is not none %} + {{ eda_output }} + {% endif %} + competition_description_template: system: |- You are a data science assistant that extracts structured information from unstructured text.