feat: add eda to data science scenario (#639)

* add eda to data science scenario

* fix CI

rdagent/components/coder/data_science/feature/__init__.py

@@ -55,6 +55,7 @@ class FeatureMultiProcessEvolvingStrategy(MultiProcessEvolvingStrategy):
 
         # 2. code
         system_prompt = T(".prompts:feature_coder.system").r(
+            competition_info=self.scen.get_scenario_all_desc(),
             task_desc=feature_information_str,
             data_loader_code=workspace.file_dict.get("load_data.py"),
             queried_similar_successful_knowledge=queried_similar_successful_knowledge,

rdagent/components/coder/data_science/feature/eval.py

@@ -1,4 +1,5 @@
 import json
+import re
 from pathlib import Path
 
 from rdagent.app.data_science.conf import DS_RD_SETTING
@@ -60,6 +61,7 @@ class FeatureCoSTEEREvaluator(CoSTEEREvaluator):
 
         if "main.py" in implementation.file_dict:
             workflow_stdout = implementation.execute(env=de, entry="python main.py")
+            workflow_stdout = re.sub(r"=== Start of EDA part ===(.*)=== End of EDA part ===", "", workflow_stdout)
         else:
             workflow_stdout = None
 

rdagent/components/coder/data_science/feature/prompts.yaml

@@ -5,6 +5,9 @@ feature_coder:
 
     ## Task Description
     {{ task_desc }}
+    
+    ## Competition Information for This Task
+    {{ competition_info }}
 
     {% if queried_similar_successful_knowledge|length != 0 or queried_former_failed_knowledge|length != 0 %}
     ## Relevant Information for This Task
@@ -31,7 +34,8 @@ feature_coder:
 
     ## Guidelines
     1. If feature engineering is unnecessary or should be combined with model training, you may skip this step.
-    2. The function input is the output of the following data loader:
+    2. Be cautious of any column drop in the code. Dropping a column easily without any more attempts, it may not be a good practice.
+    3. The function input is the output of the following data loader:
     ```python
     {{ data_loader_code }}
     ```