refactor: use dynamic input path and update template loader (#792)

* refactor: use dynamic input path and update template loader

* fix: update include syntax for data source in prompts.yaml

* add customization path

* docs: update prompts for ensemble scoring and metric direction

* chore: remove obsolete data_science/share.yaml file

rdagent/components/coder/data_science/ensemble/eval.py

@@ -47,7 +47,11 @@ class EnsembleCoSTEEREvaluator(CoSTEEREvaluator):
             )
 
         env = get_ds_env(
-            extra_volumes={f"{DS_RD_SETTING.local_data_path}/sample/{self.scen.competition}": "/kaggle/input"}
+            extra_volumes={
+                f"{DS_RD_SETTING.local_data_path}/sample/{self.scen.competition}": T(
+                    "scenarios.data_science.share:scen.input_path"
+                ).r()
+            }
         )
 
         fname = "test/ensemble_test.txt"

rdagent/components/coder/data_science/feature/eval.py

@@ -43,7 +43,11 @@ class FeatureCoSTEEREvaluator(CoSTEEREvaluator):
             )
 
         env = get_ds_env(
-            extra_volumes={f"{DS_RD_SETTING.local_data_path}/sample/{self.scen.competition}": "/kaggle/input"}
+            extra_volumes={
+                f"{DS_RD_SETTING.local_data_path}/sample/{self.scen.competition}": T(
+                    "scenarios.data_science.share:scen.input_path"
+                ).r()
+            }
         )
 
         # TODO: do we need to clean the generated temporary content?

rdagent/components/coder/data_science/model/eval.py

@@ -57,7 +57,11 @@ class ModelGeneralCaseSpecEvaluator(CoSTEEREvaluator):
             )
 
         env = get_ds_env(
-            extra_volumes={f"{DS_RD_SETTING.local_data_path}/sample/{self.scen.competition}": "/kaggle/input"}
+            extra_volumes={
+                f"{DS_RD_SETTING.local_data_path}/sample/{self.scen.competition}": T(
+                    "scenarios.data_science.share:scen.input_path"
+                ).r()
+            }
         )
 
         if_model_removed = False

rdagent/components/coder/data_science/pipeline/eval.py

@@ -52,7 +52,11 @@ class PipelineCoSTEEREvaluator(CoSTEEREvaluator):
             )
 
         env = get_ds_env(
-            extra_volumes={f"{DS_RD_SETTING.local_data_path}/sample/{self.scen.competition}": "/kaggle/input"}
+            extra_volumes={
+                f"{DS_RD_SETTING.local_data_path}/sample/{self.scen.competition}": T(
+                    "scenarios.data_science.share:scen.input_path"
+                ).r()
+            }
         )
 
         # Clean the scores.csv & submission.csv.
@@ -97,7 +101,7 @@ class PipelineCoSTEEREvaluator(CoSTEEREvaluator):
                 score_ret_code = 1
 
         # Check submission file
-        base_check_code = (DIRNAME / "eval_tests" / "submission_format_test.txt").read_text()
+        base_check_code = T(".eval_tests.submission_format_test", ftype="txt").r()
         implementation.inject_files(**{"test/submission_format_test.py": base_check_code})
         # stdout += "----Submission Check 1-----\n"
         submission_check_out, submission_ret_code = implementation.execute_ret_code(

rdagent/components/coder/data_science/pipeline/eval_tests/submission_format_test.txt

@@ -25,13 +25,13 @@ find . | grep -i sample | grep -i submission | grep -v sample_submission.csv | g
 """
 
 # Find sample submission file dynamically
-input_dir = Path("/kaggle/input")
+input_dir = Path("{% include "scenarios.data_science.share:scen.input_path" %}")
 # Look for common variations of sample submission filenames
 sample_submission_files = list(input_dir.glob("*sample_submission*.csv")) + list(
     input_dir.glob("*sampleSubmission*.csv")
 )
 
-assert sample_submission_files, "Error: No sample submission file found in /kaggle/input/"
+assert sample_submission_files, "Error: No sample submission file found in {% include "scenarios.data_science.share:scen.input_path" %}"
 
 # Use first matching file
 sample_submission_name = sample_submission_files[0].name

rdagent/components/coder/data_science/pipeline/prompts.yaml

@@ -37,7 +37,7 @@ pipeline_coder:
 
 
     ## Guidelines
-    1. Ensure that the dataset is loaded strictly from `/kaggle/input/`, following the exact folder structure described in the **Data Folder Description**, and do not attempt to load data from the current directory (`./`).
+    1. Ensure that the dataset is loaded strictly from `{% include "scenarios.data_science.share:scen.input_path" %}`, following the exact folder structure described in the **Data Folder Description**, and do not attempt to load data from the current directory (`./`).
     2. You should avoid using logging module to output information in your generated code, and instead use the print() function.
     
     ## Exploratory Data Analysis (EDA) part(Required):

rdagent/components/coder/data_science/raw_data_loader/__init__.py

@@ -226,7 +226,11 @@ class DataLoaderCoSTEER(CoSTEER):
         new_exp = super().develop(exp)
 
         env = get_ds_env(
-            extra_volumes={f"{DS_RD_SETTING.local_data_path}/{self.scen.competition}": "/kaggle/input"},
+            extra_volumes={
+                f"{DS_RD_SETTING.local_data_path}/{self.scen.competition}": T(
+                    "scenarios.data_science.share:scen.input_path"
+                ).r()
+            },
             running_timeout_period=DS_RD_SETTING.full_timeout,
         )
 

rdagent/components/coder/data_science/raw_data_loader/eval.py

@@ -46,7 +46,11 @@ class DataLoaderCoSTEEREvaluator(CoSTEEREvaluator):
             )
 
         env = get_ds_env(
-            extra_volumes={f"{DS_RD_SETTING.local_data_path}/sample/{self.scen.competition}": "/kaggle/input"}
+            extra_volumes={
+                f"{DS_RD_SETTING.local_data_path}/sample/{self.scen.competition}": T(
+                    "scenarios.data_science.share:scen.input_path"
+                ).r()
+            }
         )
 
         # TODO: do we need to clean the generated temporary content?

rdagent/components/coder/data_science/raw_data_loader/prompts.yaml

@@ -48,7 +48,7 @@ spec:
           - `test_ids` (DT): Identifiers for the test data.
         - Docstring Requirements:
           - Describe the purpose of the function.
-          - Specify the data source location (`/kaggle/input/`).
+          - Specify the data source location (`{% include "scenarios.data_science.share:scen.input_path" %}`).
           - Clearly define the structure and type of the output.
           - Inferred data shape to each input and output data variables. To uncertain dimension, use -1.
       2. Notes:
@@ -268,7 +268,7 @@ data_loader_coder:
     {% endif %}
 
     ## Guidelines
-    1. Ensure that the dataset is loaded strictly from `/kaggle/input/`, following the exact folder structure described in the **Data Folder Description**, and do not attempt to load data from the current directory (`./`).
+    1. Ensure that the dataset is loaded strictly from `{% include "scenarios.data_science.share:scen.input_path" %}`, following the exact folder structure described in the **Data Folder Description**, and do not attempt to load data from the current directory (`./`).
     2. You should avoid using logging module to output information in your generated code, and instead use the print() function.
     3. You should use the following cache decorator to cache the results of the function:
     ```python

rdagent/components/coder/data_science/share/eval.py

@@ -45,7 +45,9 @@ class ModelDumpEvaluator(CoSTEEREvaluator):
             )
         env = get_ds_env()
         env.conf.extra_volumes = {
-            f"{DS_RD_SETTING.local_data_path}/{'sample/' if self.data_type == 'sample' else ''}{self.scen.competition}": "/kaggle/input"
+            f"{DS_RD_SETTING.local_data_path}/{'sample/' if self.data_type == 'sample' else ''}{self.scen.competition}": T(
+                "scenarios.data_science.share:scen.input_path"
+            ).r()
         }
 
         # 2) check the result and stdout after reruning the model.

rdagent/components/coder/data_science/share/prompts.yaml

@@ -55,7 +55,7 @@ dump_model_eval:
 
 docdev:
   system: |-
-    You are a skilled developer and a Kaggle grandmaster. Your task is to create documentation for a data science solution.
+    {% include "scenarios.data_science.share:scen.role" %}  Your task is to create documentation for a data science solution.
 
     You will be given:
     - a list of files in the folder.

rdagent/components/coder/data_science/workflow/eval.py

@@ -55,7 +55,11 @@ class WorkflowGeneralCaseSpecEvaluator(CoSTEEREvaluator):
             )
 
         env = get_ds_env(
-            extra_volumes={f"{DS_RD_SETTING.local_data_path}/sample/{self.scen.competition}": "/kaggle/input"}
+            extra_volumes={
+                f"{DS_RD_SETTING.local_data_path}/sample/{self.scen.competition}": T(
+                    "scenarios.data_science.share:scen.input_path"
+                ).r()
+            }
         )
 
         # # DockerEnv for MLEBench submission validation
@@ -119,7 +123,7 @@ class WorkflowGeneralCaseSpecEvaluator(CoSTEEREvaluator):
                 score_ret_code = 1
 
         # Check submission file
-        base_check_code = (DIRNAME / "eval_tests" / "submission_format_test.txt").read_text()
+        base_check_code = T(".eval_tests.submission_format_test", ftype="txt").r()
         implementation.inject_files(**{"test/submission_format_test.py": base_check_code})
         # stdout += "----Submission Check 1-----\n"
         submission_check_out, submission_ret_code = implementation.execute_ret_code(

rdagent/components/coder/data_science/workflow/eval_tests/submission_format_test.txt

@@ -22,12 +22,12 @@ find . | grep -i sample | grep -i submission | grep -v sample_submission.csv | g
 """
 
 # Find sample submission file dynamically
-input_dir = Path("/kaggle/input")
+input_dir = Path("{% include "scenarios.data_science.share:scen.input_path" %}")
 # Look for common variations of sample submission filenames
 sample_submission_files = list(input_dir.glob("*sample_submission*.csv")) + \
                          list(input_dir.glob("*sampleSubmission*.csv"))
 
-assert sample_submission_files, "Error: No sample submission file found in /kaggle/input/"
+assert sample_submission_files, "Error: No sample submission file found in {% include "scenarios.data_science.share:scen.input_path" %}"
 
 # Use first matching file
 sample_submission_name = sample_submission_files[0].name