refactor: add custom data setting for data science scene (#967)

* add custom data setting for the data science scene

* fix ci?

* fix ci

* add custom data as an example

* fix ci

* add package

* fix test_import ci error

rdagent/app/data_science/conf.py

@@ -12,7 +12,11 @@ class DataScienceBasePropSetting(KaggleBasePropSetting):
     # Main components
     ## Scen
     scen: str = "rdagent.scenarios.data_science.scen.KaggleScen"
-    """Scenario class for data mining model"""
+    """
+    Scenario class for data science tasks.
+    - For Kaggle competitions, use: "rdagent.scenarios.data_science.scen.KaggleScen"
+    - For custom data science scenarios, use: "rdagent.scenarios.data_science.scen.DataScienceScen"
+    """
 
     hypothesis_gen: str = "rdagent.scenarios.data_science.proposal.exp_gen.proposal.DSProposalV2ExpGen"
     """Hypothesis generation class"""

rdagent/components/coder/model_coder/evolving_strategy.py

@@ -10,10 +10,7 @@ from rdagent.components.coder.CoSTEER.knowledge_management import (
     CoSTEERQueriedKnowledge,
     CoSTEERQueriedKnowledgeV2,
 )
-from rdagent.components.coder.model_coder.model import (
-    ModelFBWorkspace,
-    ModelTask,
-)
+from rdagent.components.coder.model_coder.model import ModelFBWorkspace, ModelTask
 from rdagent.core.experiment import FBWorkspace
 from rdagent.oai.llm_conf import LLM_SETTINGS
 from rdagent.oai.llm_utils import APIBackend

rdagent/scenarios/data_science/example/arf-12-hour-prediction-task/description.md

@@ -0,0 +1,62 @@
+# ARF 12-Hour Prediction Task
+
+## Overview
+
+### Description
+
+Acute Respiratory Failure (ARF) is a life-threatening condition that often develops rapidly in critically ill patients. Accurate early prediction of ARF is crucial in intensive care units (ICUs) to enable timely clinical interventions and resource allocation. In this task, you are asked to build a machine learning model that predicts whether a patient will develop ARF within the next **12 hours**, based on multivariate clinical time series data.
+
+The dataset is extracted from electronic health records (EHRs) and preprocessed using the **FIDDLE** pipeline to generate structured temporal features for each patient.
+
+### Objective
+
+Your goal is to develop a binary classification model that takes a 12-hour time series as input and outputs the probability of ARF onset in the following 12 hours.
+
+---
+
+## Data Description
+
+The dataset is divided into two directories:
+
+* `train/`
+
+  * `ARF_12h.csv`: `ID` & `ARF_ONSET_HOUR` & Binary labels (`ARF_LABEL`) for training samples.
+  * `X.npz`: 3D sparse array of shape `(N, T, D)`:
+
+    * `N`: number of training samples
+    * `T`: time steps (one per hour)
+    * `D`: number of features
+
+* `test/`
+
+  * `ARF_12h.csv`: `ID` & `ARF_ONSET_HOUR`.
+  * `X.npz`: Test feature set in the same format as training data.
+
+The `.npz` files store sparse matrices and are loaded using the [`sparse`](https://github.com/pydata/sparse) library. Each matrix is converted to dense format before model input. (DO NOT USE scipy.sparse)
+e.g. 
+```
+import sparse
+X = sparse.load_npz("<url>").todense()
+```
+Then, you can use `X.transpose(0, 2, 1)` to transpose the shape of X from (N, T, D) to (N, D, T)
+
+---
+
+## Modeling
+
+Each sample is a multivariate time series representing 12 hours of clinical observations. Your model should learn temporal and cross-feature dependencies to estimate the likelihood of ARF.
+
+* **Output**: Probability score ∈ \[0, 1]
+* **Loss Function**: `CrossEntropyLoss` or equivalent
+* **Evaluation Metric**: **AUROC** (Area Under the Receiver Operating Characteristic Curve)
+
+---
+
+## Submission Format
+```
+ID,ARF_LABEL
+0,0.473
+1,0.652
+2,0.129
+...
+```

rdagent/scenarios/data_science/example/arf-12-hour-prediction-task/sample.py

@@ -0,0 +1,71 @@
+import shutil
+from pathlib import Path
+
+import numpy as np
+import pandas as pd
+import sparse
+from tqdm import tqdm
+
+
+def sample_and_copy_subfolder(
+    input_dir: Path,
+    output_dir: Path,
+    min_frac: float,
+    min_num: int,
+    seed: int = 42,
+):
+    np.random.seed(seed)
+
+    feature_path = input_dir / "X.npz"
+    label_path = input_dir / "ARF_12h.csv"
+
+    X_sparse = sparse.load_npz(feature_path)
+    df_label = pd.read_csv(label_path)
+
+    N = X_sparse.shape[0]
+    n_keep = max(int(N * min_frac), min_num)
+    idx = np.random.choice(N, n_keep, replace=False)
+
+    X_sample = X_sparse[idx]
+    df_sample = df_label.iloc[idx].reset_index(drop=True)
+
+    output_dir.mkdir(parents=True, exist_ok=True)
+    sparse.save_npz(output_dir / "X.npz", X_sample)
+    df_sample.to_csv(output_dir / "ARF_12h.csv", index=False)
+
+    print(f"[INFO] Sampled {n_keep} of {N} from {input_dir.name}")
+
+
+def create_debug_data(
+    dataset_path: str,
+    output_path: str,
+    min_frac: float = 0.02,
+    min_num: int = 10,
+):
+    dataset_root = Path(dataset_path) / "arf-12-hour-prediction-task"
+    output_root = Path(output_path)
+
+    for sub in ["train", "test"]:
+        input_dir = dataset_root / sub
+        output_dir = output_root / sub
+        print(f"\n[PROCESS] {sub} subset")
+        sample_and_copy_subfolder(
+            input_dir=input_dir,
+            output_dir=output_dir,
+            min_frac=min_frac,
+            min_num=min_num,
+            seed=42 if sub == "train" else 123,
+        )
+
+    print(f"\n[INFO] Sampling complete → Output in: {output_root}")
+
+
+if __name__ == "__main__" or globals().get("__name__") == "<run_path>":
+    dataset_path = globals().get("dataset_path", "./")
+    output_path = globals().get("output_path", "./sample")
+    create_debug_data(
+        dataset_path=dataset_path,
+        output_path=output_path,
+        min_frac=0.02,
+        min_num=10,
+    )

rdagent/scenarios/data_science/example/eval/arf-12-hour-prediction-task/grade.py

@@ -0,0 +1,75 @@
+import json
+
+import pandas as pd
+from sklearn.metrics import roc_auc_score
+
+
+class InvalidSubmissionError(Exception):
+    """
+    A custom exception for when the agent submission cannot be graded.
+    """
+
+    pass
+
+
+def prepare_for_auroc_metric(submission: pd.DataFrame, answers: pd.DataFrame, id_col: str, target_col: str) -> dict:
+
+    # Answers checks
+    assert id_col in answers.columns, f"answers dataframe should have an {id_col} column"
+    assert target_col in answers.columns, f"answers dataframe should have a {target_col} column"
+
+    # Submission checks
+    if id_col not in submission.columns:
+        raise InvalidSubmissionError(f"Submission should have an {id_col} column")
+    if target_col not in submission.columns:
+        raise InvalidSubmissionError(f"Submission should have a {target_col} column")
+    if len(submission) != len(answers):
+        raise InvalidSubmissionError(f"Submission and answers should have the same number of rows")
+    try:
+        pd.to_numeric(submission[target_col])
+    except ValueError:
+        raise InvalidSubmissionError(
+            f"Expected {target_col} column to be numeric, got {submission[target_col].dtype} instead"
+        )
+    if submission[target_col].min() < 0 or submission[target_col].max() > 1:
+        raise InvalidSubmissionError(
+            f"Submission {target_col} column should contain probabilities,"
+            " and therefore contain values between 0 and 1 inclusive"
+        )
+    # Sort
+    submission = submission.sort_values(id_col)
+    answers = answers.sort_values(id_col)
+
+    if (submission[id_col].values != answers[id_col].values).any():
+        raise InvalidSubmissionError(f"Submission and answers should have the same {id_col} values")
+
+    roc_auc_inputs = {
+        "y_true": answers[target_col].to_numpy(),
+        "y_score": submission[target_col].to_numpy(),
+    }
+
+    return roc_auc_inputs
+
+
+def grade(submission: pd.DataFrame, answers: pd.DataFrame) -> float:
+    roc_auc_inputs = prepare_for_auroc_metric(
+        submission=submission, answers=answers, id_col="ID", target_col="ARF_LABEL"
+    )
+    return roc_auc_score(y_true=roc_auc_inputs["y_true"], y_score=roc_auc_inputs["y_score"])
+
+
+if __name__ == "__main__":
+    submission_path = "submission.csv"
+    gt_submission_path = "submission_test.csv"
+    submission = pd.read_csv(submission_path)
+    answers = pd.read_csv(gt_submission_path)
+    score = grade(submission=submission, answers=answers)
+
+    print(
+        json.dumps(
+            {
+                "competition_id": "arf-12-hour-prediction-task",
+                "score": score,
+            }
+        )
+    )

rdagent/scenarios/data_science/example/eval/arf-12-hour-prediction-task/valid.py

@@ -0,0 +1,21 @@
+from pathlib import Path
+
+# Check if our submission file exists
+assert Path("submission.csv").exists(), "Error: submission.csv not found"
+
+submission_lines = Path("submission.csv").read_text().splitlines()
+test_lines = Path("submission_test.csv").read_text().splitlines()
+
+is_valid = len(submission_lines) == len(test_lines)
+
+if is_valid:
+    message = "submission.csv and submission_test.csv have the same number of lines."
+else:
+    message = (
+        f"submission.csv has {len(submission_lines)} lines, while submission_test.csv has {len(test_lines)} lines."
+    )
+
+print(message)
+
+if not is_valid:
+    raise AssertionError("Submission is invalid")

rdagent/scenarios/data_science/scen/__init__.py

@@ -1,4 +1,5 @@
 import json
+import runpy
 from pathlib import Path
 from typing import Dict
 
@@ -35,7 +36,17 @@ class DataScienceScen(Scenario):
         if DS_RD_SETTING.sample_data:
             self.debug_path = f"{local_path}/sample/{competition}"
             if not Path(self.debug_path).exists():
-                create_debug_data(competition, dataset_path=local_path)
+                sample_py_path = Path(local_path) / competition / "sample.py"
+                if sample_py_path.exists():
+                    runpy.run_path(
+                        str(sample_py_path),
+                        init_globals={
+                            "dataset_path": str(local_path),
+                            "output_path": str(self.debug_path),
+                        },
+                    )
+                else:
+                    create_debug_data(competition, dataset_path=local_path)
         else:
             self.debug_path = f"{local_path}/{competition}"
 

rdagent/scenarios/qlib/developer/feedback.py

@@ -5,11 +5,7 @@ from typing import Dict
 import pandas as pd
 
 from rdagent.core.experiment import Experiment
-from rdagent.core.proposal import (
-    Experiment2Feedback,
-    HypothesisFeedback,
-    Trace,
-)
+from rdagent.core.proposal import Experiment2Feedback, HypothesisFeedback, Trace
 from rdagent.log import rdagent_logger as logger
 from rdagent.oai.llm_utils import APIBackend
 from rdagent.scenarios.qlib.experiment.quant_experiment import QlibQuantScenario

requirements.txt

@@ -65,4 +65,7 @@ genson
 # mlflow
 mlflow
 azureml-mlflow
-types-pytz
+types-pytz
+
+# data science scenario for custom dataset
+sparse

test/utils/test_import.py

@@ -17,6 +17,8 @@ class TestRDAgentImports(unittest.TestCase):
     def import_all_modules_from_directory(directory):
         for file in directory.joinpath("rdagent").rglob("*.py"):
             fstr = str(file)
+            if "example" in fstr:
+                continue
             if "meta_tpl" in fstr:
                 continue
             if "template" in fstr or "tpl" in fstr: