diff --git a/rdagent/app/data_science/conf.py b/rdagent/app/data_science/conf.py index fa770888..a0c88319 100644 --- a/rdagent/app/data_science/conf.py +++ b/rdagent/app/data_science/conf.py @@ -12,7 +12,11 @@ class DataScienceBasePropSetting(KaggleBasePropSetting): # Main components ## Scen scen: str = "rdagent.scenarios.data_science.scen.KaggleScen" - """Scenario class for data mining model""" + """ + Scenario class for data science tasks. + - For Kaggle competitions, use: "rdagent.scenarios.data_science.scen.KaggleScen" + - For custom data science scenarios, use: "rdagent.scenarios.data_science.scen.DataScienceScen" + """ hypothesis_gen: str = "rdagent.scenarios.data_science.proposal.exp_gen.proposal.DSProposalV2ExpGen" """Hypothesis generation class""" diff --git a/rdagent/components/coder/model_coder/evolving_strategy.py b/rdagent/components/coder/model_coder/evolving_strategy.py index 24f84388..9605d915 100644 --- a/rdagent/components/coder/model_coder/evolving_strategy.py +++ b/rdagent/components/coder/model_coder/evolving_strategy.py @@ -10,10 +10,7 @@ from rdagent.components.coder.CoSTEER.knowledge_management import ( CoSTEERQueriedKnowledge, CoSTEERQueriedKnowledgeV2, ) -from rdagent.components.coder.model_coder.model import ( - ModelFBWorkspace, - ModelTask, -) +from rdagent.components.coder.model_coder.model import ModelFBWorkspace, ModelTask from rdagent.core.experiment import FBWorkspace from rdagent.oai.llm_conf import LLM_SETTINGS from rdagent.oai.llm_utils import APIBackend diff --git a/rdagent/scenarios/data_science/example/arf-12-hour-prediction-task/description.md b/rdagent/scenarios/data_science/example/arf-12-hour-prediction-task/description.md new file mode 100644 index 00000000..f3a49095 --- /dev/null +++ b/rdagent/scenarios/data_science/example/arf-12-hour-prediction-task/description.md @@ -0,0 +1,62 @@ +# ARF 12-Hour Prediction Task + +## Overview + +### Description + +Acute Respiratory Failure (ARF) is a life-threatening condition that often develops rapidly in critically ill patients. Accurate early prediction of ARF is crucial in intensive care units (ICUs) to enable timely clinical interventions and resource allocation. In this task, you are asked to build a machine learning model that predicts whether a patient will develop ARF within the next **12 hours**, based on multivariate clinical time series data. + +The dataset is extracted from electronic health records (EHRs) and preprocessed using the **FIDDLE** pipeline to generate structured temporal features for each patient. + +### Objective + +Your goal is to develop a binary classification model that takes a 12-hour time series as input and outputs the probability of ARF onset in the following 12 hours. + +--- + +## Data Description + +The dataset is divided into two directories: + +* `train/` + + * `ARF_12h.csv`: `ID` & `ARF_ONSET_HOUR` & Binary labels (`ARF_LABEL`) for training samples. + * `X.npz`: 3D sparse array of shape `(N, T, D)`: + + * `N`: number of training samples + * `T`: time steps (one per hour) + * `D`: number of features + +* `test/` + + * `ARF_12h.csv`: `ID` & `ARF_ONSET_HOUR`. + * `X.npz`: Test feature set in the same format as training data. + +The `.npz` files store sparse matrices and are loaded using the [`sparse`](https://github.com/pydata/sparse) library. Each matrix is converted to dense format before model input. (DO NOT USE scipy.sparse) +e.g. +``` +import sparse +X = sparse.load_npz("").todense() +``` +Then, you can use `X.transpose(0, 2, 1)` to transpose the shape of X from (N, T, D) to (N, D, T) + +--- + +## Modeling + +Each sample is a multivariate time series representing 12 hours of clinical observations. Your model should learn temporal and cross-feature dependencies to estimate the likelihood of ARF. + +* **Output**: Probability score ∈ \[0, 1] +* **Loss Function**: `CrossEntropyLoss` or equivalent +* **Evaluation Metric**: **AUROC** (Area Under the Receiver Operating Characteristic Curve) + +--- + +## Submission Format +``` +ID,ARF_LABEL +0,0.473 +1,0.652 +2,0.129 +... +``` diff --git a/rdagent/scenarios/data_science/example/arf-12-hour-prediction-task/sample.py b/rdagent/scenarios/data_science/example/arf-12-hour-prediction-task/sample.py new file mode 100644 index 00000000..4a6e36e5 --- /dev/null +++ b/rdagent/scenarios/data_science/example/arf-12-hour-prediction-task/sample.py @@ -0,0 +1,71 @@ +import shutil +from pathlib import Path + +import numpy as np +import pandas as pd +import sparse +from tqdm import tqdm + + +def sample_and_copy_subfolder( + input_dir: Path, + output_dir: Path, + min_frac: float, + min_num: int, + seed: int = 42, +): + np.random.seed(seed) + + feature_path = input_dir / "X.npz" + label_path = input_dir / "ARF_12h.csv" + + X_sparse = sparse.load_npz(feature_path) + df_label = pd.read_csv(label_path) + + N = X_sparse.shape[0] + n_keep = max(int(N * min_frac), min_num) + idx = np.random.choice(N, n_keep, replace=False) + + X_sample = X_sparse[idx] + df_sample = df_label.iloc[idx].reset_index(drop=True) + + output_dir.mkdir(parents=True, exist_ok=True) + sparse.save_npz(output_dir / "X.npz", X_sample) + df_sample.to_csv(output_dir / "ARF_12h.csv", index=False) + + print(f"[INFO] Sampled {n_keep} of {N} from {input_dir.name}") + + +def create_debug_data( + dataset_path: str, + output_path: str, + min_frac: float = 0.02, + min_num: int = 10, +): + dataset_root = Path(dataset_path) / "arf-12-hour-prediction-task" + output_root = Path(output_path) + + for sub in ["train", "test"]: + input_dir = dataset_root / sub + output_dir = output_root / sub + print(f"\n[PROCESS] {sub} subset") + sample_and_copy_subfolder( + input_dir=input_dir, + output_dir=output_dir, + min_frac=min_frac, + min_num=min_num, + seed=42 if sub == "train" else 123, + ) + + print(f"\n[INFO] Sampling complete → Output in: {output_root}") + + +if __name__ == "__main__" or globals().get("__name__") == "": + dataset_path = globals().get("dataset_path", "./") + output_path = globals().get("output_path", "./sample") + create_debug_data( + dataset_path=dataset_path, + output_path=output_path, + min_frac=0.02, + min_num=10, + ) diff --git a/rdagent/scenarios/data_science/example/arf-12-hour-prediction-task/test/ARF_12h.csv b/rdagent/scenarios/data_science/example/arf-12-hour-prediction-task/test/ARF_12h.csv new file mode 100644 index 00000000..e69de29b diff --git a/rdagent/scenarios/data_science/example/arf-12-hour-prediction-task/test/X.npz b/rdagent/scenarios/data_science/example/arf-12-hour-prediction-task/test/X.npz new file mode 100644 index 00000000..e69de29b diff --git a/rdagent/scenarios/data_science/example/arf-12-hour-prediction-task/train/ARF_12h.csv b/rdagent/scenarios/data_science/example/arf-12-hour-prediction-task/train/ARF_12h.csv new file mode 100644 index 00000000..e69de29b diff --git a/rdagent/scenarios/data_science/example/arf-12-hour-prediction-task/train/X.npz b/rdagent/scenarios/data_science/example/arf-12-hour-prediction-task/train/X.npz new file mode 100644 index 00000000..e69de29b diff --git a/rdagent/scenarios/data_science/example/eval/arf-12-hour-prediction-task/grade.py b/rdagent/scenarios/data_science/example/eval/arf-12-hour-prediction-task/grade.py new file mode 100644 index 00000000..3fc329bd --- /dev/null +++ b/rdagent/scenarios/data_science/example/eval/arf-12-hour-prediction-task/grade.py @@ -0,0 +1,75 @@ +import json + +import pandas as pd +from sklearn.metrics import roc_auc_score + + +class InvalidSubmissionError(Exception): + """ + A custom exception for when the agent submission cannot be graded. + """ + + pass + + +def prepare_for_auroc_metric(submission: pd.DataFrame, answers: pd.DataFrame, id_col: str, target_col: str) -> dict: + + # Answers checks + assert id_col in answers.columns, f"answers dataframe should have an {id_col} column" + assert target_col in answers.columns, f"answers dataframe should have a {target_col} column" + + # Submission checks + if id_col not in submission.columns: + raise InvalidSubmissionError(f"Submission should have an {id_col} column") + if target_col not in submission.columns: + raise InvalidSubmissionError(f"Submission should have a {target_col} column") + if len(submission) != len(answers): + raise InvalidSubmissionError(f"Submission and answers should have the same number of rows") + try: + pd.to_numeric(submission[target_col]) + except ValueError: + raise InvalidSubmissionError( + f"Expected {target_col} column to be numeric, got {submission[target_col].dtype} instead" + ) + if submission[target_col].min() < 0 or submission[target_col].max() > 1: + raise InvalidSubmissionError( + f"Submission {target_col} column should contain probabilities," + " and therefore contain values between 0 and 1 inclusive" + ) + # Sort + submission = submission.sort_values(id_col) + answers = answers.sort_values(id_col) + + if (submission[id_col].values != answers[id_col].values).any(): + raise InvalidSubmissionError(f"Submission and answers should have the same {id_col} values") + + roc_auc_inputs = { + "y_true": answers[target_col].to_numpy(), + "y_score": submission[target_col].to_numpy(), + } + + return roc_auc_inputs + + +def grade(submission: pd.DataFrame, answers: pd.DataFrame) -> float: + roc_auc_inputs = prepare_for_auroc_metric( + submission=submission, answers=answers, id_col="ID", target_col="ARF_LABEL" + ) + return roc_auc_score(y_true=roc_auc_inputs["y_true"], y_score=roc_auc_inputs["y_score"]) + + +if __name__ == "__main__": + submission_path = "submission.csv" + gt_submission_path = "submission_test.csv" + submission = pd.read_csv(submission_path) + answers = pd.read_csv(gt_submission_path) + score = grade(submission=submission, answers=answers) + + print( + json.dumps( + { + "competition_id": "arf-12-hour-prediction-task", + "score": score, + } + ) + ) diff --git a/rdagent/scenarios/data_science/example/eval/arf-12-hour-prediction-task/submission_test.csv b/rdagent/scenarios/data_science/example/eval/arf-12-hour-prediction-task/submission_test.csv new file mode 100644 index 00000000..e69de29b diff --git a/rdagent/scenarios/data_science/example/eval/arf-12-hour-prediction-task/valid.py b/rdagent/scenarios/data_science/example/eval/arf-12-hour-prediction-task/valid.py new file mode 100644 index 00000000..250dfb28 --- /dev/null +++ b/rdagent/scenarios/data_science/example/eval/arf-12-hour-prediction-task/valid.py @@ -0,0 +1,21 @@ +from pathlib import Path + +# Check if our submission file exists +assert Path("submission.csv").exists(), "Error: submission.csv not found" + +submission_lines = Path("submission.csv").read_text().splitlines() +test_lines = Path("submission_test.csv").read_text().splitlines() + +is_valid = len(submission_lines) == len(test_lines) + +if is_valid: + message = "submission.csv and submission_test.csv have the same number of lines." +else: + message = ( + f"submission.csv has {len(submission_lines)} lines, while submission_test.csv has {len(test_lines)} lines." + ) + +print(message) + +if not is_valid: + raise AssertionError("Submission is invalid") diff --git a/rdagent/scenarios/data_science/scen/__init__.py b/rdagent/scenarios/data_science/scen/__init__.py index 9d765053..808297e8 100644 --- a/rdagent/scenarios/data_science/scen/__init__.py +++ b/rdagent/scenarios/data_science/scen/__init__.py @@ -1,4 +1,5 @@ import json +import runpy from pathlib import Path from typing import Dict @@ -35,7 +36,17 @@ class DataScienceScen(Scenario): if DS_RD_SETTING.sample_data: self.debug_path = f"{local_path}/sample/{competition}" if not Path(self.debug_path).exists(): - create_debug_data(competition, dataset_path=local_path) + sample_py_path = Path(local_path) / competition / "sample.py" + if sample_py_path.exists(): + runpy.run_path( + str(sample_py_path), + init_globals={ + "dataset_path": str(local_path), + "output_path": str(self.debug_path), + }, + ) + else: + create_debug_data(competition, dataset_path=local_path) else: self.debug_path = f"{local_path}/{competition}" diff --git a/rdagent/scenarios/qlib/developer/feedback.py b/rdagent/scenarios/qlib/developer/feedback.py index 5fa68f26..58fcfbd9 100644 --- a/rdagent/scenarios/qlib/developer/feedback.py +++ b/rdagent/scenarios/qlib/developer/feedback.py @@ -5,11 +5,7 @@ from typing import Dict import pandas as pd from rdagent.core.experiment import Experiment -from rdagent.core.proposal import ( - Experiment2Feedback, - HypothesisFeedback, - Trace, -) +from rdagent.core.proposal import Experiment2Feedback, HypothesisFeedback, Trace from rdagent.log import rdagent_logger as logger from rdagent.oai.llm_utils import APIBackend from rdagent.scenarios.qlib.experiment.quant_experiment import QlibQuantScenario diff --git a/requirements.txt b/requirements.txt index e2475dd2..71f3d68c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -65,4 +65,7 @@ genson # mlflow mlflow azureml-mlflow -types-pytz \ No newline at end of file +types-pytz + +# data science scenario for custom dataset +sparse \ No newline at end of file diff --git a/test/utils/test_import.py b/test/utils/test_import.py index 89c0b530..7a966b43 100644 --- a/test/utils/test_import.py +++ b/test/utils/test_import.py @@ -17,6 +17,8 @@ class TestRDAgentImports(unittest.TestCase): def import_all_modules_from_directory(directory): for file in directory.joinpath("rdagent").rglob("*.py"): fstr = str(file) + if "example" in fstr: + continue if "meta_tpl" in fstr: continue if "template" in fstr or "tpl" in fstr: