From 319ed4096101d0826df82b778a881991315a0ca5 Mon Sep 17 00:00:00 2001 From: Suhan Cui <51844791+SH-Src@users.noreply.github.com> Date: Mon, 5 Aug 2024 13:28:50 +0800 Subject: [PATCH] fix: fix several bugs in data mining scenario (#147) * fix * ci * demo --- .../data_mining/developer/feedback.py | 1 + .../experiment/model_experiment.py | 21 ++++++++++++++++++- .../data_mining/proposal/model_proposal.py | 3 +++ 3 files changed, 24 insertions(+), 1 deletion(-) diff --git a/rdagent/scenarios/data_mining/developer/feedback.py b/rdagent/scenarios/data_mining/developer/feedback.py index af18f0c4..5a96609b 100644 --- a/rdagent/scenarios/data_mining/developer/feedback.py +++ b/rdagent/scenarios/data_mining/developer/feedback.py @@ -46,6 +46,7 @@ class DMModelHypothesisExperiment2Feedback(HypothesisExperiment2Feedback): context=context, last_hypothesis=SOTA_hypothesis, last_task=SOTA_experiment.sub_tasks[0].get_task_information() if SOTA_hypothesis else None, + last_code=SOTA_experiment.sub_workspace_list[0].code_dict.get("model.py") if SOTA_hypothesis else None, last_result=SOTA_experiment.result if SOTA_hypothesis else None, hypothesis=hypothesis, exp=exp, diff --git a/rdagent/scenarios/data_mining/experiment/model_experiment.py b/rdagent/scenarios/data_mining/experiment/model_experiment.py index 216524cf..ad366663 100644 --- a/rdagent/scenarios/data_mining/experiment/model_experiment.py +++ b/rdagent/scenarios/data_mining/experiment/model_experiment.py @@ -41,7 +41,26 @@ class DMModelScenario(Scenario): @property def rich_style_description(self) -> str: - return "Below is MIMIC Model Evolving Automatic R&D Demo." + return """ +### MIMIC-III Model Evolving Automatic R&D Demo + +#### [Overview](#_summary) + +The demo showcases the iterative process of hypothesis generation, knowledge construction, and decision-making in model construction in a clinical prediction task. The model should predict whether a patient would suffer from Acute Respiratory Failure (ARF) based on first 12 hours ICU monitoring data. + +#### [Automated R&D](#_rdloops) + +- **[R (Research)](#_research)** + - Iteration of ideas and hypotheses. + - Continuous learning and knowledge construction. + +- **[D (Development)](#_development)** + - Evolving code generation and model refinement. + - Automated implementation and testing of models. + +#### [Objective](#_summary) + +To demonstrate the dynamic evolution of models through the R&D loop, emphasizing how each iteration enhances the model performance and reliability. The performane is measured by the AUROC score (Area Under the Receiver Operating Characteristic), which is a commonly used metric for binary classification. """ def get_scenario_all_desc(self) -> str: return f"""Background of the scenario: diff --git a/rdagent/scenarios/data_mining/proposal/model_proposal.py b/rdagent/scenarios/data_mining/proposal/model_proposal.py index bd51bba5..fc60e2eb 100644 --- a/rdagent/scenarios/data_mining/proposal/model_proposal.py +++ b/rdagent/scenarios/data_mining/proposal/model_proposal.py @@ -54,6 +54,9 @@ class DMModelHypothesisGen(ModelHypothesisGen): hypothesis=response_dict["hypothesis"], reason=response_dict["reason"], concise_reason=response_dict["concise_reason"], + concise_observation=response_dict["concise_observation"], + concise_justification=response_dict["concise_justification"], + concise_knowledge=response_dict["concise_knowledge"], ) return hypothesis