From 9e34b4e855cbd709cd077f529950b8e1f5c01486 Mon Sep 17 00:00:00 2001 From: Utsab Dahal <134686066+utsab345@users.noreply.github.com> Date: Fri, 10 Oct 2025 20:34:46 +0545 Subject: [PATCH] fix: model/factor experiment filtering in Qlib proposals (#1257) * Fix model/factor experiment filtering in Qlib proposals * fix(dockerfile): install coreutils to resolve timeout command error (#1260) * chore: remove unused experiment_list and target_list from Qlib proposals * fix CI * fix target list --------- Co-authored-by: Xu Yang --- .../qlib/proposal/factor_proposal.py | 16 +-- .../scenarios/qlib/proposal/model_proposal.py | 12 +- test/notebook/testfiles/main.py | 19 ++- test/notebook/testfiles/main2.py | 22 ++-- .../testfiles/main_missing_main_fn.py | 19 ++- .../testfiles/main_missing_sections.py | 19 ++- test/qlib/test_model_factor_proposal.py | 112 ++++++++++++++++++ 7 files changed, 155 insertions(+), 64 deletions(-) create mode 100644 test/qlib/test_model_factor_proposal.py diff --git a/rdagent/scenarios/qlib/proposal/factor_proposal.py b/rdagent/scenarios/qlib/proposal/factor_proposal.py index 0ff088dd..5ce9b70d 100644 --- a/rdagent/scenarios/qlib/proposal/factor_proposal.py +++ b/rdagent/scenarios/qlib/proposal/factor_proposal.py @@ -64,13 +64,14 @@ class QlibFactorHypothesis2Experiment(FactorHypothesis2Experiment): scenario = trace.scen.get_scenario_all_desc(action="factor") else: scenario = trace.scen.get_scenario_all_desc() + experiment_output_format = T("scenarios.qlib.prompts:factor_experiment_output_format").r() if len(trace.hist) == 0: hypothesis_and_feedback = "No previous hypothesis and feedback available since it's the first round." else: specific_trace = Trace(trace.scen) - for i in range(len(trace.hist) - 1, -1, -1): # Reverse iteration + for i in range(len(trace.hist) - 1, -1, -1): if not hasattr(trace.hist[i][0].hypothesis, "action") or trace.hist[i][0].hypothesis.action == "factor": specific_trace.hist.insert(0, trace.hist[i]) if len(specific_trace.hist) > 0: @@ -81,18 +82,12 @@ class QlibFactorHypothesis2Experiment(FactorHypothesis2Experiment): else: hypothesis_and_feedback = "No previous hypothesis and feedback available." - experiment_list: List[FactorExperiment] = [t[0] for t in trace.hist] - - factor_list = [] - for experiment in experiment_list: - factor_list.extend(experiment.sub_tasks) - return { "target_hypothesis": str(hypothesis), "scenario": scenario, "hypothesis_and_feedback": hypothesis_and_feedback, "experiment_output_format": experiment_output_format, - "target_list": factor_list, + "target_list": [], "RAG": None, }, True @@ -114,10 +109,11 @@ class QlibFactorHypothesis2Experiment(FactorHypothesis2Experiment): ) exp = QlibFactorExperiment(tasks, hypothesis=hypothesis) - exp.based_experiments = [QlibFactorExperiment(sub_tasks=[])] + [t[0] for t in trace.hist if t[1]] + exp.based_experiments = [QlibFactorExperiment(sub_tasks=[])] + [ + t[0] for t in trace.hist if t[1] and isinstance(t[0], FactorExperiment) + ] unique_tasks = [] - for task in tasks: duplicate = False for based_exp in exp.based_experiments: diff --git a/rdagent/scenarios/qlib/proposal/model_proposal.py b/rdagent/scenarios/qlib/proposal/model_proposal.py index 133017d5..6bd417ad 100644 --- a/rdagent/scenarios/qlib/proposal/model_proposal.py +++ b/rdagent/scenarios/qlib/proposal/model_proposal.py @@ -87,7 +87,7 @@ class QlibModelHypothesis2Experiment(ModelHypothesis2Experiment): hypothesis_and_feedback = "No previous hypothesis and feedback available since it's the first round." else: specific_trace = Trace(trace.scen) - for i in range(len(trace.hist) - 1, -1, -1): # Reverse iteration + for i in range(len(trace.hist) - 1, -1, -1): if not hasattr(trace.hist[i][0].hypothesis, "action") or trace.hist[i][0].hypothesis.action == "model": if last_experiment is None: last_experiment = trace.hist[i][0] @@ -120,12 +120,6 @@ class QlibModelHypothesis2Experiment(ModelHypothesis2Experiment): else "No SOTA hypothesis and feedback available since previous experiments were not accepted." ) - experiment_list: List[ModelExperiment] = [t[0] for t in trace.hist] - - model_list = [] - for experiment in experiment_list: - model_list.extend(experiment.sub_tasks) - return { "target_hypothesis": str(hypothesis), "scenario": scenario, @@ -133,7 +127,7 @@ class QlibModelHypothesis2Experiment(ModelHypothesis2Experiment): "last_hypothesis_and_feedback": last_hypothesis_and_feedback, "SOTA_hypothesis_and_feedback": sota_hypothesis_and_feedback, "experiment_output_format": experiment_output_format, - "target_list": model_list, + "target_list": [], "RAG": "Note, the training data consists of less than 1 million samples for the training set and approximately 250,000 samples for the validation set. Please design the hyperparameters accordingly and control the model size. This has a significant impact on the training results. If you believe that the previous model itself is good but the training hyperparameters or model hyperparameters are not optimal, you can return the same model and adjust these parameters instead.", }, True @@ -161,5 +155,5 @@ class QlibModelHypothesis2Experiment(ModelHypothesis2Experiment): ) ) exp = QlibModelExperiment(tasks, hypothesis=hypothesis) - exp.based_experiments = [t[0] for t in trace.hist if t[1]] + exp.based_experiments = [t[0] for t in trace.hist if t[1] and isinstance(t[0], ModelExperiment)] return exp diff --git a/test/notebook/testfiles/main.py b/test/notebook/testfiles/main.py index 14c9db5e..a9f46145 100644 --- a/test/notebook/testfiles/main.py +++ b/test/notebook/testfiles/main.py @@ -1,24 +1,21 @@ +import argparse import os +import random import sys import time -import random + +import albumentations as A +import cv2 import numpy as np import pandas as pd - +import timm import torch import torch.nn as nn import torch.optim as optim -from torch.utils.data import Dataset, DataLoader - -import timm -import albumentations as A from albumentations.pytorch import ToTensorV2 - +from sklearn.metrics import confusion_matrix, roc_auc_score from sklearn.model_selection import StratifiedKFold -from sklearn.metrics import roc_auc_score, confusion_matrix - -import cv2 -import argparse +from torch.utils.data import DataLoader, Dataset parser = argparse.ArgumentParser() parser.add_argument('--debug', action='store_true', help='Run in debug mode') diff --git a/test/notebook/testfiles/main2.py b/test/notebook/testfiles/main2.py index 5ce74205..0a0b7935 100644 --- a/test/notebook/testfiles/main2.py +++ b/test/notebook/testfiles/main2.py @@ -1,24 +1,22 @@ -import os -import time import argparse +import os import random -import numpy as np -import pandas as pd -from PIL import Image +import time from glob import glob +import albumentations as A +import cv2 +import numpy as np +import pandas as pd import torch import torch.nn as nn import torch.optim as optim -from torch.utils.data import Dataset, DataLoader import torchvision - -import albumentations as A from albumentations.pytorch import ToTensorV2 -import cv2 - -from sklearn.model_selection import StratifiedShuffleSplit +from PIL import Image from sklearn.metrics import log_loss +from sklearn.model_selection import StratifiedShuffleSplit +from torch.utils.data import DataLoader, Dataset # ========= Debug mode handling ========== parser = argparse.ArgumentParser() @@ -233,7 +231,7 @@ def main(): class EfficientNetB0_4ch(nn.Module): def __init__(self, pretrained=True): super().__init__() - from torchvision.models import efficientnet_b0, EfficientNet_B0_Weights + from torchvision.models import EfficientNet_B0_Weights, efficientnet_b0 if pretrained: wts = EfficientNet_B0_Weights.DEFAULT net = efficientnet_b0(weights=wts) diff --git a/test/notebook/testfiles/main_missing_main_fn.py b/test/notebook/testfiles/main_missing_main_fn.py index d4289ed5..f6f3b360 100644 --- a/test/notebook/testfiles/main_missing_main_fn.py +++ b/test/notebook/testfiles/main_missing_main_fn.py @@ -1,24 +1,21 @@ +import argparse import os +import random import sys import time -import random + +import albumentations as A +import cv2 import numpy as np import pandas as pd - +import timm import torch import torch.nn as nn import torch.optim as optim -from torch.utils.data import Dataset, DataLoader - -import timm -import albumentations as A from albumentations.pytorch import ToTensorV2 - +from sklearn.metrics import confusion_matrix, roc_auc_score from sklearn.model_selection import StratifiedKFold -from sklearn.metrics import roc_auc_score, confusion_matrix - -import cv2 -import argparse +from torch.utils.data import DataLoader, Dataset parser = argparse.ArgumentParser() parser.add_argument('--debug', action='store_true', help='Run in debug mode') diff --git a/test/notebook/testfiles/main_missing_sections.py b/test/notebook/testfiles/main_missing_sections.py index 009a1fcf..ab6200fc 100644 --- a/test/notebook/testfiles/main_missing_sections.py +++ b/test/notebook/testfiles/main_missing_sections.py @@ -1,24 +1,21 @@ +import argparse import os +import random import sys import time -import random + +import albumentations as A +import cv2 import numpy as np import pandas as pd - +import timm import torch import torch.nn as nn import torch.optim as optim -from torch.utils.data import Dataset, DataLoader - -import timm -import albumentations as A from albumentations.pytorch import ToTensorV2 - +from sklearn.metrics import confusion_matrix, roc_auc_score from sklearn.model_selection import StratifiedKFold -from sklearn.metrics import roc_auc_score, confusion_matrix - -import cv2 -import argparse +from torch.utils.data import DataLoader, Dataset parser = argparse.ArgumentParser() parser.add_argument('--debug', action='store_true', help='Run in debug mode') diff --git a/test/qlib/test_model_factor_proposal.py b/test/qlib/test_model_factor_proposal.py new file mode 100644 index 00000000..875aecf8 --- /dev/null +++ b/test/qlib/test_model_factor_proposal.py @@ -0,0 +1,112 @@ +from unittest.mock import Mock, patch + +import pytest + +from rdagent.core.proposal import Hypothesis, Trace +from rdagent.scenarios.qlib.proposal.factor_proposal import ( + QlibFactorHypothesis2Experiment, +) +from rdagent.scenarios.qlib.proposal.model_proposal import ( + QlibModelHypothesis2Experiment, +) + + +@pytest.fixture +def mixed_model_trace(): + trace = Trace(scen=Mock()) + model_task = Mock() + model_task.name = "model_task_1" + factor_task = Mock() + factor_task.name = "factor_task_1" + trace.hist = [ + (Mock(sub_tasks=[model_task], hypothesis=Mock(action="model")), Mock()), + (Mock(sub_tasks=[factor_task], hypothesis=Mock(action="factor")), Mock()), + ] + return trace + + +@pytest.fixture +def mixed_factor_trace(): + trace = Trace(scen=Mock()) + factor_task = Mock() + factor_task.factor_name = "factor_task_1" + model_task = Mock() + model_task.name = "model_task_1" + trace.hist = [ + (Mock(sub_tasks=[factor_task], hypothesis=Mock(action="factor")), Mock()), + (Mock(sub_tasks=[model_task], hypothesis=Mock(action="model")), Mock()), + ] + return trace + + +def test_model_proposal_import(): + assert QlibModelHypothesis2Experiment is not None + + +def test_factor_proposal_import(): + assert QlibFactorHypothesis2Experiment is not None + + +def test_model_filtering(mixed_model_trace): + converter = QlibModelHypothesis2Experiment() + hypothesis = Hypothesis( + hypothesis="test", + reason="r", + concise_reason="cr", + concise_observation="co", + concise_justification="cj", + concise_knowledge="ck", + ) + with patch("rdagent.utils.agent.tpl.T.r", return_value="mocked"): + context, ok = converter.prepare_context(hypothesis, mixed_model_trace) + + target_list = context.get("target_list", []) + assert ok is True + names = [getattr(task, "name", "") for task in target_list] + assert all("model" in name for name in names) + + +def test_factor_filtering(mixed_factor_trace): + converter = QlibFactorHypothesis2Experiment() + hypothesis = Hypothesis( + hypothesis="test", + reason="r", + concise_reason="cr", + concise_observation="co", + concise_justification="cj", + concise_knowledge="ck", + ) + with patch("rdagent.utils.agent.tpl.T.r", return_value="mocked"): + context, ok = converter.prepare_context(hypothesis, mixed_factor_trace) + + target_list = context.get("target_list", []) + assert ok is True + factor_names = [getattr(task, "factor_name", "") for task in target_list] + assert all("factor" in name for name in factor_names) + + +@pytest.mark.parametrize( + "converter_class, trace_fixture, expected_type", + [ + (QlibModelHypothesis2Experiment, "mixed_model_trace", "ModelExperiment"), + (QlibFactorHypothesis2Experiment, "mixed_factor_trace", "FactorExperiment"), + ], +) +def test_code_inspection(converter_class, trace_fixture, request, expected_type): + converter = converter_class() + trace = request.getfixturevalue(trace_fixture) + hypothesis = Hypothesis( + hypothesis="test", + reason="r", + concise_reason="cr", + concise_observation="co", + concise_justification="cj", + concise_knowledge="ck", + ) + with patch("rdagent.utils.agent.tpl.T.r", return_value="mocked"): + context, ok = converter.prepare_context(hypothesis, trace) + + target_list = context.get("target_list", []) + assert ok is True + if target_list: + assert target_list[0].__class__.__name__ == expected_type