fix: model/factor experiment filtering in Qlib proposals (#1257)

* Fix model/factor experiment filtering in Qlib proposals

* fix(dockerfile): install coreutils to resolve timeout command error (#1260)

* chore: remove unused experiment_list and target_list from Qlib proposals

* fix CI

* fix target list

---------

Co-authored-by: Xu Yang <peteryang@vip.qq.com>

rdagent/scenarios/qlib/proposal/factor_proposal.py

@@ -64,13 +64,14 @@ class QlibFactorHypothesis2Experiment(FactorHypothesis2Experiment):
             scenario = trace.scen.get_scenario_all_desc(action="factor")
         else:
             scenario = trace.scen.get_scenario_all_desc()
+
         experiment_output_format = T("scenarios.qlib.prompts:factor_experiment_output_format").r()
 
         if len(trace.hist) == 0:
             hypothesis_and_feedback = "No previous hypothesis and feedback available since it's the first round."
         else:
             specific_trace = Trace(trace.scen)
-            for i in range(len(trace.hist) - 1, -1, -1):  # Reverse iteration
+            for i in range(len(trace.hist) - 1, -1, -1):
                 if not hasattr(trace.hist[i][0].hypothesis, "action") or trace.hist[i][0].hypothesis.action == "factor":
                     specific_trace.hist.insert(0, trace.hist[i])
             if len(specific_trace.hist) > 0:
@@ -81,18 +82,12 @@ class QlibFactorHypothesis2Experiment(FactorHypothesis2Experiment):
             else:
                 hypothesis_and_feedback = "No previous hypothesis and feedback available."
 
-        experiment_list: List[FactorExperiment] = [t[0] for t in trace.hist]
-
-        factor_list = []
-        for experiment in experiment_list:
-            factor_list.extend(experiment.sub_tasks)
-
         return {
             "target_hypothesis": str(hypothesis),
             "scenario": scenario,
             "hypothesis_and_feedback": hypothesis_and_feedback,
             "experiment_output_format": experiment_output_format,
-            "target_list": factor_list,
+            "target_list": [],
             "RAG": None,
         }, True
 
@@ -114,10 +109,11 @@ class QlibFactorHypothesis2Experiment(FactorHypothesis2Experiment):
             )
 
         exp = QlibFactorExperiment(tasks, hypothesis=hypothesis)
-        exp.based_experiments = [QlibFactorExperiment(sub_tasks=[])] + [t[0] for t in trace.hist if t[1]]
+        exp.based_experiments = [QlibFactorExperiment(sub_tasks=[])] + [
+            t[0] for t in trace.hist if t[1] and isinstance(t[0], FactorExperiment)
+        ]
 
         unique_tasks = []
-
         for task in tasks:
             duplicate = False
             for based_exp in exp.based_experiments:

rdagent/scenarios/qlib/proposal/model_proposal.py

@@ -87,7 +87,7 @@ class QlibModelHypothesis2Experiment(ModelHypothesis2Experiment):
             hypothesis_and_feedback = "No previous hypothesis and feedback available since it's the first round."
         else:
             specific_trace = Trace(trace.scen)
-            for i in range(len(trace.hist) - 1, -1, -1):  # Reverse iteration
+            for i in range(len(trace.hist) - 1, -1, -1):
                 if not hasattr(trace.hist[i][0].hypothesis, "action") or trace.hist[i][0].hypothesis.action == "model":
                     if last_experiment is None:
                         last_experiment = trace.hist[i][0]
@@ -120,12 +120,6 @@ class QlibModelHypothesis2Experiment(ModelHypothesis2Experiment):
             else "No SOTA hypothesis and feedback available since previous experiments were not accepted."
         )
 
-        experiment_list: List[ModelExperiment] = [t[0] for t in trace.hist]
-
-        model_list = []
-        for experiment in experiment_list:
-            model_list.extend(experiment.sub_tasks)
-
         return {
             "target_hypothesis": str(hypothesis),
             "scenario": scenario,
@@ -133,7 +127,7 @@ class QlibModelHypothesis2Experiment(ModelHypothesis2Experiment):
             "last_hypothesis_and_feedback": last_hypothesis_and_feedback,
             "SOTA_hypothesis_and_feedback": sota_hypothesis_and_feedback,
             "experiment_output_format": experiment_output_format,
-            "target_list": model_list,
+            "target_list": [],
             "RAG": "Note, the training data consists of less than 1 million samples for the training set and approximately 250,000 samples for the validation set. Please design the hyperparameters accordingly and control the model size. This has a significant impact on the training results. If you believe that the previous model itself is good but the training hyperparameters or model hyperparameters are not optimal, you can return the same model and adjust these parameters instead.",
         }, True
 
@@ -161,5 +155,5 @@ class QlibModelHypothesis2Experiment(ModelHypothesis2Experiment):
                 )
             )
         exp = QlibModelExperiment(tasks, hypothesis=hypothesis)
-        exp.based_experiments = [t[0] for t in trace.hist if t[1]]
+        exp.based_experiments = [t[0] for t in trace.hist if t[1] and isinstance(t[0], ModelExperiment)]
         return exp

test/notebook/testfiles/main.py

@@ -1,24 +1,21 @@
+import argparse
 import os
+import random
 import sys
 import time
-import random
+
+import albumentations as A
+import cv2
 import numpy as np
 import pandas as pd
-
+import timm
 import torch
 import torch.nn as nn
 import torch.optim as optim
-from torch.utils.data import Dataset, DataLoader
-
-import timm
-import albumentations as A
 from albumentations.pytorch import ToTensorV2
-
+from sklearn.metrics import confusion_matrix, roc_auc_score
 from sklearn.model_selection import StratifiedKFold
-from sklearn.metrics import roc_auc_score, confusion_matrix
-
-import cv2
-import argparse
+from torch.utils.data import DataLoader, Dataset
 
 parser = argparse.ArgumentParser()
 parser.add_argument('--debug', action='store_true', help='Run in debug mode')

test/notebook/testfiles/main2.py

@@ -1,24 +1,22 @@
-import os
-import time
 import argparse
+import os
 import random
-import numpy as np
-import pandas as pd
-from PIL import Image
+import time
 from glob import glob
 
+import albumentations as A
+import cv2
+import numpy as np
+import pandas as pd
 import torch
 import torch.nn as nn
 import torch.optim as optim
-from torch.utils.data import Dataset, DataLoader
 import torchvision
-
-import albumentations as A
 from albumentations.pytorch import ToTensorV2
-import cv2
-
-from sklearn.model_selection import StratifiedShuffleSplit
+from PIL import Image
 from sklearn.metrics import log_loss
+from sklearn.model_selection import StratifiedShuffleSplit
+from torch.utils.data import DataLoader, Dataset
 
 # ========= Debug mode handling ==========
 parser = argparse.ArgumentParser()
@@ -233,7 +231,7 @@ def main():
     class EfficientNetB0_4ch(nn.Module):
         def __init__(self, pretrained=True):
             super().__init__()
-            from torchvision.models import efficientnet_b0, EfficientNet_B0_Weights
+            from torchvision.models import EfficientNet_B0_Weights, efficientnet_b0
             if pretrained:
                 wts = EfficientNet_B0_Weights.DEFAULT
                 net = efficientnet_b0(weights=wts)

test/notebook/testfiles/main_missing_main_fn.py

@@ -1,24 +1,21 @@
+import argparse
 import os
+import random
 import sys
 import time
-import random
+
+import albumentations as A
+import cv2
 import numpy as np
 import pandas as pd
-
+import timm
 import torch
 import torch.nn as nn
 import torch.optim as optim
-from torch.utils.data import Dataset, DataLoader
-
-import timm
-import albumentations as A
 from albumentations.pytorch import ToTensorV2
-
+from sklearn.metrics import confusion_matrix, roc_auc_score
 from sklearn.model_selection import StratifiedKFold
-from sklearn.metrics import roc_auc_score, confusion_matrix
-
-import cv2
-import argparse
+from torch.utils.data import DataLoader, Dataset
 
 parser = argparse.ArgumentParser()
 parser.add_argument('--debug', action='store_true', help='Run in debug mode')

test/notebook/testfiles/main_missing_sections.py

@@ -1,24 +1,21 @@
+import argparse
 import os
+import random
 import sys
 import time
-import random
+
+import albumentations as A
+import cv2
 import numpy as np
 import pandas as pd
-
+import timm
 import torch
 import torch.nn as nn
 import torch.optim as optim
-from torch.utils.data import Dataset, DataLoader
-
-import timm
-import albumentations as A
 from albumentations.pytorch import ToTensorV2
-
+from sklearn.metrics import confusion_matrix, roc_auc_score
 from sklearn.model_selection import StratifiedKFold
-from sklearn.metrics import roc_auc_score, confusion_matrix
-
-import cv2
-import argparse
+from torch.utils.data import DataLoader, Dataset
 
 parser = argparse.ArgumentParser()
 parser.add_argument('--debug', action='store_true', help='Run in debug mode')

test/qlib/test_model_factor_proposal.py

@@ -0,0 +1,112 @@
+from unittest.mock import Mock, patch
+
+import pytest
+
+from rdagent.core.proposal import Hypothesis, Trace
+from rdagent.scenarios.qlib.proposal.factor_proposal import (
+    QlibFactorHypothesis2Experiment,
+)
+from rdagent.scenarios.qlib.proposal.model_proposal import (
+    QlibModelHypothesis2Experiment,
+)
+
+
+@pytest.fixture
+def mixed_model_trace():
+    trace = Trace(scen=Mock())
+    model_task = Mock()
+    model_task.name = "model_task_1"
+    factor_task = Mock()
+    factor_task.name = "factor_task_1"
+    trace.hist = [
+        (Mock(sub_tasks=[model_task], hypothesis=Mock(action="model")), Mock()),
+        (Mock(sub_tasks=[factor_task], hypothesis=Mock(action="factor")), Mock()),
+    ]
+    return trace
+
+
+@pytest.fixture
+def mixed_factor_trace():
+    trace = Trace(scen=Mock())
+    factor_task = Mock()
+    factor_task.factor_name = "factor_task_1"
+    model_task = Mock()
+    model_task.name = "model_task_1"
+    trace.hist = [
+        (Mock(sub_tasks=[factor_task], hypothesis=Mock(action="factor")), Mock()),
+        (Mock(sub_tasks=[model_task], hypothesis=Mock(action="model")), Mock()),
+    ]
+    return trace
+
+
+def test_model_proposal_import():
+    assert QlibModelHypothesis2Experiment is not None
+
+
+def test_factor_proposal_import():
+    assert QlibFactorHypothesis2Experiment is not None
+
+
+def test_model_filtering(mixed_model_trace):
+    converter = QlibModelHypothesis2Experiment()
+    hypothesis = Hypothesis(
+        hypothesis="test",
+        reason="r",
+        concise_reason="cr",
+        concise_observation="co",
+        concise_justification="cj",
+        concise_knowledge="ck",
+    )
+    with patch("rdagent.utils.agent.tpl.T.r", return_value="mocked"):
+        context, ok = converter.prepare_context(hypothesis, mixed_model_trace)
+
+    target_list = context.get("target_list", [])
+    assert ok is True
+    names = [getattr(task, "name", "") for task in target_list]
+    assert all("model" in name for name in names)
+
+
+def test_factor_filtering(mixed_factor_trace):
+    converter = QlibFactorHypothesis2Experiment()
+    hypothesis = Hypothesis(
+        hypothesis="test",
+        reason="r",
+        concise_reason="cr",
+        concise_observation="co",
+        concise_justification="cj",
+        concise_knowledge="ck",
+    )
+    with patch("rdagent.utils.agent.tpl.T.r", return_value="mocked"):
+        context, ok = converter.prepare_context(hypothesis, mixed_factor_trace)
+
+    target_list = context.get("target_list", [])
+    assert ok is True
+    factor_names = [getattr(task, "factor_name", "") for task in target_list]
+    assert all("factor" in name for name in factor_names)
+
+
+@pytest.mark.parametrize(
+    "converter_class, trace_fixture, expected_type",
+    [
+        (QlibModelHypothesis2Experiment, "mixed_model_trace", "ModelExperiment"),
+        (QlibFactorHypothesis2Experiment, "mixed_factor_trace", "FactorExperiment"),
+    ],
+)
+def test_code_inspection(converter_class, trace_fixture, request, expected_type):
+    converter = converter_class()
+    trace = request.getfixturevalue(trace_fixture)
+    hypothesis = Hypothesis(
+        hypothesis="test",
+        reason="r",
+        concise_reason="cr",
+        concise_observation="co",
+        concise_justification="cj",
+        concise_knowledge="ck",
+    )
+    with patch("rdagent.utils.agent.tpl.T.r", return_value="mocked"):
+        context, ok = converter.prepare_context(hypothesis, trace)
+
+    target_list = context.get("target_list", [])
+    assert ok is True
+    if target_list:
+        assert target_list[0].__class__.__name__ == expected_type