fix: model/factor experiment filtering in Qlib proposals (#1257)

* Fix model/factor experiment filtering in Qlib proposals

* fix(dockerfile): install coreutils to resolve timeout command error (#1260)

* chore: remove unused experiment_list and target_list from Qlib proposals

* fix CI

* fix target list

---------

Co-authored-by: Xu Yang <peteryang@vip.qq.com>

test/notebook/testfiles/main.py

@@ -1,24 +1,21 @@
+import argparse
 import os
+import random
 import sys
 import time
-import random
+
+import albumentations as A
+import cv2
 import numpy as np
 import pandas as pd
-
+import timm
 import torch
 import torch.nn as nn
 import torch.optim as optim
-from torch.utils.data import Dataset, DataLoader
-
-import timm
-import albumentations as A
 from albumentations.pytorch import ToTensorV2
-
+from sklearn.metrics import confusion_matrix, roc_auc_score
 from sklearn.model_selection import StratifiedKFold
-from sklearn.metrics import roc_auc_score, confusion_matrix
-
-import cv2
-import argparse
+from torch.utils.data import DataLoader, Dataset
 
 parser = argparse.ArgumentParser()
 parser.add_argument('--debug', action='store_true', help='Run in debug mode')

test/notebook/testfiles/main2.py

@@ -1,24 +1,22 @@
-import os
-import time
 import argparse
+import os
 import random
-import numpy as np
-import pandas as pd
-from PIL import Image
+import time
 from glob import glob
 
+import albumentations as A
+import cv2
+import numpy as np
+import pandas as pd
 import torch
 import torch.nn as nn
 import torch.optim as optim
-from torch.utils.data import Dataset, DataLoader
 import torchvision
-
-import albumentations as A
 from albumentations.pytorch import ToTensorV2
-import cv2
-
-from sklearn.model_selection import StratifiedShuffleSplit
+from PIL import Image
 from sklearn.metrics import log_loss
+from sklearn.model_selection import StratifiedShuffleSplit
+from torch.utils.data import DataLoader, Dataset
 
 # ========= Debug mode handling ==========
 parser = argparse.ArgumentParser()
@@ -233,7 +231,7 @@ def main():
     class EfficientNetB0_4ch(nn.Module):
         def __init__(self, pretrained=True):
             super().__init__()
-            from torchvision.models import efficientnet_b0, EfficientNet_B0_Weights
+            from torchvision.models import EfficientNet_B0_Weights, efficientnet_b0
             if pretrained:
                 wts = EfficientNet_B0_Weights.DEFAULT
                 net = efficientnet_b0(weights=wts)

test/notebook/testfiles/main_missing_main_fn.py

@@ -1,24 +1,21 @@
+import argparse
 import os
+import random
 import sys
 import time
-import random
+
+import albumentations as A
+import cv2
 import numpy as np
 import pandas as pd
-
+import timm
 import torch
 import torch.nn as nn
 import torch.optim as optim
-from torch.utils.data import Dataset, DataLoader
-
-import timm
-import albumentations as A
 from albumentations.pytorch import ToTensorV2
-
+from sklearn.metrics import confusion_matrix, roc_auc_score
 from sklearn.model_selection import StratifiedKFold
-from sklearn.metrics import roc_auc_score, confusion_matrix
-
-import cv2
-import argparse
+from torch.utils.data import DataLoader, Dataset
 
 parser = argparse.ArgumentParser()
 parser.add_argument('--debug', action='store_true', help='Run in debug mode')

test/notebook/testfiles/main_missing_sections.py

@@ -1,24 +1,21 @@
+import argparse
 import os
+import random
 import sys
 import time
-import random
+
+import albumentations as A
+import cv2
 import numpy as np
 import pandas as pd
-
+import timm
 import torch
 import torch.nn as nn
 import torch.optim as optim
-from torch.utils.data import Dataset, DataLoader
-
-import timm
-import albumentations as A
 from albumentations.pytorch import ToTensorV2
-
+from sklearn.metrics import confusion_matrix, roc_auc_score
 from sklearn.model_selection import StratifiedKFold
-from sklearn.metrics import roc_auc_score, confusion_matrix
-
-import cv2
-import argparse
+from torch.utils.data import DataLoader, Dataset
 
 parser = argparse.ArgumentParser()
 parser.add_argument('--debug', action='store_true', help='Run in debug mode')

test/qlib/test_model_factor_proposal.py

@@ -0,0 +1,112 @@
+from unittest.mock import Mock, patch
+
+import pytest
+
+from rdagent.core.proposal import Hypothesis, Trace
+from rdagent.scenarios.qlib.proposal.factor_proposal import (
+    QlibFactorHypothesis2Experiment,
+)
+from rdagent.scenarios.qlib.proposal.model_proposal import (
+    QlibModelHypothesis2Experiment,
+)
+
+
+@pytest.fixture
+def mixed_model_trace():
+    trace = Trace(scen=Mock())
+    model_task = Mock()
+    model_task.name = "model_task_1"
+    factor_task = Mock()
+    factor_task.name = "factor_task_1"
+    trace.hist = [
+        (Mock(sub_tasks=[model_task], hypothesis=Mock(action="model")), Mock()),
+        (Mock(sub_tasks=[factor_task], hypothesis=Mock(action="factor")), Mock()),
+    ]
+    return trace
+
+
+@pytest.fixture
+def mixed_factor_trace():
+    trace = Trace(scen=Mock())
+    factor_task = Mock()
+    factor_task.factor_name = "factor_task_1"
+    model_task = Mock()
+    model_task.name = "model_task_1"
+    trace.hist = [
+        (Mock(sub_tasks=[factor_task], hypothesis=Mock(action="factor")), Mock()),
+        (Mock(sub_tasks=[model_task], hypothesis=Mock(action="model")), Mock()),
+    ]
+    return trace
+
+
+def test_model_proposal_import():
+    assert QlibModelHypothesis2Experiment is not None
+
+
+def test_factor_proposal_import():
+    assert QlibFactorHypothesis2Experiment is not None
+
+
+def test_model_filtering(mixed_model_trace):
+    converter = QlibModelHypothesis2Experiment()
+    hypothesis = Hypothesis(
+        hypothesis="test",
+        reason="r",
+        concise_reason="cr",
+        concise_observation="co",
+        concise_justification="cj",
+        concise_knowledge="ck",
+    )
+    with patch("rdagent.utils.agent.tpl.T.r", return_value="mocked"):
+        context, ok = converter.prepare_context(hypothesis, mixed_model_trace)
+
+    target_list = context.get("target_list", [])
+    assert ok is True
+    names = [getattr(task, "name", "") for task in target_list]
+    assert all("model" in name for name in names)
+
+
+def test_factor_filtering(mixed_factor_trace):
+    converter = QlibFactorHypothesis2Experiment()
+    hypothesis = Hypothesis(
+        hypothesis="test",
+        reason="r",
+        concise_reason="cr",
+        concise_observation="co",
+        concise_justification="cj",
+        concise_knowledge="ck",
+    )
+    with patch("rdagent.utils.agent.tpl.T.r", return_value="mocked"):
+        context, ok = converter.prepare_context(hypothesis, mixed_factor_trace)
+
+    target_list = context.get("target_list", [])
+    assert ok is True
+    factor_names = [getattr(task, "factor_name", "") for task in target_list]
+    assert all("factor" in name for name in factor_names)
+
+
+@pytest.mark.parametrize(
+    "converter_class, trace_fixture, expected_type",
+    [
+        (QlibModelHypothesis2Experiment, "mixed_model_trace", "ModelExperiment"),
+        (QlibFactorHypothesis2Experiment, "mixed_factor_trace", "FactorExperiment"),
+    ],
+)
+def test_code_inspection(converter_class, trace_fixture, request, expected_type):
+    converter = converter_class()
+    trace = request.getfixturevalue(trace_fixture)
+    hypothesis = Hypothesis(
+        hypothesis="test",
+        reason="r",
+        concise_reason="cr",
+        concise_observation="co",
+        concise_justification="cj",
+        concise_knowledge="ck",
+    )
+    with patch("rdagent.utils.agent.tpl.T.r", return_value="mocked"):
+        context, ok = converter.prepare_context(hypothesis, trace)
+
+    target_list = context.get("target_list", [])
+    assert ok is True
+    if target_list:
+        assert target_list[0].__class__.__name__ == expected_type