test: 441 deep property-based tests across CoSTEER, workflow, core, LLM utils, and formatting

- costeer_deep: 112 tests (knowledge base, feedback, evaluators, auto-fixer) - workflow_deep: 84 tests (RDLoop, proposals, traces, hypothesis/pickle) - core_deep: 74 tests (developer, evaluator, exceptions, experiment, scenario) - llm_utils_deep: 49 tests (embeddings, APIBackend, edge cases, Unicode/NaN) - utils_deep: 122 tests (shrink_text, templates, md5_hash, property-based, stress)
2026-08-04 10:47:43 +00:00 · 2026-05-10 22:14:11 +02:00
parent 79c2560c3f
commit ee3ef28cde
5 changed files with 3062 additions and 0 deletions
@@ -0,0 +1,520 @@
+"""Deep tests for rdagent.core: developer.py, evaluation.py, and related abstractions."""
+
+from __future__ import annotations
+
+import pickle
+import sys
+from pathlib import Path
+from typing import Any
+from unittest.mock import MagicMock, patch
+
+import pytest
+
+PROJECT_ROOT = Path(__file__).parent.parent.parent
+sys.path.insert(0, str(PROJECT_ROOT))
+
+
+# =============================================================================
+# Import safety
+# =============================================================================
+
+CORE_MODULES = [
+    "rdagent.core.developer",
+    "rdagent.core.evaluation",
+    "rdagent.core.experiment",
+    "rdagent.core.proposal",
+    "rdagent.core.scenario",
+    "rdagent.core.evolving_framework",
+    "rdagent.core.evolving_agent",
+    "rdagent.core.conf",
+    "rdagent.core.exception",
+    "rdagent.core.utils",
+]
+
+
+class TestCoreModuleImports:
+    @pytest.mark.parametrize("module_name", CORE_MODULES)
+    def test_module_importable(self, module_name: str) -> None:
+        import importlib
+        mod = importlib.import_module(module_name)
+        assert mod is not None
+
+
+# =============================================================================
+# Feedback
+# =============================================================================
+
+
+class TestFeedback:
+    def test_default_is_acceptable_returns_true(self) -> None:
+        from rdagent.core.evaluation import Feedback
+        fb = Feedback()
+        assert fb.is_acceptable() is True
+
+    def test_default_finished_returns_true(self) -> None:
+        from rdagent.core.evaluation import Feedback
+        fb = Feedback()
+        assert fb.finished() is True
+
+    def test_default_bool_is_true(self) -> None:
+        from rdagent.core.evaluation import Feedback
+        fb = Feedback()
+        assert bool(fb) is True
+
+    def test_is_acceptable_calls_bool(self) -> None:
+        from rdagent.core.evaluation import Feedback
+
+        class FalseFeedback(Feedback):
+            def __bool__(self) -> bool:
+                return False
+
+        fb = FalseFeedback()
+        assert fb.is_acceptable() is False
+
+    def test_finished_can_be_overridden(self) -> None:
+        from rdagent.core.evaluation import Feedback
+
+        class CustomFinish(Feedback):
+            def __bool__(self) -> bool:
+                return False
+
+            def finished(self) -> bool:
+                return True
+
+        fb = CustomFinish()
+        assert fb.finished() is True
+        assert bool(fb) is False
+
+    def test_pickle_safety(self) -> None:
+        from rdagent.core.evaluation import Feedback
+        fb = Feedback()
+        data = pickle.dumps(fb)
+        fb2 = pickle.loads(data)
+        assert isinstance(fb2, Feedback)
+        assert bool(fb2) is True
+
+
+# =============================================================================
+# Evaluator / EvaluableObj
+# =============================================================================
+
+
+class TestEvaluator:
+    def test_evaluator_is_abstract(self) -> None:
+        from rdagent.core.evaluation import Evaluator
+        assert hasattr(Evaluator, "evaluate")
+
+    def test_concrete_evaluator_must_implement_evaluate(self) -> None:
+        from rdagent.core.evaluation import Evaluator, Feedback
+
+        class ConcreteEvaluator(Evaluator):
+            def evaluate(self, eo) -> Feedback:
+                return Feedback()
+
+        ev = ConcreteEvaluator()
+        result = ev.evaluate(None)
+        assert isinstance(result, Feedback)
+
+    @pytest.mark.parametrize("input_eo", [None, "string", {"key": "value"}, [1, 2, 3]])
+    def test_concrete_evaluator_accepts_any_input(self, input_eo: Any) -> None:
+        from rdagent.core.evaluation import Evaluator, Feedback
+
+        class FlexibleEvaluator(Evaluator):
+            def evaluate(self, eo) -> Feedback:
+                return Feedback()
+
+        ev = FlexibleEvaluator()
+        result = ev.evaluate(input_eo)
+        assert isinstance(result, Feedback)
+
+
+class TestEvaluableObj:
+    def test_evaluable_obj_is_importable(self) -> None:
+        from rdagent.core.evaluation import EvaluableObj
+        assert EvaluableObj is not None
+
+    def test_evaluable_obj_can_be_instantiated(self) -> None:
+        from rdagent.core.evaluation import EvaluableObj
+        obj = EvaluableObj()
+        assert isinstance(obj, EvaluableObj)
+
+
+# =============================================================================
+# Developer
+# =============================================================================
+
+
+class TestDeveloperBase:
+    def test_developer_is_importable(self) -> None:
+        from rdagent.core.developer import Developer
+        assert Developer is not None
+
+    def test_developer_stores_scenario(self) -> None:
+        from rdagent.core.developer import Developer
+        from rdagent.core.experiment import ASpecificExp
+
+        class ConcreteDev(Developer[ASpecificExp]):
+            def develop(self, exp: ASpecificExp) -> ASpecificExp:
+                return exp
+
+        scen = MagicMock()
+        dev = ConcreteDev(scen)
+        assert dev.scen is scen
+
+    def test_develop_modifies_in_place(self) -> None:
+        from rdagent.core.developer import Developer
+        from rdagent.core.experiment import ASpecificExp
+
+        class ModifyingDev(Developer[ASpecificExp]):
+            def develop(self, exp: ASpecificExp) -> ASpecificExp:
+                exp._modified = True
+                return exp
+
+        dev = ModifyingDev(MagicMock())
+        exp = MagicMock()
+        result = dev.develop(exp)
+        assert result._modified is True
+
+
+# =============================================================================
+# Experiment classes
+# =============================================================================
+
+
+class TestExperiment:
+    def test_task_has_get_task_information(self) -> None:
+        from rdagent.core.experiment import Task
+        assert hasattr(Task, "get_task_information")
+
+    def test_workspace_has_execute(self) -> None:
+        from rdagent.core.experiment import Workspace
+        assert hasattr(Workspace, "execute")
+
+    def test_fb_workspace_inject_files(self) -> None:
+        from rdagent.core.experiment import FBWorkspace
+        ws = FBWorkspace()
+        ws.inject_files(**{"factor.py": "def calc(): pass"})
+        code = ws.all_codes
+        assert "def calc" in code
+
+    def test_fb_workspace_copy_returns_new_instance(self) -> None:
+        from rdagent.core.experiment import FBWorkspace
+        ws = FBWorkspace()
+        ws.inject_files(**{"test.py": "x=1"})
+        ws2 = ws.copy()
+        assert ws2 is not ws
+        assert ws2.all_codes == ws.all_codes
+
+    def test_fb_workspace_pickle_safety(self) -> None:
+        from rdagent.core.experiment import FBWorkspace
+        ws = FBWorkspace()
+        ws.inject_files(**{"factor.py": "x=1", "utils.py": "y=2"})
+        data = pickle.dumps(ws)
+        ws2 = pickle.loads(data)
+        assert isinstance(ws2, FBWorkspace)
+
+    @pytest.mark.parametrize("files", [
+        {},
+        {"a.py": ""},
+        {"a.py": "x=1", "b.py": "y=2", "c.py": "z=3"},
+    ])
+    def test_fb_workspace_file_variants(self, files: dict) -> None:
+        from rdagent.core.experiment import FBWorkspace
+        ws = FBWorkspace()
+        ws.inject_files(**files)
+        assert isinstance(ws.all_codes, str)
+
+    def test_aspecific_exp_is_importable(self) -> None:
+        from rdagent.core.experiment import ASpecificExp
+        assert ASpecificExp is not None
+
+
+# =============================================================================
+# EvoStep
+# =============================================================================
+
+
+class TestEvoStep:
+    def test_default_construction(self) -> None:
+        from rdagent.core.evolving_framework import EvoStep
+        es = EvoStep(evolvable_subjects="evo")
+        assert es.evolvable_subjects == "evo"
+        assert es.queried_knowledge is None
+        assert es.feedback is None
+
+    def test_full_construction(self) -> None:
+        from rdagent.core.evolving_framework import EvoStep, QueriedKnowledge
+        qk = QueriedKnowledge()
+        es = EvoStep(evolvable_subjects="subj", queried_knowledge=qk, feedback="fb")
+        assert es.queried_knowledge is qk
+        assert es.feedback == "fb"
+
+    def test_equality_by_reference(self) -> None:
+        from rdagent.core.evolving_framework import EvoStep
+        es1 = EvoStep(evolvable_subjects="a")
+        es2 = EvoStep(evolvable_subjects="a")
+        assert es1 == es2
+
+    def test_pickle_safety(self) -> None:
+        from rdagent.core.evolving_framework import EvoStep
+        es = EvoStep(evolvable_subjects="subj", feedback="good")
+        data = pickle.dumps(es)
+        es2 = pickle.loads(data)
+        assert es2.evolvable_subjects == "subj"
+        assert es2.feedback == "good"
+
+    def test_with_none_values(self) -> None:
+        from rdagent.core.evolving_framework import EvoStep
+        es = EvoStep(evolvable_subjects=None, queried_knowledge=None, feedback=None)
+        assert es.evolvable_subjects is None
+
+
+# =============================================================================
+# Knowledge and QueriedKnowledge
+# =============================================================================
+
+
+class TestKnowledge:
+    def test_knowledge_is_importable(self) -> None:
+        from rdagent.core.evolving_framework import Knowledge
+        assert Knowledge is not None
+
+    def test_knowledge_can_be_instantiated(self) -> None:
+        from rdagent.core.evolving_framework import Knowledge
+        k = Knowledge()
+        assert isinstance(k, Knowledge)
+
+    def test_knowledge_pickle_safety(self) -> None:
+        from rdagent.core.evolving_framework import Knowledge
+        k = Knowledge()
+        data = pickle.dumps(k)
+        k2 = pickle.loads(data)
+        assert isinstance(k2, Knowledge)
+
+
+class TestQueriedKnowledge:
+    def test_default_construction(self) -> None:
+        from rdagent.core.evolving_framework import QueriedKnowledge
+        qk = QueriedKnowledge()
+        assert isinstance(qk, QueriedKnowledge)
+
+    def test_pickle_safety(self) -> None:
+        from rdagent.core.evolving_framework import QueriedKnowledge
+        qk = QueriedKnowledge()
+        data = pickle.dumps(qk)
+        qk2 = pickle.loads(data)
+        assert isinstance(qk2, QueriedKnowledge)
+
+
+# =============================================================================
+# RAGStrategy / RAGEvaluator
+# =============================================================================
+
+
+class TestRAGStrategy:
+    def test_rag_strategy_has_methods(self) -> None:
+        from rdagent.core.evolving_framework import RAGStrategy
+        assert hasattr(RAGStrategy, "generate_knowledge")
+        assert hasattr(RAGStrategy, "query")
+
+
+class TestRAGEvaluator:
+    def test_rage_evaluator_is_importable(self) -> None:
+        from rdagent.core.evolving_agent import RAGEvaluator
+        assert RAGEvaluator is not None
+
+
+# =============================================================================
+# EvolvingKnowledgeBase
+# =============================================================================
+
+
+class TestEvolvingKnowledgeBase:
+    def test_has_query_method(self) -> None:
+        from rdagent.core.evolving_framework import EvolvingKnowledgeBase
+        assert hasattr(EvolvingKnowledgeBase, "query")
+
+    def test_takes_optional_path_argument(self) -> None:
+        from rdagent.core.evolving_framework import EvolvingKnowledgeBase
+        kb = EvolvingKnowledgeBase(path=Path("/tmp/test"))
+        assert kb.path == Path("/tmp/test")
+
+
+# =============================================================================
+# EvolvableSubjects
+# =============================================================================
+
+
+class TestEvolvableSubjects:
+    def test_evolvable_subjects_is_importable(self) -> None:
+        from rdagent.core.evolving_framework import EvolvableSubjects
+        assert EvolvableSubjects is not None
+
+    def test_evolvable_subjects_has_clone_method(self) -> None:
+        from rdagent.core.evolving_framework import EvolvableSubjects
+        assert hasattr(EvolvableSubjects, "clone")
+
+    def test_evolvable_subjects_is_instantiable(self) -> None:
+        from rdagent.core.evolving_framework import EvolvableSubjects
+        es = EvolvableSubjects()
+        assert es is not None
+
+
+# =============================================================================
+# Scenario
+# =============================================================================
+
+
+class TestScenario:
+    def test_scenario_is_abstract(self) -> None:
+        from rdagent.core.scenario import Scenario
+        assert hasattr(Scenario, "__abstractmethods__")
+
+    def test_scenario_has_rich_style_description(self) -> None:
+        from rdagent.core.scenario import Scenario
+        assert hasattr(Scenario, "rich_style_description")
+
+    def test_scenario_has_background(self) -> None:
+        from rdagent.core.scenario import Scenario
+        assert hasattr(Scenario, "background")
+
+    def test_scenario_source_data_default(self) -> None:
+        from rdagent.core.scenario import Scenario
+
+        class NoDataScen(Scenario):
+            @property
+            def background(self) -> str: return "bg"
+            @property
+            def rich_style_description(self) -> str: return "rd"
+            def get_scenario_all_desc(self, **kw) -> str: return "ad"
+            def get_runtime_environment(self) -> str: return "re"
+
+        scen = NoDataScen()
+        assert scen.source_data == ""
+
+
+# =============================================================================
+# Exception classes
+# =============================================================================
+
+
+class TestExceptionClasses:
+    def test_llm_unavailable_error_is_exception(self) -> None:
+        from rdagent.core.exception import LLMUnavailableError
+        assert issubclass(LLMUnavailableError, Exception)
+
+    def test_llm_unavailable_error_string_message(self) -> None:
+        from rdagent.core.exception import LLMUnavailableError
+        with pytest.raises(LLMUnavailableError, match="test message"):
+            raise LLMUnavailableError("test message")
+
+    def test_llm_unavailable_error_pickle(self) -> None:
+        from rdagent.core.exception import LLMUnavailableError
+        e = LLMUnavailableError("pickle me")
+        data = pickle.dumps(e)
+        e2 = pickle.loads(data)
+        assert str(e2) == "pickle me"
+
+    @pytest.mark.parametrize("message", ["", "short", "multi\nline\nmessage"])
+    def test_llm_unavailable_error_message_variants(self, message: str) -> None:
+        from rdagent.core.exception import LLMUnavailableError
+        e = LLMUnavailableError(message)
+        assert str(e) == message
+
+
+# =============================================================================
+# Conf module
+# =============================================================================
+
+
+class TestConfModule:
+    def test_rd_agent_settings_is_importable(self) -> None:
+        from rdagent.core.conf import RD_AGENT_SETTINGS
+        assert RD_AGENT_SETTINGS is not None
+
+    def test_rd_agent_settings_has_multi_proc_n(self) -> None:
+        from rdagent.core.conf import RD_AGENT_SETTINGS
+        assert hasattr(RD_AGENT_SETTINGS, "multi_proc_n")
+
+    def test_rd_agent_settings_get_max_parallel(self) -> None:
+        from rdagent.core.conf import RD_AGENT_SETTINGS
+        result = RD_AGENT_SETTINGS.get_max_parallel()
+        assert isinstance(result, int)
+
+
+# =============================================================================
+# Utils module
+# =============================================================================
+
+
+class TestCoreUtils:
+    def test_import_class_with_valid_path(self) -> None:
+        from rdagent.core.utils import import_class
+        cls = import_class("rdagent.core.evaluation.Feedback")
+        from rdagent.core.evaluation import Feedback
+        assert cls is Feedback
+
+    def test_import_class_raises_on_invalid_path(self) -> None:
+        from rdagent.core.utils import import_class
+        with pytest.raises((ValueError, ImportError, ModuleNotFoundError)):
+            import_class("rdagent.nonexistent.Class")
+
+
+# =============================================================================
+# Pickle safety
+# =============================================================================
+
+
+class TestPickleSafety:
+    def test_feedback_list_picklable(self) -> None:
+        from rdagent.core.evaluation import Feedback
+        items = [Feedback(), Feedback(), Feedback()]
+        data = pickle.dumps(items)
+        loaded = pickle.loads(data)
+        assert [bool(x) for x in loaded] == [True, True, True]
+
+    def test_evo_step_with_none_feedback_pickle(self) -> None:
+        from rdagent.core.evolving_framework import EvoStep
+        es = EvoStep(evolvable_subjects="s", queried_knowledge=None, feedback=None)
+        data = pickle.dumps(es)
+        es2 = pickle.loads(data)
+        assert es2.feedback is None
+
+
+# =============================================================================
+# Edge cases
+# =============================================================================
+
+
+class TestEdgeCases:
+    def test_evo_step_with_all_none(self) -> None:
+        from rdagent.core.evolving_framework import EvoStep
+        es = EvoStep(evolvable_subjects=None, queried_knowledge=None, feedback=None)
+        assert es.evolvable_subjects is None
+
+    def test_feedback_bool_edge_cases(self) -> None:
+        from rdagent.core.evaluation import Feedback
+
+        class AlwaysTrue(Feedback):
+            def __bool__(self): return True
+
+        class AlwaysFalse(Feedback):
+            def __bool__(self): return False
+
+        assert bool(AlwaysTrue()) is True
+        assert bool(AlwaysFalse()) is False
+
+
+# =============================================================================
+# Generic types
+# =============================================================================
+
+
+class TestGenericTypes:
+    def test_aspecific_exp_importable(self) -> None:
+        from rdagent.core.experiment import ASpecificExp
+        assert ASpecificExp is not None
+
+    def test_developer_is_importable(self) -> None:
+        from rdagent.core.developer import Developer
+        assert Developer is not None