mirror of
https://github.com/NicolasBohn/NexQuant.git
synced 2026-07-28 16:07:46 +00:00
TPTBusiness
a469692141
- costeer_deep: 112 tests (knowledge base, feedback, evaluators, auto-fixer) - workflow_deep: 84 tests (RDLoop, proposals, traces, hypothesis/pickle) - core_deep: 74 tests (developer, evaluator, exceptions, experiment, scenario) - llm_utils_deep: 49 tests (embeddings, APIBackend, edge cases, Unicode/NaN) - utils_deep: 122 tests (shrink_text, templates, md5_hash, property-based, stress)
521 lines
18 KiB
Python
521 lines
18 KiB
Python
"""Deep tests for rdagent.core: developer.py, evaluation.py, and related abstractions."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import pickle
|
|
import sys
|
|
from pathlib import Path
|
|
from typing import Any
|
|
from unittest.mock import MagicMock, patch
|
|
|
|
import pytest
|
|
|
|
PROJECT_ROOT = Path(__file__).parent.parent.parent
|
|
sys.path.insert(0, str(PROJECT_ROOT))
|
|
|
|
|
|
# =============================================================================
|
|
# Import safety
|
|
# =============================================================================
|
|
|
|
CORE_MODULES = [
|
|
"rdagent.core.developer",
|
|
"rdagent.core.evaluation",
|
|
"rdagent.core.experiment",
|
|
"rdagent.core.proposal",
|
|
"rdagent.core.scenario",
|
|
"rdagent.core.evolving_framework",
|
|
"rdagent.core.evolving_agent",
|
|
"rdagent.core.conf",
|
|
"rdagent.core.exception",
|
|
"rdagent.core.utils",
|
|
]
|
|
|
|
|
|
class TestCoreModuleImports:
|
|
@pytest.mark.parametrize("module_name", CORE_MODULES)
|
|
def test_module_importable(self, module_name: str) -> None:
|
|
import importlib
|
|
mod = importlib.import_module(module_name)
|
|
assert mod is not None
|
|
|
|
|
|
# =============================================================================
|
|
# Feedback
|
|
# =============================================================================
|
|
|
|
|
|
class TestFeedback:
|
|
def test_default_is_acceptable_returns_true(self) -> None:
|
|
from rdagent.core.evaluation import Feedback
|
|
fb = Feedback()
|
|
assert fb.is_acceptable() is True
|
|
|
|
def test_default_finished_returns_true(self) -> None:
|
|
from rdagent.core.evaluation import Feedback
|
|
fb = Feedback()
|
|
assert fb.finished() is True
|
|
|
|
def test_default_bool_is_true(self) -> None:
|
|
from rdagent.core.evaluation import Feedback
|
|
fb = Feedback()
|
|
assert bool(fb) is True
|
|
|
|
def test_is_acceptable_calls_bool(self) -> None:
|
|
from rdagent.core.evaluation import Feedback
|
|
|
|
class FalseFeedback(Feedback):
|
|
def __bool__(self) -> bool:
|
|
return False
|
|
|
|
fb = FalseFeedback()
|
|
assert fb.is_acceptable() is False
|
|
|
|
def test_finished_can_be_overridden(self) -> None:
|
|
from rdagent.core.evaluation import Feedback
|
|
|
|
class CustomFinish(Feedback):
|
|
def __bool__(self) -> bool:
|
|
return False
|
|
|
|
def finished(self) -> bool:
|
|
return True
|
|
|
|
fb = CustomFinish()
|
|
assert fb.finished() is True
|
|
assert bool(fb) is False
|
|
|
|
def test_pickle_safety(self) -> None:
|
|
from rdagent.core.evaluation import Feedback
|
|
fb = Feedback()
|
|
data = pickle.dumps(fb)
|
|
fb2 = pickle.loads(data)
|
|
assert isinstance(fb2, Feedback)
|
|
assert bool(fb2) is True
|
|
|
|
|
|
# =============================================================================
|
|
# Evaluator / EvaluableObj
|
|
# =============================================================================
|
|
|
|
|
|
class TestEvaluator:
|
|
def test_evaluator_is_abstract(self) -> None:
|
|
from rdagent.core.evaluation import Evaluator
|
|
assert hasattr(Evaluator, "evaluate")
|
|
|
|
def test_concrete_evaluator_must_implement_evaluate(self) -> None:
|
|
from rdagent.core.evaluation import Evaluator, Feedback
|
|
|
|
class ConcreteEvaluator(Evaluator):
|
|
def evaluate(self, eo) -> Feedback:
|
|
return Feedback()
|
|
|
|
ev = ConcreteEvaluator()
|
|
result = ev.evaluate(None)
|
|
assert isinstance(result, Feedback)
|
|
|
|
@pytest.mark.parametrize("input_eo", [None, "string", {"key": "value"}, [1, 2, 3]])
|
|
def test_concrete_evaluator_accepts_any_input(self, input_eo: Any) -> None:
|
|
from rdagent.core.evaluation import Evaluator, Feedback
|
|
|
|
class FlexibleEvaluator(Evaluator):
|
|
def evaluate(self, eo) -> Feedback:
|
|
return Feedback()
|
|
|
|
ev = FlexibleEvaluator()
|
|
result = ev.evaluate(input_eo)
|
|
assert isinstance(result, Feedback)
|
|
|
|
|
|
class TestEvaluableObj:
|
|
def test_evaluable_obj_is_importable(self) -> None:
|
|
from rdagent.core.evaluation import EvaluableObj
|
|
assert EvaluableObj is not None
|
|
|
|
def test_evaluable_obj_can_be_instantiated(self) -> None:
|
|
from rdagent.core.evaluation import EvaluableObj
|
|
obj = EvaluableObj()
|
|
assert isinstance(obj, EvaluableObj)
|
|
|
|
|
|
# =============================================================================
|
|
# Developer
|
|
# =============================================================================
|
|
|
|
|
|
class TestDeveloperBase:
|
|
def test_developer_is_importable(self) -> None:
|
|
from rdagent.core.developer import Developer
|
|
assert Developer is not None
|
|
|
|
def test_developer_stores_scenario(self) -> None:
|
|
from rdagent.core.developer import Developer
|
|
from rdagent.core.experiment import ASpecificExp
|
|
|
|
class ConcreteDev(Developer[ASpecificExp]):
|
|
def develop(self, exp: ASpecificExp) -> ASpecificExp:
|
|
return exp
|
|
|
|
scen = MagicMock()
|
|
dev = ConcreteDev(scen)
|
|
assert dev.scen is scen
|
|
|
|
def test_develop_modifies_in_place(self) -> None:
|
|
from rdagent.core.developer import Developer
|
|
from rdagent.core.experiment import ASpecificExp
|
|
|
|
class ModifyingDev(Developer[ASpecificExp]):
|
|
def develop(self, exp: ASpecificExp) -> ASpecificExp:
|
|
exp._modified = True
|
|
return exp
|
|
|
|
dev = ModifyingDev(MagicMock())
|
|
exp = MagicMock()
|
|
result = dev.develop(exp)
|
|
assert result._modified is True
|
|
|
|
|
|
# =============================================================================
|
|
# Experiment classes
|
|
# =============================================================================
|
|
|
|
|
|
class TestExperiment:
|
|
def test_task_has_get_task_information(self) -> None:
|
|
from rdagent.core.experiment import Task
|
|
assert hasattr(Task, "get_task_information")
|
|
|
|
def test_workspace_has_execute(self) -> None:
|
|
from rdagent.core.experiment import Workspace
|
|
assert hasattr(Workspace, "execute")
|
|
|
|
def test_fb_workspace_inject_files(self) -> None:
|
|
from rdagent.core.experiment import FBWorkspace
|
|
ws = FBWorkspace()
|
|
ws.inject_files(**{"factor.py": "def calc(): pass"})
|
|
code = ws.all_codes
|
|
assert "def calc" in code
|
|
|
|
def test_fb_workspace_copy_returns_new_instance(self) -> None:
|
|
from rdagent.core.experiment import FBWorkspace
|
|
ws = FBWorkspace()
|
|
ws.inject_files(**{"test.py": "x=1"})
|
|
ws2 = ws.copy()
|
|
assert ws2 is not ws
|
|
assert ws2.all_codes == ws.all_codes
|
|
|
|
def test_fb_workspace_pickle_safety(self) -> None:
|
|
from rdagent.core.experiment import FBWorkspace
|
|
ws = FBWorkspace()
|
|
ws.inject_files(**{"factor.py": "x=1", "utils.py": "y=2"})
|
|
data = pickle.dumps(ws)
|
|
ws2 = pickle.loads(data)
|
|
assert isinstance(ws2, FBWorkspace)
|
|
|
|
@pytest.mark.parametrize("files", [
|
|
{},
|
|
{"a.py": ""},
|
|
{"a.py": "x=1", "b.py": "y=2", "c.py": "z=3"},
|
|
])
|
|
def test_fb_workspace_file_variants(self, files: dict) -> None:
|
|
from rdagent.core.experiment import FBWorkspace
|
|
ws = FBWorkspace()
|
|
ws.inject_files(**files)
|
|
assert isinstance(ws.all_codes, str)
|
|
|
|
def test_aspecific_exp_is_importable(self) -> None:
|
|
from rdagent.core.experiment import ASpecificExp
|
|
assert ASpecificExp is not None
|
|
|
|
|
|
# =============================================================================
|
|
# EvoStep
|
|
# =============================================================================
|
|
|
|
|
|
class TestEvoStep:
|
|
def test_default_construction(self) -> None:
|
|
from rdagent.core.evolving_framework import EvoStep
|
|
es = EvoStep(evolvable_subjects="evo")
|
|
assert es.evolvable_subjects == "evo"
|
|
assert es.queried_knowledge is None
|
|
assert es.feedback is None
|
|
|
|
def test_full_construction(self) -> None:
|
|
from rdagent.core.evolving_framework import EvoStep, QueriedKnowledge
|
|
qk = QueriedKnowledge()
|
|
es = EvoStep(evolvable_subjects="subj", queried_knowledge=qk, feedback="fb")
|
|
assert es.queried_knowledge is qk
|
|
assert es.feedback == "fb"
|
|
|
|
def test_equality_by_reference(self) -> None:
|
|
from rdagent.core.evolving_framework import EvoStep
|
|
es1 = EvoStep(evolvable_subjects="a")
|
|
es2 = EvoStep(evolvable_subjects="a")
|
|
assert es1 == es2
|
|
|
|
def test_pickle_safety(self) -> None:
|
|
from rdagent.core.evolving_framework import EvoStep
|
|
es = EvoStep(evolvable_subjects="subj", feedback="good")
|
|
data = pickle.dumps(es)
|
|
es2 = pickle.loads(data)
|
|
assert es2.evolvable_subjects == "subj"
|
|
assert es2.feedback == "good"
|
|
|
|
def test_with_none_values(self) -> None:
|
|
from rdagent.core.evolving_framework import EvoStep
|
|
es = EvoStep(evolvable_subjects=None, queried_knowledge=None, feedback=None)
|
|
assert es.evolvable_subjects is None
|
|
|
|
|
|
# =============================================================================
|
|
# Knowledge and QueriedKnowledge
|
|
# =============================================================================
|
|
|
|
|
|
class TestKnowledge:
|
|
def test_knowledge_is_importable(self) -> None:
|
|
from rdagent.core.evolving_framework import Knowledge
|
|
assert Knowledge is not None
|
|
|
|
def test_knowledge_can_be_instantiated(self) -> None:
|
|
from rdagent.core.evolving_framework import Knowledge
|
|
k = Knowledge()
|
|
assert isinstance(k, Knowledge)
|
|
|
|
def test_knowledge_pickle_safety(self) -> None:
|
|
from rdagent.core.evolving_framework import Knowledge
|
|
k = Knowledge()
|
|
data = pickle.dumps(k)
|
|
k2 = pickle.loads(data)
|
|
assert isinstance(k2, Knowledge)
|
|
|
|
|
|
class TestQueriedKnowledge:
|
|
def test_default_construction(self) -> None:
|
|
from rdagent.core.evolving_framework import QueriedKnowledge
|
|
qk = QueriedKnowledge()
|
|
assert isinstance(qk, QueriedKnowledge)
|
|
|
|
def test_pickle_safety(self) -> None:
|
|
from rdagent.core.evolving_framework import QueriedKnowledge
|
|
qk = QueriedKnowledge()
|
|
data = pickle.dumps(qk)
|
|
qk2 = pickle.loads(data)
|
|
assert isinstance(qk2, QueriedKnowledge)
|
|
|
|
|
|
# =============================================================================
|
|
# RAGStrategy / RAGEvaluator
|
|
# =============================================================================
|
|
|
|
|
|
class TestRAGStrategy:
|
|
def test_rag_strategy_has_methods(self) -> None:
|
|
from rdagent.core.evolving_framework import RAGStrategy
|
|
assert hasattr(RAGStrategy, "generate_knowledge")
|
|
assert hasattr(RAGStrategy, "query")
|
|
|
|
|
|
class TestRAGEvaluator:
|
|
def test_rage_evaluator_is_importable(self) -> None:
|
|
from rdagent.core.evolving_agent import RAGEvaluator
|
|
assert RAGEvaluator is not None
|
|
|
|
|
|
# =============================================================================
|
|
# EvolvingKnowledgeBase
|
|
# =============================================================================
|
|
|
|
|
|
class TestEvolvingKnowledgeBase:
|
|
def test_has_query_method(self) -> None:
|
|
from rdagent.core.evolving_framework import EvolvingKnowledgeBase
|
|
assert hasattr(EvolvingKnowledgeBase, "query")
|
|
|
|
def test_takes_optional_path_argument(self) -> None:
|
|
from rdagent.core.evolving_framework import EvolvingKnowledgeBase
|
|
kb = EvolvingKnowledgeBase(path=Path("/tmp/test"))
|
|
assert kb.path == Path("/tmp/test")
|
|
|
|
|
|
# =============================================================================
|
|
# EvolvableSubjects
|
|
# =============================================================================
|
|
|
|
|
|
class TestEvolvableSubjects:
|
|
def test_evolvable_subjects_is_importable(self) -> None:
|
|
from rdagent.core.evolving_framework import EvolvableSubjects
|
|
assert EvolvableSubjects is not None
|
|
|
|
def test_evolvable_subjects_has_clone_method(self) -> None:
|
|
from rdagent.core.evolving_framework import EvolvableSubjects
|
|
assert hasattr(EvolvableSubjects, "clone")
|
|
|
|
def test_evolvable_subjects_is_instantiable(self) -> None:
|
|
from rdagent.core.evolving_framework import EvolvableSubjects
|
|
es = EvolvableSubjects()
|
|
assert es is not None
|
|
|
|
|
|
# =============================================================================
|
|
# Scenario
|
|
# =============================================================================
|
|
|
|
|
|
class TestScenario:
|
|
def test_scenario_is_abstract(self) -> None:
|
|
from rdagent.core.scenario import Scenario
|
|
assert hasattr(Scenario, "__abstractmethods__")
|
|
|
|
def test_scenario_has_rich_style_description(self) -> None:
|
|
from rdagent.core.scenario import Scenario
|
|
assert hasattr(Scenario, "rich_style_description")
|
|
|
|
def test_scenario_has_background(self) -> None:
|
|
from rdagent.core.scenario import Scenario
|
|
assert hasattr(Scenario, "background")
|
|
|
|
def test_scenario_source_data_default(self) -> None:
|
|
from rdagent.core.scenario import Scenario
|
|
|
|
class NoDataScen(Scenario):
|
|
@property
|
|
def background(self) -> str: return "bg"
|
|
@property
|
|
def rich_style_description(self) -> str: return "rd"
|
|
def get_scenario_all_desc(self, **kw) -> str: return "ad"
|
|
def get_runtime_environment(self) -> str: return "re"
|
|
|
|
scen = NoDataScen()
|
|
assert scen.source_data == ""
|
|
|
|
|
|
# =============================================================================
|
|
# Exception classes
|
|
# =============================================================================
|
|
|
|
|
|
class TestExceptionClasses:
|
|
def test_llm_unavailable_error_is_exception(self) -> None:
|
|
from rdagent.core.exception import LLMUnavailableError
|
|
assert issubclass(LLMUnavailableError, Exception)
|
|
|
|
def test_llm_unavailable_error_string_message(self) -> None:
|
|
from rdagent.core.exception import LLMUnavailableError
|
|
with pytest.raises(LLMUnavailableError, match="test message"):
|
|
raise LLMUnavailableError("test message")
|
|
|
|
def test_llm_unavailable_error_pickle(self) -> None:
|
|
from rdagent.core.exception import LLMUnavailableError
|
|
e = LLMUnavailableError("pickle me")
|
|
data = pickle.dumps(e)
|
|
e2 = pickle.loads(data)
|
|
assert str(e2) == "pickle me"
|
|
|
|
@pytest.mark.parametrize("message", ["", "short", "multi\nline\nmessage"])
|
|
def test_llm_unavailable_error_message_variants(self, message: str) -> None:
|
|
from rdagent.core.exception import LLMUnavailableError
|
|
e = LLMUnavailableError(message)
|
|
assert str(e) == message
|
|
|
|
|
|
# =============================================================================
|
|
# Conf module
|
|
# =============================================================================
|
|
|
|
|
|
class TestConfModule:
|
|
def test_rd_agent_settings_is_importable(self) -> None:
|
|
from rdagent.core.conf import RD_AGENT_SETTINGS
|
|
assert RD_AGENT_SETTINGS is not None
|
|
|
|
def test_rd_agent_settings_has_multi_proc_n(self) -> None:
|
|
from rdagent.core.conf import RD_AGENT_SETTINGS
|
|
assert hasattr(RD_AGENT_SETTINGS, "multi_proc_n")
|
|
|
|
def test_rd_agent_settings_get_max_parallel(self) -> None:
|
|
from rdagent.core.conf import RD_AGENT_SETTINGS
|
|
result = RD_AGENT_SETTINGS.get_max_parallel()
|
|
assert isinstance(result, int)
|
|
|
|
|
|
# =============================================================================
|
|
# Utils module
|
|
# =============================================================================
|
|
|
|
|
|
class TestCoreUtils:
|
|
def test_import_class_with_valid_path(self) -> None:
|
|
from rdagent.core.utils import import_class
|
|
cls = import_class("rdagent.core.evaluation.Feedback")
|
|
from rdagent.core.evaluation import Feedback
|
|
assert cls is Feedback
|
|
|
|
def test_import_class_raises_on_invalid_path(self) -> None:
|
|
from rdagent.core.utils import import_class
|
|
with pytest.raises((ValueError, ImportError, ModuleNotFoundError)):
|
|
import_class("rdagent.nonexistent.Class")
|
|
|
|
|
|
# =============================================================================
|
|
# Pickle safety
|
|
# =============================================================================
|
|
|
|
|
|
class TestPickleSafety:
|
|
def test_feedback_list_picklable(self) -> None:
|
|
from rdagent.core.evaluation import Feedback
|
|
items = [Feedback(), Feedback(), Feedback()]
|
|
data = pickle.dumps(items)
|
|
loaded = pickle.loads(data)
|
|
assert [bool(x) for x in loaded] == [True, True, True]
|
|
|
|
def test_evo_step_with_none_feedback_pickle(self) -> None:
|
|
from rdagent.core.evolving_framework import EvoStep
|
|
es = EvoStep(evolvable_subjects="s", queried_knowledge=None, feedback=None)
|
|
data = pickle.dumps(es)
|
|
es2 = pickle.loads(data)
|
|
assert es2.feedback is None
|
|
|
|
|
|
# =============================================================================
|
|
# Edge cases
|
|
# =============================================================================
|
|
|
|
|
|
class TestEdgeCases:
|
|
def test_evo_step_with_all_none(self) -> None:
|
|
from rdagent.core.evolving_framework import EvoStep
|
|
es = EvoStep(evolvable_subjects=None, queried_knowledge=None, feedback=None)
|
|
assert es.evolvable_subjects is None
|
|
|
|
def test_feedback_bool_edge_cases(self) -> None:
|
|
from rdagent.core.evaluation import Feedback
|
|
|
|
class AlwaysTrue(Feedback):
|
|
def __bool__(self): return True
|
|
|
|
class AlwaysFalse(Feedback):
|
|
def __bool__(self): return False
|
|
|
|
assert bool(AlwaysTrue()) is True
|
|
assert bool(AlwaysFalse()) is False
|
|
|
|
|
|
# =============================================================================
|
|
# Generic types
|
|
# =============================================================================
|
|
|
|
|
|
class TestGenericTypes:
|
|
def test_aspecific_exp_importable(self) -> None:
|
|
from rdagent.core.experiment import ASpecificExp
|
|
assert ASpecificExp is not None
|
|
|
|
def test_developer_is_importable(self) -> None:
|
|
from rdagent.core.developer import Developer
|
|
assert Developer is not None
|