mirror of
https://github.com/NicolasBohn/NexQuant.git
synced 2026-07-27 15:37:44 +00:00
test: add test import (#242)
* add test import * format with isort * format with black * merge main * fix pytest error * fix pytest error * fix pytest error * fix pytest error * format with black * fix pytest error * fix pytest error * fix pytest error * fix pytest error * fix pytest error * format with isort * Exclude entrance * Add offline test * auto-lint * update coverage rate --------- Co-authored-by: Young <afe.young@gmail.com>
This commit is contained in:
Linlang
@@ -20,7 +20,7 @@ jobs:
|
||||
- run: env | sort
|
||||
- run: make dev
|
||||
- name: lint test docs and build
|
||||
run: make lint docs-gen # test docs build
|
||||
run: make lint docs-gen test-offline # test docs build
|
||||
strategy:
|
||||
matrix:
|
||||
python-version:
|
||||
|
||||
@@ -141,10 +141,21 @@ test-run:
|
||||
$(PIPRUN) python -m coverage run --concurrency=multiprocessing -m pytest --ignore test/scripts
|
||||
$(PIPRUN) python -m coverage combine
|
||||
|
||||
test-run-offline:
|
||||
# some test that does not require api calling
|
||||
$(PIPRUN) python -m coverage erase
|
||||
$(PIPRUN) python -m coverage run --concurrency=multiprocessing -m pytest -m "offline" --ignore test/scripts
|
||||
$(PIPRUN) python -m coverage combine
|
||||
|
||||
# Generate coverage report for terminal and xml.
|
||||
# TODO: we may have higher coverage rate if we have more test
|
||||
test: test-run
|
||||
$(PIPRUN) python -m coverage report --fail-under 80
|
||||
$(PIPRUN) python -m coverage xml --fail-under 80
|
||||
$(PIPRUN) python -m coverage report --fail-under 20 # 80
|
||||
$(PIPRUN) python -m coverage xml --fail-under 20 # 80
|
||||
|
||||
test-offline: test-run-offline
|
||||
$(PIPRUN) python -m coverage report --fail-under 20 # 80
|
||||
$(PIPRUN) python -m coverage xml --fail-under 20 # 80
|
||||
|
||||
########################################################################################
|
||||
# Package
|
||||
|
||||
@@ -15,27 +15,28 @@ from rdagent.scenarios.qlib.factor_experiment_loader.json_loader import (
|
||||
FactorTestCaseLoaderFromJsonFile,
|
||||
)
|
||||
|
||||
# 1.read the settings
|
||||
bs = BenchmarkSettings()
|
||||
if __name__ == "__main__":
|
||||
# 1.read the settings
|
||||
bs = BenchmarkSettings()
|
||||
|
||||
# 2.read and prepare the eval_data
|
||||
test_cases = FactorTestCaseLoaderFromJsonFile().load(bs.bench_data_path)
|
||||
# 2.read and prepare the eval_data
|
||||
test_cases = FactorTestCaseLoaderFromJsonFile().load(bs.bench_data_path)
|
||||
|
||||
# 3.declare the method to be tested and pass the arguments.
|
||||
# 3.declare the method to be tested and pass the arguments.
|
||||
|
||||
scen: Scenario = import_class(FACTOR_PROP_SETTING.scen)()
|
||||
generate_method = import_class(bs.bench_method_cls)(scen=scen)
|
||||
# 4.declare the eval method and pass the arguments.
|
||||
eval_method = FactorImplementEval(
|
||||
method=generate_method,
|
||||
test_cases=test_cases,
|
||||
scen=scen,
|
||||
catch_eval_except=True,
|
||||
test_round=bs.bench_test_round,
|
||||
)
|
||||
scen: Scenario = import_class(FACTOR_PROP_SETTING.scen)()
|
||||
generate_method = import_class(bs.bench_method_cls)(scen=scen)
|
||||
# 4.declare the eval method and pass the arguments.
|
||||
eval_method = FactorImplementEval(
|
||||
method=generate_method,
|
||||
test_cases=test_cases,
|
||||
scen=scen,
|
||||
catch_eval_except=True,
|
||||
test_round=bs.bench_test_round,
|
||||
)
|
||||
|
||||
# 5.run the eval
|
||||
res = eval_method.eval()
|
||||
# 5.run the eval
|
||||
res = eval_method.eval()
|
||||
|
||||
# 6.save the result
|
||||
logger.log_object(res)
|
||||
# 6.save the result
|
||||
logger.log_object(res)
|
||||
|
||||
@@ -7,35 +7,36 @@ from rdagent.scenarios.qlib.experiment.model_experiment import (
|
||||
QlibModelScenario,
|
||||
)
|
||||
|
||||
DIRNAME = Path(__file__).absolute().resolve().parent
|
||||
if __name__ == "__main__":
|
||||
DIRNAME = Path(__file__).absolute().resolve().parent
|
||||
|
||||
from rdagent.components.coder.model_coder.benchmark.eval import ModelImpValEval
|
||||
from rdagent.components.coder.model_coder.one_shot import ModelCodeWriter
|
||||
from rdagent.components.coder.model_coder.benchmark.eval import ModelImpValEval
|
||||
from rdagent.components.coder.model_coder.one_shot import ModelCodeWriter
|
||||
|
||||
bench_folder = DIRNAME.parent.parent / "components" / "coder" / "model_coder" / "benchmark"
|
||||
mtl = ModelTaskLoaderJson(str(bench_folder / "model_dict.json"))
|
||||
bench_folder = DIRNAME.parent.parent / "components" / "coder" / "model_coder" / "benchmark"
|
||||
mtl = ModelTaskLoaderJson(str(bench_folder / "model_dict.json"))
|
||||
|
||||
task_l = mtl.load()
|
||||
task_l = mtl.load()
|
||||
|
||||
task_l = [t for t in task_l if t.name == "A-DGN"] # FIXME: other models does not work well
|
||||
task_l = [t for t in task_l if t.name == "A-DGN"] # FIXME: other models does not work well
|
||||
|
||||
model_experiment = QlibModelExperiment(sub_tasks=task_l)
|
||||
# mtg = ModelCodeWriter(scen=QlibModelScenario())
|
||||
mtg = ModelCoSTEER(scen=QlibModelScenario())
|
||||
model_experiment = QlibModelExperiment(sub_tasks=task_l)
|
||||
# mtg = ModelCodeWriter(scen=QlibModelScenario())
|
||||
mtg = ModelCoSTEER(scen=QlibModelScenario())
|
||||
|
||||
model_experiment = mtg.develop(model_experiment)
|
||||
model_experiment = mtg.develop(model_experiment)
|
||||
|
||||
# TODO: Align it with the benchmark framework after @wenjun's refine the evaluation part.
|
||||
# Currently, we just handcraft a workflow for fast evaluation.
|
||||
# TODO: Align it with the benchmark framework after @wenjun's refine the evaluation part.
|
||||
# Currently, we just handcraft a workflow for fast evaluation.
|
||||
|
||||
mil = ModelWsLoader(bench_folder / "gt_code")
|
||||
mil = ModelWsLoader(bench_folder / "gt_code")
|
||||
|
||||
mie = ModelImpValEval()
|
||||
# Evaluation:
|
||||
eval_l = []
|
||||
for impl in model_experiment.sub_workspace_list:
|
||||
print(impl.target_task)
|
||||
gt_impl = mil.load(impl.target_task)
|
||||
eval_l.append(mie.evaluate(gt_impl, impl))
|
||||
mie = ModelImpValEval()
|
||||
# Evaluation:
|
||||
eval_l = []
|
||||
for impl in model_experiment.sub_workspace_list:
|
||||
print(impl.target_task)
|
||||
gt_impl = mil.load(impl.target_task)
|
||||
eval_l.append(mie.evaluate(gt_impl, impl))
|
||||
|
||||
print(eval_l)
|
||||
print(eval_l)
|
||||
|
||||
@@ -1,112 +0,0 @@
|
||||
import pickle
|
||||
|
||||
from rdagent.app.qlib_rd_loop.conf import PROP_SETTING
|
||||
from rdagent.core.developer import Developer
|
||||
from rdagent.core.exception import ModelEmptyError
|
||||
from rdagent.core.proposal import (
|
||||
Hypothesis2Experiment,
|
||||
HypothesisExperiment2Feedback,
|
||||
HypothesisGen,
|
||||
Trace,
|
||||
)
|
||||
from rdagent.core.scenario import Scenario
|
||||
from rdagent.core.utils import import_class
|
||||
from rdagent.log import rdagent_logger as logger
|
||||
|
||||
|
||||
# TODO: we can design a workflow that can automatically save session and traceback in the future
|
||||
class Model_RD_Agent:
|
||||
def __init__(self):
|
||||
self.scen: Scenario = import_class(PROP_SETTING.model_scen)()
|
||||
self.hypothesis_gen: HypothesisGen = import_class(PROP_SETTING.model_hypothesis_gen)(self.scen)
|
||||
self.hypothesis2experiment: Hypothesis2Experiment = import_class(PROP_SETTING.model_hypothesis2experiment)()
|
||||
self.qlib_model_coder: Developer = import_class(PROP_SETTING.model_coder)(self.scen)
|
||||
self.qlib_model_runner: Developer = import_class(PROP_SETTING.model_runner)(self.scen)
|
||||
self.qlib_model_summarizer: HypothesisExperiment2Feedback = import_class(PROP_SETTING.model_summarizer)(
|
||||
self.scen
|
||||
)
|
||||
self.trace = Trace(scen=self.scen)
|
||||
|
||||
def generate_hypothesis(self):
|
||||
hypothesis = self.hypothesis_gen.gen(self.trace)
|
||||
self.dump_objects(hypothesis=hypothesis, trace=self.trace, filename="step_hypothesis.pkl")
|
||||
return hypothesis
|
||||
|
||||
def convert_hypothesis(self, hypothesis):
|
||||
exp = self.hypothesis2experiment.convert(hypothesis, self.trace)
|
||||
self.dump_objects(exp=exp, hypothesis=hypothesis, trace=self.trace, filename="step_experiment.pkl")
|
||||
return exp
|
||||
|
||||
def generate_code(self, exp):
|
||||
exp = self.qlib_model_coder.develop(exp)
|
||||
self.dump_objects(exp=exp, trace=self.trace, filename="step_code.pkl")
|
||||
return exp
|
||||
|
||||
def run_experiment(self, exp):
|
||||
exp = self.qlib_model_runner.develop(exp)
|
||||
self.dump_objects(exp=exp, trace=self.trace, filename="step_run.pkl")
|
||||
return exp
|
||||
|
||||
def generate_feedback(self, exp, hypothesis):
|
||||
feedback = self.qlib_model_summarizer.generate_feedback(exp, hypothesis, self.trace)
|
||||
self.dump_objects(
|
||||
exp=exp, hypothesis=hypothesis, feedback=feedback, trace=self.trace, filename="step_feedback.pkl"
|
||||
)
|
||||
return feedback
|
||||
|
||||
def append_to_trace(self, hypothesis, exp, feedback):
|
||||
self.trace.hist.append((hypothesis, exp, feedback))
|
||||
self.dump_objects(trace=self.trace, filename="step_trace.pkl")
|
||||
|
||||
def dump_objects(self, exp=None, hypothesis=None, feedback=None, trace=None, filename="dumped_objects.pkl"):
|
||||
with open(filename, "wb") as f:
|
||||
pickle.dump((exp, hypothesis, feedback, trace or self.trace), f)
|
||||
|
||||
def load_objects(self, filename):
|
||||
with open(filename, "rb") as f:
|
||||
return pickle.load(f)
|
||||
|
||||
|
||||
def process_steps(agent):
|
||||
# Load trace if available
|
||||
try:
|
||||
_, _, _, trace = agent.load_objects("step_trace.pkl")
|
||||
agent.trace = trace
|
||||
print(trace.get_sota_hypothesis_and_experiment())
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
|
||||
# # # Step 1: Generate hypothesis
|
||||
# try:
|
||||
# _, hypothesis, _, _ = agent.load_objects('step_hypothesis.pkl')
|
||||
# except FileNotFoundError:
|
||||
hypothesis = agent.generate_hypothesis()
|
||||
|
||||
# # # Step 2: Convert hypothesis
|
||||
# try:
|
||||
# exp, _, _, _ = agent.load_objects('step_experiment.pkl')
|
||||
# except FileNotFoundError:
|
||||
# exp = agent.convert_hypothesis(hypothesis)
|
||||
|
||||
# # # Step 3: Generate code
|
||||
# try:
|
||||
# exp, _, _, _ = agent.load_objects('step_code.pkl')
|
||||
# except FileNotFoundError:
|
||||
# exp = agent.generate_code(exp)
|
||||
|
||||
# # # Step 4: Run experiment
|
||||
# try:
|
||||
# exp, _, _, _ = agent.load_objects('step_run.pkl')
|
||||
# except FileNotFoundError:
|
||||
# exp = agent.run_experiment(exp)
|
||||
|
||||
# # Step 5: Generate feedback
|
||||
# feedback = agent.generate_feedback(exp, hypothesis)
|
||||
|
||||
# # Step 6: Append to trace
|
||||
# agent.append_to_trace(hypothesis, exp, feedback)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
agent = Model_RD_Agent()
|
||||
process_steps(agent)
|
||||
@@ -1,92 +0,0 @@
|
||||
"""
|
||||
This file will be removed in the future and replaced by
|
||||
- rdagent/app/model_implementation/eval.py
|
||||
"""
|
||||
|
||||
import os
|
||||
|
||||
# randomly generate a input graph, node_feature and edge_index
|
||||
# 1000 nodes, 128 dim node feature, 2000 edges
|
||||
import torch
|
||||
from dotenv import load_dotenv
|
||||
|
||||
from rdagent.components.coder.model_coder.CoSTEER.evaluators import (
|
||||
shape_evaluator,
|
||||
value_evaluator,
|
||||
)
|
||||
from rdagent.oai.llm_utils import APIBackend
|
||||
|
||||
assert load_dotenv()
|
||||
formula_info = {
|
||||
"name": "Anti-Symmetric Deep Graph Network (A-DGN)",
|
||||
"description": "A framework for stable and non-dissipative DGN design. It ensures long-range information preservation between nodes and prevents gradient vanishing or explosion during training.",
|
||||
"formulation": "x_u^{(l)} = x_u^{(l-1)} + \\epsilon \\sigma \\left( W^T x_u^{(l-1)} + \\Phi(X^{(l-1)}, N_u) + b \\right)",
|
||||
"variables": {
|
||||
"x_u^{(l)}": "The state of node u at layer l",
|
||||
"\\epsilon": "The step size in the Euler discretization",
|
||||
"\\sigma": "A monotonically non-decreasing activation function",
|
||||
"W": "An anti-symmetric weight matrix",
|
||||
"X^{(l-1)}": "The node feature matrix at layer l-1",
|
||||
"N_u": "The set of neighbors of node u",
|
||||
"b": "A bias vector",
|
||||
},
|
||||
}
|
||||
|
||||
system_prompt = "You are an assistant whose job is to answer user's question."
|
||||
user_prompt = "With the following given information, write a python code using pytorch and torch_geometric to implement the model. This model is in the graph learning field, only have one layer. The input will be node_feature [num_nodes, dim_feature] and edge_index [2, num_edges], and they should be loaded from the files 'node_features.pt' and 'edge_index.pt'. There is not edge attribute or edge weight as input. The model should detect the node_feature and edge_index shape, if there is Linear transformation layer in the model, the input and output shape should be consistent. The in_channels is the dimension of the node features. You code should contain additional 'if __name__ == '__main__', where you should load the node_feature and edge_index from the files and run the model, and save the output to a file 'llm_output.pt'. Implement the model forward function based on the following information: model formula information. 1. model name: {}, 2. model description: {}, 3. model formulation: {}, 4. model variables: {}. You must complete the forward function as far as you can do.".format(
|
||||
formula_info["name"],
|
||||
formula_info["description"],
|
||||
formula_info["formulation"],
|
||||
formula_info["variables"],
|
||||
)
|
||||
|
||||
resp = APIBackend(use_chat_cache=False).build_messages_and_create_chat_completion(user_prompt, system_prompt)
|
||||
|
||||
print(resp)
|
||||
|
||||
# take the code part from the response and save it to a file, the code is covered in the ```python``` block
|
||||
code = resp.split("```python")[1].split("```")[0]
|
||||
with open("llm_code.py", "w") as f:
|
||||
f.write(code)
|
||||
|
||||
average_shape_eval = []
|
||||
average_value_eval = []
|
||||
for test_mode in ["zeros", "ones", "randn"]:
|
||||
if test_mode == "zeros":
|
||||
node_feature = torch.zeros(1000, 128)
|
||||
elif test_mode == "ones":
|
||||
node_feature = torch.ones(1000, 128)
|
||||
elif test_mode == "randn":
|
||||
node_feature = torch.randn(1000, 128)
|
||||
edge_index = torch.randint(0, 1000, (2, 2000))
|
||||
|
||||
torch.save(node_feature, "node_features.pt")
|
||||
torch.save(edge_index, "edge_index.pt")
|
||||
|
||||
try:
|
||||
os.system("python llm_code.py")
|
||||
except:
|
||||
print("Error in running the LLM code")
|
||||
os.system("python gt_code.py")
|
||||
os.system("rm edge_index.pt")
|
||||
os.system("rm node_features.pt")
|
||||
# load the output and print the shape
|
||||
|
||||
try:
|
||||
llm_output = torch.load("llm_output.pt")
|
||||
except:
|
||||
llm_output = None
|
||||
gt_output = torch.load("gt_output.pt")
|
||||
|
||||
average_shape_eval.append(shape_evaluator(llm_output, gt_output)[1])
|
||||
average_value_eval.append(value_evaluator(llm_output, gt_output)[1])
|
||||
|
||||
print("Shape evaluation: ", average_shape_eval[-1])
|
||||
print("Value evaluation:super().develop(task_l) ", average_value_eval[-1])
|
||||
|
||||
os.system("rm llm_output.pt")
|
||||
os.system("rm gt_output.pt")
|
||||
os.system("rm llm_code.py")
|
||||
|
||||
print("Average shape evaluation: ", sum(average_shape_eval) / len(average_shape_eval))
|
||||
print("Average value evaluation: ", sum(average_value_eval) / len(average_value_eval))
|
||||
@@ -70,3 +70,8 @@ st-theme
|
||||
# kaggle crawler
|
||||
selenium
|
||||
kaggle
|
||||
|
||||
seaborn
|
||||
|
||||
# This is a temporary package installed to pass the test_import test
|
||||
xgboost
|
||||
|
||||
@@ -0,0 +1,46 @@
|
||||
import importlib
|
||||
import os
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.mark.offline
|
||||
class TestRDAgentImports(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.rdagent_directory = Path(__file__).resolve().parent.parent.parent
|
||||
cls.modules = list(cls.import_all_modules_from_directory(cls.rdagent_directory))
|
||||
|
||||
@staticmethod
|
||||
def import_all_modules_from_directory(directory):
|
||||
for file in directory.joinpath("rdagent").rglob("*.py"):
|
||||
fstr = str(file)
|
||||
if "meta_tpl" in fstr:
|
||||
continue
|
||||
if "_template" in fstr:
|
||||
continue
|
||||
if (
|
||||
fstr.endswith("rdagent/log/ui/app.py")
|
||||
or fstr.endswith("rdagent/app/cli.py")
|
||||
or fstr.endswith("rdagent/app/CI/run.py")
|
||||
):
|
||||
# the entrance points
|
||||
continue
|
||||
|
||||
yield fstr[fstr.index("rdagent") : -3].replace("/", ".")
|
||||
|
||||
def test_import_modules(self):
|
||||
print(self.modules)
|
||||
for module_name in self.modules:
|
||||
with self.subTest(module=module_name):
|
||||
try:
|
||||
print(module_name)
|
||||
importlib.import_module(module_name)
|
||||
except Exception as e:
|
||||
self.fail(f"Failed to import {module_name}: {e}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -1,5 +1,7 @@
|
||||
import unittest
|
||||
|
||||
import pytest
|
||||
|
||||
from rdagent.core.utils import SingletonBaseClass
|
||||
|
||||
|
||||
@@ -15,6 +17,7 @@ class A(SingletonBaseClass):
|
||||
return self.__str__()
|
||||
|
||||
|
||||
@pytest.mark.offline
|
||||
class MiscTest(unittest.TestCase):
|
||||
def test_singleton(self):
|
||||
print("a1=================")
|
||||
|
||||
Reference in New Issue
Block a user