example workflow code for model implementation (#17)

* upload example code for model implementation

* Refactor Model Implement

* add export

---------

Co-authored-by: Young <afe.young@gmail.com>
This commit is contained in:
Xinjie Shen
2024-06-21 16:41:34 +08:00
committed by GitHub
parent a126c84c92
commit f20dc4482b
21 changed files with 2712 additions and 10 deletions
+29
View File
@@ -0,0 +1,29 @@
from pathlib import Path
DIRNAME = Path(__file__).absolute().resolve().parent
from rdagent.model_implementation.benchmark.eval import ModelImpValEval
from rdagent.model_implementation.one_shot import ModelTaskGen
from rdagent.model_implementation.task import ModelImpLoader, ModelTaskLoderJson
mtl = ModelTaskLoderJson("TODO: A Path to json")
task_l = mtl.load()
mtg = ModelTaskGen()
impl_l = mtg.generate(task_l)
# TODO: Align it with the benchmark framework after @wenjun's refine the evaluation part.
# Currently, we just handcraft a workflow for fast evaluation.
mil = ModelImpLoader(DIRNAME.parent.parent / "model_implementation" / "benchmark" / "gt_code")
mie = ModelImpValEval()
# Evaluation:
eval_l = []
for impl in impl_l:
gt_impl = mil.load(impl.target_task)
eval_l.append(mie.evaluate(gt_impl, impl))
print(eval_l)