gavindiaz/NexQuant
1
0
Fork 0
mirror of https://github.com/NicolasBohn/NexQuant.git synced 2026-08-09 21:10:56 +00:00
Code Issues Packages Projects Releases Wiki Activity

Pdf2 model task (#33)

Browse Source 
* add needed dependency

* add extract_model_and_implement pipeline

* add merge_file_to_model_dict_to_model_dict

* implement `rdagent\app\model_implementation\eval.py`

* Running benchmark

* refine import

---------

Co-authored-by: Young <afe.young@gmail.com>
This commit is contained in:
Xinjie Shen
2024-06-30 23:31:00 +08:00
committed by GitHub
parent 43cb4a564b
commit b796b90cce
14 changed files with 344 additions and 45 deletions
Download Patch File Download Diff File Expand all files Collapse all files
rdagent/app/model_implementation/eval.py
+10 -5
View File
@@ -2,14 +2,18 @@ from pathlib import Path
DIRNAME = Path(__file__).absolute().resolve().parent
from rdagent.model_implementation.benchmark.eval import ModelImpValEval
from rdagent.model_implementation.one_shot import ModelTaskGen
from rdagent.model_implementation.task import ModelImpLoader, ModelTaskLoderJson
from rdagent.components.task_implementation.model_implementation.benchmark.eval import ModelImpValEval
from rdagent.components.task_implementation.model_implementation.one_shot import ModelTaskGen
from rdagent.components.task_implementation.model_implementation.task import ModelImpLoader, ModelTaskLoderJson
mtl = ModelTaskLoderJson("TODO: A Path to json")
bench_folder = DIRNAME.parent.parent / "components" / "task_implementation" / "model_implementation" / "benchmark"
mtl = ModelTaskLoderJson(str(bench_folder / "model_dict.json"))
task_l = mtl.load()
task_l = [t for t in task_l if t.key == "A-DGN"] # FIXME: other models does not work well
mtg = ModelTaskGen()
impl_l = mtg.generate(task_l)
@@ -17,12 +21,13 @@ impl_l = mtg.generate(task_l)
# TODO: Align it with the benchmark framework after @wenjun's refine the evaluation part.
# Currently, we just handcraft a workflow for fast evaluation.
mil = ModelImpLoader(DIRNAME.parent.parent / "model_implementation" / "benchmark" / "gt_code")
mil = ModelImpLoader(bench_folder / "gt_code")
mie = ModelImpValEval()
# Evaluation:
eval_l = []
for impl in impl_l:
print(impl.target_task)
gt_impl = mil.load(impl.target_task)
eval_l.append(mie.evaluate(gt_impl, impl))