mirror of
https://github.com/NicolasBohn/NexQuant.git
synced 2026-07-28 16:07:46 +00:00
Xu Yang
e0a24fb46f
* ignore result csv file * fix app scripts * rename taskgenerator to developer and generate to develop * fix a config bug in coder * fix a small bug in factor coder evaluators * remove a single logger in factor coder evaluators * fix a small bug in model coder main.py * rename Implementation to Workspace * move the prepare the inject_code into FBWorkspace to align all the behavior * fix a small bug in model feedback * remove debug lines for multi processing and simplify evaluators multi proc * add a copy function to workspace to freeze the workspace && add config prefix to speed up debugging * make hypothesisgen a abc class * use Qlib***Experiment * fix a small bug * rename Imp to Ws * rename sub_implementations to sub_workspace_list * fix a bug in feedback not presented as content in prompts * move proposal pys to proposal folder * reformat the folder * align factor and model qlib workspace and use template to handle the workspace * add a filter to evoagent to filter out false evo * align multi_proc_n into RDAGENT seeting * handle when runner gets empty experiment * fix logger merge remaining problems * fix black and isort automatically
42 lines
1.4 KiB
Python
42 lines
1.4 KiB
Python
from pathlib import Path
|
|
|
|
from rdagent.components.coder.model_coder.CoSTEER import ModelCoSTEER
|
|
from rdagent.components.loader.task_loader import ModelTaskLoaderJson, ModelWsLoader
|
|
from rdagent.scenarios.qlib.experiment.model_experiment import (
|
|
QlibModelExperiment,
|
|
QlibModelScenario,
|
|
)
|
|
|
|
DIRNAME = Path(__file__).absolute().resolve().parent
|
|
|
|
from rdagent.components.coder.model_coder.benchmark.eval import ModelImpValEval
|
|
from rdagent.components.coder.model_coder.one_shot import ModelCodeWriter
|
|
|
|
bench_folder = DIRNAME.parent.parent / "components" / "coder" / "model_coder" / "benchmark"
|
|
mtl = ModelTaskLoaderJson(str(bench_folder / "model_dict.json"))
|
|
|
|
task_l = mtl.load()
|
|
|
|
task_l = [t for t in task_l if t.name == "A-DGN"] # FIXME: other models does not work well
|
|
|
|
model_experiment = QlibModelExperiment(sub_tasks=task_l)
|
|
# mtg = ModelCodeWriter(scen=QlibModelScenario())
|
|
mtg = ModelCoSTEER(scen=QlibModelScenario())
|
|
|
|
model_experiment = mtg.develop(model_experiment)
|
|
|
|
# TODO: Align it with the benchmark framework after @wenjun's refine the evaluation part.
|
|
# Currently, we just handcraft a workflow for fast evaluation.
|
|
|
|
mil = ModelWsLoader(bench_folder / "gt_code")
|
|
|
|
mie = ModelImpValEval()
|
|
# Evaluation:
|
|
eval_l = []
|
|
for impl in model_experiment.sub_workspace_list:
|
|
print(impl.target_task)
|
|
gt_impl = mil.load(impl.target_task)
|
|
eval_l.append(mie.evaluate(gt_impl, impl))
|
|
|
|
print(eval_l)
|