mirror of
https://github.com/NicolasBohn/NexQuant.git
synced 2026-07-28 07:57:44 +00:00
炼金术师华华
5090c6153f
* File structure for supporting litellm * more litellm support * feat: Add CachedAPIBackend class and dynamic API backend retrieval function * fix: update benchmark folder path and add default values for architecture and hyperparameters * feat: add LiteLLMAPIBackend and DeprecBackend ; changed structure of the project ; with bus * fix : deprec_backend * feat: Add LiteLLMAPIBackend class and related features; update configuration and test cases. * feat: Enhance LiteLLMAPIBackend with encoder support and dynamic argument handling;Enhance log Colors * lint * fix lint... * fix: Lint * fix:make auto-lint * fix:test oai * fix:redundant _abckend.py * fix: Optimize LiteLLMAPIBackend on token counting functiona, and clean up unused code;add test on this function * feat: Add LiteLLMSettings class and update model settings usage * fix: Update LiteLLMSettings environment variable prefix and model configurations * fix : gitignore * test: Consolidate and relocate test files for litellm backend and oai * fix : lint * fix: lint * auto lint * lint * LINT * lint * chore: remove deprecated backend configuration comments * refactor: Remove unused functions and imports from deprec.py and llm_utils.py * refactor: Move md5_hash function from deprec.py to llm_utils.py * chore: Remove extra newline and add missing import in deprec.py * lint * refactor: Move md5_hash function to utils module * lint * lint * lint --------- Co-authored-by: Young <afe.young@gmail.com> Co-authored-by: Yihua Chen <v-yihuachen@microsoft.com>
43 lines
1.5 KiB
Python
43 lines
1.5 KiB
Python
from pathlib import Path
|
|
|
|
from rdagent.components.coder.model_coder import ModelCoSTEER
|
|
from rdagent.components.loader.task_loader import ModelTaskLoaderJson, ModelWsLoader
|
|
from rdagent.scenarios.qlib.experiment.model_experiment import (
|
|
QlibModelExperiment,
|
|
QlibModelScenario,
|
|
)
|
|
|
|
if __name__ == "__main__":
|
|
DIRNAME = Path(__file__).absolute().resolve().parent
|
|
|
|
from rdagent.components.coder.model_coder.benchmark.eval import ModelImpValEval
|
|
from rdagent.components.coder.model_coder.one_shot import ModelCodeWriter
|
|
|
|
bench_folder = DIRNAME.parent.parent.parent / "components" / "coder" / "model_coder" / "benchmark"
|
|
mtl = ModelTaskLoaderJson(str(bench_folder / "model_dict.json"))
|
|
|
|
task_l = mtl.load()
|
|
|
|
task_l = [t for t in task_l if t.name == "A-DGN"] # FIXME: other models does not work well
|
|
|
|
model_experiment = QlibModelExperiment(sub_tasks=task_l)
|
|
# mtg = ModelCodeWriter(scen=QlibModelScenario())
|
|
mtg = ModelCoSTEER(scen=QlibModelScenario())
|
|
|
|
model_experiment = mtg.develop(model_experiment)
|
|
|
|
# TODO: Align it with the benchmark framework after @wenjun's refine the evaluation part.
|
|
# Currently, we just handcraft a workflow for fast evaluation.
|
|
|
|
mil = ModelWsLoader(bench_folder / "gt_code")
|
|
|
|
mie = ModelImpValEval()
|
|
# Evaluation:
|
|
eval_l = []
|
|
for impl in model_experiment.sub_workspace_list:
|
|
print(impl.target_task)
|
|
gt_impl = mil.load(impl.target_task)
|
|
eval_l.append(mie.evaluate(gt_impl, impl))
|
|
|
|
print(eval_l)
|