gavindiaz/NexQuant
1
0
Fork 0
mirror of https://github.com/NicolasBohn/NexQuant.git synced 2026-07-28 07:57:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

reporeformat V2 (#23)

Browse Source 
* reformat factor implement process

* move some code to more reasonable place

* fix the bug

* add test function in factor_extract_and_implement.py

* change select factor number to ratio , add some factor implement setting and fix some bug while using knowledgebase

* change evoagent

* add abstract class EvoAgent

* add benchmark workflow

* fix some bug in llm_utils

* run wenjun's code

* fix the knowledgebase instance check

---------

Co-authored-by: xuyang1 <xuyang1@microsoft.com>
This commit is contained in:
USTCKevinF
2024-06-14 12:59:44 +08:00
committed by GitHub
parent 9e82da243b
commit ebb659a018
32 changed files with 2227 additions and 1141 deletions
Download Patch File Download Diff File Expand all files Collapse all files
rdagent/factor_implementation/evolving/evolving_strategy.py
+79 -63
View File
@@ -1,58 +1,71 @@
from __future__ import annotations
import json
import random
from abc import abstractmethod
from copy import deepcopy
from pathlib import Path
from typing import TYPE_CHECKING
from jinja2 import Template
from rdagent.core.evolving_framework import EvolvingStrategy, QueriedKnowledge
from rdagent.core.prompts import Prompts
from rdagent.core.utils import multiprocessing_wrapper
from rdagent.factor_implementation.share_modules.factor import (
FactorImplementation,
FactorImplementationTask,
FileBasedFactorImplementation,
)
from rdagent.oai.llm_utils import APIBackend
from rdagent.factor_implementation.share_modules.factor_implementation_config import (
FactorImplementSettings,
)
from rdagent.factor_implementation.share_modules.factor_implementation_utils import (
get_data_folder_intro,
from rdagent.core.task import (
TaskImplementation,
)
from rdagent.core.prompts import Prompts
from pathlib import Path
from rdagent.factor_implementation.evolving.scheduler import (
RandomSelect,
LLMSelect,
)
from rdagent.factor_implementation.share_modules.factor_implementation_utils import get_data_folder_intro
from rdagent.oai.llm_utils import APIBackend
from rdagent.core.utils import multiprocessing_wrapper
from rdagent.factor_implementation.evolving.factor import (
FactorImplementTask,
FactorEvovlingItem,
FileBasedFactorImplementation,
)
if TYPE_CHECKING:
from factor_implementation.evolving.evolvable_subjects import (
FactorImplementationList,
)
from factor_implementation.evolving.knowledge_management import (
from rdagent.factor_implementation.evolving.knowledge_management import (
FactorImplementationQueriedKnowledge,
FactorImplementationQueriedKnowledgeV1,
)
implement_prompts = Prompts(file_path=Path(__file__).parent.parent / "prompts.yaml")
class MultiProcessEvolvingStrategy(EvolvingStrategy):
@abstractmethod
def implement_one_factor(
self,
target_task: FactorImplementationTask,
target_task: FactorImplementTask,
queried_knowledge: QueriedKnowledge = None,
) -> FactorImplementation:
) -> TaskImplementation:
raise NotImplementedError
def evolve(
self,
*,
evo: FactorImplementationList,
evo: FactorEvovlingItem,
queried_knowledge: FactorImplementationQueriedKnowledge | None = None,
**kwargs,
) -> FactorImplementationList:
) -> FactorEvovlingItem:
self.num_loop += 1
new_evo = deepcopy(evo)
new_evo.corresponding_implementations = [None for _ in new_evo.target_factor_tasks]
# 1.找出需要evolve的factor
to_be_finished_task_index = []
for index, target_factor_task in enumerate(new_evo.target_factor_tasks):
target_factor_task_desc = target_factor_task.get_factor_information()
@@ -65,11 +78,27 @@ class MultiProcessEvolvingStrategy(EvolvingStrategy):
and target_factor_task_desc not in queried_knowledge.failed_task_info_set
):
to_be_finished_task_index.append(index)
if FactorImplementSettings().implementation_factors_per_round < len(to_be_finished_task_index):
to_be_finished_task_index = random.sample(
to_be_finished_task_index,
FactorImplementSettings().implementation_factors_per_round,
# 2. 选择selection方法
# if the number of factors to be implemented is larger than the limit, we need to select some of them
if FactorImplementSettings().select_ratio < 1:
# if the number of loops is equal to the select_loop, we need to select some of them
implementation_factors_per_round = int(
FactorImplementSettings().select_ratio * len(to_be_finished_task_index)
)
if FactorImplementSettings().select_method == "random":
to_be_finished_task_index = RandomSelect(
to_be_finished_task_index,
implementation_factors_per_round,
)
if FactorImplementSettings().select_method == "scheduler":
to_be_finished_task_index = LLMSelect(
to_be_finished_task_index,
implementation_factors_per_round,
new_evo,
queried_knowledge.former_traces,
)
result = multiprocessing_wrapper(
[
@@ -81,11 +110,12 @@ class MultiProcessEvolvingStrategy(EvolvingStrategy):
for index, target_index in enumerate(to_be_finished_task_index):
new_evo.corresponding_implementations[target_index] = result[index]
if result[index].target_task.factor_name in new_evo.evolve_trace:
new_evo.evolve_trace[result[index].target_task.factor_name].append(result[index])
else:
new_evo.evolve_trace[result[index].target_task.factor_name] = [result[index]]
# for target_index in to_be_finished_task_index:
# new_evo.corresponding_implementations[target_index] = self.implement_one_factor(
# new_evo.target_factor_tasks[target_index], queried_knowledge
# )
new_evo.corresponding_selection.append(to_be_finished_task_index)
return new_evo
@@ -93,9 +123,9 @@ class MultiProcessEvolvingStrategy(EvolvingStrategy):
class FactorEvolvingStrategy(MultiProcessEvolvingStrategy):
def implement_one_factor(
self,
target_task: FactorImplementationTask,
target_task: FactorImplementTask,
queried_knowledge: FactorImplementationQueriedKnowledgeV1 = None,
) -> FactorImplementation:
) -> TaskImplementation:
factor_information_str = target_task.get_factor_information()
if queried_knowledge is not None and factor_information_str in queried_knowledge.success_task_to_knowledge_dict:
@@ -117,9 +147,7 @@ class FactorEvolvingStrategy(MultiProcessEvolvingStrategy):
queried_former_failed_knowledge_to_render = queried_former_failed_knowledge
system_prompt = Template(
Prompts(file_path=Path(__file__).parent.parent / "prompts.yaml")[
"evolving_strategy_factor_implementation_v1_system"
],
implement_prompts["evolving_strategy_factor_implementation_v1_system"],
).render(
data_info=get_data_folder_intro(),
queried_former_failed_knowledge=queried_former_failed_knowledge_to_render,
@@ -132,9 +160,7 @@ class FactorEvolvingStrategy(MultiProcessEvolvingStrategy):
while True:
user_prompt = (
Template(
Prompts(file_path=Path(__file__).parent.parent / "prompts.yaml")[
"evolving_strategy_factor_implementation_v1_user"
],
implement_prompts["evolving_strategy_factor_implementation_v1_user"],
)
.render(
factor_information_str=factor_information_str,
@@ -153,9 +179,6 @@ class FactorEvolvingStrategy(MultiProcessEvolvingStrategy):
queried_former_failed_knowledge_to_render = queried_former_failed_knowledge_to_render[1:]
elif len(queried_similar_successful_knowledge_to_render) > 1:
queried_similar_successful_knowledge_to_render = queried_similar_successful_knowledge_to_render[1:]
# print(
# f"length of queried_similar_successful_knowledge_to_render: {len(queried_similar_successful_knowledge_to_render)}, length of queried_former_failed_knowledge_to_render: {len(queried_former_failed_knowledge_to_render)}"
# )
code = json.loads(
session.build_chat_completion(
@@ -173,14 +196,20 @@ class FactorEvolvingStrategy(MultiProcessEvolvingStrategy):
class FactorEvolvingStrategyWithGraph(MultiProcessEvolvingStrategy):
def __init__(self) -> None:
self.num_loop = 0
self.haveSelected = False
def implement_one_factor(
self,
target_task: FactorImplementationTask,
target_task: FactorImplementTask,
queried_knowledge,
) -> FactorImplementation:
) -> TaskImplementation:
error_summary = FactorImplementSettings().v2_error_summary
# 1. 提取因子的背景信息
target_factor_task_information = target_task.get_factor_information()
# 2. 检查该因子是否需要继续做（是否已经作对，是否做错太多）
if (
queried_knowledge is not None
and target_factor_task_information in queried_knowledge.success_task_to_knowledge_dict
@@ -189,6 +218,8 @@ class FactorEvolvingStrategyWithGraph(MultiProcessEvolvingStrategy):
elif queried_knowledge is not None and target_factor_task_information in queried_knowledge.failed_task_info_set:
return None
else:
# 3. 取出knowledge里面的经验数据（similar success、similar error、former_trace）
queried_similar_component_knowledge = (
queried_knowledge.component_with_success_task[target_factor_task_information]
if queried_knowledge is not None
@@ -208,9 +239,7 @@ class FactorEvolvingStrategyWithGraph(MultiProcessEvolvingStrategy):
queried_former_failed_knowledge_to_render = queried_former_failed_knowledge
system_prompt = Template(
Prompts(file_path=Path(__file__).parent.parent / "prompts.yaml")[
"evolving_strategy_factor_implementation_v1_system"
],
implement_prompts["evolving_strategy_factor_implementation_v1_system"],
).render(
data_info=get_data_folder_intro(),
queried_former_failed_knowledge=queried_former_failed_knowledge_to_render,
@@ -223,18 +252,17 @@ class FactorEvolvingStrategyWithGraph(MultiProcessEvolvingStrategy):
queried_similar_component_knowledge_to_render = queried_similar_component_knowledge
queried_similar_error_knowledge_to_render = queried_similar_error_knowledge
error_summary_critics = ""
# 动态地防止prompt超长
while True:
# 总结error（可选）
if (
error_summary
and len(queried_similar_error_knowledge_to_render) != 0
and len(queried_former_failed_knowledge_to_render) != 0
):
error_summary_system_prompt = (
Template(
Prompts(file_path=Path(__file__).parent.parent / "prompts.yaml")[
"evolving_strategy_error_summary_v2_system"
],
)
Template(implement_prompts["evolving_strategy_error_summary_v2_system"])
.render(
factor_information_str=target_factor_task_information,
code_and_feedback=queried_former_failed_knowledge_to_render[
@@ -248,11 +276,7 @@ class FactorEvolvingStrategyWithGraph(MultiProcessEvolvingStrategy):
)
while True:
error_summary_user_prompt = (
Template(
Prompts(file_path=Path(__file__).parent.parent / "prompts.yaml")[
"evolving_strategy_error_summary_v2_user"
],
)
Template(implement_prompts["evolving_strategy_error_summary_v2_user"])
.render(
queried_similar_component_knowledge=queried_similar_component_knowledge_to_render,
)
@@ -269,12 +293,10 @@ class FactorEvolvingStrategyWithGraph(MultiProcessEvolvingStrategy):
user_prompt=error_summary_user_prompt,
json_mode=False,
)
# 构建user_prompt。开始写代码
user_prompt = (
Template(
Prompts(file_path=Path(__file__).parent.parent / "prompts.yaml")[
"evolving_strategy_factor_implementation_v2_user"
],
implement_prompts["evolving_strategy_factor_implementation_v2_user"],
)
.render(
factor_information_str=target_factor_task_information,
@@ -301,12 +323,6 @@ class FactorEvolvingStrategyWithGraph(MultiProcessEvolvingStrategy):
elif len(queried_similar_error_knowledge_to_render) > 0:
queried_similar_error_knowledge_to_render = queried_similar_error_knowledge_to_render[:-1]
# print(
# len(queried_similar_component_knowledge_to_render),
# len(queried_similar_error_knowledge_to_render),
# len(queried_former_failed_knowledge_to_render),
# )
response = session.build_chat_completion(
user_prompt=user_prompt,
json_mode=True,