gavindiaz/NexQuant
1
0
Fork 0
mirror of https://github.com/NicolasBohn/NexQuant.git synced 2026-07-27 23:47:46 +00:00
Code Issues Packages Projects Releases Wiki Activity

Implement model (and some factor) coder with evolving (#52)

Browse Source 
* store code into FBImplementation

* fix path related bugs

* fix a bug

* fix factor related small bugs

* re-submit all model related code

* new code to model coder

* finish the model evolving code

---------

Co-authored-by: xuyang1 <xuyang1@microsoft.com>
This commit is contained in:
Xu Yang
2024-07-10 15:45:43 +08:00
committed by GitHub
parent 828a624238
commit 720994c8e0
23 changed files with 1230 additions and 284 deletions
Download Patch File Download Diff File Expand all files Collapse all files
rdagent/components/coder/factor_coder/CoSTEER/__init__.py
-1
View File
@@ -98,5 +98,4 @@ class FactorCoSTEER(TaskGenerator[FactorExperiment]):
if self.new_knowledge_base_path is not None:
pickle.dump(factor_knowledge_base, open(self.new_knowledge_base_path, "wb"))
self.knowledge_base = factor_knowledge_base
self.latest_factor_implementations = exp.sub_tasks
return factor_experiment
rdagent/components/coder/factor_coder/CoSTEER/evolving_strategy.py
+11 -6
View File
@@ -145,7 +145,7 @@ class FactorEvolvingStrategy(MultiProcessEvolvingStrategy):
implement_prompts["evolving_strategy_factor_implementation_v1_system"],
)
.render(
data_info=get_data_folder_intro(),
scenario=self.scen.get_scenario_all_desc(),
queried_former_failed_knowledge=queried_former_failed_knowledge_to_render,
)
)
@@ -154,7 +154,7 @@ class FactorEvolvingStrategy(MultiProcessEvolvingStrategy):
)
queried_similar_successful_knowledge_to_render = queried_similar_successful_knowledge
while True:
for _ in range(10): # max attempt to reduce the length of user_prompt
user_prompt = (
Environment(undefined=StrictUndefined)
.from_string(
@@ -163,6 +163,7 @@ class FactorEvolvingStrategy(MultiProcessEvolvingStrategy):
.render(
factor_information_str=factor_information_str,
queried_similar_successful_knowledge=queried_similar_successful_knowledge_to_render,
queried_former_failed_knowledge=queried_former_failed_knowledge_to_render,
)
.strip("\n")
)
@@ -187,8 +188,9 @@ class FactorEvolvingStrategy(MultiProcessEvolvingStrategy):
# ast.parse(code)
factor_implementation = FileBasedFactorImplementation(
target_task,
code,
)
factor_implementation.prepare()
factor_implementation.inject_code(**{"factor.py": code})
return factor_implementation
@@ -255,7 +257,7 @@ class FactorEvolvingStrategyWithGraph(MultiProcessEvolvingStrategy):
queried_similar_error_knowledge_to_render = queried_similar_error_knowledge
error_summary_critics = ""
# 动态地防止prompt超长
while True:
for _ in range(10): # max attempt to reduce the length of user_prompt
# 总结error（可选）
if (
error_summary
@@ -266,6 +268,7 @@ class FactorEvolvingStrategyWithGraph(MultiProcessEvolvingStrategy):
Environment(undefined=StrictUndefined)
.from_string(implement_prompts["evolving_strategy_error_summary_v2_system"])
.render(
scenario=self.scen.get_scenario_all_desc(),
factor_information_str=target_factor_task_information,
code_and_feedback=queried_former_failed_knowledge_to_render[
-1
@@ -276,7 +279,7 @@ class FactorEvolvingStrategyWithGraph(MultiProcessEvolvingStrategy):
session_summary = APIBackend(use_chat_cache=False).build_chat_session(
session_system_prompt=error_summary_system_prompt,
)
while True:
for _ in range(10): # max attempt to reduce the length of error_summary_user_prompt
error_summary_user_prompt = (
Environment(undefined=StrictUndefined)
.from_string(implement_prompts["evolving_strategy_error_summary_v2_user"])
@@ -332,5 +335,7 @@ class FactorEvolvingStrategyWithGraph(MultiProcessEvolvingStrategy):
json_mode=True,
)
code = json.loads(response)["code"]
factor_implementation = FileBasedFactorImplementation(target_task, code)
factor_implementation = FileBasedFactorImplementation(target_task)
factor_implementation.prepare()
factor_implementation.inject_code(**{"factor.py": code})
return factor_implementation
rdagent/components/coder/factor_coder/CoSTEER/scheduler.py
+1 -1
View File
@@ -53,7 +53,7 @@ def LLMSelect(
)
)
while True:
for _ in range(10): # max attempt to reduce the length of user_prompt
user_prompt = (
Environment(undefined=StrictUndefined)
.from_string(
rdagent/components/coder/factor_coder/factor.py
+16 -18
View File
@@ -67,19 +67,15 @@ class FileBasedFactorImplementation(FBImplementation):
def __init__(
self,
target_task: FactorTask,
code,
*args,
executed_factor_value_dataframe=None,
raise_exception=False,
**kwargs,
) -> None:
super().__init__(target_task)
self.code = code
super().__init__(*args, **kwargs)
self.executed_factor_value_dataframe = executed_factor_value_dataframe
self.logger = RDAgentLog()
self.raise_exception = raise_exception
self.workspace_path = Path(
FACTOR_IMPLEMENT_SETTINGS.file_based_execution_workspace,
) / str(uuid.uuid4())
@staticmethod
def link_data_to_workspace(data_path: Path, workspace_path: Path):
@@ -98,8 +94,10 @@ class FileBasedFactorImplementation(FBImplementation):
raise NotImplementedError
def prepare(self, *args, **kwargs):
# TODO move the prepare part code in execute into here
return super().prepare(*args, **kwargs)
self.workspace_path = Path(
FACTOR_IMPLEMENT_SETTINGS.file_based_execution_workspace,
) / str(uuid.uuid4())
self.workspace_path.mkdir(exist_ok=True, parents=True)
def execute(self, store_result: bool = False) -> Tuple[str, pd.DataFrame]:
"""
@@ -114,7 +112,7 @@ class FileBasedFactorImplementation(FBImplementation):
parameters:
store_result: if True, store the factor value in the instance variable, this feature is to be used in the gt implementation to avoid multiple execution on the same gt implementation
"""
if self.code is None:
if self.code_dict is None or "factor.py" not in self.code_dict:
if self.raise_exception:
raise CodeFormatException(self.FB_CODE_NOT_SET)
else:
@@ -123,7 +121,7 @@ class FileBasedFactorImplementation(FBImplementation):
with FileLock(self.workspace_path / "execution.lock"):
if FACTOR_IMPLEMENT_SETTINGS.enable_execution_cache:
# NOTE: cache the result for the same code
target_file_name = md5_hash(self.code)
target_file_name = md5_hash(self.code_dict["factor.py"])
cache_file_path = (
Path(FACTOR_IMPLEMENT_SETTINGS.implementation_execution_cache_location) / f"{target_file_name}.pkl"
)
@@ -142,10 +140,9 @@ class FileBasedFactorImplementation(FBImplementation):
source_data_path = Path(
FACTOR_IMPLEMENT_SETTINGS.file_based_execution_data_folder,
)
self.workspace_path.mkdir(exist_ok=True, parents=True)
source_data_path.mkdir(exist_ok=True, parents=True)
code_path = self.workspace_path / f"{self.target_task.factor_name}.py"
code_path.write_text(self.code)
code_path = self.workspace_path / f"factor.py"
self.link_data_to_workspace(source_data_path, self.workspace_path)
@@ -212,10 +209,11 @@ class FileBasedFactorImplementation(FBImplementation):
@staticmethod
def from_folder(task: FactorTask, path: Union[str, Path], **kwargs):
path = Path(path)
factor_path = (path / task.factor_name).with_suffix(".py")
with factor_path.open("r") as f:
code = f.read()
return FileBasedFactorImplementation(task, code=code, **kwargs)
code_dict = {}
for file_path in path.iterdir():
if file_path.suffix == ".py":
code_dict[file_path.name] = file_path.read_text()
return FileBasedFactorImplementation(target_task=task, code_dict=code_dict, **kwargs)
class FactorExperiment(Experiment[FactorTask, FileBasedFactorImplementation]): ...
rdagent/components/coder/factor_coder/prompts.yaml
+9 -7
View File
@@ -13,6 +13,8 @@ evaluator_code_feedback_v1_system: |-
If the ground truth code is provided, your critic should only consider checking whether the user's code is align with the ground truth code since the ground truth is definitely correct.
If the ground truth code is not provided, your critic should consider checking whether the user's code is reasonable and correct.
Notice that your critics are not for user to debug the code. They are sent to the coding agent to correct the code. So don't give any following items for the user to check like "Please check the code line XXX".
You should provide the suggestion to each of your critic to help the user improve the code. Please response the critic in the following format. Here is an example structure for the output:
critic 1: The critic message to critic 1
critic 2: The critic message to critic 2
@@ -47,12 +49,10 @@ evolving_strategy_factor_implementation_v1_system: |-
{% if queried_former_failed_knowledge|length != 0 %}
--------------Your former latest attempt:---------------
{% for former_failed_knowledge in queried_former_failed_knowledge %}
=====Code to implementation {{ loop.index }}=====
{{ former_failed_knowledge.implementation.code }}
=====Feedback to implementation {{ loop.index }}=====
{{ former_failed_knowledge.feedback }}
{% endfor %}
=====Code to the former implementation=====
{{ queried_former_failed_knowledge[-1].implementation.code }}
=====Feedback to the former implementation=====
{{ queried_former_failed_knowledge[-1].feedback }}
{% endif %}
Please response the code in the following json format. Here is an example structure for the JSON output:
@@ -118,7 +118,9 @@ evolving_strategy_factor_implementation_v2_user: |-
{% endif %}
evolving_strategy_error_summary_v2_system: |-
You are doing the following task:
User is trying to implement some factors in the following scenario:
{{ scenario }}
User is doing the following task:
{{factor_information_str}}
You have written some code but it meets errors like the following: