gavindiaz/NexQuant
1
0
Fork 0
mirror of https://github.com/NicolasBohn/NexQuant.git synced 2026-08-07 20:17:45 +00:00
Code Issues Packages Projects Releases Wiki Activity

Several update on the repo (see desc) (#76)

Browse Source 
* ignore result csv file

* fix app scripts

* rename taskgenerator to developer and generate to develop

* fix a config bug in coder

* fix a small bug in factor coder evaluators

* remove a single logger in factor coder evaluators

* fix a small bug in model coder main.py

* rename Implementation to Workspace

* move the prepare the inject_code into FBWorkspace to align all the behavior

* fix a small bug in model feedback

* remove debug lines for multi processing and simplify evaluators multi proc

* add a copy function to workspace to freeze the workspace && add config prefix to speed up debugging

* make hypothesisgen a abc class

* use Qlib***Experiment

* fix a small bug

* rename Imp to Ws

* rename sub_implementations to sub_workspace_list

* fix a bug in feedback not presented as content in prompts

* move proposal pys to proposal folder

* reformat the folder

* align factor and model qlib workspace and use template to handle the workspace

* add a filter to evoagent to filter out false evo

* align multi_proc_n into RDAGENT seeting

* handle when runner gets empty experiment

* fix logger merge remaining problems

* fix black and isort automatically
This commit is contained in:
Xu Yang
2024-07-17 15:00:13 +08:00
committed by GitHub
parent eee2b3c56a
commit e0a24fb46f
76 changed files with 804 additions and 702 deletions
Download Patch File Download Diff File Expand all files Collapse all files
rdagent/components/coder/model_coder/CoSTEER/evaluators.py
+24 -29
View File
@@ -11,14 +11,14 @@ from rdagent.components.coder.model_coder.conf import MODEL_IMPL_SETTINGS
from rdagent.components.coder.model_coder.CoSTEER.evolvable_subjects import (
ModelEvolvingItem,
)
from rdagent.components.coder.model_coder.model import ModelImplementation, ModelTask
from rdagent.components.coder.model_coder.model import ModelFBWorkspace, ModelTask
from rdagent.core.conf import RD_AGENT_SETTINGS
from rdagent.core.evaluation import Evaluator
from rdagent.core.evolving_framework import QueriedKnowledge
from rdagent.core.experiment import Implementation, Task
from rdagent.log import rdagent_logger as logger
from rdagent.core.experiment import Task, Workspace
from rdagent.core.prompts import Prompts
from rdagent.core.utils import multiprocessing_wrapper
from rdagent.log import rdagent_logger as logger
from rdagent.oai.llm_utils import APIBackend
evaluate_prompts = Prompts(file_path=Path(__file__).parent.parent / "prompts.yaml")
@@ -62,15 +62,15 @@ class ModelCodeEvaluator(Evaluator):
def evaluate(
self,
target_task: Task,
implementation: Implementation,
gt_implementation: Implementation,
implementation: Workspace,
gt_implementation: Workspace,
model_execution_feedback: str = "",
model_value_feedback: str = "",
):
assert isinstance(target_task, ModelTask)
assert isinstance(implementation, ModelImplementation)
assert isinstance(implementation, ModelFBWorkspace)
if gt_implementation is not None:
assert isinstance(gt_implementation, ModelImplementation)
assert isinstance(gt_implementation, ModelFBWorkspace)
model_task_information = target_task.get_task_information()
code = implementation.code
@@ -120,16 +120,16 @@ class ModelFinalEvaluator(Evaluator):
def evaluate(
self,
target_task: Task,
implementation: Implementation,
gt_implementation: Implementation,
implementation: Workspace,
gt_implementation: Workspace,
model_execution_feedback: str,
model_value_feedback: str,
model_code_feedback: str,
):
assert isinstance(target_task, ModelTask)
assert isinstance(implementation, ModelImplementation)
assert isinstance(implementation, ModelFBWorkspace)
if gt_implementation is not None:
assert isinstance(gt_implementation, ModelImplementation)
assert isinstance(gt_implementation, ModelFBWorkspace)
system_prompt = (
Environment(undefined=StrictUndefined)
@@ -219,8 +219,8 @@ class ModelCoderEvaluator(Evaluator):
def evaluate(
self,
target_task: Task,
implementation: Implementation,
gt_implementation: Implementation,
implementation: Workspace,
gt_implementation: Workspace,
queried_knowledge: QueriedKnowledge = None,
**kwargs,
) -> ModelCoderFeedback:
@@ -248,7 +248,7 @@ class ModelCoderEvaluator(Evaluator):
input_value = 0.4
param_init_value = 0.6
assert isinstance(implementation, ModelImplementation)
assert isinstance(implementation, ModelFBWorkspace)
model_execution_feedback, gen_tensor = implementation.execute(
batch_size=batch_size,
num_features=num_features,
@@ -257,7 +257,7 @@ class ModelCoderEvaluator(Evaluator):
param_init_value=param_init_value,
)
if gt_implementation is not None:
assert isinstance(gt_implementation, ModelImplementation)
assert isinstance(gt_implementation, ModelFBWorkspace)
_, gt_tensor = gt_implementation.execute(
batch_size=batch_size,
num_features=num_features,
@@ -303,26 +303,21 @@ class ModelCoderMultiEvaluator(Evaluator):
queried_knowledge: QueriedKnowledge = None,
**kwargs,
) -> List[ModelCoderFeedback]:
multi_implementation_feedback = []
calls = []
for index in range(len(evo.sub_tasks)):
corresponding_implementation = evo.sub_implementations[index]
corresponding_gt_implementation = (
evo.sub_gt_implementations[index] if evo.sub_gt_implementations is not None else None
)
calls.append(
multi_implementation_feedback = multiprocessing_wrapper(
[
(
ModelCoderEvaluator(scen=self.scen).evaluate,
(
evo.sub_tasks[index],
corresponding_implementation,
corresponding_gt_implementation,
evo.sub_workspace_list[index],
evo.sub_gt_implementations[index] if evo.sub_gt_implementations is not None else None,
queried_knowledge,
),
),
)
multi_implementation_feedback = multiprocessing_wrapper(calls, n=MODEL_IMPL_SETTINGS.evo_multi_proc_n)
)
for index in range(len(evo.sub_tasks))
],
n=RD_AGENT_SETTINGS.multi_proc_n,
)
final_decision = [
None if single_feedback is None else single_feedback.final_decision