mirror of
https://github.com/NicolasBohn/NexQuant.git
synced 2026-07-28 16:07:46 +00:00
Suhan Cui
72 lines
3.1 KiB
Python
72 lines
3.1 KiB
Python
# TODO:
|
|
# Implement to feedback.
|
|
|
|
import json
|
|
from pathlib import Path
|
|
|
|
from jinja2 import Environment, StrictUndefined
|
|
|
|
from rdagent.core.experiment import Experiment
|
|
from rdagent.core.prompts import Prompts
|
|
from rdagent.core.proposal import (
|
|
Hypothesis,
|
|
HypothesisExperiment2Feedback,
|
|
HypothesisFeedback,
|
|
Trace,
|
|
)
|
|
from rdagent.log import rdagent_logger as logger
|
|
from rdagent.oai.llm_utils import APIBackend
|
|
from rdagent.utils import convert2bool
|
|
|
|
feedback_prompts = Prompts(file_path=Path(__file__).parent.parent.parent / "qlib" / "prompts.yaml")
|
|
DIRNAME = Path(__file__).absolute().resolve().parent
|
|
|
|
|
|
class DMModelHypothesisExperiment2Feedback(HypothesisExperiment2Feedback):
|
|
"""Generated feedbacks on the hypothesis from **Executed** Implementations of different tasks & their comparisons with previous performances"""
|
|
|
|
def generate_feedback(self, exp: Experiment, hypothesis: Hypothesis, trace: Trace) -> HypothesisFeedback:
|
|
"""
|
|
The `ti` should be executed and the results should be included, as well as the comparison between previous results (done by LLM).
|
|
For example: `mlflow` of Qlib will be included.
|
|
"""
|
|
|
|
logger.info("Generating feedback...")
|
|
# Define the system prompt for hypothesis feedback
|
|
system_prompt = feedback_prompts["model_feedback_generation"]["system"]
|
|
|
|
# Define the user prompt for hypothesis feedback
|
|
context = trace.scen
|
|
SOTA_hypothesis, SOTA_experiment = trace.get_sota_hypothesis_and_experiment()
|
|
|
|
user_prompt = (
|
|
Environment(undefined=StrictUndefined)
|
|
.from_string(feedback_prompts["model_feedback_generation"]["user"])
|
|
.render(
|
|
context=context,
|
|
last_hypothesis=SOTA_hypothesis,
|
|
last_task=SOTA_experiment.sub_tasks[0].get_task_information() if SOTA_hypothesis else None,
|
|
last_code=SOTA_experiment.sub_workspace_list[0].code_dict.get("model.py") if SOTA_hypothesis else None,
|
|
last_result=SOTA_experiment.result if SOTA_hypothesis else None,
|
|
hypothesis=hypothesis,
|
|
exp=exp,
|
|
)
|
|
)
|
|
|
|
# Call the APIBackend to generate the response for hypothesis feedback
|
|
response_hypothesis = APIBackend().build_messages_and_create_chat_completion(
|
|
user_prompt=user_prompt,
|
|
system_prompt=system_prompt,
|
|
json_mode=True,
|
|
)
|
|
|
|
# Parse the JSON response to extract the feedback
|
|
response_json_hypothesis = json.loads(response_hypothesis)
|
|
return HypothesisFeedback(
|
|
observations=response_json_hypothesis.get("Observations", "No observations provided"),
|
|
hypothesis_evaluation=response_json_hypothesis.get("Feedback for Hypothesis", "No feedback provided"),
|
|
new_hypothesis=response_json_hypothesis.get("New Hypothesis", "No new hypothesis provided"),
|
|
reason=response_json_hypothesis.get("Reasoning", "No reasoning provided"),
|
|
decision=convert2bool(response_json_hypothesis.get("Decision", "false")),
|
|
)
|