Merge branch 'main' into ytli_update

rdagent/components/proposal/model_proposal.py

@@ -36,7 +36,7 @@ class ModelHypothesisGen(HypothesisGen):
             Environment(undefined=StrictUndefined)
             .from_string(prompt_dict["hypothesis_gen"]["system_prompt"])
             .render(
-                targets="factors",
+                targets="model",
                 scenario=self.scen.get_scenario_all_desc(),
                 hypothesis_output_format=context_dict["hypothesis_output_format"],
             )
@@ -45,7 +45,7 @@ class ModelHypothesisGen(HypothesisGen):
             Environment(undefined=StrictUndefined)
             .from_string(prompt_dict["hypothesis_gen"]["user_prompt"])
             .render(
-                targets="factors",
+                targets="model",
                 hypothesis_and_feedback=context_dict["hypothesis_and_feedback"],
                 RAG=context_dict["RAG"],
             )
@@ -74,7 +74,7 @@ class ModelHypothesis2Experiment(Hypothesis2Experiment[ModelExperiment]):
             Environment(undefined=StrictUndefined)
             .from_string(prompt_dict["hypothesis2experiment"]["system_prompt"])
             .render(
-                targets="factors",
+                targets="model",
                 scenario=trace.scen.get_scenario_all_desc(),
                 experiment_output_format=context["experiment_output_format"],
             )
@@ -83,7 +83,7 @@ class ModelHypothesis2Experiment(Hypothesis2Experiment[ModelExperiment]):
             Environment(undefined=StrictUndefined)
             .from_string(prompt_dict["hypothesis2experiment"]["user_prompt"])
             .render(
-                targets="factors",
+                targets="model",
                 target_hypothesis=context["target_hypothesis"],
                 hypothesis_and_feedback=context["hypothesis_and_feedback"],
                 target_list=context["target_list"],

rdagent/core/proposal.py

@@ -54,6 +54,15 @@ class Trace(Generic[ASpecificScen]):
         self.scen: ASpecificScen = scen
         self.hist: list[Tuple[Hypothesis, Experiment, HypothesisFeedback]] = []
 
+    def get_last_experiment_info(self) -> Tuple[Hypothesis, ASpecificTask, Any]:
+        """Access the last experiment result, sub-task, and the corresponding hypothesis."""
+        # TODO: The return value does not align with the signature.
+        if not self.hist:
+            return None
+        last_hypothesis, last_experiment, _ = self.hist[-1]
+        last_task = last_experiment.sub_tasks[-1]
+        last_result = last_experiment.result
+        return last_hypothesis, last_task, last_result
 
 class HypothesisGen:
     def __init__(self, scen: Scenario):

rdagent/scenarios/qlib/prompts.yaml

@@ -44,7 +44,7 @@ model_experiment_output_format: |-
               "variable or function name 1": "description of variable or function 1",
               "variable or function name 2": "description of variable or function 2"
           }
-          "model_type": "type of model 1, Tabular or TimesSeries"
+          "model_type": "type of model 1, Tabular or TimesSeries" # Should be one of "Tabular" or "TimeSeries"
       }
       # Don't add ellipsis (...) or any filler text that might cause JSON parsing errors here!
   }

rdagent/scenarios/qlib/task_generator/feedback.py

@@ -2,10 +2,8 @@
 # Implement to feedback.
 
 import json
-import pickle
 from pathlib import Path
 
-import pandas as pd
 from jinja2 import Environment, StrictUndefined
 
 from rdagent.core.experiment import Experiment
@@ -18,16 +16,12 @@ from rdagent.core.proposal import (
     Trace,
 )
 from rdagent.oai.llm_utils import APIBackend
-from rdagent.utils.env import QTDockerEnv
 
 feedback_prompts = Prompts(file_path=Path(__file__).parent.parent / "prompts.yaml")
 DIRNAME = Path(__file__).absolute().resolve().parent
 logger = RDAgentLog()
 
 
-class QlibModelHypothesisExperiment2Feedback(HypothesisExperiment2Feedback): ...
-
-
 class QlibFactorHypothesisExperiment2Feedback(HypothesisExperiment2Feedback):
     def generateFeedback(self, exp: Experiment, hypothesis: Hypothesis, trace: Trace) -> HypothesisFeedback:
         """
@@ -102,3 +96,82 @@ class QlibFactorHypothesisExperiment2Feedback(HypothesisExperiment2Feedback):
         )
 
         return hypothesis_feedback
+
+
+class QlibModelHypothesisExperiment2Feedback(HypothesisExperiment2Feedback):
+    """Generated feedbacks on the hypothesis from **Executed** Implementations of different tasks & their comparisons with previous performances"""
+
+    def generateFeedback(self, exp: Experiment, hypothesis: Hypothesis, trace: Trace) -> HypothesisFeedback:
+        """
+        The `ti` should be executed and the results should be included, as well as the comparison between previous results (done by LLM).
+        For example: `mlflow` of Qlib will be included.
+        """
+
+        # Define the system prompt for hypothesis feedback
+        sys_prompt_hypothesis = (
+            "You are a professional result analysis assistant. You will receive a result and a hypothesis. "
+            "Your task is to provide feedback on how well the result supports or refutes the hypothesis by judging from the observation of performance increase or decrease. "
+            "Please provide detailed and constructive feedback. "
+            "Example JSON Structure for Result Analysis: "
+            '{"Observations": "Your overall observations here", "Feedback for Hypothesis": "Observations related to the hypothesis", '
+            '"New Hypothesis": "Put your new hypothesis here.", "Reasoning": "Provide reasoning for the hypothesis here.", '
+            '"Decision": "True or False"}'
+        )
+
+        # Define the user prompt for hypothesis feedback
+        context = trace.scen
+        last_experiment_info = trace.get_last_experiment_info()
+
+        if last_experiment_info:
+            last_hypothesis, last_task, last_result = last_experiment_info
+            last_info_str = f"Last Round Information:\nHypothesis: {last_hypothesis.hypothesis}\nTask: {last_task}\nResult: {last_result}\n"
+        else:
+            last_info_str = "This is the first round. No previous information available."
+
+        usr_prompt_hypothesis = f"""
+            We are in an experiment of finding hypothesis and validating or rejecting them so that in the end we have a powerful model generated.
+            Here are the context: {context}. 
+            {last_info_str}
+            
+            Now let's come to this round. You will receive the result and you will evaluate if the performance increases or decreases. 
+            Hypothesis: {hypothesis.hypothesis}\n
+            Relevant Reasoning: {hypothesis.reason}\n
+            Result: {exp.result}\n
+
+            Compare and observe. Which result has a better return and lower risk? If the performance increases， the hypothesis should be considered positive (working). 
+            Hence, with the hypotheses, relevant reasoning, and results in mind (comparison), provide detailed and constructive feedback and suggest a new hypothesis. 
+        """
+
+        try:
+            # Call the APIBackend to generate the response for hypothesis feedback
+            response_hypothesis = APIBackend().build_messages_and_create_chat_completion(
+                user_prompt=usr_prompt_hypothesis,
+                system_prompt=sys_prompt_hypothesis,
+                json_mode=True,
+            )
+
+            # Parse the JSON response to extract the feedback
+            response_json_hypothesis = json.loads(response_hypothesis)
+            hypothesis_feedback = HypothesisFeedback(
+                observations=response_json_hypothesis.get("Observations", "No observations provided"),
+                hypothesis_evaluation=response_json_hypothesis.get("Feedback for Hypothesis", "No feedback provided"),
+                new_hypothesis=response_json_hypothesis.get("New Hypothesis", "No new hypothesis provided"),
+                reason=response_json_hypothesis.get("Reasoning", "No reasoning provided"),
+                decision=response_json_hypothesis.get("Decision", "false").lower() == "true",
+            )
+
+            return hypothesis_feedback
+
+        except json.JSONDecodeError as e:
+            # TODO:  (Xiao) I think raising a specific type of ERROR to make caller know sth bad has happened would be more reasonable
+            print("Error parsing JSON response from LLM for hypothesis feedback:", e)
+        except Exception as e:
+            print("An unexpected error occurred while generating hypothesis feedback:", e)
+
+        return HypothesisFeedback(
+            observations="No observations",
+            hypothesis_evaluation="No feedback",
+            new_hypothesis="No new hypothesis",
+            reason="No reasoning",
+            decision=False,
+        )