Model run with logger (#79)

.gitignore

@@ -64,6 +64,7 @@ coverage.xml
 
 # Django stuff:
 *.log
+^log
 local_settings.py
 db.sqlite3
 db.sqlite3-journal
@@ -161,4 +162,4 @@ mlruns/
 
 # possible output from coder or runner
 *.pth
-*qlib_res.csv
+*qlib_res.csv

rdagent/app/qlib_rd_loop/model.py

@@ -30,15 +30,18 @@ qlib_model_runner: Developer = import_class(PROP_SETTING.model_runner)(scen)
 qlib_model_summarizer: HypothesisExperiment2Feedback = import_class(PROP_SETTING.model_summarizer)(scen)
 
 trace = Trace(scen=scen)
-for _ in range(PROP_SETTING.evolving_n):
-    try:
-        hypothesis = hypothesis_gen.gen(trace)
-        exp = hypothesis2experiment.convert(hypothesis, trace)
-        exp = qlib_model_coder.develop(exp)
-        exp = qlib_model_runner.develop(exp)
-        feedback = qlib_model_summarizer.generateFeedback(exp, hypothesis, trace)
-
-        trace.hist.append((hypothesis, exp, feedback))
-    except ModelEmptyException as e:
-        logger.warning(e)
-        continue
+with logger.tag("model.loop"):
+    for _ in range(PROP_SETTING.evolving_n):
+        try:
+            with logger.tag("r"): # research
+                hypothesis = hypothesis_gen.gen(trace)
+                exp = hypothesis2experiment.convert(hypothesis, trace)
+            with logger.tag("d"): # develop
+                exp = qlib_model_coder.develop(exp)
+            with logger.tag("ef"): # evaluate and feedback
+                exp = qlib_model_runner.develop(exp)
+                feedback = qlib_model_summarizer.generateFeedback(exp, hypothesis, trace)
+            trace.hist.append((hypothesis, exp, feedback))
+        except ModelEmptyException as e:
+            logger.warning(e)
+            continue

rdagent/scenarios/qlib/developer/feedback.py

@@ -16,6 +16,7 @@ from rdagent.core.proposal import (
 )
 from rdagent.log import rdagent_logger as logger
 from rdagent.oai.llm_utils import APIBackend
+from rdagent.utils import convert2bool
 
 feedback_prompts = Prompts(file_path=Path(__file__).parent.parent / "prompts.yaml")
 DIRNAME = Path(__file__).absolute().resolve().parent
@@ -74,7 +75,7 @@ class QlibFactorHypothesisExperiment2Feedback(HypothesisExperiment2Feedback):
         hypothesis_evaluation = response_json.get("Feedback for Hypothesis", "No feedback provided")
         new_hypothesis = response_json.get("New Hypothesis", "No new hypothesis provided")
         reason = response_json.get("Reasoning", "No reasoning provided")
-        decision = response_json.get("Replace Best Result", "no").lower() == "yes"
+        decision = convert2bool(response_json.get("Replace Best Result", "no"))
 
         return HypothesisFeedback(
             observations=observations,
@@ -129,5 +130,5 @@ class QlibModelHypothesisExperiment2Feedback(HypothesisExperiment2Feedback):
             hypothesis_evaluation=response_json_hypothesis.get("Feedback for Hypothesis", "No feedback provided"),
             new_hypothesis=response_json_hypothesis.get("New Hypothesis", "No new hypothesis provided"),
             reason=response_json_hypothesis.get("Reasoning", "No reasoning provided"),
-            decision=str(response_json_hypothesis.get("Decision", "false")).lower() == "true",
+            decision=convert2bool(response_json_hypothesis.get("Decision", "false")),
         )

rdagent/scenarios/qlib/prompts.yaml

@@ -92,7 +92,7 @@ model_feedback_generation:
       "Feedback for Hypothesis": "Observations related to the hypothesis",
       "New Hypothesis": "Put your new hypothesis here.",
       "Reasoning": "Provide reasoning for the hypothesis here.",
-      "Decision": True or False,
+      "Decision": <true or false>,
     }
   user: |-
     We are in an experiment of finding hypothesis and validating or rejecting them so that in the end we have a powerful model generated.
@@ -113,4 +113,4 @@ model_feedback_generation:
     Result: {{exp.result}}
 
     Compare and observe. Which result has a better return and lower risk? If the performance increases, the hypothesis should be considered positive (working). 
-    Hence, with the hypotheses, relevant reasoning, and results in mind (comparison), provide detailed and constructive feedback and suggest a new hypothesis. 
+    Hence, with the hypotheses, relevant reasoning, and results in mind (comparison), provide detailed and constructive feedback and suggest a new hypothesis. 

rdagent/utils/__init__.py

@@ -35,3 +35,21 @@ def get_module_by_module_path(module_path: Union[str, ModuleType]):
         else:
             module = importlib.import_module(module_path)
     return module
+
+
+def convert2bool(value: Union[str, bool]) -> bool:
+    """
+    Motivation: the return value of LLM is not stable. Try to convert the value into bool
+    """
+    # TODO: if we have more similar functions, we can build a library to converting unstable LLM response to stable results.
+    if isinstance(value, str):
+        v = value.lower().strip()
+        if v in ["true", "yes", "ok"]:
+            return True
+        if v in ["false", "no"]:
+            return False
+        raise ValueError(f"Can not convert {value} to bool")
+    elif isinstance(value, bool):
+        return value
+    else:
+        raise ValueError(f"Unknown value type {value} to bool")