From 9f89511be6ac03e9e9f4a8211a35cbbe6449c05a Mon Sep 17 00:00:00 2001 From: you-n-g Date: Wed, 17 Jul 2024 17:29:39 +0800 Subject: [PATCH] Model run with logger (#79) --- .gitignore | 3 ++- rdagent/app/qlib_rd_loop/model.py | 27 +++++++++++--------- rdagent/scenarios/qlib/developer/feedback.py | 5 ++-- rdagent/scenarios/qlib/prompts.yaml | 4 +-- rdagent/utils/__init__.py | 18 +++++++++++++ 5 files changed, 40 insertions(+), 17 deletions(-) diff --git a/.gitignore b/.gitignore index d2ab763a..88244b47 100644 --- a/.gitignore +++ b/.gitignore @@ -64,6 +64,7 @@ coverage.xml # Django stuff: *.log +^log local_settings.py db.sqlite3 db.sqlite3-journal @@ -161,4 +162,4 @@ mlruns/ # possible output from coder or runner *.pth -*qlib_res.csv \ No newline at end of file +*qlib_res.csv diff --git a/rdagent/app/qlib_rd_loop/model.py b/rdagent/app/qlib_rd_loop/model.py index 57af0258..6a397a0c 100644 --- a/rdagent/app/qlib_rd_loop/model.py +++ b/rdagent/app/qlib_rd_loop/model.py @@ -30,15 +30,18 @@ qlib_model_runner: Developer = import_class(PROP_SETTING.model_runner)(scen) qlib_model_summarizer: HypothesisExperiment2Feedback = import_class(PROP_SETTING.model_summarizer)(scen) trace = Trace(scen=scen) -for _ in range(PROP_SETTING.evolving_n): - try: - hypothesis = hypothesis_gen.gen(trace) - exp = hypothesis2experiment.convert(hypothesis, trace) - exp = qlib_model_coder.develop(exp) - exp = qlib_model_runner.develop(exp) - feedback = qlib_model_summarizer.generateFeedback(exp, hypothesis, trace) - - trace.hist.append((hypothesis, exp, feedback)) - except ModelEmptyException as e: - logger.warning(e) - continue +with logger.tag("model.loop"): + for _ in range(PROP_SETTING.evolving_n): + try: + with logger.tag("r"): # research + hypothesis = hypothesis_gen.gen(trace) + exp = hypothesis2experiment.convert(hypothesis, trace) + with logger.tag("d"): # develop + exp = qlib_model_coder.develop(exp) + with logger.tag("ef"): # evaluate and feedback + exp = qlib_model_runner.develop(exp) + feedback = qlib_model_summarizer.generateFeedback(exp, hypothesis, trace) + trace.hist.append((hypothesis, exp, feedback)) + except ModelEmptyException as e: + logger.warning(e) + continue diff --git a/rdagent/scenarios/qlib/developer/feedback.py b/rdagent/scenarios/qlib/developer/feedback.py index 33b7c57b..27d93dd0 100644 --- a/rdagent/scenarios/qlib/developer/feedback.py +++ b/rdagent/scenarios/qlib/developer/feedback.py @@ -16,6 +16,7 @@ from rdagent.core.proposal import ( ) from rdagent.log import rdagent_logger as logger from rdagent.oai.llm_utils import APIBackend +from rdagent.utils import convert2bool feedback_prompts = Prompts(file_path=Path(__file__).parent.parent / "prompts.yaml") DIRNAME = Path(__file__).absolute().resolve().parent @@ -74,7 +75,7 @@ class QlibFactorHypothesisExperiment2Feedback(HypothesisExperiment2Feedback): hypothesis_evaluation = response_json.get("Feedback for Hypothesis", "No feedback provided") new_hypothesis = response_json.get("New Hypothesis", "No new hypothesis provided") reason = response_json.get("Reasoning", "No reasoning provided") - decision = response_json.get("Replace Best Result", "no").lower() == "yes" + decision = convert2bool(response_json.get("Replace Best Result", "no")) return HypothesisFeedback( observations=observations, @@ -129,5 +130,5 @@ class QlibModelHypothesisExperiment2Feedback(HypothesisExperiment2Feedback): hypothesis_evaluation=response_json_hypothesis.get("Feedback for Hypothesis", "No feedback provided"), new_hypothesis=response_json_hypothesis.get("New Hypothesis", "No new hypothesis provided"), reason=response_json_hypothesis.get("Reasoning", "No reasoning provided"), - decision=str(response_json_hypothesis.get("Decision", "false")).lower() == "true", + decision=convert2bool(response_json_hypothesis.get("Decision", "false")), ) diff --git a/rdagent/scenarios/qlib/prompts.yaml b/rdagent/scenarios/qlib/prompts.yaml index 22830aaa..c4c316a8 100644 --- a/rdagent/scenarios/qlib/prompts.yaml +++ b/rdagent/scenarios/qlib/prompts.yaml @@ -92,7 +92,7 @@ model_feedback_generation: "Feedback for Hypothesis": "Observations related to the hypothesis", "New Hypothesis": "Put your new hypothesis here.", "Reasoning": "Provide reasoning for the hypothesis here.", - "Decision": True or False, + "Decision": , } user: |- We are in an experiment of finding hypothesis and validating or rejecting them so that in the end we have a powerful model generated. @@ -113,4 +113,4 @@ model_feedback_generation: Result: {{exp.result}} Compare and observe. Which result has a better return and lower risk? If the performance increases, the hypothesis should be considered positive (working). - Hence, with the hypotheses, relevant reasoning, and results in mind (comparison), provide detailed and constructive feedback and suggest a new hypothesis. \ No newline at end of file + Hence, with the hypotheses, relevant reasoning, and results in mind (comparison), provide detailed and constructive feedback and suggest a new hypothesis. diff --git a/rdagent/utils/__init__.py b/rdagent/utils/__init__.py index 98365d7b..41a096ac 100644 --- a/rdagent/utils/__init__.py +++ b/rdagent/utils/__init__.py @@ -35,3 +35,21 @@ def get_module_by_module_path(module_path: Union[str, ModuleType]): else: module = importlib.import_module(module_path) return module + + +def convert2bool(value: Union[str, bool]) -> bool: + """ + Motivation: the return value of LLM is not stable. Try to convert the value into bool + """ + # TODO: if we have more similar functions, we can build a library to converting unstable LLM response to stable results. + if isinstance(value, str): + v = value.lower().strip() + if v in ["true", "yes", "ok"]: + return True + if v in ["false", "no"]: + return False + raise ValueError(f"Can not convert {value} to bool") + elif isinstance(value, bool): + return value + else: + raise ValueError(f"Unknown value type {value} to bool")