diff --git a/rdagent/app/general_model/general_model.py b/rdagent/app/general_model/general_model.py index c1ca49ce..281ee77c 100644 --- a/rdagent/app/general_model/general_model.py +++ b/rdagent/app/general_model/general_model.py @@ -30,18 +30,15 @@ def extract_models_and_implement(report_file_path: str) -> None: Returns: None """ - with logger.tag("init"): - scenario = GeneralModelScenario() - logger.log_object(scenario, tag="scenario") - with logger.tag("r"): - # Save Relevant Images - img = extract_first_page_screenshot_from_pdf(report_file_path) - logger.log_object(img, tag="pdf_image") - exp = ModelExperimentLoaderFromPDFfiles().load(report_file_path) - logger.log_object(exp, tag="load_experiment") - with logger.tag("d"): - exp = QlibModelCoSTEER(scenario).develop(exp) - logger.log_object(exp, tag="developed_experiment") + scenario = GeneralModelScenario() + logger.log_object(scenario, tag="scenario") + # Save Relevant Images + img = extract_first_page_screenshot_from_pdf(report_file_path) + logger.log_object(img, tag="pdf_image") + exp = ModelExperimentLoaderFromPDFfiles().load(report_file_path) + logger.log_object(exp, tag="load_experiment") + exp = QlibModelCoSTEER(scenario).develop(exp) + logger.log_object(exp, tag="developed_experiment") if __name__ == "__main__": diff --git a/rdagent/app/kaggle/loop.py b/rdagent/app/kaggle/loop.py index 9f4cae81..85b5eb9d 100644 --- a/rdagent/app/kaggle/loop.py +++ b/rdagent/app/kaggle/loop.py @@ -28,90 +28,83 @@ from rdagent.scenarios.kaggle.proposal.proposal import KGTrace class KaggleRDLoop(RDLoop): def __init__(self, PROP_SETTING: BasePropSetting): - with logger.tag("init"): - scen: Scenario = import_class(PROP_SETTING.scen)(PROP_SETTING.competition) - logger.log_object(scen, tag="scenario") - knowledge_base = ( - import_class(PROP_SETTING.knowledge_base)(PROP_SETTING.knowledge_base_path, scen) - if PROP_SETTING.knowledge_base != "" - else None - ) - logger.log_object(knowledge_base, tag="knowledge_base") - self.hypothesis_gen: HypothesisGen = import_class(PROP_SETTING.hypothesis_gen)(scen) - logger.log_object(self.hypothesis_gen, tag="hypothesis generator") - self.hypothesis2experiment: Hypothesis2Experiment = import_class(PROP_SETTING.hypothesis2experiment)() - logger.log_object(self.hypothesis2experiment, tag="hypothesis2experiment") - self.feature_coder: Developer = import_class(PROP_SETTING.feature_coder)(scen) - logger.log_object(self.feature_coder, tag="feature coder") - self.model_feature_selection_coder: Developer = import_class(PROP_SETTING.model_feature_selection_coder)( - scen - ) - logger.log_object(self.model_feature_selection_coder, tag="model feature selection coder") - self.model_coder: Developer = import_class(PROP_SETTING.model_coder)(scen) - logger.log_object(self.model_coder, tag="model coder") - self.feature_runner: Developer = import_class(PROP_SETTING.feature_runner)(scen) - logger.log_object(self.feature_runner, tag="feature runner") - self.model_runner: Developer = import_class(PROP_SETTING.model_runner)(scen) - logger.log_object(self.model_runner, tag="model runner") - self.summarizer: Experiment2Feedback = import_class(PROP_SETTING.summarizer)(scen) - logger.log_object(self.summarizer, tag="summarizer") - self.trace = KGTrace(scen=scen, knowledge_base=knowledge_base) - super(RDLoop, self).__init__() + scen: Scenario = import_class(PROP_SETTING.scen)(PROP_SETTING.competition) + logger.log_object(scen, tag="scenario") + knowledge_base = ( + import_class(PROP_SETTING.knowledge_base)(PROP_SETTING.knowledge_base_path, scen) + if PROP_SETTING.knowledge_base != "" + else None + ) + logger.log_object(knowledge_base, tag="knowledge_base") + self.hypothesis_gen: HypothesisGen = import_class(PROP_SETTING.hypothesis_gen)(scen) + logger.log_object(self.hypothesis_gen, tag="hypothesis generator") + self.hypothesis2experiment: Hypothesis2Experiment = import_class(PROP_SETTING.hypothesis2experiment)() + logger.log_object(self.hypothesis2experiment, tag="hypothesis2experiment") + self.feature_coder: Developer = import_class(PROP_SETTING.feature_coder)(scen) + logger.log_object(self.feature_coder, tag="feature coder") + self.model_feature_selection_coder: Developer = import_class(PROP_SETTING.model_feature_selection_coder)(scen) + logger.log_object(self.model_feature_selection_coder, tag="model feature selection coder") + self.model_coder: Developer = import_class(PROP_SETTING.model_coder)(scen) + logger.log_object(self.model_coder, tag="model coder") + self.feature_runner: Developer = import_class(PROP_SETTING.feature_runner)(scen) + logger.log_object(self.feature_runner, tag="feature runner") + self.model_runner: Developer = import_class(PROP_SETTING.model_runner)(scen) + logger.log_object(self.model_runner, tag="model runner") + self.summarizer: Experiment2Feedback = import_class(PROP_SETTING.summarizer)(scen) + logger.log_object(self.summarizer, tag="summarizer") + self.trace = KGTrace(scen=scen, knowledge_base=knowledge_base) + super(RDLoop, self).__init__() def coding(self, prev_out: dict[str, Any]): - with logger.tag("d"): # develop - if prev_out["direct_exp_gen"]["propose"].action in [ - KG_ACTION_FEATURE_ENGINEERING, - KG_ACTION_FEATURE_PROCESSING, - ]: - exp = self.feature_coder.develop(prev_out["direct_exp_gen"]["exp_gen"]) - elif prev_out["direct_exp_gen"]["propose"].action == KG_ACTION_MODEL_FEATURE_SELECTION: - exp = self.model_feature_selection_coder.develop(prev_out["direct_exp_gen"]["exp_gen"]) - else: - exp = self.model_coder.develop(prev_out["direct_exp_gen"]["exp_gen"]) - logger.log_object(exp.sub_workspace_list, tag="coder result") + if prev_out["direct_exp_gen"]["propose"].action in [ + KG_ACTION_FEATURE_ENGINEERING, + KG_ACTION_FEATURE_PROCESSING, + ]: + exp = self.feature_coder.develop(prev_out["direct_exp_gen"]["exp_gen"]) + elif prev_out["direct_exp_gen"]["propose"].action == KG_ACTION_MODEL_FEATURE_SELECTION: + exp = self.model_feature_selection_coder.develop(prev_out["direct_exp_gen"]["exp_gen"]) + else: + exp = self.model_coder.develop(prev_out["direct_exp_gen"]["exp_gen"]) + logger.log_object(exp.sub_workspace_list, tag="coder result") return exp def running(self, prev_out: dict[str, Any]): - with logger.tag("ef"): # evaluate and feedback - if prev_out["direct_exp_gen"]["propose"].action in [ - KG_ACTION_FEATURE_ENGINEERING, - KG_ACTION_FEATURE_PROCESSING, - ]: - exp = self.feature_runner.develop(prev_out["coding"]) - else: - exp = self.model_runner.develop(prev_out["coding"]) - logger.log_object(exp, tag="runner result") - if KAGGLE_IMPLEMENT_SETTING.competition in [ - "optiver-realized-volatility-prediction", - "covid19-global-forecasting-week-1", - ]: - try: - python_files_to_notebook( - KAGGLE_IMPLEMENT_SETTING.competition, exp.experiment_workspace.workspace_path - ) - except Exception as e: - logger.error(f"Merge python files to one file failed: {e}") - if KAGGLE_IMPLEMENT_SETTING.auto_submit: - csv_path = exp.experiment_workspace.workspace_path / "submission.csv" - try: - subprocess.run( - [ - "kaggle", - "competitions", - "submit", - "-f", - str(csv_path.absolute()), - "-m", - str(csv_path.parent.absolute()), - KAGGLE_IMPLEMENT_SETTING.competition, - ], - check=True, - ) - except subprocess.CalledProcessError as e: - logger.error(f"Auto submission failed: \n{e}") - except Exception as e: - logger.error(f"Other exception when use kaggle api:\n{e}") + if prev_out["direct_exp_gen"]["propose"].action in [ + KG_ACTION_FEATURE_ENGINEERING, + KG_ACTION_FEATURE_PROCESSING, + ]: + exp = self.feature_runner.develop(prev_out["coding"]) + else: + exp = self.model_runner.develop(prev_out["coding"]) + logger.log_object(exp, tag="runner result") + if KAGGLE_IMPLEMENT_SETTING.competition in [ + "optiver-realized-volatility-prediction", + "covid19-global-forecasting-week-1", + ]: + try: + python_files_to_notebook(KAGGLE_IMPLEMENT_SETTING.competition, exp.experiment_workspace.workspace_path) + except Exception as e: + logger.error(f"Merge python files to one file failed: {e}") + if KAGGLE_IMPLEMENT_SETTING.auto_submit: + csv_path = exp.experiment_workspace.workspace_path / "submission.csv" + try: + subprocess.run( + [ + "kaggle", + "competitions", + "submit", + "-f", + str(csv_path.absolute()), + "-m", + str(csv_path.parent.absolute()), + KAGGLE_IMPLEMENT_SETTING.competition, + ], + check=True, + ) + except subprocess.CalledProcessError as e: + logger.error(f"Auto submission failed: \n{e}") + except Exception as e: + logger.error(f"Other exception when use kaggle api:\n{e}") return exp diff --git a/rdagent/app/qlib_rd_loop/factor.py b/rdagent/app/qlib_rd_loop/factor.py index f13b3d8a..9a9d7174 100755 --- a/rdagent/app/qlib_rd_loop/factor.py +++ b/rdagent/app/qlib_rd_loop/factor.py @@ -17,12 +17,11 @@ class FactorRDLoop(RDLoop): skip_loop_error = (FactorEmptyError,) def running(self, prev_out: dict[str, Any]): - with logger.tag("ef"): # evaluate and feedback - exp = self.runner.develop(prev_out["coding"]) - if exp is None: - logger.error(f"Factor extraction failed.") - raise FactorEmptyError("Factor extraction failed.") - logger.log_object(exp, tag="runner result") + exp = self.runner.develop(prev_out["coding"]) + if exp is None: + logger.error(f"Factor extraction failed.") + raise FactorEmptyError("Factor extraction failed.") + logger.log_object(exp, tag="runner result") return exp diff --git a/rdagent/app/qlib_rd_loop/factor_from_report.py b/rdagent/app/qlib_rd_loop/factor_from_report.py index 24213b49..e5938af6 100644 --- a/rdagent/app/qlib_rd_loop/factor_from_report.py +++ b/rdagent/app/qlib_rd_loop/factor_from_report.py @@ -112,27 +112,26 @@ class FactorReportLoop(FactorRDLoop, metaclass=LoopMeta): self.steps = ["propose_hypo_exp", "propose", "direct_exp_gen", "coding", "running", "feedback"] def propose_hypo_exp(self, prev_out: dict[str, Any]): - with logger.tag("r"): - while True: - if FACTOR_FROM_REPORT_PROP_SETTING.is_report_limit_enabled and self.valid_pdf_file_count > 15: - break - report_file_path = self.judge_pdf_data_items[self.pdf_file_index] - logger.info(f"Processing number {self.pdf_file_index} report: {report_file_path}") - self.pdf_file_index += 1 - exp, hypothesis = extract_hypothesis_and_exp_from_reports(str(report_file_path)) - if exp is None: - continue - self.valid_pdf_file_count += 1 - exp.based_experiments = [QlibFactorExperiment(sub_tasks=[], hypothesis=hypothesis)] + [ - t[0] for t in self.trace.hist if t[1] - ] - exp.sub_workspace_list = exp.sub_workspace_list[: FACTOR_FROM_REPORT_PROP_SETTING.max_factors_per_exp] - exp.sub_tasks = exp.sub_tasks[: FACTOR_FROM_REPORT_PROP_SETTING.max_factors_per_exp] - logger.log_object(hypothesis, tag="hypothesis generation") - logger.log_object(exp.sub_tasks, tag="experiment generation") - self.current_loop_hypothesis = hypothesis - self.current_loop_exp = exp - return None + while True: + if FACTOR_FROM_REPORT_PROP_SETTING.is_report_limit_enabled and self.valid_pdf_file_count > 15: + break + report_file_path = self.judge_pdf_data_items[self.pdf_file_index] + logger.info(f"Processing number {self.pdf_file_index} report: {report_file_path}") + self.pdf_file_index += 1 + exp, hypothesis = extract_hypothesis_and_exp_from_reports(str(report_file_path)) + if exp is None: + continue + self.valid_pdf_file_count += 1 + exp.based_experiments = [QlibFactorExperiment(sub_tasks=[], hypothesis=hypothesis)] + [ + t[0] for t in self.trace.hist if t[1] + ] + exp.sub_workspace_list = exp.sub_workspace_list[: FACTOR_FROM_REPORT_PROP_SETTING.max_factors_per_exp] + exp.sub_tasks = exp.sub_tasks[: FACTOR_FROM_REPORT_PROP_SETTING.max_factors_per_exp] + logger.log_object(hypothesis, tag="hypothesis generation") + logger.log_object(exp.sub_tasks, tag="experiment generation") + self.current_loop_hypothesis = hypothesis + self.current_loop_exp = exp + return None def propose(self, prev_out: dict[str, Any]): return self.current_loop_hypothesis @@ -141,9 +140,8 @@ class FactorReportLoop(FactorRDLoop, metaclass=LoopMeta): return {"propose": self.current_loop_hypothesis, "exp_gen": self.current_loop_exp} def coding(self, prev_out: dict[str, Any]): - with logger.tag("d"): # develop - exp = self.coder.develop(prev_out["direct_exp_gen"]["exp_gen"]) - logger.log_object(exp.sub_workspace_list, tag="coder result") + exp = self.coder.develop(prev_out["direct_exp_gen"]["exp_gen"]) + logger.log_object(exp.sub_workspace_list, tag="coder result") return exp diff --git a/rdagent/app/qlib_rd_loop/quant.py b/rdagent/app/qlib_rd_loop/quant.py index 6573a537..3a3a8b34 100644 --- a/rdagent/app/qlib_rd_loop/quant.py +++ b/rdagent/app/qlib_rd_loop/quant.py @@ -31,70 +31,66 @@ class QuantRDLoop(RDLoop): ) def __init__(self, PROP_SETTING: BasePropSetting): - with logger.tag("init"): - scen: Scenario = import_class(PROP_SETTING.scen)() - logger.log_object(scen, tag="scenario") + scen: Scenario = import_class(PROP_SETTING.scen)() + logger.log_object(scen, tag="scenario") - self.hypothesis_gen: HypothesisGen = import_class(PROP_SETTING.quant_hypothesis_gen)(scen) - logger.log_object(self.hypothesis_gen, tag="quant hypothesis generator") + self.hypothesis_gen: HypothesisGen = import_class(PROP_SETTING.quant_hypothesis_gen)(scen) + logger.log_object(self.hypothesis_gen, tag="quant hypothesis generator") - self.factor_hypothesis2experiment: Hypothesis2Experiment = import_class( - PROP_SETTING.factor_hypothesis2experiment - )() - logger.log_object(self.factor_hypothesis2experiment, tag="factor hypothesis2experiment") - self.model_hypothesis2experiment: Hypothesis2Experiment = import_class( - PROP_SETTING.model_hypothesis2experiment - )() - logger.log_object(self.model_hypothesis2experiment, tag="model hypothesis2experiment") + self.factor_hypothesis2experiment: Hypothesis2Experiment = import_class( + PROP_SETTING.factor_hypothesis2experiment + )() + logger.log_object(self.factor_hypothesis2experiment, tag="factor hypothesis2experiment") + self.model_hypothesis2experiment: Hypothesis2Experiment = import_class( + PROP_SETTING.model_hypothesis2experiment + )() + logger.log_object(self.model_hypothesis2experiment, tag="model hypothesis2experiment") - self.factor_coder: Developer = import_class(PROP_SETTING.factor_coder)(scen) - logger.log_object(self.factor_coder, tag="factor coder") - self.model_coder: Developer = import_class(PROP_SETTING.model_coder)(scen) - logger.log_object(self.model_coder, tag="model coder") + self.factor_coder: Developer = import_class(PROP_SETTING.factor_coder)(scen) + logger.log_object(self.factor_coder, tag="factor coder") + self.model_coder: Developer = import_class(PROP_SETTING.model_coder)(scen) + logger.log_object(self.model_coder, tag="model coder") - self.factor_runner: Developer = import_class(PROP_SETTING.factor_runner)(scen) - logger.log_object(self.factor_runner, tag="factor runner") - self.model_runner: Developer = import_class(PROP_SETTING.model_runner)(scen) - logger.log_object(self.model_runner, tag="model runner") + self.factor_runner: Developer = import_class(PROP_SETTING.factor_runner)(scen) + logger.log_object(self.factor_runner, tag="factor runner") + self.model_runner: Developer = import_class(PROP_SETTING.model_runner)(scen) + logger.log_object(self.model_runner, tag="model runner") - self.factor_summarizer: Experiment2Feedback = import_class(PROP_SETTING.factor_summarizer)(scen) - logger.log_object(self.factor_summarizer, tag="factor summarizer") - self.model_summarizer: Experiment2Feedback = import_class(PROP_SETTING.model_summarizer)(scen) - logger.log_object(self.model_summarizer, tag="model summarizer") + self.factor_summarizer: Experiment2Feedback = import_class(PROP_SETTING.factor_summarizer)(scen) + logger.log_object(self.factor_summarizer, tag="factor summarizer") + self.model_summarizer: Experiment2Feedback = import_class(PROP_SETTING.model_summarizer)(scen) + logger.log_object(self.model_summarizer, tag="model summarizer") - self.trace = QuantTrace(scen=scen) - super(RDLoop, self).__init__() + self.trace = QuantTrace(scen=scen) + super(RDLoop, self).__init__() def direct_exp_gen(self, prev_out: dict[str, Any]): - with logger.tag("r"): # research - hypo = self._propose() - assert hypo.action in ["factor", "model"] - if hypo.action == "factor": - exp = self.factor_hypothesis2experiment.convert(hypo, self.trace) - else: - exp = self.model_hypothesis2experiment.convert(hypo, self.trace) - logger.log_object(exp.sub_tasks, tag="experiment generation") + hypo = self._propose() + assert hypo.action in ["factor", "model"] + if hypo.action == "factor": + exp = self.factor_hypothesis2experiment.convert(hypo, self.trace) + else: + exp = self.model_hypothesis2experiment.convert(hypo, self.trace) + logger.log_object(exp.sub_tasks, tag="experiment generation") return {"propose": hypo, "exp_gen": exp} def coding(self, prev_out: dict[str, Any]): - with logger.tag("d"): # development - if prev_out["direct_exp_gen"]["propose"].action == "factor": - exp = self.factor_coder.develop(prev_out["direct_exp_gen"]["exp_gen"]) - elif prev_out["direct_exp_gen"]["propose"].action == "model": - exp = self.model_coder.develop(prev_out["direct_exp_gen"]["exp_gen"]) - logger.log_object(exp, tag="coder result") + if prev_out["direct_exp_gen"]["propose"].action == "factor": + exp = self.factor_coder.develop(prev_out["direct_exp_gen"]["exp_gen"]) + elif prev_out["direct_exp_gen"]["propose"].action == "model": + exp = self.model_coder.develop(prev_out["direct_exp_gen"]["exp_gen"]) + logger.log_object(exp, tag="coder result") return exp def running(self, prev_out: dict[str, Any]): - with logger.tag("ef"): - if prev_out["direct_exp_gen"]["propose"].action == "factor": - exp = self.factor_runner.develop(prev_out["coding"]) - if exp is None: - logger.error(f"Factor extraction failed.") - raise FactorEmptyError("Factor extraction failed.") - elif prev_out["direct_exp_gen"]["propose"].action == "model": - exp = self.model_runner.develop(prev_out["coding"]) - logger.log_object(exp, tag="runner result") + if prev_out["direct_exp_gen"]["propose"].action == "factor": + exp = self.factor_runner.develop(prev_out["coding"]) + if exp is None: + logger.error(f"Factor extraction failed.") + raise FactorEmptyError("Factor extraction failed.") + elif prev_out["direct_exp_gen"]["propose"].action == "model": + exp = self.model_runner.develop(prev_out["coding"]) + logger.log_object(exp, tag="runner result") return exp def feedback(self, prev_out: dict[str, Any]): @@ -107,16 +103,14 @@ class QuantRDLoop(RDLoop): reason="", decision=False, ) - with logger.tag("ef"): # evaluate and feedback - logger.log_object(feedback, tag="feedback") + logger.log_object(feedback, tag="feedback") self.trace.hist.append((prev_out["direct_exp_gen"]["exp_gen"], feedback)) else: if prev_out["direct_exp_gen"]["propose"].action == "factor": feedback = self.factor_summarizer.generate_feedback(prev_out["running"], self.trace) elif prev_out["direct_exp_gen"]["propose"].action == "model": feedback = self.model_summarizer.generate_feedback(prev_out["running"], self.trace) - with logger.tag("ef"): - logger.log_object(feedback, tag="feedback") + logger.log_object(feedback, tag="feedback") self.trace.hist.append((prev_out["running"], feedback)) diff --git a/rdagent/components/workflow/rd_loop.py b/rdagent/components/workflow/rd_loop.py index a79a5a8b..5b315d1e 100644 --- a/rdagent/components/workflow/rd_loop.py +++ b/rdagent/components/workflow/rd_loop.py @@ -24,25 +24,24 @@ from rdagent.utils.workflow import LoopBase, LoopMeta class RDLoop(LoopBase, metaclass=LoopMeta): def __init__(self, PROP_SETTING: BasePropSetting): - with logger.tag("init"): - scen: Scenario = import_class(PROP_SETTING.scen)() - logger.log_object(scen, tag="scenario") + scen: Scenario = import_class(PROP_SETTING.scen)() + logger.log_object(scen, tag="scenario") - self.hypothesis_gen: HypothesisGen = import_class(PROP_SETTING.hypothesis_gen)(scen) - logger.log_object(self.hypothesis_gen, tag="hypothesis generator") + self.hypothesis_gen: HypothesisGen = import_class(PROP_SETTING.hypothesis_gen)(scen) + logger.log_object(self.hypothesis_gen, tag="hypothesis generator") - self.hypothesis2experiment: Hypothesis2Experiment = import_class(PROP_SETTING.hypothesis2experiment)() - logger.log_object(self.hypothesis2experiment, tag="hypothesis2experiment") + self.hypothesis2experiment: Hypothesis2Experiment = import_class(PROP_SETTING.hypothesis2experiment)() + logger.log_object(self.hypothesis2experiment, tag="hypothesis2experiment") - self.coder: Developer = import_class(PROP_SETTING.coder)(scen) - logger.log_object(self.coder, tag="coder") - self.runner: Developer = import_class(PROP_SETTING.runner)(scen) - logger.log_object(self.runner, tag="runner") + self.coder: Developer = import_class(PROP_SETTING.coder)(scen) + logger.log_object(self.coder, tag="coder") + self.runner: Developer = import_class(PROP_SETTING.runner)(scen) + logger.log_object(self.runner, tag="runner") - self.summarizer: Experiment2Feedback = import_class(PROP_SETTING.summarizer)(scen) - logger.log_object(self.summarizer, tag="summarizer") - self.trace = Trace(scen=scen) - super().__init__() + self.summarizer: Experiment2Feedback = import_class(PROP_SETTING.summarizer)(scen) + logger.log_object(self.summarizer, tag="summarizer") + self.trace = Trace(scen=scen) + super().__init__() # excluded steps def _propose(self): @@ -57,21 +56,18 @@ class RDLoop(LoopBase, metaclass=LoopMeta): # included steps def direct_exp_gen(self, prev_out: dict[str, Any]): - with logger.tag("r"): # research - hypo = self._propose() - exp = self._exp_gen(hypo) + hypo = self._propose() + exp = self._exp_gen(hypo) return {"propose": hypo, "exp_gen": exp} def coding(self, prev_out: dict[str, Any]): - with logger.tag("d"): # develop - exp = self.coder.develop(prev_out["direct_exp_gen"]["exp_gen"]) - logger.log_object(exp.sub_workspace_list, tag="coder result") + exp = self.coder.develop(prev_out["direct_exp_gen"]["exp_gen"]) + logger.log_object(exp.sub_workspace_list, tag="coder result") return exp def running(self, prev_out: dict[str, Any]): - with logger.tag("ef"): # evaluate and feedback - exp = self.runner.develop(prev_out["coding"]) - logger.log_object(exp, tag="runner result") + exp = self.runner.develop(prev_out["coding"]) + logger.log_object(exp, tag="runner result") return exp def feedback(self, prev_out: dict[str, Any]): @@ -84,11 +80,9 @@ class RDLoop(LoopBase, metaclass=LoopMeta): reason="", decision=False, ) - with logger.tag("ef"): # evaluate and feedback - logger.log_object(feedback, tag="feedback") + logger.log_object(feedback, tag="feedback") self.trace.hist.append((prev_out["direct_exp_gen"]["exp_gen"], feedback)) else: feedback = self.summarizer.generate_feedback(prev_out["running"], self.trace) - with logger.tag("ef"): # evaluate and feedback - logger.log_object(feedback, tag="feedback") + logger.log_object(feedback, tag="feedback") self.trace.hist.append((prev_out["running"], feedback)) diff --git a/rdagent/log/ui/app.py b/rdagent/log/ui/app.py index c0ceef7c..410703c2 100644 --- a/rdagent/log/ui/app.py +++ b/rdagent/log/ui/app.py @@ -111,9 +111,6 @@ if "current_tags" not in state: if "lround" not in state: state.lround = 0 # RD Loop Round -if "times" not in state: - state.times = defaultdict(lambda: defaultdict(list)) - if "erounds" not in state: state.erounds = defaultdict(int) # Evolving Rounds in each RD Loop @@ -140,7 +137,7 @@ if "alpha158_metrics" not in state: def should_display(msg: Message): - for t in state.excluded_tags: + for t in state.excluded_tags + ["debug_tpl", "debug_llm"]: if t in msg.tag.split("."): return False @@ -155,17 +152,24 @@ def get_msgs_until(end_func: Callable[[Message], bool] = lambda _: True): while True: try: msg = next(state.fs) - - # new scenario gen this tags, old version UI not have these tags. - msg.tag = re.sub(r"\.evo_loop_\d+", "", msg.tag) - msg.tag = re.sub(r"Loop_\d+\.[^.]+", "", msg.tag) - msg.tag = re.sub(r"\.\.", ".", msg.tag) - msg.tag = msg.tag.strip(".") - if should_display(msg): tags = msg.tag.split(".") - if "r" not in state.current_tags and "r" in tags: + if "hypothesis generation" in msg.tag: state.lround += 1 + + # new scenario gen this tags, old version UI not have these tags. + msg.tag = re.sub(r"\.evo_loop_\d+", "", msg.tag) + msg.tag = re.sub(r"Loop_\d+\.[^.]+", "", msg.tag) + msg.tag = re.sub(r"\.\.", ".", msg.tag) + + # remove old redundant tags + msg.tag = re.sub(r"init\.", "", msg.tag) + msg.tag = re.sub(r"r\.", "", msg.tag) + msg.tag = re.sub(r"d\.", "", msg.tag) + msg.tag = re.sub(r"ef\.", "", msg.tag) + + msg.tag = msg.tag.strip(".") + if "evolving code" not in state.current_tags and "evolving code" in tags: state.erounds[state.lround] += 1 @@ -173,7 +177,7 @@ def get_msgs_until(end_func: Callable[[Message], bool] = lambda _: True): state.last_msg = msg # Update Summary Info - if "model runner result" in tags or "factor runner result" in tags or "runner result" in tags: + if "runner result" in tags: # factor baseline exp metrics if ( isinstance(state.scenario, (QlibFactorScenario, QlibQuantScenario)) @@ -234,37 +238,26 @@ def get_msgs_until(end_func: Callable[[Message], bool] = lambda _: True): state.all_metric_series.append(sms_all) elif "hypothesis generation" in tags: state.hypotheses[state.lround] = msg.content - elif "ef" in tags and "feedback" in tags: + elif "evolving code" in tags: + msg.content = [i for i in msg.content if i] + elif "evolving feedback" in tags: + total_len = len(msg.content) + none_num = total_len - len(msg.content) + right_num = 0 + for wsf in msg.content: + if wsf.final_decision: + right_num += 1 + wrong_num = len(msg.content) - right_num + state.e_decisions[state.lround][state.erounds[state.lround]] = ( + right_num, + wrong_num, + none_num, + ) + elif "feedback" in tags and isinstance(msg.content, HypothesisFeedback): state.h_decisions[state.lround] = msg.content.decision - elif "d" in tags: - if "evolving code" in tags: - msg.content = [i for i in msg.content if i] - if "evolving feedback" in tags: - total_len = len(msg.content) - none_num = total_len - len(msg.content) - right_num = 0 - for wsf in msg.content: - if wsf.final_decision: - right_num += 1 - wrong_num = len(msg.content) - right_num - state.e_decisions[state.lround][state.erounds[state.lround]] = ( - right_num, - wrong_num, - none_num, - ) state.msgs[state.lround][msg.tag].append(msg) - # Update Times - if "init" in tags: - state.times[state.lround]["init"].append(msg.timestamp) - if "r" in tags: - state.times[state.lround]["r"].append(msg.timestamp) - if "d" in tags: - state.times[state.lround]["d"].append(msg.timestamp) - if "ef" in tags: - state.times[state.lround]["ef"].append(msg.timestamp) - # Stop Getting Logs if end_func(msg): break @@ -305,7 +298,6 @@ def refresh(same_trace: bool = False): state.last_msg = None state.current_tags = [] state.alpha158_metrics = None - state.times = defaultdict(lambda: defaultdict(list)) def evolving_feedback_window(wsf: FactorSingleFeedback | ModelSingleFeedback): @@ -471,7 +463,9 @@ def summary_window(): df = pd.DataFrame(state.metric_series) if show_true_only and len(state.hypotheses) >= len(state.metric_series): if state.alpha158_metrics is not None: - selected = ["alpha158"] + [i for i in df.index if state.h_decisions[int(i[6:])]] + selected = ["alpha158"] + [ + i for i in df.index if i == "Baseline" or state.h_decisions[int(i[6:])] + ] else: selected = [i for i in df.index if i == "Baseline" or state.h_decisions[int(i[6:])]] df = df.loc[selected] @@ -488,9 +482,9 @@ def summary_window(): elif isinstance(state.scenario, GeneralModelScenario): with st.container(border=True): st.subheader("Summary📊", divider="rainbow", anchor="_summary") - if len(state.msgs[state.lround]["d.evolving code"]) > 0: + if len(state.msgs[state.lround]["evolving code"]) > 0: # pass - ws: list[FactorFBWorkspace | ModelFBWorkspace] = state.msgs[state.lround]["d.evolving code"][-1].content + ws: list[FactorFBWorkspace | ModelFBWorkspace] = state.msgs[state.lround]["evolving code"][-1].content # All Tasks tab_names = [ @@ -498,7 +492,7 @@ def summary_window(): for w in ws ] for j in range(len(ws)): - if state.msgs[state.lround]["d.evolving feedback"][-1].content[j].final_decision: + if state.msgs[state.lround]["evolving feedback"][-1].content[j].final_decision: tab_names[j] += "✔️" else: tab_names[j] += "❌" @@ -512,7 +506,7 @@ def summary_window(): st.code(v, language="python") # Evolving Feedback - evolving_feedback_window(state.msgs[state.lround]["d.evolving feedback"][-1].content[j]) + evolving_feedback_window(state.msgs[state.lround]["evolving feedback"][-1].content[j]) def tabs_hint(): @@ -570,12 +564,12 @@ def research_window(): st.subheader(title, divider="blue", anchor="_research") if isinstance(state.scenario, SIMILAR_SCENARIOS): # pdf image - if pim := state.msgs[round]["r.extract_factors_and_implement.load_pdf_screenshot"]: + if pim := state.msgs[round]["extract_factors_and_implement.load_pdf_screenshot"]: for i in range(min(2, len(pim))): st.image(pim[i].content, use_container_width=True) # Hypothesis - if hg := state.msgs[round]["r.hypothesis generation"]: + if hg := state.msgs[round]["hypothesis generation"]: st.markdown("**Hypothesis💡**") # 🧠 h: Hypothesis = hg[0].content st.markdown( @@ -584,25 +578,20 @@ def research_window(): - **Reason**: {h.reason}""" ) - if eg := state.msgs[round]["r.experiment generation"]: + if eg := state.msgs[round]["experiment generation"]: tasks_window(eg[0].content) elif isinstance(state.scenario, GeneralModelScenario): # pdf image c1, c2 = st.columns([2, 3]) with c1: - if pim := state.msgs[round]["r.pdf_image"]: + if pim := state.msgs[0]["pdf_image"]: for i in range(len(pim)): st.image(pim[i].content, use_container_width=True) # loaded model exp with c2: - if mem := state.msgs[round]["d.load_experiment"]: - # 'load_experiment' should in 'r' now, but old version trace may in 'd', so we need to check both - # TODO: modify the way to get one message with a specific tag like 'load_experiment' in the future - me: QlibModelExperiment = mem[0].content - tasks_window(me.sub_tasks) - elif mem := state.msgs[round]["r.load_experiment"]: + if mem := state.msgs[0]["load_experiment"]: me: QlibModelExperiment = mem[0].content tasks_window(me.sub_tasks) @@ -649,7 +638,7 @@ def feedback_window(): state.scenario, (QlibModelScenario, QlibFactorScenario, QlibFactorFromReportScenario, QlibQuantScenario, KGScenario), ): - if fbr := state.msgs[round]["ef.runner result"]: + if fbr := state.msgs[round]["runner result"]: try: st.write("workspace") st.write(fbr[0].content.experiment_workspace.workspace_path) @@ -659,11 +648,11 @@ def feedback_window(): with st.expander("**Config⚙️**", expanded=True): st.markdown(state.scenario.experiment_setting, unsafe_allow_html=True) - if fbr := state.msgs[round]["ef.Quantitative Backtesting Chart"]: + if fbr := state.msgs[round]["Quantitative Backtesting Chart"]: st.markdown("**Returns📈**") fig = report_figure(fbr[0].content) st.plotly_chart(fig) - if fb := state.msgs[round]["ef.feedback"]: + if fb := state.msgs[round]["feedback"]: st.markdown("**Hypothesis Feedback🔍**") h: HypothesisFeedback = fb[0].content st.markdown( @@ -676,7 +665,7 @@ def feedback_window(): ) if isinstance(state.scenario, KGScenario): - if fbe := state.msgs[round]["ef.runner result"]: + if fbe := state.msgs[round]["runner result"]: submission_path = fbe[0].content.experiment_workspace.workspace_path / "submission.csv" st.markdown( f":green[**Exp Workspace**]: {str(fbe[0].content.experiment_workspace.workspace_path.absolute())}" @@ -723,17 +712,15 @@ def evolving_window(): else: evolving_round = 1 - ws: list[FactorFBWorkspace | ModelFBWorkspace] = state.msgs[round]["d.evolving code"][ - evolving_round - 1 - ].content + ws: list[FactorFBWorkspace | ModelFBWorkspace] = state.msgs[round]["evolving code"][evolving_round - 1].content # All Tasks tab_names = [ w.target_task.factor_name if isinstance(w.target_task, FactorTask) else w.target_task.name for w in ws ] - if len(state.msgs[round]["d.evolving feedback"]) >= evolving_round: + if len(state.msgs[round]["evolving feedback"]) >= evolving_round: for j in range(len(ws)): - if state.msgs[round]["d.evolving feedback"][evolving_round - 1].content[j].final_decision: + if state.msgs[round]["evolving feedback"][evolving_round - 1].content[j].final_decision: tab_names[j] += "✔️" else: tab_names[j] += "❌" @@ -749,8 +736,8 @@ def evolving_window(): st.code(v, language="python") # Evolving Feedback - if len(state.msgs[round]["d.evolving feedback"]) >= evolving_round: - evolving_feedback_window(state.msgs[round]["d.evolving feedback"][evolving_round - 1].content[j]) + if len(state.msgs[round]["evolving feedback"]) >= evolving_round: + evolving_feedback_window(state.msgs[round]["evolving feedback"][evolving_round - 1].content[j]) toc = """ @@ -804,12 +791,12 @@ with st.sidebar: if st.button(":green[Next Loop]", use_container_width=True): if not state.fs: refresh() - get_msgs_until(lambda m: "ef.feedback" in m.tag) + get_msgs_until(lambda m: "feedback" in m.tag and "evolving feedback" not in m.tag) if st.button("Next Step", use_container_width=True): if not state.fs: refresh() - get_msgs_until(lambda m: "d.evolving feedback" in m.tag) + get_msgs_until(lambda m: "evolving feedback" in m.tag) with st.popover(":orange[**Config⚙️**]", use_container_width=True): st.multiselect("excluded log tags", ["llm_messages"], ["llm_messages"], key="excluded_tags") @@ -891,18 +878,6 @@ with st.container(): st.markdown(state.scenario.rich_style_description + css, unsafe_allow_html=True) -def show_times(round: int): - for k, v in state.times[round].items(): - if len(v) > 1: - diff = v[-1] - v[0] - else: - diff = v[0] - v[0] - total_seconds = diff.seconds - seconds = total_seconds % 60 - minutes = total_seconds // 60 - st.markdown(f"**:blue[{k}]**: :red[**{minutes}**] minutes :orange[**{seconds}**] seconds") - - def analyze_task_completion(): st.header("Task Completion Analysis", divider="orange") @@ -924,9 +899,9 @@ def analyze_task_completion(): # For each evolving round in this loop for e_round in range(1, max_evolving_round + 1): - if len(state.msgs[loop_round]["d.evolving feedback"]) >= e_round: + if len(state.msgs[loop_round]["evolving feedback"]) >= e_round: # Get feedback for this evolving round - feedback = state.msgs[loop_round]["d.evolving feedback"][e_round - 1].content + feedback = state.msgs[loop_round]["evolving feedback"][e_round - 1].content # Count passed tasks and track their indices passed_tasks = set() @@ -944,7 +919,7 @@ def analyze_task_completion(): } completion_stats[loop_round] = { - "total_tasks": len(state.msgs[loop_round]["d.evolving feedback"][0].content), + "total_tasks": len(state.msgs[loop_round]["evolving feedback"][0].content), "rounds": tasks_passed_by_round, "max_round": max_evolving_round, } @@ -1147,14 +1122,12 @@ if state.scenario is not None: else: round = 1 - show_times(round) rf_c, d_c = st.columns([2, 2]) elif isinstance(state.scenario, GeneralModelScenario): - show_times(round) rf_c = st.container() d_c = st.container() - round = 1 + round = 0 else: st.error("Unknown Scenario!") st.stop()