From f2cb3ce9098fbb0fdc27eaafd61af696c3383cc3 Mon Sep 17 00:00:00 2001 From: Yuante Li <104308117+WinstonLiyt@users.noreply.github.com> Date: Mon, 7 Jul 2025 19:21:05 +0800 Subject: [PATCH] fix: fix some bugs in quant scen (#1026) * fix some bugs * fix ci * fix * refine * fix ci --- rdagent/core/proposal.py | 5 +- rdagent/log/ui/app.py | 83 ++++++++++--------- .../scenarios/qlib/experiment/workspace.py | 9 +- 3 files changed, 52 insertions(+), 45 deletions(-) diff --git a/rdagent/core/proposal.py b/rdagent/core/proposal.py index 50d7bba9..4bcfa805 100644 --- a/rdagent/core/proposal.py +++ b/rdagent/core/proposal.py @@ -73,8 +73,9 @@ class ExperimentFeedback(Feedback): def __str__(self) -> str: res = f"Decision: {self.decision}\nReason: {self.reason}" - if self.code_change_summary is not None: - res += "\nCode Change Summary: " + self.code_change_summary + code_change_summary = getattr(self, "code_change_summary", None) + if code_change_summary is not None: + res += "\nCode Change Summary: " + code_change_summary return res @classmethod diff --git a/rdagent/log/ui/app.py b/rdagent/log/ui/app.py index 86655e33..e73e33a3 100644 --- a/rdagent/log/ui/app.py +++ b/rdagent/log/ui/app.py @@ -130,8 +130,8 @@ if "all_metric_series" not in state: state.all_metric_series = [] # Factor Task Baseline -if "alpha158_metrics" not in state: - state.alpha158_metrics = None +if "alpha_baseline_metrics" not in state: + state.alpha_baseline_metrics = None def should_display(msg: Message): @@ -179,35 +179,42 @@ def get_msgs_until(end_func: Callable[[Message], bool] = lambda _: True): # factor baseline exp metrics if ( isinstance(state.scenario, (QlibFactorScenario, QlibQuantScenario)) - and state.alpha158_metrics is None + and state.alpha_baseline_metrics is None ): - sms = msg.content.based_experiments[0].result.loc[QLIB_SELECTED_METRICS] - sms.name = "alpha158" - state.alpha158_metrics = sms + try: + sms = msg.content.based_experiments[0].result + except AttributeError: + sms = msg.content.based_experiments[0].__dict__["result"] + sms = sms.loc[QLIB_SELECTED_METRICS] + sms.name = "Alpha Base" + state.alpha_baseline_metrics = sms - if ( - state.lround == 1 - and len(msg.content.based_experiments) > 0 - and msg.content.based_experiments[-1].result is not None - ): - sms = msg.content.based_experiments[-1].result - if isinstance( - state.scenario, - ( - QlibModelScenario, - QlibFactorFromReportScenario, - QlibFactorScenario, - QlibQuantScenario, - ), - ): - sms_all = sms - sms = sms.loc[QLIB_SELECTED_METRICS] - sms.name = f"Baseline" - state.metric_series.append(sms) - state.all_metric_series.append(sms_all) + if state.lround == 1 and len(msg.content.based_experiments) > 0: + try: + sms = msg.content.based_experiments[-1].result + except AttributeError: + sms = msg.content.based_experiments[-1].__dict__["result"] + if sms is not None: + if isinstance( + state.scenario, + ( + QlibModelScenario, + QlibFactorFromReportScenario, + QlibFactorScenario, + QlibQuantScenario, + ), + ): + sms_all = sms + sms = sms.loc[QLIB_SELECTED_METRICS] + sms.name = f"Baseline" + state.metric_series.append(sms) + state.all_metric_series.append(sms_all) # common metrics - sms = msg.content.result + try: + sms = msg.content.result + except AttributeError: + sms = msg.content.__dict__["result"] if isinstance( state.scenario, ( @@ -285,7 +292,7 @@ def refresh(same_trace: bool = False): state.all_metric_series = [] state.last_msg = None state.current_tags = [] - state.alpha158_metrics = None + state.alpha_baseline_metrics = None def evolving_feedback_window(wsf: FactorSingleFeedback | ModelSingleFeedback): @@ -378,10 +385,10 @@ def metrics_window(df: pd.DataFrame, R: int, C: int, *, height: int = 300, color hover_texts = [ hypothesis_hover_text(state.hypotheses[int(i[6:])], state.h_decisions[int(i[6:])]) for i in df.index - if i != "alpha158" and i != "Baseline" + if i != "Alpha Base" and i != "Baseline" ] - if state.alpha158_metrics is not None: - hover_texts = ["Baseline: alpha158"] + hover_texts + if state.alpha_baseline_metrics is not None: + hover_texts = ["Baseline"] + hover_texts for ci, col in enumerate(df.columns): row = ci // C + 1 col_num = ci % C + 1 @@ -401,7 +408,7 @@ def metrics_window(df: pd.DataFrame, R: int, C: int, *, height: int = 300, color ) fig.update_layout(showlegend=False, height=height) - if state.alpha158_metrics is not None: + if state.alpha_baseline_metrics is not None: for i in range(1, R + 1): # 行 for j in range(1, C + 1): # 列 fig.update_xaxes( @@ -443,15 +450,15 @@ def summary_window(): display_hypotheses(state.hypotheses, state.h_decisions, show_true_only) with chart_c: - if isinstance(state.scenario, QlibFactorScenario) and state.alpha158_metrics is not None: - df = pd.DataFrame([state.alpha158_metrics] + state.metric_series) - elif isinstance(state.scenario, QlibQuantScenario) and state.alpha158_metrics is not None: - df = pd.DataFrame([state.alpha158_metrics] + state.metric_series) + if isinstance(state.scenario, QlibFactorScenario) and state.alpha_baseline_metrics is not None: + df = pd.DataFrame([state.alpha_baseline_metrics] + state.metric_series[1:]) + elif isinstance(state.scenario, QlibQuantScenario) and state.alpha_baseline_metrics is not None: + df = pd.DataFrame([state.alpha_baseline_metrics] + state.metric_series[1:]) else: df = pd.DataFrame(state.metric_series) if show_true_only and len(state.hypotheses) >= len(state.metric_series): - if state.alpha158_metrics is not None: - selected = ["alpha158"] + [ + if state.alpha_baseline_metrics is not None: + selected = ["Alpha Base"] + [ i for i in df.index if i == "Baseline" or state.h_decisions[int(i[6:])] ] else: diff --git a/rdagent/scenarios/qlib/experiment/workspace.py b/rdagent/scenarios/qlib/experiment/workspace.py index 37797ecc..d796240a 100644 --- a/rdagent/scenarios/qlib/experiment/workspace.py +++ b/rdagent/scenarios/qlib/experiment/workspace.py @@ -33,17 +33,12 @@ class QlibFBWorkspace(FBWorkspace): ) logger.log_object(execute_qlib_log, tag="Qlib_execute_log") - # TODO: We should handle the case when Docker times out. execute_log = qtde.check_output( local_path=str(self.workspace_path), entry="python read_exp_res.py", env=run_env, ) - pattern = r"(Epoch\d+: train -[0-9\.]+, valid -[0-9\.]+|best score: -[0-9\.]+ @ \d+ epoch)" - matches = re.findall(pattern, execute_qlib_log) - execute_qlib_log = "\n".join(matches) - quantitative_backtesting_chart_path = self.workspace_path / "ret.pkl" if quantitative_backtesting_chart_path.exists(): ret_df = pd.read_pickle(quantitative_backtesting_chart_path) @@ -54,6 +49,10 @@ class QlibFBWorkspace(FBWorkspace): qlib_res_path = self.workspace_path / "qlib_res.csv" if qlib_res_path.exists(): + # Here, we ensure that the qlib experiment has run successfully before extracting information from execute_qlib_log using regex; otherwise, we keep the original experiment stdout. + pattern = r"(Epoch\d+: train -[0-9\.]+, valid -[0-9\.]+|best score: -[0-9\.]+ @ \d+ epoch)" + matches = re.findall(pattern, execute_qlib_log) + execute_qlib_log = "\n".join(matches) return pd.read_csv(qlib_res_path, index_col=0).iloc[:, 0], execute_qlib_log else: logger.error(f"File {qlib_res_path} does not exist.")