mirror of
https://github.com/NicolasBohn/NexQuant.git
synced 2026-07-27 23:47:46 +00:00
fix: fix some bugs in quant scen (#1026)
* fix some bugs * fix ci * fix * refine * fix ci
This commit is contained in:
Yuante Li
@@ -73,8 +73,9 @@ class ExperimentFeedback(Feedback):
|
||||
|
||||
def __str__(self) -> str:
|
||||
res = f"Decision: {self.decision}\nReason: {self.reason}"
|
||||
if self.code_change_summary is not None:
|
||||
res += "\nCode Change Summary: " + self.code_change_summary
|
||||
code_change_summary = getattr(self, "code_change_summary", None)
|
||||
if code_change_summary is not None:
|
||||
res += "\nCode Change Summary: " + code_change_summary
|
||||
return res
|
||||
|
||||
@classmethod
|
||||
|
||||
+45
-38
@@ -130,8 +130,8 @@ if "all_metric_series" not in state:
|
||||
state.all_metric_series = []
|
||||
|
||||
# Factor Task Baseline
|
||||
if "alpha158_metrics" not in state:
|
||||
state.alpha158_metrics = None
|
||||
if "alpha_baseline_metrics" not in state:
|
||||
state.alpha_baseline_metrics = None
|
||||
|
||||
|
||||
def should_display(msg: Message):
|
||||
@@ -179,35 +179,42 @@ def get_msgs_until(end_func: Callable[[Message], bool] = lambda _: True):
|
||||
# factor baseline exp metrics
|
||||
if (
|
||||
isinstance(state.scenario, (QlibFactorScenario, QlibQuantScenario))
|
||||
and state.alpha158_metrics is None
|
||||
and state.alpha_baseline_metrics is None
|
||||
):
|
||||
sms = msg.content.based_experiments[0].result.loc[QLIB_SELECTED_METRICS]
|
||||
sms.name = "alpha158"
|
||||
state.alpha158_metrics = sms
|
||||
try:
|
||||
sms = msg.content.based_experiments[0].result
|
||||
except AttributeError:
|
||||
sms = msg.content.based_experiments[0].__dict__["result"]
|
||||
sms = sms.loc[QLIB_SELECTED_METRICS]
|
||||
sms.name = "Alpha Base"
|
||||
state.alpha_baseline_metrics = sms
|
||||
|
||||
if (
|
||||
state.lround == 1
|
||||
and len(msg.content.based_experiments) > 0
|
||||
and msg.content.based_experiments[-1].result is not None
|
||||
):
|
||||
sms = msg.content.based_experiments[-1].result
|
||||
if isinstance(
|
||||
state.scenario,
|
||||
(
|
||||
QlibModelScenario,
|
||||
QlibFactorFromReportScenario,
|
||||
QlibFactorScenario,
|
||||
QlibQuantScenario,
|
||||
),
|
||||
):
|
||||
sms_all = sms
|
||||
sms = sms.loc[QLIB_SELECTED_METRICS]
|
||||
sms.name = f"Baseline"
|
||||
state.metric_series.append(sms)
|
||||
state.all_metric_series.append(sms_all)
|
||||
if state.lround == 1 and len(msg.content.based_experiments) > 0:
|
||||
try:
|
||||
sms = msg.content.based_experiments[-1].result
|
||||
except AttributeError:
|
||||
sms = msg.content.based_experiments[-1].__dict__["result"]
|
||||
if sms is not None:
|
||||
if isinstance(
|
||||
state.scenario,
|
||||
(
|
||||
QlibModelScenario,
|
||||
QlibFactorFromReportScenario,
|
||||
QlibFactorScenario,
|
||||
QlibQuantScenario,
|
||||
),
|
||||
):
|
||||
sms_all = sms
|
||||
sms = sms.loc[QLIB_SELECTED_METRICS]
|
||||
sms.name = f"Baseline"
|
||||
state.metric_series.append(sms)
|
||||
state.all_metric_series.append(sms_all)
|
||||
|
||||
# common metrics
|
||||
sms = msg.content.result
|
||||
try:
|
||||
sms = msg.content.result
|
||||
except AttributeError:
|
||||
sms = msg.content.__dict__["result"]
|
||||
if isinstance(
|
||||
state.scenario,
|
||||
(
|
||||
@@ -285,7 +292,7 @@ def refresh(same_trace: bool = False):
|
||||
state.all_metric_series = []
|
||||
state.last_msg = None
|
||||
state.current_tags = []
|
||||
state.alpha158_metrics = None
|
||||
state.alpha_baseline_metrics = None
|
||||
|
||||
|
||||
def evolving_feedback_window(wsf: FactorSingleFeedback | ModelSingleFeedback):
|
||||
@@ -378,10 +385,10 @@ def metrics_window(df: pd.DataFrame, R: int, C: int, *, height: int = 300, color
|
||||
hover_texts = [
|
||||
hypothesis_hover_text(state.hypotheses[int(i[6:])], state.h_decisions[int(i[6:])])
|
||||
for i in df.index
|
||||
if i != "alpha158" and i != "Baseline"
|
||||
if i != "Alpha Base" and i != "Baseline"
|
||||
]
|
||||
if state.alpha158_metrics is not None:
|
||||
hover_texts = ["Baseline: alpha158"] + hover_texts
|
||||
if state.alpha_baseline_metrics is not None:
|
||||
hover_texts = ["Baseline"] + hover_texts
|
||||
for ci, col in enumerate(df.columns):
|
||||
row = ci // C + 1
|
||||
col_num = ci % C + 1
|
||||
@@ -401,7 +408,7 @@ def metrics_window(df: pd.DataFrame, R: int, C: int, *, height: int = 300, color
|
||||
)
|
||||
fig.update_layout(showlegend=False, height=height)
|
||||
|
||||
if state.alpha158_metrics is not None:
|
||||
if state.alpha_baseline_metrics is not None:
|
||||
for i in range(1, R + 1): # 行
|
||||
for j in range(1, C + 1): # 列
|
||||
fig.update_xaxes(
|
||||
@@ -443,15 +450,15 @@ def summary_window():
|
||||
display_hypotheses(state.hypotheses, state.h_decisions, show_true_only)
|
||||
|
||||
with chart_c:
|
||||
if isinstance(state.scenario, QlibFactorScenario) and state.alpha158_metrics is not None:
|
||||
df = pd.DataFrame([state.alpha158_metrics] + state.metric_series)
|
||||
elif isinstance(state.scenario, QlibQuantScenario) and state.alpha158_metrics is not None:
|
||||
df = pd.DataFrame([state.alpha158_metrics] + state.metric_series)
|
||||
if isinstance(state.scenario, QlibFactorScenario) and state.alpha_baseline_metrics is not None:
|
||||
df = pd.DataFrame([state.alpha_baseline_metrics] + state.metric_series[1:])
|
||||
elif isinstance(state.scenario, QlibQuantScenario) and state.alpha_baseline_metrics is not None:
|
||||
df = pd.DataFrame([state.alpha_baseline_metrics] + state.metric_series[1:])
|
||||
else:
|
||||
df = pd.DataFrame(state.metric_series)
|
||||
if show_true_only and len(state.hypotheses) >= len(state.metric_series):
|
||||
if state.alpha158_metrics is not None:
|
||||
selected = ["alpha158"] + [
|
||||
if state.alpha_baseline_metrics is not None:
|
||||
selected = ["Alpha Base"] + [
|
||||
i for i in df.index if i == "Baseline" or state.h_decisions[int(i[6:])]
|
||||
]
|
||||
else:
|
||||
|
||||
@@ -33,17 +33,12 @@ class QlibFBWorkspace(FBWorkspace):
|
||||
)
|
||||
logger.log_object(execute_qlib_log, tag="Qlib_execute_log")
|
||||
|
||||
# TODO: We should handle the case when Docker times out.
|
||||
execute_log = qtde.check_output(
|
||||
local_path=str(self.workspace_path),
|
||||
entry="python read_exp_res.py",
|
||||
env=run_env,
|
||||
)
|
||||
|
||||
pattern = r"(Epoch\d+: train -[0-9\.]+, valid -[0-9\.]+|best score: -[0-9\.]+ @ \d+ epoch)"
|
||||
matches = re.findall(pattern, execute_qlib_log)
|
||||
execute_qlib_log = "\n".join(matches)
|
||||
|
||||
quantitative_backtesting_chart_path = self.workspace_path / "ret.pkl"
|
||||
if quantitative_backtesting_chart_path.exists():
|
||||
ret_df = pd.read_pickle(quantitative_backtesting_chart_path)
|
||||
@@ -54,6 +49,10 @@ class QlibFBWorkspace(FBWorkspace):
|
||||
|
||||
qlib_res_path = self.workspace_path / "qlib_res.csv"
|
||||
if qlib_res_path.exists():
|
||||
# Here, we ensure that the qlib experiment has run successfully before extracting information from execute_qlib_log using regex; otherwise, we keep the original experiment stdout.
|
||||
pattern = r"(Epoch\d+: train -[0-9\.]+, valid -[0-9\.]+|best score: -[0-9\.]+ @ \d+ epoch)"
|
||||
matches = re.findall(pattern, execute_qlib_log)
|
||||
execute_qlib_log = "\n".join(matches)
|
||||
return pd.read_csv(qlib_res_path, index_col=0).iloc[:, 0], execute_qlib_log
|
||||
else:
|
||||
logger.error(f"File {qlib_res_path} does not exist.")
|
||||
|
||||
Reference in New Issue
Block a user