From 125e6e41c681f9e4acb82cb3165044f8861f2c2f Mon Sep 17 00:00:00 2001 From: XianBW <36835909+XianBW@users.noreply.github.com> Date: Tue, 23 Jul 2024 14:48:14 +0800 Subject: [PATCH] Demo adjustment (#95) add model trace and trace obj demo --- rdagent/log/ui/app.py | 20 +++- rdagent/log/ui/st_fixed_container.py | 126 ++++++++++++++++++++ rdagent/log/ui/web.py | 168 +++++++++++++++++++++++---- 3 files changed, 286 insertions(+), 28 deletions(-) create mode 100644 rdagent/log/ui/st_fixed_container.py diff --git a/rdagent/log/ui/app.py b/rdagent/log/ui/app.py index 0649217f..58b7aa27 100644 --- a/rdagent/log/ui/app.py +++ b/rdagent/log/ui/app.py @@ -1,4 +1,18 @@ -from rdagent.log.ui.web import WebView, QlibFactorTraceWindow -from rdagent.log.storage import FileStorage +from rdagent.log.ui.web import WebView, QlibTraceWindow, TraceObjWindow, mock_msg +from rdagent.log.storage import FileStorage, Message +from rdagent.core.proposal import Trace +from pathlib import Path +import pickle -WebView(QlibFactorTraceWindow(show_common_logs=True, show_llm=True)).display(FileStorage("./log/2024-07-18_08-37-00-477228")) \ No newline at end of file + +# show logs folder +# WebView(QlibTraceWindow(show_common_logs=False, show_llm=False)).display(FileStorage("./log/2024-07-22_03-01-12-021659")) + + +# load Trace obj +with Path('./log/step_trace.pkl').open('rb') as f: + obj = pickle.load(f) + trace: Trace = obj[-1] + +# show Trace obj +# TraceObjWindow().consume_msg(mock_msg(trace)) diff --git a/rdagent/log/ui/st_fixed_container.py b/rdagent/log/ui/st_fixed_container.py new file mode 100644 index 00000000..ae3d8cc2 --- /dev/null +++ b/rdagent/log/ui/st_fixed_container.py @@ -0,0 +1,126 @@ +from typing import Literal + +import streamlit as st +from streamlit.components.v1 import html + +FIXED_CONTAINER_CSS = """ +:root {{ + --background-color: #ffffff; /* Default background color */ +}} +div[data-testid="stVerticalBlockBorderWrapper"]:has(div.fixed-container-{id}):not(:has(div.not-fixed-container)) {{ + position: {mode}; + width: inherit; + background-color: inherit; + {position}: {margin}; + z-index: 999; +}} +div[data-testid="stVerticalBlockBorderWrapper"]:has(div.fixed-container-{id}):not(:has(div.not-fixed-container)) div[data-testid="stVerticalBlock"]:has(div.fixed-container-{id}):not(:has(div.not-fixed-container)) > div[data-testid="stVerticalBlockBorderWrapper"] {{ + background-color: transparent; + width: 100%; +}} +div[data-testid="stVerticalBlockBorderWrapper"]:has(div.fixed-container-{id}):not(:has(div.not-fixed-container)) div[data-testid="stVerticalBlock"]:has(div.fixed-container-{id}):not(:has(div.not-fixed-container)) > div[data-testid="stVerticalBlockBorderWrapper"] div[data-testid="stVerticalBlockBorderWrapper"] {{ + background-color: var(--background-color); +}} +div[data-testid="stVerticalBlockBorderWrapper"]:has(div.fixed-container-{id}):not(:has(div.not-fixed-container)) div[data-testid="stVerticalBlock"]:has(div.fixed-container-{id}):not(:has(div.not-fixed-container)) > div[data-testid="element-container"] {{ + display: none; +}} +div[data-testid="stVerticalBlockBorderWrapper"]:has(div.not-fixed-container):not(:has(div[class^='fixed-container-'])) {{ + display: none; +}} +""".strip() + +FIXED_CONTAINER_JS = """ +const root = parent.document.querySelector('.stApp'); +let lastBackgroundColor = null; +function updateContainerBackground(currentBackground) { + parent.document.documentElement.style.setProperty('--background-color', currentBackground); + ; +} +function checkForBackgroundColorChange() { + const style = window.getComputedStyle(root); + const currentBackgroundColor = style.backgroundColor; + if (currentBackgroundColor !== lastBackgroundColor) { + lastBackgroundColor = currentBackgroundColor; // Update the last known value + updateContainerBackground(lastBackgroundColor); + } +} +const observerCallback = (mutationsList, observer) => { + for(let mutation of mutationsList) { + if (mutation.type === 'attributes' && (mutation.attributeName === 'class' || mutation.attributeName === 'style')) { + checkForBackgroundColorChange(); + } + } +}; +const main = () => { + checkForBackgroundColorChange(); + const observer = new MutationObserver(observerCallback); + observer.observe(root, { attributes: true, childList: false, subtree: false }); +} +// main(); +document.addEventListener("DOMContentLoaded", main); +""".strip() + + +MARGINS = { + "top": "2.875rem", + "bottom": "0", +} + + +counter = 0 + + +def st_fixed_container( + *, + height: int | None = None, + border: bool | None = None, + mode: Literal["fixed", "sticky"] = "fixed", + position: Literal["top", "bottom"] = "top", + margin: str | None = None, + transparent: bool = False, +): + if margin is None: + margin = MARGINS[position] + global counter + + fixed_container = st.container() + non_fixed_container = st.container() + css = FIXED_CONTAINER_CSS.format( + mode=mode, + position=position, + margin=margin, + id=counter, + ) + with fixed_container: + html(f"", scrolling=False, height=0) + st.markdown(f"", unsafe_allow_html=True) + st.markdown( + f"
", + unsafe_allow_html=True, + ) + with non_fixed_container: + st.markdown( + f"
", + unsafe_allow_html=True, + ) + counter += 1 + + parent_container = fixed_container if transparent else fixed_container.container() + return parent_container.container(height=height, border=border) + + +if __name__ == "__main__": + for i in range(30): + st.write(f"Line {i}") + + # with st_fixed_container(mode="sticky", position="top", border=True): + # with st_fixed_container(mode="sticky", position="bottom", border=True): + # with st_fixed_container(mode="fixed", position="top", border=True): + with st_fixed_container(mode="fixed", position="bottom", border=True): + st.write("This is a fixed container.") + st.write("This is a fixed container.") + st.write("This is a fixed container.") + + st.container(border=True).write("This is a regular container.") + for i in range(30): + st.write(f"Line {i}") \ No newline at end of file diff --git a/rdagent/log/ui/web.py b/rdagent/log/ui/web.py index 31cdc96a..3019afaa 100644 --- a/rdagent/log/ui/web.py +++ b/rdagent/log/ui/web.py @@ -4,16 +4,23 @@ import plotly.express as px from rdagent.log.base import Storage, View from rdagent.log.base import Message -from datetime import timezone +from datetime import timezone, datetime from collections import defaultdict from copy import deepcopy from typing import Callable, Type from streamlit.delta_generator import DeltaGenerator from rdagent.core.proposal import Hypothesis, HypothesisFeedback + from rdagent.scenarios.qlib.experiment.factor_experiment import QlibFactorExperiment +from rdagent.scenarios.qlib.experiment.model_experiment import QlibModelExperiment + from rdagent.components.coder.factor_coder.factor import FactorTask, FactorFBWorkspace from rdagent.components.coder.factor_coder.CoSTEER.evaluators import FactorSingleFeedback +from rdagent.components.coder.model_coder.CoSTEER.evaluators import ModelCoderFeedback +from rdagent.components.coder.model_coder.model import ModelTask, ModelFBWorkspace + + st.set_page_config(layout="wide") @@ -158,8 +165,13 @@ class ObjectsTabsWindow(StWindow): objs_dict = {self.mapper(obj): obj for obj in msg.content} elif not isinstance(msg.content, dict): raise ValueError("Message content should be a list or a dict of objects.") - - tabs = self.container.tabs(objs_dict.keys()) + + # two many tabs may cause display problem + tab_names = list(objs_dict.keys()) + tabs = [] + for i in range(0, len(tab_names), 10): + tabs.extend(self.container.tabs(tab_names[i:i+10])) + for id, obj in enumerate(objs_dict.values()): splited_msg = Message(tag=msg.tag, level=msg.level, @@ -192,6 +204,7 @@ class HypothesisFeedbackWindow(StWindow): - **Decision**: {h.decision} - **Reason**: {h.reason}""") + class FactorTaskWindow(StWindow): def consume_msg(self, msg: Message): @@ -207,6 +220,21 @@ class FactorTaskWindow(StWindow): self.container.text(f"Factor resources: {ft.factor_resources}") +class ModelTaskWindow(StWindow): + + def consume_msg(self, msg: Message): + mt: ModelTask = msg.content + + self.container.markdown(f"**Model Name**: {mt.name}") + self.container.markdown(f"**Model Type**: {mt.model_type}") + self.container.markdown(f"**Description**: {mt.description}") + self.container.markdown(f"**Formulation**: {mt.formulation}") + + variables_df = pd.DataFrame(mt.variables, index=['Value']).T + variables_df.index.name = 'Variable' + self.container.table(variables_df) + + class FactorFeedbackWindow(StWindow): def consume_msg(self, msg: Message): @@ -224,26 +252,53 @@ This implementation is {'SUCCESS' if fb.final_decision else 'FAIL'}. """) -class FactorWorkspaceWindow(StWindow): +class ModelFeedbackWindow(StWindow): def consume_msg(self, msg: Message): - fws: FactorFBWorkspace = msg.content - - # factor info - self.container.subheader('Factor info') - factor_msg = deepcopy(msg) - factor_msg.content = fws.target_task - FactorTaskWindow(self.container.container()).consume_msg(factor_msg) + mb: ModelCoderFeedback = msg.content + self.container.markdown(f"""### :blue[Model Execution Feedback] +{mb.execution_feedback} +### :blue[Model Shape Feedback] +{mb.shape_feedback} +### :blue[Model Value Feedback] +{mb.value_feedback} +### :blue[Model Code Feedback] +{mb.code_feedback} +### :blue[Model Final Feedback] +{mb.final_feedback} +### :blue[Model Final Decision] +This implementation is {'SUCCESS' if mb.final_decision else 'FAIL'}. +""") - # factor codes + +class WorkspaceWindow(StWindow): + + def consume_msg(self, msg: Message): + ws: FactorFBWorkspace | ModelFBWorkspace = msg.content + + # no workspace + if ws is None: return + + # task info + task_msg = deepcopy(msg) + task_msg.content = ws.target_task + if isinstance(ws, FactorFBWorkspace): + self.container.subheader('Factor Info') + FactorTaskWindow(self.container.container()).consume_msg(task_msg) + else: + self.container.subheader('Model Info') + ModelTaskWindow(self.container.container()).consume_msg(task_msg) + + # task codes self.container.subheader('Codes') - for k,v in fws.code_dict.items(): + for k,v in ws.code_dict.items(): self.container.markdown(f"`{k}`") self.container.code(v, language="python") # executed_factor_value_dataframe - self.container.subheader('Executed Factor Value Dataframe') - self.container.dataframe(fws.executed_factor_value_dataframe) + if isinstance(ws, FactorFBWorkspace): + self.container.subheader('Executed Factor Value Dataframe') + self.container.dataframe(ws.executed_factor_value_dataframe) class QlibFactorExpWindow(StWindow): @@ -255,20 +310,46 @@ class QlibFactorExpWindow(StWindow): ftm_msg = deepcopy(msg) ftm_msg.content = exp.sub_workspace_list ObjectsTabsWindow(self.container.expander('Factor Tasks'), - inner_class=FactorWorkspaceWindow, + inner_class=WorkspaceWindow, mapper=lambda x: x.target_task.factor_name, ).consume_msg(ftm_msg) # result self.container.subheader('Results', divider=True) - results = pd.DataFrame({f'exp {id}':e.result for id, e in enumerate(exp.based_experiments)}) + results = pd.DataFrame({f'base_exp_{id}':e.result for id, e in enumerate(exp.based_experiments)}) results['now'] = exp.result self.container.expander('results table').table(results) - self.container.expander('results chart').plotly_chart(px.bar(results, orientation='h', barmode='group')) + + try: + bar_chart = px.bar(results, orientation='h', barmode='group') + self.container.expander('results chart').plotly_chart(bar_chart) + except: + self.container.text('Results are incomplete.') -class QlibFactorTraceWindow(StWindow): +class QlibModelExpWindow(StWindow): + + def consume_msg(self, msg: Message): + exp: QlibModelExperiment = msg.content + + # model tasks + _msg = deepcopy(msg) + _msg.content = exp.sub_workspace_list + ObjectsTabsWindow(self.container.expander('Model Tasks'), + inner_class=WorkspaceWindow, + mapper=lambda x: x.target_task.name if x else 'None', + ).consume_msg(_msg) + + # result + self.container.subheader('Results', divider=True) + results = pd.DataFrame({f'base_exp_{id}':e.result for id, e in enumerate(exp.based_experiments)}) + results['now'] = exp.result + + self.container.expander('results table').table(results) + + +class QlibTraceWindow(StWindow): def __init__(self, container: 'DeltaGenerator' = st.container(), show_llm: bool = False, show_common_logs: bool = True): super().__init__(container) @@ -278,7 +359,7 @@ class QlibFactorTraceWindow(StWindow): self.current_tag = '' self.current_win = StWindow(self.container) - self.evolving_factors: list[str] = [] + self.evolving_tasks: list[str] = [] def consume_msg(self, msg: Message): @@ -304,21 +385,35 @@ class QlibFactorTraceWindow(StWindow): # hypothesis feedback self.current_win = HypothesisFeedbackWindow(self.container) elif isinstance(msg.content, QlibFactorExperiment): - # qlib exp logs self.current_win = QlibFactorExpWindow(self.container) + elif isinstance(msg.content, QlibModelExperiment): + self.current_win = QlibModelExpWindow(self.container) elif isinstance(msg.content, list): - # factor logs + if isinstance(msg.content[0], FactorTask): self.current_win = ObjectsTabsWindow(self.container.expander('Factor Tasks'), FactorTaskWindow, lambda x: x.factor_name) + elif isinstance(msg.content[0], ModelTask): + self.current_win = ObjectsTabsWindow(self.container.expander('Model Tasks'), ModelTaskWindow, lambda x: x.name) + elif isinstance(msg.content[0], FactorFBWorkspace): self.current_win = ObjectsTabsWindow(self.container.expander('Factor Workspaces'), - inner_class=FactorWorkspaceWindow, + inner_class=WorkspaceWindow, mapper=lambda x: x.target_task.factor_name) - self.evolving_factors = [m.target_task.factor_name for m in msg.content] + self.evolving_tasks = [m.target_task.factor_name for m in msg.content] + elif isinstance(msg.content[0], ModelFBWorkspace): + self.current_win = ObjectsTabsWindow(self.container.expander('Model Workspaces'), + inner_class=WorkspaceWindow, + mapper=lambda x: x.target_task.name) + self.evolving_tasks = [m.target_task.name for m in msg.content] + elif isinstance(msg.content[0], FactorSingleFeedback): self.current_win = ObjectsTabsWindow(self.container.expander('Factor Feedbacks'), inner_class=FactorFeedbackWindow, - tab_names=self.evolving_factors) + tab_names=self.evolving_tasks) + elif isinstance(msg.content[0], ModelCoderFeedback): + self.current_win = ObjectsTabsWindow(self.container.expander('Model Feedbacks'), + inner_class=ModelFeedbackWindow, + tab_names=self.evolving_tasks) else: # common logs if not self.show_common_logs: @@ -327,3 +422,26 @@ class QlibFactorTraceWindow(StWindow): self.current_win.consume_msg(msg) + +def mock_msg(obj) -> Message: + return Message(tag='mock', level='INFO', timestamp=datetime.now(), pid_trace='000', caller='mock',content=obj) + + +from rdagent.core.proposal import Trace +class TraceObjWindow(StWindow): + + def __init__(self, container: 'DeltaGenerator' = st.container()): + self.container = container + + def consume_msg(self, msg: Message): + trace:Trace = msg.content + + for id, (h, e, hf) in enumerate(trace.hist): + self.container.header(f'Trace History {id}', divider=True) + HypothesisWindow(self.container).consume_msg(mock_msg(h)) + if isinstance(e, QlibFactorExperiment): + QlibFactorExpWindow(self.container).consume_msg(mock_msg(e)) + else: + QlibModelExpWindow(self.container).consume_msg(mock_msg(e)) + HypothesisFeedbackWindow(self.container).consume_msg(mock_msg(hf)) +