gavindiaz/NexQuant
1
0
Fork 0
mirror of https://github.com/NicolasBohn/NexQuant.git synced 2026-07-27 15:37:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

chore: add a simple chatbot in LLM_LOG window in DataScience UI (#1088)

Browse Source 
* refine time show

* updates

* add chatbot in llm_log_win

* use newest litellm
This commit is contained in:
XianBW
2025-07-17 19:29:25 +08:00
committed by GitHub
parent 77725913fa
commit 221fea1e15
3 changed files with 92 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files
constraints/3.10.txt
-1
View File
@@ -5,4 +5,3 @@ psutil==6.1.0
rich==13.9.2
scipy==1.14.1
tqdm==4.66.5
litellm==1.72.4
constraints/3.11.txt
-1
View File
@@ -5,4 +5,3 @@ psutil==6.1.0
rich==13.9.2
scipy==1.14.1
tqdm==4.66.5
litellm==1.72.4
rdagent/log/ui/ds_trace.py
+92 -23
View File
@@ -4,12 +4,13 @@ import pickle
import random
import re
from collections import defaultdict
from datetime import timedelta
from datetime import time, timedelta
from pathlib import Path
import pandas as pd
import plotly.express as px
import streamlit as st
from litellm import get_valid_models
from streamlit import session_state as state
from rdagent.app.data_science.loop import DataScienceRDLoop
@@ -23,6 +24,8 @@ from rdagent.log.utils import (
extract_loopid_func_name,
is_valid_session,
)
from rdagent.oai.backend.litellm import LITELLM_SETTINGS
from rdagent.oai.llm_utils import APIBackend
from rdagent.utils.agent.tpl import T
from rdagent.utils.repo.diff import generate_diff_from_dict
@@ -39,6 +42,12 @@ if "log_path" not in state:
if "log_folder" not in state:
state.log_folder = Path("./log")
available_models = get_valid_models()
LITELLM_SETTINGS.dump_chat_cache = False
LITELLM_SETTINGS.dump_embedding_cache = False
LITELLM_SETTINGS.use_chat_cache = False
LITELLM_SETTINGS.use_embedding_cache = False
def convert_defaultdict_to_dict(d):
if isinstance(d, defaultdict):
@@ -228,7 +237,7 @@ def llm_log_win(llm_d: list):
tpl = d["obj"]["template"]
cxt = d["obj"]["context"]
rd = d["obj"]["rendered"]
with st.expander(highlight_prompts_uri(uri), expanded=False, icon="⚙️"):
with st.popover(highlight_prompts_uri(uri), icon="⚙️", use_container_width=True):
t1, t2, t3 = st.tabs([":green[**Rendered**]", ":blue[**Template**]", ":orange[**Context**]"])
with t1:
show_text(rd)
@@ -240,8 +249,10 @@ def llm_log_win(llm_d: list):
system = d["obj"].get("system", None)
user = d["obj"]["user"]
resp = d["obj"]["resp"]
with st.expander(f"**LLM**", expanded=False, icon="🤖"):
t1, t2, t3 = st.tabs([":green[**Response**]", ":blue[**User**]", ":orange[**System**]"])
with st.expander(f"**LLM**", icon="🤖", expanded=False):
t1, t2, t3, t4 = st.tabs(
[":green[**Response**]", ":blue[**User**]", ":orange[**System**]", ":violet[**ChatBot**]"]
)
with t1:
try:
rdict = json.loads(resp)
@@ -262,6 +273,50 @@ def llm_log_win(llm_d: list):
show_text(user)
with t3:
show_text(system or "No system prompt available")
with t4:
input_c, resp_c = st.columns(2)
key = hashlib.md5(resp.encode()).hexdigest()
with input_c:
btc1, btc2 = st.columns(2)
trace_model = (
state.data.get("settings", {})
.get("LITELLM_SETTINGS", {})
.get("chat_model", available_models[0])
)
trace_reasoning_effort = (
state.data.get("settings", {}).get("LITELLM_SETTINGS", {}).get("reasoning_effort", None)
)
LITELLM_SETTINGS.chat_model = btc1.selectbox(
"Chat Model",
options=available_models,
index=available_models.index(trace_model),
key=key + "_chat_model",
)
LITELLM_SETTINGS.reasoning_effort = btc2.selectbox(
"Reasoning Effort",
options=[None, "low", "medium", "high"],
index=[None, "low", "medium", "high"].index(trace_reasoning_effort),
key=key + "_reasoning_effort",
)
json_mode = st.checkbox("JSON Mode", value=False, key=key + "_json_mode")
sys_p = input_c.text_area(label="system", value=system, height="content")
user_p = input_c.text_area(label="user", value=user, height="content")
with resp_c:
if st.button("Call LLM", key=key + "_call_llm"):
with st.spinner("Calling LLM..."):
try:
resp_new = APIBackend().build_messages_and_create_chat_completion(
user_prompt=user_p,
system_prompt=sys_p,
json_mode=json_mode,
)
except Exception as e:
resp_new = f"Error: {e}"
try:
rdict = json.loads(resp_new)
st.json(rdict)
except:
st.code(resp_new, wrap_lines=True, line_numbers=True)
def hypothesis_win(hypo):
@@ -473,6 +528,16 @@ def get_llm_call_stats(llm_data: dict) -> tuple[int, int]:
return total_llm_call, total_filter_call
def timedelta_to_str(td: timedelta | None) -> str:
if isinstance(td, timedelta):
total_seconds = int(td.total_seconds())
hours = total_seconds // 3600
minutes = (total_seconds % 3600) // 60
seconds = total_seconds % 60
return f"{hours:02d}:{minutes:02d}:{seconds:02d}"
return td
def summarize_win():
st.header("Summary", divider="rainbow")
with st.container(border=True):
@@ -511,8 +576,6 @@ def summarize_win():
"Exp Gen",
"Coding",
"Running",
"Start Time (UTC+8)",
"End Time (UTC+8)",
],
index=range(min_id, max_id + 1),
)
@@ -529,18 +592,24 @@ def summarize_win():
if k not in ["component", "hypothesis", "reason"]
}
df.loc[loop, "COST($)"] = sum(tc.content["cost"] for tc in state.token_costs[loop])
# Time Stats
if loop in state.times and state.times[loop]:
df.loc[loop, "Time"] = str(sum((i.end - i.start for i in state.times[loop]), timedelta())).split(".")[0]
exp_gen_time = state.times[loop][0].end - state.times[loop][0].start
df.loc[loop, "Exp Gen"] = str(exp_gen_time).split(".")[0]
if len(state.times[loop]) > 1:
coding_time = state.times[loop][1].end - state.times[loop][1].start
df.loc[loop, "Coding"] = str(coding_time).split(".")[0]
if len(state.times[loop]) > 2:
running_time = state.times[loop][2].end - state.times[loop][2].start
df.loc[loop, "Running"] = str(running_time).split(".")[0]
df.loc[loop, "Start Time (UTC+8)"] = state.times[loop][0].start + timedelta(hours=8)
df.loc[loop, "End Time (UTC+8)"] = state.times[loop][-1].end + timedelta(hours=8)
exp_gen_time = coding_time = running_time = None
all_steps_time = timedelta()
for lpt in state.times[loop]:
all_steps_time += lpt.end - lpt.start
if lpt.step_idx == 0:
exp_gen_time = lpt.end - lpt.start
elif lpt.step_idx == 1:
coding_time = lpt.end - lpt.start
elif lpt.step_idx == 2:
running_time = lpt.end - lpt.start
df.loc[loop, "Time"] = timedelta_to_str(all_steps_time)
df.loc[loop, "Exp Gen"] = timedelta_to_str(exp_gen_time)
df.loc[loop, "Coding"] = timedelta_to_str(coding_time)
df.loc[loop, "Running"] = timedelta_to_str(running_time)
if "running" in loop_data and "no_tag" in loop_data["running"]:
try:
try:
@@ -705,10 +774,11 @@ def summarize_win():
st1.markdown("### Time Statistics")
time_stat_df = time_df.groupby("Component").sum()
time_stat_df.loc["Total"] = time_stat_df.sum()
time_stat_df.loc[:, "Exp Gen(%)"] = time_stat_df["Exp Gen"] / time_stat_df["Time"] * 100
time_stat_df.loc[:, "Coding(%)"] = time_stat_df["Coding"] / time_stat_df["Time"] * 100
time_stat_df.loc[:, "Running(%)"] = time_stat_df["Running"] / time_stat_df["Time"] * 100
time_stat_df = time_stat_df.map(lambda x: str(x).split(".")[0] if pd.notnull(x) else "0:00:00")
time_stat_df.loc[:, "Exp Gen(%)"] = (time_stat_df["Exp Gen"] / time_stat_df["Time"] * 100).round(2)
time_stat_df.loc[:, "Coding(%)"] = (time_stat_df["Coding"] / time_stat_df["Time"] * 100).round(2)
time_stat_df.loc[:, "Running(%)"] = (time_stat_df["Running"] / time_stat_df["Time"] * 100).round(2)
for col in ["Time", "Exp Gen", "Coding", "Running"]:
time_stat_df[col] = time_stat_df[col].map(timedelta_to_str)
st1.dataframe(time_stat_df)
@@ -751,7 +821,6 @@ def get_folders_sorted(log_path, sort_by_time=False):
folders = sorted(folders, key=lambda folder: folder.stat().st_mtime, reverse=True)
else:
folders = sorted(folders, key=lambda folder: folder.name)
st.write(f"Found {len(folders)} folders")
return [folder.name for folder in folders]
@@ -803,7 +872,7 @@ with st.sidebar:
st.rerun()
st.toggle("**Show LLM Log**", key="show_llm_log")
st.toggle("*Show stdout*", key="show_stdout")
st.toggle("*Show save workspace feature*", key="show_save_input")
st.toggle("*Show save workspace*", key="show_save_input")
st.markdown(
f"""
- [Summary](#summary)