From e1d6dc2dec333c05dd09ec509ac5ba39ade5f526 Mon Sep 17 00:00:00 2001 From: XianBW <36835909+XianBW@users.noreply.github.com> Date: Wed, 9 Jul 2025 14:20:33 +0800 Subject: [PATCH] chore: show llm filter calls info, remove redundant `remove_timestamp` param in Env (#1040) * show llm calls info * remove `remove_timestamp` param in Env * use cache default --- rdagent/log/ui/ds_trace.py | 79 +++++++++++++++++++++----------------- rdagent/utils/env.py | 24 +++++------- 2 files changed, 52 insertions(+), 51 deletions(-) diff --git a/rdagent/log/ui/ds_trace.py b/rdagent/log/ui/ds_trace.py index 4945b463..36326ede 100644 --- a/rdagent/log/ui/ds_trace.py +++ b/rdagent/log/ui/ds_trace.py @@ -22,6 +22,7 @@ from rdagent.log.utils import ( extract_loopid_func_name, is_valid_session, ) +from rdagent.utils.agent.tpl import T from rdagent.utils.repo.diff import generate_diff_from_dict if "show_stdout" not in state: @@ -44,7 +45,7 @@ def convert_defaultdict_to_dict(d): return d -# @st.cache_data(persist=True) +@st.cache_data(persist=True) def load_data(log_path: Path): data = defaultdict(lambda: defaultdict(dict)) llm_data = defaultdict(lambda: defaultdict(lambda: defaultdict(list))) @@ -60,8 +61,6 @@ def load_data(log_path: Path): if ei is not None: ei = int(ei) if "debug_" in msg.tag: - if "debug_tpl" in msg.tag and "filter_" in msg.content["uri"]: - continue if ei is not None: llm_data[li][fn][ei].append( { @@ -201,9 +200,9 @@ def workspace_win(workspace, cmp_workspace=None, cmp_name="last code."): def show_text(text, lang=None): """显示文本代码块""" if lang: - st.code(text, language=lang, wrap_lines=True) + st.code(text, language=lang, wrap_lines=True, line_numbers=True) elif "\n" in text: - st.code(text, language="python", wrap_lines=True) + st.code(text, language="python", wrap_lines=True, line_numbers=True) else: st.code(text, language="html", wrap_lines=True) @@ -220,6 +219,8 @@ def llm_log_win(llm_d: list): for d in llm_d: if "debug_tpl" in d["tag"]: uri = d["obj"]["uri"] + if "filter_redundant_text" in uri: + continue tpl = d["obj"]["template"] cxt = d["obj"]["context"] rd = d["obj"]["rendered"] @@ -240,32 +241,19 @@ def llm_log_win(llm_d: list): with t1: try: rdict = json.loads(resp) - if "code" in rdict: - code = rdict["code"] - st.markdown(":red[**Code in response dict:**]") - st.code(code, language="python", wrap_lines=True, line_numbers=True) - rdict.pop("code") - elif "spec" in rdict: - spec = rdict["spec"] - st.markdown(":red[**Spec in response dict:**]") - st.markdown(spec) - rdict.pop("spec") - else: - showed_keys = [] - for k, v in rdict.items(): - if k.endswith(".py"): - st.markdown(f":red[**{k}**]") - st.code(v, language="python", wrap_lines=True, line_numbers=True) - showed_keys.append(k) - for k in showed_keys: - rdict.pop(k) - st.write(":red[**Other parts (except for the code or spec) in response dict:**]") + showed_keys = [] + for k, v in rdict.items(): + if k.endswith(".py") or k.endswith(".md"): + st.markdown(f":red[**{k}**]") + st.code(v, language="python", wrap_lines=True, line_numbers=True) + showed_keys.append(k) + for k in showed_keys: + rdict.pop(k) + if len(showed_keys) > 0: + st.write(":red[**Other parts (except for the code or spec) in response dict:**]") st.json(rdict) except: - try: - st.json(resp) - except: - show_text(resp) + show_text(resp) with t2: show_text(user) with t3: @@ -466,18 +454,37 @@ def replace_ep_path(p: Path): return p +def get_llm_call_stats(llm_data: dict) -> tuple[int, int]: + total_llm_call = 0 + total_filter_call = 0 + filter_sys_prompt = T("rdagent.utils.prompts:filter_redundant_text.system").r() + for li, loop_d in llm_data.items(): + for fn, loop_fn_d in loop_d.items(): + for k, v in loop_fn_d.items(): + for d in v: + if "debug_llm" in d["tag"]: + total_llm_call += 1 + if filter_sys_prompt == d["obj"]["system"]: + total_filter_call += 1 + return total_llm_call, total_filter_call + + def summarize_win(): st.header("Summary", divider="rainbow") with st.container(border=True): min_id, max_id = get_state_data_range(state.data) - info0, info1, info2, info3 = st.columns([2, 1, 1, 1]) + info0, info1, info2, info3, info4, info5 = st.columns([1, 1, 1, 1, 1, 1]) show_trace_dag = info0.toggle("Show trace DAG", key="show_trace_dag") - with info1.popover("LITELLM_SETTINGS", icon="⚙️"): - st.write(state.data.get("SETTINGS", {}).get("LITELLM_SETTINGS", "No settings found.")) - with info2.popover("RD_AGENT_SETTINGS", icon="⚙️"): - st.write(state.data.get("SETTINGS", {}).get("RD_AGENT_SETTINGS", "No settings found.")) - with info3.popover("RDLOOP_SETTINGS", icon="⚙️"): - st.write(state.data.get("SETTINGS", {}).get("RDLOOP_SETTINGS", "No settings found.")) + with info1.popover("LITELLM", icon="⚙️"): + st.write(state.data.get("settings", {}).get("LITELLM_SETTINGS", "No settings found.")) + with info2.popover("RD_AGENT", icon="⚙️"): + st.write(state.data.get("settings", {}).get("RD_AGENT_SETTINGS", "No settings found.")) + with info3.popover("RDLOOP", icon="⚙️"): + st.write(state.data.get("settings", {}).get("RDLOOP_SETTINGS", "No settings found.")) + + llm_call, llm_filter_call = get_llm_call_stats(state.llm_data) + info4.metric("LLM Calls", llm_call) + info5.metric("LLM Filter Calls", f"{llm_filter_call}({round(llm_filter_call / llm_call * 100, 2)}%)") if show_trace_dag: st.markdown("### Trace DAG") final_trace_loop_id = max_id diff --git a/rdagent/utils/env.py b/rdagent/utils/env.py index bf45596a..e01eab72 100644 --- a/rdagent/utils/env.py +++ b/rdagent/utils/env.py @@ -213,14 +213,15 @@ class Env(Generic[ASpecificEnvConf]): local_path: str = ".", env: dict | None = None, running_extra_volume: Mapping = MappingProxyType({}), - remove_timestamp: bool = True, ) -> EnvResult: - # TODO: remove_timestamp can be implemented in a shallower way... for retry_index in range(self.conf.retry_count + 1): try: start = time.time() log_output, return_code = self._run( - entry, local_path, env, running_extra_volume=running_extra_volume, remove_timestamp=remove_timestamp + entry, + local_path, + env, + running_extra_volume=running_extra_volume, ) end = time.time() logger.info(f"Running time: {end - start} seconds") @@ -316,7 +317,10 @@ class Env(Generic[ASpecificEnvConf]): result = self.cached_run(entry_add_timeout, local_path, env, running_extra_volume) else: result = self.__run_with_retry( - entry_add_timeout, local_path, env, running_extra_volume, remove_timestamp=False + entry_add_timeout, + local_path, + env, + running_extra_volume, ) return result @@ -327,7 +331,6 @@ class Env(Generic[ASpecificEnvConf]): local_path: str = ".", env: dict | None = None, running_extra_volume: Mapping = MappingProxyType({}), - remove_timestamp: bool = True, ) -> EnvResult: """ Run the folder under the environment. @@ -364,7 +367,7 @@ class Env(Generic[ASpecificEnvConf]): ret = pickle.load(f) self.unzip_a_file_into_a_folder(str(target_folder / f"{key}.zip"), local_path) else: - ret = self.__run_with_retry(entry, local_path, env, running_extra_volume, remove_timestamp) + ret = self.__run_with_retry(entry, local_path, env, running_extra_volume) with open(target_folder / f"{key}.pkl", "wb") as f: pickle.dump(ret, f) self.zip_a_folder_into_a_file(local_path, str(target_folder / f"{key}.zip")) @@ -846,20 +849,12 @@ class DockerEnv(Env[DockerConf]): return _f() - def replace_time_info(self, input_string: str) -> str: - """To remove any time related information from the logs since it will destroy the cache mechanism""" - """We currently set this function as default, but it can be changed in the future""" - datetime_pattern = r"\b\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}(?:\.\d+)?\b" - output_string = re.sub(datetime_pattern, "[DATETIME]", input_string) - return output_string - def _run( self, entry: str | None = None, local_path: str = ".", env: dict | None = None, running_extra_volume: Mapping = MappingProxyType({}), - remove_timestamp: bool = True, **kwargs: Any, ) -> tuple[str, int]: if env is None: @@ -918,7 +913,6 @@ class DockerEnv(Env[DockerConf]): print(table) for log in logs: decoded_log = log.strip().decode() - decoded_log = self.replace_time_info(decoded_log) if remove_timestamp else decoded_log Console().print(decoded_log, markup=False) log_output += decoded_log + "\n" exit_status = container.wait()["StatusCode"]