feat: log api status to mlflow (#860)

* log api status to mlflow to frontend tracking

* fix CI

---------

Co-authored-by: Xu Yang <xuyang1@microsoft.com>
This commit is contained in:
Xu Yang
2025-05-09 12:20:49 +08:00
committed by GitHub
parent 5fecefb802
commit 1fd8abce0f
3 changed files with 20 additions and 1 deletions
+3 -1
View File
@@ -75,10 +75,12 @@ class RDAgentTimer:
class RDAgentTimerWrapper(SingletonBaseClass):
def __init__(self) -> None:
self.timer: RDAgentTimer = RDAgentTimer()
self.api_fail_count: int = 0
self.latest_api_fail_time: datetime | None = None
def replace_timer(self, timer: RDAgentTimer) -> None:
self.timer = timer
logger.info("Timer replaced successfully.")
RD_Agent_TIMER_wrapper = RDAgentTimerWrapper()
RD_Agent_TIMER_wrapper: RDAgentTimerWrapper = RDAgentTimerWrapper()
+3
View File
@@ -11,6 +11,7 @@ from datetime import datetime
from pathlib import Path
from typing import Any, Optional, cast
import pytz
from pydantic import TypeAdapter
from rdagent.core.utils import LLM_CACHE_SEED_GEN, SingletonBaseClass
@@ -349,6 +350,8 @@ class APIBackend(ABC):
content[: len(content) // 2] for content in kwargs.get("input_content_list", [])
]
else:
RD_Agent_TIMER_wrapper.api_fail_count += 1
RD_Agent_TIMER_wrapper.latest_api_fail_time = datetime.now(pytz.timezone("Asia/Shanghai"))
if (
openai_imported
and isinstance(e, openai.APITimeoutError)
+14
View File
@@ -138,6 +138,20 @@ class LoopBase:
+ current_local_datetime.year * 1e10
)
mlflow.log_metric("current_datetime", float_like_datetime)
mlflow.log_metric("api_fail_count", RD_Agent_TIMER_wrapper.api_fail_count)
lastest_api_fail_time = RD_Agent_TIMER_wrapper.latest_api_fail_time
if lastest_api_fail_time is not None:
mlflow.log_metric(
"api_fail_count",
(
lastest_api_fail_time.second
+ lastest_api_fail_time.minute * 1e2
+ lastest_api_fail_time.hour * 1e4
+ lastest_api_fail_time.day * 1e6
+ lastest_api_fail_time.month * 1e8
+ lastest_api_fail_time.year * 1e10
),
)
if self.timer.started:
if RD_AGENT_SETTINGS.enable_mlflow: